<commit_before>#!/usr/bin/py

import re
import codecs
from xml.dom.minidom import parse,parseString

metadata_fields = ['month','day','year','journalabbrv','journalid','headid','title','languages','type','journaltitle','volume','authors','issueid','issn','fpage','lpage']


class Export:
    def __init__(self):
        self.filenames = []
        filehandle = open('../../filenames.txt')
        for p in filehandle.readlines():
            mystring = p.split('\t')[0]
            mystring = re.sub('.xml\n*','',mystring)
            self.filenames.append(mystring)
    def exportall(self):
        for p in self.filenames:
            working = jstorID(p)
            #working.writeText()
            working.writeMetadata()
            
class jstorID:
    def __init__(self,string):
        self.id = string
        self.open_xml()
        self.get_metadata()
    def open_xml(self):
        self.dom = parse('../../raw/' + self.id + '.xml')
        self.text = self.dom.getElementsByTagName('text')
        self.textstring = "\n".join([t.childNodes[0].nodeValue for t in self.text])
        self.textstring = self.textstring.encode('utf-8')
    def get_metadata(self):
        self.metadata = dict()
        for item in metadata_fields:
            try:
                self.metadata[item] = self.dom.getElementsByTagName(item)[0].childNodes[0].nodeValue
            except:
                self.metadata[item] = ''
    def writeText(self):
        output = open('../../texts/' + self.id + '.txt','w')
        output.write(self.textstring)
        output.close()
    def writeMetadata(self):
        self.get_metadata()
        metadata = open ('../../metadata/main.txt','a')
        metadata.write('self.id')
        for field in metadata_fields:
            try:
                metadata.write(self.metadata['field']) + "\t"
            except KeyError:
                metadata.write("\t")

Export().exportall()<commit_msg>Couple more parser changes<commit_after>#!/usr/bin/py

import re
import codecs
from xml.dom.minidom import parse,parseString

metadata_fields = ['month','day','year','journalabbrv','journalid','headid','title','languages','type','journaltitle','volume','authors','issueid','issn','fpage','lpage']


class Export:
    def __init__(self):
        self.filenames = []
        filehandle = open('../../filenames.txt')
        for p in filehandle.readlines():
            mystring = p.split('\t')[0]
            mystring = re.sub('.xml\n*','',mystring)
            self.filenames.append(mystring)
    def exportall(self):
        for p in self.filenames:
            working = jstorID(p)
            #working.writeText()
            working.writeMetadata()
            
class jstorID:
    def __init__(self,string):
        self.id = string
        self.open_xml()
        self.get_metadata()
    def open_xml(self):
        self.dom = parse('../../raw/' + self.id + '.xml')
        self.text = self.dom.getElementsByTagName('text')
        self.textstring = "\n".join([t.childNodes[0].nodeValue for t in self.text])
        self.textstring = self.textstring.encode('utf-8')
    def get_metadata(self):
        self.metadata = dict()
        for item in metadata_fields:
            try:
                self.metadata[item] = self.dom.getElementsByTagName(item)[0].childNodes[0].nodeValue
                self.metadata[item] = self.metadata[item].encode('utf-8')
            except:
                self.metadata[item] = ''
    def writeText(self):
        output = open('../../texts/' + self.id + '.txt','w')
        output.write(self.textstring)
        output.close()
    def writeMetadata(self):
        self.get_metadata()
        metadata = open ('../../metadata/main.txt','a')
        metadata.write(self.id)
        for field in metadata_fields:
            try:
                metadata.write(self.metadata[field] + "\t")
            except KeyError:
                metadata.write("\t")
        metadata.write("\n")

Export().exportall()
{'fraction_non_alphanumeric': 0.15886203423967774, 'fraction_numerical': 0.0027693856998992953, 'mean_word_length': 2.219611021069692, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 10}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2989034', 'n_tokens_mistral': 1196, 'n_tokens_neox': 1172, 'n_words': 205}