<jupyter_start><jupyter_text>Annotate with Freeling<jupyter_code>subprocess.call("analyze -f es.cfg --server on --port 50000 --outlv tagged  --sense ukb  --workers 2 --output xml --ner --nec &", shell=True)

if not os.path.exists(outdir_xml):
    os.makedirs(outdir_xml)
    
for book_path in glob.glob(outdir_txt + "/*"):
    #print(book_path)
    book_name = book_path[-3:]
    print(book_name)

    if not os.path.exists(outdir_xml + book_name):
        os.makedirs(outdir_xml + book_name)

    
    for path_file in glob.glob(outdir_txt + book_name + "/*"):
        file_  = "".join(os.path.splitext(os.path.split(path_file)[1]))

        print(file_ )
        command = "analyzer_client " + "50000" + " < " + outdir_txt + book_name + "/" + file_ + " > " + outdir_xml + book_name + "/" + file_   
        subprocess.call(command, shell=True)<jupyter_output>1CH
b.1CH.005.019
b.1CH.008.028
b.1CH.024.013
b.1CH.006.018
b.1CH.004.001
b.1CH.027.010
b.1CH.029.017
b.1CH.024.011
b.1CH.008.029
b.1CH.006.010
b.1CH.022.017
b.1CH.025.011
b.1CH.002.012
b.1CH.015.007
b.1CH.021.023
b.1CH.008.012
b.1CH.025.013
b.1CH.012.008
b.1CH.007.014
b.1CH.015.004
b.1CH.015.018
b.1CH.024.010
b.1CH.027.009
b.1CH.012.023
b.1CH.025.007
b.1CH.014.012
b.1CH.026.015
b.1CH.018.010
b.1CH.026.009
b.1CH.011.038
b.1CH.007.039
b.1CH.015.011
b.1CH.006.009
b.1CH.007.024
b.1CH.009.037
b.1CH.026.003
b.1CH.006.045
b.1CH.017.012
b.1CH.027.011
b.1CH.003.024
b.1CH.009.014
b.1CH.006.048
b.1CH.010.011
b.1CH.011.034
b.1CH.011.044
b.1CH.002.033
b.1CH.002.046
b.1CH.006.038
b.1CH.006.028
b.1CH.027.006
b.1CH.029.010
b.1CH.026.007
b.1CH.004.033
b.1CH.024.027
b.1CH.002.049
b.1CH.004.035
b.1CH.005.001
b.1CH.013.012
b.1CH.012.011
b.1CH.019.015
b.1CH.005.025
b.1CH.002.048
b.1CH.016.030
b.1CH.024.007
b.1CH.008.002
b.1CH.012.033
b.1CH.022.014
b.1CH.012.015
b.1CH.014.008
b.1CH.024.006
b.1CH.023.025
b.[...]<jupyter_text>```por cada libro	abre cada versículo		coloca un contenedor a cada uno		sumalo a un contenedor de versículos por libro	convierte xml de Freeling en TEI	guarda archivo de TEI como standOff element de la Biblia```  Integrate in File<jupyter_code>bible_root
root_TEI = etree.fromstring( """<TEI xmlns="http://www.tei-c.org/ns/1.0">\n	<teiHeader>
		<fileDesc>
			<titleStmt>
				<title type="idno">
					<idno type="string">""" + book_name + """</idno>
                                     </title>
                                     </titleStmt>
                                     </fileDesc>
                                     </teiHeader>
                                     <text>
                                     <group>
                                     <text xml:id="b.""" + book_name + """.POSFreenling" type="morphosyntactic">
                                         <body>
                                         </body>
                                         </text>
                                         </group>
                                     </text>
                                     </TEI>
                                     """)

for book_path in glob.glob(outdir_xml + "/*"):
    
    book_name = book_path[-3:]
    print(book_name)

    root_TEI = etree.fromstring( """<TEI xmlns="http://www.tei-c.org/ns/1.0">
    <teiHeader>
		<fileDesc>
			<titleStmt>
			<title type="idno">
			<idno type="string">""" + book_name + """</idno>
            </title>
            </titleStmt>
        </fileDesc>
    </teiHeader>
 </TEI>
                                     """)

    standOff_element = etree.fromstring( '<standOff xml:id="b.' + book_name + '.POSFreenling" type="morphosyntactic">\n</standOff>')

    for verse_path in sorted(glob.glob(outdir_xml + book_name + "/*")):
        
        verse_id = verse_path[-13:]
               
        with open(verse_path, "r", errors="replace", encoding="utf-8") as fin:
            content_str = fin.read()

            content_str = '\n<seg xml:id="' + verse_id + '.POS.1" corresp="#' + verse_id + '">\n' + content_str + '\n</seg>\n'

            #print(content_str)
            
            root = etree.fromstring(content_str)
    
            standOff_element.append(root)
            

            
            
    xslt = etree.XSLT(etree.XML('''<?xml version="1.0"?>
        <xsl:stylesheet version="1.0" xmlns:xsl="http://www.w3.org/1999/XSL/Transform" xmlns="http://www.tei-c.org/ns/1.0">
        <xsl:output method="text" encoding="utf8" />
        <xsl:template match="@*|node()">
            <xsl:copy>
                <xsl:apply-templates select="@*|node()" />
            </xsl:copy>
        </xsl:template>

        <xsl:template match="token">
            <w><xsl:apply-templates select="@* | node()" /></w>
        </xsl:template>

        <xsl:template match="token[./@pos='punctuation']">
            <pc><xsl:apply-templates select="@* | node()" /></pc>
        </xsl:template>


        <xsl:template match="sentence">
            <s><xsl:apply-templates select="@* | node()" /></s>
        </xsl:template>

        <xsl:template match="sentence/@id | token/@id">
            <xsl:attribute name="n">
                <xsl:value-of select="."/>
            </xsl:attribute>
        </xsl:template>


        </xsl:stylesheet>
        '''))


    standOff_element = xslt(standOff_element)

    standOff_element =  standOff_element.getroot()
    
    """
    #print(etree.tostring(root_standOff, pretty_print=True, encoding="unicode"))
    element_text = bible_root.xpath(".//tei:TEI[.//tei:idno[@type='string']/text() = '" + book_name + "']/tei:text" ,  namespaces = namespaces )[0]
    element_text
    element_TEI = element_text.getparent()
    element_TEI.insert(element_TEI.index(element_text)+1, root_standOff)
    """

    #element_text = root_TEI.xpath(".//tei:text//tei:text", namespaces = namespaces)[0]
    print(element_text)


    root_TEI.insert(2, standOff_element)
    

    root_TEI.getroottree().write('./../../../annotated/' + book_name + '.xml', encoding="utf8")<jupyter_output>1CH
<Element {http://www.tei-c.org/ns/1.0}text at 0x7f44a8a1c648>
JER
<Element {http://www.tei-c.org/ns/1.0}text at 0x7f44a8a1c648>
1CO
<Element {http://www.tei-c.org/ns/1.0}text at 0x7f44a8a1c648>
HAB
<Element {http://www.tei-c.org/ns/1.0}text at 0x7f44a8a1c648>
JOB
<Element {http://www.tei-c.org/ns/1.0}text at 0x7f44a8a1c648>
ZEP
<Element {http://www.tei-c.org/ns/1.0}text at 0x7f44a8a1c648>
JON
<Element {http://www.tei-c.org/ns/1.0}text at 0x7f44a8a1c648>
PSA
<Element {http://www.tei-c.org/ns/1.0}text at 0x7f44a8a1c648>
2TI
<Element {http://www.tei-c.org/ns/1.0}text at 0x7f44a8a1c648>
1SA
<Element {http://www.tei-c.org/ns/1.0}text at 0x7f44a8a1c648>
EST
<Element {http://www.tei-c.org/ns/1.0}text at 0x7f44a8a1c648>
ZEC
<Element {http://www.tei-c.org/ns/1.0}text at 0x7f44a8a1c648>
LUK
<Element {http://www.tei-c.org/ns/1.0}text at 0x7f44a8a1c648>
ECC
<Element {http://www.tei-c.org/ns/1.0}text at 0x7f44a8a1c648>
EPH
<Element {http://www.tei-c.org/ns/1.0}text at 0x7f44a8a1c648>
AMO
<Eleme[...]
{'fraction_non_alphanumeric': 0.17317876055531703, 'fraction_numerical': 0.10104479748103622, 'mean_word_length': 2.1182507809013833, 'pattern_counts': {'":': 0, '<': 89, '<?xml version=': 1, '>': 88, 'https://': 0, 'lorem ipsum': 0, 'www.': 19, 'xml': 18}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 19, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '618875', 'license': 'CC-BY-4.0', 'n_tokens_mistral': 3268, 'n_tokens_neox': 2809, 'n_words': 413, 'path': 'code/python/notebooks/20210523_annotate_Freeling.ipynb', 'repo_name': 'morethanbooks/XML-TEI-Bible', 'size': '6987'}