<jupyter_start><jupyter_text>https://annotation.github.io/text-fabric/tf/convert/walker.htmlhttps://nbviewer.org/github/annotation/banks/blob/master/programs/convert.ipynb Nodetypes:1 sign2 word3 verse4 chapter5 book6 documentdicts needed:1. otype2. oslots3. book4. chapter5. verse6. trans [transcription]7.<jupyter_code>import os
import re

from tf.fabric import Fabric
from tf.convert.walker import CV<jupyter_output><empty_output><jupyter_text>Basic exploration of characters in document<jupyter_code>char_set = set()

with open('transcriptions.txt', 'r', encoding='utf-8') as file:
    for line in file:
        line = line.strip('\n')
        for char in line:
            char_set.add(char)
            
#char_set<jupyter_output><empty_output><jupyter_text>Conversion to TF<jupyter_code>#BASE = os.path.expanduser('~/github')
#ORG = 'KU'
#REPO = 'NT'
#RELATIVE = 'tf'

TF_DIR = os.path.expanduser(f'{BASE}/{ORG}/{REPO}/{RELATIVE}')

VERSION = '0.1'

#TF_PATH = f'{TF_DIR}/{VERSION}'
TF_PATH = './tf'
TF = Fabric(locations=TF_PATH, silent=True)
slotType = 'sign'
generic = {
    'name': 'TRANSCRIPTION OF THE MANUSCRIPTS CONTAINING THE NEW TESTAMENT LETTER OF JUDE',
    'author': '<NAME>',
    'contributors': '<NAME>, <NAME>',
    'converter': '<NAME>',
    'source': 'Data collected and edited by <NAME>',
    'url': 'https://easy.dans.knaw.nl/ui/datasets/id/easy-dataset:52008',
    'version': '0.1'
}
otext = {
    'fmt:text-orig-full': '{greek} ',
    'fmt:line-default': '{greek:XXX} ',
    'sectionTypes': 'book,chapter,verse',
    'sectionFeatures': 'book,chapter,verse',

}

# 'fmt:line-term': 'line#{terminator} ',
#    'structureTypes': 'book,chapter',
#    'structureFeatures': 'book,chapter',
intFeatures = {
  'chapter',
  'verse',
  'ns',
}
featureMeta = {
    'name': {
        'description': 'name of the codex or papyrus',
    },
    'book': {
        'description': 'name of a biblical book',
    },
    'chapter': {
        'description': 'chapter number in a biblical book',
    },
    'verse': {
        'description': 'verse number',
    },
    'greek': {
        'description': 'representation of a word/character in the text',
    },
    'ns': {
        'description': 'nomina sacra, divine name, often the name is abbreviated'
    },
    'vtype': {
        'description': 'verse type, has value "" if it is a verse with text, "0" if verse is not in manuscript (often verses 0 and 26)'
    },
    'wtype': {
        'description': 'word type, has value "" if it is an ordinary greek word, else indicates a lacuna in the text'
    },
    'stype': {
        'description': 'sign type, see word type'
    },
}
class Verse:
    def __init__(self, verse_string):
        """
        verse_string: string representation of a Greek verse.
        """
        self.verse_string = verse_string 
        
    def process_lacunae(self):
        """
        Checks if a verse is lacking,
        indicated with '&lac;' or '&lacfilm;' in the text.
        Returns:
        '': this is a normal verse
        'lac' or '&lacfilm;': there is a lacuna
        '0': empty verse, it is not in the manuscript
        """
        if self.verse_string == '&lac;':
            self.verse_string = ''
            return 'lac'
        
        elif self.verse_string == '&lacfilm;':
            self.verse_string = ''
            return 'lacfilm'
        
        elif self.verse_string == '0':
            self.verse_string = ''
            return 'empty'
        
        else:
            return ''
        

class Word:
    def __init__(self, word_string):
        """
        word_string: string representation of a Greek word in the text
        """
        self.word_string = word_string
        
    def process_lacunae(self):
        """
        Checks if a word is a lucuna,
        indicated with '&lac;' or '&lacfilm;' in the text.
        Returns:
        'lac' or '&lacfilm;': there is a lacuna
        '0': empty verse
        '0': Normal word
        
        """
        if self.word_string == '&lac;':
            self.word_string = ''
            return 'lac'
        
        elif self.word_string == '&lacfilm;':
            self.word_string = ''
            return 'lacfilm'
        
        elif self.word_string == '0':
            self.word_string = ''
            return 'empty'
        
        else:
            return ''
    
    def process_nomina_sacra(self):
        """
        If word has Nomina Sacra tags ('<NS>XXX</NS>'), these are removed.
        The cleaned string is returned, together with an integer:
        0: the word IS NOT a NS
        1: the word IS a NS
        """
        if self.word_string.startswith('<NS>'):
            self.word_string = self.word_string.replace('<NS>', '').replace('</NS>', '')
            return self.word_string, 1
        else:
            return self.word_string, 0
        
    
all_lacs = set()

def director(cv):
    
    no_line = 0
    
    #counter = dict(
    #  chapter=0,
    #  verse=0,
    #)
    cur = dict(
      document=None,
      book=None,
      chapter=None,
      verse=None,
      word=None
    )
    
    with open('transcriptions.txt', 'r', encoding='utf-8') as file:
        for line in file:
            line = line.strip('\n')
            
            if not line:
                continue
                
            if line.startswith('<S>'):
                
            
                for ntp in ('verse', 'chapter', 'book', 'document'):
                    cv.terminate(cur[ntp])
                    cur[ntp] = None         
                document_name = line.strip('<S>').strip('</S>')
                cur['document'] = cv.node('document')
                cv.feature(cur['document'], name=document_name)
                
                cur['book'] = cv.node('book')  
                cv.feature(cur['book'], book='letter_of_jude') 
                
                cur['chapter'] = cv.node('chapter')  
                cv.feature(cur['chapter'], chapter=1)
                
                
            elif line.startswith('<V '):
                
                # One line starts with: '<V [20]>'
                if line.startswith('<V ['):
                    line = line.replace('<V [', '<V ')
                if ']>' in line:
                    line = line.replace(']>', '>')
                
                verse_nr_re = re.compile(r'<V [0-9]+>')
                verse = verse_nr_re.search(line)
                verse = verse.group() 
                verse_number = verse.strip('<V ').strip('>')
                
                
                line = line.replace(verse, '').strip()
                if not line:
                    line = '0'

                verse = Verse(line)
                vtype = verse.process_lacunae()
                
                cur['verse'] = cv.node('verse')
                cv.feature(cur['verse'], verse=int(verse_number))
                cv.feature(cur['verse'], vtype=vtype)
                
                for w in line.split():
                    if 'lac' in w:
                        all_lacs.add(w)
                        
                    word = Word(w)
                    cur['word'] = cv.node('word')
                    
                    wtype = word.process_lacunae()
                    cv.feature(cur['word'], wtype=wtype)
                    
                    cleaned_w, ns = word.process_nomina_sacra()
                    cv.feature(cur['word'], ns=ns)
                    cv.feature(cur['word'], greek=cleaned_w)
                    
                    
                    for char in w:
                        
                        s = cv.slot()
                        cv.feature(s, greek=char)
                        cv.feature(s, stype=wtype)
                        
                    cv.terminate(cur['word'])

                cv.terminate(cur['verse'])
            
        for ntp in ('verse', 'chapter', 'book', 'document'):
            cv.terminate(cur[ntp])
            
cv = CV(TF)

good = cv.walk(
    director,
    slotType,
    otext=otext,
    generic=generic,
    intFeatures=intFeatures,
    featureMeta=featureMeta,
)

good
all_lacs<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13356122573556342, 'fraction_numerical': 0.0056159199120986445, 'mean_word_length': 1.6006349206349206, 'pattern_counts': {'":': 0, '<': 33, '<?xml version=': 0, '>': 32, 'https://': 3, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '635366', 'license': 'MIT', 'n_tokens_mistral': 2539, 'n_tokens_neox': 2431, 'n_words': 570, 'path': 'convert_letter_of_jude_to_tf.ipynb', 'repo_name': 'MartijnNaaijer/LetterOfJude', 'size': '8226'}