<jupyter_start><jupyter_text>Synonym<jupyter_code>synonymFile = 'fangzhi_synonym.owl'

read_synonym = '{0}/{1}'.format(filepath, synonymFile)
import json

with open(read_synonym , 'r') as reader:
    jf = json.loads(reader.read())

catagory = {}

# Read all nodes in first layer
for obj in jf:
    tmpId = obj['@id']
    pos = tmpId.rfind('#')
    node = tmpId[pos+1:len(tmpId)]
    
    if '(' not in node:
        continue
        
    node = node.replace('(', '').replace(')', '')
    catagory[node] = {'parent':[], 'synonym':[node]}

catagory['未歸類'] = {'parent':[], 'synonym':[]}

for obj in jf:
    if 'http://www.w3.org/2000/01/rdf-schema#subClassOf' not in obj:
        continue
        
    tmpId = obj['@id']
    pos = tmpId.rfind('#')
    subClass = tmpId[pos+1:len(tmpId)]
    subClass = subClass.replace('(', '').replace(')', '')

    tmpSubClassOf = obj['http://www.w3.org/2000/01/rdf-schema#subClassOf'][0]['@id']
    pos = tmpSubClassOf.rfind('#')
    parent = tmpSubClassOf[pos+1:len(tmpSubClassOf)]
    parent = parent.replace('(', '').replace(')', '')
    
    if parent not in catagory:
        print("ERROR!!", parent, subClass)
        
    if subClass not in catagory[parent]['synonym']:
        catagory[parent]['synonym'].append(subClass)

print(catagory)<jupyter_output>{'事紀': {'parent': [], 'synonym': ['事紀', '事記', '紀事', '記事', '邑紀事']}, '亭榭': {'parent': [], 'synonym': ['亭榭']}, '人物': {'parent': [], 'synonym': ['人物']}, '仙釋': {'parent': [], 'synonym': ['仙釋']}, '倉儲': {'parent': [], 'synonym': ['倉儲', '倉貯', '積儲', '積貯']}, '公署': {'parent': [], 'synonym': ['公署', '廨署', '文署', '署廨']}, '兵餉': {'parent': [], 'synonym': ['兵餉']}, '冢墓': {'parent': [], 'synonym': ['冢墓', '丘墓', '塚墓', '墓冢', '墓塜', '墳墓', '邱墓']}, '叢談': {'parent': [], 'synonym': ['叢談', '叢譚']}, '名勝': {'parent': [], 'synonym': ['名勝']}, '喪葬': {'parent': [], 'synonym': ['喪葬']}, '圖經': {'parent': [], 'synonym': ['圖經']}, '坊巷': {'parent': [], 'synonym': ['坊巷']}, '域市': {'parent': [], 'synonym': ['域市', '坊都', '城市', '都坊']}, '堡村': {'parent': [], 'synonym': ['堡村']}, '墟市': {'parent': [], 'synonym': ['墟市', '圩埠', '圩市', '圩集', '墟埠', '墟場', '墟屯', '墟集', '集市']}, '壇廟': {'parent': [], 'synonym': ['壇廟', '廟祠', '祠廟']}, '存留': {'parent': [], 'synonym': ['存留']}, '學宮': {'parent': [], 'synonym': ['學宮']}, '學校': {'parent': [], 'synonym': ['學校']}[...]<jupyter_text>Catagory<jupyter_code>catagoryFile = 'fangzhi_category.owl'

read_catagory = '{0}/{1}'.format(filepath, catagoryFile)
import json

# catagory = {}
with open(read_catagory , 'r') as reader:
    jf = json.loads(reader.read())
      
def findBelong(INclass, obj):
    if INclass in obj:
        return INclass

    for key in obj:
        if INclass in obj[key]['synonym']:
            return key
    return ""
    
for obj in jf:
    if 'http://www.w3.org/2000/01/rdf-schema#subClassOf' not in obj:
        continue
        
    tmpId = obj['@id']
    pos = tmpId.rfind('#')
    subClass = tmpId[pos+1:len(tmpId)]

    tmpSubClassOf = obj['http://www.w3.org/2000/01/rdf-schema#subClassOf'][0]['@id']
    pos = tmpSubClassOf.rfind('#')
    parent = tmpSubClassOf[pos+1:len(tmpSubClassOf)]
    
    OUTclass = findBelong(subClass, catagory)
    if OUTclass == "未歸類":
        continue
        
    if OUTclass == "":
        catagory[subClass] = {'parent':[parent], 'synonym':[subClass]}
    else:
        catagory[OUTclass]['parent'].append(parent)

print(catagory)<jupyter_output>{'事紀': {'parent': ['編年', '藝文'], 'synonym': ['事紀', '事記', '紀事', '記事', '邑紀事']}, '亭榭': {'parent': [], 'synonym': ['亭榭']}, '人物': {'parent': [], 'synonym': ['人物']}, '仙釋': {'parent': [], 'synonym': ['仙釋']}, '倉儲': {'parent': [], 'synonym': ['倉儲', '倉貯', '積儲', '積貯']}, '公署': {'parent': [], 'synonym': ['公署', '廨署', '文署', '署廨']}, '兵餉': {'parent': ['兵防'], 'synonym': ['兵餉']}, '冢墓': {'parent': ['勝蹟'], 'synonym': ['冢墓', '丘墓', '塚墓', '墓冢', '墓塜', '墳墓', '邱墓']}, '叢談': {'parent': [], 'synonym': ['叢談', '叢譚']}, '名勝': {'parent': [], 'synonym': ['名勝']}, '喪葬': {'parent': [], 'synonym': ['喪葬']}, '圖經': {'parent': [], 'synonym': ['圖經']}, '坊巷': {'parent': [], 'synonym': ['坊巷']}, '域市': {'parent': [], 'synonym': ['域市', '坊都', '城市', '都坊']}, '堡村': {'parent': [], 'synonym': ['堡村']}, '墟市': {'parent': [], 'synonym': ['墟市', '圩埠', '圩市', '圩集', '墟埠', '墟場', '墟屯', '墟集', '集市']}, '壇廟': {'parent': [], 'synonym': ['壇廟', '廟祠', '祠廟']}, '存留': {'parent': [], 'synonym': ['存留']}, '學宮': {'parent': [], 'synonym': ['學宮']}, '學校': {'parent': [], [...]<jupyter_text>find catagory from gatateers<jupyter_code># for idx in range(0, len(workDf)):
gatateerCata = {}

curBookId = ""
curBookName = ""

bookList = {}
bookHierarchy = {}
curLayers = {'1':'', '2':''}
for idx in range(0, len(workDf)):
    row = workDf.loc[idx]

    bookName = row['bookName']
    sectionName = row['sectionName']
    level = row['level']
    bookId = row['bookId']
    
    if curBookId == "":
        curBookId = bookId
        curBookName = bookName

    if curBookId != bookId:
        bookList[curBookName] = {'root':bookHierarchy, 'bookId':curBookId}
        
        curBookId = bookId
        curBookName = bookName
        bookHierarchy = {}
        curLayers['1'] = ''
        curLayers['2'] = ''

    if idx >= len(workDf)-1:
        bookList[bookName] = {'root':bookHierarchy, 'bookId':curBookId}
        break

    if level == 1:
        if sectionName not in bookHierarchy:
            bookHierarchy[sectionName] = {}
        curLayers['1'] = sectionName
    elif level == 2:
        try:
            if curLayers['1'] == '':
                curLayers['1'] = bookName
            if curLayers['1'] not in bookHierarchy:
                bookHierarchy[curLayers['1']] = {}
            if sectionName not in bookHierarchy[curLayers['1']]:
                bookHierarchy[curLayers['1']][sectionName] = {}
            curLayers['2'] = sectionName
        except:
            print("ERROR2:", idx, bookId, bookName, sectionName, level)
            break
    elif level == 3:
        try:
            if curLayers['1'] not in bookHierarchy:
                bookHierarchy[curLayers['1']] = {}
            if curLayers['2'] not in bookHierarchy[curLayers['1']]:
                bookHierarchy[curLayers['1']][curLayers['2']] = {}
            if sectionName not in bookHierarchy[curLayers['1']][curLayers['2']]:
                bookHierarchy[curLayers['1']][curLayers['2']][sectionName] = {}

        except:
            errRow = [idx, bookId, bookName, sectionName, level]
            print("ERROR3:", idx, bookId, bookName, sectionName, level)
            break

print(len(bookList))
# bookName: {'root':object, 'bookId':string}
print(bookList['甕安縣志'])
bookObj = bookList['甕安縣志']['root']

def findKeyInObj(key, obj):
#     print(key, obj)
    if key in obj:
#         print("found!!", key)
        return key
    
    for nKey in obj:
        if obj[nKey] != {}:
            res = findKeyInObj(key, obj[nKey])
            if res != '':
                return res
    return ''
    
# for key in bookObj:
key = findKeyInObj('農業', bookObj)
print(key)
# print(bookObj['卷首']['序'] == {})
firstTypeBooks = []
for bookName in bookList:
    for title in bookList[bookName]['root']:
        if '卷一' in title:
            firstTypeBooks.append(bookName)
            break

print(len(firstTypeBooks))
firstTypeBooks<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.2467263955892488, 'fraction_numerical': 0.009510682288077188, 'mean_word_length': 3.339712918660287, 'pattern_counts': {'":': 3, '<': 11, '<?xml version=': 0, '>': 12, 'https://': 0, 'lorem ipsum': 0, 'www.': 4, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '111646', 'license': 'MIT', 'n_tokens_mistral': 3139, 'n_tokens_neox': 3022, 'n_words': 574, 'path': 'Gazetteers/parse_gazetteers.ipynb', 'repo_name': 'VincentCheng34/StudyOnPython', 'size': '7255'}