<jupyter_start><jupyter_text>1 Pre-Processing * Open the Google Drive  * Set the Default Value<jupyter_code>from google.colab import drive
drive.mount('/content/drive')

false = False
true = True<jupyter_output>Go to this URL in a browser: https://accounts.google.com/o/oauth2/auth?client_id=947318989803-6bn6qk8qdgf4n4g3pfee6491hc0brc4i.apps.googleusercontent.com&redirect_uri=urn%3aietf%3awg%3aoauth%3a2.0%3aoob&response_type=code&scope=email%20https%3a%2f%2fwww.googleapis.com%2fauth%2fdocs.test%20https%3a%2f%2fwww.googleapis.com%2fauth%2fdrive%20https%3a%2f%2fwww.googleapis.com%2fauth%2fdrive.photos.readonly%20https%3a%2f%2fwww.googleapis.com%2fauth%2fpeopleapi.readonly

Enter your authorization code:
··········
Mounted at /content/drive<jupyter_text>1.1 Read the Dataset * Read the Raw Data from Google Drive<jupyter_code>import codecs
import json

def readFile(file):
# Read as requeired format
  dicts = []
  with codecs.open(file, 'r', 'utf-8') as items:
    for item in items:
      item = item.strip()
      dicts.append(json.loads(item))
  return dicts<jupyter_output><empty_output><jupyter_text>1.2 Format the Dataset * Remove the irrelevant symbols* Intergrate the data<jupyter_code>def unzip(dicts):
# Remove the irrelevant info
  dictTran = []

  for i in range(len(dicts)):
    t3 = json.dumps(dicts[i]['meta']['term_of_imprisonment'])[1:]
    del dicts[i]["meta"]["term_of_imprisonment"]
    t2 = str(dicts[i]['meta']).replace('{','').replace('}',',')
    t1 = "{'fact':'"+dicts[i]['fact'].replace("'",'"')+"',"
    dictTran.append(eval(t1+t2+t3))

  return dictTran<jupyter_output><empty_output><jupyter_text>* Build the CSV File<jupyter_code>import csv

def buildCsv(headers,dictTran,filePath):
# Write content to csv file
# Headers: category name, dictTran: each line contents

  with open(filePath,'w',newline='')as f:
    fCsv = csv.DictWriter(f,headers) # write header
    fCsv.writeheader()
    fCsv.writerows(dictTran) # write content<jupyter_output><empty_output><jupyter_text>* Run Functions<jupyter_code># File path
fSource = '/content/drive/My Drive/WORKS/data/Dataset/data_'
fTarget = '/content/drive/My Drive/WORKS/data/Dataset/'
headers = ['fact','accusation','criminals','punish_of_money',
    'relevant_articles','death_penalty','imprisonment','life_imprisonment']

# Run the functions
buildCsv(headers,unzip(readFile(fSource+'train.json')),(fTarget+'train.csv'))
buildCsv(headers,unzip(readFile(fSource+'test.json')),(fTarget+'test.csv'))
buildCsv(headers,unzip(readFile(fSource+'valid.json')),(fTarget+'valid.csv'))<jupyter_output><empty_output><jupyter_text>1.3 Format Keywords<jupyter_code># Disposal keywords
fLegalTerms = open("/content/drive/My Drive/WORKS/data/rawLegalTerms.txt","r")
legalTerms = fLegalTerms.read()
fLegalTerms.close() 
legalTerms=legalTerms.split('、')

with open('/content/drive/My Drive/WORKS/data/LegalTerms.txt','w') as f: 
  for i in legalTerms:
    f.write(i+"\n")<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.15716694772344014, 'fraction_numerical': 0.025295109612141653, 'mean_word_length': 7.103825136612022, 'pattern_counts': {'":': 0, '<': 24, '<?xml version=': 0, '>': 24, 'https://': 1, 'lorem ipsum': 0, 'www.': 4, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '6', 'dir': 'jupyter-structured-clean-dedup', 'id': '183929', 'license': 'Apache-2.0', 'n_tokens_mistral': 1170, 'n_tokens_neox': 1134, 'n_words': 169, 'path': 'ProcessRawData.ipynb', 'repo_name': 'TechSang/Chinese-Legal-Entity-Recognition', 'size': '2965'}