<jupyter_start><jupyter_text>d2v_general  1. Get token example from item
> Notes: use geneal('linear') tokenizition method, which means do not parse formulas<jupyter_code># coding: utf-8
import json
from tqdm import tqdm
from EduNLP.SIF.segment import seg
from EduNLP.SIF.tokenization import tokenize
from EduNLP.Pretrain import GensimWordTokenizer

def load_items():
    test_items = [
        {'ques_content':'有公式$\\FormFigureID{wrong1?}$和公式$\\FormFigureBase64{wrong2?}$，如图$\\FigureID{088f15ea-8b7c-11eb-897e-b46bfc50aa29}$,若$x,y$满足约束条件$\\SIFSep$，则$z=x+7 y$的最大值为$\\SIFBlank$'},
        {'ques_content':'如图$\\FigureID{088f15ea-8b7c-11eb-897e-b46bfc50aa29}$,若$x,y$满足约束条件$\\SIFSep$，则$z=x+7 y$的最大值为$\\SIFBlank$'},
        {'ques_content':'<div>Below is a discussion on a website.<br><table border=\1'},
    ]
    for line in test_items:
        yield line
    # """or use your jsonfile like this"""
    # work_file_path = "../../../data/OpenLUNA.json"
    # with open(work_file_path, 'r', encoding="utf-8") as f:
    #     for line in f:
    #         yield json.loads(line)


token_items = []
for item in tqdm(load_items(), "sifing"): 
    # transform content into special marks('g','m','a','s'), except text('t') and formula('f').
    # 'general' means symbolize the Formula in figure format and use 'linear' method for formula segmentation 
    tokenizer = GensimWordTokenizer(symbol="gmas", general=True)
    token_item = tokenizer(item["ques_content"])
    if token_item:
        token_items.append(token_item.tokens)

token_items[0]
len(token_items)<jupyter_output><empty_output><jupyter_text>2. Load Model and test item<jupyter_code>from urllib.request import urlopen
import os,os.path
import zipfile


def down_file(subject):
  url = "http://base.ustc.edu.cn/data/model_zoo/EduNLP/d2v/general_" + subject + "_256.zip"
  file_name = "../../../data/d2v/zip/" + url.split('/')[-1]
  u = urlopen(url)

  f = open(file_name, 'wb')  
  file_info = u.getheaders()
  print("[down file] file info : ", file_info)
  file_size_dl = 0
  block_sz = 8192
  while True: 
    buffer = u.read(block_sz) 
    if not buffer: 
      break
    file_size_dl += len(buffer) 
    f.write(buffer) 
  f.close()
  print("[down file] finish !")


def unzip_file(subject):
  zipfilename = "../../../data/d2v/zip/general_" + subject + "_256.zip"
  unziptodir = "../../../data/d2v/models/"
  print("[unzip file] start ...")
  if not os.path.exists(unziptodir):
      os.mkdir(unziptodir)
  zfobj = zipfile.ZipFile(zipfilename)
  for name in zfobj.namelist():
    name = name.replace('\\','/')
    if name.endswith('/'):
      continue
    ext_filename = os.path.join(unziptodir, name)
    ext_filename = ext_filename.replace('\\','/')
    print("save ======> ",ext_filename)
    ext_path= os.path.dirname(ext_filename)
    if not os.path.exists(ext_path) :
        os.mkdir(ext_path)
    outfile = open(ext_filename, 'wb')
    outfile.write(zfobj.read(name))
    outfile.close()
  print("[unzip file] finish !")

def getData(subject = "english"):
  """ subject = english | liberal | science |all """
  down_file(subject)
  unzip_file(subject)


work_subject = "science"
getData(work_subject)
print(token_items[0])
from EduNLP.Vector import D2V
work_subject = "science"
d2v = D2V("../../../data/d2v/models/general_" + work_subject +"_256/general_" + work_subject + "_256.bin")
print(d2v.vector_size)
d2v(token_items[0])<jupyter_output>256
{'fraction_non_alphanumeric': 0.1585261353898886, 'fraction_numerical': 0.023421879463010567, 'mean_word_length': 3.4217171717171717, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 12, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '153136', 'license': 'Apache-2.0', 'n_tokens_mistral': 1385, 'n_tokens_neox': 1318, 'n_words': 246, 'path': 'examples/pretrain/gensim/d2v_general.ipynb', 'repo_name': 'nnnyt/EduNLP', 'size': '3501'}