<jupyter_start><jupyter_text>https://github.com/boudinfl/pke/tree/master/tests<jupyter_code>get_key_phrase(spacy_model, df_2018.loc[50][separation])
get_key_phrase(spacy_model, df_2018.loc[50][separation])
get_key_phrase(spacy_model, df_2018.loc[50][separation])
get_key_phrase(spacy_model, df_2018.loc[50][separation])
get_key_phrase(spacy_model, df_2018.loc[50][separation])
!pip install git+https://github.com/boudinfl/pke.git
!pip install ginza
!pip install nltk
!pip install spacy==2.3.6
!python -m nltk.downloader stopwords
!pip install mojimoji
import pke
import ginza
import nltk
import spacy
import pandas as pd
pke.base.lang_stopwords['ja_ginza'] = 'japanese'
import re
import mojimoji
def normalize_text(text):
  # 全部全角に変換
  result = mojimoji.han_to_zen(text)

  # 全角から半角に変換（カナは除く）
  result = mojimoji.zen_to_han(result, kana=False)

  # 全ての文字を小文字に変換
  result = result.lower()
  
  return re.sub(' ','', result)
  # return re.sub(' ', '', mojimoji.zen_to_han(text)).upper().split()
path = '/content/drive/MyDrive/Colab Notebooks/pre-judgit/data.json'
with open(path, encoding='utf8')as f:
  df = pd.read_json(f)

df.info()
df_2018 = df.iloc[0:0]
cnt = 0
for i in range(len(df)):
  if df.loc[i]['公開年度'] == 2018:
    df_2018.loc[cnt] = df.loc[i]
    cnt += 1
df_2018
#分かち書きする文
separation = "事業の目的"
#separation = "事業概要"
def get_key_phrase(spacy_model, text, n = 5):
    """キーフレーズ処理

    Args:
        text ([type]): キーフレーズ対象の文字列
        n (int, optional): [description]. Defaults to 10.
        gc_collect (bool, optional): [description]. Defaults to False.

    Returns:
        [type]: [description]
    """
    #MultipartiteRankなどのキーフレーズの手法については以下のサイトが参考になります
    #https://www.ogis-ri.co.jp/otc/hiroba/technical/similar-document-search/part5.html
    extractor = pke.unsupervised.MultipartiteRank()
    extractor.load_document(input=text, language='ja_ginza', normalization=None, spacy_model= spacy_model )
    extractor.candidate_selection( pos={'NOUN', 'PROPN', 'ADJ', 'NUM'})
    extractor.candidate_weighting(threshold=0.74, method='average', alpha=1.1)
    key_phrase = extractor.get_n_best(n)
    
    return  key_phrase
    
    
#spacyに設定されているstopwordを使う
from  spacy.lang.ja import stop_words

spacy_model = spacy.load("ja_ginza")  # GiNZAモデルの読み込み
print(spacy_model)
stopwords = list(stop_words.STOP_WORDS)
nltk.corpus.stopwords.words_org = nltk.corpus.stopwords.words
nltk.corpus.stopwords.words = lambda lang : stopwords if lang == 'japanese' else nltk.corpus.stopwords.words_org(lang)
df_pke = {}
df_enpty = []
for i in range(len(df_2018)):
  if get_key_phrase(spacy_model, df_2018.loc[i][separation]) == []:
    print(i)
    df_pke[df_2018.loc[i]['プロジェクトID']] = ''
    df_enpty.append(df_2018.loc[i]['プロジェクトID'])
  else:
    pke_test = {}
    for words in get_key_phrase(spacy_model, df_2018.loc[i][separation]):
      for word in words[0].split():
        if word in pke_test:
          pke_test[word] += words[1]
        else:
          pke_test[word] = words[1]

    df_pke[df_2018.loc[i]['プロジェクトID']] = pke_test
import json
obj = []
for i in range(len(df_2018)):
  word_score = []
  if df_pke[df_2018.loc[i]['プロジェクトID']] == '':
    word_score.append({"word" : '',
                       "score" : 0})
  else:
    tmp_list = sorted(df_pke[df_2018.loc[i]['プロジェクトID']].items(), key=lambda x: x[1], reverse=True)
    for data in tmp_list:
      word_score.append({"word" : data[0],
                        "score" : data[1]})
    obj.append({'プロジェクトID':int(df_2018.loc[i]['プロジェクトID']),
                'MultipartiteRank':word_score})
  
with open('/content/drive/MyDrive/Colab Notebooks/pre-judgit/MultipartiteRank2018.json', 'w', encoding='utf8')as f:
  json.dump(obj, f, indent = 2, ensure_ascii=False)<jupyter_output><empty_output><jupyter_text>TextRank<jupyter_code>def get_key_phrase(spacy_model, text, n = 5):
    """キーフレーズ処理

    Args:
        text ([type]): キーフレーズ対象の文字列
        n (int, optional): [description]. Defaults to 10.
        gc_collect (bool, optional): [description]. Defaults to False.

    Returns:
        [type]: [description]
    """
    #MultipartiteRankなどのキーフレーズの手法については以下のサイトが参考になります
    #https://www.ogis-ri.co.jp/otc/hiroba/technical/similar-document-search/part5.html
    extractor = pke.unsupervised.TextRank()
    extractor.load_document(input=text, language='ja_ginza', normalization=None, spacy_model= spacy_model )
    extractor.candidate_selection( pos={'NOUN', 'PROPN', 'ADJ', 'NUM'})
    extractor.candidate_weighting( pos={'NOUN', 'PROPN', 'ADJ', 'NUM'})
    key_phrase = extractor.get_n_best(n)
    
    return  key_phrase
    
    
#spacyに設定されているstopwordを使う
from  spacy.lang.ja import stop_words

spacy_model = spacy.load("ja_ginza")  # GiNZAモデルの読み込み
print(spacy_model)
stopwords = list(stop_words.STOP_WORDS)
nltk.corpus.stopwords.words_org = nltk.corpus.stopwords.words
nltk.corpus.stopwords.words = lambda lang : stopwords if lang == 'japanese' else nltk.corpus.stopwords.words_org(lang)
df_pke = {}
df_enpty = []
for i in range(len(df_2018)):
  if get_key_phrase(spacy_model, df_2018.loc[i][separation]) == []:
    print(i)
    df_pke[df_2018.loc[i]['プロジェクトID']] = ''
    df_enpty.append(df_2018.loc[i]['プロジェクトID'])
  else:
    pke_test = {}
    for words in get_key_phrase(spacy_model, df_2018.loc[i][separation]):
      for word in words[0].split():
        if word in pke_test:
          pke_test[word] += words[1]
        else:
          pke_test[word] = words[1]

    df_pke[df_2018.loc[i]['プロジェクトID']] = pke_test
import json
obj = []
for i in range(len(df_2018)):
  word_score = []
  if df_pke[df_2018.loc[i]['プロジェクトID']] == '':
    word_score.append({"word" : '',
                       "score" : 0})
  else:
    tmp_list = sorted(df_pke[df_2018.loc[i]['プロジェクトID']].items(), key=lambda x: x[1], reverse=True)
    for data in tmp_list:
      word_score.append({"word" : data[0],
                        "score" : data[1]})
    obj.append({'プロジェクトID':int(df_2018.loc[i]['プロジェクトID']),
                'TextRank':word_score})
  
with open('/content/drive/MyDrive/Colab Notebooks/pre-judgit/TextRank2018.json', 'w', encoding='utf8')as f:
  json.dump(obj, f, indent = 2, ensure_ascii=False)<jupyter_output><empty_output><jupyter_text>SingleRank<jupyter_code>def get_key_phrase(spacy_model, text, n = 5):
    """キーフレーズ処理

    Args:
        text ([type]): キーフレーズ対象の文字列
        n (int, optional): [description]. Defaults to 10.
        gc_collect (bool, optional): [description]. Defaults to False.

    Returns:
        [type]: [description]
    """
    #MultipartiteRankなどのキーフレーズの手法については以下のサイトが参考になります
    #https://www.ogis-ri.co.jp/otc/hiroba/technical/similar-document-search/part5.html
    extractor = pke.unsupervised.SingleRank()
    extractor.load_document(input=text, language='ja_ginza', normalization=None, spacy_model= spacy_model )
    extractor.candidate_selection( pos={'NOUN', 'PROPN', 'ADJ', 'NUM'})
    extractor.candidate_weighting( window=10, pos={'NOUN', 'PROPN', 'ADJ', 'NUM'})
    key_phrase = extractor.get_n_best(n)
    
    return  key_phrase
    
    
#spacyに設定されているstopwordを使う
from  spacy.lang.ja import stop_words

spacy_model = spacy.load("ja_ginza")  # GiNZAモデルの読み込み
print(spacy_model)
stopwords = list(stop_words.STOP_WORDS)
nltk.corpus.stopwords.words_org = nltk.corpus.stopwords.words
nltk.corpus.stopwords.words = lambda lang : stopwords if lang == 'japanese' else nltk.corpus.stopwords.words_org(lang)
df_pke = {}
df_enpty = []
for i in range(len(df_2018)):
  if get_key_phrase(spacy_model, df_2018.loc[i][separation]) == []:
    print(i)
    df_pke[df_2018.loc[i]['プロジェクトID']] = ''
    df_enpty.append(df_2018.loc[i]['プロジェクトID'])
  else:
    pke_test = {}
    for words in get_key_phrase(spacy_model, df_2018.loc[i][separation]):
      for word in words[0].split():
        if word in pke_test:
          pke_test[word] += words[1]
        else:
          pke_test[word] = words[1]

    df_pke[df_2018.loc[i]['プロジェクトID']] = pke_test
import json
obj = []
for i in range(len(df_2018)):
  word_score = []
  if df_pke[df_2018.loc[i]['プロジェクトID']] == '':
    word_score.append({"word" : '',
                       "score" : 0})
  else:
    tmp_list = sorted(df_pke[df_2018.loc[i]['プロジェクトID']].items(), key=lambda x: x[1], reverse=True)
    for data in tmp_list:
      word_score.append({"word" : data[0],
                        "score" : data[1]})
    obj.append({'プロジェクトID':int(df_2018.loc[i]['プロジェクトID']),
                'SingleRank':word_score})
  
with open('/content/drive/MyDrive/Colab Notebooks/pre-judgit/SingleRank2018.json', 'w', encoding='utf8')as f:
  json.dump(obj, f, indent = 2, ensure_ascii=False)<jupyter_output><empty_output><jupyter_text>PositionRank<jupyter_code>def get_key_phrase(spacy_model, text, n = 5):
    """キーフレーズ処理

    Args:
        text ([type]): キーフレーズ対象の文字列
        n (int, optional): [description]. Defaults to 10.
        gc_collect (bool, optional): [description]. Defaults to False.

    Returns:
        [type]: [description]
    """
    #MultipartiteRankなどのキーフレーズの手法については以下のサイトが参考になります
    #https://www.ogis-ri.co.jp/otc/hiroba/technical/similar-document-search/part5.html
    extractor = pke.unsupervised.PositionRank()
    extractor.load_document(input=text, language='ja_ginza', normalization=None, spacy_model= spacy_model )
    extractor.candidate_selection( grammar="NP: {<ADJ>*<NOUN|PROPN|NUM>+}") #ここわからん
    extractor.candidate_weighting( window=10, pos={'NOUN', 'PROPN', 'ADJ', 'NUM'})
    key_phrase = extractor.get_n_best(n)
    
    return  key_phrase
    
    
#spacyに設定されているstopwordを使う
from  spacy.lang.ja import stop_words

spacy_model = spacy.load("ja_ginza")  # GiNZAモデルの読み込み
print(spacy_model)
stopwords = list(stop_words.STOP_WORDS)
nltk.corpus.stopwords.words_org = nltk.corpus.stopwords.words
nltk.corpus.stopwords.words = lambda lang : stopwords if lang == 'japanese' else nltk.corpus.stopwords.words_org(lang)
df_pke = {}
df_enpty = []
for i in range(len(df_2018)):
  if get_key_phrase(spacy_model, df_2018.loc[i][separation]) == []:
    print(i)
    df_pke[df_2018.loc[i]['プロジェクトID']] = ''
    df_enpty.append(df_2018.loc[i]['プロジェクトID'])
  else:
    pke_test = {}
    for words in get_key_phrase(spacy_model, df_2018.loc[i][separation]):
      for word in words[0].split():
        if word in pke_test:
          pke_test[word] += words[1]
        else:
          pke_test[word] = words[1]

    df_pke[df_2018.loc[i]['プロジェクトID']] = pke_test
import json
obj = []
for i in range(len(df_2018)):
  word_score = []
  if df_pke[df_2018.loc[i]['プロジェクトID']] == '':
    word_score.append({"word" : '',
                       "score" : 0})
  else:
    tmp_list = sorted(df_pke[df_2018.loc[i]['プロジェクトID']].items(), key=lambda x: x[1], reverse=True)
    for data in tmp_list:
      word_score.append({"word" : data[0],
                        "score" : data[1]})
    obj.append({'プロジェクトID':int(df_2018.loc[i]['プロジェクトID']),
                'PositionRank':word_score})
  
with open('/content/drive/MyDrive/Colab Notebooks/pre-judgit/PositionRank2018.json', 'w', encoding='utf8')as f:
  json.dump(obj, f, indent = 2, ensure_ascii=False)<jupyter_output><empty_output><jupyter_text>TopicRank<jupyter_code>def get_key_phrase(spacy_model, text, n = 5):
    """キーフレーズ処理

    Args:
        text ([type]): キーフレーズ対象の文字列
        n (int, optional): [description]. Defaults to 10.
        gc_collect (bool, optional): [description]. Defaults to False.

    Returns:
        [type]: [description]
    """
    #MultipartiteRankなどのキーフレーズの手法については以下のサイトが参考になります
    #https://www.ogis-ri.co.jp/otc/hiroba/technical/similar-document-search/part5.html
    extractor = pke.unsupervised.TopicRank()
    extractor.load_document(input=text, language='ja_ginza', normalization=None, spacy_model= spacy_model )
    extractor.candidate_selection( pos={'NOUN', 'PROPN', 'ADJ', 'NUM'}) #ここわからん
    extractor.candidate_weighting( threshold=0.74, method='average')
    key_phrase = extractor.get_n_best(n)
    
    return  key_phrase
    
    
#spacyに設定されているstopwordを使う
from  spacy.lang.ja import stop_words

spacy_model = spacy.load("ja_ginza")  # GiNZAモデルの読み込み
print(spacy_model)
stopwords = list(stop_words.STOP_WORDS)
nltk.corpus.stopwords.words_org = nltk.corpus.stopwords.words
nltk.corpus.stopwords.words = lambda lang : stopwords if lang == 'japanese' else nltk.corpus.stopwords.words_org(lang)
df_pke = {}
df_enpty = []
for i in range(len(df_2018)):
  if get_key_phrase(spacy_model, df_2018.loc[i][separation]) == []:
    print(i)
    df_pke[df_2018.loc[i]['プロジェクトID']] = ''
    df_enpty.append(df_2018.loc[i]['プロジェクトID'])
  else:
    pke_test = {}
    for words in get_key_phrase(spacy_model, df_2018.loc[i][separation]):
      for word in words[0].split():
        if word in pke_test:
          pke_test[word] += words[1]
        else:
          pke_test[word] = words[1]

    df_pke[df_2018.loc[i]['プロジェクトID']] = pke_test
import json
obj = []
for i in range(len(df_2018)):
  word_score = []
  if df_pke[df_2018.loc[i]['プロジェクトID']] == '':
    word_score.append({"word" : '',
                       "score" : 0})
  else:
    tmp_list = sorted(df_pke[df_2018.loc[i]['プロジェクトID']].items(), key=lambda x: x[1], reverse=True)
    for data in tmp_list:
      word_score.append({"word" : data[0],
                        "score" : data[1]})
    obj.append({'プロジェクトID':int(df_2018.loc[i]['プロジェクトID']),
                'TipicRank':word_score})
  
with open('/content/drive/MyDrive/Colab Notebooks/pre-judgit/TopicRank2018.json', 'w', encoding='utf8')as f:
  json.dump(obj, f, indent = 2, ensure_ascii=False)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.15199115044247788, 'fraction_numerical': 0.02588495575221239, 'mean_word_length': 3.8886085075702956, 'pattern_counts': {'":': 0, '<': 23, '<?xml version=': 0, '>': 23, 'https://': 7, 'lorem ipsum': 0, 'www.': 5, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '5', 'dir': 'jupyter-structured-clean-dedup', 'id': '265776', 'license': 'MIT', 'n_tokens_mistral': 5847, 'n_tokens_neox': 5508, 'n_words': 825, 'path': 'ipynb/judgit/pke.ipynb', 'repo_name': 'vdslab/t-taiki0620', 'size': '13560'}