<jupyter_start><jupyter_text>作業目標: 使用Jieba進行各項的斷詞操作這份作業我們會使用中文飯店評論資料集來作為斷詞練習。 [資料集:ChnSentiCorp_htl_all](https://github.com/SophonPlus/ChineseNlpCorpus)  讀入文本資料<jupyter_code>import pandas as pd

# hint: 可利用pandas讀取CSV
###<your code>###
pd_corpus = pd.read_csv("./ChnSentiCorp_htl_all.csv")
pd_corpus.head(5)
#確認所有留言,正評價(label=1)與負評價數量(label=0)
###<your code>###
pd_positive = pd_corpus[pd_corpus['label']==1]
pd_negative = pd_corpus[pd_corpus['label']==0]
print(f'Total: {len(pd_corpus)}, Positive: {len(pd_positive)}, Negative: {len(pd_negative)}')
#移除缺失值
###<your code>###
pd_corpus = pd_corpus.dropna()<jupyter_output><empty_output><jupyter_text>建構結巴斷詞Function建構可將讀入的Pandas DataFrame的文本資料，外加一欄名為`cut`的review欄位斷詞結果<jupyter_code>import jieba
class JiebaCutingClass(object):
    '''Class to use jeiba to parse corpus from dataframe and cut the corpus
    
    Parameters
    -----------
    key_to_cut: str
        the dataframe key to parse the sentence for jieba cutting
    dic: str
        the dictionary to use for jieba, default is None (use default dictionary)
    userdict: str
        the user defined dictionary to use for jieba, default is None
    '''
    
    def __init__(self, key_to_cut:str, dic:str=None, userdict:str=None):
        
        if dic is not None:
            jieba.set_dictionary(dic)
        
        if userdict is not None:
            jieba.load_userdict(userdict)
        
        self.key_to_cut = key_to_cut
        
        #將paddle 功能開啟
        ###<your code>###
        jieba.enable_paddle()
        
    @staticmethod
    def cut_single_sentence(sentence, use_paddle=False, use_full=False, use_search=False):
        
        if use_search:
            # hint:使用收尋引擎模式進行斷詞
            ###<your code>###
            out = jieba.cut_for_search(sentence=sentence)
        else:
            # hint:非收尋引擎模式斷詞，請注意有精確模式、全模式與paddle模式
            ###<your code>###
            out = jieba.cut(sentence=sentence, 
                            cut_all=use_full, 
                            use_paddle=use_paddle)
        
        return out
            
    
    def cut_corpus(self, corpus: pd.DataFrame, mode: str) -> pd.DataFrame:
        '''Method to read and cut sentence from dataframe and append another column named cut
        
        Paremeters
        --------------
        corpus: pd.DataFrame
            Input corpus in dataframe
        mode: str
            Jieba mode to be used
        
        Return
        ----------------
        out: pd.Dataframe
            Output corpus in dataframe
        '''
        
        # checking valid mode
        if mode not in ['paddle', 'full', 'precise', 'search']:
            raise TypeError(f'only support `paddle`, `full`, `precise`, and `search` mode, but get {mode}')
            
        # cut the corpus based on mode
        # hint: 根據mode來呼叫相對應的斷詞method
        if mode == 'paddle':
            ###<your code>###
            out = self._paddle_cut(corpus)
            
        elif mode == 'full':
            ###<your code>###
            out = self._full_cut(corpus)
            
        elif mode == 'precise':
            ###<your code>###
            out = self._precise_cut(corpus)
            
        elif mode == 'search':
            ###<your code>###
            out = self._search_cut(corpus)
            
        return out
    
    def _paddle_cut(self, corpus):
        '''paddle mode
        '''
        #enable paddle
        #hint:先啟用paddle mode
        ### <your code> ###
        jieba.enable_paddlea()
        
        out = []
        # hint:將讀進的文本進行斷詞，並將結果append回out的陣列中
        for single_review in corpus[self.key_to_cut]:
            ###<your code>###
            iterator = self.cut_single_sentence(single_review, use_paddle=True, use_full=False, use_search=False)
            out.append([word for word in iterator])
        
        corpus['cut'] = out
        
        return corpus
    
    def _full_cut(self, corpus):
        '''full mode
        '''
        
        out = []
        # hint:將讀進的文本進行斷詞，並將結果append回out的陣列中
        for single_review in corpus[self.key_to_cut]:
            ###<your code>###
            iterator = self.cut_single_sentence(single_review, use_paddle=False, use_full=True, use_search=False)
            out.append(out.append([word for word in iterator]))
            
        corpus['cut'] = out
        
        return corpus
    
    def _precise_cut(self, corpus):
        '''precise mode
        '''
        
        out = []
        # hint:將讀進的文本進行斷詞，並將結果append回out的陣列中
        for single_review in corpus[self.key_to_cut]:
            ###<your code>###
            iterator = self.cut_single_sentence(single_review, use_paddle=False, use_full=False, use_search=False)
            out.append(out.append([word for word in iterator]))
        
        corpus['cut'] = out
        
        return corpus
    
    def _search_cut(self, corpus):
        '''search mode
        '''
        
        out = []
        # hint:將讀進的文本進行斷詞，並將結果append回out的陣列中
        for single_review in corpus[self.key_to_cut]:
            ###<your code>###
            iterator = self.cut_single_sentence(single_review, use_paddle=False, use_full=False, use_search=True)
            out.append([word for word in iterator])
        
        corpus['cut'] = out
        
        return corpus<jupyter_output><empty_output><jupyter_text>使用建構好的斷詞物件對文本進行斷詞<jupyter_code>### 請使用精確模式與預設字典對文本進行斷詞

# hint:請先實例化JiebaCutingClass，再進行斷詞
###<your code>###
jieba_cut = JiebaCutingClass(key_to_cut='review')
###<your code>### #為了避免處理時間過久, 這裡我們只使用前50個進行斷詞
pd_cut = jieba_cut.cut_corpus(pd_corpus.head(50), mode='search')

pd_cut.head()<jupyter_output>Paddle enabled successfully......
Building prefix dict from the default dictionary ...
Loading model from cache C:\Users\qwerz\AppData\Local\Temp\jieba.cache
Loading model cost 0.728 seconds.
Prefix dict has been built successfully.
<ipython-input-5-65173183c8db>:145: SettingWithCopyWarning: 
A value is trying to be set on a copy of a slice from a DataFrame.
Try using .loc[row_indexer,col_indexer] = value instead

See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy
  corpus['cut'] = out<jupyter_text>載入繁中字典為指定的字詞進行斷詞<jupyter_code>test_string = '我愛cupoy自然語言處理馬拉松課程'
it = jieba_cut.cut_single_sentence(test_string)
[w for w in it]
test_string = '我愛cupoy自然語言處理馬拉松課程'
jieba_cut = JiebaCutingClass(key_to_cut='', dic='./dict.txt.big')


out_string = jieba_cut.cut_single_sentence(test_string, use_paddle=True, use_full=False, use_search=False)
print(f'Paddle模式: {[string for string in out_string]}')

out_string = jieba_cut.cut_single_sentence(test_string, use_paddle=False, use_full=True, use_search=False)
print(f'全模式: {[string for string in out_string]}')

out_string = jieba_cut.cut_single_sentence(test_string, use_paddle=False, use_full=False, use_search=True)
print(f'搜尋模式: {[string for string in out_string]}')

out_string = jieba_cut.cut_single_sentence(test_string, use_paddle=False, use_full=False, use_search=False)
print(f'精確模式: {[string for string in out_string]}')<jupyter_output>Paddle enabled successfully......
Building prefix dict from C:\Users\qwerz\working\0--Projects\Github\1st-Python-NLP-Marathon\practices\Day-006\dict.txt.big ...
Loading model from cache C:\Users\qwerz\AppData\Local\Temp\jieba.ud98a3f48c3f8f691f5918f2891cd8d4a.cache
{'fraction_non_alphanumeric': 0.11461279461279461, 'fraction_numerical': 0.006868686868686869, 'mean_word_length': 2.613625304136253, 'pattern_counts': {'":': 0, '<': 33, '<?xml version=': 0, '>': 34, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '116037', 'license': 'MIT', 'n_tokens_mistral': 2839, 'n_tokens_neox': 2814, 'n_words': 494, 'path': 'practices/Day-006/day_006_practice.ipynb', 'repo_name': 'magikerwin1993/1st-NLP-Marathon', 'size': '7425'}