def parse_data(self, _file, is_train=True):

    def cut(x, list, ignore, max_len, is_summ):
        if (isinstance(x, float) and math.isnan(x)):
            if is_summ:
                list.append(WORD[EOS])
            else:
                list.append('')
        else:
            x = x.split()
            if (len(x) > max_len):
                x = x[:max_len]
                ignore[0] += 1
            if is_summ:
                x += [WORD[EOS]]
            list.append(x)
    (origins, summurys) = ([], [])
    ignore_ori_nums = [0]
    ignore_sum_nums = [0]
    df = pd.read_csv(_file)
    df['original'].apply(cut, args=(origins, ignore_ori_nums, self.max_ori_len, False))
    df['summary'].apply(cut, args=(summurys, ignore_sum_nums, self.max_sum_len, True))
    if is_train:
        ori_ignore = self.dict((origins + summurys), self._min_word_count)
        self.train_origins = origins
        self.train_summurys = summurys
        self.train_labels = (df['score'].values - 1)
        print('Ignored origin counts - [{}]'.format(ori_ignore))
        print('Train data - ignore original lines - [{}]'.format(ignore_ori_nums[0]))
        print('Train data - ignore summary lines - [{}]'.format(ignore_sum_nums[0]))
    else:
        self.test_origins = origins
        self.test_summurys = summurys
        self.test_labels = (df['score'].values - 1)
        print('Test data - ignore original lines - [{}]'.format(ignore_ori_nums[0]))
        print('Test data - ignore summary lines - [{}]'.format(ignore_sum_nums[0]))
def parse_data(self, _file, is_train=True):

    def cut(x, list, ignore, max_len, is_summ):
        if (isinstance(x, float) and math.isnan(x)):
            if is_summ:
                list.append(WORD[EOS])
            else:
                list.append()
        else:
            x = x.split()
            if (len(x) > max_len):
                x = x[:max_len]
                ignore[0] += 1
            if is_summ:
                x += [WORD[EOS]]
            list.append(x)
    (origins, summurys) = ([], [])
    ignore_ori_nums = [0]
    ignore_sum_nums = [0]
    df = pd.read_csv(_file)
    df['original'].apply(cut, args=(origins, ignore_ori_nums, self.max_ori_len, False))
    df['summary'].apply(cut, args=(summurys, ignore_sum_nums, self.max_sum_len, True))
    if is_train:
        ori_ignore = self.dict((origins + summurys), self._min_word_count)
        self.train_origins = origins
        self.train_summurys = summurys
        self.train_labels = (df['score'].values - 1)
        print('Ignored origin counts - [{}]'.format(ori_ignore))
        print('Train data - ignore original lines - [{}]'.format(ignore_ori_nums[0]))
        print('Train data - ignore summary lines - [{}]'.format(ignore_sum_nums[0]))
    else:
        self.test_origins = origins
        self.test_summurys = summurys
        self.test_labels = (df['score'].values - 1)
        print('Test data - ignore original lines - [{}]'.format(ignore_ori_nums[0]))
        print('Test data - ignore summary lines - [{}]'.format(ignore_sum_nums[0]))