<jupyter_start><jupyter_text>Data Preprocessing<jupyter_code>import pandas as pd
import re

df = pd.read_csv('/content/gdrive/My Drive/Cognitive Computing/texts_and_fin.csv')
# Define a function to count words in the docs
def word_count(s):
    a = s.split(' ')
    return len(a)

df['txt_len'] = df['text'].map(word_count)
df['txt_len'].hist()
len(df)

# Get rid of documents with more than 20,000 words
df2 = df[df['txt_len'] < 20000]

# Eliminate text outside of the phrases "Check the appropriate box" and "Pursuant to the requirements"
# 初次文本过滤，用正则匹配取出字符串“Check the appropriate box”与“Pursuant to the requirements”中的文本内容
new_text = []
for i in range(len(df2)):
    try:
        a = re.search(r'Check\sthe\sappropriate\sbox(.*?)Pursuant\sto\sthe\srequirements', df2['text'].iloc[i]).group(1)
        new_text.append(a)
    except AttributeError:
        new_text.append(None)
        
df2['filtered_text'] = new_text
df2 = df2.dropna()

# Add an end tag to the end of filtered text so that we will have a consistent string in each row
def add_end_tag(s):
    a = s + '<end_tag>'
    return a

df2['filtered_text2'] = df2['filtered_text'].map(add_end_tag)

# Elinimate the text before the word "On"
# 二次文本过滤，公司披露的实际信息以“On”开头，因此取出每行中以“On”开头之后的文本内容
new_text = []
for i in range(len(df2)):
    try:
        a = re.search(r'On\s(.*?)\<end\_tag\>', df2['filtered_text2'].iloc[i]).group(1)
        new_text.append(a)
    except AttributeError:
        new_text.append(None)

df2['filtered_text3'] = new_text
df2 = df2.dropna()

df3 = df2[['ticker', 'signal', 'release_date', 'filtered_text3']]
df3.to_csv('texts_and_fin2.csv')<jupyter_output><empty_output><jupyter_text>BERT Model (a majority of this code is copied from the BERT Tutorial) Bert官方Github：https://github.com/google-research/bert官方的中文预训练权重：chinese_L-12_H-768_A-12.zip https://storage.googleapis.com/bert_models/2018_11_03/chinese_L-12_H-768_A-12.zip例子所在Github：https://github.com/bojone/bert_in_keras/<jupyter_code>import tensorflow as tf
import tensorflow_hub as hub
from datetime import datetime
from tensorflow import keras
import bert
from bert import run_classifier
from bert import optimization
from bert import tokenization
import os
import re
import pandas as pd
import numpy as np

df = pd.read_csv('texts_and_fin2.csv')
df.head()
# Run this cell for a function for oversampling

def oversample(X,y):
    # Get number of rows with imbalanced class
    target = y.sum().idxmax()
    n = y[target].sum()
    # identify imbalanced targets
    imbalanced = y.drop(target,axis=1)
    #For each target, create a dataframe of randomly sampled rows, append to list
    append_list =  [y.loc[y[col]==1].sample(n=n-y[col].sum(),replace=True,random_state=20) for col in imbalanced.columns]
    append_list.append(y)
    y = pd.concat(append_list,axis=0)
    # match y indexes on other inputs
    X = X.loc[y.index]
    assert (y.index.all() == X.index.all())
    return X, y

df = df.rename(columns = {"filtered_text3": "filtered_text"})
df = df.sort_values(by='release_date', ascending=True, axis=0)
testNum = int(len(df) * -.1)
X_train = df['filtered_text'][:testNum].dropna()
y_train = pd.get_dummies(columns=['signal'],data=df['signal'])[:testNum].dropna().iloc[:, :]
test = df.loc[list(set(list(df.index)) - set(list(X_train.index)))]
X_test = test['filtered_text'].dropna()
y_test = test['signal'].dropna()

X_train, y_train = oversample(X_train, y_train)

# Recreate the signal variable
y_train["signal"] = np.nan

for i, y in y_train.iterrows():
    if str(type(y_train.loc[i])) == "<class 'pandas.core.frame.DataFrame'>": # If an index only has one observation, it draws up an error if we try to use the indexer agaon
        # They're usually classed as a series while the ones with many observations are considered a df. This is a way to get
        # rid of them
        if y_train.loc[i].iloc[0, 0] == 1:
            y_train.loc[i, "signal"] = "down"
        elif y_train.loc[i].iloc[0, 1] == 1:
            y_train.loc[i, "signal"] = "stay"
        else:
            y_train.loc[i, "signal"] = "up"
    else: # If they only have one observation, we settle it here instead
        if y_train.loc[i][0] == 1:
              y_train.loc[i, "signal"] = "down"
        elif y_train.loc[i][1] == 1:
            y_train.loc[i, "signal"] = "stay"
        else:
            y_train.loc[i, "signal"] = "up"
            
X_train2 = X_train.reset_index(drop = True)
y_train2 = y_train['signal'].reset_index(drop = True)

data = pd.concat([X_train2, y_train2], axis = 1)
data.rename(columns = {"filtered_text":"doc"}, inplace = True)

X_test = X_test.reset_index(drop = True)
y_test = y_test.reset_index(drop = True)

train = data
test = pd.concat([X_test, y_test], axis = 1) 
test.rename(columns = {"filtered_text":"doc"}, inplace = True)
train.head()<jupyter_output><empty_output><jupyter_text>BERT Model<jupyter_code># input data columns
DATA_COLUMN = 'doc'      # input data is the "doc" column
LABEL_COLUMN = 'signal'  # output data is the "signal" column
label_list = ['up', 'down', 'stay']   # label_list is the list of labels("up" ,"down" and "stay" for Stock prices were up, down and remain unchanged 

# Use bert library "run_classifier" model to transform our data into a format BERT understands
# 说明：将输入的数据转为bert模型输入的数据格式，主要有下面两个步骤：

"""
1、创建“InputExamples”，即使用“run_classifier”进行处理
    
    * text_a 是我们想要分类的数据，在本例中，即为data['doc']对应的列数据
    * text_b 是我们想要让模型理解句子的数据，可将其看成是text_a的label。
    
      举个栗子，机器翻译中，输入“我是一只喵”对应“I am a cat”，那么text_a 对应“我是一只喵”，text_b 对应“I am a cat”。
      再举个栗子，问答场景中，text_a 对应问题，text_b 对应答案。
      本例是个分类问题，我们只关心text_a和对应label
      
    * label这里对应“up、down、stay”，如果是二分类问题，则对应 “True”或者“False”等。


"""
train_InputExamples = train.apply(lambda x: bert.run_classifier.InputExample(guid=None, # Globally unique ID for bookkeeping, unused in this example
                                                                   text_a = x[DATA_COLUMN], 
                                                                   text_b = None, 
                                                                   label = x[LABEL_COLUMN]), axis = 1)

test_InputExamples = test.apply(lambda x: bert.run_classifier.InputExample(guid=None, # Globally unique ID for bookkeeping, unused in this example
                                                                   text_a = x[DATA_COLUMN], 
                                                                   text_b = None, 
                                                                   label = x[LABEL_COLUMN]), axis = 1)

"""
2、对我们输入的数据进行预处理，使其与BERT模型训练的数据相匹配。

   提示：不要紧张，这部分数据预处理的代码直接copy运行就好，下面我们对其主要实现的功能进行说明：

    加载google为我们训练好的bert模型参数，即bert-weight
    文本统一转为小写（bert在训练时使用小写字母）
    分词（(i.e. "I am cat" -> ["I", "am", "cat"])
    把单词分成单词块(i.e. "calling" -> ["call", "##ing"])
    使用BERT提供的vocab.txt文件将单词映射到索引
    为每一段文本添加标记，句子开始标记“CLS”，句子与句子之间的分割标记“SEP”
    为每个输入添加“index”和“segment”标记
"""

# 在线加载bert-weight
BERT_MODEL_HUB = "https://tfhub.dev/google/bert_uncased_L-12_H-768_A-12/1"


# 本地加载bert-weight
BERT_MODEL_HUB = hub.load_module_spec("/root/None/Bert/Bert_Weight/bert_uncased_L-12_H-768_A-12")

# 下载bert-weight
"""
Windows直接访问链接即可：https://storage.googleapis.com/tfhub-modules/google/bert_uncased_L-12_H-768_A-12/1.tar.gz

Linux下载：wget https://storage.googleapis.com/tfhub-modules/google/bert_uncased_L-12_H-768_A-12/1.tar.gz

解压到指定目录：tar -zxvf 1.tar.gz -C  /root/None/Bert/Bert_Weight/bert_uncased_L-12_H-768_A-12  
"""

# 我们刚刚了解到我们使用的BERT模型需要小写的数据，存储在tokenization_info["do_lower_case"]中
def create_tokenizer_from_hub_module():
    """Get the vocab file and casing info from the Hub module."""
    with tf.Graph().as_default():
        bert_module = hub.Module(BERT_MODEL_HUB)
        tokenization_info = bert_module(signature="tokenization_info", as_dict=True)

        with tf.Session() as sess:
            vocab_file, do_lower_case = sess.run([tokenization_info["vocab_file"],tokenization_info["do_lower_case"]])
      
    return bert.tokenization.FullTokenizer(vocab_file=vocab_file, do_lower_case=do_lower_case)

tokenizer = create_tokenizer_from_hub_module()


# 使用tokenizer分词，即run_classifier.convert_examples_to_features 将上述我们定义的"InputExample"转换为Bert能够理解的特性

# We'll set sequences to be at most 128 tokens long.
MAX_SEQ_LENGTH = 128
# Convert our train and test features to InputFeatures that BERT understands.
train_features = bert.run_classifier.convert_examples_to_features(train_InputExamples, label_list, MAX_SEQ_LENGTH, tokenizer)
test_features = bert.run_classifier.convert_examples_to_features(test_InputExamples, label_list, MAX_SEQ_LENGTH, tokenizer)
def create_model(is_predicting, input_ids, input_mask, segment_ids, labels,num_labels):
    """Creates a classification model."""
    bert_module = hub.Module(BERT_MODEL_HUB,trainable=True)
    
    bert_inputs = dict(input_ids=input_ids,input_mask=input_mask,segment_ids=segment_ids)
    
    bert_outputs = bert_module(inputs=bert_inputs,signature="tokens",as_dict=True)
    
    # Use "pooled_output" for classification tasks on an entire sentence.
    # Use "sequence_outputs" for token-level output.
    output_layer = bert_outputs["pooled_output"]
    
    hidden_size = output_layer.shape[-1].value

    # Create our own layer to tune for politeness data.
    output_weights = tf.get_variable(   # initialize W
        "output_weights", [num_labels, hidden_size],
        initializer=tf.truncated_normal_initializer(stddev=0.02))

    output_bias = tf.get_variable(
        "output_bias", [num_labels],  # initialize b
        
        initializer=tf.zeros_initializer())
    
    # loss functiong definition 
    with tf.variable_scope("loss"):
        # Dropout helps prevent overfitting
        output_layer = tf.nn.dropout(output_layer, keep_prob=0.9)

        logits = tf.matmul(output_layer, output_weights, transpose_b=True)
        logits = tf.nn.bias_add(logits, output_bias)
        log_probs = tf.nn.log_softmax(logits, axis=-1) # log交叉商损失

        # Convert labels into one-hot encoding
        one_hot_labels = tf.one_hot(labels, depth=num_labels, dtype=tf.float32)

        predicted_labels = tf.squeeze(tf.argmax(log_probs, axis=-1, output_type=tf.int32))
        # If we're predicting, we want predicted labels and the probabiltiies.
        if is_predicting:
            return (predicted_labels, log_probs)
        
        # If we're train/eval, compute loss between predicted and actual label
        per_example_loss = -tf.reduce_sum(one_hot_labels * log_probs, axis=-1) # 对输入张量所有元素求和
        loss = tf.reduce_mean(per_example_loss)
        
        return (loss, predicted_labels, log_probs)

# model_fn_builder actually creates our model function
# using the passed parameters for num_labels, learning_rate, etc.
def model_fn_builder(num_labels, learning_rate, num_train_steps,num_warmup_steps):
    """Returns `model_fn` closure for TPUEstimator."""
    def model_fn(features, labels, mode, params):  # pylint: disable=unused-argument
        """The `model_fn` for TPUEstimator."""
        input_ids = features["input_ids"]
        input_mask = features["input_mask"]
        segment_ids = features["segment_ids"]
        label_ids = features["label_ids"]

        is_predicting = (mode == tf.estimator.ModeKeys.PREDICT)

        # TRAIN and EVAL
        if not is_predicting:
            (loss, predicted_labels, log_probs) = create_model(
                is_predicting, input_ids, input_mask, segment_ids, label_ids, num_labels)

            train_op = bert.optimization.create_optimizer(
              loss, learning_rate, num_train_steps, num_warmup_steps, use_tpu=False)

          # Calculate evaluation metrics. 
            def metric_fn(label_ids, predicted_labels):
                accuracy = tf.metrics.accuracy(label_ids, predicted_labels)
                return {"eval_accuracy": accuracy}

            eval_metrics = metric_fn(label_ids, predicted_labels)

            if mode == tf.estimator.ModeKeys.TRAIN:
                return tf.estimator.EstimatorSpec(mode=mode,
                loss=loss,
                train_op=train_op)
            else:
                return tf.estimator.EstimatorSpec(mode=mode, loss=loss, eval_metric_ops=eval_metrics)
        else:
            (predicted_labels, log_probs) = create_model(is_predicting, input_ids, input_mask, segment_ids, label_ids, num_labels)

            predictions = {'probabilities': log_probs, 'labels': predicted_labels}
            return tf.estimator.EstimatorSpec(mode, predictions=predictions)

        # Return the actual model function in the closure
    return model_fn
# Compute train and warmup steps from batch size
# These hyperparameters are copied from this colab notebook (https://colab.sandbox.google.com/github/tensorflow/tpu/blob/master/tools/colab/bert_finetuning_with_cloud_tpus.ipynb)
BATCH_SIZE = 32
LEARNING_RATE = 2e-5
NUM_TRAIN_EPOCHS = 3.0
# Warmup is a period of time where the learning rate 
# is small and gradually increases--usually helps training.
WARMUP_PROPORTION = 0.1
# Model configs
SAVE_CHECKPOINTS_STEPS = 500
SAVE_SUMMARY_STEPS = 100

# Compute # train and warmup steps from batch size
num_train_steps = int(len(train_features) / BATCH_SIZE * NUM_TRAIN_EPOCHS)
num_warmup_steps = int(num_train_steps * WARMUP_PROPORTION)

# Specify output directory and number of checkpoint steps to save
run_config = tf.estimator.RunConfig()

model_fn = model_fn_builder(
  num_labels=len(label_list),
  learning_rate=LEARNING_RATE,
  num_train_steps=num_train_steps,
  num_warmup_steps=num_warmup_steps)

estimator = tf.estimator.Estimator(
  model_fn=model_fn,
  config=run_config,
  params={"batch_size": BATCH_SIZE})

# Create an input function for training. drop_remainder = True for using TPUs.
train_input_fn = bert.run_classifier.input_fn_builder(
    features=train_features,
    seq_length=MAX_SEQ_LENGTH,
    is_training=True,
    drop_remainder=False)

print(f'Beginning Training!')
current_time = datetime.now()
estimator.train(input_fn=train_input_fn, max_steps=num_train_steps)
print("Training took time ", datetime.now() - current_time)<jupyter_output>WARNING:tensorflow:Using temporary folder as model directory: /tmp/tmp_rcn2l66<jupyter_text>Evaluate model on testing data<jupyter_code>test_input_fn = run_classifier.input_fn_builder(
    features=test_features,
    seq_length=MAX_SEQ_LENGTH,
    is_training=False,
    drop_remainder=False)

estimator.evaluate(input_fn=test_input_fn, steps=None)<jupyter_output>INFO:tensorflow:Calling model_fn.
{'fraction_non_alphanumeric': 0.10839136730331655, 'fraction_numerical': 0.010963248982969041, 'mean_word_length': 3.657675016056519, 'pattern_counts': {'":': 6, '<': 19, '<?xml version=': 0, '>': 20, 'https://': 7, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 12, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '538568', 'license': 'Apache-2.0', 'n_tokens_mistral': 5362, 'n_tokens_neox': 5260, 'n_words': 1032, 'path': 'TransLearning/Bert_Example/Bert_Stock_Prediction/Data Preprocessing and BERT.ipynb', 'repo_name': 'yufanwu333/DeepLearning', 'size': '14503'}