<jupyter_start><jupyter_text>1. Pandas and Matplotlib<jupyter_code>import pandas as pd
import numpy as np
import os
import matplotlib.pyplot as plt
%matplotlib inline

dir_path = os.getcwd()
print(dir_path)<jupyter_output>/home/jovyan/alu/ipython<jupyter_text>>PS. It is important to use > to automatically display matplotlib in this notebook  DataFrameDataFrame is a 2-dimension data structure. You can define index (row) and columns name<jupyter_code># nofi.csv is gottenn from oslobors
#date_parser = lambda x: pd.datetime.strptime(x, '%d.%m.%y')
#df = pd.read_csv('../data/nofi.csv', delimiter=';', header=0, parse_dates=['Date'], date_parser=date_parser)
df = pd.read_csv('../data/nofi.csv', delimiter=';', header=0)
df.shape
# original data
df.head(5)
df.shape
# not work: rep = lambda x: x.replace(',', '.')
def rep2(x):
    return [float(i.replace(',', '.')) for i in x]
def rep3(x):
    return [i.replace(',', '.') for i in x]
def c_date(x):
    return [pd.datetime.strptime(i, '%d.%m.%y') for i in x]
df[['Siste', 'Kjøper', 'Selger','Høy', 'Lav', 'VWAP']] = \
    df[['Siste', 'Kjøper', 'Selger','Høy', 'Lav', 'VWAP']].apply(rep2)
#df[['Siste', 'Kjøper', 'Selger','Høy', 'Lav', 'VWAP']] = df[['Siste', 'Kjøper', 'Selger','Høy', 'Lav', 'VWAP']].apply(rep3).astype(float)
df[['Date']] = df[['Date']].apply(c_date)
df.head(10)<jupyter_output><empty_output><jupyter_text>SeriesSeries a dataset with one dimension. In DataFrame, each row is a serie<jupyter_code>nofi_close = pd.Series(df['Siste'].values, index=df['Date'].values, name='NOFI').sort_index()
nofi_close.head(10)
nofi_close.plot()
# must use plt.show() to display the graph. 
# Why??? Abu project dont need to do it. Answer: Using <<%matplotlib inline>> at the top<jupyter_output><empty_output><jupyter_text>Dataframe and Series methods 实例 1. Resample()<jupyter_code>df_5 = nofi_close.resample('5D').mean()
df_5.plot()<jupyter_output><empty_output><jupyter_text>2. info()<jupyter_code>df[['Siste', 'Antall handler totalt']].plot(subplots=True, style=['r', 'g'], grid=True)
df.info()<jupyter_output><class 'pandas.core.frame.DataFrame'>
RangeIndex: 421 entries, 0 to 420
Data columns (total 11 columns):
Date                           421 non-null datetime64[ns]
Siste                          421 non-null float64
Kjøper                         421 non-null float64
Selger                         421 non-null float64
Høy                            421 non-null float64
Lav                            421 non-null float64
Totalt omsatt (NOK)            421 non-null int64
Totalt antall aksjer omsatt    421 non-null int64
Antall off. handler            421 non-null int64
Antall handler totalt          421 non-null int64
VWAP                           421 non-null float64
dtypes: datetime64[ns](1), float64(6), int64(4)
memory usage: 36.3 KB<jupyter_text>3. describe()<jupyter_code>nofi_close.describe()<jupyter_output><empty_output><jupyter_text>4. PCT_CHANGE()<jupyter_code>changeRatio = nofi_close.pct_change()
changeRatio.head()
changeRatio.cumsum().plot()
df.columns.values[1:]<jupyter_output><empty_output><jupyter_text>5. Adding new  Columns in Dataframe 增加了2列数据- ChangeRatio: 增幅/跌幅- Amplitude: 当日振幅<jupyter_code>df1 = pd.DataFrame(df.iloc[:, 1:].values, index=df['Date'], columns=df.columns.values[1:]).sort_index()
df1 = df1.assign(ChangeRatio=df1['Siste'].pct_change()*100);df1.head()
df1 = df1.assign(Amplitude = (df1['Høy']-df1['Lav'])/df1['Siste'].shift(1)*100);df1.head()<jupyter_output><empty_output><jupyter_text>6. Handle 'nan' value (see P85)Functions: fillna, dropna<jupyter_code>df1.fillna(0, inplace=False);df1.head()<jupyter_output><empty_output><jupyter_text>7. DataFrame conditional selection and sorting<jupyter_code>df1.sort_values(by='Amplitude', ascending=False)[:5]
df1.sort_values(by='ChangeRatio', ascending=True)[1:10]
df1[df1.ChangeRatio>0].sort_values(by='Amplitude')[:5]<jupyter_output><empty_output><jupyter_text>8.  寻找异常涨幅<jupyter_code>nofi = df1.copy();nofi.head()
print(nofi.ChangeRatio.size)
nofi.ChangeRatio.hist(bins=80)<jupyter_output>421<jupyter_text>根据上图和下边的分类（Divide the data into 10 potions）. (2.883, 7.31] 可以认为是异常<jupyter_code>cats = pd.qcut(np.abs(nofi.ChangeRatio), 10)
cats.value_counts()<jupyter_output><empty_output><jupyter_text>nofi.Amplitude.hist(bins=80)<jupyter_code>cats = pd.qcut(np.abs(nofi.Amplitude), 10)
cats.value_counts()<jupyter_output><empty_output><jupyter_text>(4.312, 14.652] 对振幅来说就是异常  9. 星期几是好日子添加2列数据- positive: 0 下跌； 1 上涨- weekday: 1 周一； 5 周五使用crosstable显示统计结果<jupyter_code>nofi['positive'] = np.where(nofi.ChangeRatio > 0, 1, 0);
#nofi['weekday'] = pd.Series(pd.to_datetime(nofi.index)).dt.dayofweek;nofi.head()
nofi['weekday'] = pd.to_datetime(nofi.index).dayofweek+1; nofi.head()

xt = pd.crosstab(nofi.weekday, nofi.positive);xt
xt_pct = xt.div(xt.sum(1).astype(float), axis=0);xt_pct
nofi.pivot_table(['positive'], index=['weekday'])<jupyter_output><empty_output><jupyter_text>10. 跳空缺口添加新的一列 ***jump_power***.<jupyter_code>jump_threshold = nofi.Siste.median() * 0.03
print(f"jump_threshold={jump_threshold}")
nofi['jump_power'] = np.where(np.abs(nofi['Lav'] - nofi['Siste'].shift(1)) > jump_threshold, 
                              (nofi['Lav'] - nofi['Siste'].shift(1))/jump_threshold, np.nan);
nofi.tail()
jp = nofi[nofi.jump_power.notnull()]; 
print(jp.shape)
jp.sort_values(by='jump_power', ascending=True)[:10]<jupyter_output>(55, 15)<jupyter_text>2. 可视化工具这里使用了3个frameworks:- matplotlib- bokeh- seaborn  1. matplotlib 蜡烛图的绘制<jupyter_code>import matplotlib.finance as plt_f

# 创建画布
fig, axes = plt.subplots(figsize=(18,7))

data = nofi.copy()
data['open'] = nofi['Siste'].shift(1)
data = data[-30:]

dochl = []

for i, (d, o, c, h, l) in enumerate(zip(data.index, data.open, data.Siste, data.Høy, data.Lav)):
    d = plt_f.date2num(d)
    dochl.append((d, o, c, h, l))

plt_f.candlestick_ochl(axes, dochl, width=0.6, colorup="red", colordown="green")
axes.autoscale_view()
axes.xaxis_date()<jupyter_output><empty_output><jupyter_text>2. Bokeh 蜡烛图的绘制Ref https://github.com/bbfamily/abu/blob/master/abupy/MarketBu/ABuMarketDrawing.pyL194<jupyter_code>import bokeh.plotting as bp
from math import pi

def plot_bokeh_candle_dochl(date, p_open, close, high, low, symbol):
    
    bp.output_notebook()

    p = bp.figure(x_axis_type="datetime", plot_width=840, title=symbol)
    p.xaxis.major_label_orientation = pi / 4
    p.grid.grid_line_alpha = 0.3

    w = 24 * 60 * 60 * 1000
    
    mids = (p_open + close) / 2
    spans = abs(close - p_open)

    inc = close > p_open
    dec = p_open > close

    p.segment(date.to_datetime(), high, date.to_datetime(), low, color="black")
    p.rect(date.to_datetime()[inc], mids[inc], w, spans[inc], fill_color='red', line_color='red')
    p.rect(date.to_datetime()[dec], mids[dec], w, spans[dec], fill_color='green', line_color='green')
    bp.show(p)
    

plot_bokeh_candle_dochl(data.index, data['open'].values, data['Siste'].values, 
                        data['Høy'].values, data['Lav'].values, 'NOFI')<jupyter_output><empty_output><jupyter_text>3. Seaborn<jupyter_code>import seaborn as sb

nofi['ChangeRatio'].describe()
sb.distplot(nofi[1:]['ChangeRatio'])
#sb.boxplot(x='weekday', y='ChangeRatio', data=nofi)<jupyter_output><empty_output><jupyter_text>3. 常用数学方法  1. 线性回归 (Linear regression)<jupyter_code>import statsmodels.api as sm
from statsmodels import regression

def liner_regression(y):
    x = np.arange(0, len(y))
    x = sm.add_constant(x)
    model = regression.linear_model.OLS(y, x).fit()
    return model

nofi_close = nofi['Siste'];
x = np.arange(0, nofi_close.shape[0])
y = nofi_close.values;
model = liner_regression(nofi_close.values)
b = model.params[0]
k = model.params[1]
y_fit = k * x + b
plt.plot(nofi_close.index, y)
plt.plot(nofi_close.index, y_fit, 'r')
# model.summary()<jupyter_output><empty_output><jupyter_text>可以通过MAE（偏差绝对值之和）,MSE（偏差平方之和）,RMSE（偏差平方和开平方）度量y和y_fit的误差. 基本公式如下（也可用sklean framework来计算）<jupyter_code>MAE = sum(np.abs(y - y_fit)) / len(y)
MSE = sum(np.square(y - y_fit)) / len(y)
RMSE = np.sqrt(sum(np.square(y - y_fit)) / len(y))
print(f"Basic function:\n  MAE={MAE};MSE={MSE};RMSE={RMSE}")


from sklearn import metrics
sklearn_MAE = metrics.mean_absolute_error(y, y_fit)
sklearn_MSE = metrics.mean_squared_error(y, y_fit)
sklearn_RMSE = np.sqrt(sklearn_MSE)
print(f"Sklearn cacluation:\n  MAE={sklearn_MAE};MSE={sklearn_MSE};RMSE={sklearn_RMSE}")
plt.plot(nofi_close.index, y)
plt.plot(nofi_close.index, y_fit, 'r')
plt.plot(nofi_close.index, y_fit + RMSE, 'g')
plt.plot(nofi_close.index, y_fit - RMSE, 'y')<jupyter_output><empty_output><jupyter_text>2. 多项式回归 (Polynomial Regression)  1. 使用np.polynomial.Chebyshev模型<jupyter_code>import itertools

_, axes = plt.subplots(nrows=4, ncols=3, figsize=(15, 15))
axes_list = list(itertools.chain.from_iterable(axes))


poly = np.arange(1, 12)

for p_cnt, axis in zip(poly, axes_list):
    p = np.polynomial.Chebyshev.fit(x, y, p_cnt)
    y_fit = p(x)
    rmse = np.sqrt(metrics.mean_squared_error(y, y_fit))
    axis.set_title(f'{p_cnt} poly RMSE={rmse}')
    axis.plot(x, y, '', x, y_fit, 'r', x, y_fit + sklearn_RMSE, 'y')

# 观察下图，到第9次多项式就可以了。估计也是书中为什么只做了9次。见P138<jupyter_output><empty_output><jupyter_text>2. 使用sklearn做回归<jupyter_code>from sklearn import linear_model
from sklearn.preprocessing import PolynomialFeatures

POLY_DEGREE = 9
# 读取数据集
datasets_X = x
datasets_Y = y

length = len(datasets_X)
datasets_X = np.array(datasets_X).reshape([length,1])
datasets_Y = np.array(datasets_Y)
minX = min(datasets_X)
maxX = max(datasets_X)
X = np.arange(minX,maxX).reshape([-1,1])
poly_reg = PolynomialFeatures(degree = POLY_DEGREE)
X_poly = poly_reg.fit_transform(datasets_X)
lin_reg_2 = linear_model.LinearRegression()
lin_reg_2.fit(X_poly, datasets_Y)

# 图像中显示
plt.scatter(datasets_X, datasets_Y, color = 'red')
plt.plot(X, lin_reg_2.predict(poly_reg.fit_transform(X)), color = 'blue')
plt.xlabel('#day')
plt.ylabel('Price')

lin_reg_2<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.16363818893685372, 'fraction_numerical': 0.02600140548137737, 'mean_word_length': 5.468831168831169, 'pattern_counts': {'":': 0, '<': 88, '<?xml version=': 0, '>': 95, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '22', 'dir': 'jupyter-structured-clean-dedup', 'id': '581570', 'license': 'Apache-2.0', 'n_tokens_mistral': 4452, 'n_tokens_neox': 4277, 'n_words': 674, 'path': 'ipython/NOFI.ipynb', 'repo_name': 'feng1918/qfeng', 'size': '9961'}