import time
import numpy as np
import xgboost as xgb
from xgboost import plot_importance,plot_tree
from sklearn.model_selection import train_test_split
from sklearn.metrics import accuracy_score
import matplotlib
import matplotlib.pyplot as plt
import os
import pandas as pd
%matplotlib inline

def dropun(X):
    for x in X.columns:
        if x[:7]=='Unnamed':
            X=X.drop(columns=[x])
    return X

def hist(L):
    kwargs = dict(histtype='stepfilled',density=True,alpha=0.3,bins=40)
    for X in L:
        plt.hist(X, **kwargs)

自变量

导入56个因子数据，作为自变量X，以2013年为例，查看一些基本情况。

X = pd.read_csv('data/factors2013.csv')
X.describe()

X.columns

Index(['ts_code', 'trade_date', 'size', 'beta', 'betad', 'idvol', 'total_vol',
       'idskew', 'skew', 'coskew', 'turn', 'std_turn', 'volumed', 'std_dvol',
       'retnmax', 'illq', 'LM', 'sharechg', 'age', 'mom12', 'mom6', 'momchg',
       'imom', 'lagretn', 'BM', 'AM', 'LEV', 'EP', 'CFP', 'OCFP', 'DP', 'SP',
       'AG', 'LG', 'BVEG', 'INVG', 'INVchg', 'SG', 'SgINVg', 'PMG', 'TAXchg',
       'ACC', 'ACCP', 'ROE', 'ROA', 'PA', 'CT', 'cash', 'cashpr', 'RD',
       'RDsales', 'CR', 'QR', 'CFdebt', 'salecash', 'saleinv', 'CRG', 'QRG'],
      dtype='object')

factors=list(X.columns)
factors.remove('ts_code')
factors.remove('trade_date')
print(factors)

['size', 'beta', 'betad', 'idvol', 'total_vol', 'idskew', 'skew', 'coskew', 'turn', 'std_turn', 'volumed', 'std_dvol', 'retnmax', 'illq', 'LM', 'sharechg', 'age', 'mom12', 'mom6', 'momchg', 'imom', 'lagretn', 'BM', 'AM', 'LEV', 'EP', 'CFP', 'OCFP', 'DP', 'SP', 'AG', 'LG', 'BVEG', 'INVG', 'INVchg', 'SG', 'SgINVg', 'PMG', 'TAXchg', 'ACC', 'ACCP', 'ROE', 'ROA', 'PA', 'CT', 'cash', 'cashpr', 'RD', 'RDsales', 'CR', 'QR', 'CFdebt', 'salecash', 'saleinv', 'CRG', 'QRG']

X_code=set(X['ts_code'])
X_code=list(X_code)
X_code.sort()
len(X_code)

2450

months = set(x[:7] for x in X['trade_date'])
months=list(months)
months.sort()
print(months)

['2013-01', '2013-02', '2013-03', '2013-04', '2013-05', '2013-06', '2013-07', '2013-08', '2013-09', '2013-10', '2013-11', '2013-12']

X['trade_date']=pd.to_datetime(X['trade_date'])

缺失值处理

方案一：使用横截面均值

由于股票池中选取的是中国股市上的全部A股（约3000只），样本量较为充足。若一只股票的某个因子暴露值在某日缺失，则取当日其它股票对应因子值的均值填充该缺失部分；若一只股票的某个因子暴露值在连续的一段时间内均为空白，则直接剔除该股票。经过上述缺失值处理后，共剩余2510只股票可供后续的分析研究。

方案二： 使用个股时间序列估计

直接采取其他股票的当日均值进行填充是否有些不妥，该股票的因子值与其他股票的因子有关系吗？要采用均值也是采用自己的均值.
更好的算法是使用一种时间序列模型，因为因子的变化基本上是连续的，可以通过前面或者后面的因子值计算缺失的因子值。但实际上一些因子值是不具备平稳性的，使用时间序列也很难估计其值的大小.

方案三： 直接剔除缺失值

缺失值实际上很少，均低于0.03（例外：EP达到0.074144，DP达到0.146086，cashpr达到0.17541）.

如果直接删去所有缺失值的行，数据只有原来的68.27%.

X_null=X.isnull().describe()
# 计算对应属性的缺失率
X_null.loc['ratio']=1-X_null.loc['freq']/X_null.loc['count']
X_null

# 直接删除所有带有缺失值的
X_test=X.dropna(axis=0, inplace=False)
X_test.isnull().describe()

X_test.to_csv('data/factors2013-3.csv')

方案四：个股按月填补缺失值

此处采用当月自己的均值填补缺失值，如果当月记录数（或属性记录数）空值多于10（一个月有20条左右记录，缺失50%就已经很多了），剔除该股票该月所有信息（不是剔除股票，因为股票在其他时间段依然是有意义的）

但是即使填补了数据，由于缺失值太多，最后剩下 383688/560396=68.47%，不能比方案三保留多少数据。

X_88=X[X['ts_code']=='000001.SZ']
plt.plot(X_88['trade_date'],X_88['beta'])
plt.show()

X_333=X[X['ts_code']=='000333.SZ']
plt.plot(X_333['trade_date'],X_333['beta'])
plt.show()
# 全年beta属性均缺失

for stock in X_code:
#     print(stock)
    X_stock=X[X['ts_code']==stock]
    X_stock = X_stock.set_index('trade_date')
    times = 0
    for month in months:
        try:
            X_month=X_stock.loc[month]
#             print(month, len(X_month.index), (X_month.isnull().sum() > 0).any())
        except KeyError:
            times+=1
    if times!=0:
        print("stock %s has no values %d" % (stock, times))

stock 000010.SZ has no values 3
stock 000017.SZ has no values 11
stock 000035.SZ has no values 8
stock 000038.SZ has no values 1
stock 000042.SZ has no values 1
stock 000333.SZ has no values 8
stock 000403.SZ has no values 1
stock 000498.SZ has no values 1
stock 000545.SZ has no values 6
stock 000607.SZ has no values 3
stock 000622.SZ has no values 1
stock 000657.SZ has no values 1
stock 000668.SZ has no values 5
stock 000670.SZ has no values 1
stock 000672.SZ has no values 3
stock 000681.SZ has no values 1
stock 000688.SZ has no values 3
stock 000710.SZ has no values 3
stock 000712.SZ has no values 2
stock 000723.SZ has no values 2
stock 000727.SZ has no values 1
stock 000751.SZ has no values 9
stock 000757.SZ has no values 1
stock 000793.SZ has no values 1
stock 000796.SZ has no values 2
stock 000820.SZ has no values 3
stock 000831.SZ has no values 1
stock 000902.SZ has no values 2
stock 000920.SZ has no values 2
stock 000972.SZ has no values 4
stock 002072.SZ has no values 1
stock 002074.SZ has no values 2
stock 002099.SZ has no values 2
stock 002125.SZ has no values 1
stock 002127.SZ has no values 1
stock 002143.SZ has no values 2
stock 002193.SZ has no values 5
stock 002195.SZ has no values 2
stock 002196.SZ has no values 1
stock 002279.SZ has no values 2
stock 002346.SZ has no values 1
stock 002354.SZ has no values 2
stock 002356.SZ has no values 3
stock 002389.SZ has no values 1
stock 002446.SZ has no values 1
stock 002504.SZ has no values 3
stock 002506.SZ has no values 1
stock 002519.SZ has no values 1
stock 002527.SZ has no values 1
stock 002579.SZ has no values 1
stock 002602.SZ has no values 2
stock 002665.SZ has no values 2
stock 300012.SZ has no values 1
stock 300032.SZ has no values 1
stock 300044.SZ has no values 1
stock 300053.SZ has no values 2
stock 300071.SZ has no values 1
stock 300108.SZ has no values 1
stock 300116.SZ has no values 1
stock 300130.SZ has no values 2
stock 300143.SZ has no values 1
stock 300169.SZ has no values 2
stock 300220.SZ has no values 1
stock 300248.SZ has no values 2
stock 300254.SZ has no values 2
stock 300268.SZ has no values 8
stock 300278.SZ has no values 1
stock 300282.SZ has no values 3
stock 300292.SZ has no values 3
stock 300315.SZ has no values 1
stock 600023.SH has no values 11
stock 600051.SH has no values 3
stock 600058.SH has no values 5
stock 600074.SH has no values 7
stock 600110.SH has no values 2
stock 600127.SH has no values 1
stock 600146.SH has no values 2
stock 600229.SH has no values 2
stock 600234.SH has no values 5
stock 600236.SH has no values 1
stock 600247.SH has no values 1
stock 600338.SH has no values 2
stock 600370.SH has no values 2
stock 600381.SH has no values 5
stock 600385.SH has no values 8
stock 600556.SH has no values 1
stock 600572.SH has no values 1
stock 600603.SH has no values 1
stock 600606.SH has no values 6
stock 600686.SH has no values 1
stock 600698.SH has no values 6
stock 600727.SH has no values 2
stock 600728.SH has no values 2
stock 600732.SH has no values 1
stock 600734.SH has no values 2
stock 600751.SH has no values 5
stock 600780.SH has no values 3
stock 600817.SH has no values 1
stock 600980.SH has no values 1
stock 601901.SH has no values 4

X_stock=X[X['ts_code']=='000008.SZ']
X_stock = X_stock.set_index('trade_date')
X_month = X_stock.loc[months[2]]
print(X_month.isnull().sum())
(X_month.isnull().sum() > 5).any()# 任意一属性空值的个数大于5，说明应该舍弃整个月的数据

ts_code       0
size          0
beta          0
betad         0
idvol         0
total_vol     0
idskew        0
skew          0
coskew        0
turn          0
std_turn      0
volumed       0
std_dvol      0
retnmax       0
illq          0
LM            0
sharechg      0
age           0
mom12         0
mom6          0
momchg        0
imom          0
lagretn       0
BM            0
AM            0
LEV           0
EP           20
CFP           0
OCFP          0
DP            0
SP            0
AG            0
LG            0
BVEG          0
INVG          0
INVchg        0
SG            0
SgINVg        0
PMG           0
TAXchg        0
ACC           0
ACCP          0
ROE           0
ROA           0
PA            0
CT            0
cash          0
cashpr        0
RD            0
RDsales       0
CR            0
QR            0
CFdebt        0
salecash      0
saleinv       0
CRG           0
QRG           0
dtype: int64





True

(X_month.isnull().sum() > 0).any()

True

X_month=X_month.fillna(value=dict(X_month.mean())) # 按月求平均，并填补缺失值

(X_month.isnull().sum() > 0).any()

False

X_month['trade_date']=X_month.index
X1=pd.DataFrame(columns=X.columns)
X1=pd.concat([X1, X_month], axis=0, ignore_index=True)
X1

X1=pd.DataFrame(columns=X.columns) # 新建一个数据表，直接在原有表格上drop太复杂
num=0
for stock in X_code:
    num+=1
    print(stock, "%d/%d"%(num, len(X_code)), end=" ")
    
    X_stock=X[X['ts_code']==stock]
    X_stock = X_stock.set_index('trade_date')
    valid=[]
    for month in months:
        try:
            X_month = X_stock.loc[month] # 居然没加这句话，全部都是000008.SZ
            if (X_month.isnull().sum() > 10).any():
                continue # 舍弃
            else:
                valid.append(month[5:])
                X_month=X_month.fillna(value=dict(X_month.mean())) # 按月求平均，并填补缺失值
                X_month['trade_date']=X_month.index
                X1=pd.concat([X1, X_month], axis=0, ignore_index=True)
        except KeyError:
            continue # 舍弃
    print(valid)

000001.SZ 1/2450 []
000002.SZ 2/2450 ['01', '02', '03', '04', '05', '06', '07', '08', '09', '10', '11', '12']
000004.SZ 3/2450 ['01', '02', '03', '04', '05', '06', '07', '08', '09', '10', '11', '12']
000005.SZ 4/2450 ['01', '02', '03', '04', '05', '06', '07', '08', '09', '10', '11', '12']
000006.SZ 5/2450 ['01', '02', '03', '04', '05', '06', '07', '08', '09', '10', '11', '12']
000007.SZ 6/2450 ['01', '02', '03', '04', '05', '06', '07', '08', '09', '10', '11', '12']
000008.SZ 7/2450 ['04', '05', '06', '07', '08', '09', '10', '11', '12']
000009.SZ 8/2450 ['01', '02', '03', '04', '05', '06', '07', '08', '09', '10', '11', '12']
000010.SZ 9/2450 ['04', '05', '06', '07', '08', '09', '10', '11', '12']
000011.SZ 10/2450 ['01', '02', '03', '04', '05', '06', '07', '08', '09', '10', '11', '12']
000012.SZ 11/2450 ['01', '02', '03', '04', '05', '06', '07', '08', '09', '10', '11', '12']
000014.SZ 12/2450 ['01', '02', '03', '04', '05', '06', '07', '08', '09', '10', '11', '12']
000016.SZ 13/2450 ['01', '02', '03', '04', '05', '06', '07', '08', '09', '10', '11', '12']
000017.SZ 14/2450 ['12']
000018.SZ 15/2450 []
000019.SZ 16/2450 ['01', '02', '03', '04', '05', '06', '07', '08', '09', '10', '11', '12']
000020.SZ 17/2450 []
000021.SZ 18/2450 ['01', '02', '03', '04', '05', '06', '07', '08', '09', '10', '11', '12']
000022.SZ 19/2450 ['01', '02', '03', '04', '05', '06', '07', '08', '09', '10', '11', '12']
000023.SZ 20/2450 ['01', '02', '03', '04', '05', '06', '07', '08', '09', '10', '11', '12']
000025.SZ 21/2450 ['01', '02', '03', '04', '05', '06', '07', '08', '09', '10', '11', '12']
000026.SZ 22/2450 ['01', '02', '03', '04', '05', '06', '07', '08', '09', '10', '11', '12']
000027.SZ 23/2450 ['01', '02', '03', '04', '05', '06', '07', '08', '09', '10', '11', '12']
000028.SZ 24/2450 ['01', '02', '03', '04', '05', '06', '07', '08', '09', '10', '11', '12']
000029.SZ 25/2450 ['01', '02', '03', '04', '05', '06', '07', '08', '09', '10', '11', '12']
000030.SZ 26/2450 ['01']
000031.SZ 27/2450 ['01', '02', '03', '04', '05', '06', '07', '08', '09', '10', '11', '12']
000032.SZ 28/2450 ['01', '02', '03', '04', '05', '06', '07', '08', '09', '10', '11', '12']
000034.SZ 29/2450 ['01', '02', '03', '04', '05', '06', '07', '08', '09', '10', '11', '12']
000035.SZ 30/2450 ['09']
000036.SZ 31/2450 ['01', '02', '03', '04', '05', '06', '07', '08', '09', '10', '11', '12']
000037.SZ 32/2450 ['01', '02', '03', '04']
000038.SZ 33/2450 ['02', '05', '06', '07', '08', '09', '10', '11', '12']
000039.SZ 34/2450 ['01', '02', '03', '04', '05', '06', '07', '08', '09', '10', '11', '12']
000040.SZ 35/2450 ['01', '02', '03', '04', '05', '06', '07', '08', '09', '10', '11', '12']
000042.SZ 36/2450 ['02', '03', '04', '05', '06', '07', '08', '09', '10', '11', '12']
000043.SZ 37/2450 ['01', '02', '03', '04', '05', '06', '07', '08', '09', '10', '11', '12']
000045.SZ 38/2450 ['01', '02', '03']
000046.SZ 39/2450 ['01', '02', '03', '04', '05', '06', '07', '08', '09', '10', '11', '12']
000048.SZ 40/2450 ['01', '02', '03', '04', '05', '06', '07', '08', '09', '10', '11', '12']
000049.SZ 41/2450 ['01', '02', '03', '04', '05', '06', '07', '08', '09', '10', '11', '12']
000050.SZ 42/2450 ['01', '02', '03', '04', '05', '06', '07', '08', '09', '10', '11', '12']
000055.SZ 43/2450 ['01', '02', '03', '04', '05', '06', '07', '08', '09', '10', '11', '12']
000056.SZ 44/2450 ['05', '06', '07', '08', '09', '10', '11', '12']
000058.SZ 45/2450 ['01', '02', '03', '04', '05', '06', '07', '08', '09', '10', '11', '12']
000059.SZ 46/2450 ['01', '02', '03', '04', '05', '06', '07', '08', '09', '10', '11', '12']
000060.SZ 47/2450 ['01', '02', '03', '04', '05', '06', '07', '08', '09', '10', '11', '12']
000061.SZ 48/2450 ['01', '02', '03', '04', '05', '06', '07', '08', '09', '10', '11', '12']
000062.SZ 49/2450 ['01', '02', '03', '04', '05', '06', '07', '08', '09', '10', '11', '12']
000063.SZ 50/2450 ['01', '02', '03']
000065.SZ 51/2450 ['01', '02', '03', '04', '05', '06', '07', '08', '09', '10', '11', '12']
000066.SZ 52/2450 ['01', '02', '03', '04']
000068.SZ 53/2450 ['04', '05', '06', '07', '08', '09', '10', '11', '12']
000069.SZ 54/2450 ['01', '02', '03', '04', '05', '06', '07', '08', '09', '10', '11', '12']
000070.SZ 55/2450 ['01', '02', '03', '04', '05', '06', '07', '08', '09', '10', '11', '12']
000078.SZ 56/2450 ['01', '02', '03', '04', '05', '06', '07', '08', '09', '10', '11', '12']
000088.SZ 57/2450 ['01', '02', '03', '04', '05', '06', '07', '08', '09', '10', '11', '12']
000089.SZ 58/2450 ['01', '02', '03', '04', '05', '06', '07', '08', '09', '10', '11', '12']
000090.SZ 59/2450 ['01', '02', '03', '04', '05', '06', '07', '08', '09', '10', '11', '12']
000096.SZ 60/2450 []
...
603766.SH 2449/2450 []
603993.SH 2450/2450 ['11', '12']

X1.describe()

X1.to_csv('data/factors2013-4.csv')

方案四对数据进行了复杂的预处理，但是最后的结果反而不好，往往最简单的方案效果反而是最好的。

后面使用XGBoost回归是接受缺失值的存在的，因此可以不进行缺失值处理。

去极值处理

部分股票的某些因子在个别时间点会突然变大，或相对当天其他股票的该因子暴露值差别较大。这是由于金融领域数据相较其它领域的观测值而言，具有更为高频的、更加剧烈的波动特征。在采取回归法构建多因子模型时，这些个别的极端值会拉偏因子对股票收益率的解释程度，影响模型的拟合效果。因此，需要在标准化因子数据之前，先进行去极值处理。

去极值的一般思路为设定数据的上限与下限，将超出限值的数据调整为限值，常用的上下限构造标准有绝对值差中位数（mean absolute deviation，MAD）、3和百分位。

方案一： MAD

有一元序列${x_i, i=1,2,\cdots, n}$，该序列与其中位数偏差的绝对值所形成的序列的中位数称作MAD，其数学表示如下：

$$MAD=median(|x_i-median(x)|)$$

最后将x限制在 $median(x)-k\cdot MAD<=x_i<=median(x)+k\cdot MAD$ 中

X=pd.read_csv('data/factors2013-3.csv')
X=dropun(X)
X.describe()
hist([X['size']])

median=X.median()
MAD=(X-median).abs().median()

k=10
min_value=median-k*MAD
max_value=median+k*MAD
print(min_value)
print(max_value)

ACC          -9.180113e-02
ACCP         -5.251415e+00
AG           -4.927788e-01
AM           -4.923881e+00
BM           -1.359178e+00
BVEG         -4.327330e-01
CFP          -4.815810e-02
CFdebt       -3.881376e-01
CR           -4.270300e+00
CRG          -1.021016e+00
CT           -1.397819e+00
DP           -6.370424e-03
EP           -2.195483e-01
INVG         -1.156071e+00
INVchg       -1.312386e-01
LEV          -2.966400e+00
LG           -1.061879e+00
LM           -1.357009e-01
OCFP         -1.888001e-01
PA           -1.308000e-01
PMG          -3.770258e+00
QR           -3.880200e+00
QRG          -1.369995e+00
RD           -1.103895e-01
RDsales      -2.688179e-01
ROA          -1.473480e-01
ROE          -2.194150e-01
SG           -1.309019e+00
SP           -1.765197e+00
SgINVg       -1.319552e+00
TAXchg       -5.199572e+00
age          -1.374500e+04
beta         -1.008042e+00
betad        -1.987331e+00
cash         -5.665304e-01
cashpr       -3.070982e+01
coskew       -2.887433e+00
idskew       -4.365035e+00
idvol        -2.741138e+00
illq         -3.949364e-02
imom         -7.660539e-14
lagretn      -7.192280e-01
mom12        -2.119960e+00
mom6         -1.421134e+00
momchg       -1.512736e+00
retnmax      -1.203994e+01
salecash     -1.113651e+01
saleinv      -1.516437e+01
sharechg      0.000000e+00
size         -1.589781e+04
skew         -3.389709e+00
std_dvol     -1.274288e+02
std_turn     -3.401294e+00
total_vol    -1.891834e+00
trade_date             NaN
ts_code                NaN
turn         -6.072329e+00
volumed      -2.345334e+02
dtype: float64
ACC           7.637599e-02
ACCP          4.824929e+00
AG            5.913275e-01
AM            7.267486e+00
BM            2.256896e+00
BVEG          5.530870e-01
CFP           6.327857e-02
CFdebt        4.822336e-01
CR            7.323700e+00
CRG           9.867493e-01
CT            1.964784e+00
DP            7.786073e-03
EP            2.967405e-01
INVG          1.251221e+00
INVchg        1.399593e-01
LEV           3.920587e+00
LG            1.191900e+00
LM            2.006399e-01
OCFP          2.613488e-01
PA            1.682481e-01
PMG           3.805522e+00
QR            5.913800e+00
QRG           1.319362e+00
RD            1.535517e-01
RDsales       4.202944e-01
ROA           1.976120e-01
ROE           2.815450e-01
SG            1.457741e+00
SP            2.350373e+00
SgINVg        1.450347e+00
TAXchg        5.057923e+00
age           2.275500e+04
beta          2.918533e+00
betad         4.025104e+00
cash          8.611341e-01
cashpr        3.004561e+01
coskew        2.659997e+00
idskew        5.810219e+00
idvol         6.972840e+00
illq          5.161289e-02
imom          7.660539e-14
lagretn       7.529421e-01
mom12         2.173220e+00
mom6          1.507490e+00
momchg        1.566523e+00
retnmax       2.144165e+01
salecash      1.483437e+01
saleinv       1.996187e+01
sharechg      0.000000e+00
size          2.362719e+04
skew          3.875412e+00
std_dvol      1.754805e+02
std_turn      4.845325e+00
total_vol     6.973538e+00
trade_date             NaN
ts_code                NaN
turn          9.005481e+00
volumed       3.271191e+02
dtype: float64

X[X-min_value<0] = np.nan
X=X.fillna(value=dict(min_value))
X[X-max_value>0] = np.nan
X=X.fillna(value=dict(max_value))
hist([X['size']])

​

​

方案二：百分位

个人感觉MAD方法相当不科学，sharechg相当于彻底没用了（median和MAD均为0）！在金融领域，这样的处理方法显然不合理。

将因子值进行升序的排序，对排位百分位高于$1-\alpha$或排位百分位低于$\alpha$的因子值，进行类似于 MAD 、 3σ 的方法进行调整，其中α为分位点，取2.5%

X=pd.read_csv('data/factors2013.csv')# 不进行缺失值处理，原来是data/factors2013-3.csv

factors=list(X.columns)
factors.remove('ts_code')
factors.remove('trade_date')
print(factors)

X.isnull().any()

['size', 'beta', 'betad', 'idvol', 'total_vol', 'idskew', 'skew', 'coskew', 'turn', 'std_turn', 'volumed', 'std_dvol', 'retnmax', 'illq', 'LM', 'sharechg', 'age', 'mom12', 'mom6', 'momchg', 'imom', 'lagretn', 'BM', 'AM', 'LEV', 'EP', 'CFP', 'OCFP', 'DP', 'SP', 'AG', 'LG', 'BVEG', 'INVG', 'INVchg', 'SG', 'SgINVg', 'PMG', 'TAXchg', 'ACC', 'ACCP', 'ROE', 'ROA', 'PA', 'CT', 'cash', 'cashpr', 'RD', 'RDsales', 'CR', 'QR', 'CFdebt', 'salecash', 'saleinv', 'CRG', 'QRG']



ts_code       False
trade_date    False
size          False
beta           True
betad          True
idvol          True
total_vol      True
idskew         True
skew           True
coskew         True
turn           True
std_turn       True
volumed        True
std_dvol       True
retnmax        True
illq           True
LM             True
sharechg       True
age           False
mom12          True
mom6           True
momchg         True
imom           True
lagretn        True
BM            False
AM            False
LEV           False
EP             True
CFP            True
OCFP           True
DP             True
SP            False
AG             True
LG             True
BVEG           True
INVG           True
INVchg         True
SG             True
SgINVg         True
PMG            True
TAXchg         True
ACC            True
ACCP           True
ROE           False
ROA            True
PA            False
CT             True
cash           True
cashpr         True
RD            False
RDsales       False
CR             True
QR             True
CFdebt         True
salecash       True
saleinv        True
CRG            True
QRG            True
dtype: bool

alpha=0.025 
min_value=X.quantile(alpha)
max_value=X.quantile(1-alpha)

for factor in factors:
    X[factor][X[factor]<min_value[factor]]=min_value[factor]
    
for factor in factors:
    X[factor][X[factor]>max_value[factor]]=max_value[factor]

下列方法似乎无效，nan值被去掉了

alpha=0.025
min_value=X.quantile(alpha)

def set_min(x, value):
    if x is np.nan:
        return np.nan
    if x>value:
        return x
    return value
        

for factor in factors:
    X[factor] = X[factor].apply(lambda x: set_min(x, min_value[factor]))

hist([X['size']])

X.to_csv('data/factors2013-0-2.csv')

去除极端值

事实上，极端值应该直接去除（删掉所有极值数据），而不是被约束

X=pd.read_csv('data/factors2013.csv')# 不进行缺失值处理，原来是data/factors2013-3.csv

for factor in factors:
    dev = X.std()[factor]/10
    string = factor+'>='+str(min_value[factor]-dev)+' and '+factor+'<='+str(max_value[factor]+dev)
    print(string)
    try:
        X=X.query(string)
    except:
        continue

size>=466.23701849061104 and size<=56197.80253800938
beta>=0.30221090662445765 and beta<=1.670458077767895
betad>=-0.11625320236077219 and betad<=2.0717848607694846
idvol>=0.9615292601591058 and idvol<=10.032230277360437
total_vol>=1.4298433155674244 and total_vol<=10.159942820944554
idskew>=-1.286912166446209 and idskew<=6.727747089327816
skew>=-1.1853762032129542 and skew<=6.62232951119754
coskew>=-0.696590000792659 and coskew<=0.5729513849863401
turn>=0.10958406312627103 and turn<=10.249778555921342
std_turn>=0.015473350415526027 and std_turn<=5.724879523908469
volumed>=1.5086259442366883 and volumed<=400.3405305938586
std_dvol>=-0.08583099559686724 and std_dvol<=201.43869662243335
retnmax>=1.5798185967027107 and retnmax<=10.327227399697504
illq>=-0.0008883217724178357 and illq<=0.061863428262617905
LM>=0.001792631659403624 and LM<=0.1824092987459326
sharechg>=-0.038136709995187236 and sharechg<=2.054779505265279
age>=144.44450825888697 and age<=7637.555491741113
mom12>=-0.4449834517221042 and mom12<=1.4840343303713122
mom6>=-0.32713652798777554 and mom6<=1.0861104156850228
momchg>=-0.9139732133983118 and momchg<=0.5396105415189818
imom>=-3.128024526047981e-14 and imom<=3.1835356772792386e-14
lagretn>=-0.22286157090150918 and lagretn<=0.3694367399650399
BM>=0.05128918974254905 and BM<=1.2785656548593018
AM>=0.08767721649814822 and AM<=8.397712433435103
LEV>=-0.05261148458880632 and LEV<=5.904579534586556
EP>=-0.0016750571453810073 and EP<=0.2485481165738477
CFP>=-0.005495478107830383 and CFP<=0.06623998107507538
OCFP>=-0.01782867987827 and OCFP<=0.2355573893646679
DP>=-0.0007575871111245347 and DP<=0.048462608536149664
SP>=-0.0365945680130713 and SP<=3.559805103566347
AG>=-0.144951436729631 and AG<=0.572742613101132
LG>=-0.3923631689491917 and LG<=1.370618107272371
BVEG>=-0.2670375782807459 and BVEG<=2.115756578280746
INVG>=nan and INVG<=nan
INVchg>=-0.07192103574412508 and INVchg<=0.12593971605394447
SG>=-1.0757814793459466 and SG<=1.8388494793459464
SgINVg>=-0.5247619250878223 and SgINVg<=1.271522702702136
PMG>=-5.378343388790043 and PMG<=5.351935388790043
TAXchg>=-8.445006570730525 and TAXchg<=9.511201048239645
ACC>=-0.05191731583423558 and ACC<=0.02343552011219536
ACCP>=-20.93164678230037 and ACCP<=9.702867358580818
ROE>=-0.12700461612294808 and ROE<=0.19080361612294808
ROA>=-0.03125005286510647 and ROA<=0.13637005286510645
PA>=-0.038806159800909096 and PA<=0.12323211839573989
CT>=-0.009019532111409267 and CT<=1.5335358023752697
cash>=0.004282037941109826 and cash<=0.6517237171869091
cashpr>=-15.696952790937148 and cashpr<=59.17672817502837
RD>=-0.0005856246253221559 and RD<=0.15173533479616041
RDsales>=0.0033296815627923155 and RDsales<=0.5224191666458877
CR>=0.06102554460830073 and CR<=17.2501744553917
QR>=0.02127856757199137 and QR<=15.286321432428009
CFdebt>=-0.11658295537729796 and CFdebt<=1.1449027864363264
salecash>=-0.26358347170427476 and salecash<=17.547743452456704
saleinv>=nan and saleinv<=nan
CRG>=-0.539995989038383 and CRG<=1.024431778695194
QRG>=-0.5970849177829025 and QRG<=1.3735574033160003

hist([X['size']])

X.describe()

如果直接暴力删去极值，数量会从560296锐减到146900（20%左右）

这说明即使是按百分位，大多数据仍会处于有效范围之外。很难说哪种方法是有效的

X.to_csv('data/factors2013-0-3.csv')

标准化处理

不同变量的观测值往往会由不同的单位衡量，不同的因子由于意义和计算方式的不同，也存在着量纲的不一致性，造成不同因子的暴露在数值上也呈现了较大差异。对数据进行标准化处理可以将量纲统一，防止因子值量纲的差异对模型造成影响。常用的标准化方法为z-score标准化和min-max标准化。

本文使用z-score标准化：

$$x^*=\frac{x-\mu}{\sigma}$$

其中，是$\mu$所有样本数据的均值，则$\sigma$表示所有样本数据的标准差。经过z-score标准化处理后的序列均值为0，方差为1，数据围绕0上下波动。

X=pd.read_csv('data/factors2013-0-2.csv')

X1=(X-X.mean())/X.std()

X[list(factors)]=X1[list(factors)]
hist([X['size']])

X.to_csv('data/factors2013-0-2-1.csv')

因变量

数据表有(open, close, pre_close)三个属性，最后需要计算收益率的值作为最后的因变量。

检查正确性

1. 数据表中存不存在空值或缺失值
2. 有 $close_{t-1}=pre\_close_t$ 这一个pre-requisite，检查是否正确
3. 是否存在某些天的日收益率大于0.1或小于-0.1，在股市中都应该是不存在的现象

Y = pd.read_csv('data/daily2011-2017.csv')
Y.isnull().any()

ts_code       False
trade_date    False
open          False
close         False
pre_close     False
dtype: bool

Y_post=Y.drop(index=[0]) # 表示Y的后一天，只需要pre_close数据
Y_post=Y_post.drop(columns=['open', 'close'])
Y_post.columns=['ts_code1', 'trade_date1', 'post_pre_close']
Y_post.reset_index(drop=True, inplace=True)

Y_concat=pd.concat([Y,Y_post], axis=1, join='inner')
Y_concat=Y_concat[Y_concat['ts_code']==Y_concat['ts_code1']]

Y_concat = Y_concat[Y_concat['close']!=Y_concat['post_pre_close']]
Y_concat['close_diff'] = Y_concat['close']-Y_concat['post_pre_close']
Y_concat['time_diff'] = pd.to_datetime(Y_concat['trade_date1'])-pd.to_datetime(Y_concat['trade_date'])
Y_concat

虽然表内不存在空值，但是存在一些缺失值，即明明不是周末，股票前一天和后一天相差的时间多于一天。因此，k日平均收益率需要谨慎考虑计算方式。

除此之外，日收益率异常的情况在下面检查。

日收益率

计算日收益率有以下两种方式：

$$yield_t=\frac{close_t-open_t}{open_t}$$

$$yield_t=\frac{close_t-close_{t-1}}{close_{t-1}}$$

收盘价是前一交易日确定的，而第二天的开盘价格按照规则要以前一日收盘价为基准，按照新股20%，正常股票10%，ST股5%上下浮动。

由上述可知，第二天的开盘价虽然是基于前一日收盘价，但具体则由两市早上9点15分至25分集合竞价后的成交价决定，如果基于前一天的收盘价竞价时买盘强大，而且买盘挂的价格高，开盘价就会高于昨天的收盘价，反之就会低。最终形成部分股票的价格开盘价会和上一交易日的收盘价不一致，或高或低，也有相同的情况。

集合竞价也由股票内在动因决定，因子对其应该也有影响，但这里只考虑简单因素（第一种情况）

Y['yield']=(Y['close']-Y['open'])/Y['open']
Y=Y.drop(columns=['open','close','pre_close'])

hist([Y['yield']])

理论上应该是不存在日收益率大于0.1或小于-0.1的情况，如果有的话，应该予以排除，将收益率规约到[-0.1,0.1]的区间，下面就正常多了

Y[Y['yield']<-0.1]=-0.1
Y[Y['yield']>0.1]=0.1

hist([Y['yield']])

Y.to_csv('data/daily2011-2017-1.csv')

k日平均收益率

一天的收益率导致的巧合性还是远远大于股市背后的规律性，因此考虑使用k日平均收益代替日收益率。特别的，短期收益率k=5，中期收益率k=20，长期收益率k=120.

计算方法：$$yield_{k,t}=\frac{close_{t+k-1}-close_{t-1}}{k\cdot close_{t-1}}=\frac{close_{t+k-1}-pre_close_t}{k\cdot pre_close_t}$$

Y = pd.read_csv('data/daily2011-2017.csv')
Y.isnull().any()

ts_code       False
trade_date    False
open          False
close         False
pre_close     False
dtype: bool

k=5
Y1=Y.drop(index=range(k-1))
Y1=Y1.drop(columns=['open', 'pre_close'])
Y1.columns=['ts_code1', 'trade_date1', 'close1']
Y1.reset_index(drop=True, inplace=True)

Y2=pd.concat([Y,Y1], axis=1, join='inner')

Y=Y2[Y2['ts_code']==Y2['ts_code1']]
Y['yield']=(Y['close1']-Y['pre_close'])/Y['pre_close']/k

<ipython-input-93-74a265bd328f>:2: SettingWithCopyWarning: 
A value is trying to be set on a copy of a slice from a DataFrame.
Try using .loc[row_indexer,col_indexer] = value instead

See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy
  Y['yield']=(Y['close1']-Y['pre_close'])/Y['pre_close']/k

Y['time_diff'] = pd.to_datetime(Y['trade_date1'])-pd.to_datetime(Y['trade_date'])
Y=Y.drop(columns=['open','close','pre_close','ts_code1','trade_date1','close1'])

<ipython-input-94-3234fc21ba06>:1: SettingWithCopyWarning: 
A value is trying to be set on a copy of a slice from a DataFrame.
Try using .loc[row_indexer,col_indexer] = value instead

See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy
  Y['time_diff'] = pd.to_datetime(Y['trade_date1'])-pd.to_datetime(Y['trade_date'])

Y['time_diff'] = Y['time_diff'].astype('str').apply(lambda x:x[:-5]).astype('int32')  # 时间差转为int形式
hist([Y['time_diff']])

delete_index = Y[(Y['time_diff']>k*7/5 + 1) | (Y['time_diff']<k - 1)].index  
# 这里可能是周末的原因，时间差 = k * 7/5，未来尽量不丢失数据，在原有基础上分别+1和-1
print(len(delete_index)/len(Y))
Y = Y.drop(delete_index)
hist([Y['time_diff']])

0.07832959924480373

Y[Y['yield']<-0.1]=-0.1
Y[Y['yield']>0.1]=0.1
hist([Y['yield']])

Y.to_csv('data/daily2011-2017-5.csv')