机器学习模拟投资者预期分歧

# -*- coding: utf-8 -*-

import os
import numpy as np
import pandas as pd
import random
random.seed(42)
import warnings
warnings.filterwarnings("ignore")
import lightgbm as lgb

path = r'D:'
os.chdir(path)

# %% 主要代码

# 定义一个函数来随机选择50列
def extract_random_columns(dataframe, num_cols, num_iterations):
    # dataframe = ch.copy()
    all_columns = dataframe.columns.tolist()[3:]
    random_column_lists = []
    
    for _ in range(num_iterations):
        # 随机选择50列，确保每次选择不重复
        selected_columns = random.sample(all_columns, num_cols)
        random_column_lists.append(selected_columns)
    
    return random_column_lists

def calculate_mdf(ch,random_column_selections,trw,viw,tew,models='ols',param=None):
    # length为滑动窗口长度：取值{12}
    month_list = sorted(list(set(ch['month'])))
    
    mfd_temp = []

    for i in range(len(month_list)-trw-viw):
        # i = 0
        print(i)
        # month = month_list[i]
        month_train_start = month_list[i]
        month_train_end = month_list[i+trw-1]
        month_valid_start = month_list[i+trw]
        month_valid_end = month_list[i+trw+viw-1]        
        month_test = month_list[i+trw+viw]
        
        
        train_data = ch[(ch['month']>=month_train_start)&(ch['month']<=month_train_end)].iloc[:,2:]
        valid_data = ch[(ch['month']>=month_valid_start)&(ch['month']<=month_valid_end)].iloc[:,2:]
        test_data = ch[ch['month']==month_test].iloc[:,2:]
        
        stk_pool = ch[ch['month']==month_test]['stkcd'].to_list()
        
        predict = []

        for j in range(len(random_column_selections)):
            # j=0
            select_feature = random_column_selections[j]
        
            x_train = np.array(train_data[select_feature]) # 训练集 输入
            y_train = np.array(train_data['ret'])  # 训练集 输出
            
            x_valid = np.array(valid_data[select_feature]) # 验证集 输入
            y_valid = np.array(valid_data['ret'])  # 验证集 输出        
            
            x_test = np.array(test_data[select_feature])  # 测试集 输入
   
            lgb_train = lgb.Dataset(x_train, y_train) # 将数据保存到LightGBM二进制文件将使加载更快
            #lgb_eval = lgb.Dataset(x_test, y_test, reference=lgb_train)  # 创建验证数据
            
            params = {
                'task': 'train',
                'boosting_type': 'gbdt',  # 设置提升类型
                'objective': 'regression',  # 目标函数
                'metric':  'rmse',  # 评估函数#auc,mape,rmse,mae
                'n_jobs': 14,
                #'subsample': 0.5,
                #'subsample_freq': 1,
                'num_leaves': 31,   # 叶子节点数
                'learning_rate': param['learning_rate'],  # 学习速率
                'feature_fraction': 0.9,  # 建树的特征选择比例
                'bagging_fraction': 0.8,  # 建树的样本采样比例
                'bagging_freq': 5,  # k 意味着每 k 次迭代执行bagging
                'n_estimators': param['n_estimators'],  # 基学习器个数
                'verbose': -1,  # <0 显示致命的, =0 显示错误 (警告), >0 显示信息
            } #这里目前还没有验证集，之后可以考虑加入验证集进行训练
        
            model = lgb.train(params,lgb_train,verbose_eval = 5)
            y_pred = model.predict(x_test)

            predict.append(y_pred)
        
        y_pred = pd.DataFrame(predict, columns=stk_pool)
        y_pred_std = 1/y_pred.std(axis=0)

        mfd_temp.append(y_pred_std)
        
    mfd_df = pd.concat(mfd_temp, axis=1).T
    mfd_df.index = month_list[trw+viw:]

    return mfd_df

# %% 启动程序
if __name__ == '__main__':
    
    start_month = 200501 #回测开始年份
    end_month = 202212 #回测结束年份
    

    # 读取数据
    ch = pd.read_parquet(os.path.join(path, '过程数据', 'factor_ret.parquet'))
    ch = ch.sort_values(by=['month', 'stkcd'], ascending=True)
    ch = ch[(ch['month']>=start_month)&(ch['month']<=end_month)]

    # 随机选取特征
    feature_num = 50
    cycle_num = 100    
    random_feature = extract_random_columns(ch, feature_num, cycle_num)
    
    # 训练集、验证集、测试集的长度
    trw = 12
    viw = 0 #暂时未考虑验证集
    tew = 1
    
    mfd_df_lgb = calculate_mdf(ch,random_feature,trw,viw,tew,models='lgbm',param={'n_estimators':1000,'learning_rate':0.001})
    mfd_df_lgb.to_parquet(os.path.join(path,'mfd因子','mfd_lgb.parquet'))
    print('lgb finished') 

# -*- coding: utf-8 -*-

import pandas as pd
import numpy as np
import glob
import os
import seaborn as sns
import matplotlib
# matplotlib.use("Agg") #不显示图片
import matplotlib.pyplot as plt
# import seaborn as sns
import warnings
warnings.filterwarnings('ignore')

plt.rcParams['font.sans-serif'] = ['SimHei']  # 用来正常显示中文标签
plt.rcParams['axes.unicode_minus'] = False  # 用来正常显示坐标轴上的负数

path = r'F:'
os.chdir(path)

# %% 函数

# 最大回撤
def get_DD(ret):
    ret_acc = (1 + ret.fillna(0)).cumprod()
    ret_acc_max = np.maximum.accumulate(ret_acc)
    mdd_series = (ret_acc_max - ret_acc) / ret_acc_max
    mdd = mdd_series.max()

    return mdd, mdd_series


def get_benchmark_result(benchmark):

    result_benchmark = pd.DataFrame(
        columns=['年化收益', '年化波动率', '年化夏普比率', '最大回撤'], index=benchmark.columns)

    for i in range(benchmark.shape[1]):

        retport = benchmark.iloc[:, i]

        mean = np.mean(retport) * 12 * 100

        std = np.std(retport, ddof=1)*np.sqrt(12) * 100

        sr = mean/std

        mdd = get_DD(pd.Series(retport))[0] * 100

        result_benchmark.iloc[i, 0] = mean
        result_benchmark.iloc[i, 1] = std
        result_benchmark.iloc[i, 2] = sr
        result_benchmark.iloc[i, 3] = mdd

    return result_benchmark

# 换手率
def get_turnover(retn, weight):

    weight_lag = weight.mul(1+retn)
    weight_lag = weight_lag.div(weight_lag.sum(axis=1), axis=0)
    weight_lag = weight_lag.shift(1).fillna(0)

    turnover_series = np.sum(np.abs(weight-weight_lag), axis=1) / 2
    turnover_series.iloc[0] = 1

    turnover = np.mean(turnover_series) * 12

    return turnover, turnover_series

# 计算策略表现
def get_single_result(retn, weight, port_name, cost):
    # retn=ret.copy()
    turnover, turnover_series = get_turnover(retn, weight)

    retport = np.sum(weight*retn, axis=1)  # 组合收益率
    retport.name = port_name

    retport = retport - turnover_series * cost * 2  # 直接减去费用

    mean = np.mean(retport) * 12 * 100
    std = np.std(retport, ddof=1)*np.sqrt(12) * 100
    sr = mean/std

    mdd, mdd_series = get_DD(pd.Series(retport))

    res = [mean, std, sr, mdd*100, turnover]

    result = pd.DataFrame(res, columns=[port_name], index=[
                          '年化收益', '年化波动率', '年化夏普比率', '最大回撤', '年化换手率']).T
    return retport, result


# 获取结果
def get_all_result(factor, benchmark, cost, ret, stk_pool):

    result_benchmark = get_benchmark_result(benchmark)
    
    weight = factor * stk_pool
    weight = weight.rank(pct=True,axis=1)
    weight = pd.DataFrame(np.where(weight>=0.9,1,0),index=weight.index,columns=weight.columns)
    weight = weight.div(weight.sum(axis=1),axis=0)

    port_name = 'MFD策略'
    
    ret_series, result = get_single_result(ret, weight, port_name, cost)
    

    result_all = pd.concat([result, result_benchmark],
                           axis=0, join='outer') .astype(float)

    ret_series = pd.concat([ret_series, benchmark],
                           axis=1, join='inner') .astype(float)
    ret_series.columns = ['MFD策略', '沪深300', '中证500', '万得全A']

    return result_all, ret_series,weight

def figure_ret(ret_series):

    port_cumret = (1+ret_series).cumprod()

    port_cumret = port_cumret - 1

    port_cumret['MFD相对沪深300累计超额'] = port_cumret['MFD策略'] - port_cumret['沪深300']
    port_cumret.index = pd.to_datetime(
        port_cumret.index.astype(str), format='%Y%m')

    mdd_series = -get_DD(ret_series['MFD策略'])[1]
    mdd_series.index = pd.to_datetime(
        mdd_series.index.astype(str), format='%Y%m')

    # 画图
    fig = plt.figure(figsize=(15, 12))
    ax1 = fig.add_subplot(111)

    # 绘制折线图

    ax1.plot(port_cumret.index, port_cumret[
             'MFD策略'], color='red', label='MFD策略')
    ax1.plot(port_cumret.index, port_cumret[
             '沪深300'], color='aqua', label='沪深300')
    ax1.plot(port_cumret.index, port_cumret[
             '中证500'], color='gold', label='中证500')
    ax1.plot(port_cumret.index, port_cumret[
             '万得全A'], color='black', label='万得全A')

    ax1.fill_between(
        port_cumret.index, port_cumret['MFD相对沪深300累计超额'], color="skyblue", alpha=0.3, label='MFD相对沪深300累计超额')

    ax1.set_xlabel('年月')
    ax1.set_ylabel('累计收益率')
    ax1.tick_params(axis='y')
    ax1.legend(loc='upper left')  # 图标

    # 创建右边的 y 轴    # 绘制填充图
    ax2 = ax1.twinx()
    ax2.fill_between(mdd_series.index, mdd_series,
                     color='gray', alpha=0.3, label='回撤')

    ax2.set_ylabel('回撤')
    ax2.tick_params(axis='y')
    ax2.legend(loc='upper right')  # 图标
    ax2.set_ylim(-1, 0)  # y坐标轴的刻度范围
    ax2.set_title('策略表现')

    #plt.show()
    plt.savefig(os.path.join(path,  '检验结果', '累计收益.png'))
    plt.close()

    
# %% 运行程序
if __name__ == '__main__':
    
    start_month = 201101  # 回测开始年份
    end_month = 202212  # 回测结束年份
    
    # 读取基准指数数据
    benchmark = pd.read_parquet('benchmark.parquet')

    # 读取收益率数据
    ret = pd.read_parquet('ret.parquet')
    stk_pool = pd.read_parquet('stk_pool.parquet')
    
    # 读取因子
    factor = pd.read_parquet(os.path.join(path, 'mfd因子', 'mfd_lgb.parquet'))
    factor = factor[(factor.index>=start_month)&(factor.index<=end_month)]

    # 回测
    cost = 0.001  # 交易成本，双边千一
    
    result_all, ret_series,weight_final = get_all_result(factor
        , benchmark, cost, ret, stk_pool)

    figure_ret(ret_series)