#include <iostream>
#include <vector>
#include <unordered_map>
#include <memory>
#include <random>
#include <chrono>
#include <thread>
#include <mutex>
#include <cmath>
#include <queue>
#include <algorithm>
#include <Eigen/Dense>
#include <boost/asio.hpp>
#include <nlohmann/json.hpp>

using namespace std;
using namespace Eigen;
using json = nlohmann::json;
namespace asio = boost::asio;
using asio::ip::tcp;

// 广告渠道类型
enum class AdChannel {
    SEARCH,
    SOCIAL,
    DISPLAY,
    VIDEO,
    NATIVE,
    AUDIO
};

// 渠道状态
struct ChannelState {
    AdChannel type;
    double spend;
    double impressions;
    double clicks;
    double conversions;
    double ctr;
    double cvr;
    double cpa;
    double roas;
    
    // 更新渠道指标
    void update_metrics() {
        ctr = impressions > 0 ? clicks / impressions : 0;
        cvr = clicks > 0 ? conversions / clicks : 0;
        cpa = conversions > 0 ? spend / conversions : 0;
        roas = spend > 0 ? (conversions * 100.0) / spend : 0; // 假设每个转化价值$100
    }
};

// 深度确定性策略梯度(DDPG)智能体
class DDPGAgent {
private:
    // 演员网络(策略网络)
    MatrixXd actor_W1, actor_W2, actor_W3;
    VectorXd actor_b1, actor_b2, actor_b3;
    
    // 评论家网络(值函数网络)
    MatrixXd critic_W1, critic_W2, critic_W3;
    VectorXd critic_b1, critic_b2, critic_b3;
    
    // 目标网络
    MatrixXd target_actor_W1, target_actor_W2, target_actor_W3;
    VectorXd target_actor_b1, target_actor_b2, target_actor_b3;
    MatrixXd target_critic_W1, target_critic_W2, target_critic_W3;
    VectorXd target_critic_b1, target_critic_b2, target_critic_b3;
    
    // 经验回放缓冲区
    deque<tuple<VectorXd, VectorXd, double, VectorXd>> replay_buffer;
    int buffer_capacity;
    
    // 超参数
    double actor_lr;
    double critic_lr;
    double gamma;
    double tau;
    int batch_size;
    int state_dim;
    int action_dim;
    
    mutex mtx;
    random_device rd;
    mt19937 gen;
    
    // 初始化网络参数
    void initialize_weights(MatrixXd& W, int rows, int cols, double scale = 0.01) {
        W = MatrixXd::Random(rows, cols) * scale;
    }
    
    // 初始化偏置
    void initialize_biases(VectorXd& b, int size, double value = 0.1) {
        b = VectorXd::Constant(size, value);
    }
    
    // 前向传播(演员)
    VectorXd actor_forward(const VectorXd& state) {
        VectorXd h1 = (actor_W1 * state + actor_b1).array().tanh();
        VectorXd h2 = (actor_W2 * h1 + actor_b2).array().tanh();
        VectorXd action = (actor_W3 * h2 + actor_b3).array().sigmoid(); // 输出[0,1]范围
        return action;
    }
    
    // 前向传播(评论家)
    double critic_forward(const VectorXd& state, const VectorXd& action) {
        VectorXd state_action(state.size() + action.size());
        state_action << state, action;
        
        VectorXd h1 = (critic_W1 * state_action + critic_b1).array().relu();
        VectorXd h2 = (critic_W2 * h1 + critic_b2).array().relu();
        double value = (critic_W3 * h2 + critic_b3)(0);
        return value;
    }
    
    // 目标网络前向传播(演员)
    VectorXd target_actor_forward(const VectorXd& state) {
        VectorXd h1 = (target_actor_W1 * state + target_actor_b1).array().tanh();
        VectorXd h2 = (target_actor_W2 * h1 + target_actor_b2).array().tanh();
        VectorXd action = (target_actor_W3 * h2 + target_actor_b3).array().sigmoid();
        return action;
    }
    
    // 目标网络前向传播(评论家)
    double target_critic_forward(const VectorXd& state, const VectorXd& action) {
        VectorXd state_action(state.size() + action.size());
        state_action << state, action;
        
        VectorXd h1 = (target_critic_W1 * state_action + target_critic_b1).array().relu();
        VectorXd h2 = (target_critic_W2 * h1 + target_critic_b2).array().relu();
        double value = (target_critic_W3 * h2 + target_critic_b3)(0);
        return value;
    }
    
    // 软更新目标网络
    void soft_update() {
        target_actor_W1 = tau * actor_W1 + (1 - tau) * target_actor_W1;
        target_actor_W2 = tau * actor_W2 + (1 - tau) * target_actor_W2;
        target_actor_W3 = tau * actor_W3 + (1 - tau) * target_actor_W3;
        target_actor_b1 = tau * actor_b1 + (1 - tau) * target_actor_b1;
        target_actor_b2 = tau * actor_b2 + (1 - tau) * target_actor_b2;
        target_actor_b3 = tau * actor_b3 + (1 - tau) * target_actor_b3;
        
        target_critic_W1 = tau * critic_W1 + (1 - tau) * target_critic_W1;
        target_critic_W2 = tau * critic_W2 + (1 - tau) * target_critic_W2;
        target_critic_W3 = tau * critic_W3 + (1 - tau) * target_critic_W3;
        target_critic_b1 = tau * critic_b1 + (1 - tau) * target_critic_b1;
        target_critic_b2 = tau * critic_b2 + (1 - tau) * target_critic_b2;
        target_critic_b3 = tau * critic_b3 + (1 - tau) * target_critic_b3;
    }

public:
    DDPGAgent(int state_dim, int action_dim, 
             double actor_lr = 0.001, double critic_lr = 0.002,
             double gamma = 0.99, double tau = 0.005,
             int batch_size = 64, int buffer_capacity = 100000)
        : state_dim(state_dim), action_dim(action_dim),
          actor_lr(actor_lr), critic_lr(critic_lr),
          gamma(gamma), tau(tau),
          batch_size(batch_size), buffer_capacity(buffer_capacity),
          gen(rd()) {
        
        // 初始化演员网络
        initialize_weights(actor_W1, 400, state_dim);
        initialize_weights(actor_W2, 300, 400);
        initialize_weights(actor_W3, action_dim, 300);
        initialize_biases(actor_b1, 400);
        initialize_biases(actor_b2, 300);
        initialize_biases(actor_b3, action_dim);
        
        // 初始化评论家网络
        initialize_weights(critic_W1, 400, state_dim + action_dim);
        initialize_weights(critic_W2, 300, 400);
        initialize_weights(critic_W3, 1, 300);
        initialize_biases(critic_b1, 400);
        initialize_biases(critic_b2, 300);
        initialize_biases(critic_b3, 1);
        
        // 初始化目标网络
        target_actor_W1 = actor_W1;
        target_actor_W2 = actor_W2;
        target_actor_W3 = actor_W3;
        target_actor_b1 = actor_b1;
        target_actor_b2 = actor_b2;
        target_actor_b3 = actor_b3;
        
        target_critic_W1 = critic_W1;
        target_critic_W2 = critic_W2;
        target_critic_W3 = critic_W3;
        target_critic_b1 = critic_b1;
        target_critic_b2 = critic_b2;
        target_critic_b3 = critic_b3;
    }
    
    // 选择动作 (加入探索噪声)
    VectorXd select_action(const VectorXd& state, double noise_scale = 0.1) {
        lock_guard<mutex> lock(mtx);
        
        VectorXd action = actor_forward(state);
        
        // 添加OU噪声
        normal_distribution<double> dist(0.0, noise_scale);
        for (int i = 0; i < action.size(); ++i) {
            action[i] += dist(gen);
            action[i] = max(0.0, min(1.0, action[i])); // 裁剪到[0,1]范围
        }
        
        return action;
    }
    
    // 存储转移
    void store_transition(const VectorXd& state, const VectorXd& action, 
                         double reward, const VectorXd& next_state) {
        lock_guard<mutex> lock(mtx);
        
        replay_buffer.emplace_back(state, action, reward, next_state);
        if (replay_buffer.size() > buffer_capacity) {
            replay_buffer.pop_front();
        }
    }
    
    // 训练一步
    void train() {
        lock_guard<mutex> lock(mtx);
        
        if (replay_buffer.size() < batch_size) return;
        
        // 随机采样批次
        vector<size_t> indices(replay_buffer.size());
        iota(indices.begin(), indices.end(), 0);
        shuffle(indices.begin(), indices.end(), gen);
        
        for (int i = 0; i < batch_size; ++i) {
            const auto& [state, action, reward, next_state] = replay_buffer[indices[i]];
            
            // 计算目标Q值
            VectorXd next_action = target_actor_forward(next_state);
            double target_Q = target_critic_forward(next_state, next_action);
            double y = reward + gamma * target_Q;
            
            // 更新评论家
            double current_Q = critic_forward(state, action);
            double critic_loss = 0.5 * pow(y - current_Q, 2);
            
            // 省略反向传播具体实现...
            
            // 更新演员
            VectorXd actor_action = actor_forward(state);
            double actor_loss = -critic_forward(state, actor_action);
            
            // 省略反向传播具体实现...
            
            // 软更新目标网络
            soft_update();
        }
    }
    
    // 保存模型
    void save(const string& filename_prefix) {
        ofstream actor_file(filename_prefix + "_actor.txt");
        ofstream critic_file(filename_prefix + "_critic.txt");
        
        if (!actor_file || !critic_file) {
            throw runtime_error("无法保存模型");
        }
        
        // 保存演员网络
        actor_file << actor_W1 << "\n" << actor_W2 << "\n" << actor_W3 << "\n"
                  << actor_b1 << "\n" << actor_b2 << "\n" << actor_b3 << "\n";
        
        // 保存评论家网络
        critic_file << critic_W1 << "\n" << critic_W2 << "\n" << critic_W3 << "\n"
                   << critic_b1 << "\n" << critic_b2 << "\n" << critic_b3 << "\n";
    }
    
    // 加载模型
    void load(const string& filename_prefix) {
        ifstream actor_file(filename_prefix + "_actor.txt");
        ifstream critic_file(filename_prefix + "_critic.txt");
        
        if (!actor_file || !critic_file) {
            throw runtime_error("无法加载模型");
        }
        
        // 加载演员网络
        actor_file >> actor_W1 >> actor_W2 >> actor_W3 
                  >> actor_b1 >> actor_b2 >> actor_b3;
        
        // 加载评论家网络
        critic_file >> critic_W1 >> critic_W2 >> critic_W3 
                   >> critic_b1 >> critic_b2 >> critic_b3;
        
        // 同时更新目标网络
        target_actor_W1 = actor_W1;
        target_actor_W2 = actor_W2;
        target_actor_W3 = actor_W3;
        target_actor_b1 = actor_b1;
        target_actor_b2 = actor_b2;
        target_actor_b3 = actor_b3;
        
        target_critic_W1 = critic_W1;
        target_critic_W2 = critic_W2;
        target_critic_W3 = critic_W3;
        target_critic_b1 = critic_b1;
        target_critic_b2 = critic_b2;
        target_critic_b3 = critic_b3;
    }
};

// 跨渠道广告预算优化器
class CrossChannelBudgetOptimizer {
private:
    unordered_map<AdChannel, ChannelState> channels;
    DDPGAgent agent;
    double total_budget;
    double remaining_budget;
    chrono::system_clock::time_point last_update;
    mutex mtx;
    thread training_thread;
    atomic<bool> running;
    
    // 状态编码
    VectorXd encode_state() {
        VectorXd state(6 * channels.size());
        int i = 0;
        
        for (const auto& [channel, stats] : channels) {
            state[i++] = stats.spend / total_budget;
            state[i++] = stats.ctr;
            state[i++] = stats.cvr;
            state[i++] = stats.cpa;
            state[i++] = stats.roas;
            state[i++] = remaining_budget / total_budget;
        }
        
        return state;
    }
    
    // 动作解码 (分配比例)
    unordered_map<AdChannel, double> decode_action(const VectorXd& action) {
        unordered_map<AdChannel, double> allocation;
        double sum = action.sum();
        
        int i = 0;
        for (const auto& [channel, _] : channels) {
            allocation[channel] = action[i++] / sum;
        }
        
        return allocation;
    }
    
    // 训练线程
    void training_loop() {
        while (running) {
            this_thread::sleep_for(chrono::minutes(1)); // 每分钟训练一次
            
            lock_guard<mutex> lock(mtx);
            agent.train();
        }
    }
    
    // 计算奖励
    double calculate_reward() {
        double total_conversions = 0;
        double total_spend = 0;
        
        for (const auto& [_, stats] : channels) {
            total_conversions += stats.conversions;
            total_spend += stats.spend;
        }
        
        // 奖励 = 总转化价值 - 总花费 (假设每个转化价值$100)
        return total_conversions * 100.0 - total_spend;
    }

public:
    CrossChannelBudgetOptimizer(double budget, 
                               const vector<AdChannel>& channel_types,
                               int state_dim = 36, int action_dim = 6)
        : total_budget(budget), remaining_budget(budget),
          agent(state_dim, action_dim), running(false) {
        
        // 初始化渠道状态
        for (auto channel : channel_types) {
            channels[channel] = {channel, 0, 0, 0, 0, 0, 0, 0, 0};
        }
        
        last_update = chrono::system_clock::now();
    }
    
    ~CrossChannelBudgetOptimizer() {
        stop();
    }
    
    // 启动优化器
    void start() {
        if (running) return;
        running = true;
        training_thread = thread(&CrossChannelBudgetOptimizer::training_loop, this);
    }
    
    // 停止优化器
    void stop() {
        if (!running) return;
        running = false;
        if (training_thread.joinable()) {
            training_thread.join();
        }
    }
    
    // 获取预算分配
    unordered_map<AdChannel, double> get_budget_allocation() {
        lock_guard<mutex> lock(mtx);
        
        // 编码当前状态
        VectorXd state = encode_state();
        
        // 获取动作 (预算分配比例)
        VectorXd action = agent.select_action(state, 0.1); // 加入少量探索噪声
        
        // 解码为各渠道分配比例
        auto allocation = decode_action(action);
        
        // 根据剩余预算计算实际分配金额
        unordered_map<AdChannel, double> budget_allocation;
        for (const auto& [channel, ratio] : allocation) {
            budget_allocation[channel] = ratio * remaining_budget;
        }
        
        return budget_allocation;
    }
    
    // 更新渠道表现数据
    void update_channel_performance(AdChannel channel, 
                                   double spend, double impressions, 
                                   double clicks, double conversions) {
        lock_guard<mutex> lock(mtx);
        
        auto& stats = channels[channel];
        stats.spend += spend;
        stats.impressions += impressions;
        stats.clicks += clicks;
        stats.conversions += conversions;
        stats.update_metrics();
        
        remaining_budget -= spend;
        
        // 准备训练数据
        VectorXd old_state = encode_state();
        double reward = calculate_reward();
        
        // 模拟下一个状态 (简化处理)
        VectorXd new_state = encode_state();
        
        // 存储转移
        VectorXd action = VectorXd::Zero(channels.size());
        int i = 0;
        for (const auto& [ch, _] : channels) {
            action[i++] = (ch == channel) ? 1.0 : 0.0; // 简化动作表示
        }
        
        agent.store_transition(old_state, action, reward, new_state);
    }
    
    // 保存模型
    void save_model(const string& filename_prefix) {
        lock_guard<mutex> lock(mtx);
        agent.save(filename_prefix);
    }
    
    // 加载模型
    void load_model(const string& filename_prefix) {
        lock_guard<mutex> lock(mtx);
        agent.load(filename_prefix);
    }
};

// HTTP服务器
class BudgetServer {
private:
    CrossChannelBudgetOptimizer& optimizer;
    asio::io_context io_context;
    tcp::acceptor acceptor;
    atomic<bool> running;
    
    // 处理HTTP请求
    string handle_request(const string& method, const string& target, const string& body) {
        if (method == "GET" && target == "/allocation") {
            // 获取预算分配
            auto allocation = optimizer.get_budget_allocation();
            
            json response;
            for (const auto& [channel, budget] : allocation) {
                string channel_name;
                switch (channel) {
                    case AdChannel::SEARCH: channel_name = "SEARCH"; break;
                    case AdChannel::SOCIAL: channel_name = "SOCIAL"; break;
                    case AdChannel::DISPLAY: channel_name = "DISPLAY"; break;
                    case AdChannel::VIDEO: channel_name = "VIDEO"; break;
                    case AdChannel::NATIVE: channel_name = "NATIVE"; break;
                    case AdChannel::AUDIO: channel_name = "AUDIO"; break;
                }
                response[channel_name] = budget;
            }
            
            return "HTTP/1.1 200 OK\r\nContent-Type: application/json\r\n\r\n" + response.dump();
        }
        else if (method == "POST" && target == "/update") {
            // 更新渠道表现数据
            json data = json::parse(body);
            
            AdChannel channel;
            string channel_str = data["channel"];
            if (channel_str == "SEARCH") channel = AdChannel::SEARCH;
            else if (channel_str == "SOCIAL") channel = AdChannel::SOCIAL;
            else if (channel_str == "DISPLAY") channel = AdChannel::DISPLAY;
            else if (channel_str == "VIDEO") channel = AdChannel::VIDEO;
            else if (channel_str == "NATIVE") channel = AdChannel::NATIVE;
            else if (channel_str == "AUDIO") channel = AdChannel::AUDIO;
            else return "HTTP/1.1 400 Bad Request\r\n\r\n";
            
            optimizer.update_channel_performance(
                channel,
                data["spend"],
                data["impressions"],
                data["clicks"],
                data["conversions"]
            );
            
            return "HTTP/1.1 200 OK\r\n\r\n";
        }
        
        return "HTTP/1.1 404 Not Found\r\n\r\n";
    }
    
    // 处理客户端连接
    void handle_connection(tcp::socket socket) {
        try {
            asio::streambuf buffer;
            asio::read_until(socket, buffer, "\r\n\r\n");
            
            istream is(&buffer);
            string request_line;
            getline(is, request_line);
            
            vector<string> tokens;
            boost::split(tokens, request_line, boost::is_any_of(" "));
            if (tokens.size() < 3) return;
            
            string method = tokens[0];
            string target = tokens[1];
            
            // 读取请求体
            string body;
            while (is) {
                string line;
                getline(is, line);
                if (line == "\r") break;
            }
            
            while (is) {
                string line;
                getline(is, line);
                body += line;
            }
            
            // 处理请求
            string response = handle_request(method, target, body);
            
            // 发送响应
            asio::write(socket, asio::buffer(response));
        } catch (const exception& e) {
            cerr << "处理连接错误: " << e.what() << endl;
        }
    }
    
    // 接受连接
    void accept_connections() {
        while (running) {
            try {
                tcp::socket socket(io_context);
                acceptor.accept(socket);
                
                thread([this, s = move(socket)]() mutable {
                    handle_connection(move(s));
                }).detach();
            } catch (const exception& e) {
                if (running) {
                    cerr << "接受连接错误: " << e.what() << endl;
                }
            }
        }
    }

public:
    BudgetServer(CrossChannelBudgetOptimizer& opt, unsigned short port)
        : optimizer(opt), acceptor(io_context, tcp::endpoint(tcp::v4(), port)), running(false) {}
    
    ~BudgetServer() {
        stop();
    }
    
    // 启动服务器
    void start() {
        if (running) return;
        running = true;
        thread([this]() { accept_connections(); }).detach();
    }
    
    // 停止服务器
    void stop() {
        if (!running) return;
        running = false;
        acceptor.close();
    }
};

// 示例使用
int main() {
    try {
        // 1. 创建预算优化器 (总预算$100,000)
        vector<AdChannel> channels = {
            AdChannel::SEARCH,
            AdChannel::SOCIAL,
            AdChannel::DISPLAY,
            AdChannel::VIDEO,
            AdChannel::NATIVE,
            AdChannel::AUDIO
        };
        
        CrossChannelBudgetOptimizer optimizer(100000.0, channels);
        
        // 2. 加载预训练模型 (如果有)
        try {
            optimizer.load_model("budget_model");
            cout << "加载预训练模型成功" << endl;
        } catch (const exception& e) {
            cout << "未找到预训练模型，将从头开始训练: " << e.what() << endl;
        }
        
        // 3. 启动优化器
        optimizer.start();
        
        // 4. 创建HTTP服务器
        BudgetServer server(optimizer, 8080);
        server.start();
        cout << "预算服务器已启动，监听端口8080" << endl;
        
        // 5. 模拟运行 (在实际应用中应从真实数据源获取)
        cout << "按Enter键停止服务器..." << endl;
        cin.get();
        
        // 6. 停止服务
        server.stop();
        optimizer.stop();
        
        // 7. 保存模型
        optimizer.save_model("budget_model");
        cout << "模型已保存" << endl;
        
    } catch (const exception& e) {
        cerr << "错误: " << e.what() << endl;
        return 1;
    }
    
    return 0;
}

使用说明

功能特点

1. ​​深度强化学习​​：

   • DDPG算法实现

   • 演员-评论家架构

   • 经验回放缓冲区

2. ​​跨渠道优化​​：

   • 动态预算分配

   • 多目标平衡

   • 实时性能适应

3. ​​生产级特性​​：

   • 线程安全设计

   • 模型持久化

   • HTTP API接口

4. ​​复杂状态编码​​：

   • 多维度渠道指标

   • 预算状态跟踪

   • 时间因素考虑

核心组件

1. ​​DDPGAgent​​：

   • 深度确定性策略梯度实现

   • 双网络架构 (演员+评论家)

   • 目标网络稳定训练

2. ​​CrossChannelBudgetOptimizer​​：

   • 预算优化核心

   • 状态编码/解码

   • 奖励函数设计

3. ​​BudgetServer​​：

   • RESTful API服务

   • 实时数据接口

   • 高并发处理

使用方法

1. ​​初始化优化器​​：

   vector<AdChannel> channels = {...};
   CrossChannelBudgetOptimizer optimizer(total_budget, channels);

2. ​​加载模型​​：

   optimizer.load_model("model_prefix");

3. ​​启动服务​​：

   optimizer.start();
   BudgetServer server(optimizer, port);
   server.start();

4. ​​获取分配​​：

   GET /allocation

5. ​​更新数据​​：

   POST /update
   {
     "channel": "SEARCH",
     "spend": 1000,
     "impressions": 50000,
     "clicks": 500,
     "conversions": 10
   }

6. ​​保存模型​​：

   optimizer.save_model("model_prefix");

应用场景

1. ​​程序化广告购买​​：

   • 跨DSP平台优化

   • 实时竞价预算分配

   • 多渠道协同

2. ​​效果营销​​：

   • ROI最大化

   • 转化成本控制

   • 渠道组合优化

3. ​​预算管理​​：

   • 动态预算调整

   • 支出节奏控制

   • 风险分散

技术亮点

1. ​​DDPG算法​​：

   • 连续动作空间

   • 策略梯度优化

   • 适用于复杂决策

2. ​​Eigen库​​：

   • 高性能矩阵运算

   • 神经网络加速

   • 内存效率优化

3. ​​现代C++​​：

   • 多线程支持

   • RAII资源管理

   • 异常安全

4. ​​Boost.Asio​​：

   • 异步网络IO

   • 高并发服务

   • 跨平台支持

扩展建议

1. ​​高级算法​​：

   • 实现TD3或SAC

   • 添加优先级回放

   • 支持多智能体

2. ​​特征工程​​：

   • 加入时序特征

   • 实现注意力机制

   • 支持深度学习

3. ​​监控系统​​：

   • 实时可视化

   • 异常检测

   • 自动告警

4. ​​分布式部署​​：

   • 多节点协同

   • 参数服务器

   • 负载均衡

这个系统为广告运营团队提供了一个智能的跨渠道预算优化解决方案，能够自动学习各渠道表现并动态调整预算分配，最大化广告投放效果。