class Value(nn.Module):    def __init__(self, s_dim, hv_dim):        super(Value, self).__init__()
        self.net = nn.Sequential(nn.Linear(s_dim, hv_dim),                                 nn.ReLU(),                                 nn.Linear(hv_dim, hv_dim),                                 nn.ReLU(),                                 nn.Linear(hv_dim, 1))
    def forward(self, s):        """        :param s: [b, s_dim]        :return:  [b, 1]        """        value = self.net(s)
        return value

self.net = nn.Sequential(nn.Linear(s_dim, h_dim),                         nn.ReLU(),                         nn.Linear(h_dim, h_dim),                         nn.ReLU(),                         nn.Linear(h_dim, a_dim))

def select_action(self, s, sample=True):    """    :param s: [s_dim]    :return: [a_dim]    """    # forward to get action probs    # [s_dim] => [a_dim]    a_weights = self.forward(s)    a_probs = torch.sigmoid(a_weights)        # [a_dim] => [a_dim, 2]    a_probs = a_probs.unsqueeze(1)    a_probs = torch.cat([1-a_probs, a_probs], 1)    a_probs = torch.clamp(a_probs, 1e-10, 1 - 1e-10)        # [a_dim, 2] => [a_dim]    a = a_probs.multinomial(1).squeeze(1) if sample else a_probs.argmax(1)        return a

def est_adv(self, r, v, mask):    """    we save a trajectory in continuous space and it reaches the ending of current trajectory when mask=0.    :param r: reward, Tensor, [b]    :param v: estimated value, Tensor, [b]    :param mask: indicates ending for 0 otherwise 1, Tensor, [b]    :return: A(s, a), V-target(s), both Tensor    """    batchsz = v.size(0)
    # v_target is worked out by Bellman equation.    v_target = torch.Tensor(batchsz).to(device=DEVICE)    delta = torch.Tensor(batchsz).to(device=DEVICE)    A_sa = torch.Tensor(batchsz).to(device=DEVICE)
    prev_v_target = 0    prev_v = 0    prev_A_sa = 0    for t in reversed(range(batchsz)):        # mask here indicates a end of trajectory        # this value will be treated as the target value of value network.        # mask = 0 means the immediate reward is the real V(s) since it's end of trajectory.        # formula: V(s_t) = r_t + gamma * V(s_t+1)        v_target[t] = r[t] + self.gamma * prev_v_target * mask[t]
        # please refer to : https://arxiv.org/abs/1506.02438        # for generalized adavantage estimation        # formula: delta(s_t) = r_t + gamma * V(s_t+1) - V(s_t)        delta[t] = r[t] + self.gamma * prev_v * mask[t] - v[t]
        # formula: A(s, a) = delta(s_t) + gamma * lamda * A(s_t+1, a_t+1)        # here use symbol tau as lambda, but original paper uses symbol lambda.        A_sa[t] = delta[t] + self.gamma * self.tau * prev_A_sa * mask[t]
        # update previous        prev_v_target = v_target[t]        prev_v = v[t]        prev_A_sa = A_sa[t]
    # normalize A_sa    A_sa = (A_sa - A_sa.mean()) / A_sa.std()
    return A_sa, v_target