手搓大模型之only em for pos之增加依赖性

import paddle


class EmAdd(paddle.nn.Layer):
    def __init__(self, voc_size=9999, hidden_size=256):
        super(EmAdd, self).__init__()
        self.hidden_size = hidden_size
        p = 0
        while True:
            voc_size //= hidden_size
            if voc_size == 0:
                break
            else:
                p += 1
        self.em = paddle.nn.LayerList([paddle.nn.Embedding(hidden_size, hidden_size) for _ in range(p + 1)])
        # self.em_zero = paddle.nn.LayerList([paddle.nn.Embedding(hidden_size, hidden_size,padding_idx=0)
        # for _ in range(p + 1)])

    def forward(self, em_add_x):

        add = 0
        p = len(self.em) + 1
        mask = paddle.zeros(em_add_x.shape)
        for i, em in enumerate(self.em):
            # mask 是累加不等于0
            i += 1
            x0 = em_add_x % self.hidden_size ** (p - i) // self.hidden_size ** ((p - i) - 1)