import paddle
class EmAdd(paddle.nn.Layer):
def __init__(self, voc_size=9999, hidden_size=256):
super(EmAdd, self).__init__()
self.hidden_size = hidden_size
p = 0
while True:
voc_size //= hidden_size
if voc_size == 0:
break
else:
p += 1
self.em = paddle.nn.LayerList([paddle.nn.Embedding(hidden_size, hidden_size) for _ in range(p + 1)])
# self.em_zero = paddle.nn.LayerList([paddle.nn.Embedding(hidden_size, hidden_size,padding_idx=0)
# for _ in range(p + 1)])
def forward(self, em_add_x):
add = 0
p = len(self.em) + 1
mask = paddle.zeros(em_add_x.shape)
for i, em in enumerate(self.em):
# mask 是累加不等于0
i += 1
x0 = em_add_x % self.hidden_size ** (p - i) // self.hidden_size ** ((p - i) - 1)
文章来源地址https://www.toymoban.com/news/detail-676077.html
文章来源：https://www.toymoban.com/news/detail-676077.html
到了这里，关于手搓大模型之only em for pos之增加依赖性的文章就介绍完了。如果您还想了解更多内容，请在右上角搜索TOY模板网以前的文章或继续浏览下面的相关文章，希望大家以后多多支持TOY模板网！
