手搓大语言模型 使用jieba分词制作词表，词表大小几十万 加强依赖性

jieba分词词表生成与训练

import numpy as np
import paddle
import pandas as pd
from multiprocessing import Process, Manager, freeze_support
from just_mask_em import JustMaskEm, HeadLoss
from tqdm import tqdm
from glob import glob
import jieba
import warnings

warnings.