forked from TomHeaven/nudt_thesis
-
Notifications
You must be signed in to change notification settings - Fork 0
Expand file tree
/
Copy paththesis.toc
More file actions
69 lines (69 loc) · 6.02 KB
/
Copy paththesis.toc
File metadata and controls
69 lines (69 loc) · 6.02 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
\contentsline {chapter}{摘\hspace {1em}要}{i}{section*.4}%
\contentsline {chapter}{\textsf {Abstract}}{iii}{section*.5}%
\contentsline {chapter}{本文中英文对照表}{v}{chapter*.6}%
\contentsline {chapter}{\numberline {1}绪论}{1}{chapter.1}%
\contentsline {section}{\numberline {1.1}研究背景与意义}{1}{section.1.1}%
\contentsline {subsection}{\numberline {1.1.1}多模态大语言模型的训练}{1}{subsection.1.1.1}%
\contentsline {subsection}{\numberline {1.1.2}异构 GPU 集群下的并行训练}{4}{subsection.1.1.2}%
\contentsline {section}{\numberline {1.2}国内外研究现状与挑战}{6}{section.1.2}%
\contentsline {subsection}{\numberline {1.2.1}多模态大语言模型并行训练策略研究现状}{6}{subsection.1.2.1}%
\contentsline {subsection}{\numberline {1.2.2}异构 GPU 集群并行训练策略研究现状}{8}{subsection.1.2.2}%
\contentsline {subsection}{\numberline {1.2.3}研究问题与挑战}{9}{subsection.1.2.3}%
\contentsline {section}{\numberline {1.3}研究内容与贡献}{10}{section.1.3}%
\contentsline {subsection}{\numberline {1.3.1}MMoH：多模态大语言模型在异构 GPU 集群上的混合并行方法}{10}{subsection.1.3.1}%
\contentsline {subsection}{\numberline {1.3.2}HR-MMoH：多模态大语言模型在异构 GPU 集群上的重计算方法}{11}{subsection.1.3.2}%
\contentsline {section}{\numberline {1.4}论文组织结构}{11}{section.1.4}%
\contentsline {chapter}{\numberline {2}相关工作}{13}{chapter.2}%
\contentsline {section}{\numberline {2.1}多模态大语言模型及并行训练技术}{13}{section.2.1}%
\contentsline {subsection}{\numberline {2.1.1}多模态大语言模型的模型架构与训练数据}{13}{subsection.2.1.1}%
\contentsline {subsection}{\numberline {2.1.2}多模态大语言模型的训练过程}{15}{subsection.2.1.2}%
\contentsline {subsection}{\numberline {2.1.3}多维混合并行训练方法}{16}{subsection.2.1.3}%
\contentsline {subsection}{\numberline {2.1.4}零冗余优化器（ZeRO）}{20}{subsection.2.1.4}%
\contentsline {section}{\numberline {2.2}异构 GPU 集群并行训练}{21}{section.2.2}%
\contentsline {subsection}{\numberline {2.2.1}异构 GPU 集群并行训练策略}{21}{subsection.2.2.1}%
\contentsline {subsection}{\numberline {2.2.2}异构 GPU 集群并行训练的工业实践}{22}{subsection.2.2.2}%
\contentsline {section}{\numberline {2.3}重计算技术}{23}{section.2.3}%
\contentsline {subsection}{\numberline {2.3.1}重计算基本原理与分类}{23}{subsection.2.3.1}%
\contentsline {subsection}{\numberline {2.3.2}重计算的主流实现及其局限性}{24}{subsection.2.3.2}%
\contentsline {section}{\numberline {2.4}本章小结}{24}{section.2.4}%
\contentsline {chapter}{\numberline {3}多模态大语言模型在异构 GPU 集群上的高效混合并行技术}{27}{chapter.3}%
\contentsline {section}{\numberline {3.1}引言}{27}{section.3.1}%
\contentsline {subsection}{\numberline {3.1.1}问题与动机}{27}{subsection.3.1.1}%
\contentsline {subsection}{\numberline {3.1.2}MMoH 框架概览}{30}{subsection.3.1.2}%
\contentsline {section}{\numberline {3.2}MMoH 并行策略规划器}{32}{section.3.2}%
\contentsline {subsection}{\numberline {3.2.1}需求驱动的设备拓扑识别}{33}{subsection.3.2.1}%
\contentsline {subsection}{\numberline {3.2.2}复合粒度模型抽象}{35}{subsection.3.2.2}%
\contentsline {subsection}{\numberline {3.2.3}冻结感知的模型划分}{36}{subsection.3.2.3}%
\contentsline {section}{\numberline {3.3}提前终止流水线调度策略}{38}{section.3.3}%
\contentsline {subsection}{\numberline {3.3.1}冗余识别与剪枝规则}{39}{subsection.3.3.1}%
\contentsline {subsection}{\numberline {3.3.2}调度流程与实现细节}{40}{subsection.3.3.2}%
\contentsline {subsection}{\numberline {3.3.3}对成本模型与划分策略的影响}{40}{subsection.3.3.3}%
\contentsline {subsection}{\numberline {3.3.4}实现要点}{42}{subsection.3.3.4}%
\contentsline {section}{\numberline {3.4}实验验证}{43}{section.3.4}%
\contentsline {subsection}{\numberline {3.4.1}实验设置}{43}{subsection.3.4.1}%
\contentsline {subsection}{\numberline {3.4.2}实验结果}{44}{subsection.3.4.2}%
\contentsline {subsection}{\numberline {3.4.3}实验分析}{46}{subsection.3.4.3}%
\contentsline {section}{\numberline {3.5}本章小结}{48}{section.3.5}%
\contentsline {chapter}{\numberline {4}多模态大语言模型在异构 GPU 集群上的重计算技术}{51}{chapter.4}%
\contentsline {section}{\numberline {4.1}引言}{51}{section.4.1}%
\contentsline {subsection}{\numberline {4.1.1}问题与动机}{51}{subsection.4.1.1}%
\contentsline {subsection}{\numberline {4.1.2}HR-MMoH 框架概览}{52}{subsection.4.1.2}%
\contentsline {section}{\numberline {4.2}异构 GPU 重计算技术}{52}{section.4.2}%
\contentsline {subsection}{\numberline {4.2.1}现有的重计算策略与定量分析}{52}{subsection.4.2.1}%
\contentsline {subsection}{\numberline {4.2.2}异构 GPU 重计算策略设计}{59}{subsection.4.2.2}%
\contentsline {section}{\numberline {4.3}实验验证}{63}{section.4.3}%
\contentsline {subsection}{\numberline {4.3.1}实验设置}{63}{subsection.4.3.1}%
\contentsline {subsection}{\numberline {4.3.2}实验结果}{64}{subsection.4.3.2}%
\contentsline {subsection}{\numberline {4.3.3}实验分析}{68}{subsection.4.3.3}%
\contentsline {section}{\numberline {4.4}本章小结}{68}{section.4.4}%
\contentsline {chapter}{\numberline {5}总结与展望}{71}{chapter.5}%
\contentsline {section}{\numberline {5.1}全文工作总结}{71}{section.5.1}%
\contentsline {subsection}{\numberline {5.1.1}主要创新点}{71}{subsection.5.1.1}%
\contentsline {section}{\numberline {5.2}局限性及未来工作展望}{72}{section.5.2}%
\contentsline {subsection}{\numberline {5.2.1}研究局限性}{72}{subsection.5.2.1}%
\contentsline {subsection}{\numberline {5.2.2}未来研究工作}{73}{subsection.5.2.2}%
\contentsline {chapter}{致谢}{75}{chapter*.47}%
\contentsline {chapter}{参考文献}{77}{section*.48}%
\contentsline {chapter}{作者在学期间取得的学术成果}{87}{chapter*.50}%
\contentsline {chapter}{公开评阅信息}{89}{chapter*.51}%
\contentsfinish