Trinity 代码分析（二）
- 分析Trinity 代码架构
1.整体架构
-
trinity-main执行各种初始化，然后创建执行系统调用的子进程。trinity-main创建的共享内存区域用于记录各种全局信息(打开文件描述符号、执行的系统调用总数以及成功和失败的系统调用数等等)和每个子进程的各种信息(pid和执行的系统调用信息等等)。
-
trinity-watchdog确保系统正常工作。它会检查子进程是否正在运行(可能会在系统调用中被暂停)，如果没有运行，则会将其杀死。当主进程检测到其中一个子进程已终止时(因为trinity-watchdog将其终止或出于其它原因)会启动一个新的子进程来替换它。trinity-watchdog还监视共享内存区域的完整性。
1.1.trinity-main
trinity.c：
99 int main(int argc, char* argv[])
100 {
101 int ret = EXIT_SUCCESS;
102 const char taskname[13]="trinity-main";
103
106 progname = argv[0];
107
108 mainpid = getpid();
109
110 page_size = getpagesize();
111 num_online_cpus = sysconf(_SC_NPROCESSORS_ONLN);
112 max_children = num_online_cpus * 4; /* possibly overridden in params. */
113
>>114 select_syscall_tables();
115
116 create_shm();
117
118 parse_args(argc, argv);
119
120 init_uids();
121
122 change_tmp_dir();
123
124 init_shm();
125
126 init_taint_checking();
127
128 if (munge_tables() == FALSE) {
129 ret = EXIT_FAILURE;
130 goto out;
131 }
132
133 if (show_syscall_list == TRUE) {
134 dump_syscall_tables();
135 goto out;
136 }
137
138 if (show_ioctl_list == TRUE) {
139 dump_ioctls();
140 goto out;
141 }
142
143 if (show_unannotated == TRUE) {
144 show_unannotated_args();
145 goto out;
146 }
147
148 init_syscalls();
149
150 do_uid0_check();
151
152 if (do_specific_domain == TRUE)
153 find_specific_domain(specific_domain_optarg);
154
155 pids_init();
156
157 init_logging();
158
159 init_object_lists(OBJ_GLOBAL);
160
161 setup_initial_mappings();
162
163 parse_devices();
164
165 /* FIXME: Some better object construction method needed. */
166 create_futexes();
167 create_sysv_shms();
170 setup_main_signals();
171
172 no_bind_to_cpu = RAND_BOOL();
173
174 prctl(PR_SET_NAME, (unsigned long) &taskname);
175
176 if (open_fds() == FALSE) {
177 if (shm->exit_reason != STILL_RUNNING)
178 panic(EXIT_FD_INIT_FAILURE); // FIXME: Later, push this down to multiple EXIT's.
179
180 _exit(EXIT_FAILURE);
181 }
182
183 setup_ftrace();
184
185 main_loop();
186
187 destroy_global_objects();
188
189 if (is_tainted() == TRUE)
190 stop_ftrace();
191
192 output(0, "Ran %ld syscalls. Successes: %ld Failures: %ld\n",
193 shm->stats.op_count, shm->stats.successes, shm->stats.failures);
194 if (show_stats == TRUE)
195 dump_stats();
196
197 shutdown_logging();
198
199 ret = set_exit_code(shm->exit_reason);
200 out:
201
202 exit(ret);
203 }
首先设置最大子进程数max_children为CPU核心数的4倍，然后调用select_syscall_tables(); 选择系
统调用表，其中结构体struct syscalltable定义如下：
include/syscall.h：
162 struct syscalltable {
163 struct syscallentry *entry;
164 };
87 struct syscallentry {
88 void (*sanitise)(struct syscallrecord *rec);
89 void (*post)(struct syscallrecord *rec);
90 int (*init)(void);
91 char * (*decode)(struct syscallrecord *rec, unsigned int argnum);
92
93 unsigned int number;
94 unsigned int active_number;
95 const char name[80];
96 const unsigned int num_args;
97 unsigned int flags;
122 ....
123 }
syscalltable结构体中的entry是一个指向syscallentry结构体的指针，所以通过指向syscalltable结构体的指针table通过table[i].entry可以取到所有的syscallentry。在syscallentry结构体中可以看到每个系统调用中包含系统调用名、参数个数、返回值类型和每一个参数的参数名、类型、取值范围等等。
在include目录下arch-xxx.h定义了不同架构的SYSCALLS，mips如下：
include/arch-mips.h:
#define SYSCALLS syscalls_mips
其中有三个文件定义该结构体：
- include/syscalls-mips-64.h
- include/syscalls-mips-n32.h
- include/syscalls-mips-o32.h
include/syscalls-mips-64.h:
4 struct syscalltable syscalls_mips[] = {
>> 5 { .entry = &syscall_read }, /* 5000 */
>> 6 { .entry = &syscall_write },
>> 7 { .entry = &syscall_open },
>> 8 { .entry = &syscall_close },
>> 9 { .entry = &syscall_newstat },
>> 10 { .entry = &syscall_newfstat }, /* 5005 */
>> 11 { .entry = &syscall_newlstat },
>> 12 { .entry = &syscall_poll },
>> 13 { .entry = &syscall_lseek },
>> 14 { .entry = &syscall_ni_syscall },
15 /* { .entry = &syscall_mips_mmap }, */
>> 16 { .entry = &syscall_mprotect }, /* 5010 */
>> 17 { .entry = &syscall_munmap },
>> 18 { .entry = &syscall_brk },
>> 19 { .entry = &syscall_rt_sigaction },
>> 20 { .entry = &syscall_rt_sigprocmask },
>> 21 { .entry = &syscall_ioctl }, /* 5015 */
>> 22 { .entry = &syscall_pread64 },
>> 23 { .entry = &syscall_pwrite64 },
>> 24 { .entry = &syscall_readv },
25 { .entry = &syscall_writev },
26 { .entry = &syscall_access }, /* 5020 */
27 { .entry = &syscall_pipe },
28 { .entry = &syscall_select },
29 { .entry = &syscall_sched_yield },
30 { .entry = &syscall_mremap },
31 { .entry = &syscall_msync }, /* 5025 */
32 { .entry = &syscall_mincore },
33 { .entry = &syscall_madvise },
34 { .entry = &syscall_shmget },
-
create_shm();
接下来创建并初始化共享内存区域。共享内存区域shm_s结构体定义在include\shm.h中，该结构体中的children是一个二维数组，数组中的每一个元素都是指向子进程使用的childdata结构体的指针。
-
parse_args(argc, argv);
-
init_syscalls(); 初始化系统调用的过程中会调用它们的init函数。
175 void init_syscalls(void)
176 {
177 if (biarch == TRUE)
178 init_syscalls_biarch();
179 else
180 init_syscalls_uniarch();
181 }
tables-uniarch.c:
111 void init_syscalls_uniarch(void)
112 {
113 unsigned int i;
114
115 for_each_syscall(i) {
116 struct syscallentry *entry = syscalls[i].entry;
117 if (entry == NULL)
118 continue;
119
120 if (entry->flags & ACTIVE)
121 if (entry->init)
122 entry->init();
123 }
124 }
- main_loop();
708 void main_loop(void)
709 {
710 fork_children();
711
712 while (shm->exit_reason == STILL_RUNNING) {
713
714 handle_children();
720
721 while (check_all_locks() == TRUE) {
722 reap_dead_kids();
723 if (shm->exit_reason == EXIT_REACHED_COUNT)
724 kill_all_kids();
725 }
731
732 check_children_progressing();
733
734 print_stats();
735
739 if (shm->running_childs < max_children)
740 fork_children();
741 }
748
749 handle_children();
750
751 /* Are there still children running ? */
752 while (pidmap_empty() == FALSE) {
753 static unsigned int last = 0;
754
755 if (last != shm->running_childs) {
756 last = shm->running_childs;
760 }
761
762 /* Wait for all the children to exit. */
763 while (shm->running_childs > 0) {
764 taint_check();
765
766 handle_children();
767 kill_all_kids();
768 /* Give children a chance to exit before retrying. */
769 sleep(1);
770 }
771 reap_dead_kids();
772 }
773
779 }
重点分析：
-
fork_children();
子进程是由fork_children函数创建，在fork_children函数中调用了spawn_child函数，spawn_child函数中fork成功以后调用了child_process函数，接着调用random_syscall执行随机系统调用，该函数首先随机选择一个系统调用，接下来生成函数参数。如果函数参数类型为ARG_UNDEFINED则随机生成一个数作为参数，对于其余的参数类型在generic_sanitise函数中调用fill_arg函数生成。在fill_arg函数中根据参数类型argtype调用不同的函数生成对应的参数。
参数类型：
include/syscall.h:
44 enum argtype {
45 ARG_UNDEFINED,
46 ARG_FD,
47 ARG_LEN,
48 ARG_ADDRESS,
49 ARG_MODE_T,
50 ARG_NON_NULL_ADDRESS,
51 ARG_PID,
52 ARG_RANGE,
53 ARG_OP,
54 ARG_LIST,
55 ARG_CPU,
56 ARG_PATHNAME,
57 ARG_IOVEC,
58 ARG_IOVECLEN,
59 ARG_SOCKADDR,
60 ARG_SOCKADDRLEN,
61 ARG_MMAP,
62 ARG_SOCKETINFO,
63 };
参数生成：
generate-args.c:
372 void generic_sanitise(struct syscallrecord *rec)
373 {
374 struct syscallentry *entry;
375 unsigned int call;
376
377 call = rec->nr;
378 entry = syscalls[call].entry;
379
380 if (entry->arg1type != 0)
381 rec->a1 = fill_arg(rec, 1);
382 if (entry->arg2type != 0)
383 rec->a2 = fill_arg(rec, 2);
384 if (entry->arg3type != 0)
385 rec->a3 = fill_arg(rec, 3);
386 if (entry->arg4type != 0)
387 rec->a4 = fill_arg(rec, 4);
388 if (entry->arg5type != 0)
389 rec->a5 = fill_arg(rec, 5);
390 if (entry->arg6type != 0)
391 rec->a6 = fill_arg(rec, 6);
392 }
287 static unsigned long fill_arg(struct syscallrecord *rec, unsigned int argnum)
288 {
289 struct syscallentry *entry;
290 unsigned int call;
291 enum argtype argtype;
292
293 call = rec->nr;
294 entry = syscalls[call].entry;
295
298
299 argtype = get_argtype(entry, argnum);
300
301 switch (argtype) {
302 case ARG_UNDEFINED:
303 if (RAND_BOOL())
304 return (unsigned long) rand64();
305 return (unsigned long) get_writable_address(page_size);
306
307 case ARG_FD:
308 if (RAND_BOOL()) {
309 unsigned int i;
310 /* If this is the 2nd or more ARG_FD, make it unique */
311 for (i = 0; i < argnum; i++) {
312 enum argtype arg;
313 arg = get_argtype(entry, i);
314 if (arg == ARG_FD)
315 return get_new_random_fd();
316 }
317 }
318 return get_random_fd();
319
320 case ARG_LEN:
321 return (unsigned long) get_len();
-
进入while循环，只要子进程还在运行，就一直执行下面这些操作：
首先在handle_children函数中等待子进程停止的信号。如果1秒之后没有接收到则返回。如果接收到则找到相应的子进程然后调用handle_child函数处理。
在handle_child函数中如果子进程是正常终止则记录该子进程已经退出，删除它的所有有关引用，重新创建子进程；如果子进程是异常终止或者暂停则调用相应的处理函数handle_childsig，handle_childsig函数除了做一些记录，其它的处理和handle_child函数大致相同。
refer to
- https://zhuanlan.zhihu.com/p/62512231
refer to
- https://lwn.net/Articles/536173/