c++的CAS与内存屏障: 内联汇编的实际应用场景(S1封装系统调用)
代码可在这里查看: https://github.com/hi-quasars/quark/blob/master/tests/tst-syscall.cc
封装了3个系统调用:write(2),exit(2),gittid(2).-
update notes:
- 20180708 initial version.
- 20180709 refine syscall wrapper.
- 20180709 refine version 2 for generic syscall wrapper.
上一篇主要讨论了内联汇编语法层面的东西，现在接着深入研究下实际应用的场景.
封装系统调用
一些非posix标准的系统调用，可能需要通过glibc提供的syscall函数来调用.(unistd.h下的syscall(2))
在x86架构下，x86_64linux上较新glibc的syscall函数最终调用的指令为syscall汇编指令,而较旧的x86_64架构下的glibc或x86架构可能调用int 0x80 指令.
Linux Assembly X86
-
syscall指令vsint 0x08
看wiki上的解释
syscall
The x86_64 architecture introduced a dedicated instruction to make a syscall. It does not access the interrupt descriptor table and is faster.
- 实验1，x86_64下的write系统调用封装.
这里用c++模板封装了个简单的类型转换层.
大概的层次关系如下:
| 层次 | 代码片段 | 备注 |
|---|---|---|
| api层 | write(int, const char*, int) | 参数有类型，不同系统调用参数个数不同 |
| 胶水层 | DoSysCall... | 类型转换和参数个数匹配,如int类型参数需要转成64位寄存器的合法操作数 |
| 系统调用 | asm(...) | 参数无类型（只有长度），参数个数明确.（1-6） |
- 这个例子简单粗暴地进行了复制和十分冗长的类型转换.
这个例子接受参数个数为3的系统调用(不算系统调用号)
1
2 /*
3 *
4 *
5 * from the glibc wiki page https://en.wikibooks.org/wiki/X86_Assembly/Interfacing_with_Linux
6 - int 0x80
7 On both Linux x86 and Linux x86_64 systems you can make a syscall by calling interrupt 0x80 using the int $0x80 command. Parameters are passed by setting the general purpose registers as following:
8
9 Param 1 Param 2 Param 3 Param 4 Param 5 Param 6
10 eax ebx ecx edx esi edi ebp
11 Return value
12 eax
13 The syscall numbers are described in the Linux generated file $build/arch/x86/include/generated/uapi/asm/unistd_32.h or $build/usr/include/asm/unistd_32.h. The latter could also be present on your Linux system, just omit the $build.
14 All registers are preserved during the syscall.
15
16 - syscall
17 The x86_64 architecture introduced a dedicated instruction to make a syscall. It does not access the interrupt descriptor table and is faster. Parameters are passed by setting the general purpose registers as following:
18
19 Param 1 Param 2 Param 3 Param 4 Param 5 Param 6
20 rax rdi rsi rdx r10 r8 r9
21 Return value
22 rax
23 The syscall numbers are described in the Linux generated file $build/usr/include/asm/unistd_64.h. This file could also be present on your Linux system, just omit the $build.
24 All registers, except rcx and r11 (and the return value, rax), are preserved during the syscall.
25 * */
26
27 /// for x86 & x86_64
28 #include
29 #include
30 #include
31 #include
32
33 template
34 struct byte { /// this type is a pod type.
35 char ctn[len];
36 };
37
38
39 template
40 struct byte_t {
41 byte itn;
42 };
43
44 template
45 uint64_t* FetchCtnPtr(T* x) {
46 return reinterpret_cast(x->itn.ctn);
47 }
48
49 template
50 int DoSysCall0(uint64_t SysCallNR, T1 a1, T2 a2, T3 a3)
51 {
52 uint64_t ret;
53 byte_t x1;
54 byte_t x2;
55 byte_t x3;
56 const char * ptr;
57 memcpy(static_cast(x1.itn.ctn), static_cast(&a1), sizeof(a1));
58 memcpy(static_cast(x2.itn.ctn), static_cast(&a2), sizeof(a2));
59 memcpy(static_cast(x3.itn.ctn), static_cast(&a3), sizeof(a3));
60
61 printf("%d %s %d\n", *(reinterpret_cast(x1.itn.ctn)),
62 (ptr = reinterpret_cast(*(FetchCtnPtr(&x2)))) == NULL ? "NULL" : ptr,
63 *(reinterpret_cast(x3.itn.ctn)));
64
65 asm(
66 "movq %1, %%rax\n\t" /// SysCall Number
67 "movq %2, %%rdi\n\t" /// Para 1
68 "movq %3, %%rsi\n\t" /// Para 2
69 "movq %4, %%rdx\n\t" /// Para 3
70 "syscall\n\t" /// Call it
71 "movq %%rax, %0\n\t" /// Fetch Return Value
72 :"=r"(ret)
73 :"r"(SysCallNR),"r"(*(FetchCtnPtr(&x1))), "r"(*(FetchCtnPtr(&x2))), "r"(*(FetchCtnPtr(&x3)))
74 :"%rax","%rdi", "%rsi", "%rdx"
75 );
76 return (int)ret;
77 }
78
79
80 int Write(int fd, const char* str, int num) {
81 return DoSysCall0(SYS_write, fd, str, num);
82 }
83
84 int main() {
85 int ret;
86 const char * str1 = "Hello Syscall\n";
87 ret = Write(1, str1, strlen(str1));
88 std::cout << "Ret: " << ret << std::endl;
89 return 0;
90 }
运行结果如下：
result
从这个例子可看出，glibc在包裹函数做的的事情应该是包括：
- 入参/出参类型转换.(系统调用在上层的参数类型可能不同)
- 填充寄存器.(架构相关,填充哪些寄存器)
- 调用中断指令
- 最后，总结一下这个部分
glibc的syscall函数是根据不同架构、不同版本内核由脚本生成来的，扩展性比自己写syscall_wrapper好得多.
建议还是使用syscall函数而非自己封装一个，毕竟这些脏活累活吃力不讨好.
需要自己封装时，一定是非常少见的情况了(估计这辈子都碰不到)，这有对添加system call的glibc现状做一些讨论.lwn - 这里对类型封装部分进行一些重构, 采用type_traits和enable_if等trick对类型转换部分优化了一下.
Here is an improve version of syscall wrapper
Compile: g++ tst-syscall.cc -o test -std=c++11 or g++ tst-syscall.cc -o test
1
2 /*
3 *
4 *
5 * from the glibc wiki page https://en.wikibooks.org/wiki/X86_Assembly/Interfacing_with_Linux
6 - int 0x80
7 On both Linux x86 and Linux x86_64 systems you can make a syscall by calling interrupt 0x80 using the int $0x80 command. Parameters are passed by setting the general purpose registers as following:
8
9 Param 1 Param 2 Param 3 Param 4 Param 5 Param 6
10 eax ebx ecx edx esi edi ebp
11 Return value
12 eax
13 The syscall numbers are described in the Linux generated file $build/arch/x86/include/generated/uapi/asm/unistd_32.h or $build/usr/include/asm/unistd_32.h. The latter could also be present on your Linux system, just omit the $build.
14 All registers are preserved during the syscall.
15
16 - syscall
17 The x86_64 architecture introduced a dedicated instruction to make a syscall. It does not access the interrupt descriptor table and is faster. Parameters are passed by setting the general purpose registers as following:
18
19 Param 1 Param 2 Param 3 Param 4 Param 5 Param 6
20 rax rdi rsi rdx r10 r8 r9
21 Return value
22 rax
23 The syscall numbers are described in the Linux generated file $build/usr/include/asm/unistd_64.h. This file could also be present on your Linux system, just omit the $build.
24 All registers, except rcx and r11 (and the return value, rax), are preserved during the syscall.
25 * */
26
27 /// for x86 & x86_64
28 #include
29 #include
30 #include
31 #include
32
33
34
35 struct b4 {
36 char buf[4];
37 };
38 struct b8 {
39 char buf[8];
40 };
41
42 #if __cplusplus >= 201103L
43
44 #include
45 typedef std::true_type true_type;
46 typedef std::false_type false_type;
47
48 #else
49 template
50 struct integer_constant {
51 typedef T value_type;
52 enum { value = Val };
53 };
54 typedef integer_constant true_type;
55 typedef integer_constant false_type;
56 #endif
57
58 template
59 struct is_b4 {
60 typedef false_type value_type;
61 const bool value = false;
62 };
63
64 template <>
65 struct is_b4 {
66 typedef true_type value_type;
67 const bool value = true;
68 };
69
70 template
71 struct is_b8 {
72 typedef false_type value_type;
73 static const bool value = false;
74 };
75
76 template <>
77 struct is_b8 {
78 typedef true_type value_type;
79 static const bool value = true;
80 };
81
82 template
83 struct enable_if_my {
84 };
85 template
86 struct enable_if_my {
87 typedef T type;
88 };
89
90
91 template
92 void makeb48(b4* b, T* x) {
93 memcpy(static_cast(b->buf), static_cast(x), sizeof(T));
94 }
95 template
96 void makeb48(b8* b, T* x) {
97 memcpy(static_cast(b->buf), static_cast(x), sizeof(T));
98 }
99 template::value, T>::type* = nullptr>
100 uint32_t* CtnPtr(T *b){
101 return reinterpret_cast(b->buf);
102 }
103 template::value, T>::type* = nullptr> /// should be ::type* , or compiler cannot infer the tempalte argument.
104 uint64_t* CtnPtr(T *b){
105 return reinterpret_cast(b->buf);
106 }
107
108 template
109 int DoSysCall0(uint64_t SysCallNR, T1 a1, T2 a2, T3 a3)
110 {
111 uint64_t ret;
112 b8 x1;
113 b8 x2;
114 b8 x3;
115 const char * ptr;
116 makeb48(&x1, &a1);
117 makeb48(&x2, &a2);
118 makeb48(&x3, &a3);
119
120 printf("%d %s %d\n", *(reinterpret_cast(CtnPtr(&x1))),
121 (ptr = reinterpret_cast(*(CtnPtr(&x2)))) == NULL ? "NULL" : ptr,
122 *(reinterpret_cast(CtnPtr(&x3))));
123
124 asm(
125 "movq %1, %%rax\n\t" /// SysCall Number
126 "movq %2, %%rdi\n\t" /// Para 1
127 "movq %3, %%rsi\n\t" /// Para 2
128 "movq %4, %%rdx\n\t" /// Para 3
129 "syscall\n\t" /// Call it
130 "movq %%rax, %0\n\t" /// Fetch Return Value
131 :"=r"(ret)
132 :"r"(SysCallNR),"r"(*(CtnPtr(&x1))), "r"(*(CtnPtr(&x2))), "r"(*(CtnPtr(&x3)))
133 :"%rax","%rdi", "%rsi", "%rdx"
134 );
135 return (int)ret;
136 }
137
138
139 int Write(int fd, const char* str, int num) {
140 return DoSysCall0(SYS_write, fd, str, num);
141 }
142
143 int main() {
144 int ret;
145 const char * str1 = "Hello Syscall\n";
146 ret = Write(1, str1, strlen(str1));
147 std::cout << "Ret: " << ret << std::endl;
148 return 0;
149 }
improve version of syscall wrapper
- 重构第二个版本，添加系统调用的通用化支持
1
2 /*
3 *
4 *
5 * from the glibc wiki page https://en.wikibooks.org/wiki/X86_Assembly/Interfacing_with_Linux
6 - int 0x80
7 On both Linux x86 and Linux x86_64 systems you can make a syscall by calling interrupt 0x80 using the int $0x80 command. Parameters are passed by setting the general purpose registers as following:
8
9 Param 1 Param 2 Param 3 Param 4 Param 5 Param 6
10 eax ebx ecx edx esi edi ebp
11 Return value
12 eax
13 The syscall numbers are described in the Linux generated file $build/arch/x86/include/generated/uapi/asm/unistd_32.h or $build/usr/include/asm/unistd_32.h. The latter could also be present on your Linux system, just omit the $build.
14 All registers are preserved during the syscall.
15
16 - syscall
17 The x86_64 architecture introduced a dedicated instruction to make a syscall. It does not access the interrupt descriptor table and is faster. Parameters are passed by setting the general purpose registers as following:
18
19 Param 1 Param 2 Param 3 Param 4 Param 5 Param 6
20 rax rdi rsi rdx r10 r8 r9
21 Return value
22 rax
23 The syscall numbers are described in the Linux generated file $build/usr/include/asm/unistd_64.h. This file could also be present on your Linux system, just omit the $build.
24 All registers, except rcx and r11 (and the return value, rax), are preserved during the syscall.
25 * */
26
27 /// for x86 & x86_64
28 #include
29 #include
30 #include
31 #include
32
33
34
35 struct b4 {
36 char buf[4];
37 };
38 struct b8 {
39 char buf[8];
40 };
41
42 #if __cplusplus >= 201103L
43
44 #include
45 typedef std::true_type true_type;
46 typedef std::false_type false_type;
47
48 #else
49 template
50 struct integer_constant {
51 typedef T value_type;
52 enum { value = Val };
53 };
54 typedef integer_constant true_type;
55 typedef integer_constant false_type;
56 #endif
57
58 template
59 struct is_b4 {
60 typedef false_type value_type;
61 const bool value = false;
62 };
63
64 template <>
65 struct is_b4 {
66 typedef true_type value_type;
67 const bool value = true;
68 };
69
70 template
71 struct is_b8 {
72 typedef false_type value_type;
73 static const bool value = false;
74 };
75
76 template <>
77 struct is_b8 {
78 typedef true_type value_type;
79 static const bool value = true;
80 };
81
82 template
83 struct enable_if_my {
84 };
85 template
86 struct enable_if_my {
87 typedef T type;
88 };
89
90
91 template
92 void makeb48(b4* b, T* x) {
93 memcpy(static_cast(b->buf), static_cast(x), sizeof(T));
94 }
95 template
96 void makeb48(b8* b, T* x) {
97 memcpy(static_cast(b->buf), static_cast(x), sizeof(T));
98 }
99 template::value, T>::type* = nullptr>
100 uint32_t* CtnPtr(T *b){
101 return reinterpret_cast(b->buf);
102 }
103 template::value, T>::type* = nullptr> /// should be ::type* , or compiler cannot infer the tempalte argument.
104 uint64_t* CtnPtr(T *b){
105 return reinterpret_cast(b->buf);
106 }
107
108
109 #define SysCallCmd0 "movq %1, %%rax\n\t"
110 #define SysCallCmd1 SysCallCmd0 "movq %2, %%rdi\n\t"
111 #define SysCallCmd2 SysCallCmd1 "movq %3, %%rsi\n\t"
112 #define SysCallCmd3 SysCallCmd2 "movq %4, %%rdx\n\t"
113 #define SysCallCmd4 SysCallCmd3 "movq %5, %%r10\n\t"
114 #define SysCallCmd5 SysCallCmd4 "movq %6, %%r8\n\t"
115 #define SysCallCmd6 SysCallCmd5 "movq %7, %%r9\n\t"
116
117 #define SysCallRet "movq %%rax,%0\n\t"
118
119 #define OutputReg(x) "=r"(x)
120 #define InputReg(x) "r"(x)
121 #define SysCallInRegList0(nr,ofs) InputReg(nr)
122 #define SysCallInRegList1(nr,a1) InputReg(nr),InputReg(a1)
123 #define SysCallInRegList2(nr,a1,a2) SysCallInRegList1(nr,a1),InputReg(a2)
124 #define SysCallInRegList3(nr,a1,a2,a3) SysCallInRegList2(nr,a1,a2),InputReg(a3)
125 #define SysCallInRegList4(nr,a1,a2,a3,a4) SysCallInRegList3(nr,a1,a2,a3),InputReg(a4)
126 #define SysCallInRegList5(nr,a1,a2,a3,a4,a5) SysCallInRegList4(nr,a1,a2,a3,a4),InputReg(a5)
127 #define SysCallInRegList6(nr,a1,a2,a3,a4,a5,a6) SysCallInRegList5(nr,a1,a2,a3,a4,a5),InputReg(a6)
128
129 #define SysCallCrobList0 "%rax"
130 #define SysCallCrobList1 SysCallCrobList0,"%rdi"
131 #define SysCallCrobList2 SysCallCrobList1,"%rsi"
132 #define SysCallCrobList3 SysCallCrobList2,"%rdx"
133 #define SysCallCrobList4 SysCallCrobList3,"%r10"
134 #define SysCallCrobList5 SysCallCrobList4,"%r8"
135 #define SysCallCrobList6 SysCallCrobList5,"%r9"
136
137
138 #define SysCallTemplateX8664(ret, NR, i, ...) do { asm( \
139 SysCallCmd##i \
140 "syscall\n\t" \
141 SysCallRet \
142 :OutputReg(ret) \
143 :SysCallInRegList##i(NR, __VA_ARGS__) \
144 :SysCallCrobList##i \
145 ); } while(0)
146
147 #define X86_64DefSysCall0(ret, NR) SysCallTemplateX8664(ret, NR, 0)
148 #define X86_64DefSysCall1(ret, NR, a1) SysCallTemplateX8664(ret, NR, 1, a1)
149 #define X86_64DefSysCall2(ret, NR, a1, a2) SysCallTemplateX8664(ret, NR, 2, a1, a2)
150
151 #define X86_64DefSysCall3(ret, NR, a1, a2, a3) \
152 SysCallTemplateX8664(ret, NR, 3, a1, a2, a3)
153 #define X86_64DefSysCall4(ret, NR, a1, a2, a3, a4) \
154 SysCallTemplateX8664(ret, NR, 4, a1, a2, a3, a4)
155 #define X86_64DefSysCall5(ret, NR, a1, a2, a3, a4, a5) \
156 SysCallTemplateX8664(ret, NR, 5, a1, a2, a3, a4, a5)
157 #define X86_64DefSysCall6(ret, NR, a1, a2, a3, a4, a5, a6) \
158 SysCallTemplateX8664(ret, NR, 6, a1, a2, a3, a4, a5, a6)
159
160
161 int DoSysCall(uint64_t SysCallNR)
162 {
163 uint64_t ret;
164 X86_64DefSysCall0(ret, SysCallNR);
165 return (int)ret;
166 }
167
168 template
169 int DoSysCall(uint64_t SysCallNR, T1 a1)
170 {
171 uint64_t ret;
172 b8 x1;
173 makeb48(&x1, &a1);
174 X86_64DefSysCall1(ret, SysCallNR, (*CtnPtr(&x1)));
175 return (int)ret;
176 }
177
178 template
179 int DoSysCall(uint64_t SysCallNR, T1 a1, T2 a2)
180 {
181 uint64_t ret;
182 b8 x1, x2;
183 makeb48(&x1, &a1);
184 makeb48(&x2, &a2);
185 X86_64DefSysCall2(ret, SysCallNR, (*CtnPtr(&x1)), (*CtnPtr(&x2)));
186 return (int)ret;
187 }
188
189 template
190 int DoSysCall(uint64_t SysCallNR, T1 a1, T2 a2, T3 a3)
191 {
192 uint64_t ret;
193 b8 x1, x2, x3;
194 makeb48(&x1, &a1);
195 makeb48(&x2, &a2);
196 makeb48(&x3, &a3);
197 X86_64DefSysCall3(ret, SysCallNR, (*CtnPtr(&x1)), (*CtnPtr(&x2)), (*CtnPtr(&x3)));
198 return (int)ret;
199 }
200
201 template
202 int DoSysCall(uint64_t SysCallNR, T1 a1, T2 a2, T3 a3, T4 a4)
203 {
204 uint64_t ret;
205 b8 x1, x2, x3, x4;
206 makeb48(&x1, &a1);
207 makeb48(&x2, &a2);
208 makeb48(&x3, &a3);
209 makeb48(&x4, &a4);
210 X86_64DefSysCall4(ret, SysCallNR, (*CtnPtr(&x1)), (*CtnPtr(&x2)), (*CtnPtr(&x3)), (*CtnPtr(&x4)));
211 return (int)ret;
212 }
213
214 template
215 int DoSysCall(uint64_t SysCallNR, T1 a1, T2 a2, T3 a3, T4 a4, T5 a5)
216 {
217 uint64_t ret;
218 b8 x1, x2, x3, x4, x5;
219 makeb48(&x1, &a1);
220 makeb48(&x2, &a2);
221 makeb48(&x3, &a3);
222 makeb48(&x4, &a4);
223 makeb48(&x5, &a5);
224 X86_64DefSysCall5(ret, SysCallNR, (*CtnPtr(&x1)), (*CtnPtr(&x2)), (*CtnPtr(&x3)), (*CtnPtr(&x4)), (*CtnPtr(&x5)));
225 return (int)ret;
226 }
227
228 template
229 int DoSysCall(uint64_t SysCallNR, T1 a1, T2 a2, T3 a3, T4 a4, T5 a5, T6 a6)
230 {
231 uint64_t ret;
232 b8 x1, x2, x3, x4, x5, x6;
233 makeb48(&x1, &a1);
234 makeb48(&x2, &a2);
235 makeb48(&x3, &a3);
236 makeb48(&x4, &a4);
237 makeb48(&x5, &a5);
238 makeb48(&x6, &a6);
239 X86_64DefSysCall6(ret, SysCallNR, (*CtnPtr(&x1)), (*CtnPtr(&x2)), (*CtnPtr(&x3)), (*CtnPtr(&x4)), (*CtnPtr(&x5)), (*CtnPtr(&x6)));
240 return (int)ret;
241 }
242
243 template
244 int DoSysCall0(uint64_t SysCallNR, T1 a1, T2 a2, T3 a3)
245 {
246 uint64_t ret;
247 b8 x1;
248 b8 x2;
249 b8 x3;
250 const char * ptr;
251 makeb48(&x1, &a1);
252 makeb48(&x2, &a2);
253 makeb48(&x3, &a3);
254
255 printf("%d %s %d\n", *(reinterpret_cast(CtnPtr(&x1))),
256 (ptr = reinterpret_cast(*(CtnPtr(&x2)))) == NULL ? "NULL" : ptr,
257 *(reinterpret_cast(CtnPtr(&x3))));
258
259 asm(
260 "movq %1, %%rax\n\t" /// SysCall Number
261 "movq %2, %%rdi\n\t" /// Para 1
262 "movq %3, %%rsi\n\t" /// Para 2
263 "movq %4, %%rdx\n\t" /// Para 3
264 "syscall\n\t" /// Call it
265 "movq %%rax, %0\n\t" /// Fetch Return Value
266 :"=r"(ret)
267 :"r"(SysCallNR),"r"(*(CtnPtr(&x1))), "r"(*(CtnPtr(&x2))), "r"(*(CtnPtr(&x3)))
268 :"%rax","%rdi", "%rsi", "%rdx"
269 );
270 return (int)ret;
271 }
272
273
274 int Write(int fd, const char* str, int num) {
275 return DoSysCall(SYS_write, fd, str, num);
276 }
277
278 int Exit(int code) {
279 return DoSysCall(SYS_exit, code);
280 }
281
282 int Gettid() {
283 return DoSysCall(SYS_gettid);
284 }
285
286 int main() {
287 int ret;
288 const char * str1 = "Hello Syscall\n";
289 ret = Write(1, str1, strlen(str1));
290 std::cout << "Ret: " << ret << std::endl;
291
292 std::cout << "Tid: " << Gettid() << std::endl;
293 return 0;
294 }
添加tid
内存屏障和CAS放入下一篇.
内存屏障
tbd...
CAS
tbd...