eBPF Talk: 手撕 XDP 程序
前段时间，老板让我 手撕 分析 一个陌生的 XDP 程序。
责任重过山头，莫敢推辞，只能硬着头皮上了。
陌生的 XDP 程序
没有源代码，且发现该 XDP 程序还没带上 debug 信息；分析的难度急剧飙升。
不过，根据 XDP 程序开发的套路，还是能够从 bpf 汇编分析一下 XDP 程序大概的处理逻辑。以下为部分 bpf 汇编：
# bpftool p d x i 21712
0: (bf) r3 = r1
1: (b7) r0 = 1
2: (79) r1 = *(u64 *)(r3 +8)
3: (79) r4 = *(u64 *)(r3 +0)
4: (bf) r2 = r4
5: (07) r2 += 14
6: (2d) if r2 > r1 goto pc+144
7: (71) r6 = *(u8 *)(r4 +13)
8: (67) r6 <<= 8
9: (71) r2 = *(u8 *)(r4 +12)
10: (4f) r6 |= r2
11: (15) if r6 == 0xa888 goto pc+3
12: (b7) r2 = 14
13: (b7) r5 = 0
14: (55) if r6 != 0x81 goto pc+7
15: (bf) r2 = r4
16: (07) r2 += 18
17: (b7) r0 = 1
18: (2d) if r2 > r1 goto pc+132
19: (b7) r2 = 18
20: (b7) r5 = 4
21: (69) r6 = *(u16 *)(r4 +16)
22: (bf) r0 = r6
23: (57) r0 &= 65535
24: (15) if r0 == 0x81 goto pc+2
25: (15) if r0 == 0xa888 goto pc+1
26: (05) goto pc+12
27: (bf) r7 = r2
28: (07) r7 += 4
29: (bf) r6 = r4
30: (0f) r6 += r7
31: (b7) r0 = 1
32: (2d) if r6 > r1 goto pc+118
33: (bf) r0 = r4
34: (07) r0 += 2
35: (0f) r0 += r2
36: (07) r5 += 4
37: (69) r6 = *(u16 *)(r0 +0)
38: (bf) r2 = r7
39: (57) r6 &= 65535
40: (b7) r0 = 2
41: (15) if r6 == 0x8 goto pc+1
42: (05) goto pc+108
43: (bf) r7 = r4
44: (0f) r7 += r2
45: (bf) r6 = r7
46: (07) r6 += 20
47: (b7) r0 = 1
48: (2d) if r6 > r1 goto pc+102
49: (71) r2 = *(u8 *)(r7 +9)
50: (b7) r0 = 2
51: (55) if r2 != 0x6 goto pc+99
52: (bf) r2 = r6
53: (07) r2 += 20
54: (b7) r0 = 1
55: (2d) if r2 > r1 goto pc+95
56: (69) r1 = *(u16 *)(r6 +2)
57: (b7) r0 = 2
58: (55) if r1 != 0xfcff goto pc+92
59: (69) r1 = *(u16 *)(r6 +12)
60: (57) r1 &= 512
61: (b7) r0 = 2
62: (15) if r1 == 0x0 goto pc+88
63: ...
131: (b7) r8 = 0
132: (79) r1 = *(u64 *)(r10 -48)
133: (05) goto pc+29
134: (18) r1 = 0xa726f72726520
136: (7b) *(u64 *)(r10 -8) = r1
137: (18) r1 = 0x67616c665f706374
139: (7b) *(u64 *)(r10 -16) = r1
140: (18) r1 = 0x206563616c706572
142: (7b) *(u64 *)(r10 -24) = r1
143: (18) r1 = 0x5f6d7573635f346c
145: (7b) *(u64 *)(r10 -32) = r1
146: (bf) r1 = r10
147: (07) r1 += -32
148: (b7) r2 = 32
149: (85) call bpf_trace_printk#-55408
150: (b7) r0 = 1
151: (95) exit
152: ...
196: (b7) r1 = 0
197: (73) *(u8 *)(r10 -2) = r1
198: (b7) r1 = 2674
199: (6b) *(u16 *)(r10 -4) = r1
200: (b7) r1 = 1869771365
201: (63) *(u32 *)(r10 -8) = r1
202: (18) r1 = 0x207165735f6b6361
204: (7b) *(u64 *)(r10 -16) = r1
205: (18) r1 = 0x206563616c706572
207: (7b) *(u64 *)(r10 -24) = r1
208: (18) r1 = 0x5f6d7573635f346c
210: (7b) *(u64 *)(r10 -32) = r1
211: (bf) r1 = r10
212: (07) r1 += -32
213: (b7) r2 = 31
214: (05) goto pc-66
215: ...
参考
enum xdp_action {
XDP_ABORTED = 0,
XDP_DROP,
XDP_PASS,
XDP_TX,
XDP_REDIRECT,
};
给 bpf 汇编加上一些注释：
# bpftool p d x i 21712
0: (bf) r3 = r1
1: (b7) r0 = 1 // XDP_DROP
2: (79) r1 = *(u64 *)(r3 +8)
3: (79) r4 = *(u64 *)(r3 +0)
4: (bf) r2 = r4
5: (07) r2 += 14
6: (2d) if r2 > r1 goto pc+144
7: (71) r6 = *(u8 *)(r4 +13)
8: (67) r6 <<= 8
9: (71) r2 = *(u8 *)(r4 +12)
10: (4f) r6 |= r2
11: (15) if r6 == 0xa888 goto pc+3 // VLAN 8021AD
12: (b7) r2 = 14
13: (b7) r5 = 0
14: (55) if r6 != 0x81 goto pc+7 // VLAN 8021Q
15: (bf) r2 = r4
16: (07) r2 += 18
17: (b7) r0 = 1 // XDP_DROP
18: (2d) if r2 > r1 goto pc+132
19: (b7) r2 = 18
20: (b7) r5 = 4
21: (69) r6 = *(u16 *)(r4 +16)
22: (bf) r0 = r6
23: (57) r0 &= 65535
24: (15) if r0 == 0x81 goto pc+2 // VLAN 8021Q
25: (15) if r0 == 0xa888 goto pc+1 // VLAN 8021AD
26: (05) goto pc+12
27: (bf) r7 = r2
28: (07) r7 += 4
29: (bf) r6 = r4
30: (0f) r6 += r7
31: (b7) r0 = 1 // XDP_DROP
32: (2d) if r6 > r1 goto pc+118
33: (bf) r0 = r4
34: (07) r0 += 2
35: (0f) r0 += r2
36: (07) r5 += 4
37: (69) r6 = *(u16 *)(r0 +0)
38: (bf) r2 = r7
39: (57) r6 &= 65535
40: (b7) r0 = 2 // XDP_PASS
41: (15) if r6 == 0x8 goto pc+1 // ETH_P_IP
42: (05) goto pc+108
43: (bf) r7 = r4
44: (0f) r7 += r2
45: (bf) r6 = r7
46: (07) r6 += 20 // iphdr
47: (b7) r0 = 1 // XDP_DROP
48: (2d) if r6 > r1 goto pc+102
49: (71) r2 = *(u8 *)(r7 +9)
50: (b7) r0 = 2 // XDP_PASS
51: (55) if r2 != 0x6 goto pc+99 // IPPROTO_TCP
52: (bf) r2 = r6
53: (07) r2 += 20 // tcphdr
54: (b7) r0 = 1 // XDP_DROP
55: (2d) if r2 > r1 goto pc+95
56: (69) r1 = *(u16 *)(r6 +2)
57: (b7) r0 = 2 // XDP_PASS
58: (55) if r1 != 0xfcff goto pc+92 // dport 65532
59: (69) r1 = *(u16 *)(r6 +12)
60: (57) r1 &= 512
61: (b7) r0 = 2 // XDP_PASS
62: (15) if r1 == 0x0 goto pc+88 // tcp flags, not SYN
63: ...
131: (b7) r8 = 0
132: (79) r1 = *(u64 *)(r10 -48)
133: (05) goto pc+29
134: (18) r1 = 0xa726f72726520
136: (7b) *(u64 *)(r10 -8) = r1
137: (18) r1 = 0x67616c665f706374
139: (7b) *(u64 *)(r10 -16) = r1
140: (18) r1 = 0x206563616c706572
142: (7b) *(u64 *)(r10 -24) = r1
143: (18) r1 = 0x5f6d7573635f346c
145: (7b) *(u64 *)(r10 -32) = r1
146: (bf) r1 = r10
147: (07) r1 += -32
148: (b7) r2 = 32
149: (85) call bpf_trace_printk#-55408 // print "l4_csum_replace tcp_flag error\n"
150: (b7) r0 = 1 // XDP_DROP
151: (95) exit
152: ...
196: (b7) r1 = 0
197: (73) *(u8 *)(r10 -2) = r1
198: (b7) r1 = 2674
199: (6b) *(u16 *)(r10 -4) = r1
200: (b7) r1 = 1869771365
201: (63) *(u32 *)(r10 -8) = r1
202: (18) r1 = 0x207165735f6b6361
204: (7b) *(u64 *)(r10 -16) = r1
205: (18) r1 = 0x206563616c706572
207: (7b) *(u64 *)(r10 -24) = r1
208: (18) r1 = 0x5f6d7573635f346c // l4_csum_replace ack_seq erro
210: (7b) *(u64 *)(r10 -32) = r1
211: (bf) r1 = r10
212: (07) r1 += -32
213: (b7) r2 = 31
214: (05) goto pc-66
215: ...
结合两个 bpf_trace_printk() 可知，这个 XDP 程序对目的端口为 65532 的 TCP SYN 包进行了网络包修改。
尽管分析的结果还不够透彻，但已满足老板的预期，就不再继续分析了。
分析过程
XDP 套路一：判断协议
在 XDP 程序里处理网络包，首先需要判断的就是当前网络包是不是预期的那个；如果不是，则 XDP_PASS。
而在，判断的过程中，如果发现是畸形包，则直接 XDP_DROP 。
上面的 XDP 程序就判断了 ETH_P_8021Q、ETH_P_8021AD、ETH_P_IP 和 IPPROTO_TCP 等协议。
比如：
eth = (typeof(eth)) xdp->data;
if (eth->h_proto == ETH_P_8021Q || eth->h_proto == ETH_P_8021AD)
eth = (void *) (eth + 1) + sizeof(struct vlan_hdr);
XDP 套路二：判断范围
在读取网络包内容前，必须先判断读取的内存有没有超出网络包内容的范围了。
上面的 XDP 程序里，如果超出网络包内容的范围，就 XDP_PASS。
比如：
eth = (typeof(eth)) xdp->data;
if ((void *) (eth + 1) > xdp->data_end)
return XDP_PASS;
XDP 套路三：bpf_trace_printk() 的使用
其实，这是大部分 bpf 程序的套路了。
这是因为 bpf_trace_printk() 所使用的字符串都保存到栈上，如：
134: (18) r1 = 0xa726f72726520
136: (7b) *(u64 *)(r10 -8) = r1
137: (18) r1 = 0x67616c665f706374
139: (7b) *(u64 *)(r10 -16) = r1
140: (18) r1 = 0x206563616c706572
142: (7b) *(u64 *)(r10 -24) = r1
143: (18) r1 = 0x5f6d7573635f346c
145: (7b) *(u64 *)(r10 -32) = r1
146: (bf) r1 = r10
147: (07) r1 += -32
148: (b7) r2 = 32
149: (85) call bpf_trace_printk#-55408 // print "l4_csum_replace tcp_flag error\n"
通过 r1 = 0xa726f72726520; *(u64 *)(r10 -8) = r1 的方式，将字符串保存到栈上，然后准备好 bpf_trace_printk() 的参数 R1 和 R2，最后 call bpf_trace_printk#-55408；这里，R1 是指向字符串保存的栈空间的地址的指针，R2 是字符串大小 32 字节。
所以，可以将保存到栈上的字符串给翻译出来：
s := []uint64{0x5f6d7573635f346c, 0x206563616c706572, 0x67616c665f706374, 0xa726f72726520}
b := (*byte)(unsafe.Pointer(&s[0]))
fmt.Println(unsafe.String(b, len(s)*8))
// l4_csum_replace tcp_flag error
这里就是 bpf_printk("l4_csum_replace tcp_flag error\n") 了。
小结
只要掌握了那些 XDP（bpf）程序的开发套路，分析陌生的 XDP 程序就不在话下。