谢太数据库: monitoring network activity of processes 查看进程网络活动

monitoring network activity of processes
监控进程网络活动

1. 用strace跟踪进程发出的网络相关的系统调用

strace -t -e trace=network -e signal=\!alrm -p $(pgrep -u s3op2 emulator)

[root@DEV-Blur-DB-1 ~]# strace -t -e trace=network -e signal=\!alrm -p $(pgrep -u s3op2 emulator)
Process 9744 attached - interrupt to quit
[ Process PID=9744 runs in 32 bit mode. ]
08:00:38 recv(11, "OPEN\36\2\0\0\0\0\0\0007\0\0\0\261\21\0\0\260\257\272\261shell:ex"..., 8729, 0) = 79
08:00:38 send(11, "OKAY\1\0\0\0\36\2\0\0\0\0\0\0\0\0\0\0\260\264\276\246", 24, 0) = 24
...
08:01:30 recv(11, "OKAY\36\2\0\0\1\0\0\0\0\0\0\0\0\0\0\0\260\264\276\246", 4162, 0) = 24
08:01:30 socket(PF_INET, SOCK_STREAM, IPPROTO_IP) = 12
08:01:30 setsockopt(12, SOL_SOCKET, SO_REUSEADDR, [1], 4) = 0
08:01:30 setsockopt(12, SOL_SOCKET, SO_OOBINLINE, [1], 4) = 0
08:01:30 connect(12, {sa_family=AF_INET, sin_port=htons(5222), sin_addr=inet_addr("124.65.150.30")}, 16) = -1 EINPROGRESS (Operation now in progress)
08:01:30 send(12, "", 0, 0)             = 0
08:01:31 send(11, "WRTE\1\0\0\0\36\2\0\0\246\0\0\0\0379\0\0\250\255\253\27203-26 08"..., 190, 0) = 190
08:01:31 recv(11, "OKAY\36\2\0\0\1\0\0\0\0\0\0\0\0\0\0\0\260\264\276\246", 4138, 0) = 24
08:01:31 send(12, "<stream:stream token=\"fake-token"..., 207, 0) = 207
08:01:31 recv(12, "<stream:stream xmlns=\'jabber:cli"..., 8760, 0) = 182
08:01:31 send(11, "WRTE\1\0\0\0\36\2\0\0c\0\0\0N\37\0\0\250\255\253\27203-26 08"..., 123, 0) = 123
08:01:31 recv(11, "OKAY\36\2\0\0\1\0\0\0\0\0\0\0\0\0\0\0\260\264\276\246", 4114, 0) = 24
08:01:31 send(11, "WRTE\1\0\0\0\36\2\0\0\205\0\0\0\21*\0\0\250\255\253\27203-26 08"..., 157, 0) = 157
08:01:31 recv(11, "OKAY\36\2\0\0\1\0\0\0\0\0\0\0\0\0\0\0\260\264\276\246", 4090, 0) = 24
08:01:31 recv(12, "<stream:features><starttls xmlns"..., 8578, 0) = 136
...
08:03:45 recv(11, "OKAY\36\2\0\0\1\0\0\0\0\0\0\0\0\0\0\0\260\264\276\246", 2362, 0) = 24
08:03:45 send(11, "WRTE\1\0\0\0\36\2\0\0t\0\0\0B%\0\0\250\255\253\27203-26 08"..., 140, 0) = 140
08:03:45 recv(11, "OKAY\36\2\0\0\1\0\0\0\0\0\0\0\0\0\0\0\260\264\276\246", 2338, 0) = 24
Process 9744 detached

或用aix,solaris下的truss

2. tcpdump跟踪网络活动

tcpdump -ibond0 host 124.65.150.30

[root@DEV-Blur-DB-1 ~]# tcpdump -ibond0 host 124.65.150.30
tcpdump: verbose output suppressed, use -v or -vv for full protocol decode
listening on bond0, link-type EN10MB (Ethernet), capture size 96 bytes
08:01:30.963078 IP DEV-Blur-DB-1.s3lab.mot.com.40181 > 124.65.150.30.xmpp-client: S 2339214547:2339214547(0) win 5840 <mss 1460,sackOK,timestamp 3879708500 0,nop,wscale 7>
08:01:30.963687 IP 124.65.150.30.xmpp-client > DEV-Blur-DB-1.s3lab.mot.com.40181: S 4093453715:4093453715(0) ack 2339214548 win 4380 <mss 1460,nop,wscale 0,nop,nop,timestamp 1103908394 3879708500,sackOK,eol>
08:01:30.963712 IP DEV-Blur-DB-1.s3lab.mot.com.40181 > 124.65.150.30.xmpp-client: . ack 1 win 46 <nop,nop,timestamp 3879708500 1103908394>
08:01:31.143569 IP DEV-Blur-DB-1.s3lab.mot.com.40181 > 124.65.150.30.xmpp-client: P 1:208(207) ack 1 win 46 <nop,nop,timestamp 3879708680 1103908394>
08:01:31.144866 IP 124.65.150.30.xmpp-client > DEV-Blur-DB-1.s3lab.mot.com.40181: P 1:183(182) ack 208 win 4587 <nop,nop,timestamp 1103908575 3879708680>
08:01:31.144900 IP DEV-Blur-DB-1.s3lab.mot.com.40181 > 124.65.150.30.xmpp-client: . ack 183 win 54 <nop,nop,timestamp 3879708681 1103908575>
08:01:31.244338 IP 124.65.150.30.xmpp-client > DEV-Blur-DB-1.s3lab.mot.com.40181: P 183:319(136) ack 208 win 4587 <nop,nop,timestamp 1103908675 3879708681>
08:01:31.244383 IP DEV-Blur-DB-1.s3lab.mot.com.40181 > 124.65.150.30.xmpp-client: . ack 319 win 63 <nop,nop,timestamp 3879708781 1103908675>
08:01:34.726795 IP DEV-Blur-DB-1.s3lab.mot.com.40181 > 124.65.150.30.xmpp-client: P 208:274(66) ack 319 win 63 <nop,nop,timestamp 3879712263 1103908675>
08:01:34.728264 IP 124.65.150.30.xmpp-client > DEV-Blur-DB-1.s3lab.mot.com.40181: P 319:1130(811) ack 274 win 4653 <nop,nop,timestamp 1103912158 3879712263>
08:01:34.728294 IP DEV-Blur-DB-1.s3lab.mot.com.40181 > 124.65.150.30.xmpp-client: . ack 1130 win 76 <nop,nop,timestamp 3879712265 1103912158>
08:01:35.932557 IP DEV-Blur-DB-1.s3lab.mot.com.40181 > 124.65.150.30.xmpp-client: P 274:464(190) ack 1130 win 76 <nop,nop,timestamp 3879713469 1103912158>
08:01:35.935533 IP 124.65.150.30.xmpp-client > DEV-Blur-DB-1.s3lab.mot.com.40181: P 1130:1181(51) ack 464 win 4843 <nop,nop,timestamp 1103913366 3879713469>
08:01:35.935588 IP DEV-Blur-DB-1.s3lab.mot.com.40181 > 124.65.150.30.xmpp-client: . ack 1181 win 76 <nop,nop,timestamp 3879713472 1103913366>
08:01:36.499741 IP DEV-Blur-DB-1.s3lab.mot.com.40181 > 124.65.150.30.xmpp-client: P 464:925(461) ack 1181 win 76 <nop,nop,timestamp 3879714036 1103913366>
08:01:36.501393 IP 124.65.150.30.xmpp-client > DEV-Blur-DB-1.s3lab.mot.com.40181: P 1181:1394(213) ack 925 win 5304 <nop,nop,timestamp 1103913932 3879714036>
08:01:36.501422 IP DEV-Blur-DB-1.s3lab.mot.com.40181 > 124.65.150.30.xmpp-client: . ack 1394 win 88 <nop,nop,timestamp 3879714038 1103913932>
08:01:36.601453 IP 124.65.150.30.xmpp-client > DEV-Blur-DB-1.s3lab.mot.com.40181: P 1394:1697(303) ack 925 win 5304 <nop,nop,timestamp 1103914032 3879714038>
08:01:36.601495 IP DEV-Blur-DB-1.s3lab.mot.com.40181 > 124.65.150.30.xmpp-client: . ack 1697 win 101 <nop,nop,timestamp 3879714138 1103914032>
08:03:06.161609 IP DEV-Blur-DB-1.s3lab.mot.com.41925 > 124.65.150.30.teradataordbms: S 2434143153:2434143153(0) win 5840 <mss 1460,sackOK,timestamp 3879803697 0,nop,wscale 7>
08:03:06.162257 IP 124.65.150.30.teradataordbms > DEV-Blur-DB-1.s3lab.mot.com.41925: S 674537425:674537425(0) ack 2434143154 win 4380 <mss 1460,nop,wscale 0,nop,nop,timestamp 1104003593 3879803697,sackOK,eol>
...

85 packets captured
171 packets received by filter
0 packets dropped by kernel
[root@DEV-Blur-DB-1 ~]#

不能跟踪某一个进程

也可用其它工具, 比如wireshark, solaris的snoop等

3. auditctl审计进程的系统调用
添加审计规则

auditctl -a entry,always -F arch=b32 -S socketcall -F pid=$(pgrep -u s3op2 emulator) -k emulator
auditctl -l
ausearch -ts today -k emulator
tail -f /var/log/audit/audit.log

查看audit.log日志或ausearch查看

[root@DEV-Blur-DB-1 ~]# auditctl -a entry,always -F arch=b32 -S socketcall -F pid=$(pgrep -u s3op2 emulator) -k emulator
[root@DEV-Blur-DB-1 ~]# auditctl -l
LIST_RULES: entry,always arch=1073741827 (0x40000003) pid=9744 (0x2610) key=emulator syscall=socketcall
[root@DEV-Blur-DB-1 ~]# tail -f /var/log/audit/audit.log
...
type=SYSCALL msg=audit(1238054490.902:157569): arch=40000003 syscall=102 success=yes exit=209 a0=9 a1=ff918fe4 a2=ab906ac a3=ab906ac items=0 ppid=9743 pid=9744 auid=4294967295 uid=1003 gid=1002 euid=1003 suid=1003 fsuid=1003 egid=1002 sgid=1002 fsgid=1002 tty=pts61 ses=4294967295 comm="emulator" exe="/data/s3op2/morrison_viper_0_6_10_emulator_linux/bin/emulator" key="emulator"
type=SYSCALL msg=audit(1238054490.902:157570): arch=40000003 syscall=102 success=yes exit=24 a0=a a1=ff919000 a2=ff919200 a3=9abca0 items=0 ppid=9743 pid=9744 auid=4294967295 uid=1003 gid=1002 euid=1003 suid=1003 fsuid=1003 egid=1002 sgid=1002 fsgid=1002 tty=pts61 ses=4294967295 comm="emulator" exe="/data/s3op2/morrison_viper_0_6_10_emulator_linux/bin/emulator" key="emulator"
type=SYSCALL msg=audit(1238054490.961:157571): arch=40000003 syscall=102 success=yes exit=12 a0=1 a1=ff918ea4 a2=ab8ce88 a3=2000 items=0 ppid=9743 pid=9744 auid=4294967295 uid=1003 gid=1002 euid=1003 suid=1003 fsuid=1003 egid=1002 sgid=1002 fsgid=1002 tty=pts61 ses=4294967295 comm="emulator" exe="/data/s3op2/morrison_viper_0_6_10_emulator_linux/bin/emulator" key="emulator"
type=SYSCALL msg=audit(1238054490.961:157572): arch=40000003 syscall=102 success=yes exit=0 a0=e a1=ff918e84 a2=ab8ce88 a3=2000 items=0 ppid=9743 pid=9744 auid=4294967295 uid=1003 gid=1002 euid=1003 suid=1003 fsuid=1003 egid=1002 sgid=1002 fsgid=1002 tty=pts61 ses=4294967295 comm="emulator" exe="/data/s3op2/morrison_viper_0_6_10_emulator_linux/bin/emulator" key="emulator"
type=SYSCALL msg=audit(1238054490.961:157573): arch=40000003 syscall=102 success=yes exit=0 a0=e a1=ff918e84 a2=ab8ce88 a3=2000 items=0 ppid=9743 pid=9744 auid=4294967295 uid=1003 gid=1002 euid=1003 suid=1003 fsuid=1003 egid=1002 sgid=1002 fsgid=1002 tty=pts61 ses=4294967295 comm="emulator" exe="/data/s3op2/morrison_viper_0_6_10_emulator_linux/bin/emulator" key="emulator"
type=SYSCALL msg=audit(1238054490.962:157574): arch=40000003 syscall=102 success=no exit=-115 a0=3 a1=ff918ea4 a2=ab8ce88 a3=2000 items=0 ppid=9743 pid=9744 auid=4294967295 uid=1003 gid=1002 euid=1003 suid=1003 fsuid=1003 egid=1002 sgid=1002 fsgid=1002 tty=pts61 ses=4294967295 comm="emulator" exe="/data/s3op2/morrison_viper_0_6_10_emulator_linux/bin/emulator" key="emulator"
type=SOCKADDR msg=audit(1238054490.962:157574): saddr=020014667C41961E40CF990A0F930A08
type=SYSCALL msg=audit(1238054490.964:157575): arch=40000003 syscall=102 success=yes exit=0 a0=9 a1=ff919080 a2=ff919180 a3=9abca0 items=0 ppid=9743 pid=9744 auid=4294967295 uid=1003 gid=1002 euid=1003 suid=1003 fsuid=1003 egid=1002 sgid=1002 fsgid=1002 tty=pts61 ses=4294967295 comm="emulator" exe="/data/s3op2/morrison_viper_0_6_10_emulator_linux/bin/emulator" key="emulator"
type=SYSCALL msg=audit(1238054491.073:157576): arch=40000003 syscall=102 success=yes exit=190 a0=9 a1=ff918fe4 a2=ab906ac a3=ab906ac items=0 ppid=9743 pid=9744 auid=4294967295 uid=1003 gid=1002 euid=1003 suid=1003 fsuid=1003 egid=1002 sgid=1002 fsgid=1002 tty=pts61 ses=4294967295 comm="emulator" exe="/data/s3op2/morrison_viper_0_6_10_emulator_linux/bin/emulator" key="emulator"
type=SYSCALL msg=audit(1238054491.073:157577): arch=40000003 syscall=102 success=yes exit=24 a0=a a1=ff919000 a2=ff919200 a3=9abca0 items=0 ppid=9743 pid=9744 auid=4294967295 uid=1003 gid=1002 euid=1003 suid=1003 fsuid=1003 egid=1002 sgid=1002 fsgid=1002 tty=pts61 ses=4294967295 comm="emulator" exe="/data/s3op2/morrison_viper_0_6_10_emulator_linux/bin/emulator" key="emulator"
...

不好用, 结果看不懂, 还不如strace
syscall=102表示sys_socketcall, a0=?是socketcall函数的第一个参数, 这里表示套接字调用, 参考/usr/include/linux/net.h

删除规则

[root@DEV-Blur-DB-1 ~]# auditctl -d entry,always -F arch=b32 -S socketcall -F pid=$(pgrep -u s3op2 emulator) -k emulator
[root@DEV-Blur-DB-1 ~]# auditctl -l
No rules
[root@DEV-Blur-DB-1 ~]#

另外参考
/usr/include/asm-x86_64/unistd.h
/usr/include/asm-i386/unistd.h
/usr/include/linux/audit.h
等头文件

4. systemtap
需要安装当前内核版本对应的kernel-debuginfo-common和kernel-debuginfo
RPM安装包可从debuginfo.centos.org下载

[root@DEV-Blur-DB-1 ~]# rpm -ivh kernel-debuginfo-common-2.6.18-92.el5.x86_64.rpm
Preparing...                ########################################### [100%]
   1:kernel-debuginfo-common########################################### [100%]
[root@DEV-Blur-DB-1 ~]# rpm -ivh kernel-debuginfo-2.6.18-92.el5.x86_64.rpm
Preparing...                ########################################### [100%]
   1:kernel-debuginfo       ########################################### [100%]
[root@DEV-Blur-DB-1 ~]# ll

cat /usr/share/doc/systemtap-0.6.2/examples/socket-trace.stp
#! /usr/bin/env stap
probe kernel.function("*@net/socket.c").call {
  printf ("%s -> %s\n", thread_indent(1), probefunc())
}
probe kernel.function("*@net/socket.c").return {
  printf ("%s <- %s\n", thread_indent(-1), probefunc())
}
stap -v /usr/share/doc/systemtap-0.6.2/examples/socket-trace.stp

不会用，仅供参考
SystemTap Beginners Guide

solaris下可用dtrace

-fin-

谢太数据库

谢太的最爱

Blog Archive

Thursday, March 26, 2009

monitoring network activity of processes 查看进程网络活动

No comments:

Followers