5.Linux常用排查命令

可以使用一下命令查使用内存最多的10个线程    
ps aux | sort -k4nr | head -n 10

 

可以使用一下命令查使用CPU最多的10个线程    
ps aux | sort -k3nr | head -n 10

 

查看最消耗CPU的线程
# ps H -eo pid,pcpu|sort -nk2|tail
46062 12.4
46062 13.7
46062 16.2
46062 28.2
46062 28.8
46062 83.3
46062 84.4
46062 84.5
46062 89.5
46062 91.3

 

查看最消耗MEM的线程
# ps H -eo pid,pmem|sort -nk2|tail  
46062  5.2
46062  5.2
46062  5.2
46062  5.2
46062  5.2
46062  5.2
46062  5.2
46062  5.2
46062  5.2
46062  5.2

 

查看最消耗CPU的线程及应用程序
# ps H -eo pid,pcpu,command|sort -nk2|tail        
46062 12.4 /usr/sbin/mysqld --basedir=/usr --datadir=/export/mysql/data --plugin-dir=/usr/lib64/mysql/plugin --user=mysql --log-error=/export/mysql/log/error.log --pid-file=/var/run/mysqld/mysqld.pid --socket=/var/lib/mysql/mysql.sock --port=3306
46062 13.7 /usr/sbin/mysqld --basedir=/usr --datadir=/export/mysql/data --plugin-dir=/usr/lib64/mysql/plugin --user=mysql --log-error=/export/mysql/log/error.log --pid-file=/var/run/mysqld/mysqld.pid --socket=/var/lib/mysql/mysql.sock --port=3306
46062 16.2 /usr/sbin/mysqld --basedir=/usr --datadir=/export/mysql/data --plugin-dir=/usr/lib64/mysql/plugin --user=mysql --log-error=/export/mysql/log/error.log --pid-file=/var/run/mysqld/mysqld.pid --socket=/var/lib/mysql/mysql.sock --port=3306
46062 28.7 /usr/sbin/mysqld --basedir=/usr --datadir=/export/mysql/data --plugin-dir=/usr/lib64/mysql/plugin --user=mysql --log-error=/export/mysql/log/error.log --pid-file=/var/run/mysqld/mysqld.pid --socket=/var/lib/mysql/mysql.sock --port=3306
46062 28.8 /usr/sbin/mysqld --basedir=/usr --datadir=/export/mysql/data --plugin-dir=/usr/lib64/mysql/plugin --user=mysql --log-error=/export/mysql/log/error.log --pid-file=/var/run/mysqld/mysqld.pid --socket=/var/lib/mysql/mysql.sock --port=3306
46062 83.3 /usr/sbin/mysqld --basedir=/usr --datadir=/export/mysql/data --plugin-dir=/usr/lib64/mysql/plugin --user=mysql --log-error=/export/mysql/log/error.log --pid-file=/var/run/mysqld/mysqld.pid --socket=/var/lib/mysql/mysql.sock --port=3306
46062 84.4 /usr/sbin/mysqld --basedir=/usr --datadir=/export/mysql/data --plugin-dir=/usr/lib64/mysql/plugin --user=mysql --log-error=/export/mysql/log/error.log --pid-file=/var/run/mysqld/mysqld.pid --socket=/var/lib/mysql/mysql.sock --port=3306
46062 84.5 /usr/sbin/mysqld --basedir=/usr --datadir=/export/mysql/data --plugin-dir=/usr/lib64/mysql/plugin --user=mysql --log-error=/export/mysql/log/error.log --pid-file=/var/run/mysqld/mysqld.pid --socket=/var/lib/mysql/mysql.sock --port=3306
46062 89.5 /usr/sbin/mysqld --basedir=/usr --datadir=/export/mysql/data --plugin-dir=/usr/lib64/mysql/plugin --user=mysql --log-error=/export/mysql/log/error.log --pid-file=/var/run/mysqld/mysqld.pid --socket=/var/lib/mysql/mysql.sock --port=3306
46062 91.3 /usr/sbin/mysqld --basedir=/usr --datadir=/export/mysql/data --plugin-dir=/usr/lib64/mysql/plugin --user=mysql --log-error=/export/mysql/log/error.log --pid-file=/var/run/mysqld/mysqld.pid --socket=/var/lib/mysql/mysql.sock --port=3306

 

查看最消耗MEM的线程及应用程序
# ps H -eo pid,pmem,command|sort -nk2|tail  
46062  5.2 /usr/sbin/mysqld --basedir=/usr --datadir=/export/mysql/data --plugin-dir=/usr/lib64/mysql/plugin --user=mysql --log-error=/export/mysql/log/error.log --pid-file=/var/run/mysqld/mysqld.pid --socket=/var/lib/mysql/mysql.sock --port=3306
46062  5.2 /usr/sbin/mysqld --basedir=/usr --datadir=/export/mysql/data --plugin-dir=/usr/lib64/mysql/plugin --user=mysql --log-error=/export/mysql/log/error.log --pid-file=/var/run/mysqld/mysqld.pid --socket=/var/lib/mysql/mysql.sock --port=3306
46062  5.2 /usr/sbin/mysqld --basedir=/usr --datadir=/export/mysql/data --plugin-dir=/usr/lib64/mysql/plugin --user=mysql --log-error=/export/mysql/log/error.log --pid-file=/var/run/mysqld/mysqld.pid --socket=/var/lib/mysql/mysql.sock --port=3306
46062  5.2 /usr/sbin/mysqld --basedir=/usr --datadir=/export/mysql/data --plugin-dir=/usr/lib64/mysql/plugin --user=mysql --log-error=/export/mysql/log/error.log --pid-file=/var/run/mysqld/mysqld.pid --socket=/var/lib/mysql/mysql.sock --port=3306
46062  5.2 /usr/sbin/mysqld --basedir=/usr --datadir=/export/mysql/data --plugin-dir=/usr/lib64/mysql/plugin --user=mysql --log-error=/export/mysql/log/error.log --pid-file=/var/run/mysqld/mysqld.pid --socket=/var/lib/mysql/mysql.sock --port=3306
46062  5.2 /usr/sbin/mysqld --basedir=/usr --datadir=/export/mysql/data --plugin-dir=/usr/lib64/mysql/plugin --user=mysql --log-error=/export/mysql/log/error.log --pid-file=/var/run/mysqld/mysqld.pid --socket=/var/lib/mysql/mysql.sock --port=3306
46062  5.2 /usr/sbin/mysqld --basedir=/usr --datadir=/export/mysql/data --plugin-dir=/usr/lib64/mysql/plugin --user=mysql --log-error=/export/mysql/log/error.log --pid-file=/var/run/mysqld/mysqld.pid --socket=/var/lib/mysql/mysql.sock --port=3306
46062  5.2 /usr/sbin/mysqld --basedir=/usr --datadir=/export/mysql/data --plugin-dir=/usr/lib64/mysql/plugin --user=mysql --log-error=/export/mysql/log/error.log --pid-file=/var/run/mysqld/mysqld.pid --socket=/var/lib/mysql/mysql.sock --port=3306
46062  5.2 /usr/sbin/mysqld --basedir=/usr --datadir=/export/mysql/data --plugin-dir=/usr/lib64/mysql/plugin --user=mysql --log-error=/export/mysql/log/error.log --pid-file=/var/run/mysqld/mysqld.pid --socket=/var/lib/mysql/mysql.sock --port=3306
46062  5.2 /usr/sbin/mysqld --basedir=/usr --datadir=/export/mysql/data --plugin-dir=/usr/lib64/mysql/plugin --user=mysql --log-error=/export/mysql/log/error.log --pid-file=/var/run/mysqld/mysqld.pid --socket=/var/lib/mysql/mysql.sock --port=3306

 

 

根据进程查应用程序
# ps aux|fgrep 46062
root     33140  0.0  0.0 100952   684 pts/1    S+   11:27   0:00 fgrep 46062
mysql    46062  565  5.2 14796316 1723368 ?    Sl   Feb11 1209085:27 /usr/sbin/mysqld --basedir=/usr --datadir=/export/mysql/data --plugin-dir=/usr/lib64/mysql/plugin --user=mysql --log-error=/export/mysql/log/error.log --pid-file=/var/run/mysqld/mysqld.pid --socket=/var/lib/mysql/mysql.sock --port=3306

 

根据端口查询进程和应用程序
lsof -i:3306/mysql(程序名称)
COMMAND   PID  USER   FD   TYPE    DEVICE SIZE/OFF NODE NAME
mysqld  46062 mysql   13u  IPv6  69109648      0t0  TCP *:mysql (LISTEN)
mysqld  46062 mysql   19u  IPv6 522548147      0t0  TCP 10.159.33.15:mysql->10.159.32.173:50842 (ESTABLISHED)
mysqld  46062 mysql   25u  IPv6 522451730      0t0  TCP 10.159.33.15:mysql->10.159.32.170:44889 (ESTABLISHED)
mysqld  46062 mysql   26u  IPv6 522527700      0t0  TCP 10.159.33.15:mysql->10.159.32.170:49604 (ESTABLISHED)
mysqld  46062 mysql   28u  IPv6 522540747      0t0  TCP 10.159.33.15:mysql->10.159.32.171:60664 (ESTABLISHED)
mysqld  46062 mysql   29u  IPv6 522547817      0t0  TCP 10.159.33.15:mysql->10.159.32.170:51505 (ESTABLISHED)
mysqld  46062 mysql   30u  IPv6 522446872      0t0  TCP 10.159.33.15:mysql->10.159.32.170:44404 (ESTABLISHED)
mysqld  46062 mysql   31u  IPv6 522551513      0t0  TCP 10.159.33.15:mysql->10.159.32.173:52172 (ESTABLISHED)
mysqld  46062 mysql   32u  IPv6 522504208      0t0  TCP 10.159.33.15:mysql->10.159.32.172:36145 (ESTABLISHED)
mysqld  46062 mysql   33u  IPv6 522540748      0t0  TCP 10.159.33.15:mysql->10.159.32.171:60665 (ESTABLISHED)
mysqld  46062 mysql   34u  IPv6 522499270      0t0  TCP 10.159.33.15:mysql->10.159.32.171:60117 (ESTABLISHED)
mysqld  46062 mysql   35u  IPv6 522449331      0t0  TCP 10.159.33.15:mysql->10.159.32.173:38880 (ESTABLISHED)
mysqld  46062 mysql   38u  IPv6 522499271      0t0  TCP 10.159.33.15:mysql->10.159.32.171:60118 (ESTABLISHED)
mysqld  46062 mysql   40u  IPv6 522493845      0t0  TCP 10.159.33.15:mysql->10.159.32.172:57554 (ESTABLISHED)
mysqld  46062 mysql   44u  IPv6 522470228      0t0  TCP 10.159.33.15:mysql->10.159.32.170:45947 (ESTABLISHED)
mysqld  46062 mysql   47u  IPv6 522479678      0t0  TCP 10.159.33.15:mysql->10.159.32.170:46452 (ESTABLISHED)

 

查看某一程序的进程，线程信息。每一列依次为用户ID，PID，PPID，LWP，第六列NLWP(LWP(线程ID)和NLWP(线程的个数))
# ps -eLf | grep mysql
root      3672     1  3672  0    1  2014 ?        00:00:00 mysql -uroot
root     21124     1 21124  0    1  2014 ?        00:00:00 /bin/sh /usr/bin/mysqld_safe --datadir=/apps/mysql/data --socket=/var/lib/mysql/mysql.sock --pid-file=/var/run/mysqld/mysqld.pid --basedir=/usr --user=mysql
mysql    21502 21124 21502  0   44  2014 ?        00:04:19 /usr/sbin/mysqld --basedir=/usr --datadir=/apps/mysql/data --plugin-dir=/usr/lib64/mysql/plugin --user=mysql --log-error=/export/mysql/log/error.log --pid-file=/var/run/mysqld/mysqld.pid --socket=/var/lib/mysql/mysql.sock --port=3306
mysql    21502 21124 21504  0   44  2014 ?        00:21:12 /usr/sbin/mysqld --basedir=/usr --datadir=/apps/mysql/data --plugin-dir=/usr/lib64/mysql/plugin --user=mysql --log-error=/export/mysql/log/error.log --pid-file=/var/run/mysqld/mysqld.pid --socket=/var/lib/mysql/mysql.sock --port=3306
mysql    21502 21124 21505  0   44  2014 ?        00:30:39 /usr/sbin/mysqld --basedir=/usr --datadir=/apps/mysql/data --plugin-dir=/usr/lib64/mysql/plugin --user=mysql --log-error=/export/mysql/log/error.log --pid-file=/var/run/mysqld/mysqld.pid --socket=/var/lib/mysql/mysql.sock --port=3306
mysql    21502 21124 21506  0   44  2014 ?        2-11:43:19 /usr/sbin/mysqld --basedir=/usr --datadir=/apps/mysql/data --plugin-dir=/usr/lib64/mysql/plugin --user=mysql --log-error=/export/mysql/log/error.log --pid-file=/var/run/mysqld/mysqld.pid --socket=/var/lib/mysql/mysql.sock --port=3306
mysql    21502 21124 21507  0   44  2014 ?        2-11:17:25 /usr/sbin/mysqld --basedir=/usr --datadir=/apps/mysql/data --plugin-dir=/usr/lib64/mysql/plugin --user=mysql --log-error=/export/mysql/log/error.log --pid-file=/var/run/mysqld/mysqld.pid --socket=/var/lib/mysql/mysql.sock --port=3306

每一列依次为用户ID，进程ID，父进程ID，线程ID，运行该线程的CPU的序号
# ps -eo ruser,pid,ppid,lwp,psr,args -L |grep mysql
root      3672     1  3672   2 mysql -uroot
root     21124     1 21124   0 /bin/sh /usr/bin/mysqld_safe --datadir=/apps/mysql/data --socket=/var/lib/mysql/mysql.sock --pid-file=/var/run/mysqld/mysqld.pid --basedir=/usr --user=mysql
mysql    21502 21124 21502   2 /usr/sbin/mysqld --basedir=/usr --datadir=/apps/mysql/data --plugin-dir=/usr/lib64/mysql/plugin --user=mysql --log-error=/export/mysql/log/error.log --pid-file=/var/run/mysqld/mysqld.pid --socket=/var/lib/mysql/mysql.sock --port=3306
mysql    21502 21124 21504   1 /usr/sbin/mysqld --basedir=/usr --datadir=/apps/mysql/data --plugin-dir=/usr/lib64/mysql/plugin --user=mysql --log-error=/export/mysql/log/error.log --pid-file=/var/run/mysqld/mysqld.pid --socket=/var/lib/mysql/mysql.sock --port=3306
mysql    21502 21124 21505  13 /usr/sbin/mysqld --basedir=/usr --datadir=/apps/mysql/data --plugin-dir=/usr/lib64/mysql/plugin --user=mysql --log-error=/export/mysql/log/error.log --pid-file=/var/run/mysqld/mysqld.pid --socket=/var/lib/mysql/mysql.sock --port=3306
mysql    21502 21124 21506   5 /usr/sbin/mysqld --basedir=/usr --datadir=/apps/mysql/data --plugin-dir=/usr/lib64/mysql/plugin --user=mysql --log-error=/export/mysql/log/error.log --pid-file=/var/run/mysqld/mysqld.pid --socket=/var/lib/mysql/mysql.sock --port=3306
mysql    21502 21124 21507   1 /usr/sbin/mysqld --basedir=/usr --datadir=/apps/mysql/data --plugin-dir=/usr/lib64/mysql/plugin --user=mysql --log-error=/export/mysql/log/error.log --pid-file=/var/run/mysqld/mysqld.pid --socket=/var/lib/mysql/mysql.sock --port=3306

 

查看线程top H
Tasks: 1824 total,   5 running, 1819 sleeping,   0 stopped,   0 zombie
Cpu(s): 30.9%us,  1.7%sy,  0.0%ni, 63.1%id,  4.2%wa,  0.0%hi,  0.1%si,  0.0%st
Mem:  32827220k total, 32262032k used,   565188k free,   243652k buffers
Swap: 33554424k total,  1036436k used, 32517988k free, 28803144k cached

  PID USER      PR  NI  VIRT  RES  SHR S %CPU %MEM    TIME+  COMMAND                                                                               
23451 mysql     20   0 14.1g 1.6g 7492 R 99.7  5.2  68435:19 mysqld                                                                                
27083 mysql     20   0 14.1g 1.6g 7492 R 99.4  5.2  73519:03 mysqld                                                                                
43328 mysql     20   0 14.1g 1.6g 7492 R 99.0  5.2  73468:59 mysqld                                                                                
19728 mysql     20   0 14.1g 1.6g 7492 R 98.7  5.2  29:20.28 mysqld                                                                                
37088 mysql     20   0 14.1g 1.6g 7492 S 53.9  5.2 361:18.74 mysqld                                                                                
11254 mysql     20   0 14.1g 1.6g 7492 S 51.6  5.2   2301:20 mysqld                                                                                
34694 mysql     20   0 14.1g 1.6g 7492 S 46.4  5.2   1648:14 mysqld                                                                                
38157 mysql     20   0 14.1g 1.6g 7492 S 39.0  5.2 226:09.02 mysqld                                                                                
39131 mysql     20   0 14.1g 1.6g 7492 S 35.1  5.2  28:27.61 mysqld                                                                                
29578 mysql     20   0 14.1g 1.6g 7492 S 35.1  5.2   0:58.99 mysqld                                                                                
21886 mysql     20   0 14.1g 1.6g 7492 S 25.6  5.2 237:17.99 mysqld                                                                                
46072 mysql     20   0 14.1g 1.6g 7492 S 14.0  5.2  17152:01 mysqld                                                                                
46073 mysql     20   0 14.1g 1.6g 7492 S 14.0  5.2  17973:15 mysqld

 

查看进程top
Tasks: 444 total,   1 running, 443 sleeping,   0 stopped,   0 zombie
Cpu(s): 28.9%us,  1.7%sy,  0.0%ni, 65.0%id,  4.2%wa,  0.0%hi,  0.1%si,  0.0%st
Mem:  32827220k total, 32255008k used,   572212k free,   243728k buffers
Swap: 33554424k total,  1036436k used, 32517988k free, 28802036k cached

  PID USER      PR  NI  VIRT  RES  SHR S %CPU %MEM    TIME+  COMMAND                                                                               
46062 mysql     20   0 14.1g 1.6g 7492 S 737.6  5.2  20731,01 mysqld                                                                               
 1754 root      20   0     0    0    0 S  0.3  0.0   1003:34 jbd2/dm-3-8                                                                           
46066 root      20   0 15300 1516  932 R  0.3  0.0   0:00.02 top                                                                                   
    1 root      20   0 19356  544  380 S  0.0  0.0   0:23.08 init                                                                                  
    2 root      20   0     0    0    0 S  0.0  0.0   0:04.40 kthreadd                                                                              
    3 root      RT   0     0    0    0 S  0.0  0.0  12:59.81 migration/0                                                                           
    4 root      20   0     0    0    0 S  0.0  0.0  19:02.84 ksoftirqd/0                                                                           
    5 root      RT   0     0    0    0 S  0.0  0.0   0:00.00 migration/0

         

查看每个cpu的状态top 1
top - 10:51:59 up 125 days, 14:56, 24 users,  load average: 1.46, 1.25, 1.14
Tasks: 427 total,   1 running, 426 sleeping,   0 stopped,   0 zombie
%Cpu0  :  0.3 us,  1.0 sy,  0.0 ni,  0.0 id, 98.7 wa,  0.0 hi,  0.0 si,  0.0 st
%Cpu1  :  0.3 us,  0.3 sy,  0.0 ni, 99.3 id,  0.0 wa,  0.0 hi,  0.0 si,  0.0 st
%Cpu2  :  0.0 us,  0.0 sy,  0.0 ni,100.0 id,  0.0 wa,  0.0 hi,  0.0 si,  0.0 st
%Cpu3  : 48.2 us,  9.8 sy,  0.0 ni, 41.6 id,  0.0 wa,  0.0 hi,  0.3 si,  0.0 st
KiB Mem:  16434272 total, 14468204 used,  1966068 free,  6771280 buffers
KiB Swap:   524284 total,   524128 used,      156 free.  1317920 cached Mem

  PID USER      PR  NI    VIRT    RES    SHR S  %CPU %MEM     TIME+ COMMAND                                                                  
12112 ubuntu    20   0  312324 186356   4372 S  47.5  1.1   1:07.38 python                                                                   
11540 ubuntu    20   0  161300  35400   4380 S   8.0  0.2   1:32.66 python                                                                   
23402 root      20   0   26908  20032    692 D   0.7  0.1   1:48.01 updatedb.mlocat                                                          
10682 ubuntu    20   0  105672   2088   1024 S   0.3  0.0   0:00.13 sshd                                                                     
12277 ubuntu    20   0   26132   2504   1140 R   0.3  0.0   0:00.83 top                                                                      
25898 ubuntu    20   0 2485820  31160   1464 S   0.3  0.2   1:11.79 phantomjs                                                                
    1 root      20   0   37204   5716    652 S   0.0  0.0   1:27.24 init                                                                     
    2 root      20   0       0      0      0 S   0.0  0.0   0:02.94 kthreadd                                                                 
    3 root      20   0       0      0      0 S   0.0  0.0   0:06.37 ksoftirqd/0                                                              
    5 root       0 -20       0      0      0 S   0.0  0.0   0:00.00 kworker/0:0H  
   
cpu各项指标说明：
us：用户态使用的cpu时间比
sy：系统态使用的cpu时间比
ni：用做nice加权的进程分配的用户态cpu时间比
id：空闲的cpu时间比
wa：cpu等待磁盘写入完成时间
hi：硬中断消耗时间
si：软中断消耗时间
st：虚拟机偷取时间

 

查看物理CPU个数
# cat /proc/cpuinfo |grep "physical id"|sort |uniq|wc -l

查看CPU核心数
# cat /proc/cpuinfo |grep "cores"|uniq 

查看逻辑CPU个数
# cat /proc/cpuinfo |grep "processor"|wc -l