• hadoop3 无法启动 查找内存消耗原因


    [root@hadoop3 hadoop]# xlo
    Starting namenodes on [hadoop3]
    上一次登录:三 12月 27 15:07:11 CST 2017pts/24 上
    hadoop3: ERROR: Cannot set priority of namenode process 9025
    Starting datanodes
    上一次登录:三 12月 27 15:16:59 CST 2017pts/24 上
    hadoop3: ERROR: Cannot set priority of datanode process 9236
    hadoop1: ERROR: Cannot set priority of datanode process 22612
    Starting secondary namenodes [hadoop3]
    上一次登录:三 12月 27 15:17:01 CST 2017pts/24 上
    Starting resourcemanager
    上一次登录:三 12月 27 15:17:04 CST 2017pts/24 上
    Starting nodemanagers
    上一次登录:三 12月 27 15:17:08 CST 2017pts/24 上
    [root@hadoop3 hadoop]#

    [root@hadoop3 hadoop]# cat ~/.bashrc
    # .bashrc

    # User specific aliases and functions

    alias rm='rm -i'
    alias cp='cp -i'
    alias mv='mv -i'
    alias xlpy='ps -aux|grep python|grep -v grep'
    alias xlfg='free -g'
    alias xlfm='free -m'
    alias xlce='crontab -e'
    alias xlcl='crontab -l'
    alias xlrp='python '
    alias xlhdfsf='/usr/local/hadoop/bin/hdfs namenode -format'
    alias xlo='/usr/local/hadoop/sbin/start-all.sh'
    alias xlc='/usr/local/hadoop/sbin/stop-all.sh'

    # Source global definitions
    if [ -f /etc/bashrc ]; then
    . /etc/bashrc
    fi

    [root@hadoop3 hadoop]#

    [root@hadoop3 ~]# xlfg
                  total        used        free      shared  buff/cache   available
    Mem:             15           2           0           0          12          12
    Swap:             7           0           7
    [root@hadoop3 ~]# 
    [root@hadoop3 ~]# ps -eo rss,pmem,vsize,args | sort -k 1 -r -n 
    484708  2.9 6007736 /usr/local/jdk/bin/java -Dproc_namenode -Djava.net.preferIPv4Stack=true -Dhdfs.audit.logger=INFO,NullAppender -Dhadoop.security.logger=INFO,RFAS -Dyarn.log.dir=/usr/local/hadoop/logs -Dyarn.log.file=hadoop-root-namenode-hadoop3.log -Dyarn.home.dir=/usr/local/hadoop -Dyarn.root.logger=INFO,console -Djava.library.path=/usr/local/hadoop/lib/native -Dhadoop.log.dir=/usr/local/hadoop/logs -Dhadoop.log.file=hadoop-root-namenode-hadoop3.log -Dhadoop.home.dir=/usr/local/hadoop -Dhadoop.id.str=root -Dhadoop.root.logger=INFO,RFA -Dhadoop.policy.file=hadoop-policy.xml org.apache.hadoop.hdfs.server.namenode.NameNode
    383044  2.3 6177364 /usr/local/jdk/bin/java -Dproc_resourcemanager -Djava.net.preferIPv4Stack=true -Dyarn.log.dir=/usr/local/hadoop/logs -Dyarn.log.file=hadoop-root-resourcemanager-hadoop3.log -Dyarn.home.dir=/usr/local/hadoop -Dyarn.root.logger=INFO,console -Djava.library.path=/usr/local/hadoop/lib/native -Dhadoop.log.dir=/usr/local/hadoop/logs -Dhadoop.log.file=hadoop-root-resourcemanager-hadoop3.log -Dhadoop.home.dir=/usr/local/hadoop -Dhadoop.id.str=root -Dhadoop.root.logger=INFO,RFA -Dhadoop.policy.file=hadoop-policy.xml -Dhadoop.security.logger=INFO,NullAppender org.apache.hadoop.yarn.server.resourcemanager.ResourceManager
    381180  2.3 6043360 /usr/local/jdk/bin/java -Dproc_nodemanager -Djava.net.preferIPv4Stack=true -Dyarn.log.dir=/usr/local/hadoop/logs -Dyarn.log.file=hadoop-root-nodemanager-hadoop3.log -Dyarn.home.dir=/usr/local/hadoop -Dyarn.root.logger=INFO,console -Djava.library.path=/usr/local/hadoop/lib/native -Dhadoop.log.dir=/usr/local/hadoop/logs -Dhadoop.log.file=hadoop-root-nodemanager-hadoop3.log -Dhadoop.home.dir=/usr/local/hadoop -Dhadoop.id.str=root -Dhadoop.root.logger=INFO,RFA -Dhadoop.policy.file=hadoop-policy.xml -Dhadoop.security.logger=INFO,NullAppender org.apache.hadoop.yarn.server.nodemanager.NodeManager
    375868  2.3 5962452 /usr/local/jdk/bin/java -Dproc_secondarynamenode -Djava.net.preferIPv4Stack=true -Dhdfs.audit.logger=INFO,NullAppender -Dhadoop.security.logger=INFO,RFAS -Dyarn.log.dir=/usr/local/hadoop/logs -Dyarn.log.file=hadoop-root-secondarynamenode-hadoop3.log -Dyarn.home.dir=/usr/local/hadoop -Dyarn.root.logger=INFO,console -Djava.library.path=/usr/local/hadoop/lib/native -Dhadoop.log.dir=/usr/local/hadoop/logs -Dhadoop.log.file=hadoop-root-secondarynamenode-hadoop3.log -Dhadoop.home.dir=/usr/local/hadoop -Dhadoop.id.str=root -Dhadoop.root.logger=INFO,RFA -Dhadoop.policy.file=hadoop-policy.xml org.apache.hadoop.hdfs.server.namenode.SecondaryNameNode
    335036  2.0 5998380 /usr/local/jdk/bin/java -Dproc_datanode -Djava.net.preferIPv4Stack=true -Dhadoop.security.logger=ERROR,RFAS -Dyarn.log.dir=/usr/local/hadoop/logs -Dyarn.log.file=hadoop-root-datanode-hadoop3.log -Dyarn.home.dir=/usr/local/hadoop -Dyarn.root.logger=INFO,console -Djava.library.path=/usr/local/hadoop/lib/native -Dhadoop.log.dir=/usr/local/hadoop/logs -Dhadoop.log.file=hadoop-root-datanode-hadoop3.log -Dhadoop.home.dir=/usr/local/hadoop -Dhadoop.id.str=root -Dhadoop.root.logger=INFO,RFA -Dhadoop.policy.file=hadoop-policy.xml org.apache.hadoop.hdfs.server.datanode.DataNode
    68436  0.4 110564 /usr/lib/systemd/systemd-journald
    59936  0.3 3077632 /usr/local/phantomjs/bin/phantomjs --cookies-file=/tmp/tmprwyetcbf --webdriver=49908
    40744  0.2 552392 python /home/data/crontab_chk_url/personas/trunk/plugins/spider/chk_url_status_insert_tmp_error_30min.py
    39820  0.2 352972 /usr/sbin/rsyslogd -n
    38052  0.2 478916 python /home/data/crontab_chk_url/personas/trunk/plugins/spider/chk_url_status_insert_tmp_error_30min.py
    36612  0.2 478660 python /home/data/crontab_chk_url/personas/trunk/plugins/spider/chk_url_status_insert_tmp_error_30min.py
    36112  0.2 2838472 /usr/local/phantomjs/bin/phantomjs --cookies-file=/tmp/tmp8_stcm62 --webdriver=33070
    35688  0.2 478660 python /home/data/crontab_chk_url/personas/trunk/plugins/spider/chk_url_status_insert_tmp_error_30min.py
    34256  0.2 552392 python /home/data/crontab_chk_url/personas/trunk/plugins/spider/chk_url_status_insert_tmp_error_30min.py
    33968  0.2 404672 python /home/data/crontab_chk_url/personas/trunk/plugins/spider/chk_url_status_insert_tmp_error_30min.py
    33572  0.2 404672 python /home/data/crontab_chk_url/personas/trunk/plugins/spider/chk_url_status_insert_tmp_error_30min.py
    33220  0.2 2839092 /usr/local/phantomjs/bin/phantomjs --cookies-file=/tmp/tmpq9k5ga_d --webdriver=37099
    12152  0.0 529480 /usr/lib/polkit-1/polkitd --no-debug
     8132  0.0 439592 /usr/sbin/NetworkManager --no-daemon
     6424  0.0 144460 sshd: root@notty
     5724  0.0 143932 sshd: root@pts/28,pts/29,pts/30,pts/31,pts/32
     5712  0.0 143932 sshd: root@pts/26,pts/21,pts/22
     5708  0.0 144064 sshd: root@pts/24,pts/6,pts/7
     5704  0.0 144092 sshd: root@notty
     5700  0.0 143932 sshd: root@pts/1
     5684  0.0 144068 sshd: root@pts/0
     5676  0.0 143932 sshd: root@pts/2
     5668  0.0 143932 sshd: root@pts/8
     5668  0.0 143932 sshd: root@pts/27
     5668  0.0 143932 sshd: root@pts/23
     4024  0.0  49748 /usr/lib/systemd/systemd --switched-root --system --deserialize 21
     3800  0.0  83840 /usr/sbin/sendmail -FCronDaemon -i -odi -oem -oi -t -f root
     3792  0.0  83840 /usr/sbin/sendmail -s zh_cn_xiaomi_buqiang_jiankong g1e2t3@qq.com
     3664  0.0  80936 /usr/sbin/sshd -D
     2700  0.0  51404 /usr/sbin/wpa_supplicant -u -f /var/log/wpa_supplicant.log -c /etc/wpa_supplicant/wpa_supplicant.conf -u -f /var/log/wpa_supplicant.log -P /var/run/wpa_supplicant.pid
     2532  0.0 176076 /usr/sbin/CROND -n
     2488  0.0 176076 /usr/sbin/CROND -n
     2488  0.0 176076 /usr/sbin/CROND -n
     2488  0.0 176076 /usr/sbin/CROND -n
     2488  0.0 176076 /usr/sbin/CROND -n
     2488  0.0 176076 /usr/sbin/CROND -n
     2488  0.0 176076 /usr/sbin/CROND -n
     2488  0.0 176076 /usr/sbin/CROND -n
     2488  0.0 176076 /usr/sbin/CROND -n
     2356  0.0  43704 /usr/lib/systemd/systemd-udevd
     2264  0.0 115636 -bash
     2252  0.0 146328 top
     2224  0.0 115528 -bash
     2212  0.0  22008 nload
     2204  0.0 146316 top
     2204  0.0 115608 -bash
     2192  0.0 115652 -bash
     2188  0.0 115608 -bash
     2184  0.0 115528 -bash
     2184  0.0 115480 -bash
     2144  0.0 115528 -bash
     2136  0.0  53248 /usr/libexec/openssh/sftp-server
     2136  0.0 115516 -bash
     2116  0.0 115480 -bash
     2116  0.0 115480 -bash
     2112  0.0 115480 -bash
     2112  0.0 115400 -bash
     2112  0.0 115400 -bash
     2108  0.0 115400 -bash
     2108  0.0 115400 -bash
     2104  0.0 115400 -bash
     1792  0.0  24724 /bin/dbus-daemon --system --address=systemd: --nofork --nopidfile --systemd-activation
     1724  0.0  24208 /usr/lib/systemd/systemd-logind
     1632  0.0 124192 /usr/sbin/crond -n
     1584  0.0  49020 /sbin/auditd -n
     1400  0.0 192564 /usr/sbin/lvmetad -f
     1396  0.0 113132 /bin/sh -c cd /home/data/xl_project;cat r1 > rtmp;date >r1; free -g >>r1;cat rtmp>>r1;/usr/sbin/sendmail -s 'zh_cn_xiaomi_buqiang_jiankong'  g1e2t3@qq.com < r1;
     1276  0.0 137392 ps -eo rss,pmem,vsize,args
     1228  0.0 113132 /bin/sh -c python /home/data/crontab_chk_url/personas/trunk/plugins/spider/chk_url_status_insert_tmp_error_30min.py &> /dev/null;
     1228  0.0 113132 /bin/sh -c python /home/data/crontab_chk_url/personas/trunk/plugins/spider/chk_url_status_insert_tmp_error_30min.py &> /dev/null;
     1228  0.0 113132 /bin/sh -c python /home/data/crontab_chk_url/personas/trunk/plugins/spider/chk_url_status_insert_tmp_error_30min.py &> /dev/null;
     1224  0.0 113132 /bin/sh -c python /home/data/crontab_chk_url/personas/trunk/plugins/spider/chk_url_status_insert_tmp_error_30min.py &> /dev/null;
     1224  0.0 113132 /bin/sh -c python /home/data/crontab_chk_url/personas/trunk/plugins/spider/chk_url_status_insert_tmp_error_30min.py &> /dev/null;
     1224  0.0 113132 /bin/sh -c python /home/data/crontab_chk_url/personas/trunk/plugins/spider/chk_url_status_insert_tmp_error_30min.py &> /dev/null;
     1220  0.0 113132 /bin/sh -c python /home/data/crontab_chk_url/personas/trunk/plugins/spider/chk_url_status_insert_tmp_error_30min.py &> /dev/null;
     1192  0.0  21480 /usr/sbin/irqbalance --foreground
      920  0.0 126696 sort -k 1 -r -n
      824  0.0 110048 /sbin/agetty --noclear tty1 linux
      RSS %MEM    VSZ COMMAND
        0  0.0      0 [xfs_mru_cache]
        0  0.0      0 [xfs-data/sda1]
        0  0.0      0 [xfs-data/dm-2]
        0  0.0      0 [xfs-data/dm-0]
        0  0.0      0 [xfs-conv/sda1]
        0  0.0      0 [xfs-conv/dm-2]
        0  0.0      0 [xfs-conv/dm-0]
        0  0.0      0 [xfs-cil/sda1]
        0  0.0      0 [xfs-cil/dm-2]
        0  0.0      0 [xfs-cil/dm-0]
        0  0.0      0 [xfs-buf/sda1]
        0  0.0      0 [xfs-buf/dm-2]
        0  0.0      0 [xfs-buf/dm-0]
        0  0.0      0 [xfsalloc]
        0  0.0      0 [xfsaild/sda1]
        0  0.0      0 [xfsaild/dm-2]
        0  0.0      0 [xfsaild/dm-0]
        0  0.0      0 [writeback]
        0  0.0      0 [watchdog/7]
        0  0.0      0 [watchdog/6]
        0  0.0      0 [watchdog/5]
        0  0.0      0 [watchdog/4]
        0  0.0      0 [watchdog/3]
        0  0.0      0 [watchdog/2]
        0  0.0      0 [watchdog/1]
        0  0.0      0 [watchdog/0]
        0  0.0      0 [ttm_swap]
        0  0.0      0 [scsi_tmf_5]
        0  0.0      0 [scsi_tmf_4]
        0  0.0      0 [scsi_tmf_3]
        0  0.0      0 [scsi_tmf_2]
        0  0.0      0 [scsi_tmf_1]
        0  0.0      0 [scsi_tmf_0]
        0  0.0      0 [scsi_eh_5]
        0  0.0      0 [scsi_eh_4]
        0  0.0      0 [scsi_eh_3]
        0  0.0      0 [scsi_eh_2]
        0  0.0      0 [scsi_eh_1]
        0  0.0      0 [scsi_eh_0]
        0  0.0      0 [rcu_sched]
        0  0.0      0 [rcuos/9]
        0  0.0      0 [rcuos/8]
        0  0.0      0 [rcuos/7]
        0  0.0      0 [rcuos/6]
        0  0.0      0 [rcuos/5]
        0  0.0      0 [rcuos/4]
        0  0.0      0 [rcuos/3]
        0  0.0      0 [rcuos/2]
        0  0.0      0 [rcuos/15]
        0  0.0      0 [rcuos/14]
        0  0.0      0 [rcuos/13]
        0  0.0      0 [rcuos/12]
        0  0.0      0 [rcuos/11]
        0  0.0      0 [rcuos/10]
        0  0.0      0 [rcuos/1]
        0  0.0      0 [rcuos/0]
        0  0.0      0 [rcuob/9]
        0  0.0      0 [rcuob/8]
        0  0.0      0 [rcuob/7]
        0  0.0      0 [rcuob/6]
        0  0.0      0 [rcuob/5]
        0  0.0      0 [rcuob/4]
        0  0.0      0 [rcuob/3]
        0  0.0      0 [rcuob/2]
        0  0.0      0 [rcuob/15]
        0  0.0      0 [rcuob/14]
        0  0.0      0 [rcuob/13]
        0  0.0      0 [rcuob/12]
        0  0.0      0 [rcuob/11]
        0  0.0      0 [rcuob/10]
        0  0.0      0 [rcuob/1]
        0  0.0      0 [rcuob/0]
        0  0.0      0 [rcu_bh]
        0  0.0      0 [phantomjs] <defunct>
        0  0.0      0 [phantomjs] <defunct>
        0  0.0      0 [phantomjs] <defunct>
        0  0.0      0 [phantomjs] <defunct>
        0  0.0      0 [perf]
        0  0.0      0 [netns]
        0  0.0      0 [migration/7]
        0  0.0      0 [migration/6]
        0  0.0      0 [migration/5]
        0  0.0      0 [migration/4]
        0  0.0      0 [migration/3]
        0  0.0      0 [migration/2]
        0  0.0      0 [migration/1]
        0  0.0      0 [migration/0]
        0  0.0      0 [md]
        0  0.0      0 [kworker/u32:2]
        0  0.0      0 [kworker/u32:1]
        0  0.0      0 [kworker/7:2]
        0  0.0      0 [kworker/7:1H]
        0  0.0      0 [kworker/7:0H]
        0  0.0      0 [kworker/7:0]
        0  0.0      0 [kworker/6:1H]
        0  0.0      0 [kworker/6:1]
        0  0.0      0 [kworker/6:0H]
        0  0.0      0 [kworker/6:0]
        0  0.0      0 [kworker/5:1H]
        0  0.0      0 [kworker/5:1]
        0  0.0      0 [kworker/5:0H]
        0  0.0      0 [kworker/5:0]
        0  0.0      0 [kworker/4:2H]
        0  0.0      0 [kworker/4:2]
        0  0.0      0 [kworker/4:1]
        0  0.0      0 [kworker/4:0H]
        0  0.0      0 [kworker/3:2H]
        0  0.0      0 [kworker/3:1H]
        0  0.0      0 [kworker/3:1]
        0  0.0      0 [kworker/3:0]
        0  0.0      0 [kworker/2:2H]
        0  0.0      0 [kworker/2:1H]
        0  0.0      0 [kworker/2:1]
        0  0.0      0 [kworker/2:0]
        0  0.0      0 [kworker/1:2]
        0  0.0      0 [kworker/1:1H]
        0  0.0      0 [kworker/1:0H]
        0  0.0      0 [kworker/1:0]
        0  0.0      0 [kworker/0:2]
        0  0.0      0 [kworker/0:1H]
        0  0.0      0 [kworker/0:0H]
        0  0.0      0 [kworker/0:0]
        0  0.0      0 [kthrotld]
        0  0.0      0 [kthreadd]
        0  0.0      0 [kswapd0]
        0  0.0      0 [ksoftirqd/7]
        0  0.0      0 [ksoftirqd/6]
        0  0.0      0 [ksoftirqd/5]
        0  0.0      0 [ksoftirqd/4]
        0  0.0      0 [ksoftirqd/3]
        0  0.0      0 [ksoftirqd/2]
        0  0.0      0 [ksoftirqd/1]
        0  0.0      0 [ksoftirqd/0]
        0  0.0      0 [ksmd]
        0  0.0      0 [kpsmoused]
        0  0.0      0 [kmpath_rdacd]
        0  0.0      0 [kipmi0]
        0  0.0      0 [kintegrityd]
        0  0.0      0 [khungtaskd]
        0  0.0      0 [khugepaged]
        0  0.0      0 [khelper]
        0  0.0      0 [kdmflush]
        0  0.0      0 [kdmflush]
        0  0.0      0 [kdmflush]
        0  0.0      0 [kdevtmpfs]
        0  0.0      0 [kblockd]
        0  0.0      0 [kauditd]
        0  0.0      0 [ipv6_addrconf]
        0  0.0      0 [fsnotify_mark]
        0  0.0      0 [events_power_ef]
        0  0.0      0 [edac-poller]
        0  0.0      0 [deferwq]
        0  0.0      0 [crypto]
        0  0.0      0 [bioset]
        0  0.0      0 [bioset]
        0  0.0      0 [bioset]
        0  0.0      0 [bioset]
        0  0.0      0 [ata_sff]
    [root@hadoop3 ~]# 
    

      

    http://www.cnblogs.com/JemBai/archive/2011/06/21/2086184.html

    ps - 按进程消耗内存多少排序

    ps -eo rss,pmem,pcpu,vsize,args |  sort -k 1 -r -n | less

    解析一下:

    ps 都知道,是linux,unix显示进程信息的, -e 是显示所有进程, -o是定制显示信息的格式

    rss: resident set size, 表示进程占用RAM(内存)的大小,单位是KB
    pmem: %M, 占用内存的百分比

    pcpu:%C,占用cpu的百分比

    vsize: 表示进程占用的虚拟内存的大小,KB

    args:进程名(command)

    sort命令对ps结果进行排序

    -k 1 :按第一个参数 rss进行排序

    -r:逆序

    -n:numeric,按数字来排序

     
     
     
     

    1:在命令行提示符执行top命令

    2:输入大写P,则结果按CPU占用降序排序。输入大写M,结果按内存占用降序排序。(注:大写P可以在capslock状态输入p,或者按Shift+p)

    ps -aux | grep phantomjs | cut -c 10-15 | xargs kill -9   

    手动杀死进程

    free内存变为2G,namenode可以启动,但datanode仍然不可以启动,此处说明“腾出内存”方向正确,进一步查内存消耗:已经不能通过手动杀死进程来实现腾出内存,故因为缓存占用大量内存考虑释放缓存。

  • 相关阅读:
    configure.ac:91: error: possibly undefined macro: AC_SEARCH_LIBS
    debian上安装tmux
    ssh: Bad configuration option: usedns
    mysql启动失败“MySQL Daemon failed to start”
    批量查询文件的编码格式
    mysql计算QPS
    网络资源汇总
    jQuery Mobile + HTML5
    配置SQL Server 2008 R2 Reporting Services
    分享一个C#创建Barcode的DLL
  • 原文地址:https://www.cnblogs.com/rsapaper/p/8126067.html
Copyright © 2020-2023  润新知