环境
192.168.1.101 host101
192.168.1.102 host102
1.安装配置host101
[root@host101 ~]# cat /etc/hosts |grep 192
192.168.1.101 host101
192.168.1.102 host102
[root@host101 ~]# rpm -ivh jdk-8u91-linux-x64.rpm
[root@host101 ~]# tar -zxvf hadoop-2.6.4.tar.gz
[root@host101 ~]# mv hadoop-2.6.4 /usr/local/hadoop
[root@host101 ~]# cd /usr/local/hadoop/
[root@host101 hadoop]# vim etc/hadoop/hadoop-env.sh
export JAVA_HOME=/usr/java/latest
export HADOOP_PREFIX=/usr/local/hadoop
[root@host101 hadoop]# vim etc/hadoop/slaves
host101
host102
[root@host101 hadoop]# vim etc/hadoop/core-site.xml
<configuration>
<property>
<name>fs.defaultFS</name>
<value>hdfs://host101:9000</value>
</property>
</configuration>
[root@host101 hadoop]# mkdir -p /hadoop/
[root@host101 hadoop]# vim etc/hadoop/hdfs-site.xml
<configuration>
<property>
<name>dfs.replication</name>
<value>1</value>
</property>
<property>
<name>dfs.namenode.name.dir</name>
<value>/hadoop/name/</value>
</property>
<property>
<name>dfs.datanode.data.dir</name>
<value>/hadoop/data/</value>
</property>
</configuration>
[root@host101 hadoop]# vim mapred-site.xml
<configuration>
<property>
<name>mapred.job.tracker</name>
<value>host101:9001</value>
</property>
</configuration>
[root@host101 ~]# ssh-keygen
[root@host101 ~]# ssh-copy-id host101
[root@host101 ~]# ssh-copy-id host102
2.安装配置host102
[root@host102 ~]# scp host101:/root/hadoop-2.6.4.tar.gz .
[root@host102 ~]# scp host101:/root/jdk-8u91-linux-x64.rpm .
[root@host102 ~]# rpm -ivh jdk-8u91-linux-x64.rpm
[root@host102 ~]# tar -zxvf hadoop-2.6.4.tar.gz
[root@host102 ~]# mv hadoop-2.6.4 /usr/local/hadoop
[root@host102 ~]# ssh-keygen
[root@host102 ~]# ssh-copy-id host101
[root@host102 ~]# ssh-copy-id host102
[root@host102 etc]# cd /usr/local/hadoop/etc/hadoop/
[root@host102 hadoop]# scp host101:/usr/local/hadoop/etc/hadoop/mapred-site.xml .
[root@host102 hadoop]# scp host101:/usr/local/hadoop/etc/hadoop/slaves .
[root@host102 hadoop]# scp host101:/usr/local/hadoop/etc/hadoop/hdfs-site.xml .
[root@host102 hadoop]# scp host101:/usr/local/hadoop/etc/hadoop/hadoop-env.sh .
[root@host102 hadoop]# scp host101:/usr/local/hadoop/etc/hadoop/core-site.xml .
3.启动hadoop集群
[root@host101 hadoop]# sbin/start-all.sh
This script is Deprecated. Instead use start-dfs.sh and start-yarn.sh
Starting namenodes on [host101]
host101: starting namenode, logging to /usr/local/hadoop/logs/hadoop-root-namenode-host101.out
host101: starting datanode, logging to /usr/local/hadoop/logs/hadoop-root-datanode-host101.out
host102: starting datanode, logging to /usr/local/hadoop/logs/hadoop-root-datanode-host102.out
Starting secondary namenodes [0.0.0.0]
0.0.0.0: starting secondarynamenode, logging to /usr/local/hadoop/logs/hadoop-root-secondarynamenode-host101.out
starting yarn daemons
starting resourcemanager, logging to /usr/local/hadoop/logs/yarn-root-resourcemanager-host101.out
host101: starting nodemanager, logging to /usr/local/hadoop/logs/yarn-root-nodemanager-host101.out
host102: starting nodemanager, logging to /usr/local/hadoop/logs/yarn-root-nodemanager-host102.out
[root@host101 hadoop]# bin/hdfs dfs -mkdir /eric
[root@host101 hadoop]# bin/hdfs dfs -ls /
Found 1 items
drwxr-xr-x - root supergroup 0 2016-07-06 12:09 /eric
[root@host101 hadoop]# bin/hadoop dfsadmin -report
DEPRECATED: Use of this script to execute hdfs command is deprecated.
Instead use the hdfs command for it.
Configured Capacity: 37576769536 (35.00 GB)
Present Capacity: 29447094272 (27.42 GB)
DFS Remaining: 29447086080 (27.42 GB)
DFS Used: 8192 (8 KB)
DFS Used%: 0.00%
Under replicated blocks: 0
Blocks with corrupt replicas: 0
Missing blocks: 0
-------------------------------------------------
Live datanodes (2):
Name: 192.168.1.101:50010 (host101)
Hostname: host101
Decommission Status : Normal
Configured Capacity: 18788384768 (17.50 GB)
DFS Used: 4096 (4 KB)
Non DFS Used: 3870842880 (3.61 GB)
DFS Remaining: 14917537792 (13.89 GB)
DFS Used%: 0.00%
DFS Remaining%: 79.40%
Configured Cache Capacity: 0 (0 B)
Cache Used: 0 (0 B)
Cache Remaining: 0 (0 B)
Cache Used%: 100.00%
Cache Remaining%: 0.00%
Xceivers: 1
Last contact: Wed Jul 06 12:10:07 CST 2016
Name: 192.168.1.102:50010 (host102)
Hostname: host102
Decommission Status : Normal
Configured Capacity: 18788384768 (17.50 GB)
DFS Used: 4096 (4 KB)
Non DFS Used: 4258832384 (3.97 GB)
DFS Remaining: 14529548288 (13.53 GB)
DFS Used%: 0.00%
DFS Remaining%: 77.33%
Configured Cache Capacity: 0 (0 B)
Cache Used: 0 (0 B)
Cache Remaining: 0 (0 B)
Cache Used%: 100.00%
Cache Remaining%: 0.00%
Xceivers: 1
Last contact: Wed Jul 06 12:10:07 CST 2016
[root@host101 hadoop]# jps
3920 DataNode
3811 NameNode
4056 SecondaryNameNode
4299 Jps
4. 测试集群
NameNode http://192.168.1.101:50070/dfshealth.html ResourceManager http://192.168.1.101:8088/cluster http://192.168.1.101:8042/node [root@host101 hadoop]# bin/hadoop fs -mkdir /eric/input [root@host101 hadoop]# bin/hadoop fs -copyFromLocal etc/hadoop/*.xml /eric/input [root@host101 hadoop]# bin/hadoop jar share/hadoop/mapreduce/hadoop-mapreduce-examples-2.6.4.jar grep /eric/input /eric/output 'dfs[a-z.]+' [root@host101 hadoop]# bin/hadoop fs -ls /eric/output/ Found 2 items -rw-r--r-- 1 root supergroup 0 2016-07-06 12:38 /eric/output/_SUCCESS -rw-r--r-- 1 root supergroup 77 2016-07-06 12:38 /eric/output/part-r-00000 [root@host101 hadoop]# bin/hadoop fs -cat /eric/output/part-r-00000 1 dfsadmin 1 dfs.replication 1 dfs.namenode.name.dir 1 dfs.datanode.data.dir [root@host101 hadoop]# sbin/stop-all.sh This script is Deprecated. Instead use stop-dfs.sh and stop-yarn.sh Stopping namenodes on [host101] host101: stopping namenode host101: stopping datanode host102: stopping datanode Stopping secondary namenodes [0.0.0.0] 0.0.0.0: stopping secondarynamenode stopping yarn daemons stopping resourcemanager host101: stopping nodemanager host102: no nodemanager to stop no proxyserver to stop
5. 动态添加节点
[root@host101 hadoop]# echo "192.168.1.161 host161" >> /etc/hosts [root@host102 hadoop]# echo "192.168.1.161 host161" >> /etc/hosts [root@host101 hadoop]# ssh-copy-id host161 [root@host102 hadoop]# ssh-copy-id host161 [root@host161 ~]# ssh-copy-id host161 [root@host161 ~]# ssh-copy-id host101 [root@host161 ~]# ssh-copy-id host102 [root@host102 ~]# scp host101:/root/hadoop-2.6.4.tar.gz . [root@host102 ~]# scp host101:/root/jdk-8u91-linux-x64.rpm . [root@host102 ~]# rpm -ivh jdk-8u91-linux-x64.rpm [root@host102 ~]# tar -zxvf hadoop-2.6.4.tar.gz [root@host102 ~]# mv hadoop-2.6.4 /usr/local/hadoop [root@host101 hadoop]# echo 'host161' >> etc/hadoop/slaves [root@host102 hadoop]# echo 'host161' >> etc/hadoop/slaves [root@host161 hadoop]# scp host101:/usr/local/hadoop/etc/hadoop/mapred-site.xml . [root@host161 hadoop]# scp host101:/usr/local/hadoop/etc/hadoop/slaves . [root@host161 hadoop]# scp host101:/usr/local/hadoop/etc/hadoop/hdfs-site.xml . [root@host161 hadoop]# scp host101:/usr/local/hadoop/etc/hadoop/hadoop-env.sh . [root@host161 hadoop]# scp host101:/usr/local/hadoop/etc/hadoop/core-site.xml . [root@host161 hadoop]# sbin/hadoop-daemon.sh start datanode starting datanode, logging to /usr/local/hadoop/logs/hadoop-root-datanode-host161.out [root@host101 hadoop]# bin/hadoop dfsadmin -report DEPRECATED: Use of this script to execute hdfs command is deprecated. Instead use the hdfs command for it. Configured Capacity: 56365154304 (52.49 GB) Present Capacity: 44354347008 (41.31 GB) DFS Remaining: 44192788480 (41.16 GB) DFS Used: 161558528 (154.07 MB) DFS Used%: 0.36% Under replicated blocks: 0 Blocks with corrupt replicas: 0 Missing blocks: 0 ------------------------------------------------- Live datanodes (3): Name: 192.168.1.101:50010 (host101) Hostname: host101 Decommission Status : Normal Configured Capacity: 18788384768 (17.50 GB) DFS Used: 161546240 (154.06 MB) Non DFS Used: 3873861632 (3.61 GB) DFS Remaining: 14752976896 (13.74 GB) DFS Used%: 0.86% DFS Remaining%: 78.52% Configured Cache Capacity: 0 (0 B) Cache Used: 0 (0 B) Cache Remaining: 0 (0 B) Cache Used%: 100.00% Cache Remaining%: 0.00% Xceivers: 1 Last contact: Wed Jul 06 16:02:19 CST 2016 Name: 192.168.1.161:50010 (host161) Hostname: host161 Decommission Status : Normal Configured Capacity: 18788384768 (17.50 GB) DFS Used: 4096 (4 KB) Non DFS Used: 3877494784 (3.61 GB) DFS Remaining: 14910885888 (13.89 GB) DFS Used%: 0.00% DFS Remaining%: 79.36% Configured Cache Capacity: 0 (0 B) Cache Used: 0 (0 B) Cache Remaining: 0 (0 B) Cache Used%: 100.00% Cache Remaining%: 0.00% Xceivers: 1 Last contact: Wed Jul 06 16:02:20 CST 2016 Name: 192.168.1.102:50010 (host102) Hostname: host102 Decommission Status : Normal Configured Capacity: 18788384768 (17.50 GB) DFS Used: 8192 (8 KB) Non DFS Used: 4259450880 (3.97 GB) DFS Remaining: 14528925696 (13.53 GB) DFS Used%: 0.00% DFS Remaining%: 77.33% Configured Cache Capacity: 0 (0 B) Cache Used: 0 (0 B) Cache Remaining: 0 (0 B) Cache Used%: 100.00% Cache Remaining%: 0.00% Xceivers: 1 Last contact: Wed Jul 06 16:02:19 CST 2016