Hadoop脚本:重新初始化所有节点

Hadoop脚本:重新初始化所有节点

在Hadoop集群调试的初期，经常需要重新初始化HDFS，下面的脚本可以实现此点

#!/usr/bin/php
<?php

require_once '/home/hadoop/bin/common/hadoop.inc';

$masters = f2a(HADOOP_HOME . '/conf/masters');
$slaves = f2a(HADOOP_HOME . '/conf/slaves');

foreach($masters as $master)
{
        echo "Resetting master $master...\n";
        rexec($master, "rm -rf ". HADOOP_NAME_DIR. "/* ");
}

foreach($slaves as $slave)
{
        echo "Resetting slave $slave ...\n";
        rexec($slave, "rm -rf " . HADOOP_TMP_DIR. "/* ". HADOOP_DATA_DIR."/*");

        echo "Clearing $slave logs...\n";
        rexec($slave, "rm -rf " . HADOOP_HOME. "/logs/*");
}

?>

Hadoop的配置如下:
<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>

<!-- Put site-specific property overrides in this file. -->

<configuration>
<property>
    <name>fs.default.name</name>
    <value>hdfs://master:9000</value>
</property>
<property>
    <name>mapred.job.tracker</name>
    <value>hdfs://master:9001</value>
</property>
<property>
    <name>dfs.name.dir</name>
    <value>/home/hadoop/name</value>
</property>
<property>
    <name>dfs.data.dir</name>
    <value>/home/hadoop/data</value>
</property>
<property>
    <name>dfs.replication</name>
    <value>2</value>
</property>
<property>
    <name>hadoop.tmp.dir</name>
    <value>/home/hadoop/tmp/</value>
</property>
<property>
    <name>dfs.block.size</name>
    <value>67108864</value>
</property>
</configuration>

Hadoop.inc内容如下：
<?php

define('HADOOP_HOME', '/home/hadoop/hadoop-0.18.1');
define('HADOOP_TMP_DIR','/home/hadoop/tmp');
define('HADOOP_NAME_DIR','/home/hadoop/name');
define('HADOOP_DATA_DIR','/home/hadoop/data');

function rexec($host, $cmd)
{
        $connection = ssh2_connect($host, 22);
        // ssh2_auth_password($connection, 'username', 'password');
        if (ssh2_auth_pubkey_file($connection, 'hadoop',
                          '/home/hadoop/.ssh/id_dsa.pub',
                          '/home/hadoop/.ssh/id_dsa')) {
                 // echo "Public Key Authentication Successful\n";
        } else {
                echo("Accessing $host: Public Key Authentication Failed\n");
        return '';
        }
        $stream = ssh2_exec($connection, $cmd);
        /*
        while($line = fgets($stream)) {
                flush();
                echo $line."<br />";
        }
        */
        stream_set_blocking($stream, true);
        return stream_get_contents($stream);
}

function ping_host($host)
{
    $connection = @ssh2_connect($host, 22);
        // ssh2_auth_password($connection, 'username', 'password');
    if( $connection === false)
        return false;
        if (@ssh2_auth_pubkey_file($connection, 'hadoop',
                          '/home/hadoop/.ssh/id_dsa.pub',
                          '/home/hadoop/.ssh/id_dsa')) {
                 // echo "Public Key Authentication Successful\n";
        return true;
        } else {
                // echo("Accessing $host: Public Key Authentication Failed\n");
                return false;
        }
}

function f2a( $filename )
{
    return file( $filename, FILE_IGNORE_NEW_LINES | FILE_SKIP_EMPTY_LINES);
}

?>