Hadoop编程实现之HDFS

HDFS原理图：

下面我们来写一个基于HDFS的demo，该demo主要实现的是将HDFS上的一个文件内容读取出来并保存到另一个文件上的功能。

1.辅助类

这个类主要是用来获取hdfs文件系统连接的

public class HdfsUtils {
    
    /**
     * @return
     * @throws Exception
     */
    public static FileSystem getFileSystem() throws Exception{
        
        Configuration conf = new Configuration() ;
        conf.set("fs.defaultFS", "hdfs://192.168.1.109:8020");
        
        FileSystem fileSystem = FileSystem.get(conf) ;
        
        return fileSystem ;
    }
    /**
     * @param pOpenUri
     * @param pUser
     * @return
     * @throws Exception
     * @throws InterruptedException
     * @throws URISyntaxException
     */
    public static FileSystem getFileSystemByUser(String pOpenUri,String pUser) throws Exception, InterruptedException, URISyntaxException{
        
        Configuration conf = new Configuration() ;
        conf.set("fs.defaultFS", "hdfs://192.168.1.109:8020");
        
        FileSystem fileSystem = FileSystem.get(new URI(pOpenUri), conf, pUser) ;
        
        return fileSystem ;
        
    }
    
    /**
     * @param pUser
     * @return
     * @throws Exception
     * @throws InterruptedException
     * @throws URISyntaxException
     */
    public static FileSystem getFileSystemByUser(String pUser) throws Exception, InterruptedException, URISyntaxException{
        
        String fileUri = "/home/test/test.txt" ;
        
        Configuration conf = new Configuration() ;
        conf.set("fs.defaultFS", "hdfs://192.168.1.109:8020");
        
        FileSystem fileSystem = FileSystem.get(new URI(fileUri), conf, pUser) ;
        
        return fileSystem ;
        
    }
    
    

}

2.主类

这个类主要是用来进行文件读写和创建的

public class HdfsFsTest {

    public static void main(String[] args) {

        String fileUri = "/home/test/test.txt";
        String fileOutputUrl = "/home/test/out.txt";
        try {

            writeFileToHdfs(fileUri, fileOutputUrl);
            System.out.println("DONE!");

        } catch (Exception e) {

            e.printStackTrace();
        }

    }

    public static void writeFileToHdfs(String pOpenUri, String pOutputUrl)
            throws Exception {

        FileSystem fileSystem = null;
        FSDataInputStream fileInputStream = null;
        FSDataOutputStream fileOutputStream = null;
        int buffSize = 4096;

        try {

            fileSystem = HdfsUtils.getFileSystem();

            fileInputStream = fileSystem.open(new Path(pOpenUri));
            fileOutputStream = fileSystem.create(new Path(pOutputUrl));
            IOUtils.copyBytes(fileInputStream, fileOutputStream, buffSize,
                    false);

        } catch (Exception e) {
            throw e;
        } finally {
            IOUtils.closeStream(fileInputStream);

            IOUtils.closeStream(fileOutputStream);
        }

    }

}

3.运行结果

运行成功！