hdfs工具类

hdfs：

  Hadoop分布式文件系统(HDFS)是指被设计成适合运行在通用硬件(commodity hardware)上的分布式文件系统（Distributed File System）。它和现有的分布式文件系统有很多共同点。但同时，它和其他的分布式文件系统的区别也是很明显的。HDFS是一个高度容错性的系统，适合部署在廉价的机器上。HDFS能提供高吞吐量的数据访问，非常适合大规模数据集上的应用。HDFS放宽了一部分POSIX约束，来实现流式读取文件系统数据的目的。HDFS在最开始是作为Apache Nutch搜索引擎项目的基础架构而开发的。HDFS是Apache Hadoop Core项目的一部分。

Java操作工具类：

import lombok.RequiredArgsConstructor;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.*;
import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.util.Progressable;
import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
import org.springframework.stereotype.Component;

import java.io.*;
import java.net.URI;
import java.net.URISyntaxException;

/**
 * hdfs工具
 */
@Component
@RequiredArgsConstructor
public class HdfsUtils {

    //配置链接虚拟机的IP
    public static final String HDFS_PATH = "hdfs://192.168.10.150:8020";
    //hdfs文件系统
    FileSystem fileSystem = null;
    //获取环境对象
    Configuration configuration = null;

    /**
     * 新建目录
     *
     * @throws IOException
     */
    @Test
    public void mkdir() throws IOException {
        fileSystem.mkdirs(new Path("/user/filestore/history/2021/10/11/"));
    }
    //hadoop fs -ls /hdfsapi/test

    /**
     * 创建文件
     *
     * @throws IOException
     */
    @Test
    public void create() throws IOException {
        //创建文件
        FSDataOutputStream outputStream = fileSystem.create(new Path("/user/filestore/test/l.txt"));
        outputStream.write("hello hadoop".getBytes());
        outputStream.flush();
        outputStream.close();
    }

    /**
     * 打印
     *
     * @throws IOException
     */
    @Test
    public void cat() throws IOException {
        FSDataInputStream inputStream = fileSystem.open(new Path("/user/filestore/test/b.txt"));
        IOUtils.copyBytes(inputStream, System.out, 1024);
        inputStream.close();
    }

    /**
     * 重命名
     *
     * @throws IOException
     */
    @Test
    public void rename() throws IOException {
        Path oldPath = new Path("/user/filestore/current/22/76/33/t_file_history.sql");
        Path newPath = new Path("/user/filestore/history/2021/10/11/t_file_history.sql");
        Assert.assertTrue(fileSystem.rename(oldPath, newPath));
    }

    /**
     * 上传本地文件到hdfs
     *
     * @throws Exception
     */
    @Test
    public void copyFromLocalFile() throws Exception {
        Path oldPath = new Path("D:\file/aa/20210908_0600.png");
        Path newPath = new Path("/user/filestore/filehistory/2021");
        fileSystem.copyFromLocalFile(oldPath, newPath);
    }

    /**
     * 上传本地文件到hdfs 带进度条
     *
     * @throws Exception
     */
    @Test
    public void copyFromLocalFileWithProgress() throws Exception {
//        Path oldPath = new Path("E:\ideaIU-2018.1.exe");
//        Path newPath = new Path("/user/filestore/test");
//        fileSystem.copyFromLocalFile(oldPath,newPath);

        InputStream in = new BufferedInputStream(
                new FileInputStream(
                        new File("E:\文档.txt")));

        FSDataOutputStream outputStream = fileSystem.create(new Path("/user/filestore/test/文档.txt"), new Progressable() {
            @Override
            public void progress() {
                System.out.print(".");
                //带进度提醒信息
            }
        });

        IOUtils.copyBytes(in, outputStream, 4096);

    }

    /**
     * 下载文件到本地
     *
     * @throws Exception
     */
    @Test
    public void copyToLocalFile() throws Exception {
//        获取输入流
        InputStream in = fileSystem.open(new Path("/user/filestore/filehistory/2021/20210908_0600.png"));
//        获取输出流
        OutputStream outputStream = new FileOutputStream(new File("D:\file/a.png"));
        IOUtils.copyBytes(in, outputStream, configuration);
        in.close();
        outputStream.close();
    }

    /**
     * 列出所有的文件
     *
     * @throws URISyntaxException
     * @throws IOException
     */
    @Test
    public void listFiles() throws Exception {
        FileStatus[] fileStatuses = fileSystem.listStatus(new Path("/user"));
        for (FileStatus fileStatus : fileStatuses) {
            String isDir = fileStatus.isDirectory() ? "文件夹" : "文件";
            short relication = fileStatus.getReplication();
            long len = fileStatus.getLen();
            String path = fileStatus.getPath().toString();
            System.out.println(isDir + ":" + relication + ":" + len + ":" + path);
        }
    }


    @Test
    public void upset() throws URISyntaxException, IOException {
        //上传文件,路径大家记得改一下
        String file = "D:/hadoop/output/test.txt";
        InputStream inputStream = new FileInputStream(new File(file));
        FSDataOutputStream outputStream = fileSystem.create(new Path("/user/filestore/park/aaa.txt"));
        IOUtils.copyBytes(inputStream, outputStream, configuration);
//        fileSystem.copyFromLocalFile();底层是调用了IOUtils.copyBytes()
    }

    @Test
    public void download() throws URISyntaxException, IOException {
//        获取输入流
        InputStream in = fileSystem.open(new Path("/park/2.txt"));
//        获取输出流
        String file = "D:/hadoop/output/test.txt";
        OutputStream outputStream = new FileOutputStream(new File(file));
        IOUtils.copyBytes(in, outputStream, configuration);
        in.close();
        outputStream.close();
    }


    /**
     * 删除hdfs文件
     *
     * @throws Exception
     */
    @Test
    public void deleteFile() throws Exception {
        Path path = new Path("/user/filestore/history");
        fileSystem.delete(path, true);
    }

    @Test
    public void demo1() throws URISyntaxException, IOException {
        configuration = new Configuration();
        fileSystem = (FileSystem) FileSystem.get(new URI(HDFS_PATH), configuration);
//        1、在hdfs创建目录teacher。
//        2、在teacher目录下上传文件score.txt。
        String file = "D:/hadoop/score.txt";
        InputStream inputStream = new FileInputStream(new File(file));
        OutputStream outputStream = fileSystem.create(new Path("/hdfs/teacher/score.txt"));
        IOUtils.copyBytes(inputStream, outputStream, configuration);
//        3、在hdfs创建目录student，并在student目录下创建新目录Tom、LiMing、Jerry.
        fileSystem.mkdirs(new Path("/hdfs/student/Tom"));
        fileSystem.mkdirs(new Path("/hdfs/student/LiMing"));
        fileSystem.mkdirs(new Path("/hdfs/student/Jerry"));
//        4、在Tom目录下上传information.txt,同时上传到LiMing、Jerry目录下。
        file = "D:/hadoop/information.txt";
        inputStream = new FileInputStream(new File(file));
        outputStream = fileSystem.create(new Path("/hdfs/student/Tom/information.txt"));
        IOUtils.copyBytes(inputStream, outputStream, configuration);
//        file = "D:/hadoop/information.txt";
        inputStream = new FileInputStream(new File(file));

        outputStream = fileSystem.create(new Path("/hdfs/student/LiMing/information.txt"));
        IOUtils.copyBytes(inputStream, outputStream, configuration);
//        file = "D:/hadoop/information.txt";
        inputStream = new FileInputStream(new File(file));

        outputStream = fileSystem.create(new Path("/hdfs/student/Jerry/information.txt"));
        IOUtils.copyBytes(inputStream, outputStream, configuration);
//        5、将student重命名为MyStudent。
        fileSystem.rename(new Path("/hdfs/student"), new Path("/hdfs/MyStudent"));
//        6、将Tom下的information.txt下载到E:/tom目录中
        file = "E:/tom";
        inputStream = fileSystem.open(new Path("/hdfs/MyStudent/Tom/information.txt"));
        outputStream = new FileOutputStream(new File(file));
        IOUtils.copyBytes(inputStream, outputStream, configuration);
//        7、将teacher下的score.txt也下载到此目录
        inputStream = fileSystem.open(new Path("/hdfs/teacher/score.txt"));
        outputStream = new FileOutputStream(new File(file));
        IOUtils.copyBytes(inputStream, outputStream, configuration);
//        8、删除hdfs中的Tom、LiMing目录
        fileSystem.delete(new Path("/hdfs/Tom"), true);
        fileSystem.delete(new Path("/hdfs/LiMing"), true);
        inputStream.close();
        outputStream.close();
    }


    @Before
    public void setUp() throws Exception {
        configuration = new Configuration();
        fileSystem = FileSystem.get(new URI(HDFS_PATH), configuration, "root");

        System.out.println("HDFSApp setUp");

    }

    @After
    public void tearDown() throws Exception {
        configuration = null;
        fileSystem = null;

        System.out.println("HDFSApp tearDown");
    }

}