最终效果图
目录树实体类:
/**
* @Author: DaleyZou
* @Description: hdfs 目录结构的实体类,用于展示目录树的支撑操作
* @Date: Created in 10:58 2018-9-27
* @Modified By:
*/
public class HdfsFolder {
private String id; // 目录节点的id
private String name; // 目录节点的名字
private List<HdfsFolder> children; // 此时这个目录下的孩子节点
set get 方法
}
定义所有方法共享常量:
// hdfs路径
private static String url = "hdfs://XXX:8020";
// 要读取的目录在hdfs下的相对路径
private static String USERFILEPATH = "/user/XXX";
初始化FileSystem对象
/**
* 初始化FileSystem对象便于操作文件
*
* @return
*/
public static FileSystem getFileSystem() throws IOException, URISyntaxException {
//读取配置文件
Configuration conf = new Configuration();
conf.set("fs.hdfs.impl",org.apache.hadoop.hdfs.DistributedFileSystem.class.getName());
//文件系统
FileSystem fs = null;
if (org.apache.commons.lang.StringUtils.isBlank(url)) {
//返回默认文件系统,如果在hadoop集群下运行,使用此种方法可直接获取默认文件系统;
fs = FileSystem.get(conf);
} else {
//返回指定的文件系统,如果在本地测试,需要此种方法获取文件系统;
URI uri = new URI(url.trim());
fs = FileSystem.get(uri, conf);
}
return fs;
}
根据路径递归读取 hdfs目录
public static void listFolderByCondition(String folder, HdfsFolder hdfsFolder) throws Exception{
FileSystem fs = HdfsUtils.getFileSystem();
FileStatus[] filesStatus = fs.listStatus(new Path(folder));
List<HdfsFolder> children = new ArrayList<>();
for (FileStatus file : filesStatus) {
if (file.isFile()){
HdfsFolder temp = new HdfsFolder();
temp.setId(folder + "/" + file.getPath().getName());
temp.setName(file.getPath().getName());
children.add(temp);
}
if (file.isDirectory()){
HdfsFolder temp = new HdfsFolder();
temp.setId(folder + "/" + file.getPath().getName());
temp.setName(file.getPath().getName());
listFolderByCondition(folder + "/" + file.getPath().getName(), temp);
children.add(temp);
}
}
hdfsFolder.setChildren(children);
}
/**
* @Description: 根据路径递归读取 hdfs目录
* @date: 17:15 2018-9-27
* @param: folder 需要读取目录的hdfs路径
* @return: hdfs目录信息
*/
public static HdfsFolder listFolder(String folder)
throws Exception {
FileSystem fs = HdfsUtils.getFileSystem();
FileStatus[] filesStatus = fs.listStatus(new Path(folder));
HdfsFolder hdfsFolder = new HdfsFolder(); // 目录结果
hdfsFolder.setId(folder);
hdfsFolder.setName("根目录");
List<HdfsFolder> children = new ArrayList<>();
for (FileStatus file : filesStatus) {
if (file.isFile()){
HdfsFolder temp = new HdfsFolder();
temp.setId(folder + "/" + file.getPath().getName());
temp.setName(file.getPath().getName());
children.add(temp);
}
if (file.isDirectory()){
HdfsFolder temp = new HdfsFolder();
temp.setId(folder + "/" + file.getPath().getName());
temp.setName(file.getPath().getName());
listFolderByCondition(folder + "/" + file.getPath().getName(), temp);
children.add(temp);
}
}
hdfsFolder.setChildren(children);
return hdfsFolder;
}
根据文件全路径读取文件的内容
/**
* @Description: 根据文件全路径读取文件的内容
* @date: 10:36 2018-9-28
* @param: fileName 文件全路径
* @return: 文件内容
*/
public static String checkFile(String fileName)
throws Exception {
FileSystem fs = getFileSystem();
FSDataInputStream inStream = fs.open(new Path(fileName));
BufferedReader br = new BufferedReader(new InputStreamReader(inStream, "UTf-8"));
StringBuffer buffer = new StringBuffer();
try {
String line;
line = br.readLine();
while (line != null) {
buffer.append(line).append("<br>");
line = br.readLine();
}
} finally {
br.close();
inStream.close();
}
return buffer.toString();
}