HDFS (Hadoop Distributed File System)
分布式存储系统
支持海量数据的存储,成百上千的计算机组成存储集群,HDFS可以运行在低成本的硬件之上,具有的高容错、高可靠性、高可扩展性、高吞吐率等特征,非常适合大规模数据集上的应用。
优点
- 高容错性
- 适合批处理
- 适合大数据处理
- 流式文件访问
- 可构建在廉价机器上
缺点
- 不适合低延迟数据访问
- 不适合小文件存取
- 不适合并发写入、文件随机修改
HDFS操作
命令操作HDFS
# 显示目录 / 下的文件
hdfs dfs -ls /
# 新建文件夹,绝对路径
hdfs dfs -mkdir /test
# 上传文件
hdfs dfs -put test.txt /test/
# 下载文件
hdfs dfs -get /test/test.txt
# 输出文件内容
hdfs dfs -cat /test/test.txt
Web端操作HDFS
打开http://192.168.9.200:9870/,可以对HDFS进行操作
文件在这里管理
可视化操作还是简单一些
Java操作HDFS
- 添加依赖
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-client</artifactId>
<version>3.3.2</version>
</dependency>
- 建立连接
public void setUp() throws Exception {
System.out.println("开始建立与HDFS的连接");
configuration = new Configuration();
fileSystem = FileSystem.get(new URI(HDFS_PATH), configuration, "hadoop");
}
- 文件操作
Configuration configuration = null;
FileSystem fileSystem = null;
public static final String HDFS_PATH = "hdfs://192.168.9.200:9000";
/**
* 在 hdfs中新建文件夹
*
* @throws Exception
*/
@Test
public void mkdir() throws Exception {
fileSystem.mkdirs(new Path("/JavaDemo/test"));
}
/**
* 创建文件
*
* @throws Exception
*/
@Test
public void create() throws Exception {
FSDataOutputStream outputStream = fileSystem.create(new Path("/JavaDemo/test/haha.txt"));
outputStream.write("hello bigdata from javaDemo".getBytes());
outputStream.flush();
outputStream.close();
}
/**
* 查看文件 hdfs -fs -cat file
*
* @throws Exception
*/
@Test
public void cat() throws Exception {
FSDataInputStream in = fileSystem.open(new Path("/JavaDemo/test/haha.txt"));
IOUtils.copyBytes(in, System.out, 1024);
in.close();
}
/**
* 重命名文件
*
* @throws Exception
*/
@Test
public void rename() throws Exception {
Path oldPath = new Path("/JavaDemo/test/haha.txt");
Path newPath = new Path("/JavaDemo/test/hehe.txt");
fileSystem.rename(oldPath, newPath);
}
/**
* 上传文件到HDFS
*
* @throws Exception
*/
@Test
public void copyFromLocalFile() throws Exception {
Path loacalPath = new Path("hello.txt");
Path hdfsPath = new Path("/");
fileSystem.copyFromLocalFile(loacalPath, hdfsPath);
}
/**
* 上传文件到HDFS带进度信息
*
* @throws Exception
*/
@Test
public void copyFromLocalFileWithProgress() throws Exception {
InputStream in = new BufferedInputStream(Files.newInputStream(new File("hbase-2.2.7-bin.tar.gz").toPath()));
FSDataOutputStream ouput = fileSystem.create(new Path("/JavaDemo/test/hbase-2.2.7-bin.tar.gz"), () -> {
System.out.print(".");
});
IOUtils.copyBytes(in, ouput, 4096);
}
/**
* 下载文件到HDFS
*
* @throws Exception
*/
@Test
public void copyToLocalFile() throws Exception {
Path hdfsPath = new Path("/JavaDemo/test/haha.txt");
Path loacalPath = new Path("./haha.txt");
// useRawLocalFileSystem
fileSystem.copyToLocalFile(false, hdfsPath, loacalPath, true);
}
/**
* 查看某个目录下所有文件
*
* @throws Exception
*/
@Test
public void listFiles() throws Exception {
FileStatus[] fileStatuses = fileSystem.listStatus(new Path("/"));
for (FileStatus f : fileStatuses) {
String isDir = f.isDirectory() ? "文件夹" : "文件";
short replication = f.getReplication();
long len = f.getLen();
String path = f.getPath().toString();
System.out.println(isDir + "\t" + replication + "\t" + len + "\t" + path);
}
}
/**
* 删除文件
*
* @throws Exception
*/
@Test
public void delete() throws IOException {
fileSystem.delete(new Path("/JavaDemo/haha.txt"), true);
}
- 关闭连接
public void tearDown() {
configuration = null;
fileSystem = null;
System.out.println("关闭与HDFS的连接");
}
完整测试文件文章来源:https://www.toymoban.com/news/detail-468973.html
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.*;
import org.apache.hadoop.io.IOUtils;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
import java.io.*;
import java.net.URI;
import java.nio.file.Files;
/**
* @author Gettler
* Java 操作HDFS
*/
public class HdfsDemo {
Configuration configuration = null;
FileSystem fileSystem = null;
public static final String HDFS_PATH = "hdfs://192.168.9.200:9000";
/**
* 在 hdfs中新建文件夹
*
* @throws Exception
*/
@Test
public void mkdir() throws Exception {
fileSystem.mkdirs(new Path("/JavaDemo/test"));
}
/**
* 创建文件
*
* @throws Exception
*/
@Test
public void create() throws Exception {
FSDataOutputStream outputStream = fileSystem.create(new Path("/JavaDemo/test/haha.txt"));
outputStream.write("hello bigdata from javaDemo".getBytes());
outputStream.flush();
outputStream.close();
}
/**
* 查看文件 hdfs -fs -cat file
*
* @throws Exception
*/
@Test
public void cat() throws Exception {
FSDataInputStream in = fileSystem.open(new Path("/JavaDemo/test/haha.txt"));
IOUtils.copyBytes(in, System.out, 1024);
in.close();
}
/**
* 重命名文件
*
* @throws Exception
*/
@Test
public void rename() throws Exception {
Path oldPath = new Path("/JavaDemo/test/haha.txt");
Path newPath = new Path("/JavaDemo/test/hehe.txt");
fileSystem.rename(oldPath, newPath);
}
/**
* 上传文件到HDFS
*
* @throws Exception
*/
@Test
public void copyFromLocalFile() throws Exception {
Path loacalPath = new Path("hello.txt");
Path hdfsPath = new Path("/");
fileSystem.copyFromLocalFile(loacalPath, hdfsPath);
}
/**
* 上传文件到HDFS带进度信息
*
* @throws Exception
*/
@Test
public void copyFromLocalFileWithProgress() throws Exception {
InputStream in = new BufferedInputStream(Files.newInputStream(new File("hbase-2.2.7-bin.tar.gz").toPath()));
FSDataOutputStream ouput = fileSystem.create(new Path("/JavaDemo/test/hbase-2.2.7-bin.tar.gz"), () -> {
System.out.print(".");
});
IOUtils.copyBytes(in, ouput, 4096);
}
/**
* 下载文件到HDFS
*
* @throws Exception
*/
@Test
public void copyToLocalFile() throws Exception {
Path hdfsPath = new Path("/JavaDemo/test/haha.txt");
Path loacalPath = new Path("./haha.txt");
// useRawLocalFileSystem
fileSystem.copyToLocalFile(false, hdfsPath, loacalPath, true);
}
/**
* 查看某个目录下所有文件
*
* @throws Exception
*/
@Test
public void listFiles() throws Exception {
FileStatus[] fileStatuses = fileSystem.listStatus(new Path("/"));
for (FileStatus f : fileStatuses) {
String isDir = f.isDirectory() ? "文件夹" : "文件";
short replication = f.getReplication();
long len = f.getLen();
String path = f.getPath().toString();
System.out.println(isDir + "\t" + replication + "\t" + len + "\t" + path);
}
}
/**
* 删除文件
*
* @throws Exception
*/
@Test
public void delete() throws IOException {
fileSystem.delete(new Path("/JavaDemo/haha.txt"), true);
}
//测试之前执行的代码
@Before
public void setUp() throws Exception {
System.out.println("开始建立与HDFS的连接");
configuration = new Configuration();
fileSystem = FileSystem.get(new URI(HDFS_PATH), configuration, "hadoop");
}
//测试之完执行的代码
@After
public void tearDown() {
configuration = null;
fileSystem = null;
System.out.println("关闭与HDFS的连接");
}
}
9999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999ssssssssss999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999ssssssssss999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999ssssssssss999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999ssssssssss999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999ssssssssss99999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999文章来源地址https://www.toymoban.com/news/detail-468973.html
到了这里,关于Hadoop学习笔记之HDFS的文章就介绍完了。如果您还想了解更多内容,请在右上角搜索TOY模板网以前的文章或继续浏览下面的相关文章,希望大家以后多多支持TOY模板网!