Win10 IDEA连接假造机中的Hadoop（HDFS） - IT评测·应用市场-qidao123.com

ip a

复制代码

sudo ufw disable

复制代码

# 位置可能不一样，和Hadoop安装位置有关
cd /usr/local/hadoop/etc/hadoop
vim core-site.xml

复制代码

cd /usr/local/hadoop/ #目录可能不一样，修改成自己的目录
./sbin/stop-dfs.sh # 关闭hadoop
./sbin/start-dfs.sh #启动hadoop
jps # 判断是否启动成功

复制代码

<properties>
<hadoop.version>3.3.5</hadoop.version>
</properties>
<dependencies>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-common</artifactId>
<version>${hadoop.version}</version>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-hdfs</artifactId>
<version>${hadoop.version}</version>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-client</artifactId>
<version>${hadoop.version}</version>
</dependency>
</dependencies>

复制代码

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.*;
import java.io.IOException;
public class Test01 {
public static void main(String[] args) throws IOException {
Configuration conf = new Configuration();
// 设置用户名（一定要，不然默认用户名是win的用户名）
System.setProperty("HADOOP_USER_NAME","hadoop");
// IP地址修改成虚拟机的ip
conf.set("fs.defaultFS","hdfs://192.168.111.131:9000");
conf.set("fs.hdfs.impl","org.apache.hadoop.hdfs.DistributedFileSystem");
FileSystem fs = FileSystem.get(conf);
RemoteIterator<LocatedFileStatus> files = fs.listFiles(new Path("/"), true);
while (files.hasNext()) {
FileStatus fileStatus = files.next();
System.out.println(fileStatus.getPath().toString());
}
fs.close(); //关闭hdfs
}
}

复制代码

@REM 设置IP
SET BigDataLANIP=192.168.111.131
@REM 设置命令以管理员身份运行
%1 start "" mshta vbscript:CreateObject("Shell.Application").ShellExecute("cmd.exe","/c %~s0 ::","","runas",1)(window.close)&&exit
@REM 清空所有转发规则
netsh interface portproxy reset
@REM 转发9000
netsh interface portproxy add v4tov4 listenport=9000 connectport=9000 connectaddress=%BigDataLANIP%
@REM 转发9870（HDFS的web管理界面）
netsh interface portproxy add v4tov4 listenport=9870 connectport=9870 connectaddress=%BigDataLANIP%
echo "succeed"
timeout /t 5 /nobreak >nul

复制代码

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.*;
import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStreamReader;
public class Test02 {
public static void main(String[] args) throws IOException {
Configuration conf = new Configuration();
// 设置用户名（一定要，不然默认用户名是win的用户名）
System.setProperty("HADOOP_USER_NAME","hadoop");
conf.set("fs.defaultFS","hdfs://localhost:9000");
conf.set("fs.hdfs.impl","org.apache.hadoop.hdfs.DistributedFileSystem");
// 列出根目录下的所有文件和文件夹
FileSystem fs = FileSystem.get(conf);
Path file = new Path("/");
FileStatus[] fileStatuses = fs.listStatus(file);
for (FileStatus fileStatus : fileStatuses){
System.out.println(fileStatus.getPath());
}
// 创建一个新的文件 test.txt 在HDFS的 /user/hadoop/test 目录下（如果目录不存在，则先创建目录）。
Path dirPath = new Path("/user/hadoop/test");
if(!fs.exists(dirPath)){
fs.mkdirs(dirPath);
}
Path remotePath = new Path("/user/hadoop/test/test.txt");
FSDataOutputStream outputStream = fs.create(remotePath);
outputStream.close();
// 向 test.txt 文件中写入一段指定的文本内容（如“Hello, HDFS!”）。
FSDataOutputStream outputStream2 = fs.create(remotePath);
String s = "Hello, HDFS!";
outputStream2.write(s.getBytes());
outputStream2.close();
// 读取 test.txt 文件的内容，并打印到控制台。
FSDataInputStream inputStream = fs.open(remotePath);
BufferedReader d = new BufferedReader(new InputStreamReader(inputStream));
String line = null;
while ((line = d.readLine()) != null)
System.out.println(line);
// 关闭与HDFS的连接。
fs.close();
}
}

复制代码