Java来操作连接hdfs

本帖最后由 yuntian0215 于 2017-7-7 16:55 编辑

/**
* <p>内容描述：操作hadoop</p>
* @author lvjie
* @date 2017年7月7日上午11:53:50
*/
public class UseHadoop {
      //读取集群文件夹下的文件
      public void login(FileSystem fs) throws IOException{
            Path path =new Path("/log/20170329");
            FileStatus[] files=fs.listStatus(path);
            for(FileStatus file :files){
                     System.out.println(file.isDirectory());
                     System.out.println(file.getPath().getName());
                     System.out.println(file.getModificationTime());
                     System.out.println(file.getLen());
            }
      }
      /**
      * 创建文件
      * @throws Exception
      */
      public void mkFile(FileSystem fs)throws Exception{
            //在根目录下创建zs文件夹
            Path path =new Path("/zs");
            fs.mkdirs(path);
      }
      /**
      * 下载集群指定文件到本地
      * @throws Exception
      */
      public void open(FileSystem fs) throws Exception{
            //下载集群/user/root/input/kmeans.txt文件到D:/1111.txt
            Path path =new Path("/user/root/input/kmeans.txt");
            FSDataInputStream in = fs.open(path);
            FileUtils.copyInputStreamToFile(in, new File("D:/1111.txt"));
      }
      /**
      * 上传文件到集群
      * @throws Exception
      */
      public void upload(FileSystem fs)throws Exception{
            //上传D:/1111.txt到集群/user/root/output/1111.txt
            Path path =new Path("/user/root/output/1111.txt");
            FSDataOutputStream out = fs.create(path);
            FileUtils.copyFile(new File("D:/1111.txt"), out);
      }
      /**
      * 删除文件
      * @throws IOException
      */
      public void delFile(FileSystem fs) throws IOException{
            //删除集群zs文件
            Path dstPath = new Path("/zs");
            if(fs.exists(dstPath)){
                     fs.delete(dstPath, true) ;
                     }else{
                              return ;
                     }
      }

      public static void main(String[] args) {
            UseHadoop use = new UseHadoop();
            try {
                     FileSystem fs =null;
                     Configuration config =new Configuration();
                     config.set("fs.defaultFS", "hdfs://node1:9000");
                     fs =FileSystem.get(config);

                     //use.login(fs);
                     use.mkFile(fs);
                     fs.close();
            } catch (Exception e) {
                     // TODO Auto-generated catch block
                     e.printStackTrace();
            }
      }
}

当然jar包需要自己导入
我使用的maven:
       <dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-common</artifactId>
<version>2.5.2</version>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-hdfs</artifactId>
<version>2.5.2</version>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-client</artifactId>
<version>2.5.2</version>
</dependency>