问问：通过JAVA 的API读取HDFS上的文件报错、、

如题：小弟通过 eclipse写了一段代码来读取自己的虚拟机上的伪分布的hadoop 集群的文件，不知为何一直报错请大神们看看

[mw_shl_code=java,true]package com.hadoop.hdfs;

import java.io.IOException;
import java.io.InputStream;
import java.net.URL;

import org.apache.hadoop.fs.FsUrlStreamHandlerFactory;
import org.apache.hadoop.io.IOUtils;

public class hdfsurltest {

      //让java程序识别hdfs文件系统的URL
      static {
            URL.setURLStreamHandlerFactory(new FsUrlStreamHandlerFactory());
      }

      public static void main(String[] args) {

            InputStream is = null;

            try{
                  String path = "hdfs://192.168.1.52:8020/user/data/one/one.txt";
                     URL fileUrl = new URL(path);

                     is = fileUrl.openStream();

                     IOUtils.copyBytes(is,System.out,4096,false);


            }catch(IOException e){
                     e.printStackTrace();
            }finally{
                     IOUtils.closeStream(is);
            }

      }
}[/mw_shl_code]

报错来啦：：：：
java.net.MalformedURLException: unknown protocol: hdfs
      at java.net.URL.<init>(Unknown Source)
      at java.net.URL.<init>(Unknown Source)
      at java.net.URL.<init>(Unknown Source)
      at com.beifeng.hadoop.hdfs.hdfsurltest.main(hdfsurltest.java:25)

这是哪里没写好呢，求老司机们指点指点，再次感激不尽！！！

qcbb001 · 发表于 2016-12-13 09:10:50

URL fileUrl = new URL(path);
这里是不对的，hadoop不识别url的，应该是用URI。
楼主可以找到对应版本的操作。

下面是一个例子，基于hadoop-0.20.2，
[mw_shl_code=java,true]package cn.cvu.hdfs;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;

public class Query {

      public static void main(String[] args) throws Exception {
            Configuration conf = new Configuration();
            conf.addResource(new Path("/home/hm/hadoop-1.1.2/conf/core-site.xml"));
            FileSystem fs = FileSystem.get(conf);
            Path path = new Path("/test.txt");
            FSDataInputStream is = fs.open(path);
            FileStatus stat = fs.getFileStatus(path);
            byte[] buffer = new byte[Integer.parseInt(String.valueOf(stat.getLen()))];
            is.readFully(0, buffer);
            is.close();
            fs.close();
            System.out.println(new String(buffer));
      }
}[/mw_shl_code]

easthome001 · 发表于 2016-12-13 09:26:43

运行这个试试
[mw_shl_code=java,true]import java.io.InputStream;

import java.net.URI;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.*;
import org.apache.hadoop.io.IOUtils;

publicclass FileSystemCat {
publicstaticvoid main(String[] args) throws Exception {
      String uri ="hdfs://192.168.1.52:8020/user/data/one/one.txt";
      Configuration conf = new Configuration();
FileSystem fs = FileSystem. get(URI.create (uri), conf);
InputStream in = null;
try {
      in = fs.open( new Path(uri));
      IOUtils.copyBytes(in, System.out, 4096, false);
      } finally {
      IOUtils.closeStream(in);
      }
}
}[/mw_shl_code]