最近在写一个pagerank计算的MapReduce程序,可是MapReduceDriver配置是出现了点,从输出来看Reduce没有运行,不知是啥原因,请高手指教输入文件格式是
url0 55url1;url2;url3;...
....
输出文件格式应当同输入文件格式
但是现在的输出文件格式是Mapper输出的格式
MapReduceDriver代码如下,其他部分代码见附件
[ol][*]import java.io.IOException;
[*]
[*]import org.apache.hadoop.conf.Configuration;
[*]import org.apache.hadoop.conf.Configured;
[*]import org.apache.hadoop.fs.FileSystem;
[*]import org.apache.hadoop.fs.Path;
[*]import org.apache.hadoop.io.ObjectWritable;
[*]import org.apache.hadoop.io.Text;
[*]import org.apache.hadoop.mapreduce.Job;
[*]import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
[*]import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
[*]import org.apache.hadoop.util.Tool;
[*]import org.apache.hadoop.util.ToolRunner;
[*]
[*]
[*]public class PageRankDriver extends Configured implements Tool {
[*]
[*] public int run(String[] args) throws Exception {
[*] Configuration conf = getConf();
[*] Job job = new Job(conf, "PageRank");
[*] job.setJarByClass(PageRankDriver.class);
[*] job.setMapperClass(PageRankMapper.class);
[*] job.setReducerClass(PageRankReducer.class);
[*] job.setOutputKeyClass(Text.class);
[*] job.setOutputValueClass(ObjectWritable.class);
[*]// job.setInputFormatClass(KeyValueTextInputFormat.class);
[*]
[*] FileInputFormat.addInputPath(job, new Path(args[0]));
[*] FileOutputFormat.setOutputPath(job, new Path(args[1]));
[*]
[*] FileSystem fs = FileSystem.get(conf);
[*] fs.delete(new Path(args[1]), true);
[*] int res = job.waitForCompletion(true) ? 0 : 1;
[*]// if (res == 0) {
[*]// fs.delete(new Path(args[0]), true);
[*]// fs.rename(new Path(args[1]), new Path(args[0]));
[*]// fs.delete(new Path(args[0] + "/_logs"), true);
[*]// }
[*] return res;
[*] }
[*]
[*] public static void main(String[] args) throws IOException {
[*]
[*] int exitCode = 0;
[*]// for (int i = 0; i 复制代码 |
|