分享

请教MapReduce难题!

xukunddp 发表于 2013-10-26 15:14:48 [显示全部楼层] 回帖奖励 阅读模式 关闭右栏 4 6482
最近开始研究Mapreduce了,但是发现里面所有的examples都是用旧的API写的,现在想将其用新的API重写,重写后代码如下,但是总出现错误,还请教高手指点。
[ol]
  • package cn.edu.neu.mapreduce;
  • import java.io.IOException;
  • import java.util.Random;
  • import java.util.regex.Matcher;
  • import java.util.regex.Pattern;
  • import org.apache.hadoop.conf.Configuration;
  • import org.apache.hadoop.conf.Configured;
  • import org.apache.hadoop.fs.FileSystem;
  • import org.apache.hadoop.fs.Path;
  • import org.apache.hadoop.io.LongWritable;
  • import org.apache.hadoop.io.Text;
  • import org.apache.hadoop.mapreduce.Job;
  • import org.apache.hadoop.mapreduce.Mapper;
  • import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
  • import org.apache.hadoop.mapreduce.lib.input.SequenceFileInputFormat;
  • import org.apache.hadoop.mapreduce.lib.map.InverseMapper;
  • import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
  • import org.apache.hadoop.mapreduce.lib.output.SequenceFileOutputFormat;
  • import org.apache.hadoop.mapreduce.lib.reduce.LongSumReducer;
  • import org.apache.hadoop.util.GenericOptionsParser;
  • import org.apache.hadoop.util.Tool;
  • import org.apache.hadoop.util.ToolRunner;
  • public class Grep extends Configured implements Tool {
  • private Grep() {
  • }
  • @Override
  • public int run(String[] args) throws Exception {
  • // TODO Auto-generated method stub
  • if (args.length   []");
  • ToolRunner.printGenericCommandUsage(System.out);
  • return -1;
  • }
  • Path tempDir = new Path("grep-temp-"
  • + Integer.toString(new Random().nextInt(Integer.MAX_VALUE)));
  • String[] otherArgs = new GenericOptionsParser(getConf(), args)
  • .getRemainingArgs();
  • Job job = new Job(getConf(), "Grep");
  • try {
  • job.setJobName("grep-search");
  • job.setJarByClass(Grep.class);
  • FileInputFormat.setInputPaths(job, otherArgs[0]);
  • job.setMapperClass(RegexMapper.class);
  • job.getConfiguration().set("mapred.mapper.regex", otherArgs[2]);
  • if (otherArgs.length == 4)
  • job.getConfiguration().set("mapred.mapper.regex.group",
  • otherArgs[3]);
  • job.setCombinerClass(LongSumReducer.class);
  • job.setReducerClass(LongSumReducer.class);
  • FileOutputFormat.setOutputPath(job, tempDir);
  • job.setOutputFormatClass(SequenceFileOutputFormat.class);
  • job.setOutputKeyClass(Text.class);
  • job.setOutputValueClass(LongWritable.class);
  • job.waitForCompletion(true);
  • Job sortJob = new Job(getConf(), "Sort");
  • sortJob.setJobName("grep-sort");
  • FileInputFormat.setInputPaths(sortJob, tempDir);
  • sortJob.setInputFormatClass(SequenceFileInputFormat.class);
  • sortJob.setMapperClass(InverseMapper.class);
  • sortJob.setNumReduceTasks(1);
  • FileOutputFormat.setOutputPath(sortJob, new Path(otherArgs[1]));
  • sortJob.setOutputKeyClass(LongWritable.DecreasingComparator.class);
  • job.waitForCompletion(true);
  • } finally {
  • FileSystem.get(job.getConfiguration()).delete(tempDir, true);
  • }
  • return 0;
  • }
  • /**
  • * @param args
  • */
  • public static void main(String[] args) throws Exception {
  • // TODO Auto-generated method stub
  • int res = ToolRunner.run(new Configuration(), new Grep(), args);
  • System.exit(res);
  • }
  • public class RegexMapper extends Mapper {
  • private Pattern pattern;
  • private int group;
  • public void map(K key, Text value, Context context) throws IOException,
  • InterruptedException {
  • pattern = Pattern.compile(context.getConfiguration().get(
  • "mapred.mapper.regex"));
  • group = context.getConfiguration().getInt(
  • "mapred.mapper.regex.group", 0);
  • String text = value.toString();
  • Matcher matcher = pattern.matcher(text);
  • while (matcher.find()) {
  • try {
  • context.write(new Text(matcher.group(group)),
  • new LongWritable(1));
  • } catch (InterruptedException e) {
  • // TODO Auto-generated catch block
  • e.printStackTrace();
  • }
  • }
  • }
  • }
  • }[/ol]复制代码
    错误提示:
    09/12/24 15:26:50 INFO input.FileInputFormat: Total input paths to process : 1
    09/12/24 15:26:51 INFO mapred.JobClient: Running job: job_200912241430_0008
    09/12/24 15:26:52 INFO mapred.JobClient:  map 0% reduce 0%
    09/12/24 15:27:02 INFO mapred.JobClient: Task Id : attempt_200912241430_0008_m_000000_0, Status : FAILED
    java.lang.RuntimeException: java.lang.NoSuchMethodException: cn.edu.neu.mapreduce.Grep$RegexMapper.()
    at org.apache.hadoop.util.ReflectionUtils.newInstance(ReflectionUtils.java:115)
    at org.apache.hadoop.mapred.MapTask.runNewMapper(MapTask.java:532)
    at org.apache.hadoop.mapred.MapTask.run(MapTask.java:305)
    at org.apache.hadoop.mapred.Child.main(Child.java:170)
    Caused by: java.lang.NoSuchMethodException: cn.edu.neu.mapreduce.Grep$RegexMapper.()
    at java.lang.Class.getConstructor0(Class.java:2706)
    at java.lang.Class.getDeclaredConstructor(Class.java:1985)
    at org.apache.hadoop.util.ReflectionUtils.newInstance(ReflectionUtils.java:109)
    ... 3 more
  • 已有(4)人评论

    跳转到指定楼层
    sq331335144 发表于 2013-10-26 15:14:48
    dddddddddddddddddddddddddddddddddddddd
    回复

    使用道具 举报

    louisthy 发表于 2013-10-26 15:14:48
    MapTask.java:532这行的代码是啥吗?
    回复

    使用道具 举报

    jooli 发表于 2013-10-26 15:14:48
    [ol]
  • org.apache.hadoop.mapreduce.Mapper[I] mapper =
  • (org.apache.hadoop.mapreduce.Mapper[I])
  • ReflectionUtils.newInstance(taskContext.getMapperClass(), job);[/ol]复制代码
  • 回复

    使用道具 举报

    mexiang 发表于 2013-10-26 15:14:48
    我也遇到相同的,把主类里的内部类和函数改为static修饰就可以了。。。
    回复

    使用道具 举报

    您需要登录后才可以回帖 登录 | 立即注册

    本版积分规则

    关闭

    推荐上一条 /2 下一条