最近开始研究Mapreduce了,但是发现里面所有的examples都是用旧的API写的,现在想将其用新的API重写,重写后代码如下,但是总出现错误,还请教高手指点。
[ol]package cn.edu.neu.mapreduce;import java.io.IOException;import java.util.Random;import java.util.regex.Matcher;import java.util.regex.Pattern;import org.apache.hadoop.conf.Configuration;import org.apache.hadoop.conf.Configured;import org.apache.hadoop.fs.FileSystem;import org.apache.hadoop.fs.Path;import org.apache.hadoop.io.LongWritable;import org.apache.hadoop.io.Text;import org.apache.hadoop.mapreduce.Job;import org.apache.hadoop.mapreduce.Mapper;import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;import org.apache.hadoop.mapreduce.lib.input.SequenceFileInputFormat;import org.apache.hadoop.mapreduce.lib.map.InverseMapper;import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;import org.apache.hadoop.mapreduce.lib.output.SequenceFileOutputFormat;import org.apache.hadoop.mapreduce.lib.reduce.LongSumReducer;import org.apache.hadoop.util.GenericOptionsParser;import org.apache.hadoop.util.Tool;import org.apache.hadoop.util.ToolRunner;public class Grep extends Configured implements Tool {private Grep() {}@Overridepublic int run(String[] args) throws Exception {// TODO Auto-generated method stubif (args.length []");ToolRunner.printGenericCommandUsage(System.out);return -1;}Path tempDir = new Path("grep-temp-"+ Integer.toString(new Random().nextInt(Integer.MAX_VALUE)));String[] otherArgs = new GenericOptionsParser(getConf(), args).getRemainingArgs();Job job = new Job(getConf(), "Grep");try {job.setJobName("grep-search");job.setJarByClass(Grep.class);FileInputFormat.setInputPaths(job, otherArgs[0]);job.setMapperClass(RegexMapper.class);job.getConfiguration().set("mapred.mapper.regex", otherArgs[2]);if (otherArgs.length == 4)job.getConfiguration().set("mapred.mapper.regex.group",otherArgs[3]);job.setCombinerClass(LongSumReducer.class);job.setReducerClass(LongSumReducer.class);FileOutputFormat.setOutputPath(job, tempDir);job.setOutputFormatClass(SequenceFileOutputFormat.class);job.setOutputKeyClass(Text.class);job.setOutputValueClass(LongWritable.class);job.waitForCompletion(true);Job sortJob = new Job(getConf(), "Sort");sortJob.setJobName("grep-sort");FileInputFormat.setInputPaths(sortJob, tempDir);sortJob.setInputFormatClass(SequenceFileInputFormat.class);sortJob.setMapperClass(InverseMapper.class);sortJob.setNumReduceTasks(1);FileOutputFormat.setOutputPath(sortJob, new Path(otherArgs[1]));sortJob.setOutputKeyClass(LongWritable.DecreasingComparator.class);job.waitForCompletion(true);} finally {FileSystem.get(job.getConfiguration()).delete(tempDir, true);}return 0;}/*** @param args*/public static void main(String[] args) throws Exception {// TODO Auto-generated method stubint res = ToolRunner.run(new Configuration(), new Grep(), args);System.exit(res);}public class RegexMapper extends Mapper {private Pattern pattern;private int group;public void map(K key, Text value, Context context) throws IOException,InterruptedException {pattern = Pattern.compile(context.getConfiguration().get("mapred.mapper.regex"));group = context.getConfiguration().getInt("mapred.mapper.regex.group", 0);String text = value.toString();Matcher matcher = pattern.matcher(text);while (matcher.find()) {try {context.write(new Text(matcher.group(group)),new LongWritable(1));} catch (InterruptedException e) {// TODO Auto-generated catch blocke.printStackTrace();}}}}}[/ol]复制代码
错误提示:
09/12/24 15:26:50 INFO input.FileInputFormat: Total input paths to process : 1
09/12/24 15:26:51 INFO mapred.JobClient: Running job: job_200912241430_0008
09/12/24 15:26:52 INFO mapred.JobClient: map 0% reduce 0%
09/12/24 15:27:02 INFO mapred.JobClient: Task Id : attempt_200912241430_0008_m_000000_0, Status : FAILED
java.lang.RuntimeException: java.lang.NoSuchMethodException: cn.edu.neu.mapreduce.Grep$RegexMapper.()
at org.apache.hadoop.util.ReflectionUtils.newInstance(ReflectionUtils.java:115)
at org.apache.hadoop.mapred.MapTask.runNewMapper(MapTask.java:532)
at org.apache.hadoop.mapred.MapTask.run(MapTask.java:305)
at org.apache.hadoop.mapred.Child.main(Child.java:170)
Caused by: java.lang.NoSuchMethodException: cn.edu.neu.mapreduce.Grep$RegexMapper.()
at java.lang.Class.getConstructor0(Class.java:2706)
at java.lang.Class.getDeclaredConstructor(Class.java:1985)
at org.apache.hadoop.util.ReflectionUtils.newInstance(ReflectionUtils.java:109)
... 3 more |
|