java – 在Mapreduce wordcount作业中找不到Class

前端之家收集整理的这篇文章主要介绍了java – 在Mapreduce wordcount作业中找不到Class前端之家小编觉得挺不错的,现在分享给大家,也给大家做个参考。
我试图在hadoop中运行一个wordcount工作,但总是得到一个类没有找到异常.我发布我写的类和我用来运行该命令的命令
  1. import java.io.IOException;
  2. import java.util.*;
  3. import org.apache.hadoop.fs.Path;
  4. import org.apache.hadoop.conf.*;
  5. import org.apache.hadoop.io.*;
  6. import org.apache.hadoop.mapreduce.*;
  7. import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
  8. import org.apache.hadoop.mapreduce.lib.input.TextInputFormat;
  9. import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
  10. import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat;
  11.  
  12. public class WordCount {
  13.  
  14. public static class Map extends Mapper<LongWritable,Text,IntWritable> {
  15. private final static IntWritable one = new IntWritable(1);
  16. private Text word = new Text();
  17.  
  18. public void map(LongWritable key,Text value,Context context) throws IOException,InterruptedException {
  19. String line = value.toString();
  20. StringTokenizer tokenizer = new StringTokenizer(line);
  21. while (tokenizer.hasMoreTokens()) {
  22. word.set(tokenizer.nextToken());
  23. context.write(word,one);
  24. }
  25. }
  26. }
  27.  
  28. public static class Reduce extends Reducer<Text,IntWritable,IntWritable> {
  29.  
  30. public void reduce(Text key,Iterable<IntWritable> values,Context context)
  31. throws IOException,InterruptedException {
  32. int sum = 0;
  33. for (IntWritable val : values) {
  34. sum += val.get();
  35. }
  36. context.write(key,new IntWritable(sum));
  37. }
  38. }
  39.  
  40. public static void main(String[] args) throws Exception {
  41. Configuration conf = new Configuration();
  42.  
  43. Job job = new Job(conf,"WordCount");
  44.  
  45. job.setOutputKeyClass(Text.class);
  46. job.setOutputValueClass(IntWritable.class);
  47.  
  48. job.setMapperClass(Map.class);
  49. job.setReducerClass(Reduce.class);
  50.  
  51. job.setInputFormatClass(TextInputFormat.class);
  52. job.setOutputFormatClass(TextOutputFormat.class);
  53.  
  54. FileInputFormat.addInputPath(job,new Path(args[0]));
  55. FileOutputFormat.setOutputPath(job,new Path(args[1]));
  56.  
  57. job.waitForCompletion(true);
  58. job.setJarByClass(WordCount.class);
  59. }
  60.  
  61. }

wordcount.jar导出到我的下载文件
这是我用来运行这个工作的命令

  1. jeet@jeet-Vostro-2520:~/Downloads$hadoop jar wordcount.jar org.gamma.WordCount /user/jeet/getty/gettysburg.txt /user/jeet/getty/out

在这种情况下,我的mapreduce工作是启动的,但它正在结束于进程的中间.打印异常树.

  1. 14/01/27 13:16:02 WARN mapred.JobClient: Use GenericOptionsParser for parsing the arguments. Applications should implement Tool for the same.
  2. 14/01/27 13:16:02 WARN mapred.JobClient: No job jar file set. User classes may not be found. See JobConf(Class) or JobConf#setJar(String).
  3. 14/01/27 13:16:02 INFO input.FileInputFormat: Total input paths to process : 1
  4. 14/01/27 13:16:02 INFO util.NativeCodeLoader: Loaded the native-hadoop library
  5. 14/01/27 13:16:02 WARN snappy.LoadSnappy: Snappy native library not loaded
  6. 14/01/27 13:16:03 INFO mapred.JobClient: Running job: job_201401271247_0001
  7. 14/01/27 13:16:04 INFO mapred.JobClient: map 0% reduce 0%
  8. 14/01/27 13:16:11 INFO mapred.JobClient: Task Id : attempt_201401271247_0001_m_000000_0,Status : Failed
  9. java.lang.RuntimeException: java.lang.ClassNotFoundException: org.gamma.WordCount$Map
  10. at org.apache.hadoop.conf.Configuration.getClass(Configuration.java:849)
  11. at org.apache.hadoop.mapreduce.JobContext.getMapperClass(JobContext.java:199)
  12. at org.apache.hadoop.mapred.MapTask.runNewMapper(MapTask.java:719)
  13. at org.apache.hadoop.mapred.MapTask.run(MapTask.java:370)
  14. at org.apache.hadoop.mapred.Child$4.run(Child.java:255)
  15. at java.security.AccessController.doPrivileged(Native Method)
  16. at javax.security.auth.Subject.doAs(Subject.java:415)
  17. at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1149)
  18. at org.apache.hadoop.mapred.Child.main(Child.java:249)
  19. Caused by: java.lang.ClassNotFoundException: org.gamma.WordCount$Map
  20. at java.net.URLClassLoader$1.run(URLClassLoader.java:366)
  21. at java.net.URLClassLoader$1.run(URLClassLoader.java:355)
  22. at java.security.AccessController.doPrivileged(Native Method)
  23. at java.net.URLClassLoader.findClass(URLClassLoader.java:354)
  24. at java.lang.ClassLoader.loadClass(ClassLoader.java:425)
  25. at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:308)
  26. at java.lang.ClassLoader.loadClass(ClassLoader.java:358)
  27. at java.lang.Class.forName0(Native Method)
  28. at java.lang.Class.forName(Class.java:270)
  29. at org.apache.hadoop.conf.Configuration.getClassByName(Configuration.java:802)
  30. at org.apache.hadoop.conf.Configuration.getClass(Configuration.java:847)
  31. ... 8 more
  32.  
  33. 14/01/27 13:16:16 INFO mapred.JobClient: Task Id : attempt_201401271247_0001_m_000000_1,Status : Failed
  34. java.lang.RuntimeException: java.lang.ClassNotFoundException: org.gamma.WordCount$Map
  35. at org.apache.hadoop.conf.Configuration.getClass(Configuration.java:849)
  36. at org.apache.hadoop.mapreduce.JobContext.getMapperClass(JobContext.java:199)
  37. at org.apache.hadoop.mapred.MapTask.runNewMapper(MapTask.java:719)
  38. at org.apache.hadoop.mapred.MapTask.run(MapTask.java:370)
  39. at org.apache.hadoop.mapred.Child$4.run(Child.java:255)
  40. at java.security.AccessController.doPrivileged(Native Method)
  41. at javax.security.auth.Subject.doAs(Subject.java:415)
  42. at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1149)
  43. at org.apache.hadoop.mapred.Child.main(Child.java:249)
  44. Caused by: java.lang.ClassNotFoundException: org.gamma.WordCount$Map
  45. at java.net.URLClassLoader$1.run(URLClassLoader.java:366)
  46. at java.net.URLClassLoader$1.run(URLClassLoader.java:355)
  47. at java.security.AccessController.doPrivileged(Native Method)
  48. at java.net.URLClassLoader.findClass(URLClassLoader.java:354)
  49. at java.lang.ClassLoader.loadClass(ClassLoader.java:425)
  50. at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:308)
  51. at java.lang.ClassLoader.loadClass(ClassLoader.java:358)
  52. at java.lang.Class.forName0(Native Method)
  53. at java.lang.Class.forName(Class.java:270)
  54. at org.apache.hadoop.conf.Configuration.getClassByName(Configuration.java:802)
  55. at org.apache.hadoop.conf.Configuration.getClass(Configuration.java:847)
  56. ... 8 more
  57.  
  58. 14/01/27 13:16:20 INFO mapred.JobClient: Task Id : attempt_201401271247_0001_m_000000_2,Status : Failed
  59. java.lang.RuntimeException: java.lang.ClassNotFoundException: org.gamma.WordCount$Map
  60. at org.apache.hadoop.conf.Configuration.getClass(Configuration.java:849)
  61. at org.apache.hadoop.mapreduce.JobContext.getMapperClass(JobContext.java:199)
  62. at org.apache.hadoop.mapred.MapTask.runNewMapper(MapTask.java:719)
  63. at org.apache.hadoop.mapred.MapTask.run(MapTask.java:370)
  64. at org.apache.hadoop.mapred.Child$4.run(Child.java:255)
  65. at java.security.AccessController.doPrivileged(Native Method)
  66. at javax.security.auth.Subject.doAs(Subject.java:415)
  67. at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1149)
  68. at org.apache.hadoop.mapred.Child.main(Child.java:249)
  69. Caused by: java.lang.ClassNotFoundException: org.gamma.WordCount$Map
  70. at java.net.URLClassLoader$1.run(URLClassLoader.java:366)
  71. at java.net.URLClassLoader$1.run(URLClassLoader.java:355)
  72. at java.security.AccessController.doPrivileged(Native Method)
  73. at java.net.URLClassLoader.findClass(URLClassLoader.java:354)
  74. at java.lang.ClassLoader.loadClass(ClassLoader.java:425)
  75. at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:308)
  76. at java.lang.ClassLoader.loadClass(ClassLoader.java:358)
  77. at java.lang.Class.forName0(Native Method)
  78. at java.lang.Class.forName(Class.java:270)
  79. at org.apache.hadoop.conf.Configuration.getClassByName(Configuration.java:802)
  80. at org.apache.hadoop.conf.Configuration.getClass(Configuration.java:847)
  81. ... 8 more
  82.  
  83. 14/01/27 13:16:26 INFO mapred.JobClient: Job complete: job_201401271247_0001
  84. 14/01/27 13:16:26 INFO mapred.JobClient: Counters: 7
  85. 14/01/27 13:16:26 INFO mapred.JobClient: Job Counters
  86. 14/01/27 13:16:26 INFO mapred.JobClient: SLOTS_MILLIS_MAPS=20953
  87. 14/01/27 13:16:26 INFO mapred.JobClient: Total time spent by all reduces waiting after reserving slots (ms)=0
  88. 14/01/27 13:16:26 INFO mapred.JobClient: Total time spent by all maps waiting after reserving slots (ms)=0
  89. 14/01/27 13:16:26 INFO mapred.JobClient: Launched map tasks=4
  90. 14/01/27 13:16:26 INFO mapred.JobClient: Data-local map tasks=4
  91. 14/01/27 13:16:26 INFO mapred.JobClient: SLOTS_MILLIS_REDUCES=0
  92. 14/01/27 13:16:26 INFO mapred.JobClient: Failed map tasks=1
  93.  
  94. somebody please please help i think i am very close of it

解决方法

尝试添加这个
  1. Job job = new Job(conf,"wordcount");
  2. job.setJarByClass(WordCount.class);

猜你在找的Java相关文章