炼数成金hadoop视频干货05

时间：2021-12-06 08:33:11

视频地址：http://pan.baidu.com/s/1dDEgKwD

这一节是讲师助教带着动手操作写简单的开发环境的部署和两个实例

开发环境的部署：http://www.cnblogs.com/admln/p/test-deployDevelopment.html

第一个实例就是wordcount

第二个实例

 package testHadoop;

 import java.io.IOException;

 import org.apache.hadoop.conf.Configuration;

 import org.apache.hadoop.conf.Configured;

 import org.apache.hadoop.fs.Path;

 import org.apache.hadoop.io.LongWritable;

 import org.apache.hadoop.io.Text;

 import org.apache.hadoop.mapred.TextOutputFormat;

 import org.apache.hadoop.mapreduce.Job;

 import org.apache.hadoop.mapreduce.Mapper;

 import org.apache.hadoop.mapreduce.Reducer;

 import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;

 import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;

 import org.apache.hadoop.util.Tool;

 import org.apache.hadoop.util.ToolRunner;

 @SuppressWarnings("deprecation")

 public class ReverseIndex extends Configured implements Tool{

     enum Counter{

         LINESKIP;

     }

     public static class Map extends Mapper<LongWritable,Text,Text,Text> {

         public void map(LongWritable key,Text value,Context context) throws IOException, InterruptedException {

             String line = value.toString();

             try {

                 String[] lineSplit = line.split(" ");

                 String anum = lineSplit[0];

                 String bnum = lineSplit[1];

                 context.write(new Text(bnum), new Text(anum));

             }catch(java.lang.ArrayIndexOutOfBoundsException e) {

                 context.getCounter(Counter.LINESKIP).increment(1);

                 return;

             }

         }

     }

     public static class Reduce extends Reducer<Text,Text,Text,Text> {

         public void reduce(Text key,Iterable<Text> values,Context context) throws IOException, InterruptedException {

             String valueString;

             String out = "";

             for(Text value:values) {

                 valueString = value.toString();

                 out += valueString+"|";

             }

             context.write(key, new Text(out));

         }

     }

     public int run(String[] args) throws Exception {

         Configuration conf = getConf();

         Job job = new Job(conf,"ReverseIndex");

         job.setJarByClass(ReverseIndex.class);

         FileInputFormat.addInputPath(job, new Path(args[0]));

         FileOutputFormat.setOutputPath(job, new Path(args[1]));

         job.setMapperClass(Map.class);

         job.setReducerClass(Reduce.class);

         //job.setOutputFormatClass(TextOutputFormat.class);

         job.setOutputKeyClass(Text.class);

         job.setOutputValueClass(Text.class);

         job.waitForCompletion(true);

         return job.isSuccessful()?0:1;

     }

     public static void main(String[] args) throws Exception {

         int res = ToolRunner.run(new Configuration(), new ReverseIndex(),args);

         System.exit(res);

     }

 }

eclipse中运行没问题后打包用集群运行的时候遇到一个小问题

炼数成金hadoop视频干货05

版本不合。原来我编译的时候在windows下面用的JDK7，Linux中hadoop JDK是1.6。

把源码在Linux上1.6编译一下就可以了。

实践过程中还学到一个小知识，在运行程序命令中输入输出路径如果用例如 input output这样的，就是默认在HDFS上/user/用户名/下面的input和output

如果用例如/input /output这样的路径就是在HDFS根目录下的input 和output

相关文章