如何在Ubuntu的idea上运行Hadoop程序

时间：2021-03-31 05:21:12

如何在Ubuntu的idea上运行Hadoop程序

一、前言

在idea上运行Hadoop程序，需要使用Hadoop的相关库，Ubuntu为Hadoop的运行提供了良好的支持。

二、操作方法

首先我们需要创建一个maven项目，然后在pom.xml中进行设置，导入必要的包，最后写出mapreduce程序即可。

如何在Ubuntu的idea上运行Hadoop程序

其中pom.xml文件如下：

 <?xml version="1.0" encoding="UTF-8"?>

 <project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"

   xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">

   <modelVersion>4.0.0</modelVersion>

   <groupId>com.zyr.bigdata</groupId>

   <artifactId>MapReduce</artifactId>

   <version>1.0-SNAPSHOT</version>

   <name>MapReduce</name>

   <!-- FIXME change it to the project's website -->

   <url>http://www.example.com</url>

   <properties>

     <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>

     <maven.compiler.source>1.7</maven.compiler.source>

     <maven.compiler.target>1.7</maven.compiler.target>

   </properties>

   <dependencies>

     <dependency>

       <groupId>junit</groupId>

       <artifactId>junit</artifactId>

       <version>4.11</version>

       <scope>test</scope>

     </dependency>

       <dependency>

         <groupId>org.apache.hadoop</groupId>

         <artifactId>hadoop-core</artifactId>

         <version>1.2.1</version>

       </dependency>

       <dependency>

         <groupId>org.apache.hadoop</groupId>

         <artifactId>hadoop-common</artifactId>

         <version>2.9.0</version>

       </dependency>

   </dependencies>

   <build>

     <pluginManagement><!-- lock down plugins versions to avoid using Maven defaults (may be moved to parent pom) -->

       <plugins>

         <!-- clean lifecycle, see https://maven.apache.org/ref/current/maven-core/lifecycles.html#clean_Lifecycle -->

         <plugin>

           <artifactId>maven-clean-plugin</artifactId>

           <version>3.1.0</version>

         </plugin>

         <!-- default lifecycle, jar packaging: see https://maven.apache.org/ref/current/maven-core/default-bindings.html#Plugin_bindings_for_jar_packaging -->

         <plugin>

           <artifactId>maven-resources-plugin</artifactId>

           <version>3.0.2</version>

         </plugin>

         <plugin>

           <artifactId>maven-compiler-plugin</artifactId>

           <version>3.8.0</version>

         </plugin>

         <plugin>

           <artifactId>maven-surefire-plugin</artifactId>

           <version>2.22.1</version>

         </plugin>

         <plugin>

           <artifactId>maven-jar-plugin</artifactId>

           <version>3.0.2</version>

         </plugin>

         <plugin>

           <artifactId>maven-install-plugin</artifactId>

           <version>2.5.2</version>

         </plugin>

         <plugin>

           <artifactId>maven-deploy-plugin</artifactId>

           <version>2.8.2</version>

         </plugin>

         <!-- site lifecycle, see https://maven.apache.org/ref/current/maven-core/lifecycles.html#site_Lifecycle -->

         <plugin>

           <artifactId>maven-site-plugin</artifactId>

           <version>3.7.1</version>

         </plugin>

         <plugin>

           <artifactId>maven-project-info-reports-plugin</artifactId>

           <version>3.0.0</version>

         </plugin>

       </plugins>

     </pluginManagement>

   </build>

 </project>

    然后是编写代码：

package com.zyr.bigdata;

import java.io.IOException;

import java.util.StringTokenizer;

import org.apache.hadoop.conf.Configuration;

import org.apache.hadoop.fs.Path;

import org.apache.hadoop.io.IntWritable;

import org.apache.hadoop.io.Text;

import org.apache.hadoop.mapreduce.Job;

import org.apache.hadoop.mapreduce.Mapper;

import org.apache.hadoop.mapreduce.Reducer;

import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;

import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;

public class WordCount {

    public static class TokenizerMapper

            extends Mapper<Object, Text, Text, IntWritable> {

        private final static IntWritable one = new IntWritable(1);

        private Text word = new Text();

        public void map(Object key, Text value, Context context

        ) throws IOException, InterruptedException {

            StringTokenizer itr = new StringTokenizer(value.toString());

            while (itr.hasMoreTokens()) {

                word.set(itr.nextToken());

                context.write(word, one);

            }

        }

    }

    public static class IntSumReducer

            extends Reducer<Text, IntWritable, Text, IntWritable> {

        private IntWritable result = new IntWritable();

        public void reduce(Text key, Iterable<IntWritable> values,

                           Context context

        ) throws IOException, InterruptedException {

            int sum = 0;

            for (IntWritable val : values) {

                sum += val.get();

            }

            result.set(sum);

            context.write(key, result);

        }

    }

    public static void main(String[] args) throws Exception {

        Configuration conf = new Configuration();

        Job job = Job.getInstance(conf, "word count");

        job.setJarByClass(WordCount.class);

        job.setMapperClass(TokenizerMapper.class);

        job.setCombinerClass(IntSumReducer.class);

        job.setReducerClass(IntSumReducer.class);

        job.setOutputKeyClass(Text.class);

        job.setOutputValueClass(IntWritable.class);

        FileInputFormat.addInputPath(job, new Path(args[0]));

        FileOutputFormat.setOutputPath(job, new Path(args[1]));

        System.exit(job.waitForCompletion(true) ? 0 : 1);

    }

}

    因为是mapreduce的词频统计，因此需要读入文件，在src同级创建input文件夹，里面放入文档即可。

如何在Ubuntu的idea上运行Hadoop程序

   最后需要配置执行程序的设置（在run->edit configure中新建application即可)：

如何在Ubuntu的idea上运行Hadoop程序

    然后运行程序，会生成相应的output文件夹，打开即可查看结果。 如何在Ubuntu的idea上运行Hadoop程序

      至此，最简单的mapreduce程序就完成了，需要注意的是使用maven我们没有在ubuntu上安装相应的Hadoop，因为这是最简单的单机环境，使用的是本地文件系统，但是如果使用分布式的时候就必须需要本地安装Hadoop来提供访问服务了。

相关文章