hadoop一键安装伪分布式

hadoop伪分布式和hive在openSUSE中的安装

在git上的路径为：https://github.com/huabingood/hadoop--------/tree/master

各个文件夹内容

一.lib文件夹，主要存放使用到的jar包

1. dom4j.jar 主要是用来解析xml文档的jar包
2. JavaStudy.jar 具体执行解析xml文档的jar包
3. mysql-connector-java-5.1.42-bin.jar hive中需要的Java连接MySQL数据库的驱动jar包

二.software文件夹，主要存放要安装的hadoop生态安装包

1. hadoop-2.6.0-cdh5.10.0.tar.gz CDH的hadoop安装包
2. hive-1.1.0-cdh5.10.0.tar.gz CDH的hive安装包

三.script文件夹，主要是一键安装的脚本

1. main.sh 主调函数，顺序执行安装步骤
2. install-env.sh 主要是安装需要用到的各种参数，比如安装目录，MySQL连接信息，xml配置信息等。有部分配置需要根据不同的机器进行修改。
3. functions.sh 安装步骤，主要是供main.sh调用的函数。不需要修改

如何执行

一.完成Linux环境的基本配置

1. 安装好Java
2. 安装好mysql
3. 停用防火墙
4. 配置网络，并能ping通外网
5. 设置好主机名

二.创建文件夹用于存放安装问价夹

1. root用户下修改/opt文件的权限
chown 用户名 /opt
2. 普通用户创建文件存放脚本的文件夹
mkdir /opt/install
3. 将安装文件夹放到刚才创建的文件夹中
4.  赋予脚本的执行权限
chmod 777 functions.sh install-env.sh main.sh

修改install-env.sh配置自己的安装环境

1. 添加脚本的安装路径
2. 添加MySQL的连接参数
3. 给出要运行的Java的安装路径

main.sh代码


 #!/bin/bash

 source ./install-env.sh
 source ./functions.sh

 # 创建文件安装路径
 creatFile ${INSTALL_PATH}
 # 创建hadoop临时文件夹的存放路径
 mkdir -p ${HADOOP_TMP_DIR}

 # 解压hadoop，hive到安装目录下
 myTar hadoop ${INSTALL_PATH}
 myTar hive ${INSTALL_PATH}

 # 修改hadoop中非xml的配置文件
 notXMLHadoop ${INSTALL_PATH} ${HADOOP_PROFILEPATH}
 isSuccess $? "非XML修改成功" "非XML修改失败，请检查原因！！！"

 # 修改hadoop中xml的配置文件
 alterHadoopXml "${coreSite[@]}"
 alterHadoopXml "${hdfsSite[@]}"

 # 配置pid的文件路径
 alterBashrc
 isSuccess $? "bashrc修改成功" "bashrc修改失败，请检查原因！！！"
 # 格式化NameNode
 formatNameNode

 # 修改hive的的非xml配置文件
 setHiveNotXML
 isSuccess $? "非XML修改成功" "非XML修改失败，请检查原因！！！"

 # 修改hive的xml的配置文件hive-site.xml
 alterHiveXml "${hiveSite[@]}"

 # 将Java的mysql的jar包驱动放到hive中的lib文件夹里面
 hiveOver

install-env.sh代码

 #!/bin/bash

 INSTALL_PATH="/opt/hadoop"  # 请输入安装hadoop的绝对路径。改路径下应该没有内容！
 # mysql的配置情况
 mysqlIP="192.168.59.100"
 mysqlPort=""
 mysqlDatabase="hive_test"
 mysqlUser="root"
 mysqlPwd=""
 mysqlConnetor="mysql-connector-java-5.1.42-bin.jar"

 # 这里存放自己的JAVA_HOME
 MY_JAVA_HOME="/opt/softWare/jdk1.8.0_131"

 # 以下内容请不要修改！！！
 HADOOP_PROFILEPATH="/etc/hadoop"
 # hadoop临时文件的存放路径
 HADOOP_TMP_DIR="${INSTALL_PATH}/tmp/hadoop"
 HIVE_TMP_DIR="${INSTALL_PATH}/tmp/hive"
 # 需要往一下文件中修改其中的JAVA_HOME
 myArray=("hadoop-env.sh" "mapred-env.sh" "yarn-env.sh")

 # hadoop中需要修改的xml配置文件的文件名和修改属性
 coreSite=("core-site.xml" "fs.defaultFS" "hdfs://`hostname`:9000" "hadoop.tmp.dir" "${HADOOP_TMP_DIR}")
 hdfsSite=("hdfs-site.xml" "dfs.replication" "")

 # hive日志文件的存放路径
 hiveLogs="${INSTALL_PATH}/tmp/hive_logs"
 # hive-site.xml文件的配置
 hiveSite=("hive-site.xml" "javax.jdo.option.ConnectionURL" "jdbc:mysql://${mysqlIP}:${mysqlPort}/${mysqlDatabase}?createDatabaseIfNotExist=true&useSSL=false&characterEncoding=utf8" "javax.jdo.option.ConnectionDriverName" "com.mysql.jdbc.Driver" "javax.jdo.option.ConnectionUserName" "${mysqlUser}" "javax.jdo.option.ConnectionPassword" "${mysqlPwd}" "hive.exec.scratchdir" "${HIVE_TMP_DIR}" "system:java.io.tmpdir" "${HIVE_TMP_DIR}" "hive.cli.print.header" "true" "hive.cli.print.current.db" "true" "hive.metastore.warehouse.dir" "/opt/hadoop/warehouse")

functions.sh代码

 #!/bin/bash

 source ./install-env.sh

 # 创建文件夹，并清空文件夹
 function creatFile(){
     if [ -e $ ]
     then
         echo "文件夹已经存在！将清空${1}!!!\n"
         isContinue "清空${1}" "将要清空${1}"
         rm -rf ${}/*
         isSuccess $? "${1}清空成功，准备解压文件。" "${1}清空失败，请检查原因！！！"
     else
         mkdir ${1}
         isSuccess $? "${1}创建成功，准备解压文件。" "${1}创建失败，请检查原因！！！"
     fi
 }

 # 询问是否继续进项操作
 function isContinue(){
     echo -e "\e[1;31m是否${1}【y/n】\e[0m"
     read choose
     if [ "${choose}" = "y" ] || [ "${choose}" = "yes" ] || [ "${choose}" = "Y" ] || [ "${choose}" = "YES" ]
     then
         echo -e "\e[1;31m${1}\e[0m"
     else
         exit
     fi
 }

 # 判断前一个操作是否成功，如果错误就全部推倒重来
 function isSuccess(){
     if [ $1 -eq 0 ]
     then
         echo "${2}"
     else
         echo -e "\e[1;31m${3}\e[0m"
         rm -rf ${INSTALL_PATH}
         exit
     fi
 }

 # 解压缩文件
 function myTar(){
     filePath=`find ../software -name ${1}* | awk -F "/" '{print $NF}'`
     tar -zxvf ../software/${filePath} -C ${2} 1>>/dev/null 2>>/dev/null
     isSuccess $? "${1}解压成功。" "${2}解压失败，请查找原因！！！"
 }

 # 修改hadoop的非xml类的配置文件
 function notXMLHadoop(){
     filePath=`ls ${1} | grep hadoop`
     mkdir ${HADOOP_TMP_DIR}/myPID

     for i in ${myArray[@]}
     do
         sed -i '/export JAVA_HOME=$/d' ${1}/${filePath}${2}/${i}
         #sed -i "/export JAVA_HOME=${MY_JAVA_HOME}/d" ${1}/${filePath}${2}/${i}
         sed -i "3a export JAVA_HOME=${MY_JAVA_HOME}" ${1}/${filePath}${2}/${i}
         isSuccess $? "$i修改成功" "$i修改失败"

     done

     # 修改各个env.sh文件，修改pid文件存放的路径

     sed -i "s@export HADOOP_PID_DIR=.*@HADOOP_PID_DIR=${HADOOP_TMP_DIR}/myPID@g" ${1}/${filePath}${2}/hadoop-env.sh

 #    sed -i "/export HADOOP_MAPRED_PID_DIR=${HADOOP_TMP_DIR}\/myPID/d" ${1}/${filePath}${2}/mapred-env.sh
     sed -i "4a export HADOOP_MAPRED_PID_DIR=${HADOOP_TMP_DIR}/myPID" ${1}/${filePath}${2}/mapred-env.sh

 #    sed -i "/export YARN_PID_DIR=${HADOOP_TMP_DIR}\/myPID/d" ${1}/${filePath}${2}/yarn-env.sh
     sed -i "4a export YARN_PID_DIR=${HADOOP_TMP_DIR}/myPID" ${1}/${filePath}${2}/yarn-env.sh

     # 修改slaves文件
     sed -i "1a `hostname`" ${1}/${filePath}${2}/slaves
     isSuccess $? "slaves修改成功" "slaves修改失败"
     sed -i '1d' ${1}/${filePath}${2}/slaves
 }

 # 修改hadoop中的xml类的配置文件
 function alterHadoopXml(){

     myArray=($@)
     filePath=`ls ${INSTALL_PATH} | grep hadoop`
     for ((i=1;i<${#myArray[@]};i+=2))
     do
         j=$((i+1))
         java -jar ../lib/JavaStudy.jar /${INSTALL_PATH}/${filePath}${HADOOP_PROFILEPATH}/${myArray[0]} add ${myArray[$i]} ${myArray[$j]}
         isSuccess $? "${myArray[0]}中的${myArray[$i]}修改成功" "${myArray[0]}中的${myArray[$i]}修改失败，请检查原因！！！"
     done
 }

 # 修改当前用户的.bashrc文件，添加hadoop中pid等配置的存放路径
 function alterBashrc(){
     bashrcPathe=~/.bashrc
     mkdir ${HADOOP_TMP_DIR}/myPID
     sed -i "/myPIDDir=/d" ${bashrcPathe}
     sed -i "\$a myPIDDir=${HADOOP_TMP_DIR}/myPID" ${bashrcPathe}
     sed -i "/export HADOOP_PID_DIR=${myPIDDir}/d" ${bashrcPathe}
     sed -i '$a export HADOOP_PID_DIR=${myPIDDir}' ${bashrcPathe}
     sed -i '/export YARN_PID_DIR=${myPIDDir}/d' ${bashrcPathe}
     sed -i '$a export YARN_PID_DIR=${myPIDDir}' ${bashrcPathe}
     sed -i '/export HADOOP_MAPRED_PID_DIR=${myPIDDir}/d' ${bashrcPathe}
     sed -i '$a export HADOOP_MAPRED_PID_DIR=${myPIDDir}' ${bashrcPathe}
     sed -i '/export HBASE_PID_DIR=${myPIDDir}/d' ${bashrcPathe}
     sed -i '$a export HBASE_PID_DIR=${myPIDDir}' ${bashrcPathe}

     source ~/.bashrc
 }

 # 格式化NameNode
 function formatNameNode(){
     filePath=`ls ${INSTALL_PATH} | grep hadoop`
     ${INSTALL_PATH}/${filePath}/bin/hdfs namenode -format 1>>/dev/null 2>>/dev/null
     if [ $? -ne 0 ]
     then
         exit
     fi
     isSuccess $? "NameNode格式化成功！" "NameNode格式化失败，请查找原因！！！"
 }

 # 配置好hive的配置文件
 function setHiveNotXML(){
     echo "开始修改hive的非xml配置文件"
     filePath=`ls ${INSTALL_PATH} | grep hive`
     cp ${INSTALL_PATH}/${filePath}/conf/hive-env.sh.template ${INSTALL_PATH}/${filePath}/conf/hive-env.sh
     cp ${INSTALL_PATH}/${filePath}/conf/hive-exec-log4j.properties.template ${INSTALL_PATH}/${filePath}/conf/hive-exec-log4j.properties
     cp ${INSTALL_PATH}/${filePath}/conf/hive-log4j.properties.template ${INSTALL_PATH}/${filePath}/conf/hive-log4j.properties
     touch  ${INSTALL_PATH}/${filePath}/conf/hive-site.xml
     echo '<?xml version="1.0" encoding="UTF-8" standalone="no"?>' > ${INSTALL_PATH}/${filePath}/conf/hive-site.xml
     sed -i '1a <?xml-stylesheet type="text/xsl" href="configuration.xsl"?>' ${INSTALL_PATH}/${filePath}/conf/hive-site.xml
     sed -i '2a <configuration>' ${INSTALL_PATH}/${filePath}/conf/hive-site.xml
     sed -i '3a </configuration>' ${INSTALL_PATH}/${filePath}/conf/hive-site.xml
     # 修改hive-env.sh
     hadoopPath=`ls ${INSTALL_PATH} | grep hadoop`
     sed -i "\$a HADOOP_HOME=${INSTALL_PATH}/${hadoopPath}" ${INSTALL_PATH}/${filePath}/conf/hive-env.sh
     sed -i "\$a HIVE_CONF_DIR=${INSTALL_PATH}/${filePath}/conf" ${INSTALL_PATH}/${filePath}/conf/hive-env.sh
     sed -i "\$a export JAVA_HOME=${MY_JAVA_HOME}" ${INSTALL_PATH}/${filePath}/conf/hive-env.sh

     # 修改hive-log4j.properties文件
     mkdir ${hiveLogs}
     sed -i "[e-mail]=.*@hive.log.dir=${hiveLogs}@g" ${INSTALL_PATH}/${filePath}/conf/hive-log4j.properties
 }

 # 修改hive的xml文件
 function alterHiveXml(){
     myArray=($@)
     filePath=`ls ${INSTALL_PATH} | grep hive`
     for ((i=1;i<${#myArray[@]};i+=2))
     do
         j=$((i+1))
         java -jar ../lib/JavaStudy.jar /${INSTALL_PATH}/${filePath}/conf/${myArray[0]} add ${myArray[$i]} ${myArray[$j]}
         isSuccess $? "${myArray[0]}中的${myArray[$i]}修改成功" "${myArray[0]}中的${myArray[$i]}修改失败，请检查原因！！！"
     done
 }

 # 创建hive的配置文件夹，并Java的mysql驱动jar包复制到hive中的lib文件夹中
 function hiveOver(){
     mkdir ${HIVE_TMP_DIR}
     filePath=`ls ${INSTALL_PATH} | grep hive`
     cp ../lib/${mysqlConnetor} ${INSTALL_PATH}/${filePath}/lib
     isSuccess $? "hadoop与hive的伪分布式安装成功，just have fun!" "复制mysql的jar包失败，请检查原因！！！"
 }

JavaStudy.jar代码

package com.huabingood;

import org.dom4j.Document;
import org.dom4j.DocumentException;
import org.dom4j.Element;
import org.dom4j.io.OutputFormat;
import org.dom4j.io.SAXReader;
import org.dom4j.io.XMLWriter;

import java.io.*;

/**
 * Created by 杨华彬 on 2017/5/4.
 */
public class TestXML {
    public static void main(String[] args) {
        /*String[] args = new String[4];
        args[0] = "D:\\core-site.xml";
        args[1] = "add";
        args[2] = "name";
        args[3] = "1234567";*/

        // 创建解析器，准备解析指定路径的xml文档
        SAXReader saxReader = new SAXReader();
        Document doc = null;
        try {
            doc = saxReader.read(new File(args[0]));
        } catch (DocumentException e) {
            e.printStackTrace();
        }

        addNode(args, doc);

        write2XML(args[0], doc);
    }

    /**
     * 获取参数后，自动创建name和value节点，将根据传入的值设置name和value的值
     *
     * @param args 传递的是创建一个节点的参数。参数1表示xml文档的路径，
     *             参数2表示是创建节点还是修改节点，参数3表示name节点的值，参数4表示
     *             value节点的值。
     */
    public static void addNode(String[] args, Document doc) {

        // 获取根节点
        Element root = doc.getRootElement();
        // 创建根节点下的子节点
        Element parentNode = root.addElement("property");

        // 创建name和value节点
        Element nameChild = parentNode.addElement("name");
        nameChild.setText(args[2]);
        Element valueChild = parentNode.addElement("value");
        valueChild.setText(args[3]);
    }

    /**
     * 将修改内容写到文档中去
     *
     * @param file 修改的文件路径
     * @param doc  前面传进来的xml文档对象模型。
     */
    public static void write2XML(String file, Document doc) {

        // 设置排版格式为缩进格式，字符为utf-8
        OutputFormat of = OutputFormat.createPrettyPrint();
        of.setEncoding("utf-8");

        // 创建写入流，准备写入对象
        File myFile = new File(file);
        OutputStreamWriter osw = null;
        FileOutputStream fos = null;
        try {
            fos = new FileOutputStream(myFile);
            osw = new OutputStreamWriter(fos, "utf-8");
        } catch (FileNotFoundException e) {
            e.printStackTrace();
        } catch (Exception e) {
            e.printStackTrace();
        }

        XMLWriter xmlWriter = new XMLWriter(osw, of);

        // 开始写入内容
        try {
            xmlWriter.write(doc);
            xmlWriter.flush();

        } catch (IOException e) {
            e.printStackTrace();
        } finally {
            try {
                xmlWriter.close();
            } catch (IOException e) {
                e.printStackTrace();
            }
        }

    }
}