spark简单文件配置

时间：2023-03-10 03:36:13
spark简单文件配置 
cd /usr/local/spark/spark-2.2.1-bin-hadoop2.7/conf

cp slaves.template slaves

cp spark-env.sh.template spark-env.sh

修改slaves和spark-env.sh文件
slaves

hadoop-3

hadoop-4

hadoop-5

spark-env.sh

export JAVA_HOME=/usr/lib/jvm/java-1.8.0-openjdk

export SCALA_HOME=/usr/share/scala

export HADOOP_HOME=/usr/local/hadoop/hadoop-2.7.3

export SPARK_MASTER_IP=192.168.1.116

export SPARK_MASTER_PORT=7077

export SPARK_MASTER_WEBUI_PORT=7070

export SPARK_WORKER_CORES=2

export SPARK_WORKER_MEMORY=1024m

export SPARK_WORKER_INSTANCES=2

export SPARK_CLASSPATH=$HBASE_HOME/lib/hbase-protocol-1.2.4.jar:$HBASE_HOME/lib/hbase-common-1.2.4.jar:$HBASE_HOME/lib/htrace-core-3.1.0-incubating.jar:$HBAS

E_HOME/lib/hbase-server-1.2.4.jar:$HBASE_HOME/lib/hbase-client-1.2.4.jar:$HBASE_HOME/lib/metrics-core-2.2.0.jar:$SPARK_CLASSPATH

export SPARK_LOCAL_DIR="/mnt/spark/tmp"

export SPARK_JAVA_OPTS="-Dspark.storage.blockManagerHeartBeatMs=60000-Dspark.local.dir=$SPARK_LOCAL_DIR -XX:+PrintGCDetails -XX:+PrintGCTimeStamps -Xloggc:$S

PARK_HOME/logs/gc.log -XX:+UseConcMarkSweepGC -XX:+UseCMSCompactAtFullCollection -XX:CMSInitiatingOccupancyFraction=60"

相关文章