1.配置环境说明
spark:5台zybo板，192.168.1.1master，其它4台为slave
hadoop：192.168.1.1（外接SanDisk )
2.单节点hadoop测试：
如果出现内存不足情况如下：
查看当前虚拟内存容量：
free -m
cd /mnt
mkdir swap
cd swap/
创建一个swap文件
dd if=/dev/zero of=swapfile bs=1024 count=1000000
把生成的文件转换成swap文件
mkswap swapfile
激活swap文件
swapon swapfile
free -m
通过测试：
3.spark + hadoop 测试
SPARK_MASTER_IP=192.168.1.1 ./sbin/start-all.sh
MASTER=spark://192.168.1.1:7077 ./bin/pyspark
file = sc.textFile("hdfs://192.168.1.1:9000/in/file")
counts = file.flatMap(lambda line: line.split(" ")) \
.map(lambda word: (word, 1)) \
.reduceByKey(lambda a, b: a + b)
counts.saveAsTextFile("hdfs://192.168.1.1:9000/out/mycount")
counts.saveAsTextFile("/mnt/mycount")
counts.collect()
counts.collect()
错误1：
java.net.ConnectException: Call From zynq/192.168.1.1 to spark1:8020 failed on connection exception: java.net.ConnectException: Connection refused; For more details see: http://wiki.apache.org/hadoop/ConnectionRefused
这是由于我们用root启动hadoop，而spark要远程操作hadoop系统，没有权限引起的
解决：如果是测试环境，可以取消hadoop hdfs的用户权限检查。打开etc/hadoop/hdfs-site.xml，找到dfs.permissions属性修改为false（默认为true）OK了。
<property>
<name>dfs.permissions</name>
<value>false</value>
</property>
4.附：我的配置文件
go.sh:
#! /bin/sh - mount /dev/sda1 /mnt/
cd /mnt/swap/
swapon swapfile
free -m cd /root/hadoop-2.4.0/
sbin/hadoop-daemon.sh start namenode
sbin/hadoop-daemon.sh start datanode
sbin/hadoop-daemon.sh start secondarynamenode
sbin/yarn-daemon.sh start resourcemanager
sbin/yarn-daemon.sh start nodemanager
sbin/mr-jobhistory-daemon.sh start historyserver jps
while [ `netstat -ntlp | grep 9000` -eq `echo` ]
do
sleep 1
done
netstat -ntlp
echo hadoop start successfully cd /root/spark-0.9.1-bin-hadoop2
SPARK_MASTER_IP=192.168.1.1 ./sbin/start-all.sh
jps
while [ `netstat -ntlp | grep 7077` -eq `echo` ]
do
sleep 1
done
netstat -ntlp
echo spark start successfully
/etc/hosts
#127.0.0.1 localhost zynq
192.168.1.1 spark1 localhost zynq
#192.168.1.1 spark1
192.168.1.2 spark2
192.168.1.3 spark3
192.168.1.4 spark4
192.168.1.5 spark5
192.168.1.100 sparkMaster
#::1 localhost ip6-localhost ip6-loopback
/etc/profile
export PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:$PATH
export JAVA_HOME=/usr/lib/jdk1.7.0_55
export JRE_HOME=${JAVA_HOME}/jre
export CLASSPATH=.:$JAVA_HOME/lib/tools.jar
export PATH=$JAVA_HOME/bin:$PATH
export HADOOP_HOME=/root/hadoop-2.4.0 export PATH=$PATH:$HADOOP_HOME/bin
export HADOOP_COMMON_LIB_NATIVE_DIR=$HADOOP_HOME/lib/native
export HADOOP_OPTS="-Djava.library.path=$HADOOP_HOME/lib/native"
export HADOOP_MAPRED_HOME=$HADOOP_HOME
export HADOOP_COMMON_HOME=$HADOOP_HOME
export HADOOP_HDFS_HOME=$HADOOP_HOME
export YARN_HOME=$HADOOP_HOME
export HADOOP_CONF_DIR=$HADOOP_HOME/etc/hadoop ifconfig eth2 hw ether 00:0a:35:00:01:01
ifconfig eth2 192.168.1.1/24 up
HADOOP_HOME/etc/hadoop/yarn-site.xml
<configuration>
<property>
<name>yarn.nodemanager.aux-services</name>
<value>mapreduce_shuffle</value>
</property> <property>
<name>yarn.nodemanager.aux-services.mapreduce_shuffle.class</name>
<value>org.apache.hadoop.mapred.ShuffleHandler</value>
</property> </configuration>
HADOOP_HOME/etc/hadoop/core-site.xml
<configuration>
<property>
<name>fs.default.name</name>
<value>hdfs://localhost:9000</value>
</property> <property>
<name>hadoop.tmp.dir</name>
<value>/mnt/hadoop/tmp</value>
</property>
</configuration>
HADOOP_HOME/etc/hadoop/hdfs-site.xml
<configuration>
<property>
<name>dfs.replication</name>
<value>1</value>
</property>
<property>
<name>dfs.permissions</name>
<value>false</value>
</property> <property>
<name>dfs.namenode.rpc-address</name>
<value>192.168.1.1:9000</value>
</property> <property>
<name>dfs.datanode.data.dir</name>
<value>/mnt/datanode</value>
</property>
<property>
<name>dfs.namenode.name.dir</name>
<value>/mnt/namenode</value>
</property>
</configuration>
done