Spark伪分布式安装

#下载scala
wget https://downloads.lightbend.com/scala/2.13.6/scala-2.13.6.tgz
#解压文件
tar -xvf scala-2.13.6.tgz
#配置环境变量
vim /etc/profile
#添加配置项
#scala2.13.6
export SCALA_HOME=/usr/local/scala-2.13.6
export PATH=$PATH:$SCALA_HOME/bin
#保存后，使环境变量立即生效
source /etc/profile

2.下载安装Spark，这里使用spark-3.2.0-bin-hadoop3.2-scala2.13版本

#下载Spark
wget https://dlcdn.apache.org/spark/spark-3.2.0/spark-3.2.0-bin-hadoop3.2-scala2.13.tgz
#解压文件
tar -xvf spark-3.2.0-bin-hadoop3.2-scala2.13.tgz
mv spark-3.2.0-bin-hadoop3.2-scala2.13 spark-3.2.0
#配置环境变量
vim /etc/profile
#添加配置项
#spark3.2.0
export SPARK_HOME=/usr/local/spark-3.2.0
export PATH=$PATH:$SPARK_HOME/bin
#保存后，使环境变量立即生效
source /etc/profile

#配置文件目录
cd /usr/local/spark-3.2.0/conf/
#1.由模板spark-env.sh.template修改配置
cp spark-env.sh.template spark-env.sh
vim spark-env.sh
#添加配置项
export JAVA_HOME=/usr/local/jdk
export HADOOP_HOME=/usr/local/hadoop-3.2.1
export SCALA_HOME=/usr/local/scala-2.13.6
export HADOOP_CONF_DIR=/usr/local/hadoop-3.2.1/etc/hadoop
export SPARK_MASTER_HOST=hadoop0
export SPARK_PID_DIR=/usr/local/spark-3.2.0/data/pid
export SPARK_LOCAL_DIRS=/usr/local/spark-3.2.0/data/spark_shuffle
export SPARK_EXECUTOR_MEMORY=1G
export SPARK_WORKER_MEMORY=4G
#2.由模板spark-env.sh.template修改配置
cp spark-defaults.conf.template spark-defaults.conf
vim spark-defaults.conf
#添加配置项
spark.master                     spark://hadoop0:7077
spark.eventLog.enabled           true
spark.eventLog.dir               hdfs://hadoop0:9000/eventLog
spark.serializer                 org.apache.spark.serializer.KryoSerializer
spark.driver.memory              1g
spark.jars.packages     Azure:mmlspark:0.12
#创建spark.eventLog.dir对应的目录
hadoop fs -mkdir /eventLog

#启动spark
/usr/local/spark-3.2.0/sbin/start-all.sh
#启动后，使用jps查看是否有Master和Worker两个进程

/usr/local/spark-3.2.0/examples/src/main/ 目录下有各种语言版本的示例程序可以测试。

/usr/local/spark-3.2.0/bin/run-example SparkPi
#使用Spark Shell
spark-shell

启动Spark Shell后，可以通过http://hadoop0:4040/和http://hadoop0:8080/查看Spark的运行信息。