MAC下Anaconda+Pyspark安装配置详细步骤

sudo vi ~/.bash_profile #bash_profile是当前用户的环境变量文件

export SPARK_HOME="/Users/sherry/documents/spark/spark-3.1.2-bin-hadoop2.7" #spark文件的完整解压目录
export PATH=${PATH}:${SPARK_HOME}/bin

source ~/.bash_profile #让修改的bash_profile生效
echo $PATH #查看环境变量，可以看到新增的路径名

vi ~/.zshrc

if [ -f ~/.bash_profile ]; then
        source ~/.bash_profile
fi 

from pyspark import SparkContext, SparkConf

if __name__ == "__main__":
    spark_conf = SparkConf()\
        .setAppName('Python_Spark_WordCount')\
        .setMaster('local[2]') 
    #使用spark最原始的api进行数据分析
    sc = SparkContext(conf=spark_conf)
    sc.setLogLevel('WARN')
    print (sc)
   # ------创建RDD，需要分析的数据 --------------------------------------
    def local_rdd(spark_context):
        datas = ['hadoop spark','hadoop Hive spark','hadoop hive spark',\
                 'hadoop python spark','hadoop python spark',]
        return spark_context.parallelize(datas)

    rdd = local_rdd(sc)
    print (rdd.count())
    print (rdd.first())
    sc.stop()

import os
#添加spark安装目录
os.environ['SPARK_HOME'] ='/Users/sherry/documents/spark/spark-3.1.2-bin-hadoop2.7'