spark source

我会带着你远行 2023-10-18 18:21 31阅读 0赞

#Run on a Spark standalone cluster
    ./bin/spark-submit \
      --class org.apache.spark.examples.SparkPi \
      --master spark://207.184.161.138:7077 \
      --executor-memory 20G \
      --total-executor-cores 100 \
      /path/to/examples.jar \

1、用户提交任务
    org/apache/spark/deploy/SparkSubmit.scala
    	main
    		submit
    			doRunMain
    				prepareSubmit
    					isStandaloneCluster => childMainClass = "org.apache.spark.deploy.Client"
    					isYarnCluster => childMainClass = "org.apache.spark.deploy.yarn.Client"
    				runMain
    					反射运行 mainClass 的main方法
    org/apache/spark/deploy/Client.scala
    	main
    		org.apache.spark.deploy.ClientEndpoint
    			onStart
    				ayncSendToMasterAndForwardReply
    					self.send(v)
    org/apache/spark/rpc/netty/NettyRpcEnv.scala
    	send
    		def send(message: Any)
    		nettyEnv.send(RequestMessage(nettyEnv.address, this, message))
    			def send(message: RequestMessag)
    			postToOutbox(message.receiver, OneWayOutboxMessage(serialize(message)))
    				targetOutbox.send(message)
    org.apache.spark.rpc.netty.Outbox#send
    	drainOutbox()
    		message.sendWith(_client)
    org.apache.spark.rpc.netty.OneWayOutboxMessage#sendWith
    	client.send(content)
    org.apache.spark.network.client.TransportClient#send
    
    2、Master接收客户端消息，调度Driver
    org/apache/spark/deploy/master/Master.scala
    
    org.apache.spark.deploy.master.Master#receiveAndReply
    	RequestSubmitDriver(description)
    		schedule()
    		org.apache.spark.deploy.master.Master#schedule
    			launchDriver(worker, driver) //启动Driver
    			startExecutorsOnWorkers() //启动Executors
    
    3、Worker接收Master消息，启动Driver
    org/apache/spark/deploy/worker/Worker.scala
    
    org.apache.spark.deploy.worker.Worker#receive
    	LaunchDriver(driverId, driverDesc)
    	LaunchExecutor(masterUrl, appId, execId, appDesc, cores_, memory_)

refer:  
【Spark2.0源码学习】-1.概述 - hframe - 博客园 https://www.cnblogs.com/hframe/p/6735123.html  
Spark1.2源码系列（一）spark-submit提交作业过程 - 岑玉海 - 博客园 https://www.cnblogs.com/cenyuhai/p/3775687.html