Spark 3.1 Standalone集群搭建

拼搏现实的明天。 2022-09-04 05:54 233阅读 0赞

### 1、下载 ###

[http://spark.apache.org/downloads.html][http_spark.apache.org_downloads.html]  
![在这里插入图片描述][watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L2NoZW5neXVxaWFuZw_size_16_color_FFFFFF_t_70]

[https://www.apache.org/dyn/closer.lua/spark/spark-3.1.2/spark-3.1.2-bin-hadoop3.2.tgz][https_www.apache.org_dyn_closer.lua_spark_spark-3.1.2_spark-3.1.2-bin-hadoop3.2.tgz]

### 2、解压配置 ###

（1）解压

[root@node2 app]# ls
    nacos  spark-3.1.2-bin-hadoop3.2.tgz
    [root@node2 app]# tar -zxvf spark-3.1.2-bin-hadoop3.2.tgz

（2）重命名

[root@node2 app]# mv spark-3.1.2-bin-hadoop3.2 spark-3.1.2
    [root@node2 app]# cd spark-3.1.2/
    [root@node2 spark-3.1.2]# ls
    bin  conf  data  examples  jars  kubernetes  LICENSE  licenses  NOTICE  python  R  README.md  RELEASE  sbin  yarn
    [root@node2 spark-3.1.2]# ll

（3）配置workers（旧版本是slaves）

[root@node2 spark-3.1.2]# cd conf/
    [root@node2 conf]# ls
    fairscheduler.xml.template  log4j.properties.template  metrics.properties.template  spark-defaults.conf.template  spark-env.sh.template  workers.template
    [root@node2 conf]# vi workers.template 
    [root@node2 conf]# cp workers.template workers
    [root@node2 conf]# vi workers
    [root@node2 conf]# cat workers
    #
    # Licensed to the Apache Software Foundation (ASF) under one or more
    # contributor license agreements.  See the NOTICE file distributed with
    # this work for additional information regarding copyright ownership.
    # The ASF licenses this file to You under the Apache License, Version 2.0
    # (the "License"); you may not use this file except in compliance with
    # the License.  You may obtain a copy of the License at
    #
    #    http://www.apache.org/licenses/LICENSE-2.0
    #
    # Unless required by applicable law or agreed to in writing, software
    # distributed under the License is distributed on an "AS IS" BASIS,
    # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    # See the License for the specific language governing permissions and
    # limitations under the License.
    #
    
    # A Spark Worker will be started on each of the machines listed below.
    node1
    node2
    node3
    [root@node2 conf]#

（4）配置spark-env.sh

[root@node2 conf]# cp spark-env.sh.template spark-env.sh
    [root@node2 conf]# vi spark-env.sh

添加一行

export JAVA_HOME=/opt/jdk1.8.0_161

（5）分发

[root@node2 app]# scp -r spark-3.1.2  node1:/app
    [root@node2 app]# scp -r spark-3.1.2  node3:/app

### 3、启动集群 ###

[root@node1 app]# cd spark-3.1.2
    [root@node1 spark-3.1.2]# sbin/start-master.sh 
    starting org.apache.spark.deploy.master.Master, logging to /app/spark-3.1.2/logs/spark-root-org.apache.spark.deploy.master.Master-1-node1.out
    [root@node1 spark-3.1.2]# jps
    6148 PaloFe
    11689 Jps
    11325 Master  
    [root@node1 spark-3.1.2]# sbin/start-slaves.sh 
    This script is deprecated, use start-workers.sh
    node1: starting org.apache.spark.deploy.worker.Worker, logging to /app/spark-3.1.2/logs/spark-root-org.apache.spark.deploy.worker.Worker-1-node1.out
    node3: starting org.apache.spark.deploy.worker.Worker, logging to /app/spark-3.1.2/logs/spark-root-org.apache.spark.deploy.worker.Worker-1-node3.out
    node2: starting org.apache.spark.deploy.worker.Worker, logging to /app/spark-3.1.2/logs/spark-root-org.apache.spark.deploy.worker.Worker-1-node2.out
    [root@node1 spark-3.1.2]# jps
    6148 PaloFe
    14983 Jps
    14204 Worker
    11325 Master
    [root@node1 spark-3.1.2]#

[root@node2 app]# curl node1:8080
    <!DOCTYPE html><html>
          <head>
            <meta http-equiv="Content-type" content="text/html; charset=utf-8"/><meta name="viewport" content="width=device-width, initial-scale=1"/><link rel="stylesheet" href="/static/bootstrap.min.css" type="text/css"/><link rel="stylesheet" href="/static/vis-timeline-graph2d.min.css" type="text/css"/><link rel="stylesheet" href="/static/webui.css" type="text/css"/><link rel="stylesheet" href="/static/timeline-view.css" type="text/css"/><script src="/static/sorttable.js"></script><script src="/static/jquery-3.5.1.min.js"></script><script src="/static/vis-timeline-graph2d.min.js"></script><script src="/static/bootstrap.bundle.min.js"></script><script src="/static/initialize-tooltips.js"></script><script src="/static/table.js"></script><script src="/static/timeline-view.js"></script><script src="/static/log-view.js"></script><script src="/static/webui.js"></script><script>setUIRoot('')</script>
            
            <link rel="shortcut icon" href="/static/spark-logo-77x50px-hd.png"></link>
            <title>Spark Master at spark://node1:7077</title>
          </head>
          <body>
            <div class="container-fluid">
              <div class="row">
                <div class="col-12">
                  <h3 style="vertical-align: middle; display: inline-block;">
                    <a style="text-decoration: none" href="/">
                      <img src="/static/spark-logo-77x50px-hd.png"/>
                      <span class="version" style="margin-right: 15px;">3.1.2</span>
                    </a>
                    Spark Master at spark://node1:7077
                  </h3>
                </div>
              </div>
              <div class="row">
                <div class="col-12">
                  <div class="row">
              <div class="col-12">
                <ul class="list-unstyled">
                  <li><strong>URL:</strong> spark://node1:7077</li>
                  
                  <li><strong>Alive Workers:</strong> 3</li>
                  <li><strong>Cores in use:</strong> 12 Total,
                    0 Used</li>
                  <li><strong>Memory in use:</strong>
                    19.9 GiB Total,
                    0.0 B Used</li>
                  <li><strong>Resources in use:</strong>
                    </li>
                  <li><strong>Applications:</strong>
                    0 <a href="#running-app">Running</a>,
                    0 <a href="#completed-app">Completed</a> </li>
                  <li><strong>Drivers:</strong>
                    0 Running,
                    0 Completed </li>
                  <li><strong>Status:</strong> ALIVE</li>
                </ul>
              </div>
            </div><div class="row">
              <div class="col-12">
                <span class="collapse-aggregated-workers collapse-table" onClick="collapseTable('collapse-aggregated-workers','aggregated-workers')">
                  <h4>
                    <span class="collapse-table-arrow arrow-open"></span>
                    <a>Workers (3)</a>
                  </h4>
                </span>
                <div class="aggregated-workers collapsible-table">
                  <table class="table table-bordered table-sm table-striped sortable">
          <thead><th width="" class="">Worker Id</th><th width="" class="">Address</th><th width="" class="">State</th><th width="" class="">Cores</th><th width="" class="">Memory</th><th width="" class="">Resources</th></thead>
          <tbody>
            <tr>
          <td>
            <a href="http://10.17.12.158:8081">
                  worker-20210811170616-10.17.12.158-36757
                </a>
          </td>
          <td>10.17.12.158:36757</td>
          <td>ALIVE</td>
          <td>4 (0 Used)</td>
          <td sorttable_customkey="6799.0">
            6.6 GiB
            (0.0 B Used)
          </td>
          <td></td>
        </tr><tr>
          <td>
            <a href="http://10.17.12.159:8081">
                  worker-20210811170617-10.17.12.159-33329
                </a>
          </td>
          <td>10.17.12.159:33329</td>
          <td>ALIVE</td>
          <td>4 (0 Used)</td>
          <td sorttable_customkey="6799.0">
            6.6 GiB
            (0.0 B Used)
          </td>
          <td></td>
        </tr><tr>
          <td>
            <a href="http://10.17.12.160:8081">
                  worker-20210811170617-10.17.12.160-40490
                </a>
          </td>
          <td>10.17.12.160:40490</td>
          <td>ALIVE</td>
          <td>4 (0 Used)</td>
          <td sorttable_customkey="6799.0">
            6.6 GiB
            (0.0 B Used)
          </td>
          <td></td>
        </tr>
          </tbody>
        </table>
                </div>
              </div>
            </div><div class="row">
              <div class="col-12">
                <span id="running-app" class="collapse-aggregated-activeApps collapse-table" onClick="collapseTable('collapse-aggregated-activeApps','aggregated-activeApps')">
                  <h4>
                    <span class="collapse-table-arrow arrow-open"></span>
                    <a>Running Applications (0)</a>
                  </h4>
                </span>
                <div class="aggregated-activeApps collapsible-table">
                  <table class="table table-bordered table-sm table-striped sortable">
          <thead><th width="" class="">Application ID</th><th width="" class="">Name</th><th width="" class="">Cores</th><th width="" class="">Memory per Executor</th><th width="" class="">Resources Per Executor</th><th width="" class="">Submitted Time</th><th width="" class="">User</th><th width="" class="">State</th><th width="" class="">Duration</th></thead>
          <tbody>
            
          </tbody>
        </table>
                </div>
              </div>
            </div><div>
              
            </div><div class="row">
              <div class="col-12">
                <span id="completed-app" class="collapse-aggregated-completedApps collapse-table" onClick="collapseTable('collapse-aggregated-completedApps',
                    'aggregated-completedApps')">
                  <h4>
                    <span class="collapse-table-arrow arrow-open"></span>
                    <a>Completed Applications (0)</a>
                  </h4>
                </span>
                <div class="aggregated-completedApps collapsible-table">
                  <table class="table table-bordered table-sm table-striped sortable">
          <thead><th width="" class="">Application ID</th><th width="" class="">Name</th><th width="" class="">Cores</th><th width="" class="">Memory per Executor</th><th width="" class="">Resources Per Executor</th><th width="" class="">Submitted Time</th><th width="" class="">User</th><th width="" class="">State</th><th width="" class="">Duration</th></thead>
          <tbody>
            
          </tbody>
        </table>
                </div>
              </div>
            </div><div>
              
            </div>
                </div>
              </div>
            </div>
          </body>
        </html>[root@node2 app]#

[http_spark.apache.org_downloads.html]: http://spark.apache.org/downloads.html
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L2NoZW5neXVxaWFuZw_size_16_color_FFFFFF_t_70]: /images/20220829/c23d0609e0604b3a866f6abc2bc65073.png
[https_www.apache.org_dyn_closer.lua_spark_spark-3.1.2_spark-3.1.2-bin-hadoop3.2.tgz]: https://www.apache.org/dyn/closer.lua/spark/spark-3.1.2/spark-3.1.2-bin-hadoop3.2.tgz