springboot-集成spark

落日映苍穹つ 2022-02-20 11:20 532阅读 0赞

1.pom.xml

<**dependency**>  
    <**groupId**>org.springframework.data</**groupId**>  
    <**artifactId**>spring-data-hadoop</**artifactId**>  
    <**version**>2.5.0.RELEASE</**version**>  
</**dependency**>  
<**dependency**>  
    <**groupId**>org.apache.hbase</**groupId**>  
    <**artifactId**>hbase-client</**artifactId**>  
    <**version**>1.3.2</**version**>  
    <**exclusions**>  
        <**exclusion**>  
            <**groupId**>org.slf4j</**groupId**>  
            <**artifactId**>slf4j-log4j12</**artifactId**>  
        </**exclusion**>  
        <**exclusion**>  
            <**groupId**>org.mortbay.jetty</**groupId**>  
            <**artifactId**>servlet-api-2.5</**artifactId**>  
        </**exclusion**>  
        <**exclusion**>  
            <**groupId**>org.mortbay.jetty</**groupId**>  
            <**artifactId**>servlet-api-2.5-6.1.14</**artifactId**>  
        </**exclusion**>  
    </**exclusions**>  
</**dependency**>  
<**dependency**>  
    <**groupId**>org.apache.hadoop</**groupId**>  
    <**artifactId**>hadoop-hdfs</**artifactId**>  
    <**version**>2.7.3</**version**>  
    <**exclusions**>  
        <**exclusion**>  
            <**artifactId**>servlet-api</**artifactId**>  
            <**groupId**>javax.servlet</**groupId**>  
        </**exclusion**>  
    </**exclusions**>  
</**dependency**>

2.wordcount例子

import org.apache.spark.SparkConf;
    import org.apache.spark.api.java.JavaPairRDD;
    import org.apache.spark.api.java.JavaRDD;
    import org.apache.spark.api.java.JavaSparkContext;
    import org.apache.spark.api.java.function.FlatMapFunction;
    import org.apache.spark.api.java.function.Function2;
    import org.apache.spark.api.java.function.PairFunction;
    import scala.Tuple2;
    
    import java.util.Arrays;
    import java.util.Iterator;
    import java.util.List;
    
    public class JavaSparkWordCount {
        public static void main(String[] args) {
            //配置执行
            SparkConf conf = new SparkConf().setAppName("Java_WordCount");
    
            // 创建SparkContext对象:  JavaSparkContext
            JavaSparkContext context = new JavaSparkContext(conf);
    
            //读入数据
            JavaRDD<String> lines = context.textFile(args[0]);
    
            //分词
            JavaRDD<String> words = lines.flatMap(new FlatMapFunction<String, String>() {
    
                //       @Override
                public Iterator<String> call(String line) throws Exception {
                    return Arrays.asList(line.split(" ")).iterator();
                }
    
            });
    
            //每个单词记一次数
            JavaPairRDD<String, Integer> wordOne = words.mapToPair(new PairFunction<String, String, Integer>() {
    
                //       @Override
                public Tuple2<String, Integer> call(String word) throws Exception {
                    return new Tuple2<String,Integer>(word,1);
                }
            });
    
            //执行reduceByKey的操作
            JavaPairRDD<String, Integer> count = wordOne.reduceByKey(new Function2<Integer, Integer, Integer>() {
    
                //       @Override
                public Integer call(Integer i1, Integer i2) throws Exception {
                    return i1 + i2;
                }
            });
    
    
            //执行计算，执行action操作: 把结果打印在屏幕上
            List<Tuple2<String, Integer>> result = count.collect();
    
            //输出
            for(Tuple2<String, Integer> tuple: result){
                System.out.println(tuple._1+"\t"+tuple._2);
            }
    
            //停止SparkContext对象
            context.stop();
        }
    }

3.打包运行

如果按springboot的maven打包方式打包，提交时报ClassNotFoundException

bin/spark-submit --master spark://hadoop1:7077 --class com.JavaSparkWordCount /xxx/sparktest.jar /xxx/data.txt

WARN  SparkSubmit$$anon$2:87 - Failed to load com.JavaSparkWordCount.

java.lang.ClassNotFoundException: com.JavaSparkWordCount

at java.net.URLClassLoader.findClass(URLClassLoader.java:381)

解压打出的jar包，发现main所在类文件不在根目录，而是：在BOOT-INF\\classes目录。

正确的打包方式：

按F4，或者点击idea的

![20190413154950782.png][]

![watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3FxXzQxNjY1MzU2_size_16_color_FFFFFF_t_70][]

配置完后，点build,build artifacts...打包

打完包可以在jar包的com目录找到class文件。

再提交运行ok。

更多spark知识点，参见spark相关章节（spark知识点好像还没写博客，后续补上）

[20190413154950782.png]: /images/20220220/95c83129460043c08e151bf0aff45127.png
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3FxXzQxNjY1MzU2_size_16_color_FFFFFF_t_70]: /images/20220220/5f51731ebaf04a36a5a4ca098b49ec35.png