IDEA开发java版本spark程序

如何去创建项目这里就不对讲了，可以参考 ：https://www.cnblogs.com/braveym/p/12214367.html

先在pom.xml文件里面添加spark依赖包

<dependency>
      <groupId>org.apache.spark</groupId>
      <artifactId>spark-core_2.11</artifactId>
      <version>2.3.0</version>
    </dependency>

新建一个java类

编写代码

package com.dtiantai;

import org.apache.spark.SparkConf;
import org.apache.spark.api.java.JavaPairRDD;
import org.apache.spark.api.java.JavaRDD;
import org.apache.spark.api.java.JavaSparkContext;
import org.apache.spark.api.java.function.FlatMapFunction;
import org.apache.spark.api.java.function.Function2;
import org.apache.spark.api.java.function.PairFunction;
import scala.Tuple2;

import java.util.Arrays;
import java.util.Iterator;

public class MyJavaWordCount {
    public static void main(String[] args) {
        //参数检测
        if(args.length>2){
            System.err.println("Usage: MyJavaWordCount <input> <output>");
            System.exit(1);
        }

        //创建sparkconf
         SparkConf conf = new SparkConf().setAppName("MyJavaWordCount");
         conf.setMaster("local[2]");

         JavaSparkContext sc=new JavaSparkContext(conf);

         //读取数据
        JavaRDD<String> inputRDD = sc.textFile(args[0]);

        //进行相关计算
        JavaRDD<String> words=inputRDD.flatMap(new FlatMapFunction<String, String>() {

            @Override
            public Iterator<String> call(String s) throws Exception {
                return Arrays.asList(s.split("\s+")).iterator();
            }
        });

        JavaPairRDD<String,Integer> result= words.mapToPair(new PairFunction<String, String, Integer>() {
            @Override
            public Tuple2<String, Integer> call(String s) throws Exception {
                return new Tuple2<>(s, 1);
            }
        }).reduceByKey(new Function2<Integer, Integer, Integer>() {
             @Override
             public Integer call(Integer x, Integer y) throws Exception {
                 return x+y;
             }
         });

        //保存结果
        result.saveAsTextFile(args[1]);

        //关闭sc
        sc.stop();
    }
}

配置输入输出目录

运行程序

D:SoftWareJDK8injava.exe "-javaagent:D:SoftWareIDEAIntelliJ IDEA 2019.1.3libidea_rt.jar=52468:D:SoftWareIDEAIntelliJ IDEA 2019.1.3in" -Dfile.encoding=UTF-8 -classpath D:SoftWareJDK8jrelibcharsets.jar;D:SoftWareJDK8jrelibdeploy.jar;D:SoftWareJDK8jrelibextaccess-bridge-64.jar;D:SoftWareJDK8jrelibextcldrdata.jar;D:SoftWareJDK8jrelibextdnsns.jar;D:SoftWareJDK8jrelibextjaccess.jar;D:SoftWareJDK8jrelibextjfxrt.jar;D:SoftWareJDK8jrelibextlocaledata.jar;D:SoftWareJDK8jrelibext
ashorn.jar;D:SoftWareJDK8jrelibextsunec.jar;D:SoftWareJDK8jrelibextsunjce_provider.jar;D:SoftWareJDK8jrelibextsunmscapi.jar;D:SoftWareJDK8jrelibextsunpkcs11.jar;D:SoftWareJDK8jrelibextzipfs.jar;D:SoftWareJDK8jrelibjavaws.jar;D:SoftWareJDK8jrelibjce.jar;D:SoftWareJDK8jrelibjfr.jar;D:SoftWareJDK8jrelibjfxswt.jar;D:SoftWareJDK8jrelibjsse.jar;D:SoftWareJDK8jrelibmanagement-agent.jar;D:SoftWareJDK8jrelibplugin.jar;D:SoftWareJDK8jrelib
esources.jar;D:SoftWareJDK8jrelib
t.jar;D:MyCode	estJavaAndScala	argetclasses;D:SoftWareScalalibscala-actors-2.11.0.jar;D:SoftWareScalalibscala-actors-migration_2.11-1.1.0.jar;D:SoftWareScalalibscala-library.jar;D:SoftWareScalalibscala-parser-combinators_2.11-1.0.1.jar;D:SoftWareScalalibscala-reflect.jar;D:SoftWareScalalibscala-swing_2.11-1.0.1.jar;D:SoftWareScalalibscala-xml_2.11-1.0.1.jar;D:SoftWareMavenapache-maven-3.6.1Repositoryorgapachesparkspark-core_2.112.3.0spark-core_2.11-2.3.0.jar;D:SoftWareMavenapache-maven-3.6.1Repositoryorgapacheavroavro1.7.7avro-1.7.7.jar;D:SoftWareMavenapache-maven-3.6.1Repositoryorgcodehausjacksonjackson-core-asl1.9.13jackson-core-asl-1.9.13.jar;D:SoftWareMavenapache-maven-3.6.1Repositoryorgcodehausjacksonjackson-mapper-asl1.9.13jackson-mapper-asl-1.9.13.jar;D:SoftWareMavenapache-maven-3.6.1Repositorycom	houghtworksparanamerparanamer2.3paranamer-2.3.jar;D:SoftWareMavenapache-maven-3.6.1Repositoryorgapachecommonscommons-compress1.4.1commons-compress-1.4.1.jar;D:SoftWareMavenapache-maven-3.6.1Repositoryorg	ukaanixz1.0xz-1.0.jar;D:SoftWareMavenapache-maven-3.6.1Repositoryorgapacheavroavro-mapred1.7.7avro-mapred-1.7.7-hadoop2.jar;D:SoftWareMavenapache-maven-3.6.1Repositoryorgapacheavroavro-ipc1.7.7avro-ipc-1.7.7.jar;D:SoftWareMavenapache-maven-3.6.1Repositoryorgapacheavroavro-ipc1.7.7avro-ipc-1.7.7-tests.jar;D:SoftWareMavenapache-maven-3.6.1Repositorycom	witterchill_2.110.8.4chill_2.11-0.8.4.jar;D:SoftWareMavenapache-maven-3.6.1Repositorycomesotericsoftwarekryo-shaded3.0.3kryo-shaded-3.0.3.jar;D:SoftWareMavenapache-maven-3.6.1Repositorycomesotericsoftwareminlog1.3.0minlog-1.3.0.jar;D:SoftWareMavenapache-maven-3.6.1Repositoryorgobjenesisobjenesis2.1objenesis-2.1.jar;D:SoftWareMavenapache-maven-3.6.1Repositorycom	witterchill-java0.8.4chill-java-0.8.4.jar;D:SoftWareMavenapache-maven-3.6.1Repositoryorgapachexbeanxbean-asm5-shaded4.4xbean-asm5-shaded-4.4.jar;D:SoftWareMavenapache-maven-3.6.1Repositoryorgapachehadoophadoop-client2.6.5hadoop-client-2.6.5.jar;D:SoftWareMavenapache-maven-3.6.1Repositoryorgapachehadoophadoop-common2.6.5hadoop-common-2.6.5.jar;D:SoftWareMavenapache-maven-3.6.1Repositorycommons-clicommons-cli1.2commons-cli-1.2.jar;D:SoftWareMavenapache-maven-3.6.1Repositoryxmlencxmlenc0.52xmlenc-0.52.jar;D:SoftWareMavenapache-maven-3.6.1Repositorycommons-httpclientcommons-httpclient3.1commons-httpclient-3.1.jar;D:SoftWareMavenapache-maven-3.6.1Repositorycommons-iocommons-io2.4commons-io-2.4.jar;D:SoftWareMavenapache-maven-3.6.1Repositorycommons-collectionscommons-collections3.2.2commons-collections-3.2.2.jar;D:SoftWareMavenapache-maven-3.6.1Repositorycommons-langcommons-lang2.6commons-lang-2.6.jar;D:SoftWareMavenapache-maven-3.6.1Repositorycommons-configurationcommons-configuration1.6commons-configuration-1.6.jar;D:SoftWareMavenapache-maven-3.6.1Repositorycommons-digestercommons-digester1.8commons-digester-1.8.jar;D:SoftWareMavenapache-maven-3.6.1Repositorycommons-beanutilscommons-beanutils1.7.0commons-beanutils-1.7.0.jar;D:SoftWareMavenapache-maven-3.6.1Repositorycommons-beanutilscommons-beanutils-core1.8.0commons-beanutils-core-1.8.0.jar;D:SoftWareMavenapache-maven-3.6.1Repositorycomgoogleprotobufprotobuf-java2.5.0protobuf-java-2.5.0.jar;D:SoftWareMavenapache-maven-3.6.1Repositorycomgooglecodegsongson2.2.4gson-2.2.4.jar;D:SoftWareMavenapache-maven-3.6.1Repositoryorgapachehadoophadoop-auth2.6.5hadoop-auth-2.6.5.jar;D:SoftWareMavenapache-maven-3.6.1Repositoryorgapachedirectoryserverapacheds-kerberos-codec2.0.0-M15apacheds-kerberos-codec-2.0.0-M15.jar;D:SoftWareMavenapache-maven-3.6.1Repositoryorgapachedirectoryserverapacheds-i18n2.0.0-M15apacheds-i18n-2.0.0-M15.jar;D:SoftWareMavenapache-maven-3.6.1Repositoryorgapachedirectoryapiapi-asn1-api1.0.0-M20api-asn1-api-1.0.0-M20.jar;D:SoftWareMavenapache-maven-3.6.1Repositoryorgapachedirectoryapiapi-util1.0.0-M20api-util-1.0.0-M20.jar;D:SoftWareMavenapache-maven-3.6.1Repositoryorgapachecuratorcurator-client2.6.0curator-client-2.6.0.jar;D:SoftWareMavenapache-maven-3.6.1Repositoryorghtracehtrace-core3.0.4htrace-core-3.0.4.jar;D:SoftWareMavenapache-maven-3.6.1Repositoryorgapachehadoophadoop-hdfs2.6.5hadoop-hdfs-2.6.5.jar;D:SoftWareMavenapache-maven-3.6.1Repositoryorgmortbayjettyjetty-util6.1.26jetty-util-6.1.26.jar;D:SoftWareMavenapache-maven-3.6.1RepositoryxercesxercesImpl2.9.1xercesImpl-2.9.1.jar;D:SoftWareMavenapache-maven-3.6.1Repositoryxml-apisxml-apis1.3.04xml-apis-1.3.04.jar;D:SoftWareMavenapache-maven-3.6.1Repositoryorgapachehadoophadoop-mapreduce-client-app2.6.5hadoop-mapreduce-client-app-2.6.5.jar;D:SoftWareMavenapache-maven-3.6.1Repositoryorgapachehadoophadoop-mapreduce-client-common2.6.5hadoop-mapreduce-client-common-2.6.5.jar;D:SoftWareMavenapache-maven-3.6.1Repositoryorgapachehadoophadoop-yarn-client2.6.5hadoop-yarn-client-2.6.5.jar;D:SoftWareMavenapache-maven-3.6.1Repositoryorgapachehadoophadoop-yarn-server-common2.6.5hadoop-yarn-server-common-2.6.5.jar;D:SoftWareMavenapache-maven-3.6.1Repositoryorgapachehadoophadoop-mapreduce-client-shuffle2.6.5hadoop-mapreduce-client-shuffle-2.6.5.jar;D:SoftWareMavenapache-maven-3.6.1Repositoryorgapachehadoophadoop-yarn-api2.6.5hadoop-yarn-api-2.6.5.jar;D:SoftWareMavenapache-maven-3.6.1Repositoryorgapachehadoophadoop-mapreduce-client-core2.6.5hadoop-mapreduce-client-core-2.6.5.jar;D:SoftWareMavenapache-maven-3.6.1Repositoryorgapachehadoophadoop-yarn-common2.6.5hadoop-yarn-common-2.6.5.jar;D:SoftWareMavenapache-maven-3.6.1Repositoryjavaxxmlindjaxb-api2.2.2jaxb-api-2.2.2.jar;D:SoftWareMavenapache-maven-3.6.1Repositoryjavaxxmlstreamstax-api1.0-2stax-api-1.0-2.jar;D:SoftWareMavenapache-maven-3.6.1Repositoryorgcodehausjacksonjackson-jaxrs1.9.13jackson-jaxrs-1.9.13.jar;D:SoftWareMavenapache-maven-3.6.1Repositoryorgcodehausjacksonjackson-xc1.9.13jackson-xc-1.9.13.jar;D:SoftWareMavenapache-maven-3.6.1Repositoryorgapachehadoophadoop-mapreduce-client-jobclient2.6.5hadoop-mapreduce-client-jobclient-2.6.5.jar;D:SoftWareMavenapache-maven-3.6.1Repositoryorgapachehadoophadoop-annotations2.6.5hadoop-annotations-2.6.5.jar;D:SoftWareMavenapache-maven-3.6.1Repositoryorgapachesparkspark-launcher_2.112.3.0spark-launcher_2.11-2.3.0.jar;D:SoftWareMavenapache-maven-3.6.1Repositoryorgapachesparkspark-kvstore_2.112.3.0spark-kvstore_2.11-2.3.0.jar;D:SoftWareMavenapache-maven-3.6.1Repositoryorgfusesourceleveldbjnileveldbjni-all1.8leveldbjni-all-1.8.jar;D:SoftWareMavenapache-maven-3.6.1Repositorycomfasterxmljacksoncorejackson-core2.6.7jackson-core-2.6.7.jar;D:SoftWareMavenapache-maven-3.6.1Repositorycomfasterxmljacksoncorejackson-annotations2.6.7jackson-annotations-2.6.7.jar;D:SoftWareMavenapache-maven-3.6.1Repositoryorgapachesparkspark-network-common_2.112.3.0spark-network-common_2.11-2.3.0.jar;D:SoftWareMavenapache-maven-3.6.1Repositoryorgapachesparkspark-network-shuffle_2.112.3.0spark-network-shuffle_2.11-2.3.0.jar;D:SoftWareMavenapache-maven-3.6.1Repositoryorgapachesparkspark-unsafe_2.112.3.0spark-unsafe_2.11-2.3.0.jar;D:SoftWareMavenapache-maven-3.6.1Repository
etjavadevjets3tjets3t0.9.4jets3t-0.9.4.jar;D:SoftWareMavenapache-maven-3.6.1Repositoryorgapachehttpcomponentshttpcore4.4.1httpcore-4.4.1.jar;D:SoftWareMavenapache-maven-3.6.1Repositoryorgapachehttpcomponentshttpclient4.5httpclient-4.5.jar;D:SoftWareMavenapache-maven-3.6.1Repositorycommons-codeccommons-codec1.15-SNAPSHOTcommons-codec-1.15-20200118.020329-6.jar;D:SoftWareMavenapache-maven-3.6.1Repositoryjavaxactivationactivation1.1.1activation-1.1.1.jar;D:SoftWareMavenapache-maven-3.6.1Repositoryorgouncycastlecprov-jdk15on1.52cprov-jdk15on-1.52.jar;D:SoftWareMavenapache-maven-3.6.1Repositorycomjamesmurtyutilsjava-xmlbuilder1.1java-xmlbuilder-1.1.jar;D:SoftWareMavenapache-maven-3.6.1Repository
etiharderase642.3.8ase64-2.3.8.jar;D:SoftWareMavenapache-maven-3.6.1Repositoryorgapachecuratorcurator-recipes2.6.0curator-recipes-2.6.0.jar;D:SoftWareMavenapache-maven-3.6.1Repositoryorgapachecuratorcurator-framework2.6.0curator-framework-2.6.0.jar;D:SoftWareMavenapache-maven-3.6.1Repositoryorgapachezookeeperzookeeper3.4.6zookeeper-3.4.6.jar;D:SoftWareMavenapache-maven-3.6.1Repositorycomgoogleguavaguava16.0.1guava-16.0.1.jar;D:SoftWareMavenapache-maven-3.6.1Repositoryjavaxservletjavax.servlet-api3.1.0javax.servlet-api-3.1.0.jar;D:SoftWareMavenapache-maven-3.6.1Repositoryorgapachecommonscommons-lang33.5commons-lang3-3.5.jar;D:SoftWareMavenapache-maven-3.6.1Repositoryorgapachecommonscommons-math33.4.1commons-math3-3.4.1.jar;D:SoftWareMavenapache-maven-3.6.1Repositorycomgooglecodefindbugsjsr3051.3.9jsr305-1.3.9.jar;D:SoftWareMavenapache-maven-3.6.1Repositoryorgslf4jslf4j-api1.7.16slf4j-api-1.7.16.jar;D:SoftWareMavenapache-maven-3.6.1Repositoryorgslf4jjul-to-slf4j1.7.16jul-to-slf4j-1.7.16.jar;D:SoftWareMavenapache-maven-3.6.1Repositoryorgslf4jjcl-over-slf4j1.7.16jcl-over-slf4j-1.7.16.jar;D:SoftWareMavenapache-maven-3.6.1Repositorylog4jlog4j1.2.17log4j-1.2.17.jar;D:SoftWareMavenapache-maven-3.6.1Repositoryorgslf4jslf4j-log4j121.7.16slf4j-log4j12-1.7.16.jar;D:SoftWareMavenapache-maven-3.6.1Repositorycom
ingcompress-lzf1.0.3compress-lzf-1.0.3.jar;D:SoftWareMavenapache-maven-3.6.1Repositoryorgxerialsnappysnappy-java1.1.2.6snappy-java-1.1.2.6.jar;D:SoftWareMavenapache-maven-3.6.1Repositoryorglz4lz4-java1.4.0lz4-java-1.4.0.jar;D:SoftWareMavenapache-maven-3.6.1Repositorycomgithublubenzstd-jni1.3.2-2zstd-jni-1.3.2-2.jar;D:SoftWareMavenapache-maven-3.6.1Repositoryorg
oaringbitmapRoaringBitmap0.5.11RoaringBitmap-0.5.11.jar;D:SoftWareMavenapache-maven-3.6.1Repositorycommons-netcommons-net2.2commons-net-2.2.jar;D:SoftWareMavenapache-maven-3.6.1Repositoryorgscala-langscala-library2.11.8scala-library-2.11.8.jar;D:SoftWareMavenapache-maven-3.6.1Repositoryorgjson4sjson4s-jackson_2.113.2.11json4s-jackson_2.11-3.2.11.jar;D:SoftWareMavenapache-maven-3.6.1Repositoryorgjson4sjson4s-core_2.113.2.11json4s-core_2.11-3.2.11.jar;D:SoftWareMavenapache-maven-3.6.1Repositoryorgjson4sjson4s-ast_2.113.2.11json4s-ast_2.11-3.2.11.jar;D:SoftWareMavenapache-maven-3.6.1Repositoryorgscala-langscalap2.11.0scalap-2.11.0.jar;D:SoftWareMavenapache-maven-3.6.1Repositoryorgscala-langscala-compiler2.11.0scala-compiler-2.11.0.jar;D:SoftWareMavenapache-maven-3.6.1Repositoryorgscala-langmodulesscala-xml_2.111.0.1scala-xml_2.11-1.0.1.jar;D:SoftWareMavenapache-maven-3.6.1Repositoryorgscala-langmodulesscala-parser-combinators_2.111.0.1scala-parser-combinators_2.11-1.0.1.jar;D:SoftWareMavenapache-maven-3.6.1Repositoryorgglassfishjerseycorejersey-client2.22.2jersey-client-2.22.2.jar;D:SoftWareMavenapache-maven-3.6.1Repositoryjavaxws
sjavax.ws.rs-api2.0.1javax.ws.rs-api-2.0.1.jar;D:SoftWareMavenapache-maven-3.6.1Repositoryorgglassfishhk2hk2-api2.4.0-b34hk2-api-2.4.0-b34.jar;D:SoftWareMavenapache-maven-3.6.1Repositoryorgglassfishhk2hk2-utils2.4.0-b34hk2-utils-2.4.0-b34.jar;D:SoftWareMavenapache-maven-3.6.1Repositoryorgglassfishhk2externalaopalliance-repackaged2.4.0-b34aopalliance-repackaged-2.4.0-b34.jar;D:SoftWareMavenapache-maven-3.6.1Repositoryorgglassfishhk2externaljavax.inject2.4.0-b34javax.inject-2.4.0-b34.jar;D:SoftWareMavenapache-maven-3.6.1Repositoryorgglassfishhk2hk2-locator2.4.0-b34hk2-locator-2.4.0-b34.jar;D:SoftWareMavenapache-maven-3.6.1Repositoryorgjavassistjavassist3.18.1-GAjavassist-3.18.1-GA.jar;D:SoftWareMavenapache-maven-3.6.1Repositoryorgglassfishjerseycorejersey-common2.22.2jersey-common-2.22.2.jar;D:SoftWareMavenapache-maven-3.6.1Repositoryjavaxannotationjavax.annotation-api1.2javax.annotation-api-1.2.jar;D:SoftWareMavenapache-maven-3.6.1Repositoryorgglassfishjerseyundles
epackagedjersey-guava2.22.2jersey-guava-2.22.2.jar;D:SoftWareMavenapache-maven-3.6.1Repositoryorgglassfishhk2osgi-resource-locator1.0.1osgi-resource-locator-1.0.1.jar;D:SoftWareMavenapache-maven-3.6.1Repositoryorgglassfishjerseycorejersey-server2.22.2jersey-server-2.22.2.jar;D:SoftWareMavenapache-maven-3.6.1Repositoryorgglassfishjerseymediajersey-media-jaxb2.22.2jersey-media-jaxb-2.22.2.jar;D:SoftWareMavenapache-maven-3.6.1Repositoryjavaxvalidationvalidation-api1.1.0.Finalvalidation-api-1.1.0.Final.jar;D:SoftWareMavenapache-maven-3.6.1Repositoryorgglassfishjerseycontainersjersey-container-servlet2.22.2jersey-container-servlet-2.22.2.jar;D:SoftWareMavenapache-maven-3.6.1Repositoryorgglassfishjerseycontainersjersey-container-servlet-core2.22.2jersey-container-servlet-core-2.22.2.jar;D:SoftWareMavenapache-maven-3.6.1Repositoryio
etty
etty-all4.1.17.Final
etty-all-4.1.17.Final.jar;D:SoftWareMavenapache-maven-3.6.1Repositoryio
etty
etty3.9.9.Final
etty-3.9.9.Final.jar;D:SoftWareMavenapache-maven-3.6.1Repositorycomclearspringanalyticsstream2.7.0stream-2.7.0.jar;D:SoftWareMavenapache-maven-3.6.1Repositoryiodropwizardmetricsmetrics-core3.1.5metrics-core-3.1.5.jar;D:SoftWareMavenapache-maven-3.6.1Repositoryiodropwizardmetricsmetrics-jvm3.1.5metrics-jvm-3.1.5.jar;D:SoftWareMavenapache-maven-3.6.1Repositoryiodropwizardmetricsmetrics-json3.1.5metrics-json-3.1.5.jar;D:SoftWareMavenapache-maven-3.6.1Repositoryiodropwizardmetricsmetrics-graphite3.1.5metrics-graphite-3.1.5.jar;D:SoftWareMavenapache-maven-3.6.1Repositorycomfasterxmljacksoncorejackson-databind2.6.7.1jackson-databind-2.6.7.1.jar;D:SoftWareMavenapache-maven-3.6.1Repositorycomfasterxmljacksonmodulejackson-module-scala_2.112.6.7.1jackson-module-scala_2.11-2.6.7.1.jar;D:SoftWareMavenapache-maven-3.6.1Repositoryorgscala-langscala-reflect2.11.8scala-reflect-2.11.8.jar;D:SoftWareMavenapache-maven-3.6.1Repositorycomfasterxmljacksonmodulejackson-module-paranamer2.7.9jackson-module-paranamer-2.7.9.jar;D:SoftWareMavenapache-maven-3.6.1Repositoryorgapacheivyivy2.4.0ivy-2.4.0.jar;D:SoftWareMavenapache-maven-3.6.1Repositoryorooro2.0.8oro-2.0.8.jar;D:SoftWareMavenapache-maven-3.6.1Repository
et
azorvinepyrolite4.13pyrolite-4.13.jar;D:SoftWareMavenapache-maven-3.6.1Repository
etsfpy4jpy4j0.10.6py4j-0.10.6.jar;D:SoftWareMavenapache-maven-3.6.1Repositoryorgapachesparkspark-tags_2.112.3.0spark-tags_2.11-2.3.0.jar;D:SoftWareMavenapache-maven-3.6.1Repositoryorgapachecommonscommons-crypto1.0.0commons-crypto-1.0.0.jar;D:SoftWareMavenapache-maven-3.6.1Repositoryorgspark-projectsparkunused1.0.0unused-1.0.0.jar com.dtiantai.MyJavaWordCount D:MyCode	estJavaAndScaladatawc.txt D:MyCode	estJavaAndScaladataout
Using Spark's default log4j profile: org/apache/spark/log4j-defaults.properties
20/01/19 18:14:04 INFO SparkContext: Running Spark version 2.3.0
20/01/19 18:14:04 INFO SparkContext: Submitted application: MyJavaWordCount
20/01/19 18:14:04 INFO SecurityManager: Changing view acls to: admin
20/01/19 18:14:04 INFO SecurityManager: Changing modify acls to: admin
20/01/19 18:14:04 INFO SecurityManager: Changing view acls groups to: 
20/01/19 18:14:04 INFO SecurityManager: Changing modify acls groups to: 
20/01/19 18:14:04 INFO SecurityManager: SecurityManager: authentication disabled; ui acls disabled; users  with view permissions: Set(admin); groups with view permissions: Set(); users  with modify permissions: Set(admin); groups with modify permissions: Set()
20/01/19 18:14:05 INFO Utils: Successfully started service 'sparkDriver' on port 52491.
20/01/19 18:14:05 INFO SparkEnv: Registering MapOutputTracker
20/01/19 18:14:05 INFO SparkEnv: Registering BlockManagerMaster
20/01/19 18:14:05 INFO BlockManagerMasterEndpoint: Using org.apache.spark.storage.DefaultTopologyMapper for getting topology information
20/01/19 18:14:05 INFO BlockManagerMasterEndpoint: BlockManagerMasterEndpoint up
20/01/19 18:14:05 INFO DiskBlockManager: Created local directory at C:UsersadminAppDataLocalTemplockmgr-f405ac37-c1ff-4a8a-a490-1144a192f758
20/01/19 18:14:05 INFO MemoryStore: MemoryStore started with capacity 902.7 MB
20/01/19 18:14:05 INFO SparkEnv: Registering OutputCommitCoordinator
20/01/19 18:14:05 INFO Utils: Successfully started service 'SparkUI' on port 4040.
20/01/19 18:14:05 INFO SparkUI: Bound SparkUI to 0.0.0.0, and started at http://GX-GYM-D8178:4040
20/01/19 18:14:05 INFO Executor: Starting executor ID driver on host localhost
20/01/19 18:14:05 INFO Utils: Successfully started service 'org.apache.spark.network.netty.NettyBlockTransferService' on port 52504.
20/01/19 18:14:05 INFO NettyBlockTransferService: Server created on GX-GYM-D8178:52504
20/01/19 18:14:05 INFO BlockManager: Using org.apache.spark.storage.RandomBlockReplicationPolicy for block replication policy
20/01/19 18:14:05 INFO BlockManagerMaster: Registering BlockManager BlockManagerId(driver, GX-GYM-D8178, 52504, None)
20/01/19 18:14:05 INFO BlockManagerMasterEndpoint: Registering block manager GX-GYM-D8178:52504 with 902.7 MB RAM, BlockManagerId(driver, GX-GYM-D8178, 52504, None)
20/01/19 18:14:05 INFO BlockManagerMaster: Registered BlockManager BlockManagerId(driver, GX-GYM-D8178, 52504, None)
20/01/19 18:14:05 INFO BlockManager: Initialized BlockManager: BlockManagerId(driver, GX-GYM-D8178, 52504, None)
20/01/19 18:14:06 INFO MemoryStore: Block broadcast_0 stored as values in memory (estimated size 214.5 KB, free 902.5 MB)
20/01/19 18:14:07 INFO MemoryStore: Block broadcast_0_piece0 stored as bytes in memory (estimated size 20.4 KB, free 902.5 MB)
20/01/19 18:14:07 INFO BlockManagerInfo: Added broadcast_0_piece0 in memory on GX-GYM-D8178:52504 (size: 20.4 KB, free: 902.7 MB)
20/01/19 18:14:07 INFO SparkContext: Created broadcast 0 from textFile at MyJavaWordCount.java:30
20/01/19 18:14:07 INFO FileInputFormat: Total input paths to process : 1
20/01/19 18:14:07 INFO deprecation: mapred.output.dir is deprecated. Instead, use mapreduce.output.fileoutputformat.outputdir
20/01/19 18:14:07 INFO SparkContext: Starting job: runJob at SparkHadoopWriter.scala:78
20/01/19 18:14:07 INFO DAGScheduler: Registering RDD 3 (mapToPair at MyJavaWordCount.java:41)
20/01/19 18:14:07 INFO DAGScheduler: Got job 0 (runJob at SparkHadoopWriter.scala:78) with 2 output partitions
20/01/19 18:14:07 INFO DAGScheduler: Final stage: ResultStage 1 (runJob at SparkHadoopWriter.scala:78)
20/01/19 18:14:07 INFO DAGScheduler: Parents of final stage: List(ShuffleMapStage 0)
20/01/19 18:14:07 INFO DAGScheduler: Missing parents: List(ShuffleMapStage 0)
20/01/19 18:14:07 INFO DAGScheduler: Submitting ShuffleMapStage 0 (MapPartitionsRDD[3] at mapToPair at MyJavaWordCount.java:41), which has no missing parents
20/01/19 18:14:07 INFO MemoryStore: Block broadcast_1 stored as values in memory (estimated size 5.1 KB, free 902.5 MB)
20/01/19 18:14:07 INFO MemoryStore: Block broadcast_1_piece0 stored as bytes in memory (estimated size 2.9 KB, free 902.5 MB)
20/01/19 18:14:07 INFO BlockManagerInfo: Added broadcast_1_piece0 in memory on GX-GYM-D8178:52504 (size: 2.9 KB, free: 902.7 MB)
20/01/19 18:14:07 INFO SparkContext: Created broadcast 1 from broadcast at DAGScheduler.scala:1039
20/01/19 18:14:07 INFO DAGScheduler: Submitting 2 missing tasks from ShuffleMapStage 0 (MapPartitionsRDD[3] at mapToPair at MyJavaWordCount.java:41) (first 15 tasks are for partitions Vector(0, 1))
20/01/19 18:14:07 INFO TaskSchedulerImpl: Adding task set 0.0 with 2 tasks
20/01/19 18:14:07 INFO TaskSetManager: Starting task 0.0 in stage 0.0 (TID 0, localhost, executor driver, partition 0, PROCESS_LOCAL, 7880 bytes)
20/01/19 18:14:07 INFO TaskSetManager: Starting task 1.0 in stage 0.0 (TID 1, localhost, executor driver, partition 1, PROCESS_LOCAL, 7880 bytes)
20/01/19 18:14:07 INFO Executor: Running task 1.0 in stage 0.0 (TID 1)
20/01/19 18:14:07 INFO Executor: Running task 0.0 in stage 0.0 (TID 0)
20/01/19 18:14:07 INFO HadoopRDD: Input split: file:/D:/MyCode/testJavaAndScala/data/wc.txt:0+32
20/01/19 18:14:07 INFO HadoopRDD: Input split: file:/D:/MyCode/testJavaAndScala/data/wc.txt:32+33
20/01/19 18:14:07 INFO Executor: Finished task 1.0 in stage 0.0 (TID 1). 1154 bytes result sent to driver
20/01/19 18:14:07 INFO Executor: Finished task 0.0 in stage 0.0 (TID 0). 1111 bytes result sent to driver
20/01/19 18:14:07 INFO TaskSetManager: Finished task 0.0 in stage 0.0 (TID 0) in 194 ms on localhost (executor driver) (1/2)
20/01/19 18:14:07 INFO TaskSetManager: Finished task 1.0 in stage 0.0 (TID 1) in 182 ms on localhost (executor driver) (2/2)
20/01/19 18:14:07 INFO TaskSchedulerImpl: Removed TaskSet 0.0, whose tasks have all completed, from pool 
20/01/19 18:14:07 INFO DAGScheduler: ShuffleMapStage 0 (mapToPair at MyJavaWordCount.java:41) finished in 0.310 s
20/01/19 18:14:07 INFO DAGScheduler: looking for newly runnable stages
20/01/19 18:14:07 INFO DAGScheduler: running: Set()
20/01/19 18:14:07 INFO DAGScheduler: waiting: Set(ResultStage 1)
20/01/19 18:14:07 INFO DAGScheduler: failed: Set()
20/01/19 18:14:07 INFO DAGScheduler: Submitting ResultStage 1 (MapPartitionsRDD[5] at saveAsTextFile at MyJavaWordCount.java:54), which has no missing parents
20/01/19 18:14:07 INFO MemoryStore: Block broadcast_2 stored as values in memory (estimated size 65.5 KB, free 902.4 MB)
20/01/19 18:14:07 INFO MemoryStore: Block broadcast_2_piece0 stored as bytes in memory (estimated size 23.4 KB, free 902.4 MB)
20/01/19 18:14:07 INFO BlockManagerInfo: Added broadcast_2_piece0 in memory on GX-GYM-D8178:52504 (size: 23.4 KB, free: 902.7 MB)
20/01/19 18:14:07 INFO SparkContext: Created broadcast 2 from broadcast at DAGScheduler.scala:1039
20/01/19 18:14:07 INFO DAGScheduler: Submitting 2 missing tasks from ResultStage 1 (MapPartitionsRDD[5] at saveAsTextFile at MyJavaWordCount.java:54) (first 15 tasks are for partitions Vector(0, 1))
20/01/19 18:14:07 INFO TaskSchedulerImpl: Adding task set 1.0 with 2 tasks
20/01/19 18:14:07 INFO TaskSetManager: Starting task 0.0 in stage 1.0 (TID 2, localhost, executor driver, partition 0, ANY, 7649 bytes)
20/01/19 18:14:07 INFO TaskSetManager: Starting task 1.0 in stage 1.0 (TID 3, localhost, executor driver, partition 1, ANY, 7649 bytes)
20/01/19 18:14:07 INFO Executor: Running task 0.0 in stage 1.0 (TID 2)
20/01/19 18:14:07 INFO Executor: Running task 1.0 in stage 1.0 (TID 3)
20/01/19 18:14:07 INFO ShuffleBlockFetcherIterator: Getting 1 non-empty blocks out of 2 blocks
20/01/19 18:14:07 INFO ShuffleBlockFetcherIterator: Getting 2 non-empty blocks out of 2 blocks
20/01/19 18:14:07 INFO ShuffleBlockFetcherIterator: Started 0 remote fetches in 8 ms
20/01/19 18:14:07 INFO ShuffleBlockFetcherIterator: Started 0 remote fetches in 4 ms
20/01/19 18:14:08 INFO FileOutputCommitter: Saved output of task 'attempt_20200119181407_0005_m_000000_0' to file:/D:/MyCode/testJavaAndScala/data/out/_temporary/0/task_20200119181407_0005_m_000000
20/01/19 18:14:08 INFO SparkHadoopMapRedUtil: attempt_20200119181407_0005_m_000000_0: Committed
20/01/19 18:14:08 INFO FileOutputCommitter: Saved output of task 'attempt_20200119181407_0005_m_000001_0' to file:/D:/MyCode/testJavaAndScala/data/out/_temporary/0/task_20200119181407_0005_m_000001
20/01/19 18:14:08 INFO SparkHadoopMapRedUtil: attempt_20200119181407_0005_m_000001_0: Committed
20/01/19 18:14:08 INFO Executor: Finished task 0.0 in stage 1.0 (TID 2). 1502 bytes result sent to driver
20/01/19 18:14:08 INFO Executor: Finished task 1.0 in stage 1.0 (TID 3). 1502 bytes result sent to driver
20/01/19 18:14:08 INFO TaskSetManager: Finished task 1.0 in stage 1.0 (TID 3) in 172 ms on localhost (executor driver) (1/2)
20/01/19 18:14:08 INFO TaskSetManager: Finished task 0.0 in stage 1.0 (TID 2) in 174 ms on localhost (executor driver) (2/2)
20/01/19 18:14:08 INFO TaskSchedulerImpl: Removed TaskSet 1.0, whose tasks have all completed, from pool 
20/01/19 18:14:08 INFO DAGScheduler: ResultStage 1 (runJob at SparkHadoopWriter.scala:78) finished in 0.205 s
20/01/19 18:14:08 INFO DAGScheduler: Job 0 finished: runJob at SparkHadoopWriter.scala:78, took 0.652025 s
20/01/19 18:14:08 INFO SparkHadoopWriter: Job job_20200119181407_0005 committed.
20/01/19 18:14:08 INFO SparkUI: Stopped Spark web UI at http://GX-GYM-D8178:4040
20/01/19 18:14:08 INFO MapOutputTrackerMasterEndpoint: MapOutputTrackerMasterEndpoint stopped!
20/01/19 18:14:08 INFO MemoryStore: MemoryStore cleared
20/01/19 18:14:08 INFO BlockManager: BlockManager stopped
20/01/19 18:14:08 INFO BlockManagerMaster: BlockManagerMaster stopped
20/01/19 18:14:08 INFO OutputCommitCoordinator$OutputCommitCoordinatorEndpoint: OutputCommitCoordinator stopped!
20/01/19 18:14:08 INFO SparkContext: Successfully stopped SparkContext
20/01/19 18:14:08 INFO ShutdownHookManager: Shutdown hook called
20/01/19 18:14:08 INFO ShutdownHookManager: Deleting directory C:UsersadminAppDataLocalTempspark-09da7888-77f6-41df-b053-d9cc5d3f1b2b

Process finished with exit code 0

查看输出结果