spark用的是cdh spark-2.0.1
package main.scala
import org.apache.spark.rdd.RDD
import org.apache.spark.{SparkConf, SparkContext}
/* Created by tomtang on 2016/12/16.
*/
object tomhelloworld {
def FILE_NAME:String = "word_count_results_"
def main(args: Array[String]) {
// if (args.length < 1) {
// println("Usage:SparkWordCount FileName");
// System.exit(1);
// }
val conf = {
new SparkConf().setMaster("local[*]").setAppName("spark_test")
}
val sc = new SparkContext(conf)
val textFile = sc.textFile("D:\IdeaProjects\first_scala\data\testdata.txt")
val wordCounts: RDD[(String, Int)] = textFile.flatMap(line => line split " ").map(
word => (word, 1)).reduceByKey((a, b) => a + b)
//print the results,for debug use.
//println("Word Count program running results:")
//wordCounts.collect().foreach(e => {
//val (k,v) = e
//println(k+"="+v)
//});
wordCounts.saveAsTextFile(FILE_NAME+System.currentTimeMillis())
println("Word Count program running results are successfully saved.")
println("Hello World")
}
}
报错:
Exception in thread "main" java.lang.NoSuchMethodError: scala.Predef$.refArrayOps([Ljava/lang/Object;)Lscala/collection/mutable/ArrayOps;
at org.apache.spark.util.Utils$.getCallSite(Utils.scala:1410)
at org.apache.spark.SparkContext.<init>(SparkContext.scala:77)
at sparkTraining.SimpleApp$.main(testScala.scala:9)
at sparkTraining.SimpleApp.main(testScala.scala)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.intellij.rt.execution.application.AppMain.main(AppMain.java:147)
参考http://blog.csdn.net/u013887486/article/details/53201900
因为没有按照文档里面的Scala版本号下载对应的Scala,下载对应的版本2.11.7
发现是scala的版本问题。
选ivy2.10.4