package stream.scala import java.io.PrintWriter import java.net.ServerSocket class LoggerSimulation { } object LoggerSimulation{ /** * 生成一个字母 * @param 字母的下标 * @return 生成的字母 */ def gennerateContent(index:Int):String = { import scala.collection.mutable.ListBuffer val charList = ListBuffer[Char](); for (i<- 65 to 90){ charList += i.toChar } val charArray = charList.toArray charArray(index).toString(); } /** * 生成随机下标 * @return 返回一个下标 */ def index = { import java.util.Random val rdm = new Random() rdm.nextInt(7) } /** * 启动一个main方法来创建一个serversockt发送消息 * @param args 端口，发送的时间间隔 */ def main(args: Array[String]): Unit = { // if (args.length !=2){ // System.err.println("Usage:<port><millisecond>") // System.exit(1); // } val listener = new ServerSocket(8888.toInt) println("已经做好连接的准备-------") while(true){ val socket = listener.accept() new Thread(){ override def run(): Unit = { println("Got client connected from:"+socket.getInetAddress) val out = new PrintWriter(socket.getOutputStream,true) while(true){ Thread.sleep(1000.toLong) val content = gennerateContent(index) println(content) out.write(content+" ") out.flush() } socket.close() } }.start() } } } -------------------------------------------------- package stream.scala import org.apache.spark.storage.StorageLevel import org.apache.spark.streaming.{Seconds, StreamingContext} import org.apache.spark.{SparkConf, SparkContext} /** * 这是一个接收来自网络端口的信息 * 参数 spark集群的主节点地址，网络通信的节点地址，网络通信的端口，每个多长时间作为一个单位进行执行任务 * local[*] localhost 8888 5 */ class NetWorkWordCount { } object NetWorkWordCount{ def main(args: Array[String]): Unit = { // if(args.length!=4){ // System.err.println("Usage:NetWorkWordCount<master> <hostname> <port> <seconds> +" + // "In local modle,<master> should be 'local[n]' with n >1 "); // System.exit(1); // } val config = new SparkConf().setAppName("NetWorkWordCount").setMaster("local[*]"); val ssc = new StreamingContext(config,Seconds(5.toInt)); val lines = ssc.socketTextStream("localhost",8888.toInt,StorageLevel.MEMORY_ONLY_SER) val words = lines.flatMap(line => line.split(" ")); val wordCount = words.map(x=>(x,1)).reduceByKey(_+_); wordCount.print() ssc.start(); ssc.awaitTermination(); } } ************************************************************************* 18/01/12 10:52:25 INFO MemoryStore: ensureFreeSpace(2304) called with curMem=112292, maxMem=490356080 18/01/12 10:52:25 INFO MemoryStore: Block broadcast_136 stored as values in memory (estimated size 2.3 KB, free 467.5 MB) 18/01/12 10:52:25 INFO MemoryStore: ensureFreeSpace(1417) called with curMem=114596, maxMem=490356080 18/01/12 10:52:25 INFO MemoryStore: Block broadcast_136_piece0 stored as bytes in memory (estimated size 1417.0 B, free 467.5 MB) 18/01/12 10:52:25 INFO BlockManagerInfo: Added broadcast_136_piece0 in memory on localhost:54439 (size: 1417.0 B, free: 467.6 MB) 18/01/12 10:52:25 INFO SparkContext: Created broadcast 136 from broadcast at DAGScheduler.scala:874 18/01/12 10:52:25 INFO DAGScheduler: Submitting 1 missing tasks from ResultStage 182 (ShuffledRDD[184] at reduceByKey at NetWorkWordCount.scala:28) 18/01/12 10:52:25 INFO TaskSchedulerImpl: Adding task set 182.0 with 1 tasks 18/01/12 10:52:25 INFO TaskSetManager: Starting task 0.0 in stage 182.0 (TID 402, localhost, PROCESS_LOCAL, 1165 bytes) 18/01/12 10:52:25 INFO Executor: Running task 0.0 in stage 182.0 (TID 402) 18/01/12 10:52:25 INFO ShuffleBlockFetcherIterator: Getting 2 non-empty blocks out of 5 blocks 18/01/12 10:52:25 INFO ShuffleBlockFetcherIterator: Started 0 remote fetches in 0 ms 18/01/12 10:52:25 INFO Executor: Finished task 0.0 in stage 182.0 (TID 402). 1028 bytes result sent to driver 18/01/12 10:52:25 INFO TaskSetManager: Finished task 0.0 in stage 182.0 (TID 402) in 2 ms on localhost (1/1) 18/01/12 10:52:25 INFO TaskSchedulerImpl: Removed TaskSet 182.0, whose tasks have all completed, from pool 18/01/12 10:52:25 INFO DAGScheduler: ResultStage 182 (print at NetWorkWordCount.scala:29) finished in 0.002 s 18/01/12 10:52:25 INFO DAGScheduler: Job 91 finished: print at NetWorkWordCount.scala:29, took 0.058539 s 18/01/12 10:52:25 INFO SparkContext: Starting job: print at NetWorkWordCount.scala:29 18/01/12 10:52:25 INFO MapOutputTrackerMaster: Size of output statuses for shuffle 45 is 167 bytes 18/01/12 10:52:25 INFO DAGScheduler: Got job 92 (print at NetWorkWordCount.scala:29) with 3 output partitions (allowLocal=true) 18/01/12 10:52:25 INFO DAGScheduler: Final stage: ResultStage 184(print at NetWorkWordCount.scala:29) 18/01/12 10:52:25 INFO DAGScheduler: Parents of final stage: List(ShuffleMapStage 183) 18/01/12 10:52:25 INFO DAGScheduler: Missing parents: List() 18/01/12 10:52:25 INFO DAGScheduler: Submitting ResultStage 184 (ShuffledRDD[184] at reduceByKey at NetWorkWordCount.scala:28), which has no missing parents 18/01/12 10:52:25 INFO MemoryStore: ensureFreeSpace(2304) called with curMem=116013, maxMem=490356080 18/01/12 10:52:25 INFO MemoryStore: Block broadcast_137 stored as values in memory (estimated size 2.3 KB, free 467.5 MB) 18/01/12 10:52:25 INFO MemoryStore: ensureFreeSpace(1417) called with curMem=118317, maxMem=490356080 18/01/12 10:52:25 INFO MemoryStore: Block broadcast_137_piece0 stored as bytes in memory (estimated size 1417.0 B, free 467.5 MB) 18/01/12 10:52:25 INFO BlockManagerInfo: Added broadcast_137_piece0 in memory on localhost:54439 (size: 1417.0 B, free: 467.6 MB) 18/01/12 10:52:25 INFO SparkContext: Created broadcast 137 from broadcast at DAGScheduler.scala:874 18/01/12 10:52:25 INFO DAGScheduler: Submitting 3 missing tasks from ResultStage 184 (ShuffledRDD[184] at reduceByKey at NetWorkWordCount.scala:28) 18/01/12 10:52:25 INFO TaskSchedulerImpl: Adding task set 184.0 with 3 tasks 18/01/12 10:52:25 INFO TaskSetManager: Starting task 0.0 in stage 184.0 (TID 403, localhost, PROCESS_LOCAL, 1165 bytes) 18/01/12 10:52:25 INFO TaskSetManager: Starting task 1.0 in stage 184.0 (TID 404, localhost, PROCESS_LOCAL, 1165 bytes) 18/01/12 10:52:25 INFO TaskSetManager: Starting task 2.0 in stage 184.0 (TID 405, localhost, PROCESS_LOCAL, 1165 bytes) 18/01/12 10:52:25 INFO Executor: Running task 1.0 in stage 184.0 (TID 404) 18/01/12 10:52:25 INFO Executor: Running task 2.0 in stage 184.0 (TID 405) 18/01/12 10:52:25 INFO Executor: Running task 0.0 in stage 184.0 (TID 403) 18/01/12 10:52:25 INFO ShuffleBlockFetcherIterator: Getting 2 non-empty blocks out of 5 blocks 18/01/12 10:52:25 INFO ShuffleBlockFetcherIterator: Started 0 remote fetches in 0 ms 18/01/12 10:52:25 INFO ShuffleBlockFetcherIterator: Getting 0 non-empty blocks out of 5 blocks 18/01/12 10:52:25 INFO ShuffleBlockFetcherIterator: Started 0 remote fetches in 0 ms 18/01/12 10:52:25 INFO ShuffleBlockFetcherIterator: Getting 1 non-empty blocks out of 5 blocks 18/01/12 10:52:25 INFO ShuffleBlockFetcherIterator: Started 0 remote fetches in 0 ms 18/01/12 10:52:25 INFO Executor: Finished task 2.0 in stage 184.0 (TID 405). 882 bytes result sent to driver 18/01/12 10:52:25 INFO Executor: Finished task 1.0 in stage 184.0 (TID 404). 1028 bytes result sent to driver 18/01/12 10:52:25 INFO Executor: Finished task 0.0 in stage 184.0 (TID 403). 1028 bytes result sent to driver 18/01/12 10:52:25 INFO TaskSetManager: Finished task 2.0 in stage 184.0 (TID 405) in 2 ms on localhost (1/3) 18/01/12 10:52:25 INFO TaskSetManager: Finished task 1.0 in stage 184.0 (TID 404) in 2 ms on localhost (2/3) 18/01/12 10:52:25 INFO TaskSetManager: Finished task 0.0 in stage 184.0 (TID 403) in 2 ms on localhost (3/3) 18/01/12 10:52:25 INFO DAGScheduler: ResultStage 184 (print at NetWorkWordCount.scala:29) finished in 0.002 s 18/01/12 10:52:25 INFO TaskSchedulerImpl: Removed TaskSet 184.0, whose tasks have all completed, from pool 18/01/12 10:52:25 INFO DAGScheduler: Job 92 finished: print at NetWorkWordCount.scala:29, took 0.005325 s ------------------------------------------- Time: 1515725545000 ms ------------------------------------------- 18/01/12 10:52:25 INFO JobScheduler: Finished job streaming job 1515725545000 ms.0 from job set of time 1515725545000 ms 18/01/12 10:52:25 INFO JobScheduler: Total delay: 0.090 s for time 1515725545000 ms (execution: 0.066 s) 18/01/12 10:52:25 INFO ShuffledRDD: Removing RDD 180 from persistence list 18/01/12 10:52:25 INFO BlockManager: Removing RDD 180 18/01/12 10:52:25 INFO MapPartitionsRDD: Removing RDD 179 from persistence list 18/01/12 10:52:25 INFO BlockManager: Removing RDD 179 18/01/12 10:52:25 INFO MapPartitionsRDD: Removing RDD 178 from persistence list 18/01/12 10:52:25 INFO BlockManager: Removing RDD 178 18/01/12 10:52:25 INFO BlockRDD: Removing RDD 177 from persistence list 18/01/12 10:52:25 INFO BlockManager: Removing RDD 177 18/01/12 10:52:25 INFO SocketInputDStream: Removing blocks of RDD BlockRDD[177] at socketTextStream at NetWorkWordCount.scala:26 of time 1515725545000 ms (D,2) (E,1) (F,2) 18/01/12 10:52:25 INFO BlockManagerInfo: Removed input-0-1515725535600 on localhost:54439 in memory (size: 8.0 B, free: 467.6 MB) 18/01/12 10:52:25 INFO BlockManagerInfo: Removed input-0-1515725536600 on localhost:54439 in memory (size: 8.0 B, free: 467.6 MB) 18/01/12 10:52:25 INFO BlockManagerInfo: Removed input-0-1515725537600 on localhost:54439 in memory (size: 8.0 B, free: 467.6 MB) 18/01/12 10:52:25 INFO BlockManagerInfo: Removed input-0-1515725538600 on localhost:54439 in memory (size: 8.0 B, free: 467.6 MB) 18/01/12 10:52:25 INFO ReceivedBlockTracker: Deleting batches ArrayBuffer(1515725535000 ms) 18/01/12 10:52:25 INFO InputInfoTracker: remove old batch metadata: 1515725535000 ms 18/01/12 10:52:25 INFO BlockManagerInfo: Removed input-0-1515725539600 on localhost:54439 in memory (size: 8.0 B, free: 467.6 MB) 18/01/12 10:52:25 INFO MemoryStore: ensureFreeSpace(8) called with curMem=119694, maxMem=490356080 18/01/12 10:52:25 INFO MemoryStore: Block input-0-1515725545600 stored as bytes in memory (estimated size 8.0 B, free 467.5 MB) 18/01/12 10:52:25 INFO BlockManagerInfo: Added input-0-1515725545600 in memory on localhost:54439 (size: 8.0 B, free: 467.6 MB) 18/01/12 10:52:25 INFO BlockGenerator: Pushed block input-0-1515725545600 18/01/12 10:52:26 INFO MemoryStore: ensureFreeSpace(8) called with curMem=119702, maxMem=490356080 18/01/12 10:52:26 INFO MemoryStore: Block input-0-1515725546600 stored as bytes in memory (estimated size 8.0 B, free 467.5 MB) 18/01/12 10:52:26 INFO BlockManagerInfo: Added input-0-1515725546600 in memory on localhost:54439 (size: 8.0 B, free: 467.6 MB) 18/01/12 10:52:26 INFO BlockGenerator: Pushed block input-0-1515725546600 18/01/12 10:52:27 INFO MemoryStore: ensureFreeSpace(8) called with curMem=119710, maxMem=490356080 18/01/12 10:52:27 INFO MemoryStore: Block input-0-1515725547600 stored as bytes in memory (estimated size 8.0 B, free 467.5 MB) 18/01/12 10:52:27 INFO BlockManagerInfo: Added input-0-1515725547600 in memory on localhost:54439 (size: 8.0 B, free: 467.6 MB) 18/01/12 10:52:27 INFO BlockGenerator: Pushed block input-0-1515725547600 18/01/12 10:52:28 INFO MemoryStore: ensureFreeSpace(8) called with curMem=119718, maxMem=490356080 18/01/12 10:52:28 INFO MemoryStore: Block input-0-1515725548600 stored as bytes in memory (estimated size 8.0 B, free 467.5 MB) 18/01/12 10:52:28 INFO BlockManagerInfo: Added input-0-1515725548600 in memory on localhost:54439 (size: 8.0 B, free: 467.6 MB) 18/01/12 10:52:28 INFO BlockGenerator: Pushed block input-0-1515725548600 18/01/12 10:52:29 INFO MemoryStore: ensureFreeSpace(8) called with curMem=119726, maxMem=490356080 18/01/12 10:52:29 INFO MemoryStore: Block input-0-1515725549600 stored as bytes in memory (estimated size 8.0 B, free 467.5 MB) 18/01/12 10:52:29 INFO BlockManagerInfo: Added input-0-1515725549600 in memory on localhost:54439 (size: 8.0 B, free: 467.6 MB) 18/01/12 10:52:29 INFO BlockGenerator: Pushed block input-0-1515725549600 18/01/12 10:52:30 INFO JobScheduler: Added jobs for time 1515725550000 ms 18/01/12 10:52:30 INFO JobScheduler: Starting job streaming job 1515725550000 ms.0 from job set of time 1515725550000 ms 18/01/12 10:52:30 INFO SparkContext: Starting job: print at NetWorkWordCount.scala:29 18/01/12 10:52:30 INFO DAGScheduler: Registering RDD 187 (map at NetWorkWordCount.scala:28) 18/01/12 10:52:30 INFO DAGScheduler: Got job 93 (print at NetWorkWordCount.scala:29) with 1 output partitions (allowLocal=true) 18/01/12 10:52:30 INFO DAGScheduler: Final stage: ResultStage 186(print at NetWorkWordCount.scala:29) 18/01/12 10:52:30 INFO DAGScheduler: Parents of final stage: List(ShuffleMapStage 185) 18/01/12 10:52:30 INFO DAGScheduler: Missing parents: List(ShuffleMapStage 185) 18/01/12 10:52:30 INFO DAGScheduler: Submitting ShuffleMapStage 185 (MapPartitionsRDD[187] at map at NetWorkWordCount.scala:28), which has no missing parents 18/01/12 10:52:30 INFO MemoryStore: ensureFreeSpace(2560) called with curMem=119734, maxMem=490356080 18/01/12 10:52:30 INFO MemoryStore: Block broadcast_138 stored as values in memory (estimated size 2.5 KB, free 467.5 MB) 18/01/12 10:52:30 INFO MemoryStore: ensureFreeSpace(1564) called with curMem=122294, maxMem=490356080 18/01/12 10:52:30 INFO MemoryStore: Block broadcast_138_piece0 stored as bytes in memory (estimated size 1564.0 B, free 467.5 MB) 18/01/12 10:52:30 INFO BlockManagerInfo: Added broadcast_138_piece0 in memory on localhost:54439 (size: 1564.0 B, free: 467.6 MB) 18/01/12 10:52:30 INFO SparkContext: Created broadcast 138 from broadcast at DAGScheduler.scala:874 18/01/12 10:52:30 INFO DAGScheduler: Submitting 5 missing tasks from ShuffleMapStage 185 (MapPartitionsRDD[187] at map at NetWorkWordCount.scala:28) 18/01/12 10:52:30 INFO TaskSchedulerImpl: Adding task set 185.0 with 5 tasks 18/01/12 10:52:30 INFO TaskSetManager: Starting task 0.0 in stage 185.0 (TID 406, localhost, NODE_LOCAL, 1277 bytes) 18/01/12 10:52:30 INFO TaskSetManager: Starting task 1.0 in stage 185.0 (TID 407, localhost, NODE_LOCAL, 1277 bytes) 18/01/12 10:52:30 INFO TaskSetManager: Starting task 2.0 in stage 185.0 (TID 408, localhost, NODE_LOCAL, 1277 bytes) 18/01/12 10:52:30 INFO Executor: Running task 1.0 in stage 185.0 (TID 407) 18/01/12 10:52:30 INFO Executor: Running task 2.0 in stage 185.0 (TID 408) 18/01/12 10:52:30 INFO Executor: Running task 0.0 in stage 185.0 (TID 406) 18/01/12 10:52:30 INFO BlockManager: Found block input-0-1515725546600 locally 18/01/12 10:52:30 INFO BlockManager: Found block input-0-1515725545600 locally 18/01/12 10:52:30 INFO BlockManager: Found block input-0-1515725547600 locally 18/01/12 10:52:30 INFO Executor: Finished task 2.0 in stage 185.0 (TID 408). 882 bytes result sent to driver 18/01/12 10:52:30 INFO TaskSetManager: Starting task 3.0 in stage 185.0 (TID 409, localhost, NODE_LOCAL, 1277 bytes) 18/01/12 10:52:30 INFO TaskSetManager: Finished task 2.0 in stage 185.0 (TID 408) in 24 ms on localhost (1/5) 18/01/12 10:52:30 INFO Executor: Running task 3.0 in stage 185.0 (TID 409) 18/01/12 10:52:30 INFO Executor: Finished task 1.0 in stage 185.0 (TID 407). 882 bytes result sent to driver 18/01/12 10:52:30 INFO BlockManager: Found block input-0-1515725548600 locally 18/01/12 10:52:30 INFO Executor: Finished task 0.0 in stage 185.0 (TID 406). 882 bytes result sent to driver 18/01/12 10:52:30 INFO TaskSetManager: Starting task 4.0 in stage 185.0 (TID 410, localhost, NODE_LOCAL, 1277 bytes) 18/01/12 10:52:30 INFO Executor: Running task 4.0 in stage 185.0 (TID 410) 18/01/12 10:52:30 INFO TaskSetManager: Finished task 0.0 in stage 185.0 (TID 406) in 28 ms on localhost (2/5) 18/01/12 10:52:30 INFO BlockManager: Found block input-0-1515725549600 locally 18/01/12 10:52:30 INFO TaskSetManager: Finished task 1.0 in stage 185.0 (TID 407) in 31 ms on localhost (3/5) 18/01/12 10:52:30 INFO Executor: Finished task 3.0 in stage 185.0 (TID 409). 882 bytes result sent to driver 18/01/12 10:52:30 INFO TaskSetManager: Finished task 3.0 in stage 185.0 (TID 409) in 9 ms on localhost (4/5) 18/01/12 10:52:30 INFO Executor: Finished task 4.0 in stage 185.0 (TID 410). 882 bytes result sent to driver 18/01/12 10:52:30 INFO TaskSetManager: Finished task 4.0 in stage 185.0 (TID 410) in 13 ms on localhost (5/5) 18/01/12 10:52:30 INFO TaskSchedulerImpl: Removed TaskSet 185.0, whose tasks have all completed, from pool 18/01/12 10:52:30 INFO DAGScheduler: ShuffleMapStage 185 (map at NetWorkWordCount.scala:28) finished in 0.041 s 18/01/12 10:52:30 INFO DAGScheduler: looking for newly runnable stages 18/01/12 10:52:30 INFO DAGScheduler: running: Set(ResultStage 0) 18/01/12 10:52:30 INFO DAGScheduler: waiting: Set(ResultStage 186) 18/01/12 10:52:30 INFO DAGScheduler: failed: Set() 18/01/12 10:52:30 INFO DAGScheduler: Missing parents for ResultStage 186: List() 18/01/12 10:52:30 INFO DAGScheduler: Submitting ResultStage 186 (ShuffledRDD[188] at reduceByKey at NetWorkWordCount.scala:28), which is now runnable 18/01/12 10:52:30 INFO MemoryStore: ensureFreeSpace(2304) called with curMem=123858, maxMem=490356080 18/01/12 10:52:30 INFO MemoryStore: Block broadcast_139 stored as values in memory (estimated size 2.3 KB, free 467.5 MB) 18/01/12 10:52:30 INFO MemoryStore: ensureFreeSpace(1417) called with curMem=126162, maxMem=490356080 18/01/12 10:52:30 INFO MemoryStore: Block broadcast_139_piece0 stored as bytes in memory (estimated size 1417.0 B, free 467.5 MB) 18/01/12 10:52:30 INFO BlockManagerInfo: Added broadcast_139_piece0 in memory on localhost:54439 (size: 1417.0 B, free: 467.6 MB) 18/01/12 10:52:30 INFO SparkContext: Created broadcast 139 from broadcast at DAGScheduler.scala:874 18/01/12 10:52:30 INFO DAGScheduler: Submitting 1 missing tasks from ResultStage 186 (ShuffledRDD[188] at reduceByKey at NetWorkWordCount.scala:28) 18/01/12 10:52:30 INFO TaskSchedulerImpl: Adding task set 186.0 with 1 tasks 18/01/12 10:52:30 INFO TaskSetManager: Starting task 0.0 in stage 186.0 (TID 411, localhost, PROCESS_LOCAL, 1165 bytes) 18/01/12 10:52:30 INFO Executor: Running task 0.0 in stage 186.0 (TID 411) 18/01/12 10:52:30 INFO ShuffleBlockFetcherIterator: Getting 0 non-empty blocks out of 5 blocks 18/01/12 10:52:30 INFO ShuffleBlockFetcherIterator: Started 0 remote fetches in 0 ms 18/01/12 10:52:30 INFO Executor: Finished task 0.0 in stage 186.0 (TID 411). 882 bytes result sent to driver 18/01/12 10:52:30 INFO TaskSetManager: Finished task 0.0 in stage 186.0 (TID 411) in 1 ms on localhost (1/1) 18/01/12 10:52:30 INFO TaskSchedulerImpl: Removed TaskSet 186.0, whose tasks have all completed, from pool 18/01/12 10:52:30 INFO DAGScheduler: ResultStage 186 (print at NetWorkWordCount.scala:29) finished in 0.002 s 18/01/12 10:52:30 INFO DAGScheduler: Job 93 finished: print at NetWorkWordCount.scala:29, took 0.049675 s 18/01/12 10:52:30 INFO SparkContext: Starting job: print at NetWorkWordCount.scala:29 18/01/12 10:52:30 INFO MapOutputTrackerMaster: Size of output statuses for shuffle 46 is 158 bytes 18/01/12 10:52:30 INFO DAGScheduler: Got job 94 (print at NetWorkWordCount.scala:29) with 3 output partitions (allowLocal=true) 18/01/12 10:52:30 INFO DAGScheduler: Final stage: ResultStage 188(print at NetWorkWordCount.scala:29) 18/01/12 10:52:30 INFO DAGScheduler: Parents of final stage: List(ShuffleMapStage 187) 18/01/12 10:52:30 INFO DAGScheduler: Missing parents: List() 18/01/12 10:52:30 INFO DAGScheduler: Submitting ResultStage 188 (ShuffledRDD[188] at reduceByKey at NetWorkWordCount.scala:28), which has no missing parents 18/01/12 10:52:30 INFO MemoryStore: ensureFreeSpace(2304) called with curMem=127579, maxMem=490356080 18/01/12 10:52:30 INFO MemoryStore: Block broadcast_140 stored as values in memory (estimated size 2.3 KB, free 467.5 MB) 18/01/12 10:52:30 INFO MemoryStore: ensureFreeSpace(1417) called with curMem=129883, maxMem=490356080 18/01/12 10:52:30 INFO MemoryStore: Block broadcast_140_piece0 stored as bytes in memory (estimated size 1417.0 B, free 467.5 MB) 18/01/12 10:52:30 INFO BlockManagerInfo: Added broadcast_140_piece0 in memory on localhost:54439 (size: 1417.0 B, free: 467.6 MB) 18/01/12 10:52:30 INFO SparkContext: Created broadcast 140 from broadcast at DAGScheduler.scala:874 18/01/12 10:52:30 INFO DAGScheduler: Submitting 3 missing tasks from ResultStage 188 (ShuffledRDD[188] at reduceByKey at NetWorkWordCount.scala:28) 18/01/12 10:52:30 INFO TaskSchedulerImpl: Adding task set 188.0 with 3 tasks 18/01/12 10:52:30 INFO TaskSetManager: Starting task 0.0 in stage 188.0 (TID 412, localhost, PROCESS_LOCAL, 1165 bytes) 18/01/12 10:52:30 INFO TaskSetManager: Starting task 1.0 in stage 188.0 (TID 413, localhost, PROCESS_LOCAL, 1165 bytes) 18/01/12 10:52:30 INFO TaskSetManager: Starting task 2.0 in stage 188.0 (TID 414, localhost, PROCESS_LOCAL, 1165 bytes) 18/01/12 10:52:30 INFO Executor: Running task 0.0 in stage 188.0 (TID 412) 18/01/12 10:52:30 INFO Executor: Running task 2.0 in stage 188.0 (TID 414) 18/01/12 10:52:30 INFO Executor: Running task 1.0 in stage 188.0 (TID 413) 18/01/12 10:52:30 INFO ShuffleBlockFetcherIterator: Getting 0 non-empty blocks out of 5 blocks 18/01/12 10:52:30 INFO ShuffleBlockFetcherIterator: Started 0 remote fetches in 1 ms 18/01/12 10:52:30 INFO Executor: Finished task 0.0 in stage 188.0 (TID 412). 882 bytes result sent to driver 18/01/12 10:52:30 INFO ShuffleBlockFetcherIterator: Getting 2 non-empty blocks out of 5 blocks 18/01/12 10:52:30 INFO ShuffleBlockFetcherIterator: Started 0 remote fetches in 0 ms 18/01/12 10:52:30 INFO TaskSetManager: Finished task 0.0 in stage 188.0 (TID 412) in 3 ms on localhost (1/3) 18/01/12 10:52:30 INFO ShuffleBlockFetcherIterator: Getting 3 non-empty blocks out of 5 blocks 18/01/12 10:52:30 INFO ShuffleBlockFetcherIterator: Started 0 remote fetches in 3 ms 18/01/12 10:52:30 INFO Executor: Finished task 1.0 in stage 188.0 (TID 413). 1048 bytes result sent to driver 18/01/12 10:52:30 INFO TaskSetManager: Finished task 1.0 in stage 188.0 (TID 413) in 4 ms on localhost (2/3) 18/01/12 10:52:30 INFO Executor: Finished task 2.0 in stage 188.0 (TID 414). 1048 bytes result sent to driver 18/01/12 10:52:30 INFO TaskSetManager: Finished task 2.0 in stage 188.0 (TID 414) in 4 ms on localhost (3/3) 18/01/12 10:52:30 INFO TaskSchedulerImpl: Removed TaskSet 188.0, whose tasks have all completed, from pool 18/01/12 10:52:30 INFO DAGScheduler: ResultStage 188 (print at NetWorkWordCount.scala:29) finished in 0.006 s 18/01/12 10:52:30 INFO DAGScheduler: Job 94 finished: print at NetWorkWordCount.scala:29, took 0.009241 s ------------------------------------------- Time: 1515725550000 ms ------------------------------------------- (B,1) (F,1) (G,2) (C,1) 18/01/12 10:52:30 INFO JobScheduler: Finished job streaming job 1515725550000 ms.0 from job set of time 1515725550000 ms 18/01/12 10:52:30 INFO JobScheduler: Total delay: 0.069 s for time 1515725550000 ms (execution: 0.063 s) 18/01/12 10:52:30 INFO ShuffledRDD: Removing RDD 184 from persistence list 18/01/12 10:52:30 INFO MapPartitionsRDD: Removing RDD 183 from persistence list 18/01/12 10:52:30 INFO BlockManager: Removing RDD 184 18/01/12 10:52:30 INFO MapPartitionsRDD: Removing RDD 182 from persistence list 18/01/12 10:52:30 INFO BlockManager: Removing RDD 183 18/01/12 10:52:30 INFO BlockRDD: Removing RDD 181 from persistence list 18/01/12 10:52:30 INFO BlockManager: Removing RDD 182 18/01/12 10:52:30 INFO SocketInputDStream: Removing blocks of RDD BlockRDD[181] at socketTextStream at NetWorkWordCount.scala:26 of time 1515725550000 ms 18/01/12 10:52:30 INFO BlockManager: Removing RDD 181 18/01/12 10:52:30 INFO BlockManagerInfo: Removed input-0-1515725540800 on localhost:54439 in memory (size: 8.0 B, free: 467.6 MB) 18/01/12 10:52:30 INFO BlockManagerInfo: Removed input-0-1515725541600 on localhost:54439 in memory (size: 8.0 B, free: 467.6 MB) 18/01/12 10:52:30 INFO BlockManagerInfo: Removed input-0-1515725542600 on localhost:54439 in memory (size: 8.0 B, free: 467.6 MB) 18/01/12 10:52:30 INFO BlockManagerInfo: Removed input-0-1515725543600 on localhost:54439 in memory (size: 8.0 B, free: 467.6 MB) 18/01/12 10:52:30 INFO ReceivedBlockTracker: Deleting batches ArrayBuffer(1515725540000 ms) 18/01/12 10:52:30 INFO InputInfoTracker: remove old batch metadata: 1515725540000 ms 18/01/12 10:52:30 INFO BlockManagerInfo: Removed input-0-1515725544600 on localhost:54439 in memory (size: 8.0 B, free: 467.6 MB) 18/01/12 10:52:30 INFO MemoryStore: ensureFreeSpace(8) called with curMem=131260, maxMem=490356080 18/01/12 10:52:30 INFO MemoryStore: Block input-0-1515725550600 stored as bytes in memory (estimated size 8.0 B, free 467.5 MB) 18/01/12 10:52:30 INFO BlockManagerInfo: Added input-0-1515725550600 in memory on localhost:54439 (size: 8.0 B, free: 467.6 MB) 18/01/12 10:52:30 INFO BlockGenerator: Pushed block input-0-1515725550600 18/01/12 10:52:32 INFO MemoryStore: ensureFreeSpace(8) called with curMem=131268, maxMem=490356080 18/01/12 10:52:32 INFO MemoryStore: Block input-0-1515725551800 stored as bytes in memory (estimated size 8.0 B, free 467.5 MB) 18/01/12 10:52:32 INFO BlockManagerInfo: Added input-0-1515725551800 in memory on localhost:54439 (size: 8.0 B, free: 467.6 MB) 18/01/12 10:52:32 INFO BlockGenerator: Pushed block input-0-1515725551800 18/01/12 10:52:33 INFO MemoryStore: ensureFreeSpace(8) called with curMem=131276, maxMem=490356080 18/01/12 10:52:33 INFO MemoryStore: Block input-0-1515725552800 stored as bytes in memory (estimated size 8.0 B, free 467.5 MB) 18/01/12 10:52:33 INFO BlockManagerInfo: Added input-0-1515725552800 in memory on localhost:54439 (size: 8.0 B, free: 467.6 MB) 18/01/12 10:52:33 INFO BlockGenerator: Pushed block input-0-1515725552800 18/01/12 10:52:34 INFO MemoryStore: ensureFreeSpace(8) called with curMem=131284, maxMem=490356080 18/01/12 10:52:34 INFO MemoryStore: Block input-0-1515725553800 stored as bytes in memory (estimated size 8.0 B, free 467.5 MB) 18/01/12 10:52:34 INFO BlockManagerInfo: Added input-0-1515725553800 in memory on localhost:54439 (size: 8.0 B, free: 467.6 MB) 18/01/12 10:52:34 INFO BlockGenerator: Pushed block input-0-1515725553800 18/01/12 10:52:35 INFO MemoryStore: ensureFreeSpace(8) called with curMem=131292, maxMem=490356080 18/01/12 10:52:35 INFO MemoryStore: Block input-0-1515725554800 stored as bytes in memory (estimated size 8.0 B, free 467.5 MB) 18/01/12 10:52:35 INFO BlockManagerInfo: Added input-0-1515725554800 in memory on localhost:54439 (size: 8.0 B, free: 467.6 MB) 18/01/12 10:52:35 INFO BlockGenerator: Pushed block input-0-1515725554800 18/01/12 10:52:35 INFO JobScheduler: Added jobs for time 1515725555000 ms 18/01/12 10:52:35 INFO SparkContext: Starting job: print at NetWorkWordCount.scala:29 18/01/12 10:52:35 INFO JobScheduler: Starting job streaming job 1515725555000 ms.0 from job set of time 1515725555000 ms 18/01/12 10:52:35 INFO DAGScheduler: Registering RDD 191 (map at NetWorkWordCount.scala:28) 18/01/12 10:52:35 INFO DAGScheduler: Got job 95 (print at NetWorkWordCount.scala:29) with 1 output partitions (allowLocal=true) 18/01/12 10:52:35 INFO DAGScheduler: Final stage: ResultStage 190(print at NetWorkWordCount.scala:29) 18/01/12 10:52:35 INFO DAGScheduler: Parents of final stage: List(ShuffleMapStage 189) 18/01/12 10:52:35 INFO DAGScheduler: Missing parents: List(ShuffleMapStage 189) 18/01/12 10:52:35 INFO DAGScheduler: Submitting ShuffleMapStage 189 (MapPartitionsRDD[191] at map at NetWorkWordCount.scala:28), which has no missing parents 18/01/12 10:52:35 INFO MemoryStore: ensureFreeSpace(2560) called with curMem=131300, maxMem=490356080 18/01/12 10:52:35 INFO MemoryStore: Block broadcast_141 stored as values in memory (estimated size 2.5 KB, free 467.5 MB) 18/01/12 10:52:35 INFO MemoryStore: ensureFreeSpace(1564) called with curMem=133860, maxMem=490356080 18/01/12 10:52:35 INFO MemoryStore: Block broadcast_141_piece0 stored as bytes in memory (estimated size 1564.0 B, free 467.5 MB) 18/01/12 10:52:35 INFO BlockManagerInfo: Added broadcast_141_piece0 in memory on localhost:54439 (size: 1564.0 B, free: 467.6 MB) 18/01/12 10:52:35 INFO SparkContext: Created broadcast 141 from broadcast at DAGScheduler.scala:874 18/01/12 10:52:35 INFO DAGScheduler: Submitting 4 missing tasks from ShuffleMapStage 189 (MapPartitionsRDD[191] at map at NetWorkWordCount.scala:28) 18/01/12 10:52:35 INFO TaskSchedulerImpl: Adding task set 189.0 with 4 tasks 18/01/12 10:52:35 INFO TaskSetManager: Starting task 0.0 in stage 189.0 (TID 415, localhost, NODE_LOCAL, 1277 bytes) 18/01/12 10:52:35 INFO TaskSetManager: Starting task 1.0 in stage 189.0 (TID 416, localhost, NODE_LOCAL, 1277 bytes) 18/01/12 10:52:35 INFO TaskSetManager: Starting task 2.0 in stage 189.0 (TID 417, localhost, NODE_LOCAL, 1277 bytes) 18/01/12 10:52:35 INFO Executor: Running task 2.0 in stage 189.0 (TID 417) 18/01/12 10:52:35 INFO BlockManager: Found block input-0-1515725552800 locally 18/01/12 10:52:35 INFO Executor: Running task 0.0 in stage 189.0 (TID 415) 18/01/12 10:52:35 INFO BlockManager: Found block input-0-1515725550600 locally 18/01/12 10:52:35 INFO Executor: Running task 1.0 in stage 189.0 (TID 416) 18/01/12 10:52:35 INFO BlockManager: Found block input-0-1515725551800 locally 18/01/12 10:52:35 INFO Executor: Finished task 2.0 in stage 189.0 (TID 417). 882 bytes result sent to driver 18/01/12 10:52:35 INFO TaskSetManager: Starting task 3.0 in stage 189.0 (TID 418, localhost, NODE_LOCAL, 1277 bytes) 18/01/12 10:52:35 INFO Executor: Running task 3.0 in stage 189.0 (TID 418) 18/01/12 10:52:35 INFO TaskSetManager: Finished task 2.0 in stage 189.0 (TID 417) in 7 ms on localhost (1/4) 18/01/12 10:52:35 INFO BlockManager: Found block input-0-1515725553800 locally 18/01/12 10:52:35 INFO Executor: Finished task 0.0 in stage 189.0 (TID 415). 882 bytes result sent to driver 18/01/12 10:52:35 INFO TaskSetManager: Finished task 0.0 in stage 189.0 (TID 415) in 9 ms on localhost (2/4) 18/01/12 10:52:35 INFO Executor: Finished task 3.0 in stage 189.0 (TID 418). 882 bytes result sent to driver 18/01/12 10:52:35 INFO TaskSetManager: Finished task 3.0 in stage 189.0 (TID 418) in 4 ms on localhost (3/4) 18/01/12 10:52:35 INFO Executor: Finished task 1.0 in stage 189.0 (TID 416). 882 bytes result sent to driver 18/01/12 10:52:35 INFO TaskSetManager: Finished task 1.0 in stage 189.0 (TID 416) in 12 ms on localhost (4/4) 18/01/12 10:52:35 INFO DAGScheduler: ShuffleMapStage 189 (map at NetWorkWordCount.scala:28) finished in 0.012 s 18/01/12 10:52:35 INFO DAGScheduler: looking for newly runnable stages 18/01/12 10:52:35 INFO TaskSchedulerImpl: Removed TaskSet 189.0, whose tasks have all completed, from pool 18/01/12 10:52:35 INFO DAGScheduler: running: Set(ResultStage 0) 18/01/12 10:52:35 INFO DAGScheduler: waiting: Set(ResultStage 190) 18/01/12 10:52:35 INFO DAGScheduler: failed: Set() 18/01/12 10:52:35 INFO DAGScheduler: Missing parents for ResultStage 190: List() 18/01/12 10:52:35 INFO DAGScheduler: Submitting ResultStage 190 (ShuffledRDD[192] at reduceByKey at NetWorkWordCount.scala:28), which is now runnable 18/01/12 10:52:35 INFO MemoryStore: ensureFreeSpace(2304) called with curMem=135424, maxMem=490356080 18/01/12 10:52:35 INFO MemoryStore: Block broadcast_142 stored as values in memory (estimated size 2.3 KB, free 467.5 MB) 18/01/12 10:52:35 INFO MemoryStore: ensureFreeSpace(1413) called with curMem=137728, maxMem=490356080 18/01/12 10:52:35 INFO MemoryStore: Block broadcast_142_piece0 stored as bytes in memory (estimated size 1413.0 B, free 467.5 MB) 18/01/12 10:52:35 INFO BlockManagerInfo: Added broadcast_142_piece0 in memory on localhost:54439 (size: 1413.0 B, free: 467.6 MB) 18/01/12 10:52:35 INFO SparkContext: Created broadcast 142 from broadcast at DAGScheduler.scala:874 18/01/12 10:52:35 INFO DAGScheduler: Submitting 1 missing tasks from ResultStage 190 (ShuffledRDD[192] at reduceByKey at NetWorkWordCount.scala:28) 18/01/12 10:52:35 INFO TaskSchedulerImpl: Adding task set 190.0 with 1 tasks 18/01/12 10:52:35 INFO TaskSetManager: Starting task 0.0 in stage 190.0 (TID 419, localhost, PROCESS_LOCAL, 1165 bytes) 18/01/12 10:52:35 INFO Executor: Running task 0.0 in stage 190.0 (TID 419) 18/01/12 10:52:35 INFO ShuffleBlockFetcherIterator: Getting 0 non-empty blocks out of 4 blocks 18/01/12 10:52:35 INFO ShuffleBlockFetcherIterator: Started 0 remote fetches in 1 ms 18/01/12 10:52:35 INFO Executor: Finished task 0.0 in stage 190.0 (TID 419). 882 bytes result sent to driver 18/01/12 10:52:35 INFO TaskSetManager: Finished task 0.0 in stage 190.0 (TID 419) in 1 ms on localhost (1/1) 18/01/12 10:52:35 INFO TaskSchedulerImpl: Removed TaskSet 190.0, whose tasks have all completed, from pool 18/01/12 10:52:35 INFO DAGScheduler: ResultStage 190 (print at NetWorkWordCount.scala:29) finished in 0.002 s 18/01/12 10:52:35 INFO DAGScheduler: Job 95 finished: print at NetWorkWordCount.scala:29, took 0.022178 s 18/01/12 10:52:35 INFO SparkContext: Starting job: print at NetWorkWordCount.scala:29 18/01/12 10:52:35 INFO MapOutputTrackerMaster: Size of output statuses for shuffle 47 is 165 bytes 18/01/12 10:52:35 INFO DAGScheduler: Got job 96 (print at NetWorkWordCount.scala:29) with 3 output partitions (allowLocal=true) 18/01/12 10:52:35 INFO DAGScheduler: Final stage: ResultStage 192(print at NetWorkWordCount.scala:29) 18/01/12 10:52:35 INFO DAGScheduler: Parents of final stage: List(ShuffleMapStage 191) 18/01/12 10:52:35 INFO DAGScheduler: Missing parents: List() 18/01/12 10:52:35 INFO DAGScheduler: Submitting ResultStage 192 (ShuffledRDD[192] at reduceByKey at NetWorkWordCount.scala:28), which has no missing parents 18/01/12 10:52:35 INFO MemoryStore: ensureFreeSpace(2304) called with curMem=139141, maxMem=490356080 18/01/12 10:52:35 INFO MemoryStore: Block broadcast_143 stored as values in memory (estimated size 2.3 KB, free 467.5 MB) 18/01/12 10:52:35 INFO MemoryStore: ensureFreeSpace(1413) called with curMem=141445, maxMem=490356080 18/01/12 10:52:35 INFO MemoryStore: Block broadcast_143_piece0 stored as bytes in memory (estimated size 1413.0 B, free 467.5 MB) 18/01/12 10:52:35 INFO BlockManagerInfo: Added broadcast_143_piece0 in memory on localhost:54439 (size: 1413.0 B, free: 467.6 MB) 18/01/12 10:52:35 INFO SparkContext: Created broadcast 143 from broadcast at DAGScheduler.scala:874 18/01/12 10:52:35 INFO DAGScheduler: Submitting 3 missing tasks from ResultStage 192 (ShuffledRDD[192] at reduceByKey at NetWorkWordCount.scala:28) 18/01/12 10:52:35 INFO TaskSchedulerImpl: Adding task set 192.0 with 3 tasks 18/01/12 10:52:35 INFO ContextCleaner: Cleaned shuffle 40 18/01/12 10:52:35 INFO ContextCleaner: Cleaned shuffle 41 18/01/12 10:52:35 INFO TaskSetManager: Starting task 0.0 in stage 192.0 (TID 420, localhost, PROCESS_LOCAL, 1165 bytes) 18/01/12 10:52:35 INFO TaskSetManager: Starting task 1.0 in stage 192.0 (TID 421, localhost, PROCESS_LOCAL, 1165 bytes) 18/01/12 10:52:35 INFO TaskSetManager: Starting task 2.0 in stage 192.0 (TID 422, localhost, PROCESS_LOCAL, 1165 bytes) 18/01/12 10:52:35 INFO Executor: Running task 1.0 in stage 192.0 (TID 421) 18/01/12 10:52:35 INFO ShuffleBlockFetcherIterator: Getting 2 non-empty blocks out of 4 blocks 18/01/12 10:52:35 INFO ShuffleBlockFetcherIterator: Started 0 remote fetches in 0 ms 18/01/12 10:52:35 INFO BlockManagerInfo: Removed broadcast_125_piece0 on localhost:54439 in memory (size: 1417.0 B, free: 467.6 MB) 18/01/12 10:52:35 INFO Executor: Running task 0.0 in stage 192.0 (TID 420) 18/01/12 10:52:35 INFO Executor: Finished task 1.0 in stage 192.0 (TID 421). 1028 bytes result sent to driver 18/01/12 10:52:35 INFO Executor: Running task 2.0 in stage 192.0 (TID 422) 18/01/12 10:52:35 INFO ShuffleBlockFetcherIterator: Getting 1 non-empty blocks out of 4 blocks 18/01/12 10:52:35 INFO ShuffleBlockFetcherIterator: Started 0 remote fetches in 0 ms 18/01/12 10:52:35 INFO ShuffleBlockFetcherIterator: Getting 1 non-empty blocks out of 4 blocks 18/01/12 10:52:35 INFO ShuffleBlockFetcherIterator: Started 0 remote fetches in 1 ms 18/01/12 10:52:35 INFO Executor: Finished task 0.0 in stage 192.0 (TID 420). 1028 bytes result sent to driver 18/01/12 10:52:35 INFO Executor: Finished task 2.0 in stage 192.0 (TID 422). 1028 bytes result sent to driver 18/01/12 10:52:35 INFO TaskSetManager: Finished task 1.0 in stage 192.0 (TID 421) in 5 ms on localhost (1/3) 18/01/12 10:52:35 INFO TaskSetManager: Finished task 0.0 in stage 192.0 (TID 420) in 6 ms on localhost (2/3) 18/01/12 10:52:35 INFO TaskSetManager: Finished task 2.0 in stage 192.0 (TID 422) in 5 ms on localhost (3/3) 18/01/12 10:52:35 INFO TaskSchedulerImpl: Removed TaskSet 192.0, whose tasks have all completed, from pool 18/01/12 10:52:35 INFO DAGScheduler: ResultStage 192 (print at NetWorkWordCount.scala:29) finished in 0.006 s 18/01/12 10:52:35 INFO DAGScheduler: Job 96 finished: print at NetWorkWordCount.scala:29, took 0.021802 s -------------------------------------------18/01/12 10:52:35 INFO ContextCleaner: Cleaned shuffle 42 Time: 1515725555000 ms ------------------------------------------- (A,1) (B,2) (C,1) 18/01/12 10:52:35 INFO JobScheduler: Finished job streaming job 1515725555000 ms.0 from job set of time 1515725555000 ms 18/01/12 10:52:35 INFO JobScheduler: Total delay: 0.056 s for time 1515725555000 ms (execution: 0.047 s) 18/01/12 10:52:35 INFO ShuffledRDD: Removing RDD 188 from persistence list 18/01/12 10:52:35 INFO MapPartitionsRDD: Removing RDD 187 from persistence list 18/01/12 10:52:35 INFO BlockManager: Removing RDD 188 18/01/12 10:52:35 INFO BlockManagerInfo: Removed broadcast_126_piece0 on localhost:54439 in memory (size: 1564.0 B, free: 467.6 MB) 18/01/12 10:52:35 INFO MapPartitionsRDD: Removing RDD 186 from persistence list 18/01/12 10:52:35 INFO BlockManager: Removing RDD 187 18/01/12 10:52:35 INFO BlockRDD: Removing RDD 185 from persistence list 18/01/12 10:52:35 INFO BlockManager: Removing RDD 186 18/01/12 10:52:35 INFO BlockManager: Removing RDD 185 18/01/12 10:52:35 INFO SocketInputDStream: Removing blocks of RDD BlockRDD[185] at socketTextStream at NetWorkWordCount.scala:26 of time 1515725555000 ms 18/01/12 10:52:35 INFO BlockManagerInfo: Removed input-0-1515725547600 on localhost:54439 in memory (size: 8.0 B, free: 467.6 MB) 18/01/12 10:52:35 INFO BlockManagerInfo: Removed input-0-1515725545600 on localhost:54439 in memory (size: 8.0 B, free: 467.6 MB) 18/01/12 10:52:35 INFO ReceivedBlockTracker: Deleting batches ArrayBuffer(1515725545000 ms) 18/01/12 10:52:35 INFO InputInfoTracker: remove old batch metadata: 1515725545000 ms 18/01/12 10:52:35 INFO BlockManagerInfo: Removed input-0-1515725546600 on localhost:54439 in memory (size: 8.0 B, free: 467.6 MB) 18/01/12 10:52:35 INFO BlockManagerInfo: Removed input-0-1515725549600 on localhost:54439 in memory (size: 8.0 B, free: 467.6 MB) 18/01/12 10:52:35 INFO BlockManagerInfo: Removed input-0-1515725548600 on localhost:54439 in memory (size: 8.0 B, free: 467.6 MB) 18/01/12 10:52:35 INFO BlockManagerInfo: Removed broadcast_127_piece0 on localhost:54439 in memory (size: 1417.0 B, free: 467.6 MB) 18/01/12 10:52:35 INFO BlockManagerInfo: Removed broadcast_128_piece0 on localhost:54439 in memory (size: 1417.0 B, free: 467.6 MB) 18/01/12 10:52:35 INFO ContextCleaner: Cleaned shuffle 43 18/01/12 10:52:35 INFO BlockManagerInfo: Removed broadcast_129_piece0 on localhost:54439 in memory (size: 1564.0 B, free: 467.6 MB) 18/01/12 10:52:35 INFO BlockManagerInfo: Removed broadcast_130_piece0 on localhost:54439 in memory (size: 1417.0 B, free: 467.6 MB) 18/01/12 10:52:35 INFO BlockManagerInfo: Removed broadcast_131_piece0 on localhost:54439 in memory (size: 1417.0 B, free: 467.6 MB) 18/01/12 10:52:35 INFO ContextCleaner: Cleaned shuffle 44 18/01/12 10:52:35 INFO BlockManagerInfo: Removed broadcast_132_piece0 on localhost:54439 in memory (size: 1563.0 B, free: 467.6 MB) 18/01/12 10:52:35 INFO BlockManagerInfo: Removed broadcast_142_piece0 on localhost:54439 in memory (size: 1413.0 B, free: 467.6 MB) 18/01/12 10:52:35 INFO BlockManagerInfo: Removed broadcast_141_piece0 on localhost:54439 in memory (size: 1564.0 B, free: 467.6 MB) 18/01/12 10:52:35 INFO BlockManagerInfo: Removed broadcast_140_piece0 on localhost:54439 in memory (size: 1417.0 B, free: 467.6 MB) 18/01/12 10:52:35 INFO BlockManagerInfo: Removed broadcast_139_piece0 on localhost:54439 in memory (size: 1417.0 B, free: 467.6 MB) 18/01/12 10:52:35 INFO BlockManagerInfo: Removed broadcast_138_piece0 on localhost:54439 in memory (size: 1564.0 B, free: 467.6 MB) 18/01/12 10:52:35 INFO BlockManagerInfo: Removed broadcast_137_piece0 on localhost:54439 in memory (size: 1417.0 B, free: 467.6 MB) 18/01/12 10:52:35 INFO BlockManagerInfo: Removed broadcast_136_piece0 on localhost:54439 in memory (size: 1417.0 B, free: 467.6 MB) 18/01/12 10:52:35 INFO BlockManagerInfo: Removed broadcast_135_piece0 on localhost:54439 in memory (size: 1564.0 B, free: 467.6 MB) 18/01/12 10:52:35 INFO ContextCleaner: Cleaned shuffle 45 18/01/12 10:52:35 INFO BlockManagerInfo: Removed broadcast_134_piece0 on localhost:54439 in memory (size: 1417.0 B, free: 467.6 MB) 18/01/12 10:52:35 INFO BlockManagerInfo: Removed broadcast_133_piece0 on localhost:54439 in memory (size: 1417.0 B, free: 467.6 MB) 18/01/12 10:52:36 INFO MemoryStore: ensureFreeSpace(8) called with curMem=73427, maxMem=490356080 18/01/12 10:52:36 INFO MemoryStore: Block input-0-1515725555800 stored as bytes in memory (estimated size 8.0 B, free 467.6 MB) 18/01/12 10:52:36 INFO BlockManagerInfo: Added input-0-1515725555800 in memory on localhost:54439 (size: 8.0 B, free: 467.6 MB) 18/01/12 10:52:36 INFO BlockGenerator: Pushed block input-0-1515725555800 18/01/12 10:52:37 INFO MemoryStore: ensureFreeSpace(8) called with curMem=73435, maxMem=490356080 18/01/12 10:52:37 INFO MemoryStore: Block input-0-1515725556800 stored as bytes in memory (estimated size 8.0 B, free 467.6 MB) 18/01/12 10:52:37 INFO BlockManagerInfo: Added input-0-1515725556800 in memory on localhost:54439 (size: 8.0 B, free: 467.6 MB) 18/01/12 10:52:37 INFO BlockGenerator: Pushed block input-0-1515725556800 18/01/12 10:52:38 INFO MemoryStore: ensureFreeSpace(8) called with curMem=73443, maxMem=490356080 18/01/12 10:52:38 INFO MemoryStore: Block input-0-1515725557800 stored as bytes in memory (estimated size 8.0 B, free 467.6 MB) 18/01/12 10:52:38 INFO BlockManagerInfo: Added input-0-1515725557800 in memory on localhost:54439 (size: 8.0 B, free: 467.6 MB) 18/01/12 10:52:38 INFO BlockGenerator: Pushed block input-0-1515725557800 18/01/12 10:52:39 INFO MemoryStore: ensureFreeSpace(8) called with curMem=73451, maxMem=490356080 18/01/12 10:52:39 INFO MemoryStore: Block input-0-1515725558800 stored as bytes in memory (estimated size 8.0 B, free 467.6 MB) 18/01/12 10:52:39 INFO BlockManagerInfo: Added input-0-1515725558800 in memory on localhost:54439 (size: 8.0 B, free: 467.6 MB) 18/01/12 10:52:39 INFO BlockGenerator: Pushed block input-0-1515725558800 18/01/12 10:52:40 INFO MemoryStore: ensureFreeSpace(8) called with curMem=73459, maxMem=490356080 18/01/12 10:52:40 INFO MemoryStore: Block input-0-1515725559800 stored as bytes in memory (estimated size 8.0 B, free 467.6 MB) 18/01/12 10:52:40 INFO BlockManagerInfo: Added input-0-1515725559800 in memory on localhost:54439 (size: 8.0 B, free: 467.6 MB) 18/01/12 10:52:40 INFO BlockGenerator: Pushed block input-0-1515725559800 18/01/12 10:52:40 INFO JobScheduler: Added jobs for time 1515725560000 ms 18/01/12 10:52:40 INFO JobScheduler: Starting job streaming job 1515725560000 ms.0 from job set of time 1515725560000 ms 18/01/12 10:52:40 INFO SparkContext: Starting job: print at NetWorkWordCount.scala:29 18/01/12 10:52:40 INFO DAGScheduler: Registering RDD 195 (map at NetWorkWordCount.scala:28) 18/01/12 10:52:40 INFO DAGScheduler: Got job 97 (print at NetWorkWordCount.scala:29) with 1 output partitions (allowLocal=true) 18/01/12 10:52:40 INFO DAGScheduler: Final stage: ResultStage 194(print at NetWorkWordCount.scala:29) 18/01/12 10:52:40 INFO DAGScheduler: Parents of final stage: List(ShuffleMapStage 193) 18/01/12 10:52:40 INFO DAGScheduler: Missing parents: List(ShuffleMapStage 193) 18/01/12 10:52:40 INFO DAGScheduler: Submitting ShuffleMapStage 193 (MapPartitionsRDD[195] at map at NetWorkWordCount.scala:28), which has no missing parents 18/01/12 10:52:40 INFO MemoryStore: ensureFreeSpace(2560) called with curMem=73467, maxMem=490356080 18/01/12 10:52:40 INFO MemoryStore: Block broadcast_144 stored as values in memory (estimated size 2.5 KB, free 467.6 MB) 18/01/12 10:52:40 INFO MemoryStore: ensureFreeSpace(1564) called with curMem=76027, maxMem=490356080 18/01/12 10:52:40 INFO MemoryStore: Block broadcast_144_piece0 stored as bytes in memory (estimated size 1564.0 B, free 467.6 MB) 18/01/12 10:52:40 INFO BlockManagerInfo: Added broadcast_144_piece0 in memory on localhost:54439 (size: 1564.0 B, free: 467.6 MB) 18/01/12 10:52:40 INFO SparkContext: Created broadcast 144 from broadcast at DAGScheduler.scala:874 18/01/12 10:52:40 INFO DAGScheduler: Submitting 5 missing tasks from ShuffleMapStage 193 (MapPartitionsRDD[195] at map at NetWorkWordCount.scala:28) 18/01/12 10:52:40 INFO TaskSchedulerImpl: Adding task set 193.0 with 5 tasks 18/01/12 10:52:40 INFO TaskSetManager: Starting task 0.0 in stage 193.0 (TID 423, localhost, NODE_LOCAL, 1277 bytes) 18/01/12 10:52:40 INFO TaskSetManager: Starting task 1.0 in stage 193.0 (TID 424, localhost, NODE_LOCAL, 1277 bytes) 18/01/12 10:52:40 INFO TaskSetManager: Starting task 2.0 in stage 193.0 (TID 425, localhost, NODE_LOCAL, 1277 bytes) 18/01/12 10:52:40 INFO Executor: Running task 2.0 in stage 193.0 (TID 425) 18/01/12 10:52:40 INFO BlockManager: Found block input-0-1515725556800 locally 18/01/12 10:52:40 INFO Executor: Running task 0.0 in stage 193.0 (TID 423) 18/01/12 10:52:40 INFO BlockManager: Found block input-0-1515725554800 locally 18/01/12 10:52:40 INFO Executor: Running task 1.0 in stage 193.0 (TID 424) 18/01/12 10:52:40 INFO BlockManager: Found block input-0-1515725555800 locally 18/01/12 10:52:40 INFO Executor: Finished task 2.0 in stage 193.0 (TID 425). 882 bytes result sent to driver 18/01/12 10:52:40 INFO TaskSetManager: Starting task 3.0 in stage 193.0 (TID 426, localhost, NODE_LOCAL, 1277 bytes) 18/01/12 10:52:40 INFO Executor: Running task 3.0 in stage 193.0 (TID 426) 18/01/12 10:52:40 INFO TaskSetManager: Finished task 2.0 in stage 193.0 (TID 425) in 4 ms on localhost (1/5) 18/01/12 10:52:40 INFO Executor: Finished task 0.0 in stage 193.0 (TID 423). 882 bytes result sent to driver 18/01/12 10:52:40 INFO BlockManager: Found block input-0-1515725557800 locally 18/01/12 10:52:40 INFO TaskSetManager: Starting task 4.0 in stage 193.0 (TID 427, localhost, NODE_LOCAL, 1277 bytes) 18/01/12 10:52:40 INFO TaskSetManager: Finished task 0.0 in stage 193.0 (TID 423) in 6 ms on localhost (2/5) 18/01/12 10:52:40 INFO Executor: Running task 4.0 in stage 193.0 (TID 427) 18/01/12 10:52:40 INFO Executor: Finished task 1.0 in stage 193.0 (TID 424). 882 bytes result sent to driver 18/01/12 10:52:40 INFO BlockManager: Found block input-0-1515725558800 locally 18/01/12 10:52:40 INFO Executor: Finished task 3.0 in stage 193.0 (TID 426). 882 bytes result sent to driver 18/01/12 10:52:40 INFO TaskSetManager: Finished task 1.0 in stage 193.0 (TID 424) in 8 ms on localhost (3/5) 18/01/12 10:52:40 INFO TaskSetManager: Finished task 3.0 in stage 193.0 (TID 426) in 4 ms on localhost (4/5) 18/01/12 10:52:40 INFO Executor: Finished task 4.0 in stage 193.0 (TID 427). 882 bytes result sent to driver 18/01/12 10:52:40 INFO TaskSetManager: Finished task 4.0 in stage 193.0 (TID 427) in 5 ms on localhost (5/5) 18/01/12 10:52:40 INFO TaskSchedulerImpl: Removed TaskSet 193.0, whose tasks have all completed, from pool 18/01/12 10:52:40 INFO DAGScheduler: ShuffleMapStage 193 (map at NetWorkWordCount.scala:28) finished in 0.011 s 18/01/12 10:52:40 INFO DAGScheduler: looking for newly runnable stages 18/01/12 10:52:40 INFO DAGScheduler: running: Set(ResultStage 0) 18/01/12 10:52:40 INFO DAGScheduler: waiting: Set(ResultStage 194) 18/01/12 10:52:40 INFO DAGScheduler: failed: Set() 18/01/12 10:52:40 INFO DAGScheduler: Missing parents for ResultStage 194: List() 18/01/12 10:52:40 INFO DAGScheduler: Submitting ResultStage 194 (ShuffledRDD[196] at reduceByKey at NetWorkWordCount.scala:28), which is now runnable 18/01/12 10:52:40 INFO MemoryStore: ensureFreeSpace(2304) called with curMem=77591, maxMem=490356080 18/01/12 10:52:40 INFO MemoryStore: Block broadcast_145 stored as values in memory (estimated size 2.3 KB, free 467.6 MB) 18/01/12 10:52:40 INFO MemoryStore: ensureFreeSpace(1417) called with curMem=79895, maxMem=490356080 18/01/12 10:52:40 INFO MemoryStore: Block broadcast_145_piece0 stored as bytes in memory (estimated size 1417.0 B, free 467.6 MB) 18/01/12 10:52:40 INFO BlockManagerInfo: Added broadcast_145_piece0 in memory on localhost:54439 (size: 1417.0 B, free: 467.6 MB) 18/01/12 10:52:40 INFO SparkContext: Created broadcast 145 from broadcast at DAGScheduler.scala:874 18/01/12 10:52:40 INFO DAGScheduler: Submitting 1 missing tasks from ResultStage 194 (ShuffledRDD[196] at reduceByKey at NetWorkWordCount.scala:28) 18/01/12 10:52:40 INFO TaskSchedulerImpl: Adding task set 194.0 with 1 tasks 18/01/12 10:52:40 INFO TaskSetManager: Starting task 0.0 in stage 194.0 (TID 428, localhost, PROCESS_LOCAL, 1165 bytes) 18/01/12 10:52:40 INFO Executor: Running task 0.0 in stage 194.0 (TID 428) 18/01/12 10:52:40 INFO ShuffleBlockFetcherIterator: Getting 2 non-empty blocks out of 5 blocks 18/01/12 10:52:40 INFO ShuffleBlockFetcherIterator: Started 0 remote fetches in 0 ms 18/01/12 10:52:40 INFO Executor: Finished task 0.0 in stage 194.0 (TID 428). 1028 bytes result sent to driver 18/01/12 10:52:40 INFO TaskSetManager: Finished task 0.0 in stage 194.0 (TID 428) in 3 ms on localhost (1/1) 18/01/12 10:52:40 INFO TaskSchedulerImpl: Removed TaskSet 194.0, whose tasks have all completed, from pool 18/01/12 10:52:40 INFO DAGScheduler: ResultStage 194 (print at NetWorkWordCount.scala:29) finished in 0.005 s 18/01/12 10:52:40 INFO DAGScheduler: Job 97 finished: print at NetWorkWordCount.scala:29, took 0.026568 s 18/01/12 10:52:40 INFO SparkContext: Starting job: print at NetWorkWordCount.scala:29 18/01/12 10:52:40 INFO MapOutputTrackerMaster: Size of output statuses for shuffle 48 is 161 bytes 18/01/12 10:52:40 INFO DAGScheduler: Got job 98 (print at NetWorkWordCount.scala:29) with 3 output partitions (allowLocal=true) 18/01/12 10:52:40 INFO DAGScheduler: Final stage: ResultStage 196(print at NetWorkWordCount.scala:29) 18/01/12 10:52:40 INFO DAGScheduler: Parents of final stage: List(ShuffleMapStage 195) 18/01/12 10:52:40 INFO DAGScheduler: Missing parents: List() 18/01/12 10:52:40 INFO DAGScheduler: Submitting ResultStage 196 (ShuffledRDD[196] at reduceByKey at NetWorkWordCount.scala:28), which has no missing parents 18/01/12 10:52:40 INFO MemoryStore: ensureFreeSpace(2304) called with curMem=81312, maxMem=490356080 18/01/12 10:52:40 INFO MemoryStore: Block broadcast_146 stored as values in memory (estimated size 2.3 KB, free 467.6 MB) 18/01/12 10:52:40 INFO MemoryStore: ensureFreeSpace(1417) called with curMem=83616, maxMem=490356080 18/01/12 10:52:40 INFO MemoryStore: Block broadcast_146_piece0 stored as bytes in memory (estimated size 1417.0 B, free 467.6 MB) 18/01/12 10:52:40 INFO BlockManagerInfo: Added broadcast_146_piece0 in memory on localhost:54439 (size: 1417.0 B, free: 467.6 MB) 18/01/12 10:52:40 INFO SparkContext: Created broadcast 146 from broadcast at DAGScheduler.scala:874 18/01/12 10:52:40 INFO DAGScheduler: Submitting 3 missing tasks from ResultStage 196 (ShuffledRDD[196] at reduceByKey at NetWorkWordCount.scala:28) 18/01/12 10:52:40 INFO TaskSchedulerImpl: Adding task set 196.0 with 3 tasks 18/01/12 10:52:40 INFO TaskSetManager: Starting task 0.0 in stage 196.0 (TID 429, localhost, PROCESS_LOCAL, 1165 bytes) 18/01/12 10:52:40 INFO TaskSetManager: Starting task 1.0 in stage 196.0 (TID 430, localhost, PROCESS_LOCAL, 1165 bytes) 18/01/12 10:52:40 INFO TaskSetManager: Starting task 2.0 in stage 196.0 (TID 431, localhost, PROCESS_LOCAL, 1165 bytes) 18/01/12 10:52:40 INFO Executor: Running task 1.0 in stage 196.0 (TID 430) 18/01/12 10:52:40 INFO Executor: Running task 0.0 in stage 196.0 (TID 429) 18/01/12 10:52:40 INFO ShuffleBlockFetcherIterator: Getting 0 non-empty blocks out of 5 blocks 18/01/12 10:52:40 INFO ShuffleBlockFetcherIterator: Started 0 remote fetches in 0 ms 18/01/12 10:52:40 INFO ShuffleBlockFetcherIterator: Getting 0 non-empty blocks out of 5 blocks 18/01/12 10:52:40 INFO ShuffleBlockFetcherIterator: Started 0 remote fetches in 0 ms 18/01/12 10:52:40 INFO Executor: Finished task 1.0 in stage 196.0 (TID 430). 882 bytes result sent to driver 18/01/12 10:52:40 INFO Executor: Finished task 0.0 in stage 196.0 (TID 429). 882 bytes result sent to driver 18/01/12 10:52:40 INFO TaskSetManager: Finished task 1.0 in stage 196.0 (TID 430) in 2 ms on localhost (1/3) 18/01/12 10:52:40 INFO Executor: Running task 2.0 in stage 196.0 (TID 431) 18/01/12 10:52:40 INFO TaskSetManager: Finished task 0.0 in stage 196.0 (TID 429) in 2 ms on localhost (2/3) 18/01/12 10:52:40 INFO ShuffleBlockFetcherIterator: Getting 3 non-empty blocks out of 5 blocks 18/01/12 10:52:40 INFO ShuffleBlockFetcherIterator: Started 0 remote fetches in 0 ms 18/01/12 10:52:40 INFO Executor: Finished task 2.0 in stage 196.0 (TID 431). 1028 bytes result sent to driver 18/01/12 10:52:40 INFO TaskSetManager: Finished task 2.0 in stage 196.0 (TID 431) in 4 ms on localhost (3/3) 18/01/12 10:52:40 INFO TaskSchedulerImpl: Removed TaskSet 196.0, whose tasks have all completed, from pool 18/01/12 10:52:40 INFO DAGScheduler: ResultStage 196 (print at NetWorkWordCount.scala:29) finished in 0.004 s 18/01/12 10:52:40 INFO DAGScheduler: Job 98 finished: print at NetWorkWordCount.scala:29, took 0.007002 s ------------------------------------------- Time: 1515725560000 ms ------------------------------------------- 18/01/12 10:52:40 INFO JobScheduler: Finished job streaming job 1515725560000 ms.0 from job set of time 1515725560000 ms 18/01/12 10:52:40 INFO JobScheduler: Total delay: 0.043 s for time 1515725560000 ms (execution: 0.037 s) 18/01/12 10:52:40 INFO ShuffledRDD: Removing RDD 192 from persistence list (D,2) (G,3) 18/01/12 10:52:40 INFO BlockManager: Removing RDD 192 18/01/12 10:52:40 INFO MapPartitionsRDD: Removing RDD 191 from persistence list 18/01/12 10:52:40 INFO BlockManager: Removing RDD 191 18/01/12 10:52:40 INFO MapPartitionsRDD: Removing RDD 190 from persistence list 18/01/12 10:52:40 INFO BlockManager: Removing RDD 190 18/01/12 10:52:40 INFO BlockRDD: Removing RDD 189 from persistence list 18/01/12 10:52:40 INFO BlockManager: Removing RDD 189 18/01/12 10:52:40 INFO SocketInputDStream: Removing blocks of RDD BlockRDD[189] at socketTextStream at NetWorkWordCount.scala:26 of time 1515725560000 ms 18/01/12 10:52:40 INFO BlockManagerInfo: Removed input-0-1515725550600 on localhost:54439 in memory (size: 8.0 B, free: 467.6 MB) 18/01/12 10:52:40 INFO BlockManagerInfo: Removed input-0-1515725551800 on localhost:54439 in memory (size: 8.0 B, free: 467.6 MB) 18/01/12 10:52:40 INFO BlockManagerInfo: Removed input-0-1515725552800 on localhost:54439 in memory (size: 8.0 B, free: 467.6 MB) 18/01/12 10:52:40 INFO BlockManagerInfo: Removed input-0-1515725553800 on localhost:54439 in memory (size: 8.0 B, free: 467.6 MB) 18/01/12 10:52:40 INFO ReceivedBlockTracker: Deleting batches ArrayBuffer(1515725550000 ms) 18/01/12 10:52:40 INFO InputInfoTracker: remove old batch metadata: 1515725550000 ms 18/01/12 10:52:41 INFO MemoryStore: ensureFreeSpace(8) called with curMem=85001, maxMem=490356080 18/01/12 10:52:41 INFO MemoryStore: Block input-0-1515725560800 stored as bytes in memory (estimated size 8.0 B, free 467.6 MB) 18/01/12 10:52:41 INFO BlockManagerInfo: Added input-0-1515725560800 in memory on localhost:54439 (size: 8.0 B, free: 467.6 MB) 18/01/12 10:52:41 INFO BlockGenerator: Pushed block input-0-1515725560800 18/01/12 10:52:42 INFO MemoryStore: ensureFreeSpace(8) called with curMem=85009, maxMem=490356080 18/01/12 10:52:42 INFO MemoryStore: Block input-0-1515725561800 stored as bytes in memory (estimated size 8.0 B, free 467.6 MB) 18/01/12 10:52:42 INFO BlockManagerInfo: Added input-0-1515725561800 in memory on localhost:54439 (size: 8.0 B, free: 467.6 MB) 18/01/12 10:52:42 INFO BlockGenerator: Pushed block input-0-1515725561800 18/01/12 10:52:43 INFO MemoryStore: ensureFreeSpace(8) called with curMem=85017, maxMem=490356080 18/01/12 10:52:43 INFO MemoryStore: Block input-0-1515725562800 stored as bytes in memory (estimated size 8.0 B, free 467.6 MB) 18/01/12 10:52:43 INFO BlockManagerInfo: Added input-0-1515725562800 in memory on localhost:54439 (size: 8.0 B, free: 467.6 MB) 18/01/12 10:52:43 INFO BlockGenerator: Pushed block input-0-1515725562800 18/01/12 10:52:44 INFO MemoryStore: ensureFreeSpace(8) called with curMem=85025, maxMem=490356080 18/01/12 10:52:44 INFO MemoryStore: Block input-0-1515725563800 stored as bytes in memory (estimated size 8.0 B, free 467.6 MB) 18/01/12 10:52:44 INFO BlockManagerInfo: Added input-0-1515725563800 in memory on localhost:54439 (size: 8.0 B, free: 467.6 MB) 18/01/12 10:52:44 INFO BlockGenerator: Pushed block input-0-1515725563800 18/01/12 10:52:45 INFO MemoryStore: ensureFreeSpace(8) called with curMem=85033, maxMem=490356080 18/01/12 10:52:45 INFO MemoryStore: Block input-0-1515725564800 stored as bytes in memory (estimated size 8.0 B, free 467.6 MB) 18/01/12 10:52:45 INFO BlockManagerInfo: Added input-0-1515725564800 in memory on localhost:54439 (size: 8.0 B, free: 467.6 MB) 18/01/12 10:52:45 INFO BlockGenerator: Pushed block input-0-1515725564800 18/01/12 10:52:45 INFO JobScheduler: Added jobs for time 1515725565000 ms 18/01/12 10:52:45 INFO JobScheduler: Starting job streaming job 1515725565000 ms.0 from job set of time 1515725565000 ms 18/01/12 10:52:45 INFO SparkContext: Starting job: print at NetWorkWordCount.scala:29 18/01/12 10:52:45 INFO DAGScheduler: Registering RDD 199 (map at NetWorkWordCount.scala:28) 18/01/12 10:52:45 INFO DAGScheduler: Got job 99 (print at NetWorkWordCount.scala:29) with 1 output partitions (allowLocal=true) 18/01/12 10:52:45 INFO DAGScheduler: Final stage: ResultStage 198(print at NetWorkWordCount.scala:29) 18/01/12 10:52:45 INFO DAGScheduler: Parents of final stage: List(ShuffleMapStage 197) 18/01/12 10:52:45 INFO DAGScheduler: Missing parents: List(ShuffleMapStage 197) 18/01/12 10:52:45 INFO DAGScheduler: Submitting ShuffleMapStage 197 (MapPartitionsRDD[199] at map at NetWorkWordCount.scala:28), which has no missing parents 18/01/12 10:52:45 INFO MemoryStore: ensureFreeSpace(2560) called with curMem=85041, maxMem=490356080 18/01/12 10:52:45 INFO MemoryStore: Block broadcast_147 stored as values in memory (estimated size 2.5 KB, free 467.6 MB) 18/01/12 10:52:45 INFO MemoryStore: ensureFreeSpace(1565) called with curMem=87601, maxMem=490356080 18/01/12 10:52:45 INFO MemoryStore: Block broadcast_147_piece0 stored as bytes in memory (estimated size 1565.0 B, free 467.6 MB) 18/01/12 10:52:45 INFO BlockManagerInfo: Added broadcast_147_piece0 in memory on localhost:54439 (size: 1565.0 B, free: 467.6 MB) 18/01/12 10:52:45 INFO SparkContext: Created broadcast 147 from broadcast at DAGScheduler.scala:874 18/01/12 10:52:45 INFO DAGScheduler: Submitting 5 missing tasks from ShuffleMapStage 197 (MapPartitionsRDD[199] at map at NetWorkWordCount.scala:28) 18/01/12 10:52:45 INFO TaskSchedulerImpl: Adding task set 197.0 with 5 tasks 18/01/12 10:52:45 INFO TaskSetManager: Starting task 0.0 in stage 197.0 (TID 432, localhost, NODE_LOCAL, 1277 bytes) 18/01/12 10:52:45 INFO TaskSetManager: Starting task 1.0 in stage 197.0 (TID 433, localhost, NODE_LOCAL, 1277 bytes) 18/01/12 10:52:45 INFO TaskSetManager: Starting task 2.0 in stage 197.0 (TID 434, localhost, NODE_LOCAL, 1277 bytes) 18/01/12 10:52:45 INFO Executor: Running task 2.0 in stage 197.0 (TID 434) 18/01/12 10:52:45 INFO BlockManager: Found block input-0-1515725561800 locally 18/01/12 10:52:45 INFO Executor: Running task 0.0 in stage 197.0 (TID 432) 18/01/12 10:52:45 INFO BlockManager: Found block input-0-1515725559800 locally 18/01/12 10:52:45 INFO Executor: Running task 1.0 in stage 197.0 (TID 433) 18/01/12 10:52:45 INFO BlockManager: Found block input-0-1515725560800 locally 18/01/12 10:52:45 INFO Executor: Finished task 2.0 in stage 197.0 (TID 434). 882 bytes result sent to driver 18/01/12 10:52:45 INFO TaskSetManager: Starting task 3.0 in stage 197.0 (TID 435, localhost, NODE_LOCAL, 1277 bytes) 18/01/12 10:52:45 INFO Executor: Running task 3.0 in stage 197.0 (TID 435) 18/01/12 10:52:45 INFO BlockManager: Found block input-0-1515725562800 locally 18/01/12 10:52:45 INFO TaskSetManager: Finished task 2.0 in stage 197.0 (TID 434) in 14 ms on localhost (1/5) 18/01/12 10:52:45 INFO Executor: Finished task 1.0 in stage 197.0 (TID 433). 882 bytes result sent to driver 18/01/12 10:52:45 INFO TaskSetManager: Starting task 4.0 in stage 197.0 (TID 436, localhost, NODE_LOCAL, 1277 bytes) 18/01/12 10:52:45 INFO TaskSetManager: Finished task 1.0 in stage 197.0 (TID 433) in 17 ms on localhost (2/5) 18/01/12 10:52:45 INFO Executor: Running task 4.0 in stage 197.0 (TID 436) 18/01/12 10:52:45 INFO BlockManager: Found block input-0-1515725563800 locally 18/01/12 10:52:45 INFO Executor: Finished task 3.0 in stage 197.0 (TID 435). 882 bytes result sent to driver 18/01/12 10:52:45 INFO TaskSetManager: Finished task 3.0 in stage 197.0 (TID 435) in 14 ms on localhost (3/5) 18/01/12 10:52:45 INFO Executor: Finished task 0.0 in stage 197.0 (TID 432). 882 bytes result sent to driver 18/01/12 10:52:45 INFO TaskSetManager: Finished task 0.0 in stage 197.0 (TID 432) in 25 ms on localhost (4/5) 18/01/12 10:52:45 INFO Executor: Finished task 4.0 in stage 197.0 (TID 436). 882 bytes result sent to driver 18/01/12 10:52:45 INFO TaskSetManager: Finished task 4.0 in stage 197.0 (TID 436) in 12 ms on localhost (5/5) 18/01/12 10:52:45 INFO TaskSchedulerImpl: Removed TaskSet 197.0, whose tasks have all completed, from pool 18/01/12 10:52:45 INFO DAGScheduler: ShuffleMapStage 197 (map at NetWorkWordCount.scala:28) finished in 0.028 s 18/01/12 10:52:45 INFO DAGScheduler: looking for newly runnable stages 18/01/12 10:52:45 INFO DAGScheduler: running: Set(ResultStage 0) 18/01/12 10:52:45 INFO DAGScheduler: waiting: Set(ResultStage 198) 18/01/12 10:52:45 INFO DAGScheduler: failed: Set() 18/01/12 10:52:45 INFO DAGScheduler: Missing parents for ResultStage 198: List() 18/01/12 10:52:45 INFO DAGScheduler: Submitting ResultStage 198 (ShuffledRDD[200] at reduceByKey at NetWorkWordCount.scala:28), which is now runnable 18/01/12 10:52:45 INFO MemoryStore: ensureFreeSpace(2304) called with curMem=89166, maxMem=490356080 18/01/12 10:52:45 INFO MemoryStore: Block broadcast_148 stored as values in memory (estimated size 2.3 KB, free 467.6 MB) 18/01/12 10:52:45 INFO MemoryStore: ensureFreeSpace(1415) called with curMem=91470, maxMem=490356080 18/01/12 10:52:45 INFO MemoryStore: Block broadcast_148_piece0 stored as bytes in memory (estimated size 1415.0 B, free 467.6 MB) 18/01/12 10:52:45 INFO BlockManagerInfo: Added broadcast_148_piece0 in memory on localhost:54439 (size: 1415.0 B, free: 467.6 MB) 18/01/12 10:52:45 INFO SparkContext: Created broadcast 148 from broadcast at DAGScheduler.scala:874 18/01/12 10:52:45 INFO DAGScheduler: Submitting 1 missing tasks from ResultStage 198 (ShuffledRDD[200] at reduceByKey at NetWorkWordCount.scala:28) 18/01/12 10:52:45 INFO TaskSchedulerImpl: Adding task set 198.0 with 1 tasks 18/01/12 10:52:45 INFO TaskSetManager: Starting task 0.0 in stage 198.0 (TID 437, localhost, PROCESS_LOCAL, 1165 bytes) 18/01/12 10:52:45 INFO Executor: Running task 0.0 in stage 198.0 (TID 437) 18/01/12 10:52:45 INFO ShuffleBlockFetcherIterator: Getting 1 non-empty blocks out of 5 blocks 18/01/12 10:52:45 INFO ShuffleBlockFetcherIterator: Started 0 remote fetches in 0 ms 18/01/12 10:52:45 INFO Executor: Finished task 0.0 in stage 198.0 (TID 437). 1028 bytes result sent to driver 18/01/12 10:52:45 INFO TaskSetManager: Finished task 0.0 in stage 198.0 (TID 437) in 2 ms on localhost (1/1) 18/01/12 10:52:45 INFO TaskSchedulerImpl: Removed TaskSet 198.0, whose tasks have all completed, from pool 18/01/12 10:52:45 INFO DAGScheduler: ResultStage 198 (print at NetWorkWordCount.scala:29) finished in 0.002 s 18/01/12 10:52:45 INFO DAGScheduler: Job 99 finished: print at NetWorkWordCount.scala:29, took 0.039103 s 18/01/12 10:52:45 INFO SparkContext: Starting job: print at NetWorkWordCount.scala:29 18/01/12 10:52:45 INFO MapOutputTrackerMaster: Size of output statuses for shuffle 49 is 170 bytes 18/01/12 10:52:45 INFO DAGScheduler: Got job 100 (print at NetWorkWordCount.scala:29) with 3 output partitions (allowLocal=true) 18/01/12 10:52:45 INFO DAGScheduler: Final stage: ResultStage 200(print at NetWorkWordCount.scala:29) 18/01/12 10:52:45 INFO DAGScheduler: Parents of final stage: List(ShuffleMapStage 199) 18/01/12 10:52:45 INFO DAGScheduler: Missing parents: List() 18/01/12 10:52:45 INFO DAGScheduler: Submitting ResultStage 200 (ShuffledRDD[200] at reduceByKey at NetWorkWordCount.scala:28), which has no missing parents 18/01/12 10:52:45 INFO MemoryStore: ensureFreeSpace(2304) called with curMem=92885, maxMem=490356080 18/01/12 10:52:45 INFO MemoryStore: Block broadcast_149 stored as values in memory (estimated size 2.3 KB, free 467.5 MB) 18/01/12 10:52:45 INFO MemoryStore: ensureFreeSpace(1415) called with curMem=95189, maxMem=490356080 18/01/12 10:52:45 INFO MemoryStore: Block broadcast_149_piece0 stored as bytes in memory (estimated size 1415.0 B, free 467.5 MB) 18/01/12 10:52:45 INFO BlockManagerInfo: Added broadcast_149_piece0 in memory on localhost:54439 (size: 1415.0 B, free: 467.6 MB) 18/01/12 10:52:45 INFO SparkContext: Created broadcast 149 from broadcast at DAGScheduler.scala:874 18/01/12 10:52:45 INFO DAGScheduler: Submitting 3 missing tasks from ResultStage 200 (ShuffledRDD[200] at reduceByKey at NetWorkWordCount.scala:28) 18/01/12 10:52:45 INFO TaskSchedulerImpl: Adding task set 200.0 with 3 tasks 18/01/12 10:52:45 INFO TaskSetManager: Starting task 0.0 in stage 200.0 (TID 438, localhost, PROCESS_LOCAL, 1165 bytes) 18/01/12 10:52:45 INFO TaskSetManager: Starting task 1.0 in stage 200.0 (TID 439, localhost, PROCESS_LOCAL, 1165 bytes) 18/01/12 10:52:45 INFO TaskSetManager: Starting task 2.0 in stage 200.0 (TID 440, localhost, PROCESS_LOCAL, 1165 bytes) 18/01/12 10:52:45 INFO Executor: Running task 2.0 in stage 200.0 (TID 440) 18/01/12 10:52:45 INFO Executor: Running task 1.0 in stage 200.0 (TID 439) 18/01/12 10:52:45 INFO Executor: Running task 0.0 in stage 200.0 (TID 438) 18/01/12 10:52:45 INFO ShuffleBlockFetcherIterator: Getting 1 non-empty blocks out of 5 blocks 18/01/12 10:52:45 INFO ShuffleBlockFetcherIterator: Started 0 remote fetches in 0 ms 18/01/12 10:52:45 INFO ShuffleBlockFetcherIterator: Getting 2 non-empty blocks out of 5 blocks 18/01/12 10:52:45 INFO ShuffleBlockFetcherIterator: Started 0 remote fetches in 1 ms 18/01/12 10:52:45 INFO ShuffleBlockFetcherIterator: Getting 1 non-empty blocks out of 5 blocks 18/01/12 10:52:45 INFO ShuffleBlockFetcherIterator: Started 0 remote fetches in 1 ms 18/01/12 10:52:45 INFO Executor: Finished task 2.0 in stage 200.0 (TID 440). 1028 bytes result sent to driver 18/01/12 10:52:45 INFO Executor: Finished task 0.0 in stage 200.0 (TID 438). 1028 bytes result sent to driver 18/01/12 10:52:45 INFO Executor: Finished task 1.0 in stage 200.0 (TID 439). 1028 bytes result sent to driver 18/01/12 10:52:45 INFO TaskSetManager: Finished task 0.0 in stage 200.0 (TID 438) in 1 ms on localhost (1/3) 18/01/12 10:52:45 INFO TaskSetManager: Finished task 2.0 in stage 200.0 (TID 440) in 2 ms on localhost (2/3) 18/01/12 10:52:45 INFO TaskSetManager: Finished task 1.0 in stage 200.0 (TID 439) in 2 ms on localhost (3/3) 18/01/12 10:52:45 INFO TaskSchedulerImpl: Removed TaskSet 200.0, whose tasks have all completed, from pool 18/01/12 10:52:45 INFO DAGScheduler: ResultStage 200 (print at NetWorkWordCount.scala:29) finished in 0.003 s 18/01/12 10:52:45 INFO DAGScheduler: Job 100 finished: print at NetWorkWordCount.scala:29, took 0.005057 s ------------------------------------------- Time: 1515725565000 ms ------------------------------------------- (D,1) (A,1) (F,2) (G,1) 18/01/12 10:52:45 INFO JobScheduler: Finished job streaming job 1515725565000 ms.0 from job set of time 1515725565000 ms 18/01/12 10:52:45 INFO JobScheduler: Total delay: 0.061 s for time 1515725565000 ms (execution: 0.048 s) 18/01/12 10:52:45 INFO ShuffledRDD: Removing RDD 196 from persistence list 18/01/12 10:52:45 INFO MapPartitionsRDD: Removing RDD 195 from persistence list 18/01/12 10:52:45 INFO BlockManager: Removing RDD 196 18/01/12 10:52:45 INFO BlockManager: Removing RDD 195 18/01/12 10:52:45 INFO MapPartitionsRDD: Removing RDD 194 from persistence list 18/01/12 10:52:45 INFO BlockManager: Removing RDD 194 18/01/12 10:52:45 INFO BlockRDD: Removing RDD 193 from persistence list 18/01/12 10:52:45 INFO BlockManager: Removing RDD 193 18/01/12 10:52:45 INFO SocketInputDStream: Removing blocks of RDD BlockRDD[193] at socketTextStream at NetWorkWordCount.scala:26 of time 1515725565000 ms 18/01/12 10:52:45 INFO BlockManagerInfo: Removed input-0-1515725554800 on localhost:54439 in memory (size: 8.0 B, free: 467.6 MB) 18/01/12 10:52:45 INFO BlockManagerInfo: Removed input-0-1515725555800 on localhost:54439 in memory (size: 8.0 B, free: 467.6 MB) 18/01/12 10:52:45 INFO BlockManagerInfo: Removed input-0-1515725556800 on localhost:54439 in memory (size: 8.0 B, free: 467.6 MB) 18/01/12 10:52:45 INFO ReceivedBlockTracker: Deleting batches ArrayBuffer(1515725555000 ms) 18/01/12 10:52:45 INFO BlockManagerInfo: Removed input-0-1515725557800 on localhost:54439 in memory (size: 8.0 B, free: 467.6 MB) 18/01/12 10:52:45 INFO InputInfoTracker: remove old batch metadata: 1515725555000 ms 18/01/12 10:52:45 INFO BlockManagerInfo: Removed input-0-1515725558800 on localhost:54439 in memory (size: 8.0 B, free: 467.6 MB) 18/01/12 10:52:46 INFO MemoryStore: ensureFreeSpace(8) called with curMem=96564, maxMem=490356080 18/01/12 10:52:46 INFO MemoryStore: Block input-0-1515725565800 stored as bytes in memory (estimated size 8.0 B, free 467.5 MB) 18/01/12 10:52:46 INFO BlockManagerInfo: Added input-0-1515725565800 in memory on localhost:54439 (size: 8.0 B, free: 467.6 MB) 18/01/12 10:52:46 INFO BlockGenerator: Pushed block input-0-1515725565800 18/01/12 10:52:47 INFO MemoryStore: ensureFreeSpace(8) called with curMem=96572, maxMem=490356080 18/01/12 10:52:47 INFO MemoryStore: Block input-0-1515725566800 stored as bytes in memory (estimated size 8.0 B, free 467.5 MB) 18/01/12 10:52:47 INFO BlockManagerInfo: Added input-0-1515725566800 in memory on localhost:54439 (size: 8.0 B, free: 467.6 MB) 18/01/12 10:52:47 INFO BlockGenerator: Pushed block input-0-1515725566800 18/01/12 10:52:48 INFO MemoryStore: ensureFreeSpace(8) called with curMem=96580, maxMem=490356080 18/01/12 10:52:48 INFO MemoryStore: Block input-0-1515725567800 stored as bytes in memory (estimated size 8.0 B, free 467.5 MB) 18/01/12 10:52:48 INFO BlockManagerInfo: Added input-0-1515725567800 in memory on localhost:54439 (size: 8.0 B, free: 467.6 MB) 18/01/12 10:52:48 INFO BlockGenerator: Pushed block input-0-1515725567800 18/01/12 10:52:49 INFO MemoryStore: ensureFreeSpace(8) called with curMem=96588, maxMem=490356080 18/01/12 10:52:49 INFO MemoryStore: Block input-0-1515725568800 stored as bytes in memory (estimated size 8.0 B, free 467.5 MB) 18/01/12 10:52:49 INFO BlockManagerInfo: Added input-0-1515725568800 in memory on localhost:54439 (size: 8.0 B, free: 467.6 MB) 18/01/12 10:52:49 INFO BlockGenerator: Pushed block input-0-1515725568800 18/01/12 10:52:50 INFO MemoryStore: ensureFreeSpace(8) called with curMem=96596, maxMem=490356080 18/01/12 10:52:50 INFO MemoryStore: Block input-0-1515725569800 stored as bytes in memory (estimated size 8.0 B, free 467.5 MB) 18/01/12 10:52:50 INFO BlockManagerInfo: Added input-0-1515725569800 in memory on localhost:54439 (size: 8.0 B, free: 467.6 MB) 18/01/12 10:52:50 INFO BlockGenerator: Pushed block input-0-1515725569800 18/01/12 10:52:50 INFO JobScheduler: Added jobs for time 1515725570000 ms 18/01/12 10:52:50 INFO SparkContext: Starting job: print at NetWorkWordCount.scala:29 18/01/12 10:52:50 INFO JobScheduler: Starting job streaming job 1515725570000 ms.0 from job set of time 1515725570000 ms 18/01/12 10:52:50 INFO DAGScheduler: Registering RDD 203 (map at NetWorkWordCount.scala:28) 18/01/12 10:52:50 INFO DAGScheduler: Got job 101 (print at NetWorkWordCount.scala:29) with 1 output partitions (allowLocal=true) 18/01/12 10:52:50 INFO DAGScheduler: Final stage: ResultStage 202(print at NetWorkWordCount.scala:29) 18/01/12 10:52:50 INFO DAGScheduler: Parents of final stage: List(ShuffleMapStage 201) 18/01/12 10:52:50 INFO DAGScheduler: Missing parents: List(ShuffleMapStage 201) 18/01/12 10:52:50 INFO DAGScheduler: Submitting ShuffleMapStage 201 (MapPartitionsRDD[203] at map at NetWorkWordCount.scala:28), which has no missing parents 18/01/12 10:52:50 INFO MemoryStore: ensureFreeSpace(2560) called with curMem=96604, maxMem=490356080 18/01/12 10:52:50 INFO MemoryStore: Block broadcast_150 stored as values in memory (estimated size 2.5 KB, free 467.5 MB) 18/01/12 10:52:50 INFO MemoryStore: ensureFreeSpace(1564) called with curMem=99164, maxMem=490356080 18/01/12 10:52:50 INFO MemoryStore: Block broadcast_150_piece0 stored as bytes in memory (estimated size 1564.0 B, free 467.5 MB) 18/01/12 10:52:50 INFO BlockManagerInfo: Added broadcast_150_piece0 in memory on localhost:54439 (size: 1564.0 B, free: 467.6 MB) 18/01/12 10:52:50 INFO SparkContext: Created broadcast 150 from broadcast at DAGScheduler.scala:874 18/01/12 10:52:50 INFO DAGScheduler: Submitting 5 missing tasks from ShuffleMapStage 201 (MapPartitionsRDD[203] at map at NetWorkWordCount.scala:28) 18/01/12 10:52:50 INFO TaskSchedulerImpl: Adding task set 201.0 with 5 tasks 18/01/12 10:52:50 INFO TaskSetManager: Starting task 0.0 in stage 201.0 (TID 441, localhost, NODE_LOCAL, 1277 bytes) 18/01/12 10:52:50 INFO TaskSetManager: Starting task 1.0 in stage 201.0 (TID 442, localhost, NODE_LOCAL, 1277 bytes) 18/01/12 10:52:50 INFO TaskSetManager: Starting task 2.0 in stage 201.0 (TID 443, localhost, NODE_LOCAL, 1277 bytes) 18/01/12 10:52:50 INFO Executor: Running task 2.0 in stage 201.0 (TID 443) 18/01/12 10:52:50 INFO BlockManager: Found block input-0-1515725566800 locally 18/01/12 10:52:50 INFO Executor: Running task 0.0 in stage 201.0 (TID 441) 18/01/12 10:52:50 INFO BlockManager: Found block input-0-1515725564800 locally 18/01/12 10:52:50 INFO Executor: Running task 1.0 in stage 201.0 (TID 442) 18/01/12 10:52:50 INFO BlockManager: Found block input-0-1515725565800 locally 18/01/12 10:52:50 INFO Executor: Finished task 2.0 in stage 201.0 (TID 443). 882 bytes result sent to driver 18/01/12 10:52:50 INFO TaskSetManager: Starting task 3.0 in stage 201.0 (TID 444, localhost, NODE_LOCAL, 1277 bytes) 18/01/12 10:52:50 INFO TaskSetManager: Finished task 2.0 in stage 201.0 (TID 443) in 12 ms on localhost (1/5) 18/01/12 10:52:50 INFO Executor: Running task 3.0 in stage 201.0 (TID 444) 18/01/12 10:52:50 INFO Executor: Finished task 0.0 in stage 201.0 (TID 441). 882 bytes result sent to driver 18/01/12 10:52:50 INFO TaskSetManager: Starting task 4.0 in stage 201.0 (TID 445, localhost, NODE_LOCAL, 1277 bytes) 18/01/12 10:52:50 INFO TaskSetManager: Finished task 0.0 in stage 201.0 (TID 441) in 14 ms on localhost (2/5) 18/01/12 10:52:50 INFO Executor: Running task 4.0 in stage 201.0 (TID 445) 18/01/12 10:52:50 INFO Executor: Finished task 1.0 in stage 201.0 (TID 442). 882 bytes result sent to driver 18/01/12 10:52:50 INFO TaskSetManager: Finished task 1.0 in stage 201.0 (TID 442) in 15 ms on localhost (3/5) 18/01/12 10:52:50 INFO BlockManager: Found block input-0-1515725568800 locally 18/01/12 10:52:50 INFO BlockManager: Found block input-0-1515725567800 locally 18/01/12 10:52:50 INFO Executor: Finished task 4.0 in stage 201.0 (TID 445). 882 bytes result sent to driver 18/01/12 10:52:50 INFO Executor: Finished task 3.0 in stage 201.0 (TID 444). 882 bytes result sent to driver 18/01/12 10:52:50 INFO TaskSetManager: Finished task 4.0 in stage 201.0 (TID 445) in 4 ms on localhost (4/5) 18/01/12 10:52:50 INFO TaskSetManager: Finished task 3.0 in stage 201.0 (TID 444) in 6 ms on localhost (5/5) 18/01/12 10:52:50 INFO TaskSchedulerImpl: Removed TaskSet 201.0, whose tasks have all completed, from pool 18/01/12 10:52:50 INFO DAGScheduler: ShuffleMapStage 201 (map at NetWorkWordCount.scala:28) finished in 0.018 s 18/01/12 10:52:50 INFO DAGScheduler: looking for newly runnable stages 18/01/12 10:52:50 INFO DAGScheduler: running: Set(ResultStage 0) 18/01/12 10:52:50 INFO DAGScheduler: waiting: Set(ResultStage 202) 18/01/12 10:52:50 INFO DAGScheduler: failed: Set() 18/01/12 10:52:50 INFO DAGScheduler: Missing parents for ResultStage 202: List() 18/01/12 10:52:50 INFO DAGScheduler: Submitting ResultStage 202 (ShuffledRDD[204] at reduceByKey at NetWorkWordCount.scala:28), which is now runnable 18/01/12 10:52:50 INFO MemoryStore: ensureFreeSpace(2304) called with curMem=100728, maxMem=490356080 18/01/12 10:52:50 INFO MemoryStore: Block broadcast_151 stored as values in memory (estimated size 2.3 KB, free 467.5 MB) 18/01/12 10:52:50 INFO MemoryStore: ensureFreeSpace(1417) called with curMem=103032, maxMem=490356080 18/01/12 10:52:50 INFO MemoryStore: Block broadcast_151_piece0 stored as bytes in memory (estimated size 1417.0 B, free 467.5 MB) 18/01/12 10:52:50 INFO BlockManagerInfo: Added broadcast_151_piece0 in memory on localhost:54439 (size: 1417.0 B, free: 467.6 MB) 18/01/12 10:52:50 INFO SparkContext: Created broadcast 151 from broadcast at DAGScheduler.scala:874 18/01/12 10:52:50 INFO DAGScheduler: Submitting 1 missing tasks from ResultStage 202 (ShuffledRDD[204] at reduceByKey at NetWorkWordCount.scala:28) 18/01/12 10:52:50 INFO TaskSchedulerImpl: Adding task set 202.0 with 1 tasks 18/01/12 10:52:50 INFO TaskSetManager: Starting task 0.0 in stage 202.0 (TID 446, localhost, PROCESS_LOCAL, 1165 bytes) 18/01/12 10:52:50 INFO Executor: Running task 0.0 in stage 202.0 (TID 446) 18/01/12 10:52:50 INFO ShuffleBlockFetcherIterator: Getting 1 non-empty blocks out of 5 blocks 18/01/12 10:52:50 INFO ShuffleBlockFetcherIterator: Started 0 remote fetches in 0 ms 18/01/12 10:52:50 INFO Executor: Finished task 0.0 in stage 202.0 (TID 446). 1028 bytes result sent to driver 18/01/12 10:52:50 INFO TaskSetManager: Finished task 0.0 in stage 202.0 (TID 446) in 2 ms on localhost (1/1) 18/01/12 10:52:50 INFO DAGScheduler: ResultStage 202 (print at NetWorkWordCount.scala:29) finished in 0.002 s 18/01/12 10:52:50 INFO TaskSchedulerImpl: Removed TaskSet 202.0, whose tasks have all completed, from pool 18/01/12 10:52:50 INFO DAGScheduler: Job 101 finished: print at NetWorkWordCount.scala:29, took 0.028394 s 18/01/12 10:52:50 INFO SparkContext: Starting job: print at NetWorkWordCount.scala:29 18/01/12 10:52:50 INFO MapOutputTrackerMaster: Size of output statuses for shuffle 50 is 165 bytes 18/01/12 10:52:50 INFO DAGScheduler: Got job 102 (print at NetWorkWordCount.scala:29) with 3 output partitions (allowLocal=true) 18/01/12 10:52:50 INFO DAGScheduler: Final stage: ResultStage 204(print at NetWorkWordCount.scala:29) 18/01/12 10:52:50 INFO DAGScheduler: Parents of final stage: List(ShuffleMapStage 203) 18/01/12 10:52:50 INFO DAGScheduler: Missing parents: List() 18/01/12 10:52:50 INFO DAGScheduler: Submitting ResultStage 204 (ShuffledRDD[204] at reduceByKey at NetWorkWordCount.scala:28), which has no missing parents 18/01/12 10:52:50 INFO MemoryStore: ensureFreeSpace(2304) called with curMem=104449, maxMem=490356080 18/01/12 10:52:50 INFO MemoryStore: Block broadcast_152 stored as values in memory (estimated size 2.3 KB, free 467.5 MB) 18/01/12 10:52:50 INFO MemoryStore: ensureFreeSpace(1417) called with curMem=106753, maxMem=490356080 18/01/12 10:52:50 INFO MemoryStore: Block broadcast_152_piece0 stored as bytes in memory (estimated size 1417.0 B, free 467.5 MB) 18/01/12 10:52:50 INFO BlockManagerInfo: Added broadcast_152_piece0 in memory on localhost:54439 (size: 1417.0 B, free: 467.6 MB) 18/01/12 10:52:50 INFO SparkContext: Created broadcast 152 from broadcast at DAGScheduler.scala:874 18/01/12 10:52:50 INFO DAGScheduler: Submitting 3 missing tasks from ResultStage 204 (ShuffledRDD[204] at reduceByKey at NetWorkWordCount.scala:28) 18/01/12 10:52:50 INFO TaskSchedulerImpl: Adding task set 204.0 with 3 tasks 18/01/12 10:52:50 INFO TaskSetManager: Starting task 0.0 in stage 204.0 (TID 447, localhost, PROCESS_LOCAL, 1165 bytes) 18/01/12 10:52:50 INFO TaskSetManager: Starting task 1.0 in stage 204.0 (TID 448, localhost, PROCESS_LOCAL, 1165 bytes) 18/01/12 10:52:50 INFO TaskSetManager: Starting task 2.0 in stage 204.0 (TID 449, localhost, PROCESS_LOCAL, 1165 bytes) 18/01/12 10:52:50 INFO Executor: Running task 0.0 in stage 204.0 (TID 447) 18/01/12 10:52:50 INFO ShuffleBlockFetcherIterator: Getting 0 non-empty blocks out of 5 blocks 18/01/12 10:52:50 INFO ShuffleBlockFetcherIterator: Started 0 remote fetches in 1 ms 18/01/12 10:52:50 INFO Executor: Finished task 0.0 in stage 204.0 (TID 447). 882 bytes result sent to driver 18/01/12 10:52:50 INFO Executor: Running task 2.0 in stage 204.0 (TID 449) 18/01/12 10:52:50 INFO Executor: Running task 1.0 in stage 204.0 (TID 448) 18/01/12 10:52:50 INFO ShuffleBlockFetcherIterator: Getting 2 non-empty blocks out of 5 blocks 18/01/12 10:52:50 INFO ShuffleBlockFetcherIterator: Started 0 remote fetches in 0 ms 18/01/12 10:52:50 INFO TaskSetManager: Finished task 0.0 in stage 204.0 (TID 447) in 4 ms on localhost (1/3) 18/01/12 10:52:50 INFO ShuffleBlockFetcherIterator: Getting 2 non-empty blocks out of 5 blocks 18/01/12 10:52:50 INFO Executor: Finished task 1.0 in stage 204.0 (TID 448). 1048 bytes result sent to driver 18/01/12 10:52:50 INFO ShuffleBlockFetcherIterator: Started 0 remote fetches in 2 ms 18/01/12 10:52:50 INFO TaskSetManager: Finished task 1.0 in stage 204.0 (TID 448) in 5 ms on localhost (2/3) 18/01/12 10:52:50 INFO Executor: Finished task 2.0 in stage 204.0 (TID 449). 1028 bytes result sent to driver 18/01/12 10:52:50 INFO TaskSetManager: Finished task 2.0 in stage 204.0 (TID 449) in 6 ms on localhost (3/3) 18/01/12 10:52:50 INFO TaskSchedulerImpl: Removed TaskSet 204.0, whose tasks have all completed, from pool 18/01/12 10:52:50 INFO DAGScheduler: ResultStage 204 (print at NetWorkWordCount.scala:29) finished in 0.006 s 18/01/12 10:52:50 INFO DAGScheduler: Job 102 finished: print at NetWorkWordCount.scala:29, took 0.009907 s ------------------------------------------- Time: 1515725570000 ms ------------------------------------------- 18/01/12 10:52:50 INFO JobScheduler: Finished job streaming job 1515725570000 ms.0 from job set of time 1515725570000 ms 18/01/12 10:52:50 INFO JobScheduler: Total delay: 0.054 s for time 1515725570000 ms (execution: 0.046 s) 18/01/12 10:52:50 INFO ShuffledRDD: Removing RDD 200 from persistence list (D,1) (B,1) (F,1) (C,2) 18/01/12 10:52:50 INFO BlockManager: Removing RDD 200 18/01/12 10:52:50 INFO MapPartitionsRDD: Removing RDD 199 from persistence list 18/01/12 10:52:50 INFO BlockManager: Removing RDD 199 18/01/12 10:52:50 INFO MapPartitionsRDD: Removing RDD 198 from persistence list 18/01/12 10:52:50 INFO BlockManager: Removing RDD 198 18/01/12 10:52:50 INFO BlockRDD: Removing RDD 197 from persistence list 18/01/12 10:52:50 INFO BlockManager: Removing RDD 197 18/01/12 10:52:50 INFO SocketInputDStream: Removing blocks of RDD BlockRDD[197] at socketTextStream at NetWorkWordCount.scala:26 of time 1515725570000 ms 18/01/12 10:52:50 INFO BlockManagerInfo: Removed input-0-1515725559800 on localhost:54439 in memory (size: 8.0 B, free: 467.6 MB) 18/01/12 10:52:50 INFO BlockManagerInfo: Removed input-0-1515725560800 on localhost:54439 in memory (size: 8.0 B, free: 467.6 MB) 18/01/12 10:52:50 INFO BlockManagerInfo: Removed input-0-1515725561800 on localhost:54439 in memory (size: 8.0 B, free: 467.6 MB) 18/01/12 10:52:50 INFO ReceivedBlockTracker: Deleting batches ArrayBuffer(1515725560000 ms) 18/01/12 10:52:50 INFO InputInfoTracker: remove old batch metadata: 1515725560000 ms 18/01/12 10:52:50 INFO BlockManagerInfo: Removed input-0-1515725563800 on localhost:54439 in memory (size: 8.0 B, free: 467.6 MB) 18/01/12 10:52:50 INFO BlockManagerInfo: Removed input-0-1515725562800 on localhost:54439 in memory (size: 8.0 B, free: 467.6 MB) 18/01/12 10:52:51 INFO MemoryStore: ensureFreeSpace(8) called with curMem=108130, maxMem=490356080 18/01/12 10:52:51 INFO MemoryStore: Block input-0-1515725570800 stored as bytes in memory (estimated size 8.0 B, free 467.5 MB) 18/01/12 10:52:51 INFO BlockManagerInfo: Added input-0-1515725570800 in memory on localhost:54439 (size: 8.0 B, free: 467.6 MB) 18/01/12 10:52:51 INFO BlockGenerator: Pushed block input-0-1515725570800 18/01/12 10:52:52 INFO MemoryStore: ensureFreeSpace(8) called with curMem=108138, maxMem=490356080 18/01/12 10:52:52 INFO MemoryStore: Block input-0-1515725571800 stored as bytes in memory (estimated size 8.0 B, free 467.5 MB) 18/01/12 10:52:52 INFO BlockManagerInfo: Added input-0-1515725571800 in memory on localhost:54439 (size: 8.0 B, free: 467.6 MB) 18/01/12 10:52:52 INFO BlockGenerator: Pushed block input-0-1515725571800 18/01/12 10:52:53 INFO MemoryStore: ensureFreeSpace(8) called with curMem=108146, maxMem=490356080 18/01/12 10:52:53 INFO MemoryStore: Block input-0-1515725572800 stored as bytes in memory (estimated size 8.0 B, free 467.5 MB) 18/01/12 10:52:53 INFO BlockManagerInfo: Added input-0-1515725572800 in memory on localhost:54439 (size: 8.0 B, free: 467.6 MB) 18/01/12 10:52:53 INFO BlockGenerator: Pushed block input-0-1515725572800 18/01/12 10:52:54 INFO MemoryStore: ensureFreeSpace(8) called with curMem=108154, maxMem=490356080 18/01/12 10:52:54 INFO MemoryStore: Block input-0-1515725573800 stored as bytes in memory (estimated size 8.0 B, free 467.5 MB) 18/01/12 10:52:54 INFO BlockManagerInfo: Added input-0-1515725573800 in memory on localhost:54439 (size: 8.0 B, free: 467.6 MB) 18/01/12 10:52:54 INFO BlockGenerator: Pushed block input-0-1515725573800 18/01/12 10:52:55 INFO MemoryStore: ensureFreeSpace(8) called with curMem=108162, maxMem=490356080 18/01/12 10:52:55 INFO MemoryStore: Block input-0-1515725574800 stored as bytes in memory (estimated size 8.0 B, free 467.5 MB) 18/01/12 10:52:55 INFO BlockManagerInfo: Added input-0-1515725574800 in memory on localhost:54439 (size: 8.0 B, free: 467.6 MB) 18/01/12 10:52:55 INFO BlockGenerator: Pushed block input-0-1515725574800 18/01/12 10:52:55 INFO JobScheduler: Added jobs for time 1515725575000 ms 18/01/12 10:52:55 INFO JobScheduler: Starting job streaming job 1515725575000 ms.0 from job set of time 1515725575000 ms 18/01/12 10:52:55 INFO SparkContext: Starting job: print at NetWorkWordCount.scala:29 18/01/12 10:52:55 INFO DAGScheduler: Registering RDD 207 (map at NetWorkWordCount.scala:28) 18/01/12 10:52:55 INFO DAGScheduler: Got job 103 (print at NetWorkWordCount.scala:29) with 1 output partitions (allowLocal=true) 18/01/12 10:52:55 INFO DAGScheduler: Final stage: ResultStage 206(print at NetWorkWordCount.scala:29) 18/01/12 10:52:55 INFO DAGScheduler: Parents of final stage: List(ShuffleMapStage 205) 18/01/12 10:52:55 INFO DAGScheduler: Missing parents: List(ShuffleMapStage 205) 18/01/12 10:52:55 INFO DAGScheduler: Submitting ShuffleMapStage 205 (MapPartitionsRDD[207] at map at NetWorkWordCount.scala:28), which has no missing parents 18/01/12 10:52:55 INFO MemoryStore: ensureFreeSpace(2560) called with curMem=108170, maxMem=490356080 18/01/12 10:52:55 INFO MemoryStore: Block broadcast_153 stored as values in memory (estimated size 2.5 KB, free 467.5 MB) 18/01/12 10:52:55 INFO MemoryStore: ensureFreeSpace(1564) called with curMem=110730, maxMem=490356080 18/01/12 10:52:55 INFO MemoryStore: Block broadcast_153_piece0 stored as bytes in memory (estimated size 1564.0 B, free 467.5 MB) 18/01/12 10:52:55 INFO BlockManagerInfo: Added broadcast_153_piece0 in memory on localhost:54439 (size: 1564.0 B, free: 467.6 MB) 18/01/12 10:52:55 INFO SparkContext: Created broadcast 153 from broadcast at DAGScheduler.scala:874 18/01/12 10:52:55 INFO DAGScheduler: Submitting 5 missing tasks from ShuffleMapStage 205 (MapPartitionsRDD[207] at map at NetWorkWordCount.scala:28) 18/01/12 10:52:55 INFO TaskSchedulerImpl: Adding task set 205.0 with 5 tasks 18/01/12 10:52:55 INFO TaskSetManager: Starting task 0.0 in stage 205.0 (TID 450, localhost, NODE_LOCAL, 1277 bytes) 18/01/12 10:52:55 INFO TaskSetManager: Starting task 1.0 in stage 205.0 (TID 451, localhost, NODE_LOCAL, 1277 bytes) 18/01/12 10:52:55 INFO TaskSetManager: Starting task 2.0 in stage 205.0 (TID 452, localhost, NODE_LOCAL, 1277 bytes) 18/01/12 10:52:55 INFO Executor: Running task 0.0 in stage 205.0 (TID 450) 18/01/12 10:52:55 INFO Executor: Running task 1.0 in stage 205.0 (TID 451) 18/01/12 10:52:55 INFO Executor: Running task 2.0 in stage 205.0 (TID 452) 18/01/12 10:52:55 INFO BlockManager: Found block input-0-1515725569800 locally 18/01/12 10:52:55 INFO BlockManager: Found block input-0-1515725571800 locally 18/01/12 10:52:55 INFO BlockManager: Found block input-0-1515725570800 locally 18/01/12 10:52:55 INFO Executor: Finished task 0.0 in stage 205.0 (TID 450). 882 bytes result sent to driver 18/01/12 10:52:55 INFO TaskSetManager: Starting task 3.0 in stage 205.0 (TID 453, localhost, NODE_LOCAL, 1277 bytes) 18/01/12 10:52:55 INFO Executor: Running task 3.0 in stage 205.0 (TID 453) 18/01/12 10:52:55 INFO TaskSetManager: Finished task 0.0 in stage 205.0 (TID 450) in 5 ms on localhost (1/5) 18/01/12 10:52:55 INFO BlockManager: Found block input-0-1515725572800 locally 18/01/12 10:52:55 INFO Executor: Finished task 2.0 in stage 205.0 (TID 452). 882 bytes result sent to driver 18/01/12 10:52:55 INFO Executor: Finished task 3.0 in stage 205.0 (TID 453). 882 bytes result sent to driver 18/01/12 10:52:55 INFO TaskSetManager: Starting task 4.0 in stage 205.0 (TID 454, localhost, NODE_LOCAL, 1277 bytes) 18/01/12 10:52:55 INFO Executor: Running task 4.0 in stage 205.0 (TID 454) 18/01/12 10:52:55 INFO BlockManager: Found block input-0-1515725573800 locally 18/01/12 10:52:55 INFO TaskSetManager: Finished task 2.0 in stage 205.0 (TID 452) in 12 ms on localhost (2/5) 18/01/12 10:52:55 INFO TaskSetManager: Finished task 3.0 in stage 205.0 (TID 453) in 9 ms on localhost (3/5) 18/01/12 10:52:55 INFO Executor: Finished task 4.0 in stage 205.0 (TID 454). 882 bytes result sent to driver 18/01/12 10:52:55 INFO TaskSetManager: Finished task 4.0 in stage 205.0 (TID 454) in 4 ms on localhost (4/5) 18/01/12 10:52:55 INFO Executor: Finished task 1.0 in stage 205.0 (TID 451). 882 bytes result sent to driver 18/01/12 10:52:55 INFO TaskSetManager: Finished task 1.0 in stage 205.0 (TID 451) in 15 ms on localhost (5/5) 18/01/12 10:52:55 INFO TaskSchedulerImpl: Removed TaskSet 205.0, whose tasks have all completed, from pool 18/01/12 10:52:55 INFO DAGScheduler: ShuffleMapStage 205 (map at NetWorkWordCount.scala:28) finished in 0.015 s 18/01/12 10:52:55 INFO DAGScheduler: looking for newly runnable stages 18/01/12 10:52:55 INFO DAGScheduler: running: Set(ResultStage 0) 18/01/12 10:52:55 INFO DAGScheduler: waiting: Set(ResultStage 206) 18/01/12 10:52:55 INFO DAGScheduler: failed: Set() 18/01/12 10:52:55 INFO DAGScheduler: Missing parents for ResultStage 206: List() 18/01/12 10:52:55 INFO DAGScheduler: Submitting ResultStage 206 (ShuffledRDD[208] at reduceByKey at NetWorkWordCount.scala:28), which is now runnable 18/01/12 10:52:55 INFO MemoryStore: ensureFreeSpace(2304) called with curMem=112294, maxMem=490356080 18/01/12 10:52:55 INFO MemoryStore: Block broadcast_154 stored as values in memory (estimated size 2.3 KB, free 467.5 MB) 18/01/12 10:52:55 INFO MemoryStore: ensureFreeSpace(1417) called with curMem=114598, maxMem=490356080 18/01/12 10:52:55 INFO MemoryStore: Block broadcast_154_piece0 stored as bytes in memory (estimated size 1417.0 B, free 467.5 MB) 18/01/12 10:52:55 INFO BlockManagerInfo: Added broadcast_154_piece0 in memory on localhost:54439 (size: 1417.0 B, free: 467.6 MB) 18/01/12 10:52:55 INFO SparkContext: Created broadcast 154 from broadcast at DAGScheduler.scala:874 18/01/12 10:52:55 INFO DAGScheduler: Submitting 1 missing tasks from ResultStage 206 (ShuffledRDD[208] at reduceByKey at NetWorkWordCount.scala:28) 18/01/12 10:52:55 INFO TaskSchedulerImpl: Adding task set 206.0 with 1 tasks 18/01/12 10:52:55 INFO TaskSetManager: Starting task 0.0 in stage 206.0 (TID 455, localhost, PROCESS_LOCAL, 1165 bytes) 18/01/12 10:52:55 INFO Executor: Running task 0.0 in stage 206.0 (TID 455) 18/01/12 10:52:55 INFO ShuffleBlockFetcherIterator: Getting 0 non-empty blocks out of 5 blocks 18/01/12 10:52:55 INFO ShuffleBlockFetcherIterator: Started 0 remote fetches in 0 ms 18/01/12 10:52:55 INFO Executor: Finished task 0.0 in stage 206.0 (TID 455). 882 bytes result sent to driver 18/01/12 10:52:55 INFO TaskSetManager: Finished task 0.0 in stage 206.0 (TID 455) in 2 ms on localhost (1/1) 18/01/12 10:52:55 INFO TaskSchedulerImpl: Removed TaskSet 206.0, whose tasks have all completed, from pool 18/01/12 10:52:55 INFO DAGScheduler: ResultStage 206 (print at NetWorkWordCount.scala:29) finished in 0.002 s 18/01/12 10:52:55 INFO DAGScheduler: Job 103 finished: print at NetWorkWordCount.scala:29, took 0.025503 s 18/01/12 10:52:55 INFO SparkContext: Starting job: print at NetWorkWordCount.scala:29 18/01/12 10:52:55 INFO MapOutputTrackerMaster: Size of output statuses for shuffle 51 is 165 bytes 18/01/12 10:52:55 INFO DAGScheduler: Got job 104 (print at NetWorkWordCount.scala:29) with 3 output partitions (allowLocal=true) 18/01/12 10:52:55 INFO DAGScheduler: Final stage: ResultStage 208(print at NetWorkWordCount.scala:29) 18/01/12 10:52:55 INFO DAGScheduler: Parents of final stage: List(ShuffleMapStage 207) 18/01/12 10:52:55 INFO DAGScheduler: Missing parents: List() 18/01/12 10:52:55 INFO DAGScheduler: Submitting ResultStage 208 (ShuffledRDD[208] at reduceByKey at NetWorkWordCount.scala:28), which has no missing parents 18/01/12 10:52:55 INFO MemoryStore: ensureFreeSpace(2304) called with curMem=116015, maxMem=490356080 18/01/12 10:52:55 INFO MemoryStore: Block broadcast_155 stored as values in memory (estimated size 2.3 KB, free 467.5 MB) 18/01/12 10:52:55 INFO MemoryStore: ensureFreeSpace(1417) called with curMem=118319, maxMem=490356080 18/01/12 10:52:55 INFO MemoryStore: Block broadcast_155_piece0 stored as bytes in memory (estimated size 1417.0 B, free 467.5 MB) 18/01/12 10:52:55 INFO BlockManagerInfo: Added broadcast_155_piece0 in memory on localhost:54439 (size: 1417.0 B, free: 467.6 MB) 18/01/12 10:52:55 INFO SparkContext: Created broadcast 155 from broadcast at DAGScheduler.scala:874 18/01/12 10:52:55 INFO DAGScheduler: Submitting 3 missing tasks from ResultStage 208 (ShuffledRDD[208] at reduceByKey at NetWorkWordCount.scala:28) 18/01/12 10:52:55 INFO TaskSchedulerImpl: Adding task set 208.0 with 3 tasks 18/01/12 10:52:55 INFO TaskSetManager: Starting task 0.0 in stage 208.0 (TID 456, localhost, PROCESS_LOCAL, 1165 bytes) 18/01/12 10:52:55 INFO TaskSetManager: Starting task 1.0 in stage 208.0 (TID 457, localhost, PROCESS_LOCAL, 1165 bytes) 18/01/12 10:52:55 INFO TaskSetManager: Starting task 2.0 in stage 208.0 (TID 458, localhost, PROCESS_LOCAL, 1165 bytes) 18/01/12 10:52:55 INFO Executor: Running task 1.0 in stage 208.0 (TID 457) 18/01/12 10:52:55 INFO ShuffleBlockFetcherIterator: Getting 2 non-empty blocks out of 5 blocks 18/01/12 10:52:55 INFO ShuffleBlockFetcherIterator: Started 0 remote fetches in 0 ms 18/01/12 10:52:55 INFO Executor: Finished task 1.0 in stage 208.0 (TID 457). 1048 bytes result sent to driver 18/01/12 10:52:55 INFO Executor: Running task 2.0 in stage 208.0 (TID 458) 18/01/12 10:52:55 INFO ShuffleBlockFetcherIterator: Getting 1 non-empty blocks out of 5 blocks 18/01/12 10:52:55 INFO ShuffleBlockFetcherIterator: Started 0 remote fetches in 0 ms 18/01/12 10:52:55 INFO Executor: Finished task 2.0 in stage 208.0 (TID 458). 1028 bytes result sent to driver 18/01/12 10:52:55 INFO Executor: Running task 0.0 in stage 208.0 (TID 456) 18/01/12 10:52:55 INFO ShuffleBlockFetcherIterator: Getting 2 non-empty blocks out of 5 blocks 18/01/12 10:52:55 INFO ShuffleBlockFetcherIterator: Started 0 remote fetches in 0 ms 18/01/12 10:52:55 INFO TaskSetManager: Finished task 1.0 in stage 208.0 (TID 457) in 5 ms on localhost (1/3) 18/01/12 10:52:55 INFO Executor: Finished task 0.0 in stage 208.0 (TID 456). 1028 bytes result sent to driver 18/01/12 10:52:55 INFO TaskSetManager: Finished task 2.0 in stage 208.0 (TID 458) in 6 ms on localhost (2/3) 18/01/12 10:52:55 INFO TaskSetManager: Finished task 0.0 in stage 208.0 (TID 456) in 6 ms on localhost (3/3) 18/01/12 10:52:55 INFO TaskSchedulerImpl: Removed TaskSet 208.0, whose tasks have all completed, from pool 18/01/12 10:52:55 INFO DAGScheduler: ResultStage 208 (print at NetWorkWordCount.scala:29) finished in 0.008 s -------------------------------------------18/01/12 10:52:55 INFO DAGScheduler: Job 104 finished: print at NetWorkWordCount.scala:29, took 0.010832 s 18/01/12 10:52:55 INFO JobScheduler: Finished job streaming job 1515725575000 ms.0 from job set of time 1515725575000 ms 18/01/12 10:52:55 INFO JobScheduler: Total delay: 0.051 s for time 1515725575000 ms (execution: 0.042 s) Time: 1515725575000 ms ------------------------------------------- (A,2) (B,1) (F,1) (C,1) 18/01/12 10:52:55 INFO ShuffledRDD: Removing RDD 204 from persistence list 18/01/12 10:52:55 INFO MapPartitionsRDD: Removing RDD 203 from persistence list 18/01/12 10:52:55 INFO BlockManager: Removing RDD 204 18/01/12 10:52:55 INFO MapPartitionsRDD: Removing RDD 202 from persistence list 18/01/12 10:52:55 INFO BlockRDD: Removing RDD 201 from persistence list 18/01/12 10:52:55 INFO SocketInputDStream: Removing blocks of RDD BlockRDD[201] at socketTextStream at NetWorkWordCount.scala:26 of time 1515725575000 ms 18/01/12 10:52:55 INFO BlockManagerInfo: Removed input-0-1515725564800 on localhost:54439 in memory (size: 8.0 B, free: 467.6 MB) 18/01/12 10:52:55 INFO BlockManager: Removing RDD 202 18/01/12 10:52:55 INFO BlockManagerInfo: Removed input-0-1515725565800 on localhost:54439 in memory (size: 8.0 B, free: 467.6 MB) 18/01/12 10:52:55 INFO BlockManagerInfo: Removed input-0-1515725566800 on localhost:54439 in memory (size: 8.0 B, free: 467.6 MB) 18/01/12 10:52:55 INFO BlockManagerInfo: Removed input-0-1515725567800 on localhost:54439 in memory (size: 8.0 B, free: 467.6 MB) 18/01/12 10:52:55 INFO BlockManager: Removing RDD 203 18/01/12 10:52:55 INFO BlockManager: Removing RDD 201 18/01/12 10:52:55 INFO ReceivedBlockTracker: Deleting batches ArrayBuffer(1515725565000 ms) 18/01/12 10:52:55 INFO InputInfoTracker: remove old batch metadata: 1515725565000 ms 18/01/12 10:52:55 INFO BlockManagerInfo: Removed input-0-1515725568800 on localhost:54439 in memory (size: 8.0 B, free: 467.6 MB)