• 搭建一个IntelliJ的Spark项目


    之前发现创建一个新项目之后,无法添加scala class


    创建新项目
      选择maven项目,然后选择simple或者quickstart;
      进入项目后,在Project Structure里面,在global libraries面板中,删除已有的scala-sdk,然后再添加;
      然后再工程中添加scala文件夹,添加一个package,在package里面就可以添加scala文件了。

    Maven

      1 <?xml version="1.0" encoding="UTF-8"?>
      2 <project xmlns="http://maven.apache.org/POM/4.0.0"
      3          xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
      4          xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
      5     <modelVersion>4.0.0</modelVersion>
      6 
      7     <groupId>com.cmiot</groupId>
      8     <artifactId>parquetToHbase</artifactId>
      9     <packaging>jar</packaging>
     10     <version>3.2.1-diy-thinrow</version>
     11 
     12     <repositories>
     13         <repository>
     14             <id>cloudera</id>
     15             <url>https://repository.cloudera.com/cloudera/cloudera-repos</url>
     16         </repository>
     17         <repository>
     18             <id>nexus</id>
     19             <url>https://maven.atlassian.com/3rdparty/</url>
     20         </repository>
     21     </repositories>
     22 
     23     <dependencies>
     24         <dependency>
     25             <groupId>com.typesafe</groupId>
     26             <artifactId>config</artifactId>
     27             <version>1.2.1</version>
     28         </dependency>
     29 
     30 
     31         <dependency>
     32             <groupId>org.apache.spark</groupId>
     33             <artifactId>spark-core_${spark.tool.version}</artifactId>
     34             <version>${spark.version}</version>
     35             <!--scope>provided</scope-->
     36             <exclusions>
     37                 <exclusion>
     38                     <groupId>org.scala-lang</groupId>
     39                     <artifactId>scala-library</artifactId>
     40                 </exclusion>
     41                 <exclusion>
     42                     <groupId>org.scala-lang</groupId>
     43                     <artifactId>scala-lang</artifactId>
     44                 </exclusion>
     45                 <exclusion>
     46                     <groupId>org.scala-lang</groupId>
     47                     <artifactId>scalap</artifactId>
     48                 </exclusion>
     49                 <exclusion>
     50                     <artifactId>slf4j-log4j12</artifactId>
     51                     <groupId>org.slf4j</groupId>
     52                 </exclusion>
     53             </exclusions>
     54         </dependency>
     55         <dependency>
     56             <groupId>org.apache.spark</groupId>
     57             <artifactId>spark-sql_${spark.tool.version}</artifactId>
     58             <version>${spark.version}</version>
     59             <!--scope>provided</scope-->
     60         </dependency>
     61         <!--<dependency>-->
     62             <!--<groupId>com.oracle</groupId>-->
     63             <!--<artifactId>ojdbc6</artifactId>-->
     64             <!--<version>11.2.0.4.0-atlassian-hosted</version>-->
     65         <!--</dependency>
     66         <dependency>
     67             <groupId>oracle</groupId>
     68             <artifactId>ojdbc</artifactId>
     69             <version>8</version>
     70         </dependency>-->
     71         <dependency>
     72             <groupId>com.github.nscala-time</groupId>
     73             <artifactId>nscala-time_${scala.version.tools}</artifactId>
     74             <version>2.0.0</version>
     75         </dependency>
     76         <dependency>
     77             <groupId>org.scala-lang</groupId>
     78             <artifactId>scala-library</artifactId>
     79             <version>${scala.version}</version>
     80         </dependency>
     81 
     82         <dependency>
     83             <groupId>org.scala-lang</groupId>
     84             <artifactId>scala-reflect</artifactId>
     85             <version>${scala.version}</version>
     86         </dependency>
     87 
     88         <dependency>
     89             <groupId>org.scala-lang</groupId>
     90             <artifactId>scala-compiler</artifactId>
     91             <version>${scala.version}</version>
     92         </dependency>
     93         <dependency>
     94             <groupId>org.scalatest</groupId>
     95             <artifactId>scalatest_${scala.version.tools}</artifactId>
     96             <version>${scala.test.version}</version>
     97             <scope>test</scope>
     98             <exclusions>
     99                 <exclusion>
    100                     <groupId>org.scala-lang</groupId>
    101                     <artifactId>scala-library</artifactId>
    102                 </exclusion>
    103             </exclusions>
    104         </dependency>
    105         <dependency>
    106             <groupId>org.scalacheck</groupId>
    107             <artifactId>scalacheck_${scala.version.tools}</artifactId>
    108             <version>${scala.check.version}</version>
    109             <scope>test</scope>
    110             <exclusions>
    111                 <exclusion>
    112                     <groupId>org.scala-lang</groupId>
    113                     <artifactId>scala-library</artifactId>
    114                 </exclusion>
    115             </exclusions>
    116         </dependency>
    117         <dependency>
    118             <groupId>junit</groupId>
    119             <artifactId>junit</artifactId>
    120             <version>${junit.version}</version>
    121             <scope>test</scope>
    122         </dependency>
    123         <dependency>
    124             <groupId>org.slf4j</groupId>
    125             <artifactId>slf4j-log4j12</artifactId>
    126             <version>${slf4j.log4j12.version}</version>
    127             <!--<scope>test</scope>-->
    128         </dependency>
    129         <dependency>
    130             <groupId>org.alluxio</groupId>
    131             <artifactId>alluxio-core-client</artifactId>
    132             <version>${alluxio.client.version}</version>
    133         </dependency>
    134         <dependency>
    135             <groupId>com.databricks</groupId>
    136             <artifactId>spark-xml_${spark.tool.version}</artifactId>
    137             <version>${spark.xml.version}</version>
    138         </dependency>
    139         <dependency>
    140             <groupId>org.apache.hbase</groupId>
    141             <artifactId>hbase-client</artifactId>
    142             <version>${hbase.version}</version>
    143             <exclusions>
    144                 <exclusion>
    145                     <groupId>javax.servlet</groupId>
    146                     <artifactId>*</artifactId>
    147                 </exclusion>
    148             </exclusions>
    149         </dependency>
    150         <dependency>
    151             <groupId>org.apache.hbase</groupId>
    152             <artifactId>hbase-server</artifactId>
    153             <version>${hbase.version}</version>
    154             <exclusions>
    155                 <exclusion>
    156                     <groupId>javax.servlet</groupId>
    157                     <artifactId>*</artifactId>
    158                 </exclusion>
    159             </exclusions>
    160         </dependency>
    161         <!--
    162         <dependency>
    163             <groupId>org.apache.hbase</groupId>
    164             <artifactId>hbase-spark</artifactId>
    165             <version>1.2.0-cdh5.10.2</version>
    166         </dependency>
    167         -->
    168     </dependencies>
    169 
    170     <properties>
    171         <hbase.version>1.2.0-cdh5.11.2</hbase.version>
    172         <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
    173         <project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
    174         <maven.compiler.encoding>UTF-8</maven.compiler.encoding>
    175         <maven.surefire.version>2.7</maven.surefire.version>
    176         <maven.compiler.version>3.1</maven.compiler.version>
    177         <maven.assembly.version>2.4</maven.assembly.version>
    178         <java.version>1.8</java.version>
    179         <scala.version>2.11.8</scala.version>
    180         <scala.version.tools>2.11</scala.version.tools>
    181         <scala.maven.version>3.2.0</scala.maven.version>
    182         <scala.check.version>1.11.4</scala.check.version>
    183         <scala.test.version>3.0.1</scala.test.version>
    184         <scala.test.maven.version>1.0</scala.test.maven.version>
    185         <junit.version>4.11</junit.version>
    186         <spark.version>2.1.0.cloudera2</spark.version>
    187         <spark.tool.version>2.11</spark.tool.version>
    188         <spark.xml.version>0.4.1</spark.xml.version>
    189         <alluxio.client.version>1.0.0</alluxio.client.version>
    190         <cobertura.version>2.7</cobertura.version>
    191         <slf4j.log4j12.version>1.7.10</slf4j.log4j12.version>
    192         <test.arg.line>-Xms1024m -Xmx1024m -XX:PermSize=512m -XX:MaxPermSize=1024m</test.arg.line>
    193     </properties>
    194 
    195     <profiles>
    196         <profile>
    197             <id>dev</id>
    198             <activation>
    199                 <activeByDefault>true</activeByDefault>
    200             </activation>
    201             <properties>
    202                 <filters.env>dev</filters.env>
    203             </properties>
    204         </profile>
    205 
    206         <profile>
    207             <id>test</id>
    208 
    209             <properties>
    210                 <filters.env>test</filters.env>
    211             </properties>
    212         </profile>
    213 
    214         <profile>
    215             <id>prod</id>
    216             <properties>
    217                 <filters.env>prod</filters.env>
    218             </properties>
    219         </profile>
    220     </profiles>
    221 
    222     <build>
    223         <resources>
    224             <resource>
    225                 <directory>${project.basedir}/src/main/resources</directory>
    226                 <filtering>true</filtering>
    227             </resource>
    228         </resources>
    229         <filters>
    230             <filter>${project.basedir}/filters/${filters.env}/conf.properties</filter>
    231         </filters>
    232         <plugins>
    233 
    234             <plugin>
    235                 <groupId>net.alchim31.maven</groupId>
    236                 <artifactId>scala-maven-plugin</artifactId>
    237                 <version>${scala.maven.version}</version>
    238                 <executions>
    239                     <execution>
    240                         <id>scala-compile</id>
    241                         <phase>process-resources</phase>
    242                         <goals>
    243                             <goal>add-source</goal>
    244                             <goal>compile</goal>
    245                         </goals>
    246                     </execution>
    247                     <execution>
    248                         <id>scala-test-compile</id>
    249                         <phase>process-test-resources</phase>
    250                         <goals>
    251                             <goal>testCompile</goal>
    252                         </goals>
    253                     </execution>
    254                 </executions>
    255             </plugin>
    256             <!---->
    257             <plugin>
    258                 <groupId>org.apache.maven.plugins</groupId>
    259                 <artifactId>maven-compiler-plugin</artifactId>
    260                 <version>${maven.compiler.version}</version>
    261                 <configuration>
    262                     <source>${java.version}</source>
    263                     <target>${java.version}</target>
    264                 </configuration>
    265             </plugin>
    266             <!--
    267             <plugin>
    268                 <groupId>org.apache.maven.plugins</groupId>
    269                 <artifactId>maven-assembly-plugin</artifactId>
    270                 <version>${maven.assembly.version}</version>
    271                 <configuration>
    272                     <descriptorRefs>
    273                         <descriptorRef>jar-with-dependencies</descriptorRef>
    274                     </descriptorRefs>
    275                     <archive>
    276                         <manifest>
    277                             <mainClass>com.cmiot.StatsApp</mainClass>
    278                         </manifest>
    279                     </archive>
    280                 </configuration>
    281                 <executions>
    282                     <execution>
    283                         <phase>package</phase>
    284                         <goals>
    285                             <goal>single</goal>
    286                         </goals>
    287                     </execution>
    288                 </executions>
    289             </plugin>
    290             <plugin>
    291                 <groupId>org.apache.maven.plugins</groupId>
    292                 <artifactId>maven-surefire-plugin</artifactId>
    293                 <version>${maven.surefire.version}</version>
    294                 <configuration>
    295                     <argLine>${test.arg.line}</argLine>
    296                 </configuration>
    297             </plugin>
    298             <plugin>
    299                 <groupId>org.scalatest</groupId>
    300                 <artifactId>scalatest-maven-plugin</artifactId>
    301                 <version>${scala.test.maven.version}</version>
    302                 <configuration>
    303                     <skipTests>false</skipTests>
    304                     <reportsDirectory>${project.build.directory}/test-reports</reportsDirectory>
    305                     <junitxml>.</junitxml>
    306                     <filereports>WDF TestSuite.txt</filereports>
    307                     <argLine>${test.arg.line}</argLine>
    308                 </configuration>
    309                 <executions>
    310                     <execution>
    311                         <id>test</id>
    312                         <goals>
    313                             <goal>test</goal>
    314                         </goals>
    315                     </execution>
    316                 </executions>
    317             </plugin>
    318             <plugin>
    319                 <groupId>org.codehaus.mojo</groupId>
    320                 <artifactId>cobertura-maven-plugin</artifactId>
    321                 <version>${cobertura.version}</version>
    322                 <configuration>
    323                     <formats>
    324                         <format>xml</format>
    325                         <format>html</format>
    326                     </formats>
    327                     <check/>
    328                     <instrumentation>
    329                         <ignores>
    330                             <ignore>com.cmiot.StatsApp</ignore>
    331                             <ignore>com.cmiot.app</ignore>
    332                         </ignores>
    333                         <excludes>
    334                             <exclude>com/cmiot/StatsApp*</exclude>
    335                             <exclude>com/cmiot/app/Stats*</exclude>
    336                         </excludes>
    337                     </instrumentation>
    338                 </configuration>
    339                 <executions>
    340                     <execution>
    341                         <phase>test</phase>
    342                         <goals>
    343                             <goal>cobertura</goal>
    344                         </goals>
    345                     </execution>
    346                 </executions>
    347             </plugin>
    348             -->
    349         </plugins>
    350     </build>
    351 </project>
  • 相关阅读:
    mapent
    java map使用
    set集合HashSet
    List集合的子类ArrayList和LinkedList
    List集合和iterator并发异常处理
    java泛型接口
    java集合---迭代器iterator
    java集合(类似python的列表)
    安装python3.7和PyCharm专业版
    python类的__slots__属性、__del__属性、上下文(__enter__和__exit__)、
  • 原文地址:https://www.cnblogs.com/xiashiwendao/p/8646830.html
Copyright © 2020-2023  润新知