flink_初识01

1.flink wordcount

package flink

import org.apache.flink.api.scala._
import org.apache.flink.core.fs.FileSystem.WriteMode
import org.apache.flink.streaming.api.scala.{DataStream, StreamExecutionEnvironment}


object FlinkDemo {
  def main(args: Array[String]): Unit = {

    val env = StreamExecutionEnvironment.getExecutionEnvironment
    //source
    val text: DataStream[String] = env.readTextFile("file:///E:\\\\sparkproject\\\\src\\\\test\\\\data\\\\flink.txt")
    //transaction
    val result = text
      .flatMap { x => x.split("\\s") }
      .map { x => (x, 1) }
      .keyBy(0)
      .sum(1)
    //sink
    result.setParallelism(1).print()
    result.writeAsText("file:///E:\\\\sparkproject\\\\src\\\\test\\\\data\\\\result.txt", WriteMode.OVERWRITE).setParallelism(1)
    env.execute("flink wordcount demo")
  }
}

pom文件

         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
    4.0.0

    com.cnsuning.sparkproject
    sparkcode
    1.0-SNAPSHOT


    
        UTF-8
        UTF-8
        2.11.8
        2.1.0.9
        2.11
    
    
        
        
            org.apache.hive
            hive-common
            1.2.2
        

        
            commons-logging
            commons-logging
            1.1.1
            jar
        

        
            org.apache.commons
            commons-lang3
            3.1
        
        
            org.slf4j
            slf4j-log4j12
            1.7.7
            runtime
        
        
            log4j
            log4j
            1.2.17
        

        
            org.apache.hadoop
            hadoop-common
            2.6.2
        
        
        
        
        
        
        
            org.apache.spark
            spark-core_2.11
            2.1.0
        
        
            org.apache.spark
            spark-streaming_2.11
            2.1.0
        

        
            org.apache.spark
            spark-streaming-kafka-0-8_2.11
            2.1.0
        
        
        
        
        
        

        
            org.apache.spark
            spark-sql_2.11
            2.1.0
        

        
            com.alibaba
            fastjson
            1.2.29
        
        
            org.apache.spark
            spark-hive_2.11
            2.1.0
            
        
        
        
            org.apache.flink
            flink-java
            1.6.1
        
        
            org.apache.flink
            flink-streaming-java_2.11
            1.6.1
        
        
        
            org.apache.flink
            flink-core
            1.6.1
        
        
        
            org.apache.flink
            flink-cep_2.11
            1.6.1
        

        
            org.apache.flink
            flink-clients_2.11
            1.6.1
        
        
            org.apache.flink
            flink-connector-wikiedits_2.11
            1.6.1
        
        
        
            org.apache.flink
            flink-streaming-scala_2.11
            1.6.1
        
        
        
        
            org.apache.flink
            flink-connector-kafka-0.11_2.11
            1.6.1
        
        
            org.apache.flink
            flink-scala_2.11
            1.6.1
            
        
        
            org.apache.flink
            flink-table_2.11
            1.6.1
            provided
        

        
        
        
        
        
        
        
        
        
        
        
        
        
        
        
        
        
        
        
        
        
        
        
            org.elasticsearch
            elasticsearch-spark-20_${spark.artifactId.version}
            6.7.1
        
        
    

    
        
            
                maven-assembly-plugin
                
                    
                        jar-with-dependencies
                    
                
            
            
                org.codehaus.mojo
                build-helper-maven-plugin
                1.8
                
                    
                        add-source
                        generate-sources
                        
                            add-source
                        
                        
                            
                                src/main/scala
                                src/test/scala
                            
                        
                    
                    
                        add-test-source
                        generate-sources
                        
                            add-test-source
                        
                        
                            
                                src/test/scala
                            
                        
                    
                
            
            
                org.apache.maven.plugins
                maven-compiler-plugin
                2.3.2
                
                    1.7
                    1.7
                    ${project.build.sourceEncoding}
                
            
            
                org.scala-tools
                maven-scala-plugin
                
                    
                        
                            compile
                            add-source
                            testCompile
                        
                    
                
                
                    2.11.8
                    src/main/scala
                    
                        -Xms64m
                        -Xmx1024m
                    
                
            
            
                org.apache.maven.plugins
                maven-release-plugin
                2.5.3
            
            
                org.apache.maven.plugins
                maven-deploy-plugin
                
                    false
                
            
            
                org.apache.maven.plugins
                maven-shade-plugin
                2.4.1
                
                    
                        package
                        
                            shade
                        
                        
                            
                                
                                    *:*
                                    
                                        META-INF/*.SF
                                        org.apache.hive
                                        META-INF/*.DSA
                                        META-INF/*.RSA
                                    
                                
                            
                            false
                        
                    
                
            
        
        
            
                src/main/resources
                true
            
            
                src/main/resources/${profiles.active}
            
        

        
        
            
                
                    org.eclipse.m2e
                    lifecycle-mapping
                    1.0.0
                    
                        
                            
                                
                                    
                                        org.codehaus.mojo
                                        build-helper-maven-plugin
                                        [1.8,)
                                        
                                            add-source
                                            add-test-source
                                        
                                    
                                    flink
                                        
                                    
                                

                                
                                    
                                        org.scala-tools
                                        maven-scala-plugin
                                        [1.8,)
                                        
                                            compile
                                            add-source
                                            testCompile
                                        
                                    
                                    
                                        
                                    
                                
                            
                        
                    
                
            
        
    

View Code

 

转载于:https://www.cnblogs.com/yin-fei/p/11163602.html