Spark任务跑完将结果以邮件的方式进行发送
工作中，一般是晚上提交任务，第二天等结果，等跑完再走等问题阻碍着我们的步伐，现开发提交完任务就可以第一时间知道，并可以发送给领导，很方便
以下代码为scala编写，主要应用于跑spark任务后的结果发送。
开始需要：构建maven项目：
添加依赖pom.xml:
com.typesafe.play
play-mailer_2.11
7.0.0
里面主要包含
- 生成邮件
- 创建发邮件者
- 创建附件并添加
- 快速发送简单信息等。
package utils
import java.io.File
import com.typesafe.config.ConfigFactory
import org.apache.spark.rdd.RDD
import play.api.libs.mailer._
/**
* @program: route
* @class_name TaskSendMail
* @author: rk
* @create: 2019-06-24 09:57
* @Description:
**/
object TaskSendMail {
/**
* 定义一个发邮件的人
* @param host STMP服务地址
* @param port STMP服务端口号
* @param user STMP服务用户邮箱
* @param password STMP服务邮箱密码
* @param timeout setSocketTomeout 默认: 60s
* @param connectionTimeout setSocketConnectionTimeout 默认：60s
* @return 返回一个可以发邮件的用户
*/
def createMailer(host:String, port: Int, user: String, password: String, timeout:Int = 10000, connectionTimeout:Int = 10000):SMTPMailer ={
// STMP服务SMTPConfiguration
val configuration = new SMTPConfiguration(
host, port, false, false, false,
Option(user), Option(password), false, timeout = Option(timeout),
connectionTimeout = Option(connectionTimeout), ConfigFactory.empty(), false
)
val mailer: SMTPMailer = new SMTPMailer(configuration)
mailer
}
/**
* 生成一封邮件
* @param subject 邮件主题
* @param from 邮件发送地址
* @param to 邮件接收地址
* @param bodyText 邮件内容
* @param bodyHtml 邮件的超文本内容
* @param charset 字符编码 默认： utf-8
* @param attachments 邮件的附件
* @return 一封邮件
*/
def createEmail(subject:String, from:String, to:Seq[String], bodyText:String = "ok", bodyHtml:String = "", charset:String = "utf-8", attachments:Seq[Attachment] = Seq.empty): Email = {
val email = Email(subject, from, to,
bodyText = Option[String](bodyText), bodyHtml = Option[String](bodyHtml),
charset= Option[String](charset),attachments = attachments
)
email
}
/**
* 生成一个附件
* @param name 附件的名字
* @param fileStr 以本地文件为附件相关参数
* @param rdd 以hdfs文件或rdd或df为附件相关参数
* @return
*/
def createAttachments(name: String, fileStr: String = "", rdd:RDD[String] = null): Attachment = {
var attachment: Attachment = null
if(fileStr.contains(":")){
val file: File = new File(fileStr)
attachment = AttachmentFile(name, file)
}else{
val data: Array[Byte] = rdd.collect().mkString("\n").getBytes()
// 根据文件类型选择MimeTypes对应的值
val mimetype = "text/plain"
attachment = AttachmentData(name, data, mimetype)
}
attachment
}
/**
* 主要针对日常简单结果的快速发送
* @param subject 邮件主题名字
* @param toStr 邮件的接收人，多名以,分割
* @param bodyText 邮件的内容
* @return 用户设备 <510109769.1.1561635225728@RAN>
*/
def dailyEmail(subject:String, toStr:String, bodyText: String):String={
val to = toStr.split(",").toList
// 阿里云企业 邮箱
val host = "smtp.mxhichina.com"
val port = 25
val user = "@.com"
val password = ""
val from = user
val mailer: SMTPMailer = TaskSendMail.createMailer(host, port, user, password)
val email: Email = TaskSendMail.createEmail(subject, from, to, bodyText = bodyText)
val userdev: String = mailer.send(email)
userdev
}
}
进行测试：
package email
import org.apache.spark.rdd.RDD
import org.apache.spark.sql.SparkSession
import play.api.libs.mailer.{Attachment, Email, SMTPMailer}
import utils.TaskSendMail
/**
* @program: route
* @class_name EmailTest
* @author: rk
* @create: 2019-06-24 11:10
* @Description:
**/
object EmailTest {
def main(args: Array[String]): Unit = {
val subject = "ais-scala发邮件"
val from = "@.com"
val to = Seq("@.com","@qq.com")
val bodyText = "发个邮件"
val bodyHtml =
"""
|
|
|发邮件成功啦
|
|
""".stripMargin
val charset = "utf-8"
// val emails: Email = email(subject, from, to)
// val emails: Email = email(subject, from, to, bodyText = bodyText, charset =charset)
// val emails: Email = email(subject, from, to, bodyHtml = bodyHtml)
val name1 = "附件1"
val file = "d:/loong/data/ais.csv"
val attachment1 = TaskSendMail.createAttachments(name1, file)
// val emails: Email = email(subject, from, to, attachments = Seq(attachment1))
val name2 = "附件2"
val session: SparkSession = SparkSession.builder().master("local[*]")
.appName(this.getClass.getSimpleName).getOrCreate()
val ais_rdd: RDD[String] = session.sparkContext.textFile("d:/loong/data/ais.csv")
val attavhment2: Attachment = TaskSendMail.createAttachments(name2, rdd = ais_rdd)
val emails: Email = TaskSendMail.createEmail(subject, from, to)
// val emails: Email = TaskSendMail.createEmail(subject, from, to, attachments = Seq(attachment1, attavhment2))
// val host = "smtp.qq.com"
//// val port = 587
//// val user = "@qq.com"
//// val password = "jsjvokandbvxbbch"
// 阿里云企业 邮箱
val host = "smtp.mxhichina.com"
val port = 25
val user = "@.com"
val password = ""
val timeout = Option(10000)
val connectionTimeout = Option(10000)
val mailerop: SMTPMailer = TaskSendMail.createMailer(host, port, user, password)
mailerop.send(emails)
println("发送成功！！！")
}
}
日常邮件发送测试：
package email
import utils.TaskSendMail
/**
* @program: route
* @class_name DailyEmail
* @author: rk
* @create: 2019-06-27 19:11
* @Description:
**/
object DailyEmail {
def main(args: Array[String]): Unit = {
val subject = "吃过饭了"
val to = "@qq.com"
val bodyText = "真的吃过了！！！"
TaskSendMail.dailyEmail(subject, to, bodyText)
println("发送成功！！！")
}
}
注：
- 不同的邮箱可能对应的端口不同，需要修改一下配置，其他的均可通用。
- 在打包到集群去运行时，需要将对应的依赖jar包加上，否则会报异常。不想打包依赖的话，可以查找我上传的指定jar包，直接解压放入到打包后的jar包中即可。