【急速入门】Elastic-Job分布式定时任务框架
前言
急速入门，分布式框架O(∩_∩)O哈哈~
Elastic-Job
分布式调度解决方案，使用jar包提供协调服务
https://mvnrepository.com/search?q=elastic-job-lite-core
- 需要注意的是，我们在使用Elastic-Job时需要整合ZooKeeper集群。
本地使用Windows版本ZooKeeper，单机版
https://zookeeper.apache.org/
- 单机启动
- 修改配置文件
- 双击启动
- (*❦ω❦)成功
Elastic-Job 分片
一个任务拆分成多个独立的任务项，每个服务获得一个或几个分片项。
- 【案例：】
如果遍历数据库某张表的作业，供两台服务器。
每台服务器执任务的50%，即服务器A执行ID尾号为奇数的数据，服务器B执行ID尾号为偶数的数据。
高可用
分片总数设置为1，多于1台服务器执行作业，1主n从。
执行服务器崩溃后，等待的服务器将启动，执行下次任务。
Demo演示
Simple作业详解
意为定时任务的简单实现，只需实现execute方法。提供了弹性的扩容和分片功能。
1.搭建项目
2.引入maven依赖
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<groupId>com.zcw</groupId>
<artifactId>zcw-java-simple-job</artifactId>
<version>1.0-SNAPSHOT</version>
<name>zcw-java-simple-job</name>
<!-- FIXME change it to the project's website -->
<url>http://www.example.com</url>
<properties>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<maven.compiler.source>8</maven.compiler.source>
<maven.compiler.target>8</maven.compiler.target>
</properties>
<dependencies>
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
<version>4.11</version>
<scope>test</scope>
</dependency>
<!--Elastic-job-lite-core-->
<dependency>
<groupId>com.dangdang</groupId>
<artifactId>elastic-job-lite-core</artifactId>
<version>2.1.5</version>
</dependency>
</dependencies>
</project>
3.实现SimpleJob接口，实现execute方法
package com.zcw.job;
import com.dangdang.ddframe.job.api.ShardingContext;
import com.dangdang.ddframe.job.api.simple.SimpleJob;
/**
* @ClassName : MySimpleJob
* @Description :
* @Author : Zhaocunwei
* @Date: 2020-06-04 13:45
*/
public class MySimpleJob implements SimpleJob {
@Override
public void execute(ShardingContext shardingContext) {
System.out.println("我是当前分片项"+shardingContext.getShardingItem()+
",总分片项："+shardingContext.getShardingTotalCount());
}
}
4.配置作业
- 定义作业核心配置（作业名称，定时策略，分片总数）
- 定义作业类型（Simple,Dataflow,Scripe,实现类的全包名）
- 定义Lite作业根配置（overwrite属性的重要性）
5.配置zookeeper的链接方式（单机版）
package com.zcw;
import com.dangdang.ddframe.job.config.JobCoreConfiguration;
import com.dangdang.ddframe.job.config.JobTypeConfiguration;
import com.dangdang.ddframe.job.config.simple.SimpleJobConfiguration;
import com.dangdang.ddframe.job.lite.api.JobScheduler;
import com.dangdang.ddframe.job.lite.config.LiteJobConfiguration;
import com.dangdang.ddframe.job.lite.internal.schedule.JobScheduleController;
import com.dangdang.ddframe.job.reg.base.CoordinatorRegistryCenter;
import com.dangdang.ddframe.job.reg.zookeeper.ZookeeperConfiguration;
import com.dangdang.ddframe.job.reg.zookeeper.ZookeeperRegistryCenter;
import com.zcw.job.MySimpleJob;
/**
* Hello world!
*
*/
public class App
{
public static void main( String[] args ) {
System.out.println( "Hello World!" );
new JobScheduler(zkCenter(),configuration()).init();
}
/**
* 注册中心
* @return
*/
public static CoordinatorRegistryCenter zkCenter(){
ZookeeperConfiguration zookeeperConfiguration = new ZookeeperConfiguration("localhost:2181",
"zcw-simple-job");
CoordinatorRegistryCenter coordinatorRegistryCenter = new ZookeeperRegistryCenter(zookeeperConfiguration);
//注册中心初始化
coordinatorRegistryCenter.init();
return coordinatorRegistryCenter;
}
/**
* Job配置
* @return
*/
public static LiteJobConfiguration configuration(){
//job核心配置
JobCoreConfiguration jobCoreConfiguration = JobCoreConfiguration
.newBuilder("mySimpleJob","0/10 * * * * ?",2)
.build();
//job类型配置
JobTypeConfiguration jobTypeConfiguration = new SimpleJobConfiguration(jobCoreConfiguration,
MySimpleJob.class.getCanonicalName());
//job根的配置（LiteJobConfiguration）
LiteJobConfiguration liteJobConfiguration = LiteJobConfiguration
.newBuilder(jobTypeConfiguration)
//启动时都要覆盖掉，注册中心的数据
.overwrite(true)
.build();
return liteJobConfiguration;
}
}
- 运行
6.复制代码， 修改端口，模拟两台服务器
dataflow流式任务
-
分为数据抓取(fetchData)和数据处理（processData）
-
流程
1.定时任务根据规则触发
2.抓取数据
3.处理数据
4.处理完数据后，再次抓取
5.若数据存在，继续处理，若不存在，则本次任务结束
6.等待任务规则，下次触发
- 应用场景：
不间歇的数据处理（比如：第三方订单的抓取） - Demo练习
【实现DataflowJob接口】
注意：DataFlowJob泛型
泛型规定了抓取数据的返回类型
1.创建实体类
package com.zcw.model;
/**
* @ClassName : Order
* @Description :
* @Author : Zhaocunwei
* @Date: 2020-06-04 17:37
*/
public class Order {
private Integer orderId;
//0 未处理；1,已处理
private Integer status;
public Integer getOrderId() {
return orderId;
}
public void setOrderId(Integer orderId) {
this.orderId = orderId;
}
public Integer getStatus() {
return status;
}
public void setStatus(Integer status) {
this.status = status;
}
@Override
public String toString() {
return "Order{" +
"orderId=" + orderId +
", status=" + status +
'}';
}
}
2.编写接口
package com.zcw.job;
import com.dangdang.ddframe.job.api.ShardingContext;
import com.dangdang.ddframe.job.api.dataflow.DataflowJob;
import com.sun.org.apache.xpath.internal.operations.Or;
import com.zcw.model.Order;
import java.time.LocalTime;
import java.util.ArrayList;
import java.util.List;
import java.util.stream.Collectors;
/**
* @ClassName : MyDataflowJob
* @Description :
* @Author : Zhaocunwei
* @Date: 2020-06-04 17:36
*/
public class MyDataflowJob implements DataflowJob<Order> {
private List<Order> orders = new ArrayList<>();
{
for(int i=0;i<100;i++){
Order order = new Order();
order.setOrderId(i+1);
order.setStatus(0);
orders.add(order);
}
}
@Override
public List<Order> fetchData(ShardingContext shardingContext) {
//订单号 % 分片总数 == 当前分片项
List<Order> orderList = orders.stream().filter(o->o.getStatus()==0)
.filter(o->o.getOrderId()%shardingContext
.getShardingTotalCount() ==shardingContext.getShardingItem())
.collect(Collectors.toList());
List<Order> subList=null;
if(orderList!=null && orderList.size()>0){
subList = orderList.subList(0,10);
}
try {
Thread.sleep(3000);
} catch (InterruptedException e) {
e.printStackTrace();
}
LocalTime time = LocalTime.now();
System.out.println(time+",我是分片项："+shardingContext.getShardingItem()+"，" +
"我抓取的数据是："+subList);
return subList;
}
@Override
public void processData(ShardingContext shardingContext, List<Order> list) {
list.forEach(o->o.setStatus(1));
try {
Thread.sleep(5000);
} catch (InterruptedException e) {
e.printStackTrace();
}
LocalTime time = LocalTime.now();
System.out.println(time+",我是分片项："+shardingContext.getShardingItem()+
",我正在处理数据");
}
}
- 修改启动类
package com.zcw;
import com.dangdang.ddframe.job.config.JobCoreConfiguration;
import com.dangdang.ddframe.job.config.JobTypeConfiguration;
import com.dangdang.ddframe.job.config.dataflow.DataflowJobConfiguration;
import com.dangdang.ddframe.job.config.simple.SimpleJobConfiguration;
import com.dangdang.ddframe.job.lite.api.JobScheduler;
import com.dangdang.ddframe.job.lite.config.LiteJobConfiguration;
import com.dangdang.ddframe.job.lite.internal.schedule.JobScheduleController;
import com.dangdang.ddframe.job.reg.base.CoordinatorRegistryCenter;
import com.dangdang.ddframe.job.reg.zookeeper.ZookeeperConfiguration;
import com.dangdang.ddframe.job.reg.zookeeper.ZookeeperRegistryCenter;
import com.zcw.job.MyDataflowJob;
import com.zcw.job.MySimpleJob;
/**
* Hello world!
*
*/
public class App
{
public static void main( String[] args ) {
System.out.println( "Hello World!" );
// new JobScheduler(zkCenter(),configuration()).init();
new JobScheduler(zkCenter(),configurationDataflow()).init();
}
/**
* 注册中心
* @return
*/
public static CoordinatorRegistryCenter zkCenter(){
ZookeeperConfiguration zookeeperConfiguration = new ZookeeperConfiguration("localhost:2181",
"zcw-simple-job");
CoordinatorRegistryCenter coordinatorRegistryCenter = new ZookeeperRegistryCenter(zookeeperConfiguration);
//注册中心初始化
coordinatorRegistryCenter.init();
return coordinatorRegistryCenter;
}
/**
* Job配置
* @return
*/
public static LiteJobConfiguration configuration(){
//job核心配置
JobCoreConfiguration jobCoreConfiguration = JobCoreConfiguration
.newBuilder("mySimpleJob","0/10 * * * * ?",2)
.build();
//job类型配置
JobTypeConfiguration jobTypeConfiguration = new SimpleJobConfiguration(jobCoreConfiguration,
MySimpleJob.class.getCanonicalName());
//job根的配置（LiteJobConfiguration）
LiteJobConfiguration liteJobConfiguration = LiteJobConfiguration
.newBuilder(jobTypeConfiguration)
//启动时都要覆盖掉，注册中心的数据
.overwrite(true)
.build();
return liteJobConfiguration;
}
/**
* Job配置
* @return
*/
public static LiteJobConfiguration configurationDataflow(){
//job核心配置
JobCoreConfiguration jobCoreConfiguration = JobCoreConfiguration
.newBuilder("myDataFlowJob","0/10 * * * * ?",2)
.build();
//job类型配置
JobTypeConfiguration jobTypeConfiguration = new DataflowJobConfiguration(
jobCoreConfiguration,
MyDataflowJob.class.getCanonicalName(),true);
//job根的配置（LiteJobConfiguration）
LiteJobConfiguration liteJobConfiguration = LiteJobConfiguration
.newBuilder(jobTypeConfiguration)
//启动时都要覆盖掉，注册中心的数据
.overwrite(true)
.build();
return liteJobConfiguration;
}
}
- 测试：
编写脚本Script
echo 我是cmd脚本，我的作业信息是：%1
- 在我们APP类里面定义方法
public static LiteJobConfiguration configurationScript(){
//job核心配置
JobCoreConfiguration jobCoreConfiguration = JobCoreConfiguration
.newBuilder("myScriptJob","0/10 * * * * ?",2)
.build();
//job类型配置
JobTypeConfiguration jobTypeConfiguration = new ScriptJobConfiguration(
jobCoreConfiguration,
"F:/test/test.cmd");
//job根的配置（LiteJobConfiguration）
LiteJobConfiguration liteJobConfiguration = LiteJobConfiguration
.newBuilder(jobTypeConfiguration)
//启动时都要覆盖掉，注册中心的数据
.overwrite(true)
.build();
return liteJobConfiguration;
}
- 运行
- 修改文件
