先上工具类
package com.zuodou.utlis;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Component;
import javax.xml.crypto.Data;
import java.io.*;
import java.net.HttpURLConnection;
import java.net.URL;
import java.net.URLEncoder;
import java.security.MessageDigest;
import java.util.*;
import java.util.Map.Entry;
import java.util.zip.GZIPInputStream;
@Component
public class ApiUtils {
private static final String SIGN_METHOD_MD5 = "md5";
private static final String CHARSET_UTF8 = "utf-8";
private static final String CONTENT_ENCODING_GZIP = "gzip";
// TOP服务地址,正式环境需要设置为https://openapi.jushuitan.com
// @Value("${erp.serverUrl}")
// public String serverUrl;
//
// @Value("${erp.appKey}")
// public String appKey; // 可替换为您的应用的appKey
//
// @Value("${erp.appSecret}")
// public String appSecret; // 可替换为您的应用的appSecret
//
// @Value("${erp.accessToken}")
// public String accessToken; // 必须替换为授权得到的真实有效accessToken
public static String serverUrl;
public static String appKey;
public static String appSecret;
public static String accessToken;
@Value("${erp.serverUrl}")
public void setServerUrl(String serverUrl) {
ApiUtils.serverUrl = serverUrl;
}
@Value("${erp.appKey}")
public void setAppKey(String appKey) {
ApiUtils.appKey = appKey;
}
@Value("${erp.appSecret}")
public void setAppSecret(String appSecret) {
ApiUtils.appSecret = appSecret;
}
@Value("${erp.accessToken}")
public void setAccessToken(String accessToken) {
ApiUtils.accessToken = accessToken;
}
/**
*
* @param fangfa
* @return
* @throws IOException
*/
public static String getSellerItem(String fangfa, String biz) throws IOException {
Map params = new HashMap();
// 公共参数
params.put("app_key",appKey);
params.put("access_token", accessToken);
params.put("timestamp", String.valueOf(System.currentTimeMillis() / 1000));
params.put("version", "2");
params.put("charset", "utf-8");
// 业务参数
params.put("biz", biz);
// 签名参数
params.put("sign", signTopRequest(params, appSecret, SIGN_METHOD_MD5));
// 调用API
return callApi(new URL(serverUrl+fangfa), params);
}
/**
* 对TOP请求进行签名。
*/
private static String signTopRequest(Map params, String secret, String signMethod) throws IOException {
// 第一步:检查参数是否已经排序
String[] keys = params.keySet().toArray(new String[0]);
Arrays.sort(keys);
// 第二步:把所有参数名和参数值串在一起
StringBuilder query = new StringBuilder();
if (SIGN_METHOD_MD5.equals(signMethod)) {
query.append(secret);
}
for (String key : keys) {
String value = params.get(key);
if (isNotEmpty(key) && isNotEmpty(value)) {
query.append(key).append(value);
}
}
return createSign(query.toString());
}
/**
* 生成新sign
*
* @param str 字符串
* @return String
*/
private static String createSign(String str) {
if (str == null || str.length() == 0) {
return null;
}
char[] hexDigits = {'0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'a', 'b', 'c', 'd', 'e', 'f'};
try {
MessageDigest mdTemp = MessageDigest.getInstance(SIGN_METHOD_MD5);
mdTemp.update(str.getBytes("UTF-8"));
byte[] md = mdTemp.digest();
int j = md.length;
char[] buf = new char[j * 2];
int k = 0;
int i = 0;
while (i < j) {
byte byte0 = md[i];
buf[k++] = hexDigits[byte0 >>> 4 & 0xf];
buf[k++] = hexDigits[byte0 & 0xf];
i++;
}
return new String(buf);
} catch (Exception e) {
return null;
}
}
private static String callApi(URL url, Map params) throws IOException {
String query = buildQuery(params, CHARSET_UTF8);
byte[] content = {};
if (query != null) {
content = query.getBytes(CHARSET_UTF8);
}
HttpURLConnection conn = null;
OutputStream out = null;
String rsp = null;
try {
conn = (HttpURLConnection) url.openConnection();
conn.setRequestMethod("POST");
conn.setDoInput(true);
conn.setDoOutput(true);
conn.setRequestProperty("Host", url.getHost());
conn.setRequestProperty("Content-Type", "application/x-www-form-urlencoded;charset=" + CHARSET_UTF8);
out = conn.getOutputStream();
out.write(content);
rsp = getResponseAsString(conn);
} finally {
if (out != null) {
out.close();
}
if (conn != null) {
conn.disconnect();
}
}
return rsp;
}
private static String buildQuery(Map params, String charset) throws IOException {
if (params == null || params.isEmpty()) {
return null;
}
StringBuilder query = new StringBuilder();
Set> entries = params.entrySet();
boolean hasParam = false;
for (Entry entry : entries) {
String name = entry.getKey();
String value = entry.getValue();
// 忽略参数名或参数值为空的参数
if (isNotEmpty(name) && isNotEmpty(value)) {
if (hasParam) {
query.append("&");
} else {
hasParam = true;
}
query.append(name).append("=").append(URLEncoder.encode(value, charset));
}
}
return query.toString();
}
private static String getResponseAsString(HttpURLConnection conn) throws IOException {
String charset = getResponseCharset(conn.getContentType());
if (conn.getResponseCode() < 400) {
String contentEncoding = conn.getContentEncoding();
if (CONTENT_ENCODING_GZIP.equalsIgnoreCase(contentEncoding)) {
return getStreamAsString(new GZIPInputStream(conn.getInputStream()), charset);
} else {
return getStreamAsString(conn.getInputStream(), charset);
}
} else {// Client Error 4xx and Server Error 5xx
throw new IOException(conn.getResponseCode() + " " + conn.getResponseMessage());
}
}
private static String getStreamAsString(InputStream stream, String charset) throws IOException {
try {
Reader reader = new InputStreamReader(stream, charset);
StringBuilder response = new StringBuilder();
final char[] buff = new char[1024];
int read = 0;
while ((read = reader.read(buff)) > 0) {
response.append(buff, 0, read);
}
return response.toString();
} finally {
if (stream != null) {
stream.close();
}
}
}
private static String getResponseCharset(String ctype) {
String charset = CHARSET_UTF8;
if (isNotEmpty(ctype)) {
String[] params = ctype.split(";");
for (String param : params) {
param = param.trim();
if (param.startsWith("charset")) {
String[] pair = param.split("=", 2);
if (pair.length == 2) {
if (isNotEmpty(pair[1])) {
charset = pair[1].trim();
}
}
break;
}
}
}
return charset;
}
private static boolean isNotEmpty(String value) {
int strLen;
if (value == null || (strLen = value.length()) == 0) {
return false;
}
for (int i = 0; i < strLen; i++) {
if ((Character.isWhitespace(value.charAt(i)) == false)) {
return true;
}
}
return false;
}
public static void main(String[] args) {
int pageNo = 1;
int pageSize = 100;
boolean hasMoreData = true;
List allData = new ArrayList<>();
while (hasMoreData) {
List currentPageData = callThirdPartyApi(pageNo, pageSize); // 调用第三方API获取当前页数据
allData.addAll(currentPageData); // 将当前页数据加入总数据集合
if (currentPageData.size() < pageSize) {
hasMoreData = false; // 当前页数据不足pageSize,表示已获取所有数据
} else {
pageNo++; // 继续获取下一页数据
}
}
// 所有数据已获取完成
System.out.println("Total data count: " + allData.size());
// 进一步处理allData集合...
}
// 调用第三方API获取指定页数据的示例方法
private static List callThirdPartyApi(int pageNo, int pageSize) {
// 调用第三方API获取指定页的数据
// 返回数据列表
return new ArrayList<>(); // 这里仅作示例,实际应调用API并返回数据
}
}
需求说明,需要把聚水潭所有的售后数据拉取到自研平台进行进一步操作。
调用接口的限制:分页,每页数量50,请求限制,一秒钟不能超过5次,一分钟不能超过一百次。
注意代码待完善:accessToken是有过期时间的,但可以在主账号设置,如果超过限制我不会进行记录会漏掉这条数据。有一些注入的地方可以删掉。我定义了一个erpDatas和erpItems类来接收数据,由于返回值是下划线的,而我项目架构是驼峰命名,会导致映射值失败,最下面是处理方法(可以不用的自行删除)。
package com.zuodou.job;
import com.zuodou.mapper.ZuodouTaskDatasMapper;
import com.zuodou.utlis.DataUtil;
import com.zuodou.utlis.TimeUtils;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import org.quartz.DisallowConcurrentExecution;
import org.quartz.Job;
import org.quartz.JobExecutionContext;
import org.quartz.JobExecutionException;
import org.springframework.beans.factory.annotation.Autowired;
import java.util.*;
/**
* 售后
*/
@Slf4j
//禁止并发执行
@DisallowConcurrentExecution
public class TaskDatas implements Job {
@Autowired
private ZuodouTaskDatasMapper zuodouTaskDatasMapper;
@Autowired
private threadService threadService;
/**
* 若参数变量名修改 QuartzJobController中也需对应修改
*/
private String parameter;
public void setParameter(String parameter) {
this.parameter = parameter;
}
private String parameterstartime;
public void setParameterstartime(String parameterstartime) {
this.parameterstartime = parameterstartime;
}
/**
* 拿售后数据
* @param jobExecutionContext
* @throws JobExecutionException
*
* token过期
* 1.请求状态
* 2.时间间隔七天
* 3.分页数据
* /open/refund/single/query
*
*/
@Override
public void execute(JobExecutionContext jobExecutionContext) throws JobExecutionException {
try {
//1.拿到数据库中最大时间 如果没有,默认多少,如果有生成一个七天时间lsit
//1.1 先判断数据库时间和当前时间是否相差七天,相差七天就减,没有相差就用当前时间当作结束时间
//2.在每个七天里面分页拿取数据,data_count 总数,page_count 有多少页
//3.把分页数据整合到list中
// for (String shopId : Shop_id) {
String start = zuodouTaskDatasMapper.maxOrdersDate(null);
String end = TimeUtils.getStringDate();//比当前时间少一分钟,防止数据重复
if (StringUtils.isBlank(start)) {
start = this.parameterstartime;
}
log.info("聚水潭售后数据拉取开始-------"+TimeUtils.date2Str(new Date()));
Date CalendarstartTime = TimeUtils.DateSwitch(start);
Date CalendarendTime = TimeUtils.DateSwitch(end);
Calendar startCal = Calendar.getInstance();
startCal.setTime(CalendarstartTime) ;
Calendar endCal = Calendar.getInstance();
endCal.setTime(CalendarendTime);
String startTime =start;
String endTime =null;
while (!start.equals(end)) { //当开始时间大于或者等于结束时间
Integer day = DataUtil.countDaynew(start,end );
if (day>7) {
startCal.add(Calendar.DAY_OF_MONTH, 7);
endTime = TimeUtils.date2Str(startCal.getTime());
Integer daya = DataUtil.countDaynew(endTime, end); //判断结束时间加上七天跟当前时间的间隔
if (daya <1) { //如果加上七天大于当前时间或者等于
endTime=end;
}
}
if (StringUtils.isBlank(endTime)){
endTime=end;
}
System.out.println(startTime+"-----3---"+endTime);
String finalStartTime = startTime;
String finalEndTime = endTime;
threadService.apithread(finalStartTime, finalEndTime);
Integer days = DataUtil.countDaynew(startTime,endTime );
Integer daya = DataUtil.countDaynew(endTime, end);
if (days==7){ //超过七天
startTime = endTime;
} else if (days>0&&daya==0){
startTime = endTime;//累计七天的结束时间
endTime = end; //结束时间为当前时间
start = end; //设置停止循环
}else if(daya<7){
endTime=end;
start=end;//用于结束循环
}
}
// }
} catch (Exception e) {
e.printStackTrace();
}
}
public static void main(String[] args) {
String start = "2023-10-09 13:41:20";
String end = TimeUtils.getStringDate();//比当前时间少一分钟,防止数据重复
if (StringUtils.isBlank(start)) {
start = "2023-08-01 00:00:00";
}
Date CalendarstartTime = TimeUtils.DateSwitch(start);
Date CalendarendTime = TimeUtils.DateSwitch(end);
Calendar startCal = Calendar.getInstance();
startCal.setTime(CalendarstartTime) ;
Calendar endCal = Calendar.getInstance();
endCal.setTime(CalendarendTime);
String startTime =start;
String endTime =null;
while (!start.equals(end)) { //当开始时间大于或者等于结束时间
Integer day = DataUtil.countDaynew(start,end );
if (day>7) {
startCal.add(Calendar.DAY_OF_MONTH, 7);
endTime = TimeUtils.date2Str(startCal.getTime());
Integer daya = DataUtil.countDaynew(endTime, end); //判断结束时间加上七天跟当前时间的间隔
if (daya <1) { //如果加上七天大于当前时间或者等于
endTime=end;
}
}
if (StringUtils.isBlank(endTime)){
endTime=end;
}
System.out.println(startTime+"-----3---"+endTime);
Integer days = DataUtil.countDaynew(startTime,endTime );
Integer daya = DataUtil.countDaynew(endTime, end);
if (days==7){ //超过七天
startTime = endTime;
} else if (days>0&&daya==0){
startTime = endTime;//累计七天的结束时间
endTime = end; //结束时间为当前时间
start = end; //设置停止循环
}else if(daya<7){
endTime=end;
start=end;//用于结束循环
}
}
}
}
package com.zuodou.job;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.node.ArrayNode;
import com.zuodou.entity.ZuodouTaskDatas;
import com.zuodou.entity.ZuodouTaskItem;
import com.zuodou.enums.CommonConstant;
import com.zuodou.enums.ErpStatusEnum;
import com.zuodou.erpmodel.erpData;
import com.zuodou.erpmodel.erpDatas;
import com.zuodou.erpmodel.erpItems;
import com.zuodou.mapper.ZuodouTaskDatasMapper;
import com.zuodou.model.*;
import com.zuodou.service.IZuodouTaskDatasService;
import com.zuodou.service.IZuodouTaskItemService;
import com.zuodou.utlis.BaseUtlis;
import com.zuodou.utlis.TimeUtils;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import org.springframework.util.CollectionUtils;
import java.io.IOException;
import java.util.*;
import static com.zuodou.utlis.ApiUtils.getSellerItem;
@Service
@Slf4j
public class threadService {
@Autowired
private IZuodouTaskItemService zuodouTaskItemService;
@Autowired
private IZuodouTaskDatasService iZuodouTaskDatasService;
@Autowired
private ZuodouTaskDatasMapper zuodouTaskDatasMapper;
public void apithread(String finalStartTime, String finalEndTime) {
log.info("执行售后任务时间"+finalStartTime+"----"+finalEndTime);
List zuodouTaskDatas = new ArrayList<>();
List updatezuodouTaskDatas = new ArrayList<>();
List zuodouTaskItemList = new ArrayList<>();
List updatezuodouTaskItemList = new ArrayList<>();
List orders = new ArrayList<>();
ObjectMapper objectMapper=new ObjectMapper();
OrderQueryParams queryParams = new OrderQueryParams();
queryParams.setModified_begin(finalStartTime).setModified_end(finalEndTime).setPage_index(1).setPage_size(50);
try {
String valueAsString = objectMapper.writeValueAsString(queryParams);
JsonNode jsonNode = objectMapper.readTree(getSellerItem("/open/refund/single/query", valueAsString));
String msg = jsonNode.get("msg").asText();
String code = jsonNode.get("code").asText();
if (StringUtils.equals(msg, "执行成功") && StringUtils.equals(code, "0")) {
Integer page_count = 0;//有多少页
JsonNode dataNode = jsonNode.get("data");
erpData erpData = null;
erpData = objectMapper.treeToValue(dataNode, erpData.class);
page_count += erpData.getPage_count();
if (page_count > 0) {
for (int j = 1; j <= page_count; j++) {
OrderQueryParams queryParamsdata_count = new OrderQueryParams();
queryParamsdata_count.setModified_begin(finalStartTime).setModified_end(finalEndTime).setPage_index(j).setPage_size(50);
String valueAsStrings = null;
valueAsStrings = objectMapper.writeValueAsString(queryParamsdata_count);
JsonNode jsonNodes = objectMapper.readTree(getSellerItem("/open/refund/single/query", valueAsStrings));
String msgs = jsonNodes.get("msg").asText();
String codes = jsonNodes.get("code").asText();
JsonNode dataNodea = jsonNodes.get("data");
if (StringUtils.equals(msgs, "执行成功") && StringUtils.equals(codes, "0")) {
log.info("售后数据执行成功"+msgs);
JsonNode dataNodes = dataNodea.get("datas");//拿到数据结构
if (dataNodes instanceof ArrayNode) {
ArrayNode ordersArrayNode = (ArrayNode) dataNodes;
for (JsonNode orderNode : ordersArrayNode) {
erpDatas order = objectMapper.treeToValue(orderNode, erpDatas.class);
orders.add(order);
}
}
} else {
log.info("售后数据执行失败"+msgs);
}
try {
// 在apithread方法中休息两秒
Thread.sleep(1000);
} catch (InterruptedException e) {
e.printStackTrace();
}
}
}
log.info("聚水潭售后数据拉取结束-----"+ TimeUtils.date2Str(new Date()));
} else {
log.info("外层分页售后"+msg);
}
try {
// 在apithread方法中休息两秒
Thread.sleep(1000);
} catch (InterruptedException e) {
e.printStackTrace();
}
insertOrdersData(orders, zuodouTaskDatas,updatezuodouTaskDatas, zuodouTaskItemList,updatezuodouTaskItemList);
log.info("开始插入售后数据----"+zuodouTaskDatas.size() + "商品数据---"+zuodouTaskItemList.size()+"需要修改售后数据--"+updatezuodouTaskDatas.size());
iZuodouTaskDatasService.saveBatch(zuodouTaskDatas);
zuodouTaskItemService.saveBatch(zuodouTaskItemList);
iZuodouTaskDatasService.updateBatchById(updatezuodouTaskDatas);
zuodouTaskItemService.updateBatchById(updatezuodouTaskItemList);
} catch (JsonProcessingException e) {
e.printStackTrace();
} catch (IOException e) {
e.printStackTrace();
} catch (IllegalAccessException e) {
e.printStackTrace();
} catch (InterruptedException e) {
e.printStackTrace();
}catch (Exception e){
e.printStackTrace();
}
}
private void insertOrdersData(List orders, List zuodouTaskOrdersList, List updatezuodouTaskOrdersList, List zuodouTaskItemList,List updatezuodouTaskItemList ) throws IllegalAccessException, IOException, InterruptedException {
if (!CollectionUtils.isEmpty(orders)){
for (erpDatas order : orders) {
ZuodouTaskDatas zuodouTaskDatas=new ZuodouTaskDatas();
BaseUtlis.copyProperties(order,zuodouTaskDatas);
String asId = zuodouTaskDatasMapper.oIdasIddatas(order.getO_id(), order.getAs_id());
// boolean exists = zuodouTaskOrdersList.stream()
// .anyMatch(obj -> obj.getOId().equals(order.getO_id()) && obj.getAsId() .equals(order.getAs_id()));
boolean exists = zuodouTaskOrdersList.stream()
.anyMatch(obj -> obj.getAsId() .equals(order.getAs_id()));
if (exists) {//如果插入的list中有那个值就直接跳过
continue;
}else if (StringUtils.isNotBlank(asId)) {
zuodouTaskDatas.setAsId(asId);
zuodouTaskDatas.setStatusName(ErpStatusEnum.getValueName(order.getStatus(), CommonConstant.STATUS));
zuodouTaskDatas.setShopStatusName(ErpStatusEnum.getValueName(order.getShop_status(),CommonConstant.SHOP_STATUS));
zuodouTaskDatas.setGoodStatusName(ErpStatusEnum.getValueName(order.getGood_status(),CommonConstant.GOOD_STATUS));
zuodouTaskDatas.setOrderStatusName(ErpStatusEnum.getValueName(order.getOrder_status(),CommonConstant.ORDER_STATUS));
zuodouTaskDatas.setOrderlabels(String.join(",",order.getOrder_labels()));
updatezuodouTaskOrdersList.add(zuodouTaskDatas);
for (erpItems item : order.getItems()) {
ZuodouTaskItem zuodouTaskItem=new ZuodouTaskItem();
BaseUtlis.copyProperties(item,zuodouTaskItem);
zuodouTaskItem.setSkuTypeName(ErpStatusEnum.getValueName(item.getSku_type(),CommonConstant.SKU_TYPE));
zuodouTaskItem.setSkuType(item.getSku_type());
updatezuodouTaskItemList.add(zuodouTaskItem);
}
}else{
zuodouTaskDatas.setStatusName(ErpStatusEnum.getValueName(order.getStatus(),CommonConstant.STATUS));
zuodouTaskDatas.setShopStatusName(ErpStatusEnum.getValueName(order.getShop_status(),CommonConstant.SHOP_STATUS));
zuodouTaskDatas.setGoodStatusName(ErpStatusEnum.getValueName(order.getGood_status(),CommonConstant.GOOD_STATUS));
zuodouTaskDatas.setOrderStatusName(ErpStatusEnum.getValueName(order.getOrder_status(),CommonConstant.ORDER_STATUS));
zuodouTaskDatas.setOrderlabels(String.join(",",order.getOrder_labels()));
if (!CollectionUtils.isEmpty(order.getItems())){
for (erpItems item : order.getItems()) {
ZuodouTaskItem zuodouTaskItem=new ZuodouTaskItem();
BaseUtlis.copyProperties(item,zuodouTaskItem);
zuodouTaskItem.setSkuTypeName(ErpStatusEnum.getValueName(item.getSku_type(),CommonConstant.SKU_TYPE));
zuodouTaskItem.setSkuType(item.getSku_type());
zuodouTaskItemList.add(zuodouTaskItem);
}
}
zuodouTaskOrdersList.add(zuodouTaskDatas);
}
}
}
}
}
public static void copyProperties(Object source, Object destination) throws IllegalAccessException { Field[] sourceFields = source.getClass().getDeclaredFields(); Field[] destinationFields = destination.getClass().getDeclaredFields(); for (Field sourceField : sourceFields) { sourceField.setAccessible(true); String sourceFieldName = sourceField.getName(); String destinationFieldName = convertToCamelCase(sourceFieldName); // 将下划线命名转换为驼峰命名 for (Field destinationField : destinationFields) { if (destinationField.getName().equals(destinationFieldName)) { destinationField.setAccessible(true); destinationField.set(destination, sourceField.get(source)); break; } } } }