import java.util.ArrayList;
import java.util.List;
import java.util.stream.Collectors;
import java.util.stream.Stream;
public class ListUtils {
/**
* 将一个list均分成n个list:
* @param source
* @param n
* @return
*/
public static List> avgList(List source, int n) {
List> result = new ArrayList<>();
//(先计算出余数)
int remainder = source.size() % n;
//然后是商
int number = source.size() / n;
//偏移量
int offset = 0;
for (int i = 0; i < n; i++) {
List value;
if (remainder > 0) {
value = source.subList(i * number + offset, (i + 1) * number + offset + 1);
remainder--;
offset++;
} else {
value = source.subList(i * number + offset, (i + 1) * number + offset);
}
result.add(value);
}
return result;
}
/**
* 将list分成每份count的数量的list:
* @param resList
* @param count
* @return
*/
public static List> splitList(List resList, int count) {
if (resList == null || count < 1) {
return null;
}
//long t1 = System.currentTimeMillis();
List> result = new ArrayList>();
int size = resList.size();
if (size <= count) {
// 数据量不足count指定的大小
result.add(resList);
} else {
int pre = size / count;
int last = size % count;
// 前面pre个集合,每个大小都是count个元素
for (int i = 0; i < pre; i++) {
List itemList = new ArrayList();
for (int j = 0; j < count; j++) {
itemList.add(resList.get(i * count + j));
}
result.add(itemList);
}
// last的进行处理
if (last > 0) {
List itemList = new ArrayList();
for (int i = 0; i < last; i++) {
itemList.add(resList.get(pre * count + i));
}
result.add(itemList);
}
}
/*long t2 = System.currentTimeMillis();
System.out.println("splitList====>>> resList.size:" + resList.size()
+ ", count:" +count + ", costs time:" + (t2 - t1) + " ms");*/
return result;
}
public static List> splitList2(List list, int count) { //效率低不推荐使用
long t1 = System.currentTimeMillis();
int limit = (list.size() + count - 1) / count;
// 方法一:使用流遍历操作
/*List> result = new ArrayList<>();
Stream.iterate(0, n -> n + 1).limit(limit).forEach(i -> {
result.add(list.stream().skip(i * count).limit(count).collect(Collectors.toList()));
});*/
//方法二:获取分割后的集合
List> result = Stream.iterate(0, n -> n + 1).limit(limit).parallel().map(a -> list.stream().skip(a * count).limit(count).parallel().collect(Collectors.toList())).collect(Collectors.toList());
long t2 = System.currentTimeMillis();
System.out.println("splitList====>>> resList.size:" + list.size()
+ ", count:" +count + ", costs time:" + (t2 - t1) + " ms");
return result;
}
public static void main(String[] args) {
/*List list=new ArrayList<>();
for (int i = 0; i < 17; i++) {
list.add(i);
}
List> avgList = avgList(list, 5);
System.out.println("avgList: " + avgList);
List> splitList = splitList(list, 5);
System.out.println("splitList: " + splitList);
ArrayList arr_list = new ArrayList<>();
for (int i = 0; i < splitList.size(); i++) {
List subList = splitList.get(i);
arr_list.addAll(subList);
}
System.out.println("arr_list: " + arr_list);*/
}
}
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.Semaphore;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.test.utils.ListUtils;
public class ConcurrencyTest {
public static void main(String[] args) {
int count = 10000000;
List listKey = new ArrayList<>();
for (int i = 0; i < count; i++) {
listKey.add(114560315500000000L + i);
}
ConcurrencyTest concurrencyTest = new ConcurrencyTest();
List valueList1 = concurrencyTest.getValueList1(listKey);
System.out.println("====>> getValueList1 valueList.size: " + valueList1.size());
}
private final static Logger log = LoggerFactory.getLogger(ConcurrencyTest.class);
/**
* 模拟一千万请求数据,创建一个线程池10个线程控制最大并发数,每个线程一次处理10万条,开启100个线程。(用于控制每批次线程的数据量业务场景)
* @param listKey 请求处理的总数据量
* @return
*/
public List getValueList1(List listKey) {
/**
(1)newCachedThreadPool 创建一个可缓存线程池,如果线程池长度超过处理需要,可灵活回收空闲线程,若无可回收,则新建线程。
(2)newFixedThreadPool 创建一个定长线程池,可控制线程最大并发数,超出的线程会在队列中等待。
(3)newScheduledThreadPool 创建一个定长线程池,支持定时及周期性任务执行。
(4)newSingleThreadExecutor 创建一个单线程化的线程池,它只会用唯一的工作线程来执行任务,保证所有任务按照指定顺序(FIFO, LIFO, 优先级)执行。
*/
//创建一个线程池
final ExecutorService executorService = Executors.newFixedThreadPool(10); //用于控制同时并发执行的线程数,使用线程池创建资源提高效率
List list_val = Collections.synchronizedList(new ArrayList<>()); //保证多线程操作的是同一个List
try {
long t1 = System.currentTimeMillis();
int max_one_batch = 100000; // 一批次最多处理100000条数据
List> newList = ListUtils.splitList(listKey, max_one_batch);
int runSize = newList.size(); // 开启的线程数
/**
* CountDownLanch 只需要在子线程执行之前, 赋予初始化countDownLanch, 并赋予线程数量为初始值。
* 每个线程执行完毕的时候, 就countDown一下。主线程只需要调用await方法, 可以等待所有子线程执行结束。
*/
final CountDownLatch countDownLatch = new CountDownLatch(runSize); //计数器
/**
* Semaphore(信号量)是用来控制同时访问特定资源的线程数量,拿到信号量的线程可以进入,否则就等待。
* 通过acquire()和release()获取和释放访问许可。
*/
final Semaphore semaphore = new Semaphore(runSize); //信号量
// 循环创建线程
for (int j = 0; j < runSize; j++) {
final int i = j;
executorService.execute(new Runnable() {
@Override
public void run() {
try {
semaphore.acquire();
// 执行程序
List subList = newList.get(i);
List sub_ret = getValue(subList);
list_val.addAll(sub_ret);
log.info(Thread.currentThread().getName() + ": 当前线程/总线程: [" + i + "/" + runSize + "]"
+ ", 处理的数据条数:" + subList.size());
semaphore.release();
} catch (Exception e) {
log.error(e.getMessage(), e);
} finally {
// 计数器减一
countDownLatch.countDown();
}
}
});
}
// 阻塞主线程等待所有线程执行完成
countDownLatch.await();
// 所有线程执行完,之后才能执行的部分
long t2 = System.currentTimeMillis();
log.info("Call getValueList1 success... ret: {} size, threadCount: {}, costs time: {} ms", list_val.size(), runSize, (t2 - t1));
return list_val;
} catch (Exception e) {
return null;
} finally {
// 关闭线程池
executorService.shutdown();
}
}
private List getValue(List listKey) {
//具体操作省略
return listKey;
}
}
测试日志记录:
01:11:17.515 [pool-1-thread-4] INFO com.test.thread.ConcurrencyTest - pool-1-thread-4: 当前线程/总线程: [3/100], 处理的数据条数:100000
01:11:17.515 [pool-1-thread-8] INFO com.test.thread.ConcurrencyTest - pool-1-thread-8: 当前线程/总线程: [7/100], 处理的数据条数:100000
01:11:17.515 [pool-1-thread-1] INFO com.test.thread.ConcurrencyTest - pool-1-thread-1: 当前线程/总线程: [0/100], 处理的数据条数:100000
01:11:17.515 [pool-1-thread-10] INFO com.test.thread.ConcurrencyTest - pool-1-thread-10: 当前线程/总线程: [9/100], 处理的数据条数:100000
01:11:17.515 [pool-1-thread-2] INFO com.test.thread.ConcurrencyTest - pool-1-thread-2: 当前线程/总线程: [1/100], 处理的数据条数:100000
01:11:17.516 [pool-1-thread-9] INFO com.test.thread.ConcurrencyTest - pool-1-thread-9: 当前线程/总线程: [8/100], 处理的数据条数:100000
01:11:17.515 [pool-1-thread-5] INFO com.test.thread.ConcurrencyTest - pool-1-thread-5: 当前线程/总线程: [4/100], 处理的数据条数:100000
01:11:17.515 [pool-1-thread-3] INFO com.test.thread.ConcurrencyTest - pool-1-thread-3: 当前线程/总线程: [2/100], 处理的数据条数:100000
01:11:17.516 [pool-1-thread-6] INFO com.test.thread.ConcurrencyTest - pool-1-thread-6: 当前线程/总线程: [5/100], 处理的数据条数:100000
01:11:17.516 [pool-1-thread-7] INFO com.test.thread.ConcurrencyTest - pool-1-thread-7: 当前线程/总线程: [6/100], 处理的数据条数:100000
01:11:17.520 [pool-1-thread-4] INFO com.test.thread.ConcurrencyTest - pool-1-thread-4: 当前线程/总线程: [10/100], 处理的数据条数:100000
01:11:17.520 [pool-1-thread-7] INFO com.test.thread.ConcurrencyTest - pool-1-thread-7: 当前线程/总线程: [19/100], 处理的数据条数:100000
01:11:17.520 [pool-1-thread-4] INFO com.test.thread.ConcurrencyTest - pool-1-thread-4: 当前线程/总线程: [20/100], 处理的数据条数:100000
01:11:17.520 [pool-1-thread-6] INFO com.test.thread.ConcurrencyTest - pool-1-thread-6: 当前线程/总线程: [18/100], 处理的数据条数:100000
01:11:17.521 [pool-1-thread-3] INFO com.test.thread.ConcurrencyTest - pool-1-thread-3: 当前线程/总线程: [17/100], 处理的数据条数:100000
01:11:17.522 [pool-1-thread-5] INFO com.test.thread.ConcurrencyTest - pool-1-thread-5: 当前线程/总线程: [16/100], 处理的数据条数:100000
01:11:17.523 [pool-1-thread-9] INFO com.test.thread.ConcurrencyTest - pool-1-thread-9: 当前线程/总线程: [15/100], 处理的数据条数:100000
01:11:17.523 [pool-1-thread-8] INFO com.test.thread.ConcurrencyTest - pool-1-thread-8: 当前线程/总线程: [14/100], 处理的数据条数:100000
01:11:17.523 [pool-1-thread-2] INFO com.test.thread.ConcurrencyTest - pool-1-thread-2: 当前线程/总线程: [13/100], 处理的数据条数:100000
01:11:17.523 [pool-1-thread-10] INFO com.test.thread.ConcurrencyTest - pool-1-thread-10: 当前线程/总线程: [12/100], 处理的数据条数:100000
01:11:17.523 [pool-1-thread-1] INFO com.test.thread.ConcurrencyTest - pool-1-thread-1: 当前线程/总线程: [11/100], 处理的数据条数:100000
01:11:17.523 [pool-1-thread-10] INFO com.test.thread.ConcurrencyTest - pool-1-thread-10: 当前线程/总线程: [29/100], 处理的数据条数:100000
01:11:17.526 [pool-1-thread-2] INFO com.test.thread.ConcurrencyTest - pool-1-thread-2: 当前线程/总线程: [28/100], 处理的数据条数:100000
01:11:17.527 [pool-1-thread-8] INFO com.test.thread.ConcurrencyTest - pool-1-thread-8: 当前线程/总线程: [27/100], 处理的数据条数:100000
01:11:17.527 [pool-1-thread-2] INFO com.test.thread.ConcurrencyTest - pool-1-thread-2: 当前线程/总线程: [32/100], 处理的数据条数:100000
01:11:17.527 [pool-1-thread-9] INFO com.test.thread.ConcurrencyTest - pool-1-thread-9: 当前线程/总线程: [26/100], 处理的数据条数:100000
01:11:17.527 [pool-1-thread-5] INFO com.test.thread.ConcurrencyTest - pool-1-thread-5: 当前线程/总线程: [25/100], 处理的数据条数:100000
01:11:17.527 [pool-1-thread-3] INFO com.test.thread.ConcurrencyTest - pool-1-thread-3: 当前线程/总线程: [24/100], 处理的数据条数:100000
01:11:17.528 [pool-1-thread-6] INFO com.test.thread.ConcurrencyTest - pool-1-thread-6: 当前线程/总线程: [23/100], 处理的数据条数:100000
01:11:17.528 [pool-1-thread-4] INFO com.test.thread.ConcurrencyTest - pool-1-thread-4: 当前线程/总线程: [22/100], 处理的数据条数:100000
01:11:17.528 [pool-1-thread-7] INFO com.test.thread.ConcurrencyTest - pool-1-thread-7: 当前线程/总线程: [21/100], 处理的数据条数:100000
01:11:17.528 [pool-1-thread-4] INFO com.test.thread.ConcurrencyTest - pool-1-thread-4: 当前线程/总线程: [39/100], 处理的数据条数:100000
01:11:17.528 [pool-1-thread-6] INFO com.test.thread.ConcurrencyTest - pool-1-thread-6: 当前线程/总线程: [38/100], 处理的数据条数:100000
01:11:17.528 [pool-1-thread-3] INFO com.test.thread.ConcurrencyTest - pool-1-thread-3: 当前线程/总线程: [37/100], 处理的数据条数:100000
01:11:17.645 [pool-1-thread-5] INFO com.test.thread.ConcurrencyTest - pool-1-thread-5: 当前线程/总线程: [36/100], 处理的数据条数:100000
01:11:17.645 [pool-1-thread-9] INFO com.test.thread.ConcurrencyTest - pool-1-thread-9: 当前线程/总线程: [35/100], 处理的数据条数:100000
01:11:17.646 [pool-1-thread-2] INFO com.test.thread.ConcurrencyTest - pool-1-thread-2: 当前线程/总线程: [34/100], 处理的数据条数:100000
01:11:17.646 [pool-1-thread-8] INFO com.test.thread.ConcurrencyTest - pool-1-thread-8: 当前线程/总线程: [33/100], 处理的数据条数:100000
01:11:17.646 [pool-1-thread-10] INFO com.test.thread.ConcurrencyTest - pool-1-thread-10: 当前线程/总线程: [31/100], 处理的数据条数:100000
01:11:17.646 [pool-1-thread-1] INFO com.test.thread.ConcurrencyTest - pool-1-thread-1: 当前线程/总线程: [30/100], 处理的数据条数:100000
01:11:17.647 [pool-1-thread-10] INFO com.test.thread.ConcurrencyTest - pool-1-thread-10: 当前线程/总线程: [48/100], 处理的数据条数:100000
01:11:17.647 [pool-1-thread-8] INFO com.test.thread.ConcurrencyTest - pool-1-thread-8: 当前线程/总线程: [47/100], 处理的数据条数:100000
01:11:17.647 [pool-1-thread-2] INFO com.test.thread.ConcurrencyTest - pool-1-thread-2: 当前线程/总线程: [46/100], 处理的数据条数:100000
01:11:17.647 [pool-1-thread-9] INFO com.test.thread.ConcurrencyTest - pool-1-thread-9: 当前线程/总线程: [45/100], 处理的数据条数:100000
01:11:17.647 [pool-1-thread-5] INFO com.test.thread.ConcurrencyTest - pool-1-thread-5: 当前线程/总线程: [44/100], 处理的数据条数:100000
01:11:17.648 [pool-1-thread-3] INFO com.test.thread.ConcurrencyTest - pool-1-thread-3: 当前线程/总线程: [43/100], 处理的数据条数:100000
01:11:17.648 [pool-1-thread-6] INFO com.test.thread.ConcurrencyTest - pool-1-thread-6: 当前线程/总线程: [42/100], 处理的数据条数:100000
01:11:17.648 [pool-1-thread-4] INFO com.test.thread.ConcurrencyTest - pool-1-thread-4: 当前线程/总线程: [41/100], 处理的数据条数:100000
01:11:17.648 [pool-1-thread-7] INFO com.test.thread.ConcurrencyTest - pool-1-thread-7: 当前线程/总线程: [40/100], 处理的数据条数:100000
01:11:17.648 [pool-1-thread-4] INFO com.test.thread.ConcurrencyTest - pool-1-thread-4: 当前线程/总线程: [57/100], 处理的数据条数:100000
01:11:17.649 [pool-1-thread-6] INFO com.test.thread.ConcurrencyTest - pool-1-thread-6: 当前线程/总线程: [56/100], 处理的数据条数:100000
01:11:17.655 [pool-1-thread-3] INFO com.test.thread.ConcurrencyTest - pool-1-thread-3: 当前线程/总线程: [55/100], 处理的数据条数:100000
01:11:17.656 [pool-1-thread-9] INFO com.test.thread.ConcurrencyTest - pool-1-thread-9: 当前线程/总线程: [53/100], 处理的数据条数:100000
01:11:17.656 [pool-1-thread-9] INFO com.test.thread.ConcurrencyTest - pool-1-thread-9: 当前线程/总线程: [62/100], 处理的数据条数:100000
01:11:17.655 [pool-1-thread-5] INFO com.test.thread.ConcurrencyTest - pool-1-thread-5: 当前线程/总线程: [54/100], 处理的数据条数:100000
01:11:17.656 [pool-1-thread-2] INFO com.test.thread.ConcurrencyTest - pool-1-thread-2: 当前线程/总线程: [52/100], 处理的数据条数:100000
01:11:17.657 [pool-1-thread-2] INFO com.test.thread.ConcurrencyTest - pool-1-thread-2: 当前线程/总线程: [65/100], 处理的数据条数:100000
01:11:17.657 [pool-1-thread-8] INFO com.test.thread.ConcurrencyTest - pool-1-thread-8: 当前线程/总线程: [51/100], 处理的数据条数:100000
01:11:17.657 [pool-1-thread-10] INFO com.test.thread.ConcurrencyTest - pool-1-thread-10: 当前线程/总线程: [50/100], 处理的数据条数:100000
01:11:17.657 [pool-1-thread-1] INFO com.test.thread.ConcurrencyTest - pool-1-thread-1: 当前线程/总线程: [49/100], 处理的数据条数:100000
01:11:17.657 [pool-1-thread-10] INFO com.test.thread.ConcurrencyTest - pool-1-thread-10: 当前线程/总线程: [68/100], 处理的数据条数:100000
01:11:17.657 [pool-1-thread-8] INFO com.test.thread.ConcurrencyTest - pool-1-thread-8: 当前线程/总线程: [67/100], 处理的数据条数:100000
01:11:17.658 [pool-1-thread-2] INFO com.test.thread.ConcurrencyTest - pool-1-thread-2: 当前线程/总线程: [66/100], 处理的数据条数:100000
01:11:17.658 [pool-1-thread-5] INFO com.test.thread.ConcurrencyTest - pool-1-thread-5: 当前线程/总线程: [64/100], 处理的数据条数:100000
01:11:17.658 [pool-1-thread-9] INFO com.test.thread.ConcurrencyTest - pool-1-thread-9: 当前线程/总线程: [63/100], 处理的数据条数:100000
01:11:17.658 [pool-1-thread-3] INFO com.test.thread.ConcurrencyTest - pool-1-thread-3: 当前线程/总线程: [61/100], 处理的数据条数:100000
01:11:17.659 [pool-1-thread-6] INFO com.test.thread.ConcurrencyTest - pool-1-thread-6: 当前线程/总线程: [60/100], 处理的数据条数:100000
01:11:17.659 [pool-1-thread-4] INFO com.test.thread.ConcurrencyTest - pool-1-thread-4: 当前线程/总线程: [59/100], 处理的数据条数:100000
01:11:17.659 [pool-1-thread-7] INFO com.test.thread.ConcurrencyTest - pool-1-thread-7: 当前线程/总线程: [58/100], 处理的数据条数:100000
01:11:17.659 [pool-1-thread-4] INFO com.test.thread.ConcurrencyTest - pool-1-thread-4: 当前线程/总线程: [77/100], 处理的数据条数:100000
01:11:17.659 [pool-1-thread-6] INFO com.test.thread.ConcurrencyTest - pool-1-thread-6: 当前线程/总线程: [76/100], 处理的数据条数:100000
01:11:17.659 [pool-1-thread-3] INFO com.test.thread.ConcurrencyTest - pool-1-thread-3: 当前线程/总线程: [75/100], 处理的数据条数:100000
01:11:17.660 [pool-1-thread-9] INFO com.test.thread.ConcurrencyTest - pool-1-thread-9: 当前线程/总线程: [74/100], 处理的数据条数:100000
01:11:17.660 [pool-1-thread-5] INFO com.test.thread.ConcurrencyTest - pool-1-thread-5: 当前线程/总线程: [73/100], 处理的数据条数:100000
01:11:17.660 [pool-1-thread-2] INFO com.test.thread.ConcurrencyTest - pool-1-thread-2: 当前线程/总线程: [72/100], 处理的数据条数:100000
01:11:17.660 [pool-1-thread-8] INFO com.test.thread.ConcurrencyTest - pool-1-thread-8: 当前线程/总线程: [71/100], 处理的数据条数:100000
01:11:17.794 [pool-1-thread-10] INFO com.test.thread.ConcurrencyTest - pool-1-thread-10: 当前线程/总线程: [70/100], 处理的数据条数:100000
01:11:17.795 [pool-1-thread-1] INFO com.test.thread.ConcurrencyTest - pool-1-thread-1: 当前线程/总线程: [69/100], 处理的数据条数:100000
01:11:17.795 [pool-1-thread-10] INFO com.test.thread.ConcurrencyTest - pool-1-thread-10: 当前线程/总线程: [86/100], 处理的数据条数:100000
01:11:17.795 [pool-1-thread-8] INFO com.test.thread.ConcurrencyTest - pool-1-thread-8: 当前线程/总线程: [85/100], 处理的数据条数:100000
01:11:17.796 [pool-1-thread-2] INFO com.test.thread.ConcurrencyTest - pool-1-thread-2: 当前线程/总线程: [84/100], 处理的数据条数:100000
01:11:17.796 [pool-1-thread-2] INFO com.test.thread.ConcurrencyTest - pool-1-thread-2: 当前线程/总线程: [90/100], 处理的数据条数:100000
01:11:17.797 [pool-1-thread-2] INFO com.test.thread.ConcurrencyTest - pool-1-thread-2: 当前线程/总线程: [91/100], 处理的数据条数:100000
01:11:17.797 [pool-1-thread-2] INFO com.test.thread.ConcurrencyTest - pool-1-thread-2: 当前线程/总线程: [92/100], 处理的数据条数:100000
01:11:17.797 [pool-1-thread-5] INFO com.test.thread.ConcurrencyTest - pool-1-thread-5: 当前线程/总线程: [83/100], 处理的数据条数:100000
01:11:17.797 [pool-1-thread-9] INFO com.test.thread.ConcurrencyTest - pool-1-thread-9: 当前线程/总线程: [82/100], 处理的数据条数:100000
01:11:17.798 [pool-1-thread-3] INFO com.test.thread.ConcurrencyTest - pool-1-thread-3: 当前线程/总线程: [81/100], 处理的数据条数:100000
01:11:17.798 [pool-1-thread-6] INFO com.test.thread.ConcurrencyTest - pool-1-thread-6: 当前线程/总线程: [80/100], 处理的数据条数:100000
01:11:17.798 [pool-1-thread-4] INFO com.test.thread.ConcurrencyTest - pool-1-thread-4: 当前线程/总线程: [79/100], 处理的数据条数:100000
01:11:17.798 [pool-1-thread-7] INFO com.test.thread.ConcurrencyTest - pool-1-thread-7: 当前线程/总线程: [78/100], 处理的数据条数:100000
01:11:17.798 [pool-1-thread-4] INFO com.test.thread.ConcurrencyTest - pool-1-thread-4: 当前线程/总线程: [98/100], 处理的数据条数:100000
01:11:17.799 [pool-1-thread-6] INFO com.test.thread.ConcurrencyTest - pool-1-thread-6: 当前线程/总线程: [97/100], 处理的数据条数:100000
01:11:17.799 [pool-1-thread-3] INFO com.test.thread.ConcurrencyTest - pool-1-thread-3: 当前线程/总线程: [96/100], 处理的数据条数:100000
01:11:17.799 [pool-1-thread-5] INFO com.test.thread.ConcurrencyTest - pool-1-thread-5: 当前线程/总线程: [94/100], 处理的数据条数:100000
01:11:17.799 [pool-1-thread-9] INFO com.test.thread.ConcurrencyTest - pool-1-thread-9: 当前线程/总线程: [95/100], 处理的数据条数:100000
01:11:17.799 [pool-1-thread-2] INFO com.test.thread.ConcurrencyTest - pool-1-thread-2: 当前线程/总线程: [93/100], 处理的数据条数:100000
01:11:17.800 [pool-1-thread-8] INFO com.test.thread.ConcurrencyTest - pool-1-thread-8: 当前线程/总线程: [89/100], 处理的数据条数:100000
01:11:17.800 [pool-1-thread-10] INFO com.test.thread.ConcurrencyTest - pool-1-thread-10: 当前线程/总线程: [88/100], 处理的数据条数:100000
01:11:17.800 [pool-1-thread-1] INFO com.test.thread.ConcurrencyTest - pool-1-thread-1: 当前线程/总线程: [87/100], 处理的数据条数:100000
01:11:17.800 [pool-1-thread-7] INFO com.test.thread.ConcurrencyTest - pool-1-thread-7: 当前线程/总线程: [99/100], 处理的数据条数:100000
01:11:17.800 [main] INFO com.test.thread.ConcurrencyTest - Call getValueList1 success... ret: 10000000 size, threadCount: 100, costs time: 629 ms
====>> getValueList1 valueList.size: 10000000
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.Semaphore;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.test.utils.ListUtils;
public class ConcurrencyTest {
public static void main(String[] args) {
int count = 10000000;
List listKey = new ArrayList<>();
for (int i = 0; i < count; i++) {
listKey.add(114560315500000000L + i);
}
ConcurrencyTest concurrencyTest = new ConcurrencyTest();
List valueList2 = concurrencyTest.getValueList2(listKey);
System.out.println("====>> getValueList2 valueList.size: " + valueList2.size());
}
private final static Logger log = LoggerFactory.getLogger(ConcurrencyTest.class);
/**
* 模拟一千万请求数据,创建一个线程池10个线程控制最大并发数,设置请求的线程数为100,平均处理这一千万请求数据。(用于控制请求线程数的业务场景)
* @param listKey 请求处理的总数据量
* @return
*/
public List getValueList2(List listKey) {
/**
(1)newCachedThreadPool 创建一个可缓存线程池,如果线程池长度超过处理需要,可灵活回收空闲线程,若无可回收,则新建线程。
(2)newFixedThreadPool 创建一个定长线程池,可控制线程最大并发数,超出的线程会在队列中等待。
(3)newScheduledThreadPool 创建一个定长线程池,支持定时及周期性任务执行。
(4)newSingleThreadExecutor 创建一个单线程化的线程池,它只会用唯一的工作线程来执行任务,保证所有任务按照指定顺序(FIFO, LIFO, 优先级)执行。
*/
//创建一个线程池
final ExecutorService executorService = Executors.newFixedThreadPool(10); //用于控制同时并发执行的线程数,使用线程池创建资源提高效率
List list_val = Collections.synchronizedList(new ArrayList<>()); //保证多线程操作的是同一个List
try {
long t1 = System.currentTimeMillis();
int threadCount = 100; //请求的线程数
List> newList = ListUtils.avgList(listKey, threadCount);
int runSize = threadCount; // 请求开启的线程数
/**
* CountDownLanch 只需要在子线程执行之前, 赋予初始化countDownLanch, 并赋予线程数量为初始值。
* 每个线程执行完毕的时候, 就countDown一下。主线程只需要调用await方法, 可以等待所有子线程执行结束。
*/
final CountDownLatch countDownLatch = new CountDownLatch(runSize); //计数器
/**
* Semaphore(信号量)是用来控制同时访问特定资源的线程数量,拿到信号量的线程可以进入,否则就等待。
* 通过acquire()和release()获取和释放访问许可。
*/
final Semaphore semaphore = new Semaphore(runSize); //信号量
// 循环创建线程
for (int j = 0; j < runSize; j++) {
final int i = j;
executorService.execute(new Runnable() {
@Override
public void run() {
try {
semaphore.acquire();
// 执行程序
List subList = newList.get(i);
List sub_ret = getValue(subList);
list_val.addAll(sub_ret);
log.info(Thread.currentThread().getName() + ": 当前线程/总线程: [" + i + "/" + runSize + "]"
+ ", 处理的数据条数:" + subList.size());
semaphore.release();
} catch (Exception e) {
log.error(e.getMessage(), e);
} finally {
// 计数器减一
countDownLatch.countDown();
}
}
});
}
// 阻塞主线程等待所有线程执行完成
countDownLatch.await();
// 所有线程执行完,之后才能执行的部分
long t2 = System.currentTimeMillis();
log.info("Call getValueList2 success... ret: {} size, threadCount: {}, costs time: {} ms", list_val.size(), runSize, (t2 - t1));
return list_val;
} catch (Exception e) {
return null;
} finally {
// 关闭线程池
executorService.shutdown();
}
}
private List getValue(List listKey) {
//具体操作省略
return listKey;
}
}
测试日志记录:
01:16:05.076 [pool-1-thread-9] INFO com.test.thread.ConcurrencyTest - pool-1-thread-9: 当前线程/总线程: [8/100], 处理的数据条数:100000
01:16:05.076 [pool-1-thread-10] INFO com.test.thread.ConcurrencyTest - pool-1-thread-10: 当前线程/总线程: [9/100], 处理的数据条数:100000
01:16:05.079 [pool-1-thread-5] INFO com.test.thread.ConcurrencyTest - pool-1-thread-5: 当前线程/总线程: [4/100], 处理的数据条数:100000
01:16:05.075 [pool-1-thread-1] INFO com.test.thread.ConcurrencyTest - pool-1-thread-1: 当前线程/总线程: [0/100], 处理的数据条数:100000
01:16:05.076 [pool-1-thread-8] INFO com.test.thread.ConcurrencyTest - pool-1-thread-8: 当前线程/总线程: [7/100], 处理的数据条数:100000
01:16:05.078 [pool-1-thread-7] INFO com.test.thread.ConcurrencyTest - pool-1-thread-7: 当前线程/总线程: [6/100], 处理的数据条数:100000
01:16:05.081 [pool-1-thread-4] INFO com.test.thread.ConcurrencyTest - pool-1-thread-4: 当前线程/总线程: [3/100], 处理的数据条数:100000
01:16:05.082 [pool-1-thread-6] INFO com.test.thread.ConcurrencyTest - pool-1-thread-6: 当前线程/总线程: [5/100], 处理的数据条数:100000
01:16:05.082 [pool-1-thread-2] INFO com.test.thread.ConcurrencyTest - pool-1-thread-2: 当前线程/总线程: [1/100], 处理的数据条数:100000
01:16:05.083 [pool-1-thread-3] INFO com.test.thread.ConcurrencyTest - pool-1-thread-3: 当前线程/总线程: [2/100], 处理的数据条数:100000
01:16:05.085 [pool-1-thread-2] INFO com.test.thread.ConcurrencyTest - pool-1-thread-2: 当前线程/总线程: [18/100], 处理的数据条数:100000
01:16:05.086 [pool-1-thread-6] INFO com.test.thread.ConcurrencyTest - pool-1-thread-6: 当前线程/总线程: [17/100], 处理的数据条数:100000
01:16:05.086 [pool-1-thread-4] INFO com.test.thread.ConcurrencyTest - pool-1-thread-4: 当前线程/总线程: [16/100], 处理的数据条数:100000
01:16:05.086 [pool-1-thread-7] INFO com.test.thread.ConcurrencyTest - pool-1-thread-7: 当前线程/总线程: [15/100], 处理的数据条数:100000
01:16:05.087 [pool-1-thread-8] INFO com.test.thread.ConcurrencyTest - pool-1-thread-8: 当前线程/总线程: [14/100], 处理的数据条数:100000
01:16:05.088 [pool-1-thread-1] INFO com.test.thread.ConcurrencyTest - pool-1-thread-1: 当前线程/总线程: [13/100], 处理的数据条数:100000
01:16:05.089 [pool-1-thread-5] INFO com.test.thread.ConcurrencyTest - pool-1-thread-5: 当前线程/总线程: [12/100], 处理的数据条数:100000
01:16:05.089 [pool-1-thread-10] INFO com.test.thread.ConcurrencyTest - pool-1-thread-10: 当前线程/总线程: [11/100], 处理的数据条数:100000
01:16:05.089 [pool-1-thread-9] INFO com.test.thread.ConcurrencyTest - pool-1-thread-9: 当前线程/总线程: [10/100], 处理的数据条数:100000
01:16:05.090 [pool-1-thread-10] INFO com.test.thread.ConcurrencyTest - pool-1-thread-10: 当前线程/总线程: [27/100], 处理的数据条数:100000
01:16:05.090 [pool-1-thread-5] INFO com.test.thread.ConcurrencyTest - pool-1-thread-5: 当前线程/总线程: [26/100], 处理的数据条数:100000
01:16:05.090 [pool-1-thread-1] INFO com.test.thread.ConcurrencyTest - pool-1-thread-1: 当前线程/总线程: [25/100], 处理的数据条数:100000
01:16:05.093 [pool-1-thread-8] INFO com.test.thread.ConcurrencyTest - pool-1-thread-8: 当前线程/总线程: [24/100], 处理的数据条数:100000
01:16:05.093 [pool-1-thread-7] INFO com.test.thread.ConcurrencyTest - pool-1-thread-7: 当前线程/总线程: [23/100], 处理的数据条数:100000
01:16:05.094 [pool-1-thread-4] INFO com.test.thread.ConcurrencyTest - pool-1-thread-4: 当前线程/总线程: [22/100], 处理的数据条数:100000
01:16:05.094 [pool-1-thread-6] INFO com.test.thread.ConcurrencyTest - pool-1-thread-6: 当前线程/总线程: [21/100], 处理的数据条数:100000
01:16:05.094 [pool-1-thread-2] INFO com.test.thread.ConcurrencyTest - pool-1-thread-2: 当前线程/总线程: [20/100], 处理的数据条数:100000
01:16:05.095 [pool-1-thread-3] INFO com.test.thread.ConcurrencyTest - pool-1-thread-3: 当前线程/总线程: [19/100], 处理的数据条数:100000
01:16:05.095 [pool-1-thread-2] INFO com.test.thread.ConcurrencyTest - pool-1-thread-2: 当前线程/总线程: [36/100], 处理的数据条数:100000
01:16:05.096 [pool-1-thread-6] INFO com.test.thread.ConcurrencyTest - pool-1-thread-6: 当前线程/总线程: [35/100], 处理的数据条数:100000
01:16:05.096 [pool-1-thread-4] INFO com.test.thread.ConcurrencyTest - pool-1-thread-4: 当前线程/总线程: [34/100], 处理的数据条数:100000
01:16:05.096 [pool-1-thread-7] INFO com.test.thread.ConcurrencyTest - pool-1-thread-7: 当前线程/总线程: [33/100], 处理的数据条数:100000
01:16:05.097 [pool-1-thread-8] INFO com.test.thread.ConcurrencyTest - pool-1-thread-8: 当前线程/总线程: [32/100], 处理的数据条数:100000
01:16:05.097 [pool-1-thread-1] INFO com.test.thread.ConcurrencyTest - pool-1-thread-1: 当前线程/总线程: [31/100], 处理的数据条数:100000
01:16:05.101 [pool-1-thread-5] INFO com.test.thread.ConcurrencyTest - pool-1-thread-5: 当前线程/总线程: [30/100], 处理的数据条数:100000
01:16:05.101 [pool-1-thread-10] INFO com.test.thread.ConcurrencyTest - pool-1-thread-10: 当前线程/总线程: [29/100], 处理的数据条数:100000
01:16:05.102 [pool-1-thread-9] INFO com.test.thread.ConcurrencyTest - pool-1-thread-9: 当前线程/总线程: [28/100], 处理的数据条数:100000
01:16:05.102 [pool-1-thread-10] INFO com.test.thread.ConcurrencyTest - pool-1-thread-10: 当前线程/总线程: [45/100], 处理的数据条数:100000
01:16:05.102 [pool-1-thread-5] INFO com.test.thread.ConcurrencyTest - pool-1-thread-5: 当前线程/总线程: [44/100], 处理的数据条数:100000
01:16:05.103 [pool-1-thread-1] INFO com.test.thread.ConcurrencyTest - pool-1-thread-1: 当前线程/总线程: [43/100], 处理的数据条数:100000
01:16:05.103 [pool-1-thread-8] INFO com.test.thread.ConcurrencyTest - pool-1-thread-8: 当前线程/总线程: [42/100], 处理的数据条数:100000
01:16:05.103 [pool-1-thread-7] INFO com.test.thread.ConcurrencyTest - pool-1-thread-7: 当前线程/总线程: [41/100], 处理的数据条数:100000
01:16:05.104 [pool-1-thread-4] INFO com.test.thread.ConcurrencyTest - pool-1-thread-4: 当前线程/总线程: [40/100], 处理的数据条数:100000
01:16:05.104 [pool-1-thread-6] INFO com.test.thread.ConcurrencyTest - pool-1-thread-6: 当前线程/总线程: [39/100], 处理的数据条数:100000
01:16:05.105 [pool-1-thread-2] INFO com.test.thread.ConcurrencyTest - pool-1-thread-2: 当前线程/总线程: [38/100], 处理的数据条数:100000
01:16:05.105 [pool-1-thread-3] INFO com.test.thread.ConcurrencyTest - pool-1-thread-3: 当前线程/总线程: [37/100], 处理的数据条数:100000
01:16:05.106 [pool-1-thread-2] INFO com.test.thread.ConcurrencyTest - pool-1-thread-2: 当前线程/总线程: [54/100], 处理的数据条数:100000
01:16:05.106 [pool-1-thread-6] INFO com.test.thread.ConcurrencyTest - pool-1-thread-6: 当前线程/总线程: [53/100], 处理的数据条数:100000
01:16:05.106 [pool-1-thread-4] INFO com.test.thread.ConcurrencyTest - pool-1-thread-4: 当前线程/总线程: [52/100], 处理的数据条数:100000
01:16:05.107 [pool-1-thread-7] INFO com.test.thread.ConcurrencyTest - pool-1-thread-7: 当前线程/总线程: [51/100], 处理的数据条数:100000
01:16:05.107 [pool-1-thread-8] INFO com.test.thread.ConcurrencyTest - pool-1-thread-8: 当前线程/总线程: [50/100], 处理的数据条数:100000
01:16:05.246 [pool-1-thread-1] INFO com.test.thread.ConcurrencyTest - pool-1-thread-1: 当前线程/总线程: [49/100], 处理的数据条数:100000
01:16:05.247 [pool-1-thread-5] INFO com.test.thread.ConcurrencyTest - pool-1-thread-5: 当前线程/总线程: [48/100], 处理的数据条数:100000
01:16:05.247 [pool-1-thread-1] INFO com.test.thread.ConcurrencyTest - pool-1-thread-1: 当前线程/总线程: [61/100], 处理的数据条数:100000
01:16:05.248 [pool-1-thread-10] INFO com.test.thread.ConcurrencyTest - pool-1-thread-10: 当前线程/总线程: [47/100], 处理的数据条数:100000
01:16:05.248 [pool-1-thread-9] INFO com.test.thread.ConcurrencyTest - pool-1-thread-9: 当前线程/总线程: [46/100], 处理的数据条数:100000
01:16:05.249 [pool-1-thread-10] INFO com.test.thread.ConcurrencyTest - pool-1-thread-10: 当前线程/总线程: [64/100], 处理的数据条数:100000
01:16:05.249 [pool-1-thread-1] INFO com.test.thread.ConcurrencyTest - pool-1-thread-1: 当前线程/总线程: [63/100], 处理的数据条数:100000
01:16:05.249 [pool-1-thread-5] INFO com.test.thread.ConcurrencyTest - pool-1-thread-5: 当前线程/总线程: [62/100], 处理的数据条数:100000
01:16:05.250 [pool-1-thread-8] INFO com.test.thread.ConcurrencyTest - pool-1-thread-8: 当前线程/总线程: [60/100], 处理的数据条数:100000
01:16:05.250 [pool-1-thread-7] INFO com.test.thread.ConcurrencyTest - pool-1-thread-7: 当前线程/总线程: [59/100], 处理的数据条数:100000
01:16:05.251 [pool-1-thread-4] INFO com.test.thread.ConcurrencyTest - pool-1-thread-4: 当前线程/总线程: [58/100], 处理的数据条数:100000
01:16:05.251 [pool-1-thread-6] INFO com.test.thread.ConcurrencyTest - pool-1-thread-6: 当前线程/总线程: [57/100], 处理的数据条数:100000
01:16:05.251 [pool-1-thread-2] INFO com.test.thread.ConcurrencyTest - pool-1-thread-2: 当前线程/总线程: [56/100], 处理的数据条数:100000
01:16:05.252 [pool-1-thread-3] INFO com.test.thread.ConcurrencyTest - pool-1-thread-3: 当前线程/总线程: [55/100], 处理的数据条数:100000
01:16:05.252 [pool-1-thread-2] INFO com.test.thread.ConcurrencyTest - pool-1-thread-2: 当前线程/总线程: [73/100], 处理的数据条数:100000
01:16:05.252 [pool-1-thread-6] INFO com.test.thread.ConcurrencyTest - pool-1-thread-6: 当前线程/总线程: [72/100], 处理的数据条数:100000
01:16:05.253 [pool-1-thread-4] INFO com.test.thread.ConcurrencyTest - pool-1-thread-4: 当前线程/总线程: [71/100], 处理的数据条数:100000
01:16:05.253 [pool-1-thread-7] INFO com.test.thread.ConcurrencyTest - pool-1-thread-7: 当前线程/总线程: [70/100], 处理的数据条数:100000
01:16:05.254 [pool-1-thread-8] INFO com.test.thread.ConcurrencyTest - pool-1-thread-8: 当前线程/总线程: [69/100], 处理的数据条数:100000
01:16:05.254 [pool-1-thread-5] INFO com.test.thread.ConcurrencyTest - pool-1-thread-5: 当前线程/总线程: [68/100], 处理的数据条数:100000
01:16:05.254 [pool-1-thread-1] INFO com.test.thread.ConcurrencyTest - pool-1-thread-1: 当前线程/总线程: [67/100], 处理的数据条数:100000
01:16:05.255 [pool-1-thread-10] INFO com.test.thread.ConcurrencyTest - pool-1-thread-10: 当前线程/总线程: [66/100], 处理的数据条数:100000
01:16:05.256 [pool-1-thread-9] INFO com.test.thread.ConcurrencyTest - pool-1-thread-9: 当前线程/总线程: [65/100], 处理的数据条数:100000
01:16:05.256 [pool-1-thread-10] INFO com.test.thread.ConcurrencyTest - pool-1-thread-10: 当前线程/总线程: [82/100], 处理的数据条数:100000
01:16:05.257 [pool-1-thread-1] INFO com.test.thread.ConcurrencyTest - pool-1-thread-1: 当前线程/总线程: [81/100], 处理的数据条数:100000
01:16:05.266 [pool-1-thread-5] INFO com.test.thread.ConcurrencyTest - pool-1-thread-5: 当前线程/总线程: [80/100], 处理的数据条数:100000
01:16:05.267 [pool-1-thread-8] INFO com.test.thread.ConcurrencyTest - pool-1-thread-8: 当前线程/总线程: [79/100], 处理的数据条数:100000
01:16:05.267 [pool-1-thread-7] INFO com.test.thread.ConcurrencyTest - pool-1-thread-7: 当前线程/总线程: [78/100], 处理的数据条数:100000
01:16:05.267 [pool-1-thread-4] INFO com.test.thread.ConcurrencyTest - pool-1-thread-4: 当前线程/总线程: [77/100], 处理的数据条数:100000
01:16:05.268 [pool-1-thread-6] INFO com.test.thread.ConcurrencyTest - pool-1-thread-6: 当前线程/总线程: [76/100], 处理的数据条数:100000
01:16:05.268 [pool-1-thread-2] INFO com.test.thread.ConcurrencyTest - pool-1-thread-2: 当前线程/总线程: [75/100], 处理的数据条数:100000
01:16:05.269 [pool-1-thread-3] INFO com.test.thread.ConcurrencyTest - pool-1-thread-3: 当前线程/总线程: [74/100], 处理的数据条数:100000
01:16:05.269 [pool-1-thread-2] INFO com.test.thread.ConcurrencyTest - pool-1-thread-2: 当前线程/总线程: [91/100], 处理的数据条数:100000
01:16:05.270 [pool-1-thread-6] INFO com.test.thread.ConcurrencyTest - pool-1-thread-6: 当前线程/总线程: [90/100], 处理的数据条数:100000
01:16:05.270 [pool-1-thread-4] INFO com.test.thread.ConcurrencyTest - pool-1-thread-4: 当前线程/总线程: [89/100], 处理的数据条数:100000
01:16:05.270 [pool-1-thread-7] INFO com.test.thread.ConcurrencyTest - pool-1-thread-7: 当前线程/总线程: [88/100], 处理的数据条数:100000
01:16:05.271 [pool-1-thread-8] INFO com.test.thread.ConcurrencyTest - pool-1-thread-8: 当前线程/总线程: [87/100], 处理的数据条数:100000
01:16:05.271 [pool-1-thread-5] INFO com.test.thread.ConcurrencyTest - pool-1-thread-5: 当前线程/总线程: [86/100], 处理的数据条数:100000
01:16:05.272 [pool-1-thread-1] INFO com.test.thread.ConcurrencyTest - pool-1-thread-1: 当前线程/总线程: [85/100], 处理的数据条数:100000
01:16:05.272 [pool-1-thread-10] INFO com.test.thread.ConcurrencyTest - pool-1-thread-10: 当前线程/总线程: [84/100], 处理的数据条数:100000
01:16:05.272 [pool-1-thread-9] INFO com.test.thread.ConcurrencyTest - pool-1-thread-9: 当前线程/总线程: [83/100], 处理的数据条数:100000
01:16:05.273 [pool-1-thread-1] INFO com.test.thread.ConcurrencyTest - pool-1-thread-1: 当前线程/总线程: [99/100], 处理的数据条数:100000
01:16:05.273 [pool-1-thread-5] INFO com.test.thread.ConcurrencyTest - pool-1-thread-5: 当前线程/总线程: [98/100], 处理的数据条数:100000
01:16:05.274 [pool-1-thread-8] INFO com.test.thread.ConcurrencyTest - pool-1-thread-8: 当前线程/总线程: [97/100], 处理的数据条数:100000
01:16:05.274 [pool-1-thread-7] INFO com.test.thread.ConcurrencyTest - pool-1-thread-7: 当前线程/总线程: [96/100], 处理的数据条数:100000
01:16:05.276 [pool-1-thread-4] INFO com.test.thread.ConcurrencyTest - pool-1-thread-4: 当前线程/总线程: [95/100], 处理的数据条数:100000
01:16:05.276 [pool-1-thread-6] INFO com.test.thread.ConcurrencyTest - pool-1-thread-6: 当前线程/总线程: [94/100], 处理的数据条数:100000
01:16:05.277 [pool-1-thread-2] INFO com.test.thread.ConcurrencyTest - pool-1-thread-2: 当前线程/总线程: [93/100], 处理的数据条数:100000
01:16:05.277 [pool-1-thread-3] INFO com.test.thread.ConcurrencyTest - pool-1-thread-3: 当前线程/总线程: [92/100], 处理的数据条数:100000
01:16:05.277 [main] INFO com.test.thread.ConcurrencyTest - Call getValueList2 success... ret: 10000000 size, threadCount: 100, costs time: 366 ms
====>> getValueList2 valueList.size: 10000000