关于大任务计算方法探索-----Java ForkJoin
本文以计算0~50000000000L累和为例，探索常用的几种方式所占用的CPU时间，当然耗用时间越少，说明性能越佳。
电脑配置为：i7-6700，16 GB内存，4核8线程。
1. java ForkJoin的invoke，采用任务对半分的方式。
2. 常规For循环。
3. java Forkjoin的invoke，采用平均分为8个任务方式。
4. 多线程Thread方式，采用平均分为8个任务方式，对应到每个线程里。
方式1和3的区别在于是否把任务合理分配到了8个CPU中。
class ForkJoinSumCalculate extends RecursiveTask<Long>{
private long start;
private long end;
private long thurshold; //可变临界值
public ForkJoinSumCalculate(long start, long end, long thurshold){
this.start = start;
this.end = end;
if(thurshold == 0){
int cpus = Runtime.getRuntime().availableProcessors();
if(cpus % 2 != 0){
cpus++;
}
this.thurshold = (end-start)/cpus;
// System.out.println("cpus="+cpus);
// System.out.println(this.thurshold);
}else{
this.thurshold = thurshold;
}
}
@Override
protected Long compute() {
long length = end-start;
if(length <= thurshold){
long sum = 0L;
for (long i=start;i<=end;i++){
sum += i;
}
return sum;
}else{
long middle = (start + end)/2;
ForkJoinSumCalculate left = new ForkJoinSumCalculate(start,middle,thurshold);
left.fork();
ForkJoinSumCalculate right = new ForkJoinSumCalculate(middle+1,end,thurshold);
right.fork();
return left.join()+right.join();
}
}
}
class ForkJoinTaskSimple extends RecursiveTask<Long> {
private long start;
private long end;
private boolean forkMe;
public ForkJoinTaskSimple(long start, long end, boolean forkMe) {
super();
this.end = end;
this.start = start;
this.forkMe = forkMe;
}
@Override
protected Long compute() {
if(!forkMe){
long sum = 0L;
for(long i=start;i<=end;i++){
sum += i;
}
return sum;
}else{
long start,end,length;
length = (this.end - this.start) / 8;
start = this.start;
end = length;
ForkJoinTaskSimple task1 = new ForkJoinTaskSimple(start,end,false);
start = end+1;
end += length;
ForkJoinTaskSimple task2 = new ForkJoinTaskSimple(start,end,false);
start = end+1;
end += length;
ForkJoinTaskSimple task3 = new ForkJoinTaskSimple(start,end,false);
start = end+1;
end += length;
ForkJoinTaskSimple task4 = new ForkJoinTaskSimple(start,end,false);
start = end+1;
end += length;
ForkJoinTaskSimple task5 = new ForkJoinTaskSimple(start,end,false);
start = end+1;
end += length;
ForkJoinTaskSimple task6 = new ForkJoinTaskSimple(start,end,false);
start = end+1;
end += length;
ForkJoinTaskSimple task7 = new ForkJoinTaskSimple(start,end,false);
start = end+1;
end = this.end;
ForkJoinTaskSimple task8 = new ForkJoinTaskSimple(start,end,false);
task1.fork();
task2.fork();
task3.fork();
task4.fork();
task5.fork();
task6.fork();
task7.fork();
task8.fork();
return task1.join()+task2.join()+task3.join()+task4.join()+task5.join()+task6.join()+task7.join()+task8.join();
}
}
}
class MultiThreadTask extends Thread{
private long s;
private long e;
public long sum;
public MultiThreadTask(long start, long end) {
s = start;
e = end;
sum = 0;
}
@Override
public synchronized void start() {
super.start();
}
@Override
public void run() {
for(long i=s;isum += i;
}
}
}
public class ForkJoinPoolTest {
public void run(){ //8秒左右
ForkJoinPool pool = new ForkJoinPool();
ForkJoinTask task = new ForkJoinSumCalculate(0,50000000000L,0);
Instant start = Instant.now();
long sum = pool.invoke(task);
Instant end = Instant.now();
System.out.println(sum);
System.out.println("使用ForkJoin,耗费时间为:"+Duration.between(start,end).toMillis());
}
public void normal(){ //13秒左右
long sum =0L;
Instant start = Instant.now();
for(long i=0;i<50000000000L;i++){
sum += i;
}
Instant end = Instant.now();
System.out.println(sum);
System.out.println("使用单线程循环,耗费时间为:"+Duration.between(start,end).toMillis());
}
public void divide(){ //11秒左右
ForkJoinPool pool = new ForkJoinPool();
ForkJoinTask task = new ForkJoinTaskSimple(0,50000000000L,true);
Instant start = Instant.now();
long sum = pool.invoke(task);
Instant end = Instant.now();
System.out.println(sum);
System.out.println("使用JorkJoin,但不是二分法分任务,耗费时间为:"+Duration.between(start,end).toMillis());
}
private long multiTheadInternal(long start,long end){
long length = (end-start)/8;
long tao = 0;
tao += length;
Thread thread1 = new MultiThreadTask(start,tao);
start = tao+1;
tao += length;
Thread thread2 = new MultiThreadTask(start,tao);
start = tao+1;
tao += length;
Thread thread3 = new MultiThreadTask(start,tao);
start = tao+1;
tao += length;
Thread thread4 = new MultiThreadTask(start,tao);
start = tao+1;
tao += length;
Thread thread5 = new MultiThreadTask(start,tao);
start = tao+1;
tao += length;
Thread thread6 = new MultiThreadTask(start,tao);
start = tao+1;
tao += length;
Thread thread7 = new MultiThreadTask(start,tao);
start = tao+1;
tao = end;
Thread thread8 = new MultiThreadTask(start,tao);
thread1.start();
thread2.start();
thread3.start();
thread4.start();
thread5.start();
thread6.start();
thread7.start();
thread8.start();
try {
thread1.join();
thread2.join();
thread3.join();
thread4.join();
thread5.join();
thread6.join();
thread7.join();
thread8.join();
}catch (InterruptedException e){
e.printStackTrace();
}
return ((MultiThreadTask) thread1).sum+((MultiThreadTask) thread2).sum+((MultiThreadTask) thread3).sum+((MultiThreadTask) thread4).sum+((MultiThreadTask) thread5).sum+
((MultiThreadTask) thread6).sum+((MultiThreadTask) thread7).sum+((MultiThreadTask) thread8).sum;
}
public void multiThread (){ //约为5秒
Instant start = Instant.now();
long sum = multiTheadInternal(0,50000000000L);
Instant end = Instant.now();
System.out.println(sum);
System.out.println("使用多线程,耗费时间为:"+Duration.between(start,end).toMillis());
}
}
运行结果如下：
CPU Intel i7-6700
-4378596987249509888 使用ForkJoin,耗费时间为:7829
-4378596987249509888 使用单线程循环,耗费时间为:14104
-4378596987249509888 使用JorkJoin,但不是二分法分任务,耗费时间为:11666
-4378596987249509888 使用多线程,耗费时间为:4669从上述结果中可以看出，其实多线程来合作处理大任务要比多进程处理大任务较好些，更比单线程好得多。以上测试均考虑最大任务数为CPU线程数，即尽量合理分配任务到CPU上。
添加使用java8 求和
public void java8(){
Instant start = Instant.now();
long sum = LongStream.rangeClosed(0,50000000000L).parallel().reduce(0L,Long::sum);
Instant end = Instant.now();
System.out.println(sum);
System.out.println("使用java8,耗费时间为:"+Duration.between(start,end).toMillis());
}结果：
CPU Intel i7-6700
-4378596987249509888 使用java8,耗费时间为:5235此时java8的求和仍然比上述多线程多花了近1秒的时间。
修改部分，以支持可变线程数：
class ForkJoinSumCalculate extends RecursiveTask<Long>{
private long start;
private long end;
private long thurshold; //可变临界值
private final static double MAGNIFICATION = 2; //此处倍率用于调节任务细分程度，下同
public ForkJoinSumCalculate(long start, long end, long thurshold){
this.start = start;
this.end = end;
if(thurshold == 0){
int cpus = Runtime.getRuntime().availableProcessors();
cpus = (int)(cpus * MAGNIFICATION);
if(cpus % 2 != 0){
cpus++;
}
this.thurshold = (end-start)/cpus;
// System.out.println("cpus="+cpus);
// System.out.println(this.thurshold);
}else{
this.thurshold = thurshold;
}
}
@Override
protected Long compute() {
long length = end-start;
if(length <= thurshold){
long sum = 0L;
for (long i=start;i<=end;i++){
sum += i;
}
return sum;
}else{
long middle = (start + end)/2;
ForkJoinSumCalculate left = new ForkJoinSumCalculate(start,middle,thurshold);
left.fork();
ForkJoinSumCalculate right = new ForkJoinSumCalculate(middle+1,end,thurshold);
right.fork();
return left.join()+right.join();
}
}
}
class ForkJoinTaskSimple extends RecursiveTask<Long> {
private long start;
private long end;
private boolean forkMe;
private final static double MAGNIFICATION = 2;
public ForkJoinTaskSimple(long start, long end, boolean forkMe) {
super();
this.end = end;
this.start = start;
this.forkMe = forkMe;
}
@Override
protected Long compute() {
if(!forkMe){
long sum = 0L;
for(long i=start;i<=end;i++){
sum += i;
}
return sum;
}else{
long start,end,length;
int cpus = Runtime.getRuntime().availableProcessors();
long sum;
cpus = (int)(cpus * MAGNIFICATION);
length = (this.end - this.start) / cpus;
start = this.start;
end = 0;
ArrayList joinTaskSimples = new ArrayList();
for(int i=0;iif(i == (cpus - 1)){
end = this.end;
}else{
end += length;
}
ForkJoinTaskSimple joinTaskSimple = new ForkJoinTaskSimple(start,end,false);
joinTaskSimple.fork();
start = end+1;
joinTaskSimples.add(joinTaskSimple);
}
sum = 0;
for(ForkJoinTaskSimple joinTaskSimple : joinTaskSimples){
sum += joinTaskSimple.join();
}
return sum;
}
}
}
class MultiThreadTask extends Thread{
private long s;
private long e;
public long sum;
public MultiThreadTask(long start, long end) {
s = start;
e = end;
sum = 0;
}
@Override
public synchronized void start() {
super.start();
}
@Override
public void run() {
for(long i=s;i<=e;i++){
sum += i;
}
}
}
public class ForkJoinPoolTest {
public void run(){ //8秒左右
ForkJoinPool pool = new ForkJoinPool();
ForkJoinTask task = new ForkJoinSumCalculate(0,50000000000L,0);
Instant start = Instant.now();
long sum = pool.invoke(task);
Instant end = Instant.now();
System.out.println(sum);
System.out.println("使用ForkJoin,耗费时间为:"+Duration.between(start,end).toMillis());
}
public void normal(){ //13秒左右
long sum =0L;
Instant start = Instant.now();
for(long i=0;i<=50000000000L;i++){
sum += i;
}
Instant end = Instant.now();
System.out.println(sum);
System.out.println("使用单线程循环,耗费时间为:"+Duration.between(start,end).toMillis());
}
public void divide(){ //11秒左右
ForkJoinPool pool = new ForkJoinPool();
ForkJoinTask task = new ForkJoinTaskSimple(0,50000000000L,true);
Instant start = Instant.now();
long sum = pool.invoke(task);
Instant end = Instant.now();
System.out.println(sum);
System.out.println("使用JorkJoin,但不是二分法分任务,耗费时间为:"+Duration.between(start,end).toMillis());
}
private final static double MAGNIFICATION = 2;
private long multiTheadInternal(long start,long end){
long length;
long tao = 0;
int cpus = Runtime.getRuntime().availableProcessors();
cpus = (int)(cpus * MAGNIFICATION);
length = (end - start) / cpus;
ArrayList threads = new ArrayList();
for(int i=0;iif(i == (cpus - 1)){
tao = end;
}else {
tao += length;
}
MultiThreadTask thread = new MultiThreadTask(start,tao);
thread.start();
threads.add(thread);
start = tao+1;
}
long sum = 0;
for(MultiThreadTask threadTask : threads ){
try {
threadTask.join();
sum += threadTask.sum;
}catch (InterruptedException e){
e.printStackTrace();
}
}
return sum;
}
public void multiThread (){ //约为5秒
Instant start = Instant.now();
long sum = multiTheadInternal(0,50000000000L);
Instant end = Instant.now();
System.out.println(sum);
System.out.println("使用多线程,耗费时间为:"+Duration.between(start,end).toMillis());
}
public void java8(){
Instant start = Instant.now();
long sum = LongStream.rangeClosed(0,50000000000L).parallel().reduce(0L,Long::sum);
Instant end = Instant.now();
System.out.println(sum);
System.out.println("使用java8,耗费时间为:"+Duration.between(start,end).toMillis());
}
} 结果如下：
CPU Intel i7-6700
-4378596987249509888 使用ForkJoin,耗费时间为:6920
-4378596987249509888 使用单线程循环,耗费时间为:13314
-4378596987249509888 使用JorkJoin,但不是二分法分任务,耗费时间为:6614
-4378596987249509888 使用多线程,耗费时间为:4700
-4378596987249509888 使用java8,耗费时间为:5123试验中发现，调节倍率，使得任务数多于CPU实际线程数在一定程度上能加快求解速度。