Linux shell 实现队列并发任务

Linux shell 实现队列并发任务

新建测试脚本：

vim concurrent-demo.sh

#!/bin/bash

#/*
# * Copyright 2017 ~ 2025 the original author or authors. 
# *
# * Licensed under the Apache License, Version 2.0 (the "License");
# * you may not use this file except in compliance with the License.
# * You may obtain a copy of the License at
# *
# *      http://www.apache.org/licenses/LICENSE-2.0
# *
# * Unless required by applicable law or agreed to in writing, software
# * distributed under the License is distributed on an "AS IS" BASIS,
# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# * See the License for the specific language governing permissions and
# * limitations under the License.
# */

# 执行任务的函数
function run() {
  local taskId=$1
  local pfileFD=$2
  echo "$(date '+%Y-%m-%d %H:%M:%S') - task ${taskId} running ..."
  sleep 2
  echo "$(date '+%Y-%m-%d %H:%M:%S') - task ${taskId} completed."
}

function executeTest() {
  # 1. 定义相关变量
  local concurrent=3 # 定义并发数, 在shell中叫 threadNum 不太合适, 因为是fork的子进程.
  local pfile="/tmp/test.fifo" # 定义管道文件路径
  local pfileFD="1234" # 定义管道文件的FD(文件句柄)
  #local pfileFD="$RANDOM" # 若本函数需多次调用时, 测试：可使用shell内置random函数, 范围:[0-32767)，不建议在生成使用，生成环境建议使用seq序列，因为当CPU过高时可能导致系统随机源不足产生重复随机数，这会导致一系列问题

  # 2. 创建Linux管道文件(写读数据会先进先出的队列, 类似java中Queue).
  [ ! -p "$pfile" ] && mkfifo $pfile
  eval "exec $pfileFD<>$pfile" # 创建文件句柄(fd), 除系统内置的0/1/2外的数字作为fd的标识即可, > 和 < 分别代表写读.
  rm -f $pfile # 创建完fd后可删除管道文件

  # 3. 初始化并发数, 即添加并发相同数量的换行符到FD, 用于后续读取换行符来控制并发数. (echo 默认标准输出换行符)
  eval "for ((i=0;i<${concurrent};i++)); do echo ; done >&${pfileFD}"

  # 4. 异步提交任务, 假设tasks数为10.
  for i in `seq 1 10`; do
    eval "read -u${pfileFD}" # 每次读一个, 读完一个就少一个(fifo队列)
    {
      run "$i" "$pfileFD" # (异步)调用真正执行任务的函数
      eval "echo >&${pfileFD}" # 每执行完一个task, 继续又添加一个换行符, 目的是为了永远保持fd里有concurrent个标识, 直到任务执行完.
    } &
  done
  wait
  eval "exec ${pfileFD}>&-" # 关闭fd(管道文件)
  exit 0
}
executeTest

执行测试：

2021-06-07 10:01:18 - task 1 running ...
2021-06-07 10:01:18 - task 2 running ...
2021-06-07 10:01:18 - task 3 running ...
2021-06-07 10:01:21 - task 1 completed.
2021-06-07 10:01:21 - task 2 completed.
2021-06-07 10:01:21 - task 3 completed.
2021-06-07 10:01:21 - task 4 running ...
2021-06-07 10:01:21 - task 5 running ...
2021-06-07 10:01:21 - task 6 running ...
2021-06-07 10:01:24 - task 4 completed.
2021-06-07 10:01:24 - task 5 completed.
2021-06-07 10:01:24 - task 6 completed.
2021-06-07 10:01:24 - task 7 running ...
2021-06-07 10:01:24 - task 8 running ...
2021-06-07 10:01:24 - task 9 running ...
2021-06-07 10:01:27 - task 8 completed.
2021-06-07 10:01:27 - task 7 completed.
2021-06-07 10:01:27 - task 9 completed.
2021-06-07 10:01:27 - task 10 running ...
2021-06-07 10:01:30 - task 10 completed.

原文参考：https://blogs.wl4g.com/archives/292