Linux schedule 5、EAS(Energy-Aware Scheduling)

5、EAS(Energy-Aware Scheduling)

5.1、smp rebalance

通过搜索关键字“energy_aware()”，来查看EAS对smp负载均衡的影响。

可以看到EAS对负载均衡的策略是这样的：在overutilized的情况下，使用传统的smp/hmp负载均衡方法；在非overutilized的情况下，使用eas的均衡方法。

EAS的负载均衡和原有方法的区别有几部分：

• 1、在EAS使能且没有overutilized的情况下，hmp负载均衡不工作；
• 2、在EAS使能且没有overutilized的情况下，smp负载均衡不工作；
• 3、在EAS使能且没有overutilized的情况下，EAS的主要工作集中在进程唤醒/新建时选择运行cpu上select_task_rq_fair()；

5.1.1、rebalance_domains()

• 1、在EAS使能且没有overutilized的情况下，hmp负载均衡不使能；

static void run_rebalance_domains(struct softirq_action *h)
{
    struct rq *this_rq = this_rq();
    enum cpu_idle_type idle = this_rq->idle_balance ?
                        CPU_IDLE : CPU_NOT_IDLE;
    int this_cpu = smp_processor_id();

    /* bypass load balance of HMP if EAS consideration */
    if ((!energy_aware() && sched_feat(SCHED_HMP)) ||
            (hybrid_support() && cpu_rq(this_cpu)->rd->overutilized))
        hmp_force_up_migration(this_cpu);

    /*
     * If this cpu has a pending nohz_balance_kick, then do the
     * balancing on behalf of the other idle cpus whose ticks are
     * stopped. Do nohz_idle_balance *before* rebalance_domains to
     * give the idle cpus a chance to load balance. Else we may
     * load balance only within the local sched_domain hierarchy
     * and abort nohz_idle_balance altogether if we pull some load.
     */
    nohz_idle_balance(this_rq, idle);
    rebalance_domains(this_rq, idle);
}

• 2、在load_balance() -> find_busiest_group()中，如果在EAS使能且没有overutilized的情况下，不进行常规的smp负载均衡；

static struct sched_group *find_busiest_group(struct lb_env *env)
{

    if (energy_aware() && !env->dst_rq->rd->overutilized && !same_clus)
        goto out_balanced;

out_balanced:
    env->imbalance = 0;
    return NULL;    
}

5.1.2、select_task_rq_fair()

参考4.1.2.3、select_task_rq_fair()这一节的详细描述。

5.2、cpufreq_sched/schedutil governor

sched governor比较传统interactive governor有以下优点：

• 1、负载变化的时间更快。interactive是20ms统计一次负载，而sched governor是在schedule_tick()中更新负载，tick的时间更短；
• 2、interactive的负载计算有问题：历史负载没有老化；历史频率除以现在频率不合理；

interactive governor的主要思想就是综合rt、cfs的负载，判断当前freq的capacity是否满足需求，是否需要调频。

5.2.1、rt request

针对CONFIG_CPU_FREQ_GOV_SCHED，rt有3条关键计算路径：

• 1、rt负载的(rq->rt_avg)的累加：scheduler_tick() -> task_tick_rt() -> update_curr_rt() -> sched_rt_avg_update()

rq->rt_avg = 累加时间分量 * 当前frq分量(最大1024)

static inline void sched_rt_avg_update(struct rq *rq, u64 rt_delta)
{
    rq->rt_avg += rt_delta * arch_scale_freq_capacity(NULL, cpu_of(rq));
}

• 2、rt负载的老化：scheduler_tick() -> __update_cpu_load() -> __update_cpu_load() -> sched_avg_update()
  或者scheduler_tick() -> task_tick_rt() -> sched_rt_update_capacity_req() -> sched_avg_update()

rq->rt_avg的老化比较简单，每个period老化1/2。

void sched_avg_update(struct rq *rq)
{
    /* (1) 默认老化周期为1s/2 = 500ms */
    s64 period = sched_avg_period();

    while ((s64)(rq_clock(rq) - rq->age_stamp) > period) {
        /*
         * Inline assembly required to prevent the compiler
         * optimising this loop into a divmod call.
         * See __iter_div_u64_rem() for another example of this.
         */
        asm("" : "+rm" (rq->age_stamp));
        rq->age_stamp += period;
        /* (2) 每个老化周期，负载老化为原来的1/2 */
        rq->rt_avg /= 2;
        rq->dl_avg /= 2;
    }
}

|→

static inline u64 sched_avg_period(void)
{
    /* (1.1) 老化周期 = sysctl_sched_time_avg/2 = 500ms */
    return (u64)sysctl_sched_time_avg * NSEC_PER_MSEC / 2;
}

• 3、rt request的更新：scheduler_tick() -> task_tick_rt() -> sched_rt_update_capacity_req() -> set_rt_cpu_capacity()

rt request的计算有点粗糙: request = rt_avg/(sched_avg_period() + delta)，rt_avg中没有加上delta时间的负载。

static void sched_rt_update_capacity_req(struct rq *rq)
{
    u64 total, used, age_stamp, avg;
    s64 delta;

    if (!sched_freq())
        return;

    /* (1) 最新的负载进行老化 */
    sched_avg_update(rq);
    /*
     * Since we're reading these variables without serialization make sure
     * we read them once before doing sanity checks on them.
     */
    age_stamp = READ_ONCE(rq->age_stamp);
    /* (2) avg=老化后的负载 */
    avg = READ_ONCE(rq->rt_avg);
    delta = rq_clock(rq) - age_stamp;

    if (unlikely(delta < 0))
        delta = 0;

    /* (3) total时间=一个老化周期+上次老化剩余时间 */
    total = sched_avg_period() + delta;

    /* (4) avg/total=request，(最大频率=1024) */
    used = div_u64(avg, total);
    if (unlikely(used > SCHED_CAPACITY_SCALE))
        used = SCHED_CAPACITY_SCALE;

    /* (5) update request */
    set_rt_cpu_capacity(rq->cpu, true, (unsigned long)(used), SCHE_ONESHOT);
}

|→

static inline void set_rt_cpu_capacity(int cpu, bool request,
                       unsigned long capacity,
                    int type)
{
#ifdef CONFIG_CPU_FREQ_SCHED_ASSIST
    if (true) {
#else
    if (per_cpu(cpu_sched_capacity_reqs, cpu).rt != capacity) {
#endif
        /* (5.1) 把RT负载更新到per_cpu(cpu_sched_capacity_reqs, cpu).rt */
        per_cpu(cpu_sched_capacity_reqs, cpu).rt = capacity;
        update_cpu_capacity_request(cpu, request, type);
    }
}

5.2.2、cfs request

同样，cfs也有3条关键计算路径：

• 1、cfs负载的(rq->rt_avg)的累加：scheduler_tick() -> task_tick_fair() -> entity_tick() -> update_load_avg()
• 2、cfs负载的老化：scheduler_tick() -> task_tick_fair() -> entity_tick() -> update_load_avg()
• 3、cfs request的更新：scheduler_tick() -> sched_freq_tick() -> set_cfs_cpu_capacity()

static void sched_freq_tick(int cpu)
{
    struct sched_capacity_reqs *scr;
    unsigned long capacity_orig, capacity_curr;
    unsigned long capacity_req;
    struct sched_domain *sd = rcu_dereference(per_cpu(sd_ea, cpu));

    if (!sched_freq())
        return;

    capacity_orig = capacity_orig_of(cpu);
    capacity_curr = capacity_curr_of(cpu);

    /* (1) 如果当前频率已经是最高频率，直接返回 
        目前只支持频率从低往高调整？
     */
    if (capacity_curr == capacity_orig)
        return;

    /*
     * To make free room for a task that is building up its "real"
     * utilization and to harm its performance the least, request
     * a jump to bigger OPP as soon as the margin of free capacity is
     * impacted (specified by capacity_margin).
     */
    scr = &per_cpu(cpu_sched_capacity_reqs, cpu);

    /* (2) 计算最新的(cfs capacity+ rt capacity) * (1126/1024) 
        放大一些，等于对capacity的需求request
        ooooo这里的计算有问题：cpu_util(cpu)是带capacity分量的，而scr->rt是不带capacity分量的，不能直接相加？
     */
    /* capacity_req which includes RT loading & capacity_margin */
    capacity_req = sum_capacity_reqs(cpu_util(cpu), scr);

    /* (3) 如果capacity request大于当前频率的capacity */
    if (capacity_curr <= capacity_req) {
        if (sd) {### 5.3.1、WALT的负载计算
            const struct sched_group_energy *const sge = sd->groups->sge;
            int nr_cap_states = sge->nr_cap_states;
            int idx, tmp_idx;
            int opp_jump_step;

            for (idx = 0; idx < nr_cap_states; idx++) {
                if (sge->cap_states[idx].cap > capacity_curr+1)
                    break;
            }

            /* (4) 尝试计算一个合理的频率等级来满足capacity request */
            if (idx < nr_cap_states/3)
                opp_jump_step = 2; /* far step */
            else
                opp_jump_step = 1; /* near step */

            tmp_idx = idx + (opp_jump_step - 1);

            idx = tmp_idx > (nr_cap_states - 1) ?
                (nr_cap_states - 1) : tmp_idx;

            if (idx)
                capacity_req = (sge->cap_states[idx].cap +
                        sge->cap_states[idx-1].cap)/2;
            else
                /* should not arrive here!*/
                capacity_req = sge->cap_states[idx].cap + 2;
        }

        /* (5) 去掉request中的capacity分量，转化成scale_freq */
        /* convert scale-invariant capacity */
        capacity_req = capacity_req * SCHED_CAPACITY_SCALE / capacity_orig_of(cpu);


        /* (6) update request， 
            ooooo这里有问题啊：capacity_req计算的时候是按照rt+cfs加起来计算的，怎么有把结果配置给了scr->cfs？
         */
        /*
         * If free room ~5% impact, jump to 1 more index hihger OPP.
         * Whatever it should be better than capacity_max.
         */
        set_cfs_cpu_capacity(cpu, true, capacity_req, SCHE_ONESHOT);
    }
}

|→

static inline void set_cfs_cpu_capacity(int cpu, bool request,
                    unsigned long capacity, int type)
{
#ifdef CONFIG_CPU_FREQ_SCHED_ASSIST
    if (true) {
#else
    if (per_cpu(cpu_sched_capacity_reqs, cpu).cfs != capacity) {
#endif
        /* (6.1) 把RT负载更新到per_cpu(cpu_sched_capacity_reqs, cpu).cfs */
        per_cpu(cpu_sched_capacity_reqs, cpu).cfs = capacity;
        update_cpu_capacity_request(cpu, request, type);
    }
}

5.2.3、freq target

void update_cpu_capacity_request(int cpu, bool request, int type)
{
    unsigned long new_capacity;
    struct sched_capacity_reqs *scr;

    /* The rq lock serializes access to the CPU's sched_capacity_reqs. */
    lockdep_assert_held(&cpu_rq(cpu)->lock);

    scr = &per_cpu(cpu_sched_capacity_reqs, cpu);

    /* (1) 综合rt、cfs的request */
    new_capacity = scr->cfs + scr->rt;
    new_capacity = new_capacity * capacity_margin_dvfs
        / SCHED_CAPACITY_SCALE;
    new_capacity += scr->dl;

#ifndef CONFIG_CPU_FREQ_SCHED_ASSIST
    if (new_capacity == scr->total)
        return;
#endif

    scr->total = new_capacity;
    if (request)
        update_fdomain_capacity_request(cpu, type);
}

|→

static void update_fdomain_capacity_request(int cpu, int type)
{
    unsigned int freq_new, cpu_tmp;
    struct gov_data *gd;
    unsigned long capacity = 0;
#ifdef CONFIG_CPU_FREQ_SCHED_ASSIST
    int cid = arch_get_cluster_id(cpu);
    struct cpumask cls_cpus;
#endif
    struct cpufreq_policy *policy = NULL;

    /*
     * Avoid grabbing the policy if possible. A test is still
     * required after locking the CPU's policy to avoid racing
     * with the governor changing.
     */
    if (!per_cpu(enabled, cpu))
        return;

#ifdef CONFIG_CPU_FREQ_SCHED_ASSIST
    gd = g_gd[cid];

    /* bail early if we are throttled */
    if (ktime_before(ktime_get(), gd->throttle))
        goto out;

    arch_get_cluster_cpus(&cls_cpus, cid);

    /* find max capacity requested by cpus in this policy */
    for_each_cpu(cpu_tmp, &cls_cpus) {
        struct sched_capacity_reqs *scr;

        if (!cpu_online(cpu_tmp))
            continue;

        scr = &per_cpu(cpu_sched_capacity_reqs, cpu_tmp);
        capacity = max(capacity, scr->total);
    }

    freq_new = capacity * arch_scale_get_max_freq(cpu) >> SCHED_CAPACITY_SHIFT;
#else
    if (likely(cpu_online(cpu)))
        policy = cpufreq_cpu_get(cpu);

    if (IS_ERR_OR_NULL(policy))
        return;

    if (policy->governor != &cpufreq_gov_sched ||
        !policy->governor_data)
        goto out;

    gd = policy->governor_data;

    /* bail early if we are throttled */
    if (ktime_before(ktime_get(), gd->throttle))
        goto out;

    /* (1) 选择policy cpus中最大的capacity */
    /* find max capacity requested by cpus in this policy */
    for_each_cpu(cpu_tmp, policy->cpus) {
        struct sched_capacity_reqs *scr;

        scr = &per_cpu(cpu_sched_capacity_reqs, cpu_tmp);
        capacity = max(capacity, scr->total);
    }

    /* (2) 把相对capacity转换成绝对freq */
    /* Convert the new maximum capacity request into a cpu frequency */
    freq_new = capacity * policy->max >> SCHED_CAPACITY_SHIFT;

    if (freq_new == gd->requested_freq)
        goto out;

#endif /* !CONFIG_CPU_FREQ_SCHED_ASSIST */

    gd->requested_freq = freq_new;
    gd->target_cpu = cpu;

    /* (3) 使用irq_work或者直接配置的方式来配置新的频率 
        直接在schedule_tick()中配置频率的方式估计不会使用，因为这样会阻塞中断
     */
    /*
     * Throttling is not yet supported on platforms with fast cpufreq
     * drivers.
     */
    if (cpufreq_driver_slow)
        irq_work_queue_on(&gd->irq_work, cpu);
    else
        cpufreq_sched_try_driver_target(cpu, policy, freq_new, type);

out:
    if (policy)
        cpufreq_cpu_put(policy);
}

5.3、WALT(Windows Assisted Load Tracking)

在qualcomm 8898中，使用了WALT作为负载计算方法，也使用了自己的负载均衡算法来使用WALT负载。代码中使用CONFIG_SCHED_HMP来标示qualcomm自己负载均衡方法。

5.3.1、WALT的负载计算

Walt的本质也是时间窗分量，结合freq分量、capacity分量等一起表达的一个负载相对值。我们首先来看看几个分量的计算方法。

• 1、cluster->efficiency计算：从dts中读取，我们可以看到，四个小核的efficiency是1024，四个大核的efficiency是1638；

static struct sched_cluster *alloc_new_cluster(const struct cpumask *cpus)
{

    cluster->efficiency = arch_get_cpu_efficiency(cpumask_first(cpus));

    if (cluster->efficiency > max_possible_efficiency)
        max_possible_efficiency = cluster->efficiency;
    if (cluster->efficiency < min_possible_efficiency)
        min_possible_efficiency = cluster->efficiency;

}

unsigned long arch_get_cpu_efficiency(int cpu)
{
    return per_cpu(cpu_efficiency, cpu);
}

static void __init parse_dt_cpu_power(void)
{

        /*
         * The CPU efficiency value passed from the device tree
         * overrides the value defined in the table_efficiency[]
         */
        if (of_property_read_u32(cn, "efficiency", &efficiency) < 0) {


        }

        per_cpu(cpu_efficiency, cpu) = efficiency;

}

从 arch/arm64/boot/dts/qcom/sdm660.dtsi读到"efficiency"配置：

    cpus {
        #address-cells = <2>;
        #size-cells = <0>;

        CPU0: cpu@0 {

            efficiency = <1024>;

        };

        CPU1: cpu@1 {

            efficiency = <1024>;

        };

        CPU2: cpu@2 {

            efficiency = <1024>;

        };

        CPU3: cpu@3 {

            efficiency = <1024>;

        };

        CPU4: cpu@100 {

            efficiency = <1638>;

        };

        CPU5: cpu@101 {

            efficiency = <1638>;

        };

        CPU6: cpu@102 {

            efficiency = <1638>;

        };

        CPU7: cpu@103 {

            efficiency = <1638>;

        };

        cpu-map {
            cluster0 {
                core0 {
                    cpu = <&CPU0>;
                };

                core1 {
                    cpu = <&CPU1>;
                };

                core2 {
                    cpu = <&CPU2>;
                };

                core3 {
                    cpu = <&CPU3>;
                };
            };

            cluster1 {
                core0 {
                    cpu = <&CPU4>;
                };

                core1 {
                    cpu = <&CPU5>;
                };

                core2 {
                    cpu = <&CPU6>;
                };

                core3 {
                    cpu = <&CPU7>;
                };
            };
        };
    }

• 2、cluster->capacity：计算和最小值的正比：capacity = 1024 * (cluster->efficiency*cluster_max_freq(cluster)) / (min_possible_efficiency*min_max_freq)
• 3、cluster->max_possible_capacity：计算和最小值的正比：capacity = 1024 * (cluster->efficiency*cluster->max_possible_freq) / (min_possible_efficiency*min_max_freq)
• 4、cluster->load_scale_factor：计算和最大值的反比：lsf = 1024 * (max_possible_efficiency*max_possible_freq) / (cluster->efficiency*cluster_max_freq(cluster))
• 5、cluster->exec_scale_factor：计算和最大值的正比：exec_scale_factor = 1024 * cluster->efficiency / max_possible_efficiency

static void update_all_clusters_stats(void)
{
    struct sched_cluster *cluster;
    u64 highest_mpc = 0, lowest_mpc = U64_MAX;

    pre_big_task_count_change(cpu_possible_mask);

    for_each_sched_cluster(cluster) {
        u64 mpc;

        /* (1) 计算cluster->capacity：capacity = efficiency * cluster_max_freq
            最小值：min_possible_efficiency*min_max_freq = 1024，
            计算和最小值的正比：capacity = 1024 * (cluster->efficiency*cluster_max_freq(cluster)) / (min_possible_efficiency*min_max_freq)
         */
        cluster->capacity = compute_capacity(cluster);

        /* (2) 计算cluster->max_possible_capacity：capacity = efficiency * cluster_max_freq
            最小值：min_possible_efficiency*min_max_freq = 1024，
            计算和最小值的正比：capacity = 1024 * (cluster->efficiency*cluster->max_possible_freq) / (min_possible_efficiency*min_max_freq)
         */
        mpc = cluster->max_possible_capacity =
            compute_max_possible_capacity(cluster);

        /* (3) 计算cluster->load_scale_factor： lsf = efficiency * cluster_max_freq
            最大值：max_possible_efficiency*max_possible_freq = 1024
            计算和最大值的反比：lsf = 1024 * (max_possible_efficiency*max_possible_freq) / (cluster->efficiency*cluster_max_freq(cluster))
         */
        cluster->load_scale_factor = compute_load_scale_factor(cluster);

        /* (4) 计算cluster->exec_scale_factor：
            最大值：max_possible_efficiency = 1024
            计算和最大值的正比：exec_scale_factor = 1024 * cluster->efficiency / max_possible_efficiency
         */
        cluster->exec_scale_factor =
            DIV_ROUND_UP(cluster->efficiency * 1024,
                     max_possible_efficiency);

        if (mpc > highest_mpc)
            highest_mpc = mpc;

        if (mpc < lowest_mpc)
            lowest_mpc = mpc;
    }

    max_possible_capacity = highest_mpc;
    min_max_possible_capacity = lowest_mpc;

    __update_min_max_capacity();
    sched_update_freq_max_load(cpu_possible_mask);
    post_big_task_count_change(cpu_possible_mask);
}

|→

static int compute_capacity(struct sched_cluster *cluster)
{
    int capacity = 1024;

    capacity *= capacity_scale_cpu_efficiency(cluster);
    capacity >>= 10;

    capacity *= capacity_scale_cpu_freq(cluster);
    capacity >>= 10;

    return capacity;
}

||→

/*
 * Return 'capacity' of a cpu in reference to "least" efficient cpu, such that
 * least efficient cpu gets capacity of 1024
 */
static unsigned long
capacity_scale_cpu_efficiency(struct sched_cluster *cluster)
{
    return (1024 * cluster->efficiency) / min_possible_efficiency;
}

||→

/*
 * Return 'capacity' of a cpu in reference to cpu with lowest max_freq
 * (min_max_freq), such that one with lowest max_freq gets capacity of 1024.
 */
static unsigned long capacity_scale_cpu_freq(struct sched_cluster *cluster)
{
    return (1024 * cluster_max_freq(cluster)) / min_max_freq;
}

|→

static int compute_load_scale_factor(struct sched_cluster *cluster)
{
    int load_scale = 1024;

    /*
     * load_scale_factor accounts for the fact that task load
     * is in reference to "best" performing cpu. Task's load will need to be
     * scaled (up) by a factor to determine suitability to be placed on a
     * (little) cpu.
     */
    load_scale *= load_scale_cpu_efficiency(cluster);
    load_scale >>= 10;

    load_scale *= load_scale_cpu_freq(cluster);
    load_scale >>= 10;

    return load_scale;
}

||→

/*
 * Return load_scale_factor of a cpu in reference to "most" efficient cpu, so
 * that "most" efficient cpu gets a load_scale_factor of 1
 */
static inline unsigned long
load_scale_cpu_efficiency(struct sched_cluster *cluster)
{
    return DIV_ROUND_UP(1024 * max_possible_efficiency,
                cluster->efficiency);
}

||→

/*
 * Return load_scale_factor of a cpu in reference to cpu with best max_freq
 * (max_possible_freq), so that one with best max_freq gets a load_scale_factor
 * of 1.
 */
static inline unsigned long load_scale_cpu_freq(struct sched_cluster *cluster)
{
    return DIV_ROUND_UP(1024 * max_possible_freq,
               cluster_max_freq(cluster));
}

• 6、cluster->max_power_cost：cluster的最大功耗 = voltage^2 * frequence
• 7、cluster->min_power_cost：cluster的最小功耗 = voltage^2 * frequence

static void sort_clusters(void)
{

    for_each_sched_cluster(cluster) {
        cluster->max_power_cost = power_cost(cluster_first_cpu(cluster),
                                   max_task_load());
        cluster->min_power_cost = power_cost(cluster_first_cpu(cluster),
                                   0);

        if (cluster->max_power_cost > tmp_max)
            tmp_max = cluster->max_power_cost;
    }
    max_power_cost = tmp_max;


}

|→

unsigned int power_cost(int cpu, u64 demand)
{
    int first, mid, last;
    struct cpu_pwr_stats *per_cpu_info = get_cpu_pwr_stats();
    struct cpu_pstate_pwr *costs;
    struct freq_max_load *max_load;
    int total_static_pwr_cost = 0;
    struct rq *rq = cpu_rq(cpu);
    unsigned int pc;

    if (!per_cpu_info || !per_cpu_info[cpu].ptable)
        /*
         * When power aware scheduling is not in use, or CPU
         * power data is not available, just use the CPU
         * capacity as a rough stand-in for real CPU power
         * numbers, assuming bigger CPUs are more power
         * hungry.
         */
        return cpu_max_possible_capacity(cpu);

    rcu_read_lock();
    max_load = rcu_dereference(per_cpu(freq_max_load, cpu));
    if (!max_load) {
        pc = cpu_max_possible_capacity(cpu);
        goto unlock;
    }

    costs = per_cpu_info[cpu].ptable;

    if (demand <= max_load->freqs[0].hdemand) {
        pc = costs[0].power;
        goto unlock;
    } else if (demand > max_load->freqs[max_load->length - 1].hdemand) {
        pc = costs[max_load->length - 1].power;
        goto unlock;
    }

    first = 0;
    last = max_load->length - 1;
    mid = (last - first) >> 1;
    while (1) {
        if (demand <= max_load->freqs[mid].hdemand)
            last = mid;
        else
            first = mid;

        if (last - first == 1)
            break;
        mid = first + ((last - first) >> 1);
    }

    pc = costs[last].power;

unlock:
    rcu_read_unlock();

    if (idle_cpu(cpu) && rq->cstate) {
        total_static_pwr_cost += rq->static_cpu_pwr_cost;
        if (rq->cluster->dstate)
            total_static_pwr_cost +=
                rq->cluster->static_cluster_pwr_cost;
    }

    return pc + total_static_pwr_cost;

}


/* qualcom的power的计算公式 = voltage^2 * frequence */
static int msm_get_power_values(int cpu, struct cpu_static_info *sp)
{
    int i = 0, j;
    int ret = 0;
    uint64_t power;

    /* Calculate dynamic power spent for every frequency using formula:
     * Power = V * V * f
     * where V = voltage for frequency
     *       f = frequency
     * */
    sp->power = allocate_2d_array_uint32_t(sp->num_of_freqs);
    if (IS_ERR_OR_NULL(sp->power))
        return PTR_ERR(sp->power);

    for (i = 0; i < TEMP_DATA_POINTS; i++) {
        for (j = 0; j < sp->num_of_freqs; j++) {
            power = sp->voltage[j] *
                        sp->table[j].frequency;
            do_div(power, 1000);
            do_div(power, 1000);
            power *= sp->voltage[j];
            do_div(power, 1000);
            sp->power[i][j] = power;
        }
    }
    return ret;
}

5.3.1.1、update_task_ravg()

walt关于进程的负载计算流程如下：

• 1、把时间分成一个个window窗口，累加时间时，需要综合efficiency和freq分量(也就是capacity)：delta = delta_time * (curr_freq/max_possible_freq) * (cluster->efficiency/max_possible_efficiency);

static inline u64 scale_exec_time(u64 delta, struct rq *rq)
{
    u32 freq;

    /* curr_freq / max_possible_freq */
    freq = cpu_cycles_to_freq(rq->cc.cycles, rq->cc.time);
    delta = DIV64_U64_ROUNDUP(delta * freq, max_possible_freq);

    /* exec_scale_factor = cluster->efficiency / max_possible_efficiency */
    delta *= rq->cluster->exec_scale_factor;
    delta >>= 10;

    return delta;
}

• 2、统计runnable状态的时间：account_busy_for_task_demand()屏蔽掉runnable以外的其他状态的时间统计；

static int account_busy_for_task_demand(struct task_struct *p, int event)
{
    /*
     * No need to bother updating task demand for exiting tasks
     * or the idle task.
     */
    /* (3.1.1) exit、idle任务不计入统计 */
    if (exiting_task(p) || is_idle_task(p))
        return 0;

    /*
     * When a task is waking up it is completing a segment of non-busy
     * time. Likewise, if wait time is not treated as busy time, then
     * when a task begins to run or is migrated, it is not running and
     * is completing a segment of non-busy time.
     */
    /* (3.1.2) 任务被wakeup，之前的等待时间不计入统计 
        SCHED_ACCOUNT_WAIT_TIME用来控制ruannable的等待时间是否计入统计，默认是计入的
     */
    if (event == TASK_WAKE || (!SCHED_ACCOUNT_WAIT_TIME &&
             (event == PICK_NEXT_TASK || event == TASK_MIGRATE)))
        return 0;

    return 1;
}

• 3、在统计时间时，可能碰到的3种组合情况：

• 4、如果一个window还没有完成，会逐步累加时间到p->ravg.sum；如果一个window完成，存储最新window负载到p->ravg.sum_history[RAVG_HIST_SIZE_MAX]中，sum_history[]一共有5个槽位；系统根据sched_window_stats_policy选择策略(RECENT、MAX、AVG、MAX_RECENT_AVG)，根据sum_history[]计算选择一个合适的值作为进程负载p->ravg.demand；同时根据sum_history[]的计算进程的负载预测p->ravg.pred_demand；
  

• 5、walt的task级别的负载是p->ravg.demand，cpu级别负载是rq->hmp_stats.cumulative_runnable_avg；

• 6、

具体的update_task_ravg()代码解析如下：

scheduler_tick() -> update_task_ravg()

↓

/* Reflect task activity on its demand and cpu's busy time statistics */
void update_task_ravg(struct task_struct *p, struct rq *rq, int event,
                        u64 wallclock, u64 irqtime)
{
    u64 runtime;

    if (!rq->window_start || sched_disable_window_stats ||
        p->ravg.mark_start == wallclock)
        return;

    lockdep_assert_held(&rq->lock);

    /* (1) 根据wallclock更新rq->window_start */
    update_window_start(rq, wallclock);

    if (!p->ravg.mark_start) {
        update_task_cpu_cycles(p, cpu_of(rq));
        goto done;
    }

    /* (2) 更新cycle、walltime的差值，用来计算cpu的当前freq */
    update_task_rq_cpu_cycles(p, rq, event, wallclock, irqtime);

    /* (3) 更新task的负载demand */
    runtime = update_task_demand(p, rq, event, wallclock);
    if (runtime)
        update_task_burst(p, rq, event, runtime);

    /* (4) 更新cpu的busy时间 */
    update_cpu_busy_time(p, rq, event, wallclock, irqtime);

    /* (5) 更新task的负载预测pred_demand */
    update_task_pred_demand(rq, p, event);
done:
    trace_sched_update_task_ravg(p, rq, event, wallclock, irqtime,
                     rq->cc.cycles, rq->cc.time,
                     p->grp ? &rq->grp_time : NULL);

    /* (6) 更新task的时间更新点：p->ravg.mark_start */
    p->ravg.mark_start = wallclock;
}

|→

static u64 update_task_demand(struct task_struct *p, struct rq *rq,
                   int event, u64 wallclock)
{
    u64 mark_start = p->ravg.mark_start;
    u64 delta, window_start = rq->window_start;
    int new_window, nr_full_windows;
    u32 window_size = sched_ravg_window;
    u64 runtime;

    new_window = mark_start < window_start;

    /* (3.1) 这是一个关键点，非runnable状态的统计需要在这里异常返回 */
    if (!account_busy_for_task_demand(p, event)) {
        if (new_window)
            /*
             * If the time accounted isn't being accounted as
             * busy time, and a new window started, only the
             * previous window need be closed out with the
             * pre-existing demand. Multiple windows may have
             * elapsed, but since empty windows are dropped,
             * it is not necessary to account those.
             */
            update_history(rq, p, p->ravg.sum, 1, event);
        return 0;
    }

    /* (3.2) 第一种情况：还在原窗口内，简单继续累加p->ravg.sum */
    if (!new_window) {
        /*
         * The simple case - busy time contained within the existing
         * window.
         */
        return add_to_task_demand(rq, p, wallclock - mark_start);
    }

    /* (3.3) 第二、三种情况：原窗口已经填满 */
    /*
     * Busy time spans at least two windows. Temporarily rewind
     * window_start to first window boundary after mark_start.
     */
    delta = window_start - mark_start;
    nr_full_windows = div64_u64(delta, window_size);
    window_start -= (u64)nr_full_windows * (u64)window_size;

    /* (3.4.1) 补全第一个窗口 */
    /* Process (window_start - mark_start) first */
    runtime = add_to_task_demand(rq, p, window_start - mark_start);

    /* (3.4.2) 把第一个窗口更新到进程task负载history中, 
        更新p->ravg.demand、p->ravg.pred_demand
     */
    /* Push new sample(s) into task's demand history */
    update_history(rq, p, p->ravg.sum, 1, event);

    /* (3.5) 如果中间有几个完整窗口，更新负载，更新history */
    if (nr_full_windows) {
        u64 scaled_window = scale_exec_time(window_size, rq);

        update_history(rq, p, scaled_window, nr_full_windows, event);
        runtime += nr_full_windows * scaled_window;
    }

    /* (3.6) 最后一个没有完成的窗口，只是简单累加时间，不更新history */
    /*
     * Roll window_start back to current to process any remainder
     * in current window.
     */
    window_start += (u64)nr_full_windows * (u64)window_size;

    /* Process (wallclock - window_start) next */
    mark_start = window_start;
    runtime += add_to_task_demand(rq, p, wallclock - mark_start);

    return runtime;
}

||→

static int account_busy_for_task_demand(struct task_struct *p, int event)
{
    /*
     * No need to bother updating task demand for exiting tasks
     * or the idle task.
     */
    /* (3.1.1) exit、idle任务不计入统计 */
    if (exiting_task(p) || is_idle_task(p))
        return 0;

    /*
     * When a task is waking up it is completing a segment of non-busy
     * time. Likewise, if wait time is not treated as busy time, then
     * when a task begins to run or is migrated, it is not running and
     * is completing a segment of non-busy time.
     */
    /* (3.1.2) 任务被wakeup，之前的等待时间不计入统计 
        SCHED_ACCOUNT_WAIT_TIME用来控制ruannable的等待时间是否计入统计，默认是计入的
     */
    if (event == TASK_WAKE || (!SCHED_ACCOUNT_WAIT_TIME &&
             (event == PICK_NEXT_TASK || event == TASK_MIGRATE)))
        return 0;

    return 1;
}

||→

static void add_to_task_demand(struct rq *rq, struct task_struct *p,
                u64 delta)
{
    /* (3.4.1) 累加窗口的时间值 */
    delta = scale_exec_time(delta, rq);
    p->ravg.sum += delta;
    if (unlikely(p->ravg.sum > walt_ravg_window))
        p->ravg.sum = walt_ravg_window;
}

static inline u64 scale_exec_time(u64 delta, struct rq *rq)
{
    u32 freq;

    /* curr_freq / max_possible_freq */
    freq = cpu_cycles_to_freq(rq->cc.cycles, rq->cc.time);
    delta = DIV64_U64_ROUNDUP(delta * freq, max_possible_freq);

    /* exec_scale_factor = cluster->efficiency / max_possible_efficiency */
    delta *= rq->cluster->exec_scale_factor;
    delta >>= 10;

    return delta;
}

||→

static void update_history(struct rq *rq, struct task_struct *p,
             u32 runtime, int samples, int event)
{
    u32 *hist = &p->ravg.sum_history[0];
    int ridx, widx;
    u32 max = 0, avg, demand, pred_demand;
    u64 sum = 0;

    /* (3.4.2.1) 不活跃的进程不进行更新 */
    /* Ignore windows where task had no activity */
    if (!runtime || is_idle_task(p) || exiting_task(p) || !samples)
        goto done;

    /* (3.4.2.2) 把新窗口的runtime推送到history stack中 */
    /* Push new 'runtime' value onto stack */
    widx = sched_ravg_hist_size - 1;
    ridx = widx - samples;
    for (; ridx >= 0; --widx, --ridx) {
        hist[widx] = hist[ridx];
        sum += hist[widx];
        if (hist[widx] > max)
            max = hist[widx];
    }

    for (widx = 0; widx < samples && widx < sched_ravg_hist_size; widx++) {
        hist[widx] = runtime;
        sum += hist[widx];
        if (hist[widx] > max)
            max = hist[widx];
    }

    p->ravg.sum = 0;

    /* (3.4.2.3) 根据sched_window_stats_policy策略(RECENT、MAX、AVG、MAX_RECENT_AVG)，
        从sum_history[]中选择合适的值作为进程负载p->ravg.demand
     */
    if (sched_window_stats_policy == WINDOW_STATS_RECENT) {
        demand = runtime;
    } else if (sched_window_stats_policy == WINDOW_STATS_MAX) {
        demand = max;
    } else {
        avg = div64_u64(sum, sched_ravg_hist_size);
        if (sched_window_stats_policy == WINDOW_STATS_AVG)
            demand = avg;
        else
            demand = max(avg, runtime);
    }

    /* (3.4.2.4) 计算进程的预测负载 */
    pred_demand = predict_and_update_buckets(rq, p, runtime);

    /*
     * A throttled deadline sched class task gets dequeued without
     * changing p->on_rq. Since the dequeue decrements hmp stats
     * avoid decrementing it here again.
     */
    /* (3.4.2.5) 更新进程负载(p->ravg.demand)到cpu负载(rq->hmp_stats.cumulative_runnable_avg)中 
        cfs中p->sched_class->fixup_hmp_sched_stats对应函数fixup_hmp_sched_stats_fair()
     */
    if (task_on_rq_queued(p) && (!task_has_dl_policy(p) ||
                        !p->dl.dl_throttled))
        p->sched_class->fixup_hmp_sched_stats(rq, p, demand,
                              pred_demand);

    p->ravg.demand = demand;
    p->ravg.pred_demand = pred_demand;

done:
    trace_sched_update_history(rq, p, runtime, samples, event);
}

|||→

static inline u32 predict_and_update_buckets(struct rq *rq,
            struct task_struct *p, u32 runtime) {

    int bidx;
    u32 pred_demand;

    /* (3.4.2.4.1) 把window负载转换成bucket index(最大10) */
    bidx = busy_to_bucket(runtime);

    /* (3.4.2.4.2) 根据index，找到历史曾经达到过的更大值，取历史的值作为预测值 */
    pred_demand = get_pred_busy(rq, p, bidx, runtime);

    /* (3.4.2.4.3) 对bucket[]中本次index权重进行增加，其他权重减少 */
    bucket_increase(p->ravg.busy_buckets, bidx);

    return pred_demand;
}

|||→

static void
fixup_hmp_sched_stats_fair(struct rq *rq, struct task_struct *p,
               u32 new_task_load, u32 new_pred_demand)
{
    /* (3.4.2.5.1) 计算task负载和预测的变化值delta */
    s64 task_load_delta = (s64)new_task_load - task_load(p);
    s64 pred_demand_delta = PRED_DEMAND_DELTA;

    /* (3.4.2.5.2) 将进程级别的delta计入cpu级别的负载统计(rq->hmp_stats)中 */
    fixup_cumulative_runnable_avg(&rq->hmp_stats, p, task_load_delta,
                      pred_demand_delta);

    /* (3.4.2.5.3) 更新cpu级别big_task的数量 */
    fixup_nr_big_tasks(&rq->hmp_stats, p, task_load_delta);
}

static inline void
fixup_cumulative_runnable_avg(struct hmp_sched_stats *stats,
                  struct task_struct *p, s64 task_load_delta,
                  s64 pred_demand_delta)
{
    if (sched_disable_window_stats)
        return;

    stats->cumulative_runnable_avg += task_load_delta;
    BUG_ON((s64)stats->cumulative_runnable_avg < 0);

    stats->pred_demands_sum += pred_demand_delta;
    BUG_ON((s64)stats->pred_demands_sum < 0);
}

void fixup_nr_big_tasks(struct hmp_sched_stats *stats,
                struct task_struct *p, s64 delta)
{
    u64 new_task_load;
    u64 old_task_load;

    if (sched_disable_window_stats)
        return;

    /* task_load按照capacity反比放大，让所有cpu处在同一级别 */
    old_task_load = scale_load_to_cpu(task_load(p), task_cpu(p));
    new_task_load = scale_load_to_cpu(delta + task_load(p), task_cpu(p));

    /* 如果进程负载 > 最大负载 * 80% (sysctl_sched_upmigrate_pct)
        该任务为big_task
     */
    if (__is_big_task(p, old_task_load) && !__is_big_task(p, new_task_load))
        stats->nr_big_tasks--;
    else if (!__is_big_task(p, old_task_load) &&
         __is_big_task(p, new_task_load))
        stats->nr_big_tasks++;

    BUG_ON(stats->nr_big_tasks < 0);
}

我们再来详细看看cpu级别的busy time计算：

static void update_cpu_busy_time(struct task_struct *p, struct rq *rq,
                 int event, u64 wallclock, u64 irqtime)
{
    int new_window, full_window = 0;
    int p_is_curr_task = (p == rq->curr);
    u64 mark_start = p->ravg.mark_start;
    u64 window_start = rq->window_start;
    u32 window_size = sched_ravg_window;
    u64 delta;
    u64 *curr_runnable_sum = &rq->curr_runnable_sum;
    u64 *prev_runnable_sum = &rq->prev_runnable_sum;
    u64 *nt_curr_runnable_sum = &rq->nt_curr_runnable_sum;
    u64 *nt_prev_runnable_sum = &rq->nt_prev_runnable_sum;
    bool new_task;
    struct related_thread_group *grp;
    int cpu = rq->cpu;
    u32 old_curr_window = p->ravg.curr_window;

    new_window = mark_start < window_start;
    if (new_window) {
        full_window = (window_start - mark_start) >= window_size;
        if (p->ravg.active_windows < USHRT_MAX)
            p->ravg.active_windows++;
    }

    new_task = is_new_task(p);

    /*
     * Handle per-task window rollover. We don't care about the idle
     * task or exiting tasks.
     */
    /* (1) 如果有新window，滚动进程窗口：p->ravg.prev_window、p->ravg.curr_window */
    if (!is_idle_task(p) && !exiting_task(p)) {
        if (new_window)
            rollover_task_window(p, full_window);
    }

    /* (2) 如果有新window且进程是rq的当前进程，
        cpu级别的窗口滚动：rq->prev_runnable_sum、rq->curr_runnable_sum
        cpu级别的进程统计窗口滚动：rq->top_tasks[prev_table]、rq->top_tasks[curr_table]
     */
    if (p_is_curr_task && new_window) {
        rollover_cpu_window(rq, full_window);
        rollover_top_tasks(rq, full_window);
    }

    /* (3) 判断哪些情况可以统计进cpu time */
    if (!account_busy_for_cpu_time(rq, p, irqtime, event))
        goto done;

    grp = p->grp;
    if (grp && sched_freq_aggregate) {
        struct group_cpu_time *cpu_time = &rq->grp_time;

        curr_runnable_sum = &cpu_time->curr_runnable_sum;
        prev_runnable_sum = &cpu_time->prev_runnable_sum;

        nt_curr_runnable_sum = &cpu_time->nt_curr_runnable_sum;
        nt_prev_runnable_sum = &cpu_time->nt_prev_runnable_sum;
    }

    /* (4) 如果时间没有达到新window，
        在cpu级别的当前负载上累加：rq->curr_runnable_sum
        在进程级别的基础上累加：p->ravg.curr_window
     */
    if (!new_window) {
        /*
         * account_busy_for_cpu_time() = 1 so busy time needs
         * to be accounted to the current window. No rollover
         * since we didn't start a new window. An example of this is
         * when a task starts execution and then sleeps within the
         * same window.
         */

        if (!irqtime || !is_idle_task(p) || cpu_is_waiting_on_io(rq))
            delta = wallclock - mark_start;
        else
            delta = irqtime;
        delta = scale_exec_time(delta, rq);
        *curr_runnable_sum += delta;
        if (new_task)
            *nt_curr_runnable_sum += delta;

        if (!is_idle_task(p) && !exiting_task(p)) {
            p->ravg.curr_window += delta;
            p->ravg.curr_window_cpu[cpu] += delta;
        }

        goto done;
    }

    /* (5) 如果时间达到新window，但是进程不是rq的当前进程
        在进程级别的基础上累加：p->ravg.prev_window、p->ravg.curr_window
        在cpu级别的当前负载上累加：rq->prev_runnable_sum、rq->curr_runnable_sum
     */
    if (!p_is_curr_task) {
        /*
         * account_busy_for_cpu_time() = 1 so busy time needs
         * to be accounted to the current window. A new window
         * has also started, but p is not the current task, so the
         * window is not rolled over - just split up and account
         * as necessary into curr and prev. The window is only
         * rolled over when a new window is processed for the current
         * task.
         *
         * Irqtime can't be accounted by a task that isn't the
         * currently running task.
         */

        if (!full_window) {
            /*
             * A full window hasn't elapsed, account partial
             * contribution to previous completed window.
             */
            delta = scale_exec_time(window_start - mark_start, rq);
            if (!exiting_task(p)) {
                p->ravg.prev_window += delta;
                p->ravg.prev_window_cpu[cpu] += delta;
            }
        } else {
            /*
             * Since at least one full window has elapsed,
             * the contribution to the previous window is the
             * full window (window_size).
             */
            delta = scale_exec_time(window_size, rq);
            if (!exiting_task(p)) {
                p->ravg.prev_window = delta;
                p->ravg.prev_window_cpu[cpu] = delta;
            }
        }

        *prev_runnable_sum += delta;
        if (new_task)
            *nt_prev_runnable_sum += delta;

        /* Account piece of busy time in the current window. */
        delta = scale_exec_time(wallclock - window_start, rq);
        *curr_runnable_sum += delta;
        if (new_task)
            *nt_curr_runnable_sum += delta;

        if (!exiting_task(p)) {
            p->ravg.curr_window = delta;
            p->ravg.curr_window_cpu[cpu] = delta;
        }

        goto done;
    }

    /* (6) 如果时间达到新window，且进程是rq的当前进程
        在进程级别的基础上累加：p->ravg.prev_window、p->ravg.curr_window
        在cpu级别的当前负载上累加：rq->prev_runnable_sum、rq->curr_runnable_sum
     */
    if (!irqtime || !is_idle_task(p) || cpu_is_waiting_on_io(rq)) {
        /*
         * account_busy_for_cpu_time() = 1 so busy time needs
         * to be accounted to the current window. A new window
         * has started and p is the current task so rollover is
         * needed. If any of these three above conditions are true
         * then this busy time can't be accounted as irqtime.
         *
         * Busy time for the idle task or exiting tasks need not
         * be accounted.
         *
         * An example of this would be a task that starts execution
         * and then sleeps once a new window has begun.
         */

        if (!full_window) {
            /*
             * A full window hasn't elapsed, account partial
             * contribution to previous completed window.
             */
            delta = scale_exec_time(window_start - mark_start, rq);
            if (!is_idle_task(p) && !exiting_task(p)) {
                p->ravg.prev_window += delta;
                p->ravg.prev_window_cpu[cpu] += delta;
            }
        } else {
            /*
             * Since at least one full window has elapsed,
             * the contribution to the previous window is the
             * full window (window_size).
             */
            delta = scale_exec_time(window_size, rq);
            if (!is_idle_task(p) && !exiting_task(p)) {
                p->ravg.prev_window = delta;
                p->ravg.prev_window_cpu[cpu] = delta;
            }
        }

        /*
         * Rollover is done here by overwriting the values in
         * prev_runnable_sum and curr_runnable_sum.
         */
        *prev_runnable_sum += delta;
        if (new_task)
            *nt_prev_runnable_sum += delta;

        /* Account piece of busy time in the current window. */
        delta = scale_exec_time(wallclock - window_start, rq);
        *curr_runnable_sum += delta;
        if (new_task)
            *nt_curr_runnable_sum += delta;

        if (!is_idle_task(p) && !exiting_task(p)) {
            p->ravg.curr_window = delta;
            p->ravg.curr_window_cpu[cpu] = delta;
        }

        goto done;
    }

    if (irqtime) {
        /*
         * account_busy_for_cpu_time() = 1 so busy time needs
         * to be accounted to the current window. A new window
         * has started and p is the current task so rollover is
         * needed. The current task must be the idle task because
         * irqtime is not accounted for any other task.
         *
         * Irqtime will be accounted each time we process IRQ activity
         * after a period of idleness, so we know the IRQ busy time
         * started at wallclock - irqtime.
         */

        BUG_ON(!is_idle_task(p));
        mark_start = wallclock - irqtime;

        /*
         * Roll window over. If IRQ busy time was just in the current
         * window then that is all that need be accounted.
         */
        if (mark_start > window_start) {
            *curr_runnable_sum = scale_exec_time(irqtime, rq);
            return;
        }

        /*
         * The IRQ busy time spanned multiple windows. Process the
         * busy time preceding the current window start first.
         */
        delta = window_start - mark_start;
        if (delta > window_size)
            delta = window_size;
        delta = scale_exec_time(delta, rq);
        *prev_runnable_sum += delta;

        /* Process the remaining IRQ busy time in the current window. */
        delta = wallclock - window_start;
        rq->curr_runnable_sum = scale_exec_time(delta, rq);

        return;
    }

done:
    /* (7) 更新cpu上的top task */
    if (!is_idle_task(p) && !exiting_task(p))
        update_top_tasks(p, rq, old_curr_window,
                    new_window, full_window);
}

|→

static void update_top_tasks(struct task_struct *p, struct rq *rq,
        u32 old_curr_window, int new_window, bool full_window)
{
    u8 curr = rq->curr_table;
    u8 prev = 1 - curr;
    u8 *curr_table = rq->top_tasks[curr];
    u8 *prev_table = rq->top_tasks[prev];
    int old_index, new_index, update_index;
    u32 curr_window = p->ravg.curr_window;
    u32 prev_window = p->ravg.prev_window;
    bool zero_index_update;

    if (old_curr_window == curr_window && !new_window)
        return;

    /* (1) 把就进程p的"当前window负载"、"旧的当前window负载"转换成index(NUM_LOAD_INDICES=1000) */
    old_index = load_to_index(old_curr_window);
    new_index = load_to_index(curr_window);

    /* (2) 如果没有新window 
        更新当前top表rq->top_tasks[curr][]中新旧index的计数
        根据index的计数是否为0，更新rq->top_tasks_bitmap[curr] bitmap中对应index的值
     */
    if (!new_window) {
        zero_index_update = !old_curr_window && curr_window;
        if (old_index != new_index || zero_index_update) {
            if (old_curr_window)
                curr_table[old_index] -= 1;
            if (curr_window)
                curr_table[new_index] += 1;
            if (new_index > rq->curr_top)
                rq->curr_top = new_index;
        }

        if (!curr_table[old_index])
            __clear_bit(NUM_LOAD_INDICES - old_index - 1,
                rq->top_tasks_bitmap[curr]);

        if (curr_table[new_index] == 1)
            __set_bit(NUM_LOAD_INDICES - new_index - 1,
                rq->top_tasks_bitmap[curr]);

        return;
    }

    /*
     * The window has rolled over for this task. By the time we get
     * here, curr/prev swaps would has already occurred. So we need
     * to use prev_window for the new index.
     */
    update_index = load_to_index(prev_window);

    if (full_window) {
        /*
         * Two cases here. Either 'p' ran for the entire window or
         * it didn't run at all. In either case there is no entry
         * in the prev table. If 'p' ran the entire window, we just
         * need to create a new entry in the prev table. In this case
         * update_index will be correspond to sched_ravg_window
         * so we can unconditionally update the top index.
         */
        if (prev_window) {
            prev_table[update_index] += 1;
            rq->prev_top = update_index;
        }

        if (prev_table[update_index] == 1)
            __set_bit(NUM_LOAD_INDICES - update_index - 1,
                rq->top_tasks_bitmap[prev]);
    } else {
        zero_index_update = !old_curr_window && prev_window;
        if (old_index != update_index || zero_index_update) {
            if (old_curr_window)
                prev_table[old_index] -= 1;

            prev_table[update_index] += 1;

            if (update_index > rq->prev_top)
                rq->prev_top = update_index;

            if (!prev_table[old_index])
                __clear_bit(NUM_LOAD_INDICES - old_index - 1,
                        rq->top_tasks_bitmap[prev]);

            if (prev_table[update_index] == 1)
                __set_bit(NUM_LOAD_INDICES - update_index - 1,
                        rq->top_tasks_bitmap[prev]);
        }
    }

    if (curr_window) {
        curr_table[new_index] += 1;

        if (new_index > rq->curr_top)
            rq->curr_top = new_index;

        if (curr_table[new_index] == 1)
            __set_bit(NUM_LOAD_INDICES - new_index - 1,
                rq->top_tasks_bitmap[curr]);
    }
}

5.3.2、基于WALT的负载均衡

5.3.2.1、load_balance()

其他部分和主干内核算法一致，这里只标识出qualcom的HMP算法特有的部分。在负载均衡部分，walt用来找出cpu；但是在负载迁移时，计算负载还是使用pelt？

• 在find_busiest_queue()中：原本是找出cfs_rq->runnable_load_avg * capacity负载最大的cpu，qualcom HMP改为找出walt runnable负载(rq->hmp_stats.cumulative_runnable_avg)最重的cpu。

run_rebalance_domains() -> rebalance_domains() -> load_balance() -> find_busiest_queue() -> find_busiest_queue_hmp()

↓

static struct rq *find_busiest_queue_hmp(struct lb_env *env,
                     struct sched_group *group)
{
    struct rq *busiest = NULL, *busiest_big = NULL;
    u64 max_runnable_avg = 0, max_runnable_avg_big = 0;
    int max_nr_big = 0, nr_big;
    bool find_big = !!(env->flags & LBF_BIG_TASK_ACTIVE_BALANCE);
    int i;
    cpumask_t cpus;

    cpumask_andnot(&cpus, sched_group_cpus(group), cpu_isolated_mask);

    /* (1) 遍历sg中的cpu */
    for_each_cpu(i, &cpus) {
        struct rq *rq = cpu_rq(i);
        u64 cumulative_runnable_avg =
                rq->hmp_stats.cumulative_runnable_avg;

        if (!cpumask_test_cpu(i, env->cpus))
            continue;


        /* (2) 考虑big_task，找出big_task最重的cpu */
        if (find_big) {
            nr_big = nr_big_tasks(rq);
            if (nr_big > max_nr_big ||
                (nr_big > 0 && nr_big == max_nr_big &&
                 cumulative_runnable_avg > max_runnable_avg_big)) {
                max_runnable_avg_big = cumulative_runnable_avg;
                busiest_big = rq;
                max_nr_big = nr_big;
                continue;
            }
        }

        /* (3) 找出walt runnable负载(rq->hmp_stats.cumulative_runnable_avg)最重的cpu */
        if (cumulative_runnable_avg > max_runnable_avg) {
            max_runnable_avg = cumulative_runnable_avg;
            busiest = rq;
        }
    }

    if (busiest_big)
        return busiest_big;

    env->flags &= ~LBF_BIG_TASK_ACTIVE_BALANCE;
    return busiest;
}

5.3.2.2、nohz_idle_balance()

• _nohz_kick_needed()：

scheduler_tick() -> trigger_load_balance() -> nohz_kick_needed() -> _nohz_kick_needed() -> nohz_kick_needed_hmp()

↓

static inline int _nohz_kick_needed_hmp(struct rq *rq, int cpu, int *type)
{
    struct sched_domain *sd;
    int i;

    if (rq->nr_running < 2)
        return 0;

    /* (1) 如果是SCHED_BOOST_ON_ALL，返回true */
    if (!sysctl_sched_restrict_cluster_spill ||
            sched_boost_policy() == SCHED_BOOST_ON_ALL)
        return 1;

    /* (2) 如果当前cpu是max cpu，返回true */
    if (cpu_max_power_cost(cpu) == max_power_cost)
        return 1;

    rcu_read_lock();
    sd = rcu_dereference_check_sched_domain(rq->sd);
    if (!sd) {
        rcu_read_unlock();
        return 0;
    }

    for_each_cpu(i, sched_domain_span(sd)) {
        if (cpu_load(i) < sched_spill_load &&
                cpu_rq(i)->nr_running <
                sysctl_sched_spill_nr_run) {
            /* Change the kick type to limit to CPUs that
             * are of equal or lower capacity.
             */
            *type = NOHZ_KICK_RESTRICT;
            break;
        }
    }
    rcu_read_unlock();
    return 1;
}

• find_new_hmp_ilb()：原本是找出nohz.idle_cpus_mask中的第一个cpu作为ilb cpu，qualcom HMP改为尝试在nohz.idle_cpus_mask中找到一个max power小于当前cpu的作为ilb cpu。

scheduler_tick() -> trigger_load_balance() -> nohz_balancer_kick() -> find_new_ilb()

↓

static inline int find_new_hmp_ilb(int type)
{
    int call_cpu = raw_smp_processor_id();
    struct sched_domain *sd;
    int ilb;

    rcu_read_lock();

    /* Pick an idle cpu "closest" to call_cpu */
    for_each_domain(call_cpu, sd) {
        for_each_cpu_and(ilb, nohz.idle_cpus_mask,
                        sched_domain_span(sd)) {

            /* (1) 尝试找到一个max power小于当前power的cpu作为ilb cpu */
            if (idle_cpu(ilb) && (type != NOHZ_KICK_RESTRICT ||
                    cpu_max_power_cost(ilb) <=
                    cpu_max_power_cost(call_cpu))) {
                rcu_read_unlock();
                reset_balance_interval(ilb);
                return ilb;
            }
        }
    }

    rcu_read_unlock();
    return nr_cpu_ids;
}

5.3.2.3、select_task_rq_fair()

• select_task_rq_fair()：使用qualcom自己的算法，综合capacity、power、idle给出一个best cpu。

select_task_rq_fair() -> select_best_cpu()

↓

/* return cheapest cpu that can fit this task */
static int select_best_cpu(struct task_struct *p, int target, int reason,
               int sync)
{
    struct sched_cluster *cluster, *pref_cluster = NULL;
    struct cluster_cpu_stats stats;
    struct related_thread_group *grp;
    unsigned int sbc_flag = 0;
    int cpu = raw_smp_processor_id();
    bool special;

    struct cpu_select_env env = {
        .p          = p,
        .reason         = reason,
        .need_idle      = wake_to_idle(p),
        .need_waker_cluster = 0,
        .sync           = sync,
        .prev_cpu       = target,
        .rtg            = NULL,
        .sbc_best_flag      = 0,
        .sbc_best_cluster_flag  = 0,
        .pack_task              = false,
    };

    rcu_read_lock();
    env.boost_policy = task_sched_boost(p) ?
            sched_boost_policy() : SCHED_BOOST_NONE;

    bitmap_copy(env.candidate_list, all_cluster_ids, NR_CPUS);
    bitmap_zero(env.backup_list, NR_CPUS);

    cpumask_and(&env.search_cpus, tsk_cpus_allowed(p), cpu_active_mask);
    cpumask_andnot(&env.search_cpus, &env.search_cpus, cpu_isolated_mask);

    init_cluster_cpu_stats(&stats);
    special = env_has_special_flags(&env);

    grp = task_related_thread_group(p);

    if (grp && grp->preferred_cluster) {
        pref_cluster = grp->preferred_cluster;
        if (!cluster_allowed(&env, pref_cluster))
            clear_bit(pref_cluster->id, env.candidate_list);
        else
            env.rtg = grp;
    } else if (!special) {
        cluster = cpu_rq(cpu)->cluster;
        if (wake_to_waker_cluster(&env)) {
            if (bias_to_waker_cpu(&env, cpu)) {
                target = cpu;
                sbc_flag = SBC_FLAG_WAKER_CLUSTER |
                       SBC_FLAG_WAKER_CPU;
                goto out;
            } else if (cluster_allowed(&env, cluster)) {
                env.need_waker_cluster = 1;
                bitmap_zero(env.candidate_list, NR_CPUS);
                __set_bit(cluster->id, env.candidate_list);
                env.sbc_best_cluster_flag =
                            SBC_FLAG_WAKER_CLUSTER;
            }
        } else if (bias_to_prev_cpu(&env, &stats)) {
            sbc_flag = SBC_FLAG_PREV_CPU;
            goto out;
        }
    }

    if (!special && is_short_burst_task(p)) {
        env.pack_task = true;
        sbc_flag = SBC_FLAG_PACK_TASK;
    }
retry:

    /* (1) 从低到高找到一个power最低，且capacity能满足task_load的cluster */
    cluster = select_least_power_cluster(&env);

    if (!cluster)
        goto out;

    /*
     * 'cluster' now points to the minimum power cluster which can satisfy
     * task's perf goals. Walk down the cluster list starting with that
     * cluster. For non-small tasks, skip clusters that don't have
     * mostly_idle/idle cpus
     */

    do {
        /* (2) 全方位统计：capacity spare、cost、idle */
        find_best_cpu_in_cluster(cluster, &env, &stats);

    } while ((cluster = next_best_cluster(cluster, &env, &stats)));


    /* (3) 从idle角度给出best cpu */
    if (env.need_idle) {
        if (stats.best_idle_cpu >= 0) {
            target = stats.best_idle_cpu;
            sbc_flag |= SBC_FLAG_IDLE_CSTATE;
        } else if (stats.least_loaded_cpu >= 0) {
            target = stats.least_loaded_cpu;
            sbc_flag |= SBC_FLAG_IDLE_LEAST_LOADED;
        }

    /* (4) 从综合角度给出best cpu */
    } else if (stats.best_cpu >= 0) {
        if (stats.best_sibling_cpu >= 0 &&
                stats.best_cpu != task_cpu(p) &&
                stats.min_cost == stats.best_sibling_cpu_cost) {
            stats.best_cpu = stats.best_sibling_cpu;
            sbc_flag |= SBC_FLAG_BEST_SIBLING;
        }
        sbc_flag |= env.sbc_best_flag;
        target = stats.best_cpu;
    } else {
        if (env.rtg && env.boost_policy == SCHED_BOOST_NONE) {
            env.rtg = NULL;
            goto retry;
        }

        /*
         * With boost_policy == SCHED_BOOST_ON_BIG, we reach here with
         * backup_list = little cluster, candidate_list = none and
         * stats->best_capacity_cpu points the best spare capacity
         * CPU among the CPUs in the big cluster.
         */
        if (env.boost_policy == SCHED_BOOST_ON_BIG &&
            stats.best_capacity_cpu >= 0)
            sbc_flag |= SBC_FLAG_BOOST_CLUSTER;
        else
            find_backup_cluster(&env, &stats);

        if (stats.best_capacity_cpu >= 0) {
            target = stats.best_capacity_cpu;
            sbc_flag |= SBC_FLAG_BEST_CAP_CPU;
        }
    }
    p->last_cpu_selected_ts = sched_ktime_clock();
out:
    sbc_flag |= env.sbc_best_cluster_flag;
    rcu_read_unlock();
    trace_sched_task_load(p, sched_boost_policy() && task_sched_boost(p),
        env.reason, env.sync, env.need_idle, sbc_flag, target);
    return target;
}

5.3.2.4、Interaction Governor & sched_load

qualcom对interactive governor进行了改造，打造成了可以使用sched_load的interactive governor。

• 1、interactive governor注册回调函数，接收sched_load变化事件；

static ssize_t store_use_sched_load(
            struct cpufreq_interactive_tunables *tunables,
            const char *buf, size_t count)
{
    int ret;
    unsigned long val;

    ret = kstrtoul(buf, 0, &val);
    if (ret < 0)
        return ret;

    if (tunables->use_sched_load == (bool) val)
        return count;

    tunables->use_sched_load = val;

    if (val)
        ret = cpufreq_interactive_enable_sched_input(tunables);
    else
        ret = cpufreq_interactive_disable_sched_input(tunables);

    if (ret) {
        tunables->use_sched_load = !val;
        return ret;
    }

    return count;
}

|→

static int cpufreq_interactive_enable_sched_input(
            struct cpufreq_interactive_tunables *tunables)
{
    int rc = 0, j;
    struct cpufreq_interactive_tunables *t;

    mutex_lock(&sched_lock);

    set_window_count++;
    if (set_window_count > 1) {
        for_each_possible_cpu(j) {
            if (!per_cpu(polinfo, j))
                continue;
            t = per_cpu(polinfo, j)->cached_tunables;
            if (t && t->use_sched_load) {
                tunables->timer_rate = t->timer_rate;
                tunables->io_is_busy = t->io_is_busy;
                break;
            }
        }
    } else {
        /* (1) 设置walt窗口大小 */
        rc = set_window_helper(tunables);
        if (rc) {
            pr_err("%s: Failed to set sched window\n", __func__);
            set_window_count--;
            goto out;
        }
        sched_set_io_is_busy(tunables->io_is_busy);
    }

    if (!tunables->use_migration_notif)
        goto out;

    migration_register_count++;
    if (migration_register_count > 1)
        goto out;
    else
        /* (2) 注册sched_load变化的回调函数 */
        atomic_notifier_chain_register(&load_alert_notifier_head,
                        &load_notifier_block);
out:
    mutex_unlock(&sched_lock);
    return rc;
}

||→

static inline int set_window_helper(
            struct cpufreq_interactive_tunables *tunables)
{
    /* 设置默认窗口size为DEFAULT_TIMER_RATE(20ms) */
    return sched_set_window(round_to_nw_start(get_jiffies_64(), tunables),
             usecs_to_jiffies(tunables->timer_rate));
}

static struct notifier_block load_notifier_block = {
    .notifier_call = load_change_callback,
};

• 2、sched_load的变化通过回调函数通知给Interaction Governor；

check_for_freq_change（） -> load_alert_notifier_head -> load_change_callback()

↓

static int load_change_callback(struct notifier_block *nb, unsigned long val,
                void *data)
{
    unsigned long cpu = (unsigned long) data;
    struct cpufreq_interactive_policyinfo *ppol = per_cpu(polinfo, cpu);
    struct cpufreq_interactive_tunables *tunables;
    unsigned long flags;

    if (!ppol || ppol->reject_notification)
        return 0;

    if (!down_read_trylock(&ppol->enable_sem))
        return 0;
    if (!ppol->governor_enabled)
        goto exit;

    tunables = ppol->policy->governor_data;
    if (!tunables->use_sched_load || !tunables->use_migration_notif)
        goto exit;

    spin_lock_irqsave(&ppol->target_freq_lock, flags);
    ppol->notif_pending = true;
    ppol->notif_cpu = cpu;
    spin_unlock_irqrestore(&ppol->target_freq_lock, flags);

    if (!hrtimer_is_queued(&ppol->notif_timer))
        hrtimer_start(&ppol->notif_timer, ms_to_ktime(1),
                  HRTIMER_MODE_REL);
exit:
    up_read(&ppol->enable_sem);
    return 0;
}

• 3、除了事件通知，interactive governor还会在20ms timer中轮询sched_load的变化来决定是否需要调频。

static void cpufreq_interactive_timer(unsigned long data)
{
    s64 now;
    unsigned int delta_time;
    u64 cputime_speedadj;
    int cpu_load;
    int pol_load = 0;
    struct cpufreq_interactive_policyinfo *ppol = per_cpu(polinfo, data);
    struct cpufreq_interactive_tunables *tunables =
        ppol->policy->governor_data;
    struct sched_load *sl = ppol->sl;
    struct cpufreq_interactive_cpuinfo *pcpu;
    unsigned int new_freq;
    unsigned int prev_laf = 0, t_prevlaf;
    unsigned int pred_laf = 0, t_predlaf = 0;
    unsigned int prev_chfreq, pred_chfreq, chosen_freq;
    unsigned int index;
    unsigned long flags;
    unsigned long max_cpu;
    int cpu, i;
    int new_load_pct = 0;
    int prev_l, pred_l = 0;
    struct cpufreq_govinfo govinfo;
    bool skip_hispeed_logic, skip_min_sample_time;
    bool jump_to_max_no_ts = false;
    bool jump_to_max = false;
    bool start_hyst = true;

    if (!down_read_trylock(&ppol->enable_sem))
        return;
    if (!ppol->governor_enabled)
        goto exit;

    now = ktime_to_us(ktime_get());

    spin_lock_irqsave(&ppol->target_freq_lock, flags);
    spin_lock(&ppol->load_lock);

    skip_hispeed_logic =
        tunables->ignore_hispeed_on_notif && ppol->notif_pending;
    skip_min_sample_time = tunables->fast_ramp_down && ppol->notif_pending;
    ppol->notif_pending = false;
    now = ktime_to_us(ktime_get());
    ppol->last_evaluated_jiffy = get_jiffies_64();

    /* (1) sched_load模式，查询最新的sched_load  */
    if (tunables->use_sched_load)
        sched_get_cpus_busy(sl, ppol->policy->cpus);
    max_cpu = cpumask_first(ppol->policy->cpus);
    i = 0;
    for_each_cpu(cpu, ppol->policy->cpus) {
        pcpu = &per_cpu(cpuinfo, cpu);

        /* (2) sched_load模式，使用sched_load来计算负载变化  */
        if (tunables->use_sched_load) {

            /* (2.1) 根据上个窗口负载，获得当前目标值 */
            t_prevlaf = sl_busy_to_laf(ppol, sl[i].prev_load);
            prev_l = t_prevlaf / ppol->target_freq;

            /* (2.2) 根据上个窗口负载预测，获得当前的预测值 */
            if (tunables->enable_prediction) {
                t_predlaf = sl_busy_to_laf(ppol,
                        sl[i].predicted_load);
                pred_l = t_predlaf / ppol->target_freq;
            }
            if (sl[i].prev_load)
                new_load_pct = sl[i].new_task_load * 100 /
                            sl[i].prev_load;
            else
                new_load_pct = 0;

        /* (3) 传统模式，使用time*freq的模式来计算负载变化  */
        } else {
            now = update_load(cpu);
            delta_time = (unsigned int)
                (now - pcpu->cputime_speedadj_timestamp);
            if (WARN_ON_ONCE(!delta_time))
                continue;
            cputime_speedadj = pcpu->cputime_speedadj;
            do_div(cputime_speedadj, delta_time);
            t_prevlaf = (unsigned int)cputime_speedadj * 100;
            prev_l = t_prevlaf / ppol->target_freq;
        }

        /* find max of loadadjfreq inside policy */
        if (t_prevlaf > prev_laf) {
            prev_laf = t_prevlaf;
            max_cpu = cpu;
        }
        pred_laf = max(t_predlaf, pred_laf);

        cpu_load = max(prev_l, pred_l);
        pol_load = max(pol_load, cpu_load);
        trace_cpufreq_interactive_cpuload(cpu, cpu_load, new_load_pct,
                          prev_l, pred_l);

        /* save loadadjfreq for notification */
        pcpu->loadadjfreq = max(t_prevlaf, t_predlaf);

        /* detect heavy new task and jump to policy->max */
        if (prev_l >= tunables->go_hispeed_load &&
            new_load_pct >= NEW_TASK_RATIO) {
            skip_hispeed_logic = true;
            jump_to_max = true;
        }
        i++;
    }
    spin_unlock(&ppol->load_lock);

    tunables->boosted = tunables->boost_val || now < tunables->boostpulse_endtime;

    /* (4) 取目标值和预测值中的较大值，作为调频目标 */
    prev_chfreq = choose_freq(ppol, prev_laf);
    pred_chfreq = choose_freq(ppol, pred_laf);
    chosen_freq = max(prev_chfreq, pred_chfreq);

    if (prev_chfreq < ppol->policy->max && pred_chfreq >= ppol->policy->max)
        if (!jump_to_max)
            jump_to_max_no_ts = true;

    if (now - ppol->max_freq_hyst_start_time <
        tunables->max_freq_hysteresis &&
        pol_load >= tunables->go_hispeed_load &&
        ppol->target_freq < ppol->policy->max) {
        skip_hispeed_logic = true;
        skip_min_sample_time = true;
        if (!jump_to_max)
            jump_to_max_no_ts = true;
    }

    new_freq = chosen_freq;
    if (jump_to_max_no_ts || jump_to_max) {
        new_freq = ppol->policy->cpuinfo.max_freq;
    } else if (!skip_hispeed_logic) {
        if (pol_load >= tunables->go_hispeed_load ||
            tunables->boosted) {
            if (ppol->target_freq < tunables->hispeed_freq)
                new_freq = tunables->hispeed_freq;
            else
                new_freq = max(new_freq,
                           tunables->hispeed_freq);
        }
    }

    if (now - ppol->max_freq_hyst_start_time <
        tunables->max_freq_hysteresis) {
        if (new_freq < ppol->policy->max &&
                ppol->policy->max <= tunables->hispeed_freq)
            start_hyst = false;
        new_freq = max(tunables->hispeed_freq, new_freq);
    }

    if (!skip_hispeed_logic &&
        ppol->target_freq >= tunables->hispeed_freq &&
        new_freq > ppol->target_freq &&
        now - ppol->hispeed_validate_time <
        freq_to_above_hispeed_delay(tunables, ppol->target_freq)) {
        trace_cpufreq_interactive_notyet(
            max_cpu, pol_load, ppol->target_freq,
            ppol->policy->cur, new_freq);
        spin_unlock_irqrestore(&ppol->target_freq_lock, flags);
        goto rearm;
    }

    ppol->hispeed_validate_time = now;

    if (cpufreq_frequency_table_target(&ppol->p_nolim, ppol->freq_table,
                       new_freq, CPUFREQ_RELATION_L,
                       &index)) {
        spin_unlock_irqrestore(&ppol->target_freq_lock, flags);
        goto rearm;
    }

    new_freq = ppol->freq_table[index].frequency;

    /*
     * Do not scale below floor_freq unless we have been at or above the
     * floor frequency for the minimum sample time since last validated.
     */
    if (!skip_min_sample_time && new_freq < ppol->floor_freq) {
        if (now - ppol->floor_validate_time <
                tunables->min_sample_time) {
            trace_cpufreq_interactive_notyet(
                max_cpu, pol_load, ppol->target_freq,
                ppol->policy->cur, new_freq);
            spin_unlock_irqrestore(&ppol->target_freq_lock, flags);
            goto rearm;
        }
    }

    /*
     * Update the timestamp for checking whether speed has been held at
     * or above the selected frequency for a minimum of min_sample_time,
     * if not boosted to hispeed_freq.  If boosted to hispeed_freq then we
     * allow the speed to drop as soon as the boostpulse duration expires
     * (or the indefinite boost is turned off). If policy->max is restored
     * for max_freq_hysteresis, don't extend the timestamp. Otherwise, it
     * could incorrectly extended the duration of max_freq_hysteresis by
     * min_sample_time.
     */

    if ((!tunables->boosted || new_freq > tunables->hispeed_freq)
        && !jump_to_max_no_ts) {
        ppol->floor_freq = new_freq;
        ppol->floor_validate_time = now;
    }

    if (start_hyst && new_freq >= ppol->policy->max && !jump_to_max_no_ts)
        ppol->max_freq_hyst_start_time = now;

    if (ppol->target_freq == new_freq &&
            ppol->target_freq <= ppol->policy->cur) {
        trace_cpufreq_interactive_already(
            max_cpu, pol_load, ppol->target_freq,
            ppol->policy->cur, new_freq);
        spin_unlock_irqrestore(&ppol->target_freq_lock, flags);
        goto rearm;
    }

    trace_cpufreq_interactive_target(max_cpu, pol_load, ppol->target_freq,
                     ppol->policy->cur, new_freq);

    ppol->target_freq = new_freq;
    spin_unlock_irqrestore(&ppol->target_freq_lock, flags);
    spin_lock_irqsave(&speedchange_cpumask_lock, flags);
    cpumask_set_cpu(max_cpu, &speedchange_cpumask);
    spin_unlock_irqrestore(&speedchange_cpumask_lock, flags);
    wake_up_process_no_notif(speedchange_task);

rearm:
    if (!timer_pending(&ppol->policy_timer))
        cpufreq_interactive_timer_resched(data, false);

    /*
     * Send govinfo notification.
     * Govinfo notification could potentially wake up another thread
     * managed by its clients. Thread wakeups might trigger a load
     * change callback that executes this function again. Therefore
     * no spinlock could be held when sending the notification.
     */
    for_each_cpu(i, ppol->policy->cpus) {
        pcpu = &per_cpu(cpuinfo, i);
        govinfo.cpu = i;
        govinfo.load = pcpu->loadadjfreq / ppol->policy->max;
        govinfo.sampling_rate_us = tunables->timer_rate;
        atomic_notifier_call_chain(&cpufreq_govinfo_notifier_list,
                       CPUFREQ_LOAD_CHANGE, &govinfo);
    }

exit:
    up_read(&ppol->enable_sem);
    return;
}

|→

void sched_get_cpus_busy(struct sched_load *busy,
             const struct cpumask *query_cpus)
{
    unsigned long flags;
    struct rq *rq;
    const int cpus = cpumask_weight(query_cpus);
    u64 load[cpus], group_load[cpus];
    u64 nload[cpus], ngload[cpus];
    u64 pload[cpus];
    unsigned int max_freq[cpus];
    int notifier_sent = 0;
    int early_detection[cpus];
    int cpu, i = 0;
    unsigned int window_size;
    u64 max_prev_sum = 0;
    int max_busy_cpu = cpumask_first(query_cpus);
    u64 total_group_load = 0, total_ngload = 0;
    bool aggregate_load = false;
    struct sched_cluster *cluster = cpu_cluster(cpumask_first(query_cpus));

    if (unlikely(cpus == 0))
        return;

    local_irq_save(flags);

    /*
     * This function could be called in timer context, and the
     * current task may have been executing for a long time. Ensure
     * that the window stats are current by doing an update.
     */

    for_each_cpu(cpu, query_cpus)
        raw_spin_lock_nested(&cpu_rq(cpu)->lock, cpu);

    window_size = sched_ravg_window;

    /*
     * We don't really need the cluster lock for this entire for loop
     * block. However, there is no advantage in optimizing this as rq
     * locks are held regardless and would prevent migration anyways
     */
    raw_spin_lock(&cluster->load_lock);

    for_each_cpu(cpu, query_cpus) {
        rq = cpu_rq(cpu);

        update_task_ravg(rq->curr, rq, TASK_UPDATE, sched_ktime_clock(),
                 0);

        account_load_subtractions(rq);

        /* (1) 获取: 
            cpu上一个窗口的负载：rq->prev_runnable_sum
            cpu上一个窗口的的新任务负载：rq->nt_prev_runnable_sum
            cpu上一个窗口的负载预测：rq->hmp_stats.pred_demands_sum
         */
        load[i] = rq->prev_runnable_sum;
        nload[i] = rq->nt_prev_runnable_sum;
        pload[i] = rq->hmp_stats.pred_demands_sum;
        rq->old_estimated_time = pload[i];

        if (load[i] > max_prev_sum) {
            max_prev_sum = load[i];
            max_busy_cpu = cpu;
        }

        /*
         * sched_get_cpus_busy() is called for all CPUs in a
         * frequency domain. So the notifier_sent flag per
         * cluster works even when a frequency domain spans
         * more than 1 cluster.
         */
        if (rq->cluster->notifier_sent) {
            notifier_sent = 1;
            rq->cluster->notifier_sent = 0;
        }
        early_detection[i] = (rq->ed_task != NULL);
        max_freq[i] = cpu_max_freq(cpu);
        i++;
    }

    raw_spin_unlock(&cluster->load_lock);

    group_load_in_freq_domain(
            &cpu_rq(max_busy_cpu)->freq_domain_cpumask,
            &total_group_load, &total_ngload);
    aggregate_load = !!(total_group_load > sched_freq_aggregate_threshold);

    i = 0;
    for_each_cpu(cpu, query_cpus) {
        group_load[i] = 0;
        ngload[i] = 0;

        if (early_detection[i])
            goto skip_early;

        rq = cpu_rq(cpu);
        if (aggregate_load) {
            if (cpu == max_busy_cpu) {
                group_load[i] = total_group_load;
                ngload[i] = total_ngload;
            }
        } else {
            group_load[i] = rq->grp_time.prev_runnable_sum;
            ngload[i] = rq->grp_time.nt_prev_runnable_sum;
        }

        load[i] += group_load[i];
        nload[i] += ngload[i];

        load[i] = freq_policy_load(rq, load[i]);
        rq->old_busy_time = load[i];

        /*
         * Scale load in reference to cluster max_possible_freq.
         *
         * Note that scale_load_to_cpu() scales load in reference to
         * the cluster max_freq.
         */
        load[i] = scale_load_to_cpu(load[i], cpu);
        nload[i] = scale_load_to_cpu(nload[i], cpu);
        pload[i] = scale_load_to_cpu(pload[i], cpu);
skip_early:
        i++;
    }

    for_each_cpu(cpu, query_cpus)
        raw_spin_unlock(&(cpu_rq(cpu))->lock);

    local_irq_restore(flags);

    i = 0;
    for_each_cpu(cpu, query_cpus) {
        rq = cpu_rq(cpu);

        if (early_detection[i]) {
            busy[i].prev_load = div64_u64(sched_ravg_window,
                            NSEC_PER_USEC);
            busy[i].new_task_load = 0;
            busy[i].predicted_load = 0;
            goto exit_early;
        }

        load[i] = scale_load_to_freq(load[i], max_freq[i],
                cpu_max_possible_freq(cpu));
        nload[i] = scale_load_to_freq(nload[i], max_freq[i],
                cpu_max_possible_freq(cpu));

        pload[i] = scale_load_to_freq(pload[i], max_freq[i],
                         rq->cluster->max_possible_freq);


        /* (2) 负载经过转换后赋值给busy: 
            cpu上一个窗口的负载：busy[i].prev_load
            cpu上一个窗口的的新任务负载：busy[i].new_task_load
            cpu上一个窗口的负载预测：busy[i].predicted_load
         */
        busy[i].prev_load = div64_u64(load[i], NSEC_PER_USEC);
        busy[i].new_task_load = div64_u64(nload[i], NSEC_PER_USEC);
        busy[i].predicted_load = div64_u64(pload[i], NSEC_PER_USEC);

exit_early:
        trace_sched_get_busy(cpu, busy[i].prev_load,
                     busy[i].new_task_load,
                     busy[i].predicted_load,
                     early_detection[i],
                     aggregate_load &&
                      cpu == max_busy_cpu);
        i++;
    }
}