Apache Calcite HepPlanner源码学习

 calcite提供了两种查询优化器：

• 基于规则的启发式优化器HepPlanner(rule-based optimizer RBO)
• 基于代价的VolcanoPlanner（cost-based optimizer CBO)

本篇介绍HepPlanner的工作原理。在HepPlannerTest.java中添加如下代码进行源码分析：

// HepPlannerTest.java
  @Test public void testHepPlanner() {
    HepProgramBuilder programBuilder = HepProgram.builder();
    // 添加join中谓词下推的rule    
    programBuilder.addRuleInstance(FilterJoinRule.FilterIntoJoinRule.FILTER_ON_JOIN);
    HepPlanner planner = new HepPlanner(programBuilder.build());
    planner.setRoot(tester.convertSqlToRel("SElECT NAME FROM EMP e JOIN DEPT d ON e.DEPTNO = d.DEPTNO WHERE COMM > 1").rel);
    RelNode newRelRoot = planner.findBestExp();
    newRelRoot.toString();
  }

setRoot

首先需要设置planner root

  // HepPlanner.java
  public void setRoot(RelNode rel) {
    root = addRelToGraph(rel);
    dumpGraph();
  }
  // 递归将rel tree中的relnode转化成hepRelVetex, 构造dag
  private HepRelVertex addRelToGraph(
      RelNode rel) {
    // Check if a transformation already produced a reference
    // to an existing vertex.
    if (graph.vertexSet().contains(rel)) {
      return (HepRelVertex) rel;
    }

    // Recursively add children, replacing this rel's inputs
    // with corresponding child vertices.
    final List<RelNode> inputs = rel.getInputs();
    final List<RelNode> newInputs = new ArrayList<>();
    for (RelNode input1 : inputs) {
      HepRelVertex childVertex = addRelToGraph(input1);
      newInputs.add(childVertex);
    }
    // 当有节点修改，调用onCopyHook进行通知
    if (!Util.equalShallow(inputs, newInputs)) {
      RelNode oldRel = rel;
      rel = rel.copy(rel.getTraitSet(), newInputs);
      onCopy(oldRel, rel);
    }
    // Compute digest first time we add to DAG,
    // otherwise can't get equivVertex for common sub-expression
    rel.recomputeDigest();

    // try to find equivalent rel only if DAG is allowed
    if (!noDag) {
      // Now, check if an equivalent vertex already exists in graph.
      Pair<String, RelDataType> key = key(rel);
      HepRelVertex equivVertex = mapDigestToVertex.get(key);
      if (equivVertex != null) {
        // Use existing vertex.
        return equivVertex;
      }
    }

    // No equivalence:  create a new vertex to represent this rel.
    HepRelVertex newVertex = new HepRelVertex(rel);
    graph.addVertex(newVertex);
    updateVertex(newVertex, rel);

    for (RelNode input : rel.getInputs()) {
      graph.addEdge(newVertex, (HepRelVertex) input);
    }

    nTransformations++;
    return newVertex;
  }

addRelToGraph计算relNode Tree中每个节点的digest（calcite中用每个relNode都有其独一无二的digest，digest与relNode是一对一的关系），并根据relNode创建相应的HepRelVertex，HepRelVertex为有向无环图中的顶点，最终将relNode Tree转化成HepRelVertex表示的有向无环图。如Test中的relNode Tree

LogicalProject(NAME=[$10])
  LogicalFilter(condition=[>($6, 1)])
    LogicalJoin(condition=[=($7, $9)], joinType=[inner])
      LogicalTableScan(table=[[CATALOG, SALES, EMP]])
      LogicalTableScan(table=[[CATALOG, SALES, DEPT]])

经过addRelToGraph转化为

findBestExp

  // HepPlanner.java
  public RelNode findBestExp() {
    assert root != null;

    executeProgram(mainProgram);

    // Get rid of everything except what's in the final plan.
    collectGarbage();

    return buildFinalPlan(root);
  }
  
  private void applyRules(
      Collection<RelOptRule> rules,
      boolean forceConversions) {
    if (currentProgram.group != null) {
      assert currentProgram.group.collecting;
      currentProgram.group.ruleSet.addAll(rules);
      return;
    }

    LOGGER.trace("Applying rule set {}", rules);
    // 有节点与规则匹配后从root节点重新开始匹配
    boolean fullRestartAfterTransformation =
        currentProgram.matchOrder != HepMatchOrder.ARBITRARY
        && currentProgram.matchOrder != HepMatchOrder.DEPTH_FIRST;

    int nMatches = 0;

    boolean fixedPoint;
    do {
      Iterator<HepRelVertex> iter = getGraphIterator(root);
      fixedPoint = true;
      while (iter.hasNext()) {
        HepRelVertex vertex = iter.next();
        for (RelOptRule rule : rules) {
          HepRelVertex newVertex =
              applyRule(rule, vertex, forceConversions);
          if (newVertex == null || newVertex == vertex) {
            continue;
          }
          ++nMatches;
          if (nMatches >= currentProgram.matchLimit) {
            return;
          }
          if (fullRestartAfterTransformation) {
            iter = getGraphIterator(root);
          } else {
            // To the extent possible, pick up where we left
            // off; have to create a new iterator because old
            // one was invalidated by transformation.
            iter = getGraphIterator(newVertex);
            if (currentProgram.matchOrder == HepMatchOrder.DEPTH_FIRST) {
              nMatches =
                  depthFirstApply(iter, rules, forceConversions, nMatches);
              if (nMatches >= currentProgram.matchLimit) {
                return;
              }
            }
            // Remember to go around again since we're
            // skipping some stuff.
            fixedPoint = false;
          }
          break;
        }
      }
    } while (!fixedPoint);
  }
  
  private HepRelVertex applyRule(
      RelOptRule rule,
      HepRelVertex vertex,
      boolean forceConversions) {
    if (!belongsToDag(vertex)) {
      return null;
    }
    RelTrait parentTrait = null;
    List<RelNode> parents = null;
    if (rule instanceof ConverterRule) {
      // Guaranteed converter rules require special casing to make sure
      // they only fire where actually needed, otherwise they tend to
      // fire to infinity and beyond.
      ConverterRule converterRule = (ConverterRule) rule;
      if (converterRule.isGuaranteed() || !forceConversions) {
        if (!doesConverterApply(converterRule, vertex)) {
          return null;
        }
        parentTrait = converterRule.getOutTrait();
      }
    } else if (rule instanceof CommonRelSubExprRule) {
      // Only fire CommonRelSubExprRules if the vertex is a common
      // subexpression.
      List<HepRelVertex> parentVertices = getVertexParents(vertex);
      if (parentVertices.size() < 2) {
        return null;
      }
      parents = new ArrayList<>();
      for (HepRelVertex pVertex : parentVertices) {
        parents.add(pVertex.getCurrentRel());
      }
    }

    final List<RelNode> bindings = new ArrayList<>();
    final Map<RelNode, List<RelNode>> nodeChildren = new HashMap<>();
    boolean match =
        matchOperands(
            rule.getOperand(),
            vertex.getCurrentRel(),
            bindings,
            nodeChildren);

    if (!match) {
      return null;
    }

    HepRuleCall call =
        new HepRuleCall(
            this,
            rule.getOperand(),
            bindings.toArray(new RelNode[0]),
            nodeChildren,
            parents);

    // Allow the rule to apply its own side-conditions.
    if (!rule.matches(call)) {
      return null;
    }

    fireRule(call);

    if (!call.getResults().isEmpty()) {
      return applyTransformationResults(
          vertex,
          call,
          parentTrait);
    }

    return null;
  }

executeProgram最终调用applyRules对HepRelVertex DAG中满足rule的subGraph进行优化。applyRules工作流程如下：

• 根据MatchOrder遍历DAG，MatchOrder分为4种: (1) ARBITRARY (2) BOTTOM_UP (3) TOP_DOWN (4) DEPTH_FIRST，对每个vertex都尝试applyRule，如果rule与vertex匹配，则根据rule对该vertex进行优化，最终得到优化后的DAG

applyRule主要由以下几个部分组成：

• matchOperands：判断rule是否适用于该vertex（subGraph)
• fireRule: 按rule对vertex（subGraph）进行优化
  • rule.onMatch
• applyTransformationResults: 将vertex(subGraph)优化结果应用到relNode Tree以及DAG中生成新的优化后的relNode Tree和DAG

buildFinalPlan

 根据优化后的DAG，得到优化后的执行计划

通过HepPlanner优化后得到最终的执行计划：

LogicalProject(NAME=[$10])
    LogicalJoin(condition=[=($7, $9)], joinType=[inner])
      LogicalFilter(condition=[>($6, 1)])
        LogicalTableScan(table=[[CATALOG, SALES, EMP]])
      LogicalTableScan(table=[[CATALOG, SALES, DEPT]])