calcite提供了两种查询优化器:
本篇介绍HepPlanner的工作原理。在HepPlannerTest.java中添加如下代码进行源码分析:
// HepPlannerTest.java
@Test public void testHepPlanner() {
HepProgramBuilder programBuilder = HepProgram.builder();
// 添加join中谓词下推的rule
programBuilder.addRuleInstance(FilterJoinRule.FilterIntoJoinRule.FILTER_ON_JOIN);
HepPlanner planner = new HepPlanner(programBuilder.build());
planner.setRoot(tester.convertSqlToRel("SElECT NAME FROM EMP e JOIN DEPT d ON e.DEPTNO = d.DEPTNO WHERE COMM > 1").rel);
RelNode newRelRoot = planner.findBestExp();
newRelRoot.toString();
}
首先需要设置planner root
// HepPlanner.java
public void setRoot(RelNode rel) {
root = addRelToGraph(rel);
dumpGraph();
}
// 递归将rel tree中的relnode转化成hepRelVetex, 构造dag
private HepRelVertex addRelToGraph(
RelNode rel) {
// Check if a transformation already produced a reference
// to an existing vertex.
if (graph.vertexSet().contains(rel)) {
return (HepRelVertex) rel;
}
// Recursively add children, replacing this rel's inputs
// with corresponding child vertices.
final List<RelNode> inputs = rel.getInputs();
final List<RelNode> newInputs = new ArrayList<>();
for (RelNode input1 : inputs) {
HepRelVertex childVertex = addRelToGraph(input1);
newInputs.add(childVertex);
}
// 当有节点修改,调用onCopyHook进行通知
if (!Util.equalShallow(inputs, newInputs)) {
RelNode oldRel = rel;
rel = rel.copy(rel.getTraitSet(), newInputs);
onCopy(oldRel, rel);
}
// Compute digest first time we add to DAG,
// otherwise can't get equivVertex for common sub-expression
rel.recomputeDigest();
// try to find equivalent rel only if DAG is allowed
if (!noDag) {
// Now, check if an equivalent vertex already exists in graph.
Pair<String, RelDataType> key = key(rel);
HepRelVertex equivVertex = mapDigestToVertex.get(key);
if (equivVertex != null) {
// Use existing vertex.
return equivVertex;
}
}
// No equivalence: create a new vertex to represent this rel.
HepRelVertex newVertex = new HepRelVertex(rel);
graph.addVertex(newVertex);
updateVertex(newVertex, rel);
for (RelNode input : rel.getInputs()) {
graph.addEdge(newVertex, (HepRelVertex) input);
}
nTransformations++;
return newVertex;
}
addRelToGraph计算relNode Tree中每个节点的digest(calcite中用每个relNode都有其独一无二的digest,digest与relNode是一对一的关系),并根据relNode创建相应的HepRelVertex,HepRelVertex为有向无环图中的顶点,最终将relNode Tree转化成HepRelVertex表示的有向无环图。如Test中的relNode Tree
LogicalProject(NAME=[$10])
LogicalFilter(condition=[>($6, 1)])
LogicalJoin(condition=[=($7, $9)], joinType=[inner])
LogicalTableScan(table=[[CATALOG, SALES, EMP]])
LogicalTableScan(table=[[CATALOG, SALES, DEPT]])
// HepPlanner.java
public RelNode findBestExp() {
assert root != null;
executeProgram(mainProgram);
// Get rid of everything except what's in the final plan.
collectGarbage();
return buildFinalPlan(root);
}
private void applyRules(
Collection<RelOptRule> rules,
boolean forceConversions) {
if (currentProgram.group != null) {
assert currentProgram.group.collecting;
currentProgram.group.ruleSet.addAll(rules);
return;
}
LOGGER.trace("Applying rule set {}", rules);
// 有节点与规则匹配后从root节点重新开始匹配
boolean fullRestartAfterTransformation =
currentProgram.matchOrder != HepMatchOrder.ARBITRARY
&& currentProgram.matchOrder != HepMatchOrder.DEPTH_FIRST;
int nMatches = 0;
boolean fixedPoint;
do {
Iterator<HepRelVertex> iter = getGraphIterator(root);
fixedPoint = true;
while (iter.hasNext()) {
HepRelVertex vertex = iter.next();
for (RelOptRule rule : rules) {
HepRelVertex newVertex =
applyRule(rule, vertex, forceConversions);
if (newVertex == null || newVertex == vertex) {
continue;
}
++nMatches;
if (nMatches >= currentProgram.matchLimit) {
return;
}
if (fullRestartAfterTransformation) {
iter = getGraphIterator(root);
} else {
// To the extent possible, pick up where we left
// off; have to create a new iterator because old
// one was invalidated by transformation.
iter = getGraphIterator(newVertex);
if (currentProgram.matchOrder == HepMatchOrder.DEPTH_FIRST) {
nMatches =
depthFirstApply(iter, rules, forceConversions, nMatches);
if (nMatches >= currentProgram.matchLimit) {
return;
}
}
// Remember to go around again since we're
// skipping some stuff.
fixedPoint = false;
}
break;
}
}
} while (!fixedPoint);
}
private HepRelVertex applyRule(
RelOptRule rule,
HepRelVertex vertex,
boolean forceConversions) {
if (!belongsToDag(vertex)) {
return null;
}
RelTrait parentTrait = null;
List<RelNode> parents = null;
if (rule instanceof ConverterRule) {
// Guaranteed converter rules require special casing to make sure
// they only fire where actually needed, otherwise they tend to
// fire to infinity and beyond.
ConverterRule converterRule = (ConverterRule) rule;
if (converterRule.isGuaranteed() || !forceConversions) {
if (!doesConverterApply(converterRule, vertex)) {
return null;
}
parentTrait = converterRule.getOutTrait();
}
} else if (rule instanceof CommonRelSubExprRule) {
// Only fire CommonRelSubExprRules if the vertex is a common
// subexpression.
List<HepRelVertex> parentVertices = getVertexParents(vertex);
if (parentVertices.size() < 2) {
return null;
}
parents = new ArrayList<>();
for (HepRelVertex pVertex : parentVertices) {
parents.add(pVertex.getCurrentRel());
}
}
final List<RelNode> bindings = new ArrayList<>();
final Map<RelNode, List<RelNode>> nodeChildren = new HashMap<>();
boolean match =
matchOperands(
rule.getOperand(),
vertex.getCurrentRel(),
bindings,
nodeChildren);
if (!match) {
return null;
}
HepRuleCall call =
new HepRuleCall(
this,
rule.getOperand(),
bindings.toArray(new RelNode[0]),
nodeChildren,
parents);
// Allow the rule to apply its own side-conditions.
if (!rule.matches(call)) {
return null;
}
fireRule(call);
if (!call.getResults().isEmpty()) {
return applyTransformationResults(
vertex,
call,
parentTrait);
}
return null;
}
executeProgram最终调用applyRules对HepRelVertex DAG中满足rule的subGraph进行优化。applyRules工作流程如下:
applyRule主要由以下几个部分组成:
根据优化后的DAG,得到优化后的执行计划
通过HepPlanner优化后得到最终的执行计划:
LogicalProject(NAME=[$10])
LogicalJoin(condition=[=($7, $9)], joinType=[inner])
LogicalFilter(condition=[>($6, 1)])
LogicalTableScan(table=[[CATALOG, SALES, EMP]])
LogicalTableScan(table=[[CATALOG, SALES, DEPT]])