hive metastore 参数 metastore.limit.partition.request 不起作用

MetastoreConf 中定义参数 metastore.limit.partition.request", 可以限制请求的分区数量，但是在实际测试中不起作用。代码分析如下：

1. MetastoreConf 中定义参数

LIMIT_PARTITION_REQUEST("metastore.limit.partition.request",
        "hive.metastore.limit.partition.request", -1,
        "This limits the number of partitions (whole partition objects) that can be requested " +
        "from the metastore for a give table. MetaStore API methods using this are: \n" +
                "get_partitions, \n" +
                "get_partitions_with_auth, \n" +
                "get_partitions_by_filter, \n" +
                "get_partitions_by_expr.\n" +
            "The default value \"-1\" means no limit."),

2. DDLTask.execute

当执行 show partitions xxx_table 时，DDLWork 类型的work.getShowPartsDesc() 不为空。

      ShowPartitionsDesc showParts = work.getShowPartsDesc();
      if (showParts != null) {
        return showPartitions(db, showParts);
      }

3. DDLTask.showPartitions

在此方法中可以看到，不论是否有分区，调用db,getPartitionNames 最后一个参数总是 -1

private int showPartitions(Hive db, ShowPartitionsDesc showParts) throws HiveException {
    // get the partitions for the table and populate the output
    String tabName = showParts.getTabName();
    Table tbl = null;
    List<String> parts = null;

    tbl = db.getTable(tabName);

    if (!tbl.isPartitioned()) {
      throw new HiveException(ErrorMsg.TABLE_NOT_PARTITIONED, tabName);
    }
    if (showParts.getPartSpec() != null) {
      parts = db.getPartitionNames(tbl.getDbName(),
          tbl.getTableName(), showParts.getPartSpec(), (short) -1);
    } else {
      parts = db.getPartitionNames(tbl.getDbName(), tbl.getTableName(), (short) -1);
    }

    // write the results in the file
    DataOutputStream outStream = getOutputStream(showParts.getResFile());
    try {
      formatter.showTablePartitions(outStream, parts);
    } catch (Exception e) {
      throw new HiveException(e, ErrorMsg.GENERIC_ERROR, "show partitions for table " + tabName);
    } finally {
      IOUtils.closeStream(outStream);
    }

    return 0;
  }

5. Hive.getPartitionNames

public List<String> getPartitionNames(String dbName, String tblName, short max)
      throws HiveException {
    List<String> names = null;
    try {
      names = getMSC().listPartitionNames(dbName, tblName, max);
    } catch (Exception e) {
      LOG.error(StringUtils.stringifyException(e));
      throw new HiveException(e);
    }
    return names;
  }

6. HiveMetastoreClient.listPartitionNames

 public List<String> listPartitionNames(String dbName, String tblName,
      short max) throws NoSuchObjectException, MetaException, TException {
    return listPartitionNames(getDefaultCatalog(conf), dbName, tblName, max);
  }

7. 调用 thrift 的 listPartitionNames

  public List<String> listPartitionNames(String catName, String dbName, String tableName,
                                         int maxParts) throws TException {
    return filterHook.filterPartitionNames(catName, dbName, tableName,
        client.get_partition_names(prependCatalogToDbName(catName, dbName, conf), tableName, shrinkMaxtoShort(maxParts)));
  }

HMSHandler.get_partition_names

HMSHandler.get_partition_names 处理 RPC 请求。
getMS 返回 RawStore，一般实现时 ObjectStore。注意 max_parts 是 -1.

@Override
    public List<String> get_partition_names(final String db_name, final String tbl_name,
                                            final short max_parts) throws NoSuchObjectException, MetaException {
        String[] parsedDbName = parseDbName(db_name, conf);
        startTableFunction("get_partition_names", parsedDbName[CAT_NAME], parsedDbName[DB_NAME], tbl_name);
        fireReadTablePreEvent(parsedDbName[CAT_NAME], parsedDbName[DB_NAME], tbl_name);
        List<String> ret = null;
        Exception ex = null;
        try {
            ret = getMS().listPartitionNames(parsedDbName[CAT_NAME], parsedDbName[DB_NAME], tbl_name,
                                             max_parts);
        } catch (Exception e) {
            ex = e;
            if (e instanceof MetaException) {
                throw (MetaException) e;
            } else {
                throw newMetaException(e);
            }
        } finally {
            endFunction("get_partition_names", ret != null, ex, tbl_name);
        }
        return ret;
    }

ObjectStore.listPartitionNames

@Override
  public List<String> listPartitionNames(String catName, String dbName, String tableName,
      short max) throws MetaException {
    List<String> pns = null;
    boolean success = false;
    try {
      openTransaction();
      LOG.debug("Executing getPartitionNames");
      pns = getPartitionNamesNoTxn(catName, dbName, tableName, max);
      success = commitTransaction();
    } finally {
      if (!success) {
        rollbackTransaction();
      }
    }
    return pns;
  }

ObjectStore.getPartitionNamesNoTxn

private List<String> getPartitionNamesNoTxn(String catName, String dbName, String tableName, short max) {
    List<String> pns = new ArrayList<>();
    if (max == 0) {
      return pns;
    }
    catName = normalizeIdentifier(catName);
    dbName = normalizeIdentifier(dbName);
    tableName = normalizeIdentifier(tableName);
    Query query =
        pm.newQuery("select partitionName from org.apache.hadoop.hive.metastore.model.MPartition "
            + "where table.database.name == t1 && table.tableName == t2 && table.database.catalogName == t3 "
            + "order by partitionName asc");
    query.declareParameters("java.lang.String t1, java.lang.String t2, java.lang.String t3");
    query.setResult("partitionName");

    if (max > 0) {
      query.setRange(0, max);
    }
    Collection<String> names = (Collection<String>) query.execute(dbName, tableName, catName);
    pns.addAll(names);

    if (query != null) {
      query.closeAll();
    }
    return pns;
  }

package.jdo

package.jdo 中 MPartition 映射为 PARTITIONS 表。

<class name="MPartition" table="PARTITIONS" identity-type="datastore" detachable="true">
      <index name="UniquePartition" unique="true">
        <column name="PART_NAME"/>
        <column name="TBL_ID"/>
      index>
      <datastore-identity>
        <column name="PART_ID"/>
      datastore-identity>
      <field name="partitionName">
        <column name="PART_NAME" length="767" jdbc-type="VARCHAR"/>
      field>
      <field name="table">
        <column name="TBL_ID"/>
      field>
      <field name="createTime">
        <column name="CREATE_TIME" jdbc-type="integer"/>
      field>
      <field name="lastAccessTime">
        <column name="LAST_ACCESS_TIME" jdbc-type="integer"/>
      field>
      <field name="values" table="PARTITION_KEY_VALS">
        <collection element-type="java.lang.String"/>
        <join>
          <column name="PART_ID"/>
        join>
        <element column="PART_KEY_VAL"/>
      field>
      <field name="sd" dependent="true">
        <column name="SD_ID"/>
      field>
      <field name="parameters" table="PARTITION_PARAMS">
        <map key-type="java.lang.String" value-type="java.lang.String"/>
        <join>
          <column name="PART_ID"/>
        join>
        <key>
           <column name="PARAM_KEY" length="256" jdbc-type="VARCHAR"/>
        key>
        <value>
           <column name="PARAM_VALUE" length="4000" jdbc-type="VARCHAR"/>
        value>
      field>
    class>