FATE —— 三. 使用DSL json conf运行作业
DSL Conf教程
本文档将提供如何使用DSL Conf运行训练和预测任务的简短教程。我们将以hetero-secureboost为例。
上载数据
在运行作业之前，需要将数据上载到数据存储。请参考此处的示例。
提交培训任务
我们可以通过Flow Client提交conf和dsl来开始培训工作。在这里，我们提交一个异类secureboost二进制分类任务，其conf和dsl位于异类secureboost示例文件夹中。
flow job submit -c ./examples/dsl/v2/hetero_secureboost/test_secureboost_train_binary_conf.json -d ./examples/dsl/v2/hetero_secureboost/test_secureboost_train_dsl.json
然后我们可以得到一条包含model_id和model_version的返回消息。一定要注意，记住model_id和model_version。
{
"data": {
"board_url": "http://127.0.0.1:8080/index.html#/dashboard?job_id=202301141152024400770&role=guest&party_id=9999",
"code": 0,
"dsl_path": "/home/mengdu/standalone_fate_install_1.10.0_release/fateflow/jobs/202301141152024400770/job_dsl.json",
"job_id": "202301141152024400770",
"logs_directory": "/home/mengdu/standalone_fate_install_1.10.0_release/fateflow/logs/202301141152024400770",
"message": "success",
"model_info": {
"model_id": "guest-9999#host-9998#model",
"model_version": "202301141152024400770"
},
"pipeline_dsl_path": "/home/mengdu/standalone_fate_install_1.10.0_release/fateflow/jobs/202301141152024400770/pipeline_dsl.json",
"runtime_conf_on_party_path": "/home/mengdu/standalone_fate_install_1.10.0_release/fateflow/jobs/202301141152024400770/guest/9999/job_runtime_on_party_conf.json",
"runtime_conf_path": "/home/mengdu/standalone_fate_install_1.10.0_release/fateflow/jobs/202301141152024400770/job_runtime_conf.json",
"train_runtime_conf_path": "/home/mengdu/standalone_fate_install_1.10.0_release/fateflow/jobs/202301141152024400770/train_runtime_conf.json"
},
"jobId": "202301141152024400770", # 一定要注意，这里的jobId不同，记得替换成自己的
"retcode": 0,
"retmsg": "success"
}检索model_id和model_version
忘记在返回的消息中保存model_id和model_version？不用担心。您可以使用“flow job config”命令查询作业的相应model_id和model_version。将jobId替换成自己的后，再执行。
flow job config -j 202301141152024400770 -r guest -p 9999 -o ./执行完后，返回信息如下
{
"data": {
"job_id": "202301141152024400770",
"model_info": {
"model_id": "guest-9999#host-9998#model", # 部署模型所需model_id
"model_version": "202301141152024400770" # 部署模型所需model_version
},
"train_runtime_conf": {}
},
"retcode": 0,
"retmsg": "download successfully, please check /home/mengdu/standalone_fate_install_1.10.0_release/job_202301141152024400770_config directory",
"directory": "/home/mengdu/standalone_fate_install_1.10.0_release/job_202301141152024400770_config"
}生成预测conf并生成预测dsl
我们使用flow_client部署预测任务中所需的组件：将jobId以及model_id和model_version替换成自己的后，再执行。
flow model deploy --model-id guest-9999#host-9998#model --model-version 202301141152024400770 --cpn-list "data_transform_0, intersection_0, hetero_secure_boost_0"执行完后，返回信息如下
{
"data": {
"detail": {
"guest": {
"9999": {
"retcode": 0,
"retmsg": "deploy model of role guest 9999 success"
}
},
"host": {
"9998": {
"retcode": 0,
"retmsg": "deploy model of role host 9998 success"
}
}
},
"guest": {
"9999": 0
},
"host": {
"9998": 0
},
"model_id": "guest-9999#host-9998#model", # 用于预测conf
"model_version": "202301141200341245360" # 用于预测conf
},
"retcode": 0,
"retmsg": "success"
}然后我们可以通过deploymodel_id和model_version获得返回消息。
预测conf- json详解
我们可以通过将model_id、model_version和数据集名称替换为您的名称来修改现有的预测conf，以生成新的预测conf。这里，我们将预测conf中的model_id和model_versin替换为部署模型作业返回的model_id和model_version。
{
"dsl_version": 2,
"initiator": {
"role": "guest",
"party_id": 10000
},
"role": {
"host": [
9999
],
"guest": [
10000
]
},
"job_parameters": {
"common": {
"job_type": "predict",
"model_id": "guest-9999#host-9998#model", <<-- 要替换，请按部署模型返回
"model_version": "202301141152024400770" <<-- 要替换，请按部署模型返回
}
},
"component_parameters": {
"role": {
"guest": {
"0": {
"reader_0": {
"table": {
"name": "breast_hetero_guest", <<-- 可以在此处设置新数据集
"namespace": "experiment"
}
}
}
},
"host": {
"0": {
"reader_0": {
"table": {
"name": "breast_hetero_host", <<-- 可以在此处设置新数据集
"namespace": "experiment"
}
}
}
}
}
}
}我们还可以生成deploymodel返回的predictconf、model_id和model_version：
flow model get-predict-conf --model-id guest-9999#host-9998#model --model-version 202301141152024400770 -o ./
{
"retcode": 0,
"retmsg": "Query predict conf successfully. File path is: /home/mengdu/standalone_fate_install_1.10.0_release/predict_conf_20230114122548.json"
}predict_conf_20230114122548.json内容如下
官网给出的代码如下
{
"data": {
"component_parameters": {
"role": {
"guest": {
"0": {
"reader_1": {
"table": {
"name": "name_to_be_filled_0", <<-- 可以在此处设置新数据集
"namespace": "namespace_to_be_filled_0" <<-- 可以在此处设置新数据集
}
}
}
},
"host": {
"0": {
"reader_1": {
"table": {
"name": "name_to_be_filled_0", <<-- 可以在此处设置新数据集
"namespace": "namespace_to_be_filled_0" <<-- 可以在此处设置新数据集
}
}
}
}
}
},
"dsl_version": 2,
"initiator": {
"party_id": 9999,
"role": "guest"
},
"job_parameters": {
"common": {
"job_type": "predict",
"model_id": "guest-9999#host-9998#model", <<-- 不需要设置
"model_version": "202301141152024400770", <<-- 不需要设置
}
},
"role": {
"guest": [
12001
],
"host": [
12000
]
}
},
"retcode": 0,
"retmsg": "success"
}
笔者生成的代码如下
{"dsl_version": 2, "role": {"host": [9998], "guest": [9999]}, "initiator": {"role": "guest", "party_id": 9999}, "job_parameters": {"common": {"job_type": "predict", "inheritance_info": {}, "computing_engine": "STANDALONE", "engines_address": {}, "federated_mode": "SINGLE", "task_parallelism": 1, "computing_partitions": 4, "federated_status_collect_type": "PUSH", "model_id": "guest-9999#host-9998#model", "model_version": "202301141152024400770", "auto_retries": 0, "auto_retry_delay": 1, "eggroll_run": {}, "spark_run": {}, "rabbitmq_run": {}, "pulsar_run": {}, "adaptation_parameters": {"task_nodes": 1, "task_cores_per_node": 4, "task_memory_per_node": 0, "request_task_cores": 4, "if_initiator_baseline": true}}}, "component_parameters": {"role": {"host": {"0": {"reader_0": {"table": {"name": "name_to_be_filled_0", "namespace": "namespace_to_be_filled_0"}}, "reader_1": {"table": {"name": "name_to_be_filled_1", "namespace": "namespace_to_be_filled_1"}}}}, "guest": {"0": {"reader_0": {"table": {"name": "name_to_be_filled_0", "namespace": "namespace_to_be_filled_0"}}, "reader_1": {"table": {"name": "name_to_be_filled_1", "namespace": "namespace_to_be_filled_1"}}}}}}}提交预测作业
然后我们可以提交一个新的预测作业（文件位置自行调整）：
flow job submit -c ./test_predict_conf.jsontest_predict_conf.json如下
官网给出的代码如下
{
"dsl_version": 2,
"initiator": {
"role": "guest",
"party_id": 9999
},
"role": {
"host": [
10000
],
"guest": [
9999
]
},
"job_parameters": {
"common": {
"model_id": "guest-10000#host-9999#model",
"model_version": "20200928174750711017114",
"job_type": "predict"
}
},
"component_parameters": {
"role": {
"guest": {
"0": {
"reader_0": {
"table": {
"name": "breast_hetero_guest",
"namespace": "experiment"
}
}
}
},
"host": {
"0": {
"reader_0": {
"table": {
"name": "breast_hetero_host",
"namespace": "experiment"
}
}
}
}
}
}
}笔者写的代码如下
{
"dsl_version": 2,
"initiator": {
"role": "guest",
"party_id": 9999
},
"role": {
"host": [
9998
],
"guest": [
9999
]
},
"job_parameters": {
"common": {
"model_id": "guest-9999#host-9998#model",
"model_version": "202301141152024400770",
"job_type": "predict"
}
},
"component_parameters": {
"role": {
"guest": {
"0": {
"reader_0": {
"table": {
"name": "breast_hetero_guest",
"namespace": "experiment"
}
}
}
},
"host": {
"0": {
"reader_0": {
"table": {
"name": "breast_hetero_host",
"namespace": "experiment"
}
}
}
}
}
}
}