TFServing—AI模型服务部署框架（二）
前一节讲了tensorflow-serving单个模型部署，本节接着介绍tfserving通过配置文件，部署多个模型，以及多个模型的不同版本。
1.本地保存多个测试模型，用于测试
2.编写配置文件，models.config，base_path表示容器中，模型的路径
model_config_list {
config {
name: 'imagecls'
base_path: '/models/imagecls'
model_platform: 'tensorflow'
model_version_policy{
specific {
versions:1
}
}
}
config {
name: 'mobilenetv3'
base_path: '/models/mobilenetv3'
model_platform: 'tensorflow'
model_version_policy{
specific {
versions: 2
}
}
}
}3.启动服务
/home/hexiong/tfmodels:/models，将本地/home/hexiong/tfmodels文件夹下的所有文件复制到容器/models文件夹下
--model_config_file中的路径是，复制后容器的路径
sudo docker run -t --rm -p 8500:8500 -p 8501:8501 -v /home/hexiong/tfmodels:/models tensorflow/serving:2.8.4-gpu --model_config_file=/models/mod
el.config4.grpc访问服务
主要几个参数：
# 模型名称
request.model_spec.name = 'mobilenetv3'
#模型签名，默认serving_default
request.model_spec.signature_name = 'serving_default'
#版本号
request.model_spec.version.value = 2
import grpc
import time
from tensorflow_serving.apis import predict_pb2, prediction_service_pb2_grpc
from tensorflow.keras.preprocessing import image
from tensorflow.keras.applications.efficientnet_v2 import decode_predictions
import numpy as np
import tensorflow as tf
options = [('grpc.max_send_message_length', 1000 * 1024 * 1024),
('grpc.max_receive_message_length', 1000 * 1024 * 1024)]
channel = grpc.insecure_channel("localhost:8500", options=options)
stub = prediction_service_pb2_grpc.PredictionServiceStub(channel)
request = predict_pb2.PredictRequest()
# request.model_spec.name = 'imagecls'
# request.model_spec.signature_name = 'serving_default'
# request.model_spec.version.value = 1
request.model_spec.name = 'mobilenetv3'
request.model_spec.signature_name = 'serving_default'
request.model_spec.version.value = 2
# 输入图片并进行请求
img = image.load_img("./images/test.png", target_size=(480, 480))
img = np.expand_dims(img, axis=0)
print(img)
request.inputs['input_1'].CopyFrom(tf.make_tensor_proto(img, tf.float32))
start = time.time()
result_future = stub.Predict.future(request, 10.0) # 10 secs timeout
res = tf.make_ndarray(result_future.result().outputs['predictions'])
print(res)
res = decode_predictions(res, top=3)
print(res)5.grpc模型状态查询，使用Postman
http://localhost:8501/v1/models/imagecls
{
"model_version_status": [
{
"version": "1",
"state": "AVAILABLE",
"status": {
"error_code": "OK",
"error_message": ""
}
}
]
}http://localhost:8501/v1/models/mobilenetv3
{
"model_version_status": [
{
"version": "2",
"state": "AVAILABLE",
"status": {
"error_code": "OK",
"error_message": ""
}
}
]
}查询元数据，可以看到输入、输出格式
http://localhost:8501/v1/models/imagecls/versions/1/metadata
{
"model_spec": {
"name": "imagecls",
"signature_name": "",
"version": "1"
},
"metadata": {
"signature_def": {
"signature_def": {
"serving_default": {
"inputs": {
"input_1": {
"dtype": "DT_FLOAT",
"tensor_shape": {
"dim": [
{
"size": "-1",
"name": ""
},
{
"size": "480",
"name": ""
},
{
"size": "480",
"name": ""
},
{
"size": "3",
"name": ""
}
],
"unknown_rank": false
},
"name": "serving_default_input_1:0"
}
},
"outputs": {
"predictions": {
"dtype": "DT_FLOAT",
"tensor_shape": {
"dim": [
{
"size": "-1",
"name": ""
},
{
"size": "1000",
"name": ""
}
],
"unknown_rank": false
},
"name": "StatefulPartitionedCall:0"
}
},
"method_name": "tensorflow/serving/predict"
},
"__saved_model_init_op": {
"inputs": {},
"outputs": {
"__saved_model_init_op": {
"dtype": "DT_INVALID",
"tensor_shape": {
"dim": [],
"unknown_rank": true
},
"name": "NoOp"
}
},
"method_name": ""
}
}
}
}
}6.模型增加、删除、版本升级过程：加载models.config文件，通过参数直接进行修改。
在imagecls，mobilenetv3基础上，增加模型test1
import grpc
from google.protobuf import text_format
from tensorflow_serving.apis import model_management_pb2
from tensorflow_serving.apis import model_service_pb2_grpc
from tensorflow_serving.config import model_server_config_pb2
def update(config, url):
channel = grpc.insecure_channel(url)
stub = model_service_pb2_grpc.ModelServiceStub(channel)
request = model_management_pb2.ReloadConfigRequest()
#更新config
config_content = open(config, "r").read()
model_server_config = model_server_config_pb2.ModelServerConfig()
model_server_config = text_format.Parse(text=config_content, message=model_server_config)
model_configs = model_server_config.model_config_list.config # 数组，存放所有模型
# 修改模型config
config_alter = model_configs[1]
# 删除模型
# del model_configs[0]
# 更改模型版本
config_alter.model_version_policy.specific.versions.extend([3])
# #添加新的config, tfmodels相应文件夹下，需要创建新模型
config_list = model_server_config_pb2.ModelConfigList()
new_config = config_list.config.add()
new_config.name = "test1"
new_config.base_path = "/models/test1"
new_config.model_platform = "tensorflow"
# 设置模型版本
new_config.model_version_policy.specific.versions.extend([1])
model_server_config.model_config_list.MergeFrom(config_list)
# requests
print(new_config)
request.config.CopyFrom(model_server_config)
request_response = stub.HandleReloadConfigRequest(request, 10)
open(config, "w").write(str(request.config))
config_file = "model.config"
url = "localhost:8500"
update(config_file, url)7.模型删除后，查询
http://localhost:8501/v1/models/imagecls
{
"model_version_status": [
{
"version": "1",
"state": "END",
"status": {
"error_code": "OK",
"error_message": ""
}
}
]
}8.grpc查询接口
import cv2
import grpc
import time
from tensorflow_serving.apis import predict_pb2, prediction_service_pb2_grpc, model_service_pb2_grpc ,get_model_status_pb2, get_model_metadata_pb2
from tensorflow.keras.preprocessing import image
from tensorflow.keras.applications.efficientnet_v2 import decode_predictions
from google.protobuf.json_format import MessageToJson
import numpy as np
import tensorflow as tf
def get_model_status():
channel = grpc.insecure_channel("localhost:8500")
stub = model_service_pb2_grpc.ModelServiceStub(channel)
request = get_model_status_pb2.GetModelStatusRequest()
request.model_spec.name = "mobilenetv3"
request.model_spec.signature_name = "serving_default"
request.model_spec.version.value = 3
response = stub.GetModelStatus (request, 10)
print(response)
def get_model_metadata():
channel = grpc.insecure_channel("localhost:8500")
stub = prediction_service_pb2_grpc.PredictionServiceStub(channel)
request = get_model_metadata_pb2.GetModelMetadataRequest()
request.model_spec.name = "mobilenetv3"
request.model_spec.signature_name = "serving_default"
request.model_spec.version.value = 3
request.metadata_field.append('signature_def')
response = stub.GetModelMetadata(request, 10)
print(MessageToJson(response))以上，tesorflow serving基本使用介绍完毕，后续可以能继续更新，tfserving k8s上部署文档。