From 3439ab6c3a74a14366b2e85b8ab9a092b250a0f3 Mon Sep 17 00:00:00 2001 From: hellolee1234 Date: Mon, 4 Sep 2023 14:38:26 +0800 Subject: [PATCH 1/4] mt m2 inference --- Mt_M2_inference/M2.cfg | 51 +++ Mt_M2_inference/client.py | 67 ++++ Mt_M2_inference/client.sh | 4 + Mt_M2_inference/client_om.sh | 7 + Mt_M2_inference/server.sh | 11 + Mt_M2_inference/server_om.sh | 11 + Mt_M2_inference/tools/ckpt_convert.py | 61 ++++ Mt_M2_inference/tools/om.sh | 3 + Mt_M2_inference/tools/saved_model2om.py | 414 ++++++++++++++++++++++++ 9 files changed, 629 insertions(+) create mode 100644 Mt_M2_inference/M2.cfg create mode 100644 Mt_M2_inference/client.py create mode 100644 Mt_M2_inference/client.sh create mode 100644 Mt_M2_inference/client_om.sh create mode 100644 Mt_M2_inference/server.sh create mode 100644 Mt_M2_inference/server_om.sh create mode 100644 Mt_M2_inference/tools/ckpt_convert.py create mode 100644 Mt_M2_inference/tools/om.sh create mode 100644 Mt_M2_inference/tools/saved_model2om.py diff --git a/Mt_M2_inference/M2.cfg b/Mt_M2_inference/M2.cfg new file mode 100644 index 00000000..81e200e0 --- /dev/null +++ b/Mt_M2_inference/M2.cfg @@ -0,0 +1,51 @@ +platform_configs { +key: "tensorflow" + value { + source_adapter_config { + [type.googleapis.com/tensorflow.serving.SavedModelBundleSourceAdapterConfig] { + legacy_config { + session_config { + graph_options { + rewrite_options { + custom_optimizers { + name: "NpuOptimizer" + parameter_map: { + key: "use_off_line" + value: { + b: true + } + } + parameter_map: { + key: "mix_compile_mode" + value: { + b: true + } + } + parameter_map: { + key: "variable_placement" + value: { + s: "Host" + } + } + parameter_map: { + key: "graph_run_mode" + value: { + i: 0 + } + } + parameter_map: { + key: "precision_mode" + value: { + s: "force_fp16" + } + } + } + remapping: OFF + } + } + } + } + } + } + } +} diff --git a/Mt_M2_inference/client.py b/Mt_M2_inference/client.py new file mode 100644 index 00000000..ca657462 --- /dev/null +++ b/Mt_M2_inference/client.py @@ -0,0 +1,67 @@ +import tensorflow as tf +from tensorflow_serving.apis import predict_pb2 +from tensorflow_serving.apis import prediction_service_pb2_grpc +import grpc +import numpy as np +import os +import time + +class PredictModelGrpc(object): + def __init__(self, model_name, inputs,input_types, output_name, socket='xxx.xxx.xxx.xxx:8500'):# xxx.xxx.xxx.xxx为服务端IP地址 + self.socket = socket + self.model_name = model_name + self.inputs = inputs + self.input_types = input_types + self.output_name = output_name + self.request, self.stub = self.__get_request() + def __get_request(self): + channel = grpc.insecure_channel(self.socket, options=[('grpc.max_send_message_length', 1024 * 1024 * 1024), + ('grpc.max_receive_message_length', + 1024 * 1024 * 1024)]) # 可设置大小 + stub = prediction_service_pb2_grpc.PredictionServiceStub(channel) + request = predict_pb2.PredictRequest() + + request.model_spec.name = self.model_name + request.model_spec.signature_name = "serving_default" + + return request, stub + def inference(self): + + t0 = time.time() + for name in self.inputs: + self.request.inputs[name].CopyFrom(tf.make_tensor_proto(self.inputs[name], dtype=self.input_types[name]))# 发送请求 + t1 = time.time() + print("request.inputs={:.3f} ms".format((t1 - t0) * 1000)) + + for i in range(100): + result = self.stub.Predict.future(self.request, 1000.0) + result.result() + t2 = time.time() + print("request serving time cost: {:.3f} ms".format((t2 - t1) * 1000)) + t1 = t2 + + + res = [] + #print(result.result().outputs['task_1']) + #print(result.result().outputs['task_2']) + #res.append(tf.make_ndarray(result.result().outputs[self.output_name])[0]) # 获取结果 + + t3 = time.time() + return res + +def gen_inputs(): + from input_config import config + inputs = {} + input_types = {} + for name in config: + input_types[name] = config[name]['dtype'] + if config[name]['dtype'] == tf.int32: + inputs[name] = np.random.randint(0,100,size=config[name]['shape']) + elif config[name]['dtype'] == tf.float32: + inputs[name] = np.random.randint(0,2,size=config[name]['shape'])*1.0 + return inputs, input_types +if __name__ == '__main__': + inputs,input_types = gen_inputs() + model = PredictModelGrpc(model_name='saved_model', inputs = inputs, input_types = input_types, output_name='MobilenetV2/Logits/output:0',socket='127.0.0.1:9999') + + res = model.inference() diff --git a/Mt_M2_inference/client.sh b/Mt_M2_inference/client.sh new file mode 100644 index 00000000..8cd30b5d --- /dev/null +++ b/Mt_M2_inference/client.sh @@ -0,0 +1,4 @@ +export LD_LIBRARY_PATH=/usr/local/Ascend/ascend-toolkit/latest/lib64:/usr/local/Ascend/ascend-toolkit/latest/lib64/plugin/opskernel:/usr/local/Ascend/ascend-toolkit/latest/lib64/plugin/nnengine:/usr/local/Ascend/ascend-toolkit/latest/lib64:/usr/local/Ascend/ascend-toolkit/latest/lib64/plugin/opskernel:/usr/local/Ascend/ascend-toolkit/latest/lib64/plugin/nnengine:/usr/local/Ascend/ascend-toolkit/latest/lib64:/usr/local/Ascend/ascend-toolkit/latest/lib64/plugin/opskernel:/usr/local/Ascend/ascend-toolkit/latest/lib64/plugin/nnengine:/usr/local/Ascend/ascend-toolkit/latest/runtime/lib64:/usr/local/Ascend/driver/lib64:/usr/local/Ascend/driver/lib64/common:/usr/local/Ascend/driver/lib64/driver::/usr/local/python3.7.5/lib +unset http_proxy +unset https_proxy +python3 client.py diff --git a/Mt_M2_inference/client_om.sh b/Mt_M2_inference/client_om.sh new file mode 100644 index 00000000..d1e2a638 --- /dev/null +++ b/Mt_M2_inference/client_om.sh @@ -0,0 +1,7 @@ +export LD_LIBRARY_PATH=/usr/local/Ascend/ascend-toolkit/latest/lib64:/usr/local/Ascend/ascend-toolkit/latest/lib64/plugin/opskernel:/usr/local/Ascend/ascend-toolkit/latest/lib64/plugin/nnengine:/usr/local/Ascend/ascend-toolkit/latest/lib64:/usr/local/Ascend/ascend-toolkit/latest/lib64/plugin/opskernel:/usr/local/Ascend/ascend-toolkit/latest/lib64/plugin/nnengine:/usr/local/Ascend/ascend-toolkit/latest/lib64:/usr/local/Ascend/ascend-toolkit/latest/lib64/plugin/opskernel:/usr/local/Ascend/ascend-toolkit/latest/lib64/plugin/nnengine:/usr/local/Ascend/ascend-toolkit/latest/runtime/lib64:/usr/local/Ascend/driver/lib64:/usr/local/Ascend/driver/lib64/common:/usr/local/Ascend/driver/lib64/driver::/usr/local/python3.7.5/lib + +unset http_proxy + +unset https_proxy + +python3 client_om.py diff --git a/Mt_M2_inference/server.sh b/Mt_M2_inference/server.sh new file mode 100644 index 00000000..9dfaa30e --- /dev/null +++ b/Mt_M2_inference/server.sh @@ -0,0 +1,11 @@ +#export ASCEND_SLOG_PRINT_TO_STDOUT=1 +#export ASCEND_GLOBAL_LOG_LEVEL=1 +#export PROFILING_MODE=true +#export PROFILING_OPTIONS='{"output":"/tmp/profiling","task_trace":"on","fp_point":"","aic_metrics":"PipeUtilization"}' +taskset -c 0-32 tensorflow_model_server \ +--model_name=saved_model \ +--model_base_path=/home/l00832016/M2_infer/mt_inference_model/saved_model/ \ +--port=9999 \ +--enable_model_warmup=true \ +--platform_config_file=M2.cfg \ + diff --git a/Mt_M2_inference/server_om.sh b/Mt_M2_inference/server_om.sh new file mode 100644 index 00000000..51afffc4 --- /dev/null +++ b/Mt_M2_inference/server_om.sh @@ -0,0 +1,11 @@ +export ASCEND_SLOG_PRINT_TO_STDOUT=1 +export ASCEND_GLOBAL_LOG_LEVEL=0 +#export PROFILING_MODE=true +#export PROFILING_OPTIONS='{"output":"/tmp/profiling","task_trace":"on","fp_point":"","aic_metrics":"PipeUtilization"}' +#/home/l00832016/M2_infer/mt_inference_model/saved_model/ \ +taskset -c 0-32 tensorflow_model_server \ +--model_name=saved_model \ +--model_base_path=/home/l00832016/M2_infer/tools/om_model/ \ +--port=9999 \ +--enable_model_warmup=true \ +--platform_config_file=M2.cfg \ diff --git a/Mt_M2_inference/tools/ckpt_convert.py b/Mt_M2_inference/tools/ckpt_convert.py new file mode 100644 index 00000000..7b7914ed --- /dev/null +++ b/Mt_M2_inference/tools/ckpt_convert.py @@ -0,0 +1,61 @@ +import numpy as np +import tensorflow as tf + +import sys + + +def convert(cpu_ckpt_path,npu_ckpt_path,output_path): + reader_cpu = tf.train.NewCheckpointReader(cpu_ckpt_path) + reader_npu = tf.train.NewCheckpointReader(npu_ckpt_path) + + var_map_npu = {} + var_names_npu = reader_npu.get_variable_to_shape_map().keys() + for var_name in var_names_npu: + new_var = reader_npu.get_tensor(var_name) + var_map_npu[var_name] = new_var + + var_names_cpu = reader_cpu.get_variable_to_shape_map().keys() + var_map_cpu = {} + for var_name in var_names_cpu: + if var_name in ('embedding_table-values','embedding_table-keys'): + print("[INFO] Hash table %s found."%var_name) + continue + elif var_name not in var_map_npu: + print("[WARN] %s not found in npu ckpt!!!"%var_name) + var_map_cpu[var_name] = tf.Variable(reader_cpu.get_tensor(var_name), name = var_name) + else: + print("[Info] Assign %s."%var_name) + var_map_cpu[var_name] = tf.Variable(var_map_npu[var_name], name=var_name) + + values = np.load("one_ascend_hash_embedding_embedding.npy") + keys = np.load("one_ascend_hash_embedding_key.npy").astype(np.int32) + + var_map_cpu['embedding_table-values'] = tf.Variable(values,name = 'embedding_table-values') + var_map_cpu['embedding_table-keys'] = tf.Variable(keys,name = 'embedding_table-keys') + + + with tf.Session() as sess: + sess.run(tf.global_variables_initializer()) + tf.train.Saver().save(sess, output_path) + + for var_name in var_names_npu: + if var_name not in var_names_cpu: + print("[WARN] Attention!!!! %s not found in cpu ckpt!!!"%var_name) + print("Convert ckpt done!") + + +def main(): + cpu_ckpt_path = sys.argv[1] + npu_ckpt_path = sys.argv[2] + output_path = sys.argv[3] + print("[INFO] cpu_ckpt_path: %s"%cpu_ckpt_path) + print("[INFO] npu_ckpt_path: %s"%npu_ckpt_path) + print("[INFO] output_path: %s"%output_path) + convert(cpu_ckpt_path,npu_ckpt_path,output_path) + + +if __name__ == "__main__": + if(len(sys.argv) < 4): + print("ERROR! Usage: python3 ckpt_convert.py cpu_ckpt_path npu_ckpt_path output_path") + else: + main() diff --git a/Mt_M2_inference/tools/om.sh b/Mt_M2_inference/tools/om.sh new file mode 100644 index 00000000..3bd0f26f --- /dev/null +++ b/Mt_M2_inference/tools/om.sh @@ -0,0 +1,3 @@ +export LD_LIBRARY_PATH=/usr/local/Ascend/ascend-toolkit/latest/lib64:/usr/local/Ascend/ascend-toolkit/latest/lib64/plugin/opskernel:/usr/local/Ascend/ascend-toolkit/latest/lib64/plugin/nnengine:/usr/local/Ascend/ascend-toolkit/latest/lib64:/usr/local/Ascend/ascend-toolkit/latest/lib64/plugin/opskernel:/usr/local/Ascend/ascend-toolkit/latest/lib64/plugin/nnengine:/usr/local/Ascend/ascend-toolkit/latest/lib64:/usr/local/Ascend/ascend-toolkit/latest/lib64/plugin/opskernel:/usr/local/Ascend/ascend-toolkit/latest/lib64/plugin/nnengine:/usr/local/Ascend/ascend-toolkit/latest/runtime/lib64:/usr/local/Ascend/driver/lib64:/usr/local/Ascend/driver/lib64/common:/usr/local/Ascend/driver/lib64/driver::/usr/local/python3.7.5/lib + +python3 saved_model2om.py --input_path=/home/l00832016/M2_infer/mt_inference_model/saved_model/1693482994/ --output_path=./om_model/ --input_shape="feat_0:9600,40;feat_1:9600,40;feat_2:9600,40;feat_3:9600,1;feat_4:9600,8;feat_5:9600,32;feat_6:9600,6;feat_7:9600,40;feat_8:9600,40;feat_9:9600,16;feat_10:9600,40;feat_11:9600,8;feat_12:9600,480;feat_13:9600,8;feat_14:9600,40;feat_15:9600,1;feat_16:9600,8;feat_18:9600,1;feat_19:9600,8;feat_20:9600,40;feat_21:9600,32;feat_22:9600,1;feat_23:9600,40;feat_25:9600,6;feat_26:9600,40;feat_27:9600,8;feat_29:9600,40;feat_30:9600,8;feat_32:9600,1;feat_33:9600,256;feat_34:9600,40;feat_35:9600,40;feat_36:9600,8;feat_37:9600,1;feat_38:9600,1;feat_39:9600,1;feat_41:9600,32;feat_43:9600,40;feat_44:9600,6;feat_46:9600,9;feat_47:9600,8;feat_48:9600,4;feat_49:9600,40;feat_50:9600,40;feat_51:9600,40;feat_52:9600,40;feat_54:9600,100;feat_56:9600,40;feat_57:9600,8;feat_58:9600,40;feat_59:9600,40;feat_60:9600,6;feat_61:9600,8;feat_62:9600,40;feat_63:9600,6;feat_64:9600,1;feat_65:9600,1;feat_66:9600,1;feat_68:9600,6;feat_69:9600,8;feat_71:9600,40;feat_72:9600,40;feat_73:9600,40;feat_74:9600,8;feat_75:9600,10;feat_76:9600,40;feat_77:9600,40;feat_78:9600,40;feat_79:9600,8;feat_80:9600,1;feat_82:9600,6;feat_83:9600,1;feat_84:9600,32;feat_85:9600,40;feat_86:9600,8;feat_87:9600,40;feat_89:9600,6;feat_90:9600,40;feat_91:9600,40;feat_92:9600,1;feat_93:9600,40;feat_95:9600,1;feat_96:9600,8;feat_97:9600,320;feat_98:9600,1;feat_99:9600,1;feat_100:9600,1;feat_102:9600,40;feat_103:9600,1;feat_104:9600,1;feat_105:9600,40;feat_106:9600,8;feat_108:9600,6;feat_109:9600,8;feat_110:9600,8;feat_113:9600,8;feat_114:9600,8;feat_115:9600,60;feat_116:9600,40;feat_117:9600,40;feat_118:9600,6;feat_119:9600,40;feat_120:9600,13;feat_121:9600,3;feat_122:9600,9;feat_123:9600,6;feat_124:9600,40;feat_125:9600,40;feat_126:9600,6;feat_127:9600,8;feat_129:9600,40;feat_130:9600,40;feat_131:9600,8;feat_132:9600,1;feat_133:9600,40;feat_134:9600,10;feat_135:9600,1;feat_136:9600,33;feat_137:9600,40;feat_139:9600,6;feat_140:9600,40;feat_141:9600,8;feat_142:9600,26;feat_143:9600,40;feat_144:9600,1;feat_145:9600,8;feat_146:9600,8;feat_147:9600,40;feat_148:9600,6;feat_149:9600,40;feat_150:9600,8;feat_151:9600,6;feat_152:9600,7;feat_153:9600,8;feat_154:9600,8;feat_155:9600,40;feat_156:9600,8;feat_158:9600,8;feat_159:9600,8;feat_160:9600,40;feat_161:9600,40;feat_163:9600,6;feat_164:9600,40;feat_165:9600,40;feat_166:9600,6;feat_167:9600,40;feat_168:9600,6;feat_169:9600,40;feat_170:9600,40;feat_172:9600,1" --soc_version="Ascend310P3" diff --git a/Mt_M2_inference/tools/saved_model2om.py b/Mt_M2_inference/tools/saved_model2om.py new file mode 100644 index 00000000..8970ed82 --- /dev/null +++ b/Mt_M2_inference/tools/saved_model2om.py @@ -0,0 +1,414 @@ +import os +import argparse +import sys +import subprocess +import copy +import contextlib +import time +import shutil + +from time import strftime, localtime +from collections import namedtuple, OrderedDict + +import tensorflow as tf +from tensorflow.python.tools import saved_model_cli +from tensorflow.python.saved_model import tag_constants +from tensorflow.python.saved_model import signature_constants +from tensorflow.core.framework import types_pb2, graph_pb2, node_def_pb2, attr_value_pb2 +from tensorflow.compat.v1 import graph_util +from six import StringIO + +NodeInfo = namedtuple('NodeInfo', ['name', 'shape', 'type', 'full_name']) +TMP_PATH = '/home/l00832016/M2_infer/tools/om_model/saved_model2om' +TMP_PB_NAME = 'model.pb' +TMP_OM_NAME = 'model' + +TYPE_MAP = { + 'FP32': 'DT_FLOAT', + 'UINT8': 'DT_UINT8', + 'FP16': 'DT_HALF', +} + + +@contextlib.contextmanager +def captured_output(): + new_out, new_err = StringIO(), StringIO() + old_out, old_err = sys.stdout, sys.stderr + + try: + sys.stdout, sys.stderr = new_out, new_err + yield sys.stdout, sys.stderr + finally: + sys.stdout, sys.stderr = old_out, old_err + + +def get_input_output_node(saved_model_dir, saved_tags, sign): + parser = saved_model_cli.create_parser() + saved_model_cli_args = parser.parse_args([ + 'show', + '--dir', saved_model_dir, + '--tag_set', saved_tags, + '--signature_def', sign + ]) + + with captured_output() as (out, err): + saved_model_cli.show(saved_model_cli_args) + + result = out.getvalue().strip() + print(result) + + input_nodes = OrderedDict() + output_nodes = OrderedDict() + method_name = "" + method_name_mark = "Method name is:" + lines = result.split('\n') + for idx, line in enumerate(result.split('\n')): + if "inputs[" in line: + parse_node_from_line(idx, input_nodes, line, lines) + if "outputs[" in line: + parse_node_from_line(idx, output_nodes, line, lines) + if method_name_mark in line: + method_name = line[len(method_name_mark):].strip() + + if not output_nodes: + raise RuntimeError("No Output Nodes found in saved_model.") + if not method_name: + method_name = None + return input_nodes, output_nodes, method_name + + +def parse_node_from_line(idx, node_dict, line, lines): + type_line = lines[idx + 1] + shape_line = lines[idx + 2] + name_line = lines[idx + 3] + node_type = type_line.split(":")[1].strip() + node_name = name_line.split(":")[1].strip() + node_full_name = name_line[name_line.index(":") + 1:].strip() + node_shape = tuple(int(shape) if shape and int(shape) != -1 else None + for shape in shape_line.split(":")[1].strip()[1:-1].split(",")) + node_dict[line[line.index("'") + 1:line.rfind("'")]] = NodeInfo(node_name, node_shape, node_type, node_full_name) + + +def saved_pb(saved_model_dir, output_nodes, output_dir, output_name, saved_tags): + with tf.Session(graph=tf.Graph()) as sess: + tf.saved_model.load(sess, saved_tags, saved_model_dir) + graph = tf.compat.v1.graph_util.convert_variables_to_constants( + sess, + input_graph_def=sess.graph.as_graph_def(), + output_node_names=[node.name for node in output_nodes.values()] + ) + + tf.io.write_graph( + graph, + output_dir, + output_name, + as_text=False + ) + + return tuple(tensor.name for tensor in graph.node) + + +def saved_model_to_pb(saved_model_dir, output_dir, output_name, saved_tags=tag_constants.SERVING, + sign=signature_constants.DEFAULT_SERVING_SIGNATURE_DEF_KEY): + input_nodes, output_nodes, method_name = get_input_output_node(saved_model_dir, saved_tags, sign) + + print("[INFO]: Save Model has [", len(output_nodes), "] outputs.") + print("[INFO]: Outputs Nodes: ", output_nodes, ".") + + saved_node_names = saved_pb(saved_model_dir, output_nodes, output_dir, output_name, [saved_tags]) + + remove_keys = [] + for key, value in input_nodes.items(): + if value.name not in saved_node_names: + remove_keys.append(key) + + for key in remove_keys: + input_nodes.pop(key) + print("[INFO]: Inputs Nodes With Shapes: ", input_nodes, ".") + print("[INFO]: Saved Model convert to Frozen Model done.") + return input_nodes, output_nodes, method_name + + +def saved_sub_graph_saved_model(new_input_nodes, new_output_nodes, saved_model_dir, tmp_path, + saved_tags=tag_constants.SERVING, + sign=signature_constants.DEFAULT_SERVING_SIGNATURE_DEF_KEY): + input_node_names_dict, output_node_names_dict, method_name = \ + get_input_output_node(saved_model_dir, saved_tags, sign) + if new_input_nodes is not None: + input_node_names_dict = parse_new_input_nodes_string(new_input_nodes) + input_node_names = {input_node.name: input_node for input_node in input_node_names_dict.values()} + else: + input_node_names = OrderedDict() + + if new_output_nodes is not None: + output_node_names_dict = parse_new_output_nodes_string(new_output_nodes) + output_node_names = list(output_node.name for output_node in output_node_names_dict.values()) + with tf.Session(graph=tf.Graph()) as sess: + tf.saved_model.load(sess, [saved_tags], saved_model_dir) + input_graph_def = sess.graph.as_graph_def() + inputs_replaced_graph_def = graph_pb2.GraphDef() + for node in input_graph_def.node: + if node.name in input_node_names: + placeholder_node = node_def_pb2.NodeDef() + placeholder_node.op = "Placeholder" + placeholder_node.name = node.name + placeholder_node.attr["dtype"].CopyFrom( + attr_value_pb2.AttrValue(type=getattr(types_pb2, input_node_names.get(node.name).type))) + inputs_replaced_graph_def.node.extend([placeholder_node]) + else: + inputs_replaced_graph_def.node.extend([copy.deepcopy(node)]) + output_graph_def = graph_util.extract_sub_graph(inputs_replaced_graph_def, output_node_names) + + with tf.Session(graph=tf.Graph()) as sess: + tf.import_graph_def(output_graph_def, name="") + graph = tf.get_default_graph() + labeling_signature = tf.saved_model.signature_def_utils.build_signature_def( + inputs={k: tf.saved_model.utils.build_tensor_info(graph.get_tensor_by_name(v.full_name)) + for k, v in input_node_names_dict.items()}, + outputs={k: tf.saved_model.utils.build_tensor_info(graph.get_tensor_by_name(v.full_name)) + for k, v in output_node_names_dict.items()}, method_name=method_name) + output_path = os.path.join(tmp_path, "tmp_saved_model") + saved_model_builder = tf.saved_model.builder.SavedModelBuilder(output_path) + saved_model_builder.add_meta_graph_and_variables( + sess, [tf.saved_model.tag_constants.SERVING], signature_def_map={ + tf.saved_model.signature_constants.DEFAULT_SERVING_SIGNATURE_DEF_KEY: labeling_signature}) + saved_model_builder.save() + return output_path + + +def parse_new_input_nodes_string(new_nodes_string): + node_dict = OrderedDict() + for new_node_string in new_nodes_string.split(";"): + node_name_split = new_node_string.split(":") + name = node_name_split[0].strip() + node_type = node_name_split[1].strip() + node_dict[node_name_split[0].strip()] = NodeInfo(name, node_type, None, + ":".join(node_name_split[2:]).strip()) + return node_dict + + +def parse_new_output_nodes_string(new_nodes_string): + node_dict = OrderedDict() + for new_node_string in new_nodes_string.split(";"): + node_name_split = new_node_string.split(":") + node_dict[node_name_split[0].strip()] = NodeInfo(node_name_split[1].strip(), None, None, + ":".join(node_name_split[1:]).strip()) + return node_dict + + +def save_hw_saved_model(input_node_dict: dict, output_node_dict: dict, output_path, method_name, om_path): + try: + from npu_bridge.helper import helper + except ImportError: + print("[ERROR]: npu_bridge is not found, HW Saved Model will not be generated.") + return + + tf.disable_eager_execution() + from tensorflow.core.protobuf.rewriter_config_pb2 import RewriterConfig + config = tf.ConfigProto(allow_soft_placement=True, log_device_placement=False) + config.graph_options.rewrite_options.remapping = RewriterConfig.OFF + config.graph_options.rewrite_options.memory_optimization = RewriterConfig.OFF + custom_op = config.graph_options.rewrite_options.custom_optimizers.add() + custom_op.name = "NpuOptimizer" + custom_op.parameter_map["graph_run_mode"].i = 1 + with tf.Session(config=config) as sess: + hw_saved_model_input_dict = OrderedDict() + for key, value in input_node_dict.items(): + hw_saved_model_input_dict[key] = tf.placeholder(shape=value.shape, + dtype=tf.DType(getattr(types_pb2, value.type)), + name=value.name) + model_data = tf.Variable(tf.io.read_file(om_path), dtype=tf.string) + gen_npu_ops = helper.get_gen_ops() + outputs = gen_npu_ops.load_and_execute_om(list(hw_saved_model_input_dict.values()), + model_data=model_data, + output_dtypes=tuple(tf.DType(getattr(types_pb2, value.type)) + for value in output_node_dict.values())) + + saved_outputs = OrderedDict() + for output_tensor, output_node in zip(outputs, output_node_dict): + saved_outputs[output_node] = tf.saved_model.utils.build_tensor_info(output_tensor) + sess.run(tf.initializers.global_variables()) + labeling_signature = tf.saved_model.signature_def_utils.build_signature_def( + inputs={k: tf.saved_model.utils.build_tensor_info(v) for k, v in hw_saved_model_input_dict.items()}, + outputs=saved_outputs, + method_name=method_name) + saved_path = output_path + f"1" + saved_model_builder = tf.saved_model.builder.SavedModelBuilder(saved_path) + saved_model_builder.add_meta_graph_and_variables( + sess, [tf.saved_model.tag_constants.SERVING], signature_def_map={ + tf.saved_model.signature_constants.DEFAULT_SERVING_SIGNATURE_DEF_KEY: labeling_signature}) + + saved_model_builder.save() + print(f"[INFO]: HW Saved Model is saved to {saved_path}.") + + +def gen_pb_txt(pb_file_path): + func2graph_path_in_home = os.path.join("Ascend", "ascend-toolkit", "latest", "atc", "python", "func2graph", + "func2graph.py") + func2graph_path = os.path.join(os.getenv("HOME"), func2graph_path_in_home) + if not os.path.exists(func2graph_path): + func2graph_path = os.path.join("/usr/local", func2graph_path_in_home) + subprocess.run(("python3", func2graph_path, "-m", pb_file_path), stdout=subprocess.PIPE) + + +def pb_to_om(pb_file_path, output_path, soc_version, input_shape, out_nodes, rest_args): + atc_path_in_home = os.path.join("Ascend", "ascend-toolkit", "latest", "bin", "atc") + atc_path = os.path.join(os.getenv("HOME"), atc_path_in_home) + if not os.path.exists(atc_path): + atc_path = os.path.join("/usr/local", atc_path_in_home) + if input_shape: + return subprocess.run((atc_path, "--framework", "3", "--model", pb_file_path, "--output", output_path, + "--soc_version", soc_version, "--input_shape", input_shape, "--out_nodes", out_nodes, + *rest_args)) + else: + return subprocess.run((atc_path, "--framework", "3", "--model", pb_file_path, "--output", output_path, + "--soc_version", soc_version, "--out_nodes", out_nodes, *rest_args)) + + +def pb_to_om_with_profiling(pb_file_path, output_path, input_shape, out_nodes, profiling, rest_args): + aoe_path_in_home = os.path.join("Ascend", "ascend-toolkit", "latest", "bin", "aoe") + aoe_path = os.path.join(os.getenv("HOME"), aoe_path_in_home) + if not os.path.exists(aoe_path): + aoe_path = os.path.join("/usr/local", aoe_path_in_home) + if input_shape: + return subprocess.run((aoe_path, "--framework", "3", "--model", pb_file_path, "--output", + output_path, "--input_shape", input_shape, "--out_nodes", out_nodes, + "--job_type", profiling, *rest_args)) + else: + return subprocess.run((aoe_path, "--framework", "3", "--model", pb_file_path, "--output", + output_path, "--out_nodes", out_nodes, "--job_type", profiling, *rest_args)) + + +def print_input_shape(input_nodes): + input_shapes = [] + for value in input_nodes.values(): + input_shapes.append(f"{value.name}:{','.join('-1' if shape is None else str(shape) for shape in value.shape)}") + input_shape_param = ";".join(input_shapes) + print(f"The input_shape of model: {input_shape_param}") + + +def get_out_nodes(output_nodes): + return ";".join(output_node.full_name for output_node in output_nodes.values()) + + +def make_temp_path(base_path, dir_name, file_name): + tmp_dir_path = os.path.join(base_path, dir_name) + os.makedirs(tmp_dir_path, exist_ok=True) + tmp_file_path = os.path.join(tmp_dir_path, file_name) + return tmp_dir_path, tmp_file_path + + +def update_output_type(output_type, output_nodes): + if not output_type: + return + if output_type in TYPE_MAP.keys(): + for name in output_nodes.keys(): + node = output_nodes[name] + output_nodes[name] = NodeInfo(node.name, node.shape, TYPE_MAP[output_type], node.full_name) + else: + for i in output_type.split(';'): + index = i.rfind(':') + node_name = i[:index] + node = output_nodes[node_name] + output_nodes[node_name] = NodeInfo(node.name, node.shape, TYPE_MAP[i[index + 1:]], node.full_name) + + +def update_input_type(input_fp16_nodes, input_nodes): + unhandled_nodes = list(input_nodes.keys()) + for name in input_fp16_nodes.split(';'): + for i in unhandled_nodes[:]: + node = input_nodes[i] + if node.name == name: + input_nodes[i] = NodeInfo(node.name, node.shape, TYPE_MAP['FP16'], node.full_name) + unhandled_nodes.remove(i) + + +def update_atc_args(type_args, rest_args): + for k, v in type_args.items(): + if v not in (None, ''): + rest_args.append(f'--{k}={v}') + + +def update_type(input_nodes, output_nodes, output_type, input_fp16_nodes): + if output_type: + update_output_type(output_type, output_nodes) + if input_fp16_nodes: + update_input_type(input_fp16_nodes, input_nodes) + + +def main(input_path, output_path, input_shape, soc_version, profiling, method_name, new_input_nodes, new_output_nodes, + type_args, rest_args): + try: + now_time = int(time.time()) + tmp_pb_path, tmp_pb_file = make_temp_path(TMP_PATH, f"pb_dir_{now_time}", TMP_PB_NAME) + tmp_om_path, tmp_om_file = make_temp_path(TMP_PATH, f"om_dir_{now_time}", TMP_OM_NAME) + if new_input_nodes is not None or new_output_nodes is not None: + input_path = saved_sub_graph_saved_model(new_input_nodes, new_output_nodes, input_path, tmp_pb_path) + input_nodes, output_nodes, saved_model_method_name = saved_model_to_pb(input_path, tmp_pb_path, TMP_PB_NAME) + if not saved_model_method_name and not method_name: + print(f"[ERROR]: The method name cannot be obtained from the input saved model." + f"Please set the parameter --method_name.") + return + if input_shape: + print(f"The input_shape of model: {input_shape}") + else: + print_input_shape(input_nodes) + gen_pb_txt(tmp_pb_file) + out_nodes = get_out_nodes(output_nodes) + update_atc_args(type_args, rest_args) + if profiling is not None: + ret = pb_to_om_with_profiling(tmp_pb_file, tmp_om_file, input_shape, out_nodes, profiling, rest_args) + else: + ret = pb_to_om(tmp_pb_file, tmp_om_file, soc_version, input_shape, out_nodes, rest_args) + if ret.returncode != 0: + return + update_type(input_nodes, output_nodes, **type_args) + om_file_path = os.path.join(tmp_om_path, os.listdir(tmp_om_path)[0]) + print(om_file_path) + print(f"[INFO]: The om model has been converted and the HW Saved Model is ready to be generated.") + method_name = method_name or saved_model_method_name + print(f"[INFO]: Use method name {method_name}.") + save_hw_saved_model(input_nodes, output_nodes, output_path, method_name, om_file_path) + finally: + print("END") + #shutil.rmtree(TMP_PATH) + + +def get_args(): + parser = argparse.ArgumentParser() + parser.add_argument("--input_path", required=True, help='SavedModel path.') + parser.add_argument("--output_path", required=True, + help="Output file path&name(needn't suffix, will add .om automatically).") + parser.add_argument("--input_shape", default='', help='Shape of input data. ' + 'Separate multiple nodes with semicolons (;). ' + 'Use double quotation marks (") to enclose each argument.' + 'E.g.: "input_name1:n1,c1,h1,w1;input_name2:n2,c2,h2,w2"') + parser.add_argument("--method_name", help='Method name for TF-Serving.') + parser.add_argument("--new_input_nodes", + help='Configure this to reselect the input node.' + 'the node format is name:type_pb:node_name' + 'Separate multiple nodes with semicolons (;).' + 'Use double quotation marks (") to enclose each argument.' + 'E.g.: "embedding:DT_FLOAT:bert/embedding/word_embeddings:0;add:DT_INT:bert/embedding/add:0"') + parser.add_argument("--new_output_nodes", + help='Configure this to reselect the output node.' + 'Separate multiple nodes with semicolons (;).' + 'Use double quotation marks (") to enclose each argument.' + 'E.g.: "loss:loss/Softmax:0"') + parser.add_argument("--output_type", default='', help="ATC Parameter: Specifies the network output node type " + "or specifies the output type of a particular output node.") + parser.add_argument("--input_fp16_nodes", default='', + help="ATC Parameter: Specifies the name of the input node whose input data type is float16.") + group = parser.add_mutually_exclusive_group(required=True) + group.add_argument("--soc_version", help="The soc version. " + "This parameter is not required when profiling is set.") + group.add_argument("--profiling", choices=['1', '2'], help="Set this parameter when profiling is required." + "(sgat: 1, opat: 2).") + return parser.parse_known_args() + + +if __name__ == "__main__": + args, unknown_args = get_args() + main(args.input_path, args.output_path, args.input_shape, args.soc_version, args.profiling, args.method_name, + args.new_input_nodes, args.new_output_nodes, + {'output_type': args.output_type, 'input_fp16_nodes': args.input_fp16_nodes}, unknown_args) -- Gitee From eb09a214a79a4a56f80c81312735c2860faefb0e Mon Sep 17 00:00:00 2001 From: hellolee1234 Date: Mon, 4 Sep 2023 19:19:14 +0800 Subject: [PATCH 2/4] rename directory --- {Mt_M2_inference => tf_serving_inference}/M2.cfg | 0 {Mt_M2_inference => tf_serving_inference}/client.py | 0 {Mt_M2_inference => tf_serving_inference}/client.sh | 0 {Mt_M2_inference => tf_serving_inference}/client_om.sh | 0 {Mt_M2_inference => tf_serving_inference}/server.sh | 0 {Mt_M2_inference => tf_serving_inference}/server_om.sh | 0 {Mt_M2_inference => tf_serving_inference}/tools/ckpt_convert.py | 0 {Mt_M2_inference => tf_serving_inference}/tools/om.sh | 0 {Mt_M2_inference => tf_serving_inference}/tools/saved_model2om.py | 0 9 files changed, 0 insertions(+), 0 deletions(-) rename {Mt_M2_inference => tf_serving_inference}/M2.cfg (100%) rename {Mt_M2_inference => tf_serving_inference}/client.py (100%) rename {Mt_M2_inference => tf_serving_inference}/client.sh (100%) rename {Mt_M2_inference => tf_serving_inference}/client_om.sh (100%) rename {Mt_M2_inference => tf_serving_inference}/server.sh (100%) rename {Mt_M2_inference => tf_serving_inference}/server_om.sh (100%) rename {Mt_M2_inference => tf_serving_inference}/tools/ckpt_convert.py (100%) rename {Mt_M2_inference => tf_serving_inference}/tools/om.sh (100%) rename {Mt_M2_inference => tf_serving_inference}/tools/saved_model2om.py (100%) diff --git a/Mt_M2_inference/M2.cfg b/tf_serving_inference/M2.cfg similarity index 100% rename from Mt_M2_inference/M2.cfg rename to tf_serving_inference/M2.cfg diff --git a/Mt_M2_inference/client.py b/tf_serving_inference/client.py similarity index 100% rename from Mt_M2_inference/client.py rename to tf_serving_inference/client.py diff --git a/Mt_M2_inference/client.sh b/tf_serving_inference/client.sh similarity index 100% rename from Mt_M2_inference/client.sh rename to tf_serving_inference/client.sh diff --git a/Mt_M2_inference/client_om.sh b/tf_serving_inference/client_om.sh similarity index 100% rename from Mt_M2_inference/client_om.sh rename to tf_serving_inference/client_om.sh diff --git a/Mt_M2_inference/server.sh b/tf_serving_inference/server.sh similarity index 100% rename from Mt_M2_inference/server.sh rename to tf_serving_inference/server.sh diff --git a/Mt_M2_inference/server_om.sh b/tf_serving_inference/server_om.sh similarity index 100% rename from Mt_M2_inference/server_om.sh rename to tf_serving_inference/server_om.sh diff --git a/Mt_M2_inference/tools/ckpt_convert.py b/tf_serving_inference/tools/ckpt_convert.py similarity index 100% rename from Mt_M2_inference/tools/ckpt_convert.py rename to tf_serving_inference/tools/ckpt_convert.py diff --git a/Mt_M2_inference/tools/om.sh b/tf_serving_inference/tools/om.sh similarity index 100% rename from Mt_M2_inference/tools/om.sh rename to tf_serving_inference/tools/om.sh diff --git a/Mt_M2_inference/tools/saved_model2om.py b/tf_serving_inference/tools/saved_model2om.py similarity index 100% rename from Mt_M2_inference/tools/saved_model2om.py rename to tf_serving_inference/tools/saved_model2om.py -- Gitee From 6e6c6badabb8934f4fe5f9fab847a9b5d399f933 Mon Sep 17 00:00:00 2001 From: hellolee1234 Date: Mon, 4 Sep 2023 19:32:14 +0800 Subject: [PATCH 3/4] fix variable names --- tf_serving_inference/M2.cfg | 51 ------------------------------- tf_serving_inference/server.sh | 4 +-- tf_serving_inference/server_om.sh | 5 ++- tf_serving_inference/tools/om.sh | 2 +- 4 files changed, 5 insertions(+), 57 deletions(-) delete mode 100644 tf_serving_inference/M2.cfg diff --git a/tf_serving_inference/M2.cfg b/tf_serving_inference/M2.cfg deleted file mode 100644 index 81e200e0..00000000 --- a/tf_serving_inference/M2.cfg +++ /dev/null @@ -1,51 +0,0 @@ -platform_configs { -key: "tensorflow" - value { - source_adapter_config { - [type.googleapis.com/tensorflow.serving.SavedModelBundleSourceAdapterConfig] { - legacy_config { - session_config { - graph_options { - rewrite_options { - custom_optimizers { - name: "NpuOptimizer" - parameter_map: { - key: "use_off_line" - value: { - b: true - } - } - parameter_map: { - key: "mix_compile_mode" - value: { - b: true - } - } - parameter_map: { - key: "variable_placement" - value: { - s: "Host" - } - } - parameter_map: { - key: "graph_run_mode" - value: { - i: 0 - } - } - parameter_map: { - key: "precision_mode" - value: { - s: "force_fp16" - } - } - } - remapping: OFF - } - } - } - } - } - } - } -} diff --git a/tf_serving_inference/server.sh b/tf_serving_inference/server.sh index 9dfaa30e..70467d4f 100644 --- a/tf_serving_inference/server.sh +++ b/tf_serving_inference/server.sh @@ -4,8 +4,8 @@ #export PROFILING_OPTIONS='{"output":"/tmp/profiling","task_trace":"on","fp_point":"","aic_metrics":"PipeUtilization"}' taskset -c 0-32 tensorflow_model_server \ --model_name=saved_model \ ---model_base_path=/home/l00832016/M2_infer/mt_inference_model/saved_model/ \ +--model_base_path=/home/l00832016/inference/mt_inference_model/saved_model/ \ --port=9999 \ --enable_model_warmup=true \ ---platform_config_file=M2.cfg \ +--platform_config_file=inference.cfg \ diff --git a/tf_serving_inference/server_om.sh b/tf_serving_inference/server_om.sh index 51afffc4..f96e74db 100644 --- a/tf_serving_inference/server_om.sh +++ b/tf_serving_inference/server_om.sh @@ -2,10 +2,9 @@ export ASCEND_SLOG_PRINT_TO_STDOUT=1 export ASCEND_GLOBAL_LOG_LEVEL=0 #export PROFILING_MODE=true #export PROFILING_OPTIONS='{"output":"/tmp/profiling","task_trace":"on","fp_point":"","aic_metrics":"PipeUtilization"}' -#/home/l00832016/M2_infer/mt_inference_model/saved_model/ \ taskset -c 0-32 tensorflow_model_server \ --model_name=saved_model \ ---model_base_path=/home/l00832016/M2_infer/tools/om_model/ \ +--model_base_path=/home/l00832016/inference/tools/om_model/ \ --port=9999 \ --enable_model_warmup=true \ ---platform_config_file=M2.cfg \ +--platform_config_file=inference.cfg \ diff --git a/tf_serving_inference/tools/om.sh b/tf_serving_inference/tools/om.sh index 3bd0f26f..0b678a01 100644 --- a/tf_serving_inference/tools/om.sh +++ b/tf_serving_inference/tools/om.sh @@ -1,3 +1,3 @@ export LD_LIBRARY_PATH=/usr/local/Ascend/ascend-toolkit/latest/lib64:/usr/local/Ascend/ascend-toolkit/latest/lib64/plugin/opskernel:/usr/local/Ascend/ascend-toolkit/latest/lib64/plugin/nnengine:/usr/local/Ascend/ascend-toolkit/latest/lib64:/usr/local/Ascend/ascend-toolkit/latest/lib64/plugin/opskernel:/usr/local/Ascend/ascend-toolkit/latest/lib64/plugin/nnengine:/usr/local/Ascend/ascend-toolkit/latest/lib64:/usr/local/Ascend/ascend-toolkit/latest/lib64/plugin/opskernel:/usr/local/Ascend/ascend-toolkit/latest/lib64/plugin/nnengine:/usr/local/Ascend/ascend-toolkit/latest/runtime/lib64:/usr/local/Ascend/driver/lib64:/usr/local/Ascend/driver/lib64/common:/usr/local/Ascend/driver/lib64/driver::/usr/local/python3.7.5/lib -python3 saved_model2om.py --input_path=/home/l00832016/M2_infer/mt_inference_model/saved_model/1693482994/ --output_path=./om_model/ --input_shape="feat_0:9600,40;feat_1:9600,40;feat_2:9600,40;feat_3:9600,1;feat_4:9600,8;feat_5:9600,32;feat_6:9600,6;feat_7:9600,40;feat_8:9600,40;feat_9:9600,16;feat_10:9600,40;feat_11:9600,8;feat_12:9600,480;feat_13:9600,8;feat_14:9600,40;feat_15:9600,1;feat_16:9600,8;feat_18:9600,1;feat_19:9600,8;feat_20:9600,40;feat_21:9600,32;feat_22:9600,1;feat_23:9600,40;feat_25:9600,6;feat_26:9600,40;feat_27:9600,8;feat_29:9600,40;feat_30:9600,8;feat_32:9600,1;feat_33:9600,256;feat_34:9600,40;feat_35:9600,40;feat_36:9600,8;feat_37:9600,1;feat_38:9600,1;feat_39:9600,1;feat_41:9600,32;feat_43:9600,40;feat_44:9600,6;feat_46:9600,9;feat_47:9600,8;feat_48:9600,4;feat_49:9600,40;feat_50:9600,40;feat_51:9600,40;feat_52:9600,40;feat_54:9600,100;feat_56:9600,40;feat_57:9600,8;feat_58:9600,40;feat_59:9600,40;feat_60:9600,6;feat_61:9600,8;feat_62:9600,40;feat_63:9600,6;feat_64:9600,1;feat_65:9600,1;feat_66:9600,1;feat_68:9600,6;feat_69:9600,8;feat_71:9600,40;feat_72:9600,40;feat_73:9600,40;feat_74:9600,8;feat_75:9600,10;feat_76:9600,40;feat_77:9600,40;feat_78:9600,40;feat_79:9600,8;feat_80:9600,1;feat_82:9600,6;feat_83:9600,1;feat_84:9600,32;feat_85:9600,40;feat_86:9600,8;feat_87:9600,40;feat_89:9600,6;feat_90:9600,40;feat_91:9600,40;feat_92:9600,1;feat_93:9600,40;feat_95:9600,1;feat_96:9600,8;feat_97:9600,320;feat_98:9600,1;feat_99:9600,1;feat_100:9600,1;feat_102:9600,40;feat_103:9600,1;feat_104:9600,1;feat_105:9600,40;feat_106:9600,8;feat_108:9600,6;feat_109:9600,8;feat_110:9600,8;feat_113:9600,8;feat_114:9600,8;feat_115:9600,60;feat_116:9600,40;feat_117:9600,40;feat_118:9600,6;feat_119:9600,40;feat_120:9600,13;feat_121:9600,3;feat_122:9600,9;feat_123:9600,6;feat_124:9600,40;feat_125:9600,40;feat_126:9600,6;feat_127:9600,8;feat_129:9600,40;feat_130:9600,40;feat_131:9600,8;feat_132:9600,1;feat_133:9600,40;feat_134:9600,10;feat_135:9600,1;feat_136:9600,33;feat_137:9600,40;feat_139:9600,6;feat_140:9600,40;feat_141:9600,8;feat_142:9600,26;feat_143:9600,40;feat_144:9600,1;feat_145:9600,8;feat_146:9600,8;feat_147:9600,40;feat_148:9600,6;feat_149:9600,40;feat_150:9600,8;feat_151:9600,6;feat_152:9600,7;feat_153:9600,8;feat_154:9600,8;feat_155:9600,40;feat_156:9600,8;feat_158:9600,8;feat_159:9600,8;feat_160:9600,40;feat_161:9600,40;feat_163:9600,6;feat_164:9600,40;feat_165:9600,40;feat_166:9600,6;feat_167:9600,40;feat_168:9600,6;feat_169:9600,40;feat_170:9600,40;feat_172:9600,1" --soc_version="Ascend310P3" +python3 saved_model2om.py --input_path=/home/l00832016/inference/mt_inference_model/saved_model/1693482994/ --output_path=./om_model/ --input_shape="feat_0:9600,40;feat_1:9600,40;feat_2:9600,40;feat_3:9600,1;feat_4:9600,8;feat_5:9600,32;feat_6:9600,6;feat_7:9600,40;feat_8:9600,40;feat_9:9600,16;feat_10:9600,40;feat_11:9600,8;feat_12:9600,480;feat_13:9600,8;feat_14:9600,40;feat_15:9600,1;feat_16:9600,8;feat_18:9600,1;feat_19:9600,8;feat_20:9600,40;feat_21:9600,32;feat_22:9600,1;feat_23:9600,40;feat_25:9600,6;feat_26:9600,40;feat_27:9600,8;feat_29:9600,40;feat_30:9600,8;feat_32:9600,1;feat_33:9600,256;feat_34:9600,40;feat_35:9600,40;feat_36:9600,8;feat_37:9600,1;feat_38:9600,1;feat_39:9600,1;feat_41:9600,32;feat_43:9600,40;feat_44:9600,6;feat_46:9600,9;feat_47:9600,8;feat_48:9600,4;feat_49:9600,40;feat_50:9600,40;feat_51:9600,40;feat_52:9600,40;feat_54:9600,100;feat_56:9600,40;feat_57:9600,8;feat_58:9600,40;feat_59:9600,40;feat_60:9600,6;feat_61:9600,8;feat_62:9600,40;feat_63:9600,6;feat_64:9600,1;feat_65:9600,1;feat_66:9600,1;feat_68:9600,6;feat_69:9600,8;feat_71:9600,40;feat_72:9600,40;feat_73:9600,40;feat_74:9600,8;feat_75:9600,10;feat_76:9600,40;feat_77:9600,40;feat_78:9600,40;feat_79:9600,8;feat_80:9600,1;feat_82:9600,6;feat_83:9600,1;feat_84:9600,32;feat_85:9600,40;feat_86:9600,8;feat_87:9600,40;feat_89:9600,6;feat_90:9600,40;feat_91:9600,40;feat_92:9600,1;feat_93:9600,40;feat_95:9600,1;feat_96:9600,8;feat_97:9600,320;feat_98:9600,1;feat_99:9600,1;feat_100:9600,1;feat_102:9600,40;feat_103:9600,1;feat_104:9600,1;feat_105:9600,40;feat_106:9600,8;feat_108:9600,6;feat_109:9600,8;feat_110:9600,8;feat_113:9600,8;feat_114:9600,8;feat_115:9600,60;feat_116:9600,40;feat_117:9600,40;feat_118:9600,6;feat_119:9600,40;feat_120:9600,13;feat_121:9600,3;feat_122:9600,9;feat_123:9600,6;feat_124:9600,40;feat_125:9600,40;feat_126:9600,6;feat_127:9600,8;feat_129:9600,40;feat_130:9600,40;feat_131:9600,8;feat_132:9600,1;feat_133:9600,40;feat_134:9600,10;feat_135:9600,1;feat_136:9600,33;feat_137:9600,40;feat_139:9600,6;feat_140:9600,40;feat_141:9600,8;feat_142:9600,26;feat_143:9600,40;feat_144:9600,1;feat_145:9600,8;feat_146:9600,8;feat_147:9600,40;feat_148:9600,6;feat_149:9600,40;feat_150:9600,8;feat_151:9600,6;feat_152:9600,7;feat_153:9600,8;feat_154:9600,8;feat_155:9600,40;feat_156:9600,8;feat_158:9600,8;feat_159:9600,8;feat_160:9600,40;feat_161:9600,40;feat_163:9600,6;feat_164:9600,40;feat_165:9600,40;feat_166:9600,6;feat_167:9600,40;feat_168:9600,6;feat_169:9600,40;feat_170:9600,40;feat_172:9600,1" --soc_version="Ascend310P3" -- Gitee From 172fd05d693f76c3e74a7bf0e38f03171d261317 Mon Sep 17 00:00:00 2001 From: hellolee1234 Date: Mon, 4 Sep 2023 19:38:33 +0800 Subject: [PATCH 4/4] fix variables names --- tf_serving_inference/inference.cfg | 51 ++++++++++++++++++++ tf_serving_inference/server.sh | 2 +- tf_serving_inference/tools/om.sh | 2 +- tf_serving_inference/tools/saved_model2om.py | 2 +- 4 files changed, 54 insertions(+), 3 deletions(-) create mode 100644 tf_serving_inference/inference.cfg diff --git a/tf_serving_inference/inference.cfg b/tf_serving_inference/inference.cfg new file mode 100644 index 00000000..81e200e0 --- /dev/null +++ b/tf_serving_inference/inference.cfg @@ -0,0 +1,51 @@ +platform_configs { +key: "tensorflow" + value { + source_adapter_config { + [type.googleapis.com/tensorflow.serving.SavedModelBundleSourceAdapterConfig] { + legacy_config { + session_config { + graph_options { + rewrite_options { + custom_optimizers { + name: "NpuOptimizer" + parameter_map: { + key: "use_off_line" + value: { + b: true + } + } + parameter_map: { + key: "mix_compile_mode" + value: { + b: true + } + } + parameter_map: { + key: "variable_placement" + value: { + s: "Host" + } + } + parameter_map: { + key: "graph_run_mode" + value: { + i: 0 + } + } + parameter_map: { + key: "precision_mode" + value: { + s: "force_fp16" + } + } + } + remapping: OFF + } + } + } + } + } + } + } +} diff --git a/tf_serving_inference/server.sh b/tf_serving_inference/server.sh index 70467d4f..50e7dd29 100644 --- a/tf_serving_inference/server.sh +++ b/tf_serving_inference/server.sh @@ -4,7 +4,7 @@ #export PROFILING_OPTIONS='{"output":"/tmp/profiling","task_trace":"on","fp_point":"","aic_metrics":"PipeUtilization"}' taskset -c 0-32 tensorflow_model_server \ --model_name=saved_model \ ---model_base_path=/home/l00832016/inference/mt_inference_model/saved_model/ \ +--model_base_path=/home/l00832016/inference/inference_model/saved_model/ \ --port=9999 \ --enable_model_warmup=true \ --platform_config_file=inference.cfg \ diff --git a/tf_serving_inference/tools/om.sh b/tf_serving_inference/tools/om.sh index 0b678a01..149f649b 100644 --- a/tf_serving_inference/tools/om.sh +++ b/tf_serving_inference/tools/om.sh @@ -1,3 +1,3 @@ export LD_LIBRARY_PATH=/usr/local/Ascend/ascend-toolkit/latest/lib64:/usr/local/Ascend/ascend-toolkit/latest/lib64/plugin/opskernel:/usr/local/Ascend/ascend-toolkit/latest/lib64/plugin/nnengine:/usr/local/Ascend/ascend-toolkit/latest/lib64:/usr/local/Ascend/ascend-toolkit/latest/lib64/plugin/opskernel:/usr/local/Ascend/ascend-toolkit/latest/lib64/plugin/nnengine:/usr/local/Ascend/ascend-toolkit/latest/lib64:/usr/local/Ascend/ascend-toolkit/latest/lib64/plugin/opskernel:/usr/local/Ascend/ascend-toolkit/latest/lib64/plugin/nnengine:/usr/local/Ascend/ascend-toolkit/latest/runtime/lib64:/usr/local/Ascend/driver/lib64:/usr/local/Ascend/driver/lib64/common:/usr/local/Ascend/driver/lib64/driver::/usr/local/python3.7.5/lib -python3 saved_model2om.py --input_path=/home/l00832016/inference/mt_inference_model/saved_model/1693482994/ --output_path=./om_model/ --input_shape="feat_0:9600,40;feat_1:9600,40;feat_2:9600,40;feat_3:9600,1;feat_4:9600,8;feat_5:9600,32;feat_6:9600,6;feat_7:9600,40;feat_8:9600,40;feat_9:9600,16;feat_10:9600,40;feat_11:9600,8;feat_12:9600,480;feat_13:9600,8;feat_14:9600,40;feat_15:9600,1;feat_16:9600,8;feat_18:9600,1;feat_19:9600,8;feat_20:9600,40;feat_21:9600,32;feat_22:9600,1;feat_23:9600,40;feat_25:9600,6;feat_26:9600,40;feat_27:9600,8;feat_29:9600,40;feat_30:9600,8;feat_32:9600,1;feat_33:9600,256;feat_34:9600,40;feat_35:9600,40;feat_36:9600,8;feat_37:9600,1;feat_38:9600,1;feat_39:9600,1;feat_41:9600,32;feat_43:9600,40;feat_44:9600,6;feat_46:9600,9;feat_47:9600,8;feat_48:9600,4;feat_49:9600,40;feat_50:9600,40;feat_51:9600,40;feat_52:9600,40;feat_54:9600,100;feat_56:9600,40;feat_57:9600,8;feat_58:9600,40;feat_59:9600,40;feat_60:9600,6;feat_61:9600,8;feat_62:9600,40;feat_63:9600,6;feat_64:9600,1;feat_65:9600,1;feat_66:9600,1;feat_68:9600,6;feat_69:9600,8;feat_71:9600,40;feat_72:9600,40;feat_73:9600,40;feat_74:9600,8;feat_75:9600,10;feat_76:9600,40;feat_77:9600,40;feat_78:9600,40;feat_79:9600,8;feat_80:9600,1;feat_82:9600,6;feat_83:9600,1;feat_84:9600,32;feat_85:9600,40;feat_86:9600,8;feat_87:9600,40;feat_89:9600,6;feat_90:9600,40;feat_91:9600,40;feat_92:9600,1;feat_93:9600,40;feat_95:9600,1;feat_96:9600,8;feat_97:9600,320;feat_98:9600,1;feat_99:9600,1;feat_100:9600,1;feat_102:9600,40;feat_103:9600,1;feat_104:9600,1;feat_105:9600,40;feat_106:9600,8;feat_108:9600,6;feat_109:9600,8;feat_110:9600,8;feat_113:9600,8;feat_114:9600,8;feat_115:9600,60;feat_116:9600,40;feat_117:9600,40;feat_118:9600,6;feat_119:9600,40;feat_120:9600,13;feat_121:9600,3;feat_122:9600,9;feat_123:9600,6;feat_124:9600,40;feat_125:9600,40;feat_126:9600,6;feat_127:9600,8;feat_129:9600,40;feat_130:9600,40;feat_131:9600,8;feat_132:9600,1;feat_133:9600,40;feat_134:9600,10;feat_135:9600,1;feat_136:9600,33;feat_137:9600,40;feat_139:9600,6;feat_140:9600,40;feat_141:9600,8;feat_142:9600,26;feat_143:9600,40;feat_144:9600,1;feat_145:9600,8;feat_146:9600,8;feat_147:9600,40;feat_148:9600,6;feat_149:9600,40;feat_150:9600,8;feat_151:9600,6;feat_152:9600,7;feat_153:9600,8;feat_154:9600,8;feat_155:9600,40;feat_156:9600,8;feat_158:9600,8;feat_159:9600,8;feat_160:9600,40;feat_161:9600,40;feat_163:9600,6;feat_164:9600,40;feat_165:9600,40;feat_166:9600,6;feat_167:9600,40;feat_168:9600,6;feat_169:9600,40;feat_170:9600,40;feat_172:9600,1" --soc_version="Ascend310P3" +python3 saved_model2om.py --input_path=/home/l00832016/inference/inference_model/saved_model/1693482994/ --output_path=./om_model/ --input_shape="feat_0:9600,40;feat_1:9600,40;feat_2:9600,40;feat_3:9600,1;feat_4:9600,8;feat_5:9600,32;feat_6:9600,6;feat_7:9600,40;feat_8:9600,40;feat_9:9600,16;feat_10:9600,40;feat_11:9600,8;feat_12:9600,480;feat_13:9600,8;feat_14:9600,40;feat_15:9600,1;feat_16:9600,8;feat_18:9600,1;feat_19:9600,8;feat_20:9600,40;feat_21:9600,32;feat_22:9600,1;feat_23:9600,40;feat_25:9600,6;feat_26:9600,40;feat_27:9600,8;feat_29:9600,40;feat_30:9600,8;feat_32:9600,1;feat_33:9600,256;feat_34:9600,40;feat_35:9600,40;feat_36:9600,8;feat_37:9600,1;feat_38:9600,1;feat_39:9600,1;feat_41:9600,32;feat_43:9600,40;feat_44:9600,6;feat_46:9600,9;feat_47:9600,8;feat_48:9600,4;feat_49:9600,40;feat_50:9600,40;feat_51:9600,40;feat_52:9600,40;feat_54:9600,100;feat_56:9600,40;feat_57:9600,8;feat_58:9600,40;feat_59:9600,40;feat_60:9600,6;feat_61:9600,8;feat_62:9600,40;feat_63:9600,6;feat_64:9600,1;feat_65:9600,1;feat_66:9600,1;feat_68:9600,6;feat_69:9600,8;feat_71:9600,40;feat_72:9600,40;feat_73:9600,40;feat_74:9600,8;feat_75:9600,10;feat_76:9600,40;feat_77:9600,40;feat_78:9600,40;feat_79:9600,8;feat_80:9600,1;feat_82:9600,6;feat_83:9600,1;feat_84:9600,32;feat_85:9600,40;feat_86:9600,8;feat_87:9600,40;feat_89:9600,6;feat_90:9600,40;feat_91:9600,40;feat_92:9600,1;feat_93:9600,40;feat_95:9600,1;feat_96:9600,8;feat_97:9600,320;feat_98:9600,1;feat_99:9600,1;feat_100:9600,1;feat_102:9600,40;feat_103:9600,1;feat_104:9600,1;feat_105:9600,40;feat_106:9600,8;feat_108:9600,6;feat_109:9600,8;feat_110:9600,8;feat_113:9600,8;feat_114:9600,8;feat_115:9600,60;feat_116:9600,40;feat_117:9600,40;feat_118:9600,6;feat_119:9600,40;feat_120:9600,13;feat_121:9600,3;feat_122:9600,9;feat_123:9600,6;feat_124:9600,40;feat_125:9600,40;feat_126:9600,6;feat_127:9600,8;feat_129:9600,40;feat_130:9600,40;feat_131:9600,8;feat_132:9600,1;feat_133:9600,40;feat_134:9600,10;feat_135:9600,1;feat_136:9600,33;feat_137:9600,40;feat_139:9600,6;feat_140:9600,40;feat_141:9600,8;feat_142:9600,26;feat_143:9600,40;feat_144:9600,1;feat_145:9600,8;feat_146:9600,8;feat_147:9600,40;feat_148:9600,6;feat_149:9600,40;feat_150:9600,8;feat_151:9600,6;feat_152:9600,7;feat_153:9600,8;feat_154:9600,8;feat_155:9600,40;feat_156:9600,8;feat_158:9600,8;feat_159:9600,8;feat_160:9600,40;feat_161:9600,40;feat_163:9600,6;feat_164:9600,40;feat_165:9600,40;feat_166:9600,6;feat_167:9600,40;feat_168:9600,6;feat_169:9600,40;feat_170:9600,40;feat_172:9600,1" --soc_version="Ascend310P3" diff --git a/tf_serving_inference/tools/saved_model2om.py b/tf_serving_inference/tools/saved_model2om.py index 8970ed82..9626bdef 100644 --- a/tf_serving_inference/tools/saved_model2om.py +++ b/tf_serving_inference/tools/saved_model2om.py @@ -19,7 +19,7 @@ from tensorflow.compat.v1 import graph_util from six import StringIO NodeInfo = namedtuple('NodeInfo', ['name', 'shape', 'type', 'full_name']) -TMP_PATH = '/home/l00832016/M2_infer/tools/om_model/saved_model2om' +TMP_PATH = '/home/l00832016/inference/tools/om_model/saved_model2om' TMP_PB_NAME = 'model.pb' TMP_OM_NAME = 'model' -- Gitee