基本思想:最近手中有个swim transformer模型，想移植手机端进行推理一下，随手记录一下遇到的问题涉及简单的转ncnn tnn mnn的流程性问题
一、首先我fork了大佬的代码https://github.com/sxj731533730/swin_transformer.git，主代码结构是华为分类垃圾的源码,然后作者参考了 swin_transformer 的架构做了修改，修改参考来自Swin-Transformer/swin_transformer.py at 5d2aede42b4b12cb0e7a2448b58820aeda604426 · microsoft/Swin-Transformer · GitHub
本实验的模型
（1）先测试一下模型，看一下分类效果(原图片和测试图片)
（2）首先将model-87.pt模型转成onnx，这一步还是比较容易的
import torch
# import trochvision
import torch.utils.data
import argparse
import onnxruntime
from model import swin_base_patch4_window12_384_in22k as create_model
import os
import cv2
import numpy as np
from torch.autograd import Variable
from onnxruntime.datasets import get_example
from PIL import Image
from torchvision import transforms
import matplotlib.pyplot as plt
import torch
import torchvision.models as models
import warnings
warnings.filterwarnings('ignore')
device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
def torch2libtorch(model_path,lib_path,dummy_input):
model = torch.load(model_path)
model = model.to(device)
model.eval（)
ts = torch.jit.trace(model, dummy_input.to(device))
ts.save(lib_path)
model = torch.load(lib_path)
model = model.to(device)
model.eval（)
pre = model(dummy_input)
print("the jit:{}".format(pre))
print("推理便签是")
print("the label id {}".format(torch.argmax(pre)))
print("jt->>模型转换成功！")
def main(args):
print("opencv 读取方式")
dummy_input = getInputCv(args.img_size) # 获得网络的输入
model = infertorch(dummy_input)
torch2onnx(args, model, dummy_input)
# 遇到一个问题，好像使用上面这个onnx
dummy_input = getInputImage(args.img_size) # 获得网络的输入
model = infertorch(dummy_input)
torch2onnx(args, model, dummy_input)
torch2libtorch(args.model_path, args.jt_model_path, dummy_input)
def getInputCv(img_size):
input = cv2.imread(args.image_path)
resized = cv2.resize(input, (img_size, img_size))
print("resize = ",resized.shape)
input = cv2.cvtColor(resized, cv2.COLOR_BGR2RGB) # hwc rgb
input = np.transpose(input, (2, 0, 1)).astype(np.float32) # chw rgb
input[0, ...] = ((input[0, ...] / 255.0) - 0.485) / 0.229
input[1, ...] = ((input[1, ...] / 255.0) - 0.456) / 0.224
input[2, ...] = ((input[2, ...] / 255.0) - 0.406) / 0.225
print("after input[0,0,0]:{}".format(input[0, 0, 0]))
now_image1 = Variable(torch.from_numpy(input).to(device))
dummy_input = now_image1.unsqueeze(0)
return dummy_input
def getInputImage(img_size):
data_transform = transforms.Compose(
[transforms.Resize(int(img_size * 1.14)),
transforms.CenterCrop(img_size),
transforms.ToTensor(),
transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])])
img_path =args.image_path
assert os.path.exists(img_path), "file: '{}' dose not exist.".format(img_path)
img = Image.open(img_path)
# [N, C, H, W]
img = data_transform(img).to(device)
# expand batch dimension
dummy_input = torch.unsqueeze(img, dim=0)
return dummy_input
def infertorch(dummy_input):
model = torch.load(args.model_path)
model = model.to(device)
model.eval（)
pre = model(dummy_input)
print("the pt:{}".format(pre))
print("推理便签是")
print("the label id {}".format(torch.argmax(pre)))
print("推理成功")
return model
def torch2onnx(args,model,dummy_input):
input_names = ['input']#模型输入的name
output_names = ['output']#模型输出的name
print("====",dummy_input.shape)
torch_out = torch.onnx._export(model, dummy_input, args.onnx_model_path,
verbose=False, input_names=input_names, output_names=output_names,opset_version=11)
# test onnx model
example_model = get_example(args.onnx_model_path)
session = onnxruntime.InferenceSession(example_model)
# get the name of the first input of the model
input_name = session.get_inputs()[0].name
#print('onnx Input Name:', input_name)
result = session.run([], {input_name: dummy_input.data.cpu().numpy()})
print("the result is {}".format(result[0]))
print("推理便签是")
print("the label id {}".format(torch.Tensor(result[0]).argmax(1)))
print("onnx->>模型转换成功！")
if __name__ == "__main__":
parser = argparse.ArgumentParser(
description="PyTorch model to onnx and ncnn")
parser.add_argument('--image_path', type=str, default=r"G:\sxj731533730\swin_transformer\image\5293F.png",
help="For image from one model_file")
parser.add_argument('--model_path', type=str, default=r"G:\sxj731533730\swin_transformer\weights\model-87.pth",
help="For training from one model_file")
parser.add_argument('--save_model_path', type=str, default=r"G:\sxj731533730\swin_transformer\weights",
help="For training from one model_file")
parser.add_argument('--onnx_model_path', type=str, default=r"G:\sxj731533730\swin_transformer\weights\model-87_384.onnx",
help="For training from one model_file")
parser.add_argument('--jt_model_path', type=str, default=r"G:\sxj731533730\swin_transformer\weights\model-87_384_jc.pt",
help="For training from one model_file")
parser.add_argument('--img_size', type=int, default=384,
help="the image size of model input")
args = parser.parse_args()
main(args)
使用cv2的读取方式，主要是为了和c++代码的结果进行输出比对，因为c++也是用opencv进行处理的；输出结果
"C:\Program Files\Python36\python.exe" G:/sxj731533730/swin_transformer/convert.py
the pt:tensor([[-0.4017, 1.4294, -1.9261, -2.4726, -1.8362, -2.6418, -7.4490, -1.5320,
-1.1203, -3.7695, -1.1836, -4.0829, -2.3729, -1.4613, -1.8079, -2.4117,
-4.8565, -4.7523, 3.5879, -3.8505, 8.5447, -1.7708, 0.2446, -0.3990,
-0.1918, -4.2638, -2.0217, -1.1995, -0.2322, 3.5632, 7.5106, 7.2231,
0.3670, 2.4798, -5.6226, -2.6248, 12.9518, -1.3014, -5.5910, -3.8710]],
device='cuda:0', grad_fn=<AddmmBackward0>)
推理便签是
the label id 36
推理成功
the result is [[-0.40169567 1.4293747 -1.9261262 -2.4725769 -1.8361415 -2.6418307
-7.449049 -1.5320071 -1.1202894 -3.7694635 -1.183548 -4.0828695
-2.3728554 -1.4613135 -1.8078789 -2.4116964 -4.856498 -4.752249
3.5878901 -3.8504772 8.544659 -1.7707771 0.24464309 -0.3990332
-0.19180095 -4.2638063 -2.0216627 -1.1995448 -0.23222005 3.5632122
7.510594 7.223117 0.36697844 2.4798257 -5.6225514 -2.6248353
12.951807 -1.301393 -5.5910325 -3.871038 ]]
推理便签是
the label id tensor([36])
onnx->>模型转换成功！
the jit:tensor([[-0.4017, 1.4294, -1.9261, -2.4726, -1.8362, -2.6418, -7.4490, -1.5320,
-1.1203, -3.7695, -1.1836, -4.0829, -2.3729, -1.4613, -1.8079, -2.4117,
-4.8565, -4.7523, 3.5879, -3.8505, 8.5447, -1.7708, 0.2446, -0.3990,
-0.1918, -4.2638, -2.0217, -1.1995, -0.2322, 3.5632, 7.5106, 7.2231,
0.3670, 2.4798, -5.6226, -2.6248, 12.9518, -1.3014, -5.5910, -3.8710]],
device='cuda:0', grad_fn=<AddmmBackward0>)
推理便签是
the label id 36
jt->>模型转换成功！
opencv 读取方式
after input[0,0,0]:-1.7754088640213013
the pt:tensor([[-0.2550, 1.3657, -1.2325, -2.7399, -2.4659, -4.2618, -6.2847, -1.8048,
-1.3820, -3.4977, -1.7706, -3.9778, -2.3289, -1.7410, -0.9606, -1.9686,
-3.7105, -4.8707, 4.5964, -4.4703, 8.4576, -2.1032, -0.1619, -0.8107,
-1.4237, -3.6186, -0.9506, -0.5722, 0.3392, 3.8951, 7.5969, 5.9791,
1.7897, 0.8427, -6.1975, -2.6404, 11.8910, -1.3842, -5.8153, -4.6445]],
device='cuda:0', grad_fn=<AddmmBackward0>)
推理便签是
the label id 36
推理成功
Warning: Constant folding - Only steps=1 can be constant folded for opset >= 10 onnx::Slice op. Constant folding not applied.
Warning: Constant folding - Only steps=1 can be constant folded for opset >= 10 onnx::Slice op. Constant folding not applied.
the result is [[-0.25496757 1.3656514 -1.232497 -2.7398527 -2.4658496 -4.26177
-6.2846794 -1.8048453 -1.3820395 -3.4977243 -1.770626 -3.9778478
-2.32888 -1.7410436 -0.96058995 -1.9685831 -3.710508 -4.8706474
4.596388 -4.4702587 8.457638 -2.1031926 -0.1618805 -0.8106793
-1.4237037 -3.6185675 -0.95063126 -0.57217664 0.33925724 3.8951244
7.5969334 5.979125 1.7897222 0.84266424 -6.197471 -2.6404266
11.891024 -1.3842105 -5.815264 -4.644536 ]]
推理便签是
the label id tensor([36])
onnx->>模型转换成功！
Process finished with exit code 0
模型结构
二 、转换模型
（1）先使用ncnn的onnx2ncnn进行模型转化~
首先先简化一下模型
G:\sxj731533730\swin_transformer\weights>python -m onnxsim model-87_384.onnx model-87_384_sim.onnx
Simplifying...
Checking 0/3...
Checking 1/3...
Checking 2/3...
Ok!
转模型出问题了
D:\ncnn\buildMinGW\install\bin>onnx2ncnn.exe G:\sxj731533730\swin_transformer\weights\model-87_384_sim.onnx G:\sxj731533730\swin_transformer\weights\model-87_384_sim.param G:\sxj731533730\swin_transformer\weights\model-87_384_sim.bin
转化模型错误，好像不支持transpose等，up已经开始维护pnnx了，可能onnx2ncnn~~
Erf not supported yet!
Unsupported transpose type !
Unsupported split axis !
Unsupported squeeze axes !
Unsupported squeeze axes !
Unsupported squeeze axes !
Erf not supported yet!
Unsupported slice step !
Unsupported slice step !
Unsupported slice step !
Unsupported slice step !
Unsupported slice step !
Unsupported slice step !
Unsupported slice step !
Unsupported slice step !
Unsupported transpose type !
Unsupported split axis !
Unsupported squeeze axes !
Unsupported squeeze axes !
Unsupported squeeze axes !
Erf not supported yet!
Unsupported transpose type !
Unsupported split axis !
Unsupported squeeze axes !
Unsupported squeeze axes !
Unsupported squeeze axes !
Erf not supported yet!
（2）那试试pnnx吧，编译在linux系统上，工具在ncnn/tools/pnnx,安装步骤参考 Windows/Linux/MacOS 编译 PNNX 步骤 - 知乎 up主~
buntu@ubuntu:~/ncnn/tools/pnnx/build/install/bin$ ./pnnx model-87_384_jc.pt inputshape=[1,3,384,384]
pnnxparam = model-87_384_jc.pnnx.param
pnnxbin =model-87_384_jc.pnnx.bin
pnnxpy = model-87_384_jc_pnnx.py
ncnnparam = model-87_384_jc.ncnn.param
ncnnbin = model-87_384_jc.ncnn.bin
ncnnpy = model-87_384_jc_ncnn.py
optlevel = 2
device = cpu
inputshape = [1,3,384,384]
inputshape2 =
customop =
moduleop =
############# pass_level0
inline module = model.BasicLayer
inline module = model.DropPath
inline module = model.Mlp
inline module = model.PatchEmbed
inline module = model.PatchMerging
inline module = model.SwinTransformerBlock
inline module = model.WindowAttention
inline module = torch.nn.modules.linear.Identity
Killed
提了issue~ up正在修复， ~ o(^▽^)o 这服务 好评五颗星~~~~ up说修复一版，但是维度我的自己改 待测试 ncnn 版本pr3521
（3）试试tnn吧，因为卷卷群有人说使用tnn转换成功了，拉取docker和映射目录 /home/ubuntu/TNN/doc/cn/user/convert.md，然后也去试试吧
ubuntu@ubuntu:~/TNN$ sudo docker run -it -v /home/ubuntu/TNN/sxj731533730:/mnt tnn-convert:latest /bin/bash
root@3140d20c74f2:/opt/TNN/tools/convert2tnn# python3 ./converter.py onnx2tnn /mnt/model-87_sim.onnx -in input_info:1,3,384,384 -optimize -debug
model-87_384_jc.pt model-87_384_sim.bin model-87_384_sim.onnx model-87_384_sim.param
root@3140d20c74f2:/opt/TNN/tools/convert2tnn# python3 ./converter.py onnx2tnn /mnt/model-87_384_sim.onnx -in input:1,3,384,384
---------- convert model, please wait a moment ----------
Converter ONNX to TNN Model...
Converter ONNX to TNN check_onnx_dim...
Converter ONNX to TNN check_onnx_dim...
Converter ONNX to TNN model succeed!
的确成功了，写个代码调用一
TNN我是用MinGW32进行编译使用 probuff编译参考 27、Clion+MinGW32编译NCNN使用_sxj731533730-CSDN博客
D:\>git clone https://github.com/Tencent/TNN.git
Cloning into 'TNN'...
remote: Enumerating objects: 46716, done.
remote: Counting objects: 100% (2132/2132), done.
remote: Compressing objects: 100% (964/964), done.
remote: Total 46716 (delta 1267), reused 1765 (delta 1022), pack-reused 44584R
Receiving objects: 100% (46716/46716), 41.76 MiB | 377.00 KiB/s, done.
Resolving deltas: 100% (33426/33426), done.
Updating files: 100% (4871/4871), done
D:\>cd TNN
D:\TNN>mkdir builMinG
F:\TNN\buildMinGW>cmake -G"MinGW Makefiles" -DOpenCV_DIR="D:\Opencv440\buildMinGW\install" -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=%cd%/install -DProtobuf_INCLUDE_DIR=D:/protobuf-3.4.0/buildMinGW/install/include -DProtobuf_LIBRARIES=D:/protobuf-3.4.0/buildMinGW/install/lib/libprotobuf.a -D Protobuf_PROTOC_EXECUTABLE=D:/protobuf-3.4.0/buildMinGW/install/bin/protoc.exe -DTNN_BUILD_SHARED=OFF -DTNN_X86_ENABLE=ON ..
D:\TNN\builMinGW>mingw32-make -j8
遇到一个问题，已经提issue 合并了
[ 99%] Building CXX object CMakeFiles/TNN.dir/source/tnn/utils/split_utils.cc.obj
D:\TNN\source\tnn\utils\split_utils.cc: In static member function 'static tnn::Status tnn::SplitUtils::SplitStr(const char*, tnn::str_arr&, const char*, bool, bool, bool, bool, bool)':
D:\TNN\source\tnn\utils\split_utils.cc:163:23: error: 'min' was not declared in this scope
int len = min((i - cursor), subs_length - 1);
修改
#if defined(WIN32) && _MSC_VER < 1300 // VC++ 6.0
//int len = min((i - cursor), subs_length - 1);
int len = (i - cursor)>=(subs_length - 1)?(subs_length - 1):(i - cursor);
#else
int len = std::min<int>(i - cursor, subs_length - 1);
#endif
D:\TNN\builMinGW>mingw32-make -j8
Scanning dependencies of target TNNCpu
[ 0%] Building CXX object source/tnn/device/cpu/CMakeFiles/TNNCpu.dir/acc/compute/compute_elewise.cc.obj[ 0%]
Building CXX object source/tnn/device/cpu/CMakeFiles/TNNCpu.dir/acc/compute/compute_int8.cc.obj
[ 0%] Building CXX object source/tnn/device/cpu/CMakeFiles/TNNCpu.dir/acc/cpu__histogram_layer_acc.cc.obj
[ 0%] Building CXX object source/tnn/device/cpu/CMakeFiles/TNNCpu.dir/acc/cpu_acos_layer_acc.cc.obj
[ 1%] Building CXX object source/tnn/device/cpu/CMakeFiles/TNNCpu.dir/acc/cpu_abs_layer_acc.cc.obj
[ 1%] [ 1%] Building CXX object source/tnn/device/cpu/CMakeFiles/TNNCpu.dir/acc/cpu_add_layer_acc.cc.objBuilding CXX object source/tnn/device/cpu/CMakeFiles/TNNCpu.dir/acc/cpu_arg_max_or_min_layer_acc.cc.obj
D:/TNN/include/tnn/core/mat.h:51:3: note: in expansion of macro 'PUBLIC'
} PUBLIC MatType;
^~~~~~
[100%] Linking CXX shared library libTNN.a
[100%] Built target TNN
cmakelist.txt（因为opencv已经安装到window10电脑上，好像直接引用方式写成和linux一样就行，如果不行就引入库使用吧） 18、window10配置minGW+opencv4.4.0+Clion2020_sxj731533730-CSDN博客
cmake_minimum_required(VERSION 3.16)
project(untitled4)
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -g -fopenmp -w ")
set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -g -fopenmp -w ")
set(CMAKE_CXX_STANDARD 11)
include_directories(${CMAKE_SOURCE_DIR}/include)
include_directories(${CMAKE_SOURCE_DIR}/include/tnn)
find_package(OpenCV REQUIRED)
add_library(libTNN SHARED IMPORTED)
set_target_properties(libTNN PROPERTIES IMPORTED_LOCATION ${CMAKE_SOURCE_DIR}/lib/libTNN.so)
add_executable(untitled4 main.cpp )
target_link_libraries(untitled4 ${OpenCV_LIBS} libTNN)
测试代码mian.cpp
#include <iostream>
#include <fstream>
#include <opencv2/imgproc.hpp>
#include <opencv2/imgcodecs.hpp>
#include <opencv2/core.hpp>
#include <opencv2/highgui.hpp>
#include <opencv2/dnn.hpp>
#include "tnn/core/macro.h"
#include "tnn/core/tnn.h"
#include "tnn/utils/blob_converter.h"
#include "tnn/utils/mat_utils.h"
#include "tnn/utils/dims_vector_utils.h"
using namespace TNN_NS;
using namespace std;
template<typename T>
std::vector<unsigned int> argsort(const std::vector<T> &arr)
{
if (arr.empty()) return {};
const unsigned int _size = arr.size();
std::vector<unsigned int> indices;
for (unsigned int i = 0; i < _size; ++i) indices.push_back(i);
std::sort(indices.begin(), indices.end(),
[&arr](const unsigned int a, const unsigned int b)
{ return arr[a] > arr[b]; });
return indices;
}
template<typename T>
std::vector<float> softmax(const T *logits, unsigned int _size, unsigned int &max_id)
{
if (_size == 0 || logits == nullptr) return {};
float max_prob = 0.f, total_exp = 0.f;
std::vector<float> softmax_probs(_size);
for (unsigned int i = 0; i < _size; ++i)
{
softmax_probs[i] = std::exp((float) logits[i]);
total_exp += softmax_probs[i];
}
for (unsigned int i = 0; i < _size; ++i)
{
softmax_probs[i] = softmax_probs[i] / total_exp;
if (softmax_probs[i] > max_prob)
{
max_id = i;
max_prob = softmax_probs[i];
}
}
return softmax_probs;
}
std::string content_buffer_from(const char *proto_or_model_path)
{
std::ifstream file(proto_or_model_path, std::ios::binary);
if (file.is_open())
{
file.seekg(0, file.end);
int size = file.tellg();
char *content = new char[size];
file.seekg(0, file.beg);
file.read(content, size);
std::string file_content;
file_content.assign(content, size);
delete[] content;
file.close();
return file_content;
} // empty buffer
else
{
#ifdef LITETNN_DEBUG
std::cout << "Can not open " << proto_or_model_path << "\n";
#endif
return "";
}
}
int main() {
int mH = 384, mW = 384;
string data, buf;
cv::Mat pic = cv::imread("/home/ubuntu/TNN/sxj731533730/image/5293F.jpg");
cv::Mat mat_pic;
cv::resize(pic, mat_pic, cv::Size(mH, mW));
cv::imwrite("B.jpg",mat_pic);
cv::cvtColor(mat_pic,mat_pic,cv::COLOR_BGR2RGB);
TNN *mNet = new TNN();
ModelConfig model_config;
std::string proto_content_buffer, model_content_buffer;
proto_content_buffer = content_buffer_from("/home/ubuntu/TNN/sxj731533730/model-87_384_sim.tnnproto");
model_content_buffer = content_buffer_from("/home/ubuntu/TNN/sxj731533730/model-87_384_sim.tnnmodel");
model_config.model_type = tnn::MODEL_TYPE_TNN;
model_config.params = {proto_content_buffer, model_content_buffer};
Status ret = mNet->Init(model_config);
TNN_NS::NetworkConfig config;
config.device_type = TNN_NS::DEVICE_X86;
config.device_id = 0;
TNN_NS::Status error;
auto net_instance = mNet->CreateInst(config, error);
vector<int> dims = {1, 3, mH, mW};
std::shared_ptr<tnn::Mat> inputM = make_shared<tnn::Mat>(TNN_NS::DEVICE_X86, tnn::N8UC3, dims, (void *)mat_pic.data);
if(!inputM->GetData())
{
std::cout<<"input mat ==nullptr"<<std::endl;
return -1;
}
MatConvertParam inputCon;
inputCon.scale= {(1.0f / 0.229f) * (1.0 / 255.f),
(1.0f / 0.224f) * (1.0 / 255.f),
(1.0f / 0.225f) * (1.0 / 255.f)};
inputCon.bias = {-0.485f * 255.f * (1.0f / 0.229f) * (1.0 / 255.f),
-0.456f * 255.f * (1.0f / 0.224f) * (1.0 / 255.f),
-0.406f * 255.f * (1.0f / 0.225f) * (1.0 / 255.f)};
auto status = net_instance->SetInputMat(inputM, inputCon, "input");
if (status != TNN_NS::TNN_OK) {
std::cout << status.description() << std::endl;
cout << "error!" << endl;
return -1;
}
status = net_instance->Forward();
if (status != TNN_NS::TNN_OK) {
std::cout << status.description() << std::endl;
return -1;
}
std:;
shared_ptr<TNN_NS::Mat> output_scores ;
TNN_NS::MatConvertParam param;
status = net_instance->GetOutputMat(output_scores, param, "output", TNN_NS::DEVICE_X86);
if (status != TNN_NS::TNN_OK) {
std::cout << status.description() << std::endl;
return -1;
}
TNN_NS::MatType matType=output_scores->GetMatType();
std::vector<int> output_dims=output_scores->GetDims();
auto logits_dims = output_scores->GetDims();
const unsigned int num_classes = logits_dims.at(1); // 40
const float *logits = (float *) output_scores->GetData();
unsigned int max_id;
std::vector<float> scores = softmax<float>(logits, num_classes, max_id);
std::vector<unsigned int> sorted_indices = argsort<float>(scores);
int top_k=5;
if (top_k > num_classes) top_k = num_classes;
const char *class_names[40]={"H_Dry cell", "H_Expired drugs","H_ointment", "K_Big bone","K_Fishbone",
"K_Fruit peel","K_Fruit pulp","K_Leaves edible orange roots","K_Tea leaves", "K_The shell", "K_leftovers", "O_Broken flowerpots and dishes",
"O_Cigarette butts", "O_Disposable snack box","O_Stained plastic", "O_chopstick","O_toothpick", "R_Charging treasure",
"R_Cosmetics bottles", "R_Courier bags","R_Edible oil drum","R_Leather shoes", "R_Metal food can",
"R_Old clothes", "R_Plastic bowl tub","R_Plastic hangers","R_Plastic toys","R_Plug wire","R_Plush toys",
"R_Shampoo bottle","R_Spice bottles","R_The bottle","R_The glass","R_cans",
"R_carton", "R_cutting board","R_drink bottle", "R_package", "R_pan","R_pillow"};
for (unsigned int i = 0; i < top_k; ++i)
{
std::cout<<"sorted="<<sorted_indices[i]<<" score= "<<scores[sorted_indices[i]]<<" name= "<<class_names[sorted_indices[i]]<<std::endl;
}
std::cout<<"finish"<<std::endl;
return 0;
}
测试结果有问题,没有置信度。。
/home/ubuntu/CLionProjects/untitled4/cmake-build-debug/untitled4
sorted=30 score= -nan name= R_Spice bottles
sorted=21 score= -nan name= R_Leather shoes
sorted=22 score= -nan name= R_Metal food can
sorted=23 score= -nan name= R_Old clothes
sorted=24 score= -nan name= R_Plastic bowl tub
finish
我试了试官方的模型，就没有问题,其中置信度就存在~，还是模型转化有问题，个人感觉
(4)、查了查好像企鹅的竞争对手mnn支持swin_transform，先完成任务， 去试试
ubuntu@ubuntu:~/MNN/build$ ./MNNConvert -f ONNX --modelFile /home/ubuntu/TNN/sxj731533730/model-87_384_sim.onnx --MNNModel /home/ubuntu/TNN/sxj731533730/model-87_384_sim.mnn --bizCode MNN
Start to Convert Other Model Format To MNN Model...
[13:18:21] /home/ubuntu/MNN/tools/converter/source/onnx/onnxConverter.cpp:30: ONNX Model ir version: 7
Start to Optimize the MNN Net...
115 op name is empty or dup, set to Const115
226 op name is empty or dup, set to Const226
247 op name is empty or dup, set to BinaryOp247
543 op name is empty or dup, set to Unsqueeze543
.....
6356 op name is empty or dup, set to Unsqueeze6356
inputTensors : [ input, ]
outputTensors: [ output, ]
Converted Success!
ubuntu@ubuntu:~/MNN/build$
先使用python和c++测试一下转换的模型是否正确
import onnxruntime
import MNN
import numpy as np
if __name__ == '__main__':
x = np.ones([1, 3, 384, 384]).astype(np.float32)
# onnx
session = onnxruntime.InferenceSession("/home/ubuntu/TNN/sxj731533730/model-87_384_sim.onnx")
inputs = {session.get_inputs()[0].name: x}
outs = session.run(None, inputs)
print('onnx result is:', outs)
# mnn
interpreter = MNN.Interpreter("/home/ubuntu/TNN/sxj731533730/model-87_384_sim.mnn")
print("mnn load")
mnn_session = interpreter.createSession()
input_tensor = interpreter.getSessionInput(mnn_session)
print(input_tensor.getShape())
tmp_input = MNN.Tensor((1, 3, 384, 384), MNN.Halide_Type_Float, x[0], MNN.Tensor_DimensionType_Caffe)
print(tmp_input.getShape())
# print(tmp_input.getData())
print(input_tensor.copyFrom(tmp_input))
input_tensor.printTensorData()
interpreter.runSession(mnn_session)
output_tensor = interpreter.getSessionOutput(mnn_session, 'output')
output_tensor.printTensorData()
output_data = np.array(output_tensor.getData())
print('mnn result is:', output_data)
测试数据
/usr/bin/python3.8 /home/ubuntu/PycharmProjects/pythonProject3/main.py
onnx result is: [array([[ 1.0238203 , 0.03342247, 1.2151959 , 0.4626484 , 0.08746481,
0.32703203, 0.25524694, 0.635553 , -1.9997537 , 1.522738 ,
-1.5540018 , -1.0275203 , 1.8694828 , -0.7319019 , -0.8286574 ,
2.8318052 , 2.0492811 , -1.6291883 , -0.8406251 , 1.1167368 ,
0.415858 , 0.10364884, 0.5550288 , -2.3504062 , 0.33003935,
1.9019886 , 0.06541784, -0.61422265, -0.9162949 , 0.46095616,
-0.45301116, -0.622448 , 0.09004217, -1.1032819 , -0.086169 ,
-1.2351818 , 0.21992171, -1.5404791 , -2.2559078 , -0.10644051]],
dtype=float32)]
mnn load
(1, 3, 384, 384)
(1, 3, 384, 384)
True
mnn result is: [ 1.02819228 0.03319301 1.20141792 0.46515477 0.07841273 0.32538143
0.26669541 0.62628251 -1.99902368 1.51477814 -1.55092072 -1.02161133
1.86537707 -0.72865325 -0.82746458 2.82264352 2.03898835 -1.62318373
-0.83734703 1.10835481 0.4171572 0.09566039 0.55101848 -2.34471846
0.32499924 1.90036523 0.07312844 -0.6077975 -0.91555941 0.45853668
-0.44608408 -0.61788797 0.08805227 -1.10021794 -0.08176206 -1.22944641
0.22302166 -1.53993964 -2.24256468 -0.09817046]
Process finished with exit code 0
c++ 我在window10上使用MinGW32进行了编译，预先需要使用MinGW32编译opencv和probuff，这些编译参考我之前的编译 27、Clion+MinGW32编译NCNN使用_sxj731533730-CSDN博客
G:\>git clone https://github.com/alibaba/MNN.git
Cloning into 'MNN'...
remote: Enumerating objects: 21332, done.
remote: Counting objects: 100% (3147/3147), done.
remote: Compressing objects: 100% (1443/1443), done.
remote: Total 21332 (delta 1709), reused 3020 (delta 1672), pack-reused 18185
Receiving objects: 100% (21332/21332), 164.05 MiB | 586.00 KiB/s, done.
Resolving deltas: 100% (14508/14508), done.
Updating files: 100% (3709/3709), done.
G:\>cd MNN
G:\MNN>mkdir buildMinGW
G:\MNN>cd buildMinGW
G:\MNN\buildMinGW>cmake -G"MinGW Makefiles" -DOpenCV_DIR="D:\Opencv440\buildMinGW\install" -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=%cd%/install -DProtobuf_INCLUDE_DIR=D:/protobuf-3.4.0/buildMinGW/install/include -DProtobuf_LIBRARIES=D:/protobuf-3.4.0/buildMinGW/install/lib/libprotobuf.a -D Protobuf_PROTOC_EXECUTABLE=D:/protobuf-3.4.0/buildMinGW/install/bin/protoc.exe -DMNN_BUILD_SHARED_LIBS=OFF -DCMAKE_BUILD_TYPE=Release ..
...
[ 98%] Linking CXX executable aoa_nlu_decoder1.out.exe
[ 98%] Built target aoa_nlu_decoder1.out
[ 99%] Linking CXX executable MNNV2Basic.out.exe
[ 99%] Built target MNNV2Basic.out
[ 99%] Linking CXX executable aoa_nlu_encoder.out.exe
[ 99%] Built target aoa_nlu_encoder.out
[100%] Linking CXX executable timeProfile.out.exe
[100%] Built target timeProfile.out
[100%] Linking CXX executable aoa_nlu_decoder2.out.exe
[100%] Built target aoa_nlu_decoder2.out
G:\MNN\buildMinGW>mingw32-make install
[ 5%] Built target MNNCore
[ 6%] Built target MNNCV
[ 10%] Built target MNNAVX
[ 14%] Built target MNN_Express
[ 15%] Built target MNNUtils
[ 16%] Built target MNNMath
mnn的cmakelist.txt
cmake_minimum_required(VERSION 3.16)
project(untitled8)
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -g -fopenmp -w ")
set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -g -fopenmp -w ")
set(CMAKE_CXX_STANDARD 11)
include_directories(${CMAKE_SOURCE_DIR}/include)
include_directories(${CMAKE_SOURCE_DIR}/include/MNN)
find_package(OpenCV REQUIRED)
add_library(libMNN SHARED IMPORTED)
set_target_properties(libMNN PROPERTIES IMPORTED_LOCATION ${CMAKE_SOURCE_DIR}/lib/libMNN.so)
add_executable(untitled8 main.cpp )
target_link_libraries(untitled8 ${OpenCV_LIBS} libMNN)
mnn的源代码
#include <iostream>
#include<opencv2/core.hpp>
#include<opencv2/imgproc.hpp>
#include<opencv2/highgui.hpp>
#include<MNN/Interpreter.hpp>
#include<MNN/ImageProcess.hpp>
using namespace std;
using namespace cv;
using namespace MNN;
int main()
{
auto net = std::shared_ptr<MNN::Interpreter>(MNN::Interpreter::createFromFile("/home/ubuntu/TNN/sxj731533730/model-87_384_sim.mnn"));//创建解释器
cout << "Interpreter created" << endl;
ScheduleConfig config;
config.numThread = 8;
config.type = MNN_FORWARD_CPU;
auto session = net->createSession(config);//创建session
cout << "session created" << endl;
auto inTensor = net->getSessionInput(session, NULL);
auto outTensor = net->getSessionInput(session, NULL);
auto _Tensor = MNN::Tensor::create<float>(inTensor->shape(), NULL, MNN::Tensor::CAFFE);
for (int i = 0; i < _Tensor->elementSize(); i++) {
_Tensor->host<float>()[i] = 1.f;
}
inTensor->copyFromHostTensor(_Tensor);
//推理
net->runSession(session);
auto output= net->getSessionOutput(session, NULL);
// MNN::Tensor feat_tensor(output, output->getDimensionType());
// output->copyToHostTensor(&feat_tensor);
// feat_tensor.print();
MNN::Tensor score_host(output, output->getDimensionType());
output->copyToHostTensor(&score_host);
auto score_ptr = score_host.host<float>();
std::vector<std::pair<float, int>> scores;
for (int i = 0; i < score_host.elementSize(); ++i) {
float score = score_ptr[i];
if(i%5!=0||i==0){
std::cout<<score<<" ,";
}else {
std::cout<<score<<std::endl;
}
}
return 0;
}
测试结果就正确了
/home/ubuntu/CLionProjects/untitled8/cmake-build-debug/untitled8
Interpreter created
session created
1.02862 ,0.0331715 ,1.20148 ,0.465368 ,0.0783655 ,0.325653
0.266529 ,0.626484 ,-1.99896 ,1.51479 ,-1.55074
-1.02167 ,1.86536 ,-0.728646 ,-0.827346 ,2.82286
2.03924 ,-1.62345 ,-0.837316 ,1.10834 ,0.417171
0.095808 ,0.550832 ,-2.34458 ,0.325013 ,1.9003
0.0731261 ,-0.608058 ,-0.915578 ,0.458301 ,-0.446057
-0.618133 ,0.0880814 ,-1.09998 ,-0.081489 ,-1.22944
0.223052 ,-1.53983 ,-2.24254 ,-0.0980031 ,
Process finished with exit code 0
测试图片的脚本
import onnxruntime
import MNN
import numpy as np
import cv2
from torch.autograd import Variable
import torch
if __name__ == '__main__':
device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
img_size=384
input=cv2.imread("/home/ubuntu/TNN/sxj731533730/image/5293F.png")
resized = cv2.resize(input, (img_size, img_size))
input = cv2.cvtColor(resized, cv2.COLOR_BGR2RGB) # hwc rgb
input = np.transpose(input, (2, 0, 1)).astype(np.float32) # chw rgb
input[0, ...] = ((input[0, ...] / 255.0) - 0.485) / 0.229
input[1, ...] = ((input[1, ...] / 255.0) - 0.456) / 0.224
input[2, ...] = ((input[2, ...] / 255.0) - 0.406) / 0.225
print("after input[0,0,0]:{}".format(input[0, 0, 0]))
now_image1 = Variable(torch.from_numpy(input).to(device))
dummy_input = now_image1.unsqueeze(0)
dummy_input=dummy_input.data.cpu().numpy()
# onnx
session = onnxruntime.InferenceSession("/home/ubuntu/TNN/sxj731533730/model-87_384_sim.onnx")
inputs = {session.get_inputs()[0].name: dummy_input}
outs = session.run(None, inputs)
print('onnx result is:', outs)
print('label id result is:', torch.Tensor(outs[0].argmax(1)))
# mnn
interpreter = MNN.Interpreter("/home/ubuntu/TNN/sxj731533730/model-87_384_sim.mnn")
print("mnn load")
mnn_session = interpreter.createSession()
input_tensor = interpreter.getSessionInput(mnn_session)
print(input_tensor.getShape())
tmp_input = MNN.Tensor((1, 3, img_size, img_size), MNN.Halide_Type_Float, dummy_input[0], MNN.Tensor_DimensionType_Caffe)
print(tmp_input.getShape())
# print(tmp_input.getData())
print(input_tensor.copyFrom(tmp_input))
input_tensor.printTensorData()
interpreter.runSession(mnn_session)
output_tensor = interpreter.getSessionOutput(mnn_session, 'output')
output_tensor.printTensorData()
output_data = np.array(output_tensor.getData())
print('mnn result is:', output_data)
print('label id result is:', output_data.argmax(0))
测试结果
/usr/bin/python3.8 /home/ubuntu/PycharmProjects/pythonProject3/main.py
after input[0,0,0]:-1.7754088640213013
onnx result is: [array([[-1.1718541 , 2.5282578 , -1.1380243 , -2.7471526 , -2.2859905 ,
-2.3880184 , -7.8221016 , -1.419074 , -0.53306574, -4.0122375 ,
-1.3708531 , -3.646379 , -3.0561714 , -2.5111208 , -2.0367544 ,
-3.4276567 , -5.1356506 , -5.571644 , 4.7279434 , -3.821684 ,
8.26333 , -1.6675351 , -0.32852614, -0.43677914, 0.75563323,
-4.2848887 , -1.9744428 , -1.2648076 , 0.08407089, 4.772733 ,
5.8138533 , 6.515413 , 1.0461919 , 1.2453866 , -6.4403944 ,
-2.84369 , 14.662944 , -1.8262477 , -5.775688 , -4.297834 ]],
dtype=float32)]
label id result is: tensor([36.])
mnn load
(1, 3, 384, 384)
(1, 3, 384, 384)
True
mnn result is: [-1.17220068 2.52835894 -1.13788819 -2.74680758 -2.28597975 -2.38775802
-7.82197857 -1.41898251 -0.53285587 -4.01219416 -1.37057006 -3.64638305
-3.05643773 -2.51107216 -2.03671527 -3.42733812 -5.13568115 -5.57139444
4.72767591 -3.82120156 8.26269436 -1.66735351 -0.32833859 -0.43691555
0.75550669 -4.28488731 -1.97452545 -1.26469791 0.08376746 4.77248907
5.81364202 6.51514387 1.04610574 1.24551427 -6.44040728 -2.84363627
14.66283894 -1.82612789 -5.77541924 -4.29777861]
label id result is: 36
Process finished with exit code 0
版本一 测试c++
#include <iostream>
#include<opencv2/core.hpp>
#include<opencv2/imgproc.hpp>
#include<opencv2/highgui.hpp>
#include<MNN/Interpreter.hpp>
#include<MNN/ImageProcess.hpp>
using namespace std;
using namespace cv;
using namespace MNN;
#define IMAGE_SIZE 384
int main() {
//cv::INTER_LINEAR为双线性插值，需要和python下的resize插值方式一致
cv::Mat img_src = cv::imread("/home/ubuntu/TNN/sxj731533730/image/5293F.png");
std::vector<float> meanVals ={-0.485f * 255.f * (1.0f / 0.229f) * (1.0 / 255.f),
-0.456f * 255.f * (1.0f / 0.224f) * (1.0 / 255.f),
-0.406f * 255.f * (1.0f / 0.225f) * (1.0 / 255.f)};
std::vector<float> normVals = {(1.0f / 0.229f) * (1.0 / 255.f),
(1.0f / 0.224f) * (1.0 / 255.f),
(1.0f / 0.225f) * (1.0 / 255.f)};
cv::Mat img_resized;
cv::resize(img_src.clone(), img_resized, cv::Size(IMAGE_SIZE, IMAGE_SIZE), cv::INTER_LINEAR);
cv::Mat img_color;
cv::cvtColor(img_resized, img_color, COLOR_BGR2RGB);
auto net = std::shared_ptr<MNN::Interpreter>(MNN::Interpreter::createFromFile("/home/ubuntu/TNN/sxj731533730/model-87_384_sim.mnn"));//创建解释器
cout << "Interpreter created" << endl;
ScheduleConfig config;
config.numThread = 8;
config.type = MNN_FORWARD_CPU;
auto session = net->createSession(config);//创建session
cout << "session created" << endl;
auto inTensor = net->getSessionInput(session, NULL);
auto outTensor = net->getSessionInput(session, NULL);
auto _Tensor = MNN::Tensor::create<float>({1,3,IMAGE_SIZE,IMAGE_SIZE}, NULL, MNN::Tensor::CAFFE);
if(_Tensor->elementSize()!=3*IMAGE_SIZE*IMAGE_SIZE)
{
std::cout<<_Tensor->elementSize()<<" "<<img_color.channels()*img_color.cols*img_color.rows<<std::endl;
std::cout<<"input shape not equal image shape"<<std::endl;
return -1;
}
std::vector<cv::Mat> rgbChannels(3);
cv::split(img_color, rgbChannels);
for (auto i = 0; i < rgbChannels.size(); i++) {
rgbChannels[i].convertTo(rgbChannels[i], CV_32FC1, normVals[i], meanVals [i]);
for(int j=0;j<rgbChannels[i].rows;j++) {
for (int k = 0; k < rgbChannels[i].cols; k++) {
_Tensor->host<float>()[i*IMAGE_SIZE*IMAGE_SIZE+j*IMAGE_SIZE+k] =rgbChannels[i].at<float>(j, k);
}
}
}
/* //这样也写可以
cv::Mat img_dst;
cv::merge(rgbChannels, img_dst);
for(int i=0;i<img_dst.channels();i++) {
for (int j = 0; j < img_dst.rows; j++) {
for (int k = 0; k < img_dst.cols; k++) {
_Tensor->host<float>()[i * IMAGE_SIZE * IMAGE_SIZE + j * IMAGE_SIZE + k] = rgbChannels[i].at<float>(j,
k);
}
}
}
*/
inTensor->copyFromHostTensor(_Tensor);
//推理
net->runSession(session);
auto output= net->getSessionOutput(session, NULL);
// MNN::Tensor feat_tensor(output, output->getDimensionType());
// output->copyToHostTensor(&feat_tensor);
// feat_tensor.print();
MNN::Tensor score_host(output, output->getDimensionType());
output->copyToHostTensor(&score_host);
auto score_ptr = score_host.host<float>();
std::vector<std::pair<float, int>> scores;
for (int i = 0; i < score_host.elementSize(); ++i) {
float score = score_ptr[i];
if(i%5!=0||i==0){
std::cout<<score<<" ,";
}else {
std::cout<<score<<std::endl;
}
}
return 0;
}
测试结果
/home/ubuntu/CLionProjects/untitled8/cmake-build-debug/untitled8
Interpreter created
session created
-1.17216 ,2.52834 ,-1.1379 ,-2.74682 ,-2.28605 ,-2.38766
-7.822 ,-1.41908 ,-0.532889 ,-4.01223 ,-1.37055
-3.64634 ,-3.05633 ,-2.51103 ,-2.03672 ,-3.42735
-5.13568 ,-5.57145 ,4.72752 ,-3.82123 ,8.26262
-1.6673 ,-0.328313 ,-0.436794 ,0.755487 ,-4.28484
-1.9745 ,-1.26462 ,0.0837567 ,4.77251 ,5.81368
6.51515 ,1.04612 ,1.24547 ,-6.44043 ,-2.84369
14.6628 ,-1.82615 ,-5.77546 ,-4.29767 ,
Process finished with exit code 0
版本二测试c++
#include <iostream>
#include <opencv2/opencv.hpp>
#include <MNN/Interpreter.hpp>
#define IMAGE_VERIFY_SIZE 384
#define CLASSES_SIZE 40
#define INPUT_NAME "input"
#define OUTPUT_NAME "output"
// mnn model input=[1, 3, 224, 224], output=[1, 1000]
int main(int argc, char* argv[]){
// create net and session
const char *mnn_model_path = "/home/ubuntu/CLionProjects/untitled3/model-87_384_sim.mnn";
const char *image_path = "/home/ubuntu/PycharmProjects/pythonProject3/5293F.png";
auto mnnNet = std::shared_ptr<MNN::Interpreter>(MNN::Interpreter::createFromFile(mnn_model_path));
MNN::ScheduleConfig netConfig;
netConfig.type = MNN_FORWARD_CPU;
netConfig.numThread = 4;
auto session = mnnNet->createSession(netConfig);
auto input = mnnNet->getSessionInput(session, INPUT_NAME);
if (input->elementSize() <= 4) {
mnnNet->resizeTensor(input, {1, 3, IMAGE_VERIFY_SIZE, IMAGE_VERIFY_SIZE});
mnnNet->resizeSession(session);
}
std::cout << "input shape: " << input->shape()[0] << " " << input->shape()[1] << " " << input->shape()[2] << " " << input->shape()[3] << std::endl;
// preprocess image
MNN::Tensor givenTensor(input, MNN::Tensor::CAFFE);
// const int inputSize = givenTensor.elementSize();
// std::cout << inputSize << std::endl;
auto inputData = givenTensor.host<float>();
cv::Mat bgr_image = cv::imread(image_path);
cv::Mat norm_image;
cv::Mat rgb_image;
cv::cvtColor(bgr_image,rgb_image,cv::COLOR_BGR2RGB);
cv::resize(rgb_image, norm_image, cv::Size(IMAGE_VERIFY_SIZE, IMAGE_VERIFY_SIZE));
for(int k = 0; k < 3; k++){
for(int i = 0; i < norm_image.rows; i++){
for(int j = 0; j < norm_image.cols; j++){
const auto src = norm_image.at<cv::Vec3b>(i, j)[k];
auto dst = 0.0;
if(k == 0) dst = (float(src) / 255.0f - 0.485) / 0.229;
if(k == 1) dst = (float(src) / 255.0f - 0.456) / 0.224;
if(k == 2) dst = (float(src) / 255.0f - 0.406) / 0.225;
inputData[k * IMAGE_VERIFY_SIZE * IMAGE_VERIFY_SIZE + i * IMAGE_VERIFY_SIZE + j] = dst;
}
}
}
input->copyFromHostTensor(&givenTensor);
// run session
mnnNet->runSession(session);
// get output data
auto output = mnnNet->getSessionOutput(session, OUTPUT_NAME);
// std::cout << "output shape: " << output->shape()[0] << " " << output->shape()[1] << std::endl;
auto output_host = std::make_shared<MNN::Tensor>(output, MNN::Tensor::CAFFE);
output->copyToHostTensor(output_host.get());
auto values = output_host->host<float>();
// post process
std::vector<float> output_values;
auto exp_sum = 0.0;
auto max_index = 0;
for(int i = 0; i < output_host->elementSize(); i++){
if(values[i] > values[max_index]) max_index = i;
output_values.push_back(values[i]);
exp_sum += std::exp(values[i]);
if(i%5!=0||i==0){
std::cout<<values[i]<<" ,";
}else {
std::cout<<values[i]<<std::endl;
}
}
std::cout << "cls id: " << max_index << std::endl;
std::cout << "cls prob: " << std::exp(output_values[max_index]) / exp_sum << std::endl;
return 0;
}
测试结果
/home/ubuntu/CLionProjects/untitled3/cmake-build-debug/untitled3
input shape: 1 3 384 384
-1.17208 ,2.52835 ,-1.13784 ,-2.74679 ,-2.28598 ,-2.38782
-7.82185 ,-1.41909 ,-0.53293 ,-4.01217 ,-1.37053
-3.64631 ,-3.05637 ,-2.51102 ,-2.03677 ,-3.42727
-5.13564 ,-5.57135 ,4.7276 ,-3.82109 ,8.2626
-1.66725 ,-0.328283 ,-0.436888 ,0.755527 ,-4.28481
-1.9745 ,-1.26475 ,0.0837912 ,4.7725 ,5.81344
6.51513 ,1.04599 ,1.24563 ,-6.4404 ,-2.84359
14.6626 ,-1.82606 ,-5.77536 ,-4.29771 ,
cls id: 36
cls prob: 0.9978
Process finished with exit code 0
参考:
感谢ncnn卷卷群的nihui大佬支持和mnn钉钉群的xiaying大佬支持
https://github.com/Tencent/ncnn
https://github.com/Tencent/TNN