获取实验环境
采用共享镜像,创建基于Ascend 310的环境,用户使用HwHiAiUser。
模型准备
格式转换
由于获取的StarGAN模型为PyTroch训练得到的ONNX格式,需要使用ATC转换为能适配昇腾的OM模型
模型测试
使用msame工具测试转换后的模型
编译代码
运行代码
实验结果
图片1
图片2
original
brownHair male young
blondHair female old
代码详解
代码结构
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 . ├── CMakeLists.txt // cmake file ├── data // 原图像 │ ├── input_1.jpg │ └── input_2.jpg ├── inc // 头文件 │ ├── dvpp_cropandpaste.h │ ├── dvpp_jpegd.h │ ├── dvpp_process.h │ ├── main.h │ └── model_process.h ├── model // 预训练模型 │ ├── aipp_nv12.cfg │ ├── fusion_result.json │ ├── StarGAN_aipp.om │ └── StarGAN.onnx ├── scripts // 编译与执行脚本 │ ├── sample_build.sh │ └── sample_run.sh ├── src // 源文件 │ ├── acl.json │ ├── CMakeLists.txt │ ├── dvpp_cropandpaste.cpp │ ├── dvpp_jpegd.cpp │ ├── dvpp_process.cpp │ ├── main.cpp │ └── model_process.cpp ├── out // 输出文件夹 └── README_CN.md
类说明
main中的共有变量
结构体
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 enum Result { SUCCESS = 0 , FAILED = 1 }; struct Resolution { uint32_t width = 0 ; uint32_t height = 0 ; }; struct ImageData { acldvppPixelFormat format; uint32_t width = 0 ; uint32_t height = 0 ; uint32_t alignWidth = 0 ; uint32_t alignHeight = 0 ; uint32_t size = 0 ; std::shared_ptr<uint8_t > data = nullptr ; };
命名空间
1 2 3 4 5 6 7 8 9 10 11 12 namespace { const char *kModelPath = "../model/StarGAN_aipp.om" ; uint32_t blackHair = 0 ; uint32_t blondHair = 1 ; uint32_t brownHair = 0 ; uint32_t male = 0 ; uint32_t young = 1 ; uint32_t deviceId = 0 ; aclrtContext context; aclrtStream stream; }
ModelProcess
该类处理与模型相关的操作。
私有成员变量
1 2 3 4 5 6 7 private : bool isDevice_; uint32_t modelId_; bool loadFlag_; aclmdlDesc *modelDesc_; aclmdlDataset *input_; aclmdlDataset *output_;
包含方法
方法名
参数与返回值
说明
LoadModel
参数 const char *modelPath
返回Result
加载模型
UnloadModel
-
卸载模型
CreateModelDesc
返回Result
创建模型描述
DestroyModelDesc
-
销毁模型描述
CreateInput
参数 void *inputDataBuffer, size_t bufferSize, void *inputDataBuffer1, size_t bufferSize1
返回Result
创建输入
DestroyInput
-
销毁输入
CreateOutput
返回Result
创建输出
DestroyOutput
-
销毁输出
Execute
返回Result
执行推理
GetModelOutputData
返回aclmdlDataset
返回模型输出结果
GetModelOutputWH
返回Result
获取模型输出dims中的宽高信息
GetModelInputWH
返回Result
获取模型输入dims中的宽高信息
DvppProcess
DVPP处理相关
私有成员变量
1 2 3 4 5 protected : int isInitOk_; aclrtStream stream_; acldvppChannelDesc *dvppChannelDesc_; bool isGlobalContext_;
包含方法
接口
说明
Result ProportionCropAndPaste(ImageData& dest, ImageData& src, uint32_t ltHorz, uint32_t ltVert, uint32_t rbHorz, uint32_t rbVert)
进行抠图和提额度
Result CvtJpegToYuv420sp(ImageData& src, ImageData& dest)
调用JPEG图片解码
Result InitResource(aclrtStream& stream)
创建通道
void DestroyResource()
销毁通道信息及通道描述符
JPEG图片解码
私有成员变量
1 2 3 4 5 6 7 8 9 10 11 private : aclrtStream stream_; acldvppChannelDesc *dvppChannelDesc_; void * decodeOutBufferDev_; acldvppPicDesc *decodeOutputDesc_; uint32_t decodeOutWidth_; uint32_t decodeOutHeight_; uint32_t decodeOutWidthStride_; uint32_t decodeOutHeightStride_; uint32_t decodeOutBufferSize_;
成员方法
接口
说明
Result InitDecodeOutputDesc(ImageData& inputImage)
初始化输出图片描述
Result Process(ImageData& dest, ImageData& src)
JPEG DVPP解码
CropAndPaste
dvpp中的VPC抠图与贴图
私有成员变量
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 private : aclrtStream stream_; void *vpcOutBufferDev_; acldvppRoiConfig *cropArea_; acldvppRoiConfig *pasteArea_; acldvppPicDesc *vpcInputDesc_; acldvppPicDesc *vpcOutputDesc_; acldvppChannelDesc *dvppChannelDesc_; uint32_t ltHorz_; uint32_t rbHorz_; uint32_t ltVert_; uint32_t rbVert_; uint32_t vpcOutBufferSize_; uint32_t originalImageWidth_ = 0 ; uint32_t originalImageHeight_ = 0 ; Resolution size_;
成员方法
接口
说明
Result ProportionProcess(ImageData &resizedImage, ImageData &srcImage)
处理接口
Result InitCropAndPasteResource(ImageData &inputImage)
初始化资源,包含输入图片描述Result InitCropAndPasteInputDesc(ImageData &inputImage)
和输出图片描述Result InitCropAndPasteOutputDesc()
void DestroyCropAndPasteResource()
销毁资源,包含抠图和贴图的区域配置、输入和输出的图片描述
执行过程
资源申请
ACL初始化以及运行资源(Device、Context、Stream)申请
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 Result InitResource () { const char *aclConfigPath = "../src/acl.json" ; aclError ret = aclInit (aclConfigPath); if (ret != ACL_SUCCESS) { ERROR_LOG ("acl init failed, errorCode = %d" , static_cast <int32_t >(ret)); return FAILED; } INFO_LOG ("acl init success" ); ret = aclrtSetDevice (deviceId); if (ret != ACL_SUCCESS) { ERROR_LOG ("acl set device %d failed, errorCode = %d" , deviceId, static_cast <int32_t >(ret)); return FAILED; } INFO_LOG ("set device %d success" , deviceId); ret = aclrtCreateContext (&context, 0 ); if (ret != ACL_SUCCESS) { ERROR_LOG ("acl create context failed, deviceId= %d, errorCode = %d" , deviceId, static_cast <int32_t >(ret)); return FAILED; } INFO_LOG ("create context success" ); ret = aclrtCreateStream (&stream); if (ret != ACL_SUCCESS) { ERROR_LOG ("acl create stream failed, deviceId= %d, errorCode = %d" , deviceId, static_cast <int32_t >(ret)); return FAILED; } INFO_LOG ("create stream success" ); return SUCCESS; }
加载模型
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 Result ModelProcess::LoadModel (const char *modelPath) { if (loadFlag_) { ERROR_LOG ("model has already been loaded" ); return FAILED; } aclError ret = aclmdlLoadFromFile (modelPath, &modelId_); if (ret != ACL_SUCCESS) { ERROR_LOG ("load model from file failed, model path is %s, errorCode is %d" , modelPath, static_cast <int32_t >(ret)); return FAILED; } loadFlag_ = true ; INFO_LOG ("load model %s success" , modelPath); return SUCCESS; }
创建模型描述
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 Result ModelProcess::CreateModelDesc () { modelDesc_ = aclmdlCreateDesc (); if (modelDesc_ == nullptr ) { ERROR_LOG ("create model description failed" ); return FAILED; } aclError ret = aclmdlGetDesc (modelDesc_,modelId_); if (ret != ACL_SUCCESS) { ERROR_LOG ("get model description failed, modelId id %u, errorCode is %d" , modelId_, static_cast <int32_t >(ret)); return FAILED; } INFO_LOG ("create model description success" ); return SUCCESS; }
获取模型输入和输出的宽和高
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 Result ModelProcess::GetModelInputWH (int &width, int &height) { if (modelDesc_ == nullptr ) { ERROR_LOG ("no model description, get input hw failed" ); return FAILED; } aclmdlIODims dims; aclError ret = aclmdlGetInputDims (modelDesc_, 0 , &dims); if (ret != ACL_SUCCESS) { ERROR_LOG ("get model input dims failed, errorCode is %d" , static_cast <int32_t >(ret)); return FAILED; } if (dims.dimCount != 4 ) { ERROR_LOG ("invalid dimsCount %zu, get input hw failed" , dims.dimCount); return FAILED; } width = dims.dims[2 ]; height = dims.dims[1 ]; INFO_LOG ("model input width %d, input height %d" , width, height); return SUCCESS; } Result ModelProcess::GetModelOutputWH (int &width, int &height) { if (modelDesc_ == nullptr ) { ERROR_LOG ("no model description, get input hw failed" ); return FAILED; } aclmdlIODims dims; aclError ret = aclmdlGetOutputDims (modelDesc_, 0 , &dims); if (ret != ACL_SUCCESS) { ERROR_LOG ("get model output dims failed, errorCode is %d" , static_cast <int32_t >(ret)); return FAILED; } if (dims.dimCount != 4 ) { ERROR_LOG ("invalid dimsCount %zu, get input hw failed" , dims.dimCount); return FAILED; } width = dims.dims[3 ]; height = dims.dims[2 ]; INFO_LOG ("model output width %d, output height %d" , width, height); return SUCCESS; }
DVPP初始化
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 Result DvppProcess::InitResource (aclrtStream &stream) { aclError aclRet; dvppChannelDesc_ = acldvppCreateChannelDesc (); if (dvppChannelDesc_ == nullptr ) { ERROR_LOG ("acldvppCreateChannelDesc Failed" ); return FAILED; } aclRet = acldvppCreateChannel (dvppChannelDesc_); if (aclRet != ACL_SUCCESS) { ERROR_LOG ("acldvppCreateChannel failed, aclRet = %d" , aclRet); return FAILED; } stream_ = stream; isInitOk_ = true ; INFO_LOG ("dvpp init resource ok" ); return SUCCESS; }
读取文件到Host内存
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 Result ReadBinFile (const std::string &fileName, void *&inputBuff, uint32_t &fileSize) { std::ifstream binFile (fileName, std::ifstream::binary) ; if (binFile.is_open () == false ) { ERROR_LOG ("open file %s failed" , fileName.c_str ()); return FAILED; } binFile.seekg (0 , binFile.end); uint32_t binFileBufferLen = binFile.tellg (); if (binFileBufferLen == 0 ) { ERROR_LOG ("binfile is empty, filename is %s" , fileName.c_str ()); binFile.close (); return FAILED; } binFile.seekg (0 , binFile.beg); void *binFileBufferData = nullptr ; aclError ret = aclrtMallocHost (&binFileBufferData, binFileBufferLen); binFile.read (static_cast <char *>(binFileBufferData), binFileBufferLen); binFile.close (); inputBuff = binFileBufferData; fileSize = binFileBufferLen; return SUCCESS; }
JPEG图片解码
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 Result DvppJpegD::Process (ImageData &dest, ImageData &src) { int ret = InitDecodeOutputDesc (src); if (ret != SUCCESS) { ERROR_LOG ("InitDecodeOutputDesc failed" ); return FAILED; } ImageData imageDevice; void *buffer = nullptr ; acldvppMalloc (&buffer, src.size); aclrtMemcpy (buffer, src.size, src.data.get (), src.size, ACL_MEMCPY_HOST_TO_DEVICE); imageDevice.width = src.width; imageDevice.height = src.height; imageDevice.size = src.size; imageDevice.data = SHARED_PTR_DVPP_BUF (buffer); aclError aclRet = acldvppJpegDecodeAsync (dvppChannelDesc_, reinterpret_cast <void *>(imageDevice.data.get ()), imageDevice.size, decodeOutputDesc_, stream_); if (aclRet != ACL_SUCCESS) { ERROR_LOG ("acldvppJpegDecodeAsync failed, ret = %d" , aclRet); return FAILED; } aclRet = aclrtSynchronizeStream (stream_); if (aclRet != ACL_SUCCESS) { ERROR_LOG ("decode aclrtSynchronizeStream failed, ret = %d" , aclRet); return FAILED; } dest.format = PIXEL_FORMAT_YUV_SEMIPLANAR_420; dest.width = decodeOutWidth_; dest.height = decodeOutHeight_; dest.alignWidth = decodeOutWidthStride_; dest.alignHeight = decodeOutHeightStride_; dest.size = decodeOutBufferSize_; dest.data = SHARED_PTR_DVPP_BUF (decodeOutBufferDev_); INFO_LOG ("convert image success" ); DestroyDecodeResource (); return SUCCESS; }
抠图和贴图
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 Result CropAndPaste::ProportionProcess (ImageData &resizedImage, ImageData &srcImage) { if (SUCCESS != InitCropAndPasteResource (srcImage)) { ERROR_LOG ("Dvpp cropandpaste failed for init error" ); return FAILED; } uint32_t cropLeftOffset = 0 ; uint32_t cropTopOffset = 0 ; uint32_t cropRightOffset = (((cropLeftOffset + originalImageWidth_) >> 1 ) << 1 ) - 1 ; uint32_t cropBottomOffset = (((cropTopOffset + originalImageHeight_) >> 1 ) << 1 ) - 1 ; cropArea_ = acldvppCreateRoiConfig (cropLeftOffset, cropRightOffset, cropTopOffset, cropBottomOffset); if (cropArea_ == nullptr ) { ERROR_LOG ("acldvppCreateRoiConfig cropArea_ failed" ); return FAILED; } uint32_t pasteWidth = rbHorz_ - ltHorz_; uint32_t pasteHeight = rbVert_ - ltVert_; float rx = (float )originalImageWidth_ / (float )pasteWidth; float ry = (float )originalImageHeight_ / (float )pasteHeight; int dx = 0 ; int dy = 0 ; float r = 0.0f ; if (rx > ry) { dx = 0 ; r = rx; dy = (pasteHeight - originalImageHeight_ / r) / 2 ; } else { dy = 0 ; r = ry; dx = (pasteWidth - originalImageWidth_ / r) / 2 ; } uint32_t pasteLeftOffset = 0 ; uint32_t pasteTopOffset = 0 ; uint32_t pasteRightOffset = pasteWidth - 2 * dx; uint32_t pasteBottomOffset = pasteHeight - 2 * dy; pasteArea_ = acldvppCreateRoiConfig (pasteLeftOffset, pasteRightOffset, pasteTopOffset, pasteBottomOffset); if (pasteArea_ == nullptr ) { ERROR_LOG ("acldvppCreateRoiConfig pasteArea_ failed" ); return FAILED; } aclError ret = acldvppVpcCropAndPasteAsync (dvppChannelDesc_, vpcInputDesc_, vpcOutputDesc_, cropArea_, pasteArea_, stream_); if (ret != ACL_SUCCESS) { ERROR_LOG ("acldvppVpcCropAndPasteAsync failed, ret = %d" , ret); return FAILED; } ret = aclrtSynchronizeStream (stream_); if (ret != ACL_SUCCESS) { ERROR_LOG ("crop and paste aclrtSynchronizeStream failed, ret = %d" , ret); return FAILED; } resizedImage.width = pasteWidth - 2 * dx; resizedImage.height = pasteHeight - 2 * dy; resizedImage.alignWidth = ALIGN_UP16 (pasteWidth); resizedImage.alignHeight = ALIGN_UP2 (pasteHeight); resizedImage.size = vpcOutBufferSize_; resizedImage.data = SHARED_PTR_DVPP_BUF (vpcOutBufferDev_); DestroyCropAndPasteResource (); return SUCCESS; }
创建模型输入并传入Device内存,在Device上创建第二个输入
该步骤主要是完成Host到Device的数据传输
1 2 3 4 5 const float inputParam[5 ] = {(float )blackHair, (float )blondHair, (float )brownHair, (float )male, (float )young};uint32_t inputParamSize = sizeof (inputParam);void *inputParamBuffer = CopyDataToDevice ((void *)inputParam, inputParamSize, ACL_MEMCPY_HOST_TO_DEVICE);Result ret = modelProcess.CreateInput (resizedImage.data.get (), resizedImage.size, inputParamBuffer, inputParamSize);
模型推理
1 2 3 4 5 6 7 8 9 10 11 12 13 14 Result ModelProcess::Execute () { aclError ret = aclmdlExecute (modelId_, input_, output_); if (ret != ACL_SUCCESS) { ERROR_LOG ("execute model failed, modelId is %u, errorCode is %d" , modelId_, static_cast <int32_t >(ret)); return FAILED; } INFO_LOG ("model execute success" ); return SUCCESS; }
数据后处理
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 aclmdlDataset *inferenceOutput = modelProcess.GetModelOutputData (); aclDataBuffer *dataBuffer = aclmdlGetDatasetBuffer (inferenceOutput, 0 ); void *dataBufferDev = aclGetDataBufferAddr (dataBuffer);size_t bufferSize = aclGetDataBufferSizeV2 (dataBuffer);void *data = nullptr ;aclrtMallocHost (&data, bufferSize);aclrtMemcpy (data, bufferSize, dataBufferDev, bufferSize, ACL_MEMCPY_DEVICE_TO_HOST);float *detectData;int picSize = bufferSize / 4 ;detectData = (float *)data; int channelSize = outWidth * outHeight;int channel = 3 ;float NORMALIZE = 127.5 ;uint8_t img[channel][channelSize];for (int i = 0 ; i < picSize; i++){ img[i / channelSize][i % channelSize] = int32_t ((detectData[i] + 1.0 ) * NORMALIZE); } cv::Mat imageMat = cv::Mat::zeros (outWidth, outHeight, CV_8UC3); for (int i = 0 ; i < outHeight; i++){ for (int j = 0 ; j < outWidth; j++) { imageMat.at <Vec3b>(i, j)[0 ] = img[2 ][i * outWidth + j]; imageMat.at <Vec3b>(i, j)[1 ] = img[1 ][i * outWidth + j]; imageMat.at <Vec3b>(i, j)[2 ] = img[0 ][i * outWidth + j]; } } stringstream sstream; sstream.str ("" ); sstream << "./output/result_" << index << ".jpg" ; cv::imwrite (sstream.str (), imageMat);
模型以及Host资源释放
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 void ModelProcess::DestroyInput () { if (input_ == nullptr ) { return ; } for (size_t i = 0 ; i < aclmdlGetDatasetNumBuffers (input_); ++i) { aclDataBuffer *dataBuffer = aclmdlGetDatasetBuffer (input_, i); void *data = aclGetDataBufferAddr (dataBuffer); (void )aclrtFree (data); (void )aclDestroyDataBuffer (dataBuffer); } (void )aclmdlDestroyDataset (input_); input_ = nullptr ; INFO_LOG ("destroy model input success" ); } void ModelProcess::DestroyOutput () { if (output_ == nullptr ) { return ; } for (size_t i = 0 ; i < aclmdlGetDatasetNumBuffers (output_); ++i) { aclDataBuffer *dataBuffer = aclmdlGetDatasetBuffer (output_, i); void *data = aclGetDataBufferAddr (dataBuffer); (void )aclrtFree (data); (void )aclDestroyDataBuffer (dataBuffer); } (void )aclmdlDestroyDataset (output_); output_ = nullptr ; INFO_LOG ("destroy model output success" ); } (void )aclrtFreeHost (data); data = nullptr ;
进程资源释放
1 2 3 4 5 6 7 aclrtDestroyStream (stream);aclrtDestroyContext (context);aclrtResetDevice (deviceId);ret = aclFinalize ();