获取实验环境

采用共享镜像,创建基于Ascend 310的环境,用户使用HwHiAiUser。

env

模型准备

格式转换

由于获取的StarGAN模型为PyTroch训练得到的ONNX格式,需要使用ATC转换为能适配昇腾的OM模型

atc

模型测试

使用msame工具测试转换后的模型
msame

编译代码

build

运行代码

run

实验结果

图片1 图片2
original
brownHair male young 1 2
blondHair female old 1 2

代码详解

代码结构

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
.
├── CMakeLists.txt // cmake file
├── data // 原图像
│   ├── input_1.jpg
│   └── input_2.jpg
├── inc // 头文件
│   ├── dvpp_cropandpaste.h
│   ├── dvpp_jpegd.h
│   ├── dvpp_process.h
│   ├── main.h
│   └── model_process.h
├── model // 预训练模型
│   ├── aipp_nv12.cfg
│   ├── fusion_result.json
│   ├── StarGAN_aipp.om
│   └── StarGAN.onnx
├── scripts // 编译与执行脚本
│   ├── sample_build.sh
│   └── sample_run.sh
├── src // 源文件
│ ├── acl.json
│ ├── CMakeLists.txt
│ ├── dvpp_cropandpaste.cpp
│ ├── dvpp_jpegd.cpp
│ ├── dvpp_process.cpp
│ ├── main.cpp
│ └── model_process.cpp
├── out // 输出文件夹
└── README_CN.md

类说明

main中的共有变量

结构体

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
enum Result {
SUCCESS = 0,
FAILED = 1
};
struct Resolution {
uint32_t width = 0;
uint32_t height = 0;
};
struct ImageData {
acldvppPixelFormat format;
uint32_t width = 0;
uint32_t height = 0;
uint32_t alignWidth = 0;
uint32_t alignHeight = 0;
uint32_t size = 0;
std::shared_ptr<uint8_t> data = nullptr;
};

命名空间

1
2
3
4
5
6
7
8
9
10
11
12
namespace
{
const char *kModelPath = "../model/StarGAN_aipp.om";
uint32_t blackHair = 0; // 黑发色,1表示使能,0表示不使能
uint32_t blondHair = 1; // 金发色,1表示使能,0表示不使能
uint32_t brownHair = 0; // 棕发色,1表示使能,0表示不使能
uint32_t male = 0; // 性别改变,1表示男性,0表示女性
uint32_t young = 1; // 年轻化,1表示年轻化,0表示老龄化
uint32_t deviceId = 0;
aclrtContext context;
aclrtStream stream;
}

ModelProcess

该类处理与模型相关的操作。

私有成员变量

1
2
3
4
5
6
7
private:
bool isDevice_;
uint32_t modelId_;
bool loadFlag_;
aclmdlDesc *modelDesc_;
aclmdlDataset *input_;
aclmdlDataset *output_;

包含方法

方法名 参数与返回值 说明
LoadModel 参数 const char *modelPath 返回Result 加载模型
UnloadModel - 卸载模型
CreateModelDesc 返回Result 创建模型描述
DestroyModelDesc - 销毁模型描述
CreateInput 参数 void *inputDataBuffer, size_t bufferSize, void *inputDataBuffer1, size_t bufferSize1 返回Result 创建输入
DestroyInput - 销毁输入
CreateOutput 返回Result 创建输出
DestroyOutput - 销毁输出
Execute 返回Result 执行推理
GetModelOutputData 返回aclmdlDataset 返回模型输出结果
GetModelOutputWH 返回Result 获取模型输出dims中的宽高信息
GetModelInputWH 返回Result 获取模型输入dims中的宽高信息

DvppProcess

DVPP处理相关

私有成员变量

1
2
3
4
5
protected:
int isInitOk_;
aclrtStream stream_;
acldvppChannelDesc *dvppChannelDesc_;
bool isGlobalContext_;

包含方法

接口 说明
Result ProportionCropAndPaste(ImageData& dest, ImageData& src, uint32_t ltHorz, uint32_t ltVert, uint32_t rbHorz, uint32_t rbVert) 进行抠图和提额度
Result CvtJpegToYuv420sp(ImageData& src, ImageData& dest) 调用JPEG图片解码
Result InitResource(aclrtStream& stream) 创建通道
void DestroyResource() 销毁通道信息及通道描述符

JPEG图片解码

私有成员变量

1
2
3
4
5
6
7
8
9
10
11
private:
aclrtStream stream_;
acldvppChannelDesc *dvppChannelDesc_;

void* decodeOutBufferDev_;
acldvppPicDesc *decodeOutputDesc_;
uint32_t decodeOutWidth_;
uint32_t decodeOutHeight_;
uint32_t decodeOutWidthStride_;
uint32_t decodeOutHeightStride_;
uint32_t decodeOutBufferSize_;

成员方法

接口 说明
Result InitDecodeOutputDesc(ImageData& inputImage) 初始化输出图片描述
Result Process(ImageData& dest, ImageData& src) JPEG DVPP解码

CropAndPaste

dvpp中的VPC抠图与贴图

私有成员变量

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
private:
aclrtStream stream_;
void *vpcOutBufferDev_;
acldvppRoiConfig *cropArea_;
acldvppRoiConfig *pasteArea_;
acldvppPicDesc *vpcInputDesc_;
acldvppPicDesc *vpcOutputDesc_;
acldvppChannelDesc *dvppChannelDesc_;

uint32_t ltHorz_;
uint32_t rbHorz_;
uint32_t ltVert_;
uint32_t rbVert_;
uint32_t vpcOutBufferSize_;
uint32_t originalImageWidth_ = 0;
uint32_t originalImageHeight_ = 0;
Resolution size_;

成员方法

接口 说明
Result ProportionProcess(ImageData &resizedImage, ImageData &srcImage) 处理接口
Result InitCropAndPasteResource(ImageData &inputImage) 初始化资源,包含输入图片描述Result InitCropAndPasteInputDesc(ImageData &inputImage)和输出图片描述Result InitCropAndPasteOutputDesc()
void DestroyCropAndPasteResource() 销毁资源,包含抠图和贴图的区域配置、输入和输出的图片描述

执行过程

资源申请

ACL初始化以及运行资源(Device、Context、Stream)申请

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
Result InitResource()
{
// 1.进行ACL初始化
const char *aclConfigPath = "../src/acl.json";
aclError ret = aclInit(aclConfigPath);
if (ret != ACL_SUCCESS)
{
ERROR_LOG("acl init failed, errorCode = %d", static_cast<int32_t>(ret));
return FAILED;
}
INFO_LOG("acl init success");
// 2.运行管理资源申请
// TODO:Device、context、stream申请
ret = aclrtSetDevice(deviceId);
if (ret != ACL_SUCCESS)
{
ERROR_LOG("acl set device %d failed, errorCode = %d", deviceId, static_cast<int32_t>(ret));
return FAILED;
}
INFO_LOG("set device %d success", deviceId);

ret = aclrtCreateContext(&context, 0);
if (ret != ACL_SUCCESS)
{
ERROR_LOG("acl create context failed, deviceId= %d, errorCode = %d", deviceId, static_cast<int32_t>(ret));
return FAILED;
}
INFO_LOG("create context success");

ret = aclrtCreateStream(&stream);
if (ret != ACL_SUCCESS)
{
ERROR_LOG("acl create stream failed, deviceId= %d, errorCode = %d", deviceId, static_cast<int32_t>(ret));
return FAILED;
}
// DONE

INFO_LOG("create stream success");
return SUCCESS;
}

加载模型

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
Result ModelProcess::LoadModel(const char *modelPath)
{
if (loadFlag_)
{
ERROR_LOG("model has already been loaded");
return FAILED;
}
// TODO:加载本地模型文件,其中模型id为modelId_
aclError ret = aclmdlLoadFromFile(modelPath, &modelId_);
if (ret != ACL_SUCCESS)
{
ERROR_LOG("load model from file failed, model path is %s, errorCode is %d", modelPath, static_cast<int32_t>(ret));
return FAILED;
}
// DONE
loadFlag_ = true;
INFO_LOG("load model %s success", modelPath);
return SUCCESS;
}

创建模型描述

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
Result ModelProcess::CreateModelDesc()
{
modelDesc_ = aclmdlCreateDesc();
if (modelDesc_ == nullptr)
{
ERROR_LOG("create model description failed");
return FAILED;
}
// TODO:获取模型描述信息,模型ID为modelId_
aclError ret = aclmdlGetDesc(modelDesc_,modelId_);
if (ret != ACL_SUCCESS)
{
ERROR_LOG("get model description failed, modelId id %u, errorCode is %d", modelId_, static_cast<int32_t>(ret));
return FAILED;
}
// DONE

INFO_LOG("create model description success");

return SUCCESS;
}

获取模型输入和输出的宽和高

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
// 获取模型输入dims中的宽高信息
Result ModelProcess::GetModelInputWH(int &width, int &height)
{
if (modelDesc_ == nullptr)
{
ERROR_LOG("no model description, get input hw failed");
return FAILED;
}
// 当前模型输入为NCHW。所以需要获取dims[1]和dims[2]
aclmdlIODims dims;
// 模型第一个输入为图片
// TODO:获取模型第一个输入的dims信息,参数为(modelDesc_, 0, &dims)
aclError ret = aclmdlGetInputDims(modelDesc_, 0, &dims);
if (ret != ACL_SUCCESS)
{
ERROR_LOG("get model input dims failed, errorCode is %d", static_cast<int32_t>(ret));
return FAILED;
}
// DONE
if (dims.dimCount != 4)
{
ERROR_LOG("invalid dimsCount %zu, get input hw failed", dims.dimCount);
return FAILED;
}
width = dims.dims[2];
height = dims.dims[1];
INFO_LOG("model input width %d, input height %d", width, height);

return SUCCESS;
}

// 获取模型输出dims中的宽高信息
Result ModelProcess::GetModelOutputWH(int &width, int &height)
{
if (modelDesc_ == nullptr)
{
ERROR_LOG("no model description, get input hw failed");
return FAILED;
}
// 当前模型输出为NHWC。所以需要获取dims[2]和dims[3]
aclmdlIODims dims;
// 模型仅有一个输出结果
// TODO:获取模型第一个输出的dims信息,参数为(modelDesc_, 0, &dims)
aclError ret = aclmdlGetOutputDims(modelDesc_, 0, &dims);
if (ret != ACL_SUCCESS)
{
ERROR_LOG("get model output dims failed, errorCode is %d", static_cast<int32_t>(ret));
return FAILED;
}
// DONE
if (dims.dimCount != 4)
{
ERROR_LOG("invalid dimsCount %zu, get input hw failed", dims.dimCount);
return FAILED;
}
width = dims.dims[3];
height = dims.dims[2];
INFO_LOG("model output width %d, output height %d", width, height);

return SUCCESS;
}

DVPP初始化

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
Result DvppProcess::InitResource(aclrtStream &stream)
{
aclError aclRet;

// TODO:创建DVPP图片处理通道描述:dvppChannelDesc_,并创建通道
dvppChannelDesc_ = acldvppCreateChannelDesc();
if (dvppChannelDesc_ == nullptr)
{
ERROR_LOG("acldvppCreateChannelDesc Failed");
return FAILED;
}

aclRet = acldvppCreateChannel(dvppChannelDesc_);
if (aclRet != ACL_SUCCESS)
{
ERROR_LOG("acldvppCreateChannel failed, aclRet = %d", aclRet);
return FAILED;
}
// Done

stream_ = stream;
isInitOk_ = true;
INFO_LOG("dvpp init resource ok");
return SUCCESS;
}

读取文件到Host内存

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
Result ReadBinFile(const std::string &fileName, void *&inputBuff, uint32_t &fileSize)
{
std::ifstream binFile(fileName, std::ifstream::binary);
if (binFile.is_open() == false)
{
ERROR_LOG("open file %s failed", fileName.c_str());
return FAILED;
}

binFile.seekg(0, binFile.end);
uint32_t binFileBufferLen = binFile.tellg();
if (binFileBufferLen == 0)
{
ERROR_LOG("binfile is empty, filename is %s", fileName.c_str());
binFile.close();
return FAILED;
}

binFile.seekg(0, binFile.beg);

void *binFileBufferData = nullptr;
aclError ret = aclrtMallocHost(&binFileBufferData, binFileBufferLen);
binFile.read(static_cast<char *>(binFileBufferData), binFileBufferLen);
binFile.close();

inputBuff = binFileBufferData;
fileSize = binFileBufferLen;
return SUCCESS;
}

JPEG图片解码

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
Result DvppJpegD::Process(ImageData &dest, ImageData &src)
{
int ret = InitDecodeOutputDesc(src);
if (ret != SUCCESS)
{
ERROR_LOG("InitDecodeOutputDesc failed");
return FAILED;
}

ImageData imageDevice;
void *buffer = nullptr;
// 将原始图片拷贝到DVPP中再进行解码
acldvppMalloc(&buffer, src.size);
aclrtMemcpy(buffer, src.size, src.data.get(), src.size, ACL_MEMCPY_HOST_TO_DEVICE);
imageDevice.width = src.width;
imageDevice.height = src.height;
imageDevice.size = src.size;
imageDevice.data = SHARED_PTR_DVPP_BUF(buffer);

// TODO:执行图片解码,输入参数为(dvppChannelDesc_, reinterpret_cast<void *>(imageDevice.data.get()), imageDevice.size, decodeOutputDesc_, stream_)
aclError aclRet = acldvppJpegDecodeAsync(dvppChannelDesc_, reinterpret_cast<void *>(imageDevice.data.get()), imageDevice.size, decodeOutputDesc_, stream_);
if (aclRet != ACL_SUCCESS)
{
ERROR_LOG("acldvppJpegDecodeAsync failed, ret = %d", aclRet);
return FAILED;
}
// DONE
// TODO:执行同步等待,输入参数为stream_
aclRet = aclrtSynchronizeStream(stream_);
if (aclRet != ACL_SUCCESS)
{
ERROR_LOG("decode aclrtSynchronizeStream failed, ret = %d", aclRet);
return FAILED;
}
// DONE

dest.format = PIXEL_FORMAT_YUV_SEMIPLANAR_420;
dest.width = decodeOutWidth_;
dest.height = decodeOutHeight_;
dest.alignWidth = decodeOutWidthStride_;
dest.alignHeight = decodeOutHeightStride_;
dest.size = decodeOutBufferSize_;
dest.data = SHARED_PTR_DVPP_BUF(decodeOutBufferDev_);
INFO_LOG("convert image success");

DestroyDecodeResource();

return SUCCESS;
}

抠图和贴图

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
Result CropAndPaste::ProportionProcess(ImageData &resizedImage, ImageData &srcImage)
{
if (SUCCESS != InitCropAndPasteResource(srcImage))
{
ERROR_LOG("Dvpp cropandpaste failed for init error");
return FAILED;
}

// 偶数
uint32_t cropLeftOffset = 0;
uint32_t cropTopOffset = 0;
// 奇数
uint32_t cropRightOffset = (((cropLeftOffset + originalImageWidth_) >> 1) << 1) - 1;
uint32_t cropBottomOffset = (((cropTopOffset + originalImageHeight_) >> 1) << 1) - 1;
// 创建抠图区域
cropArea_ = acldvppCreateRoiConfig(cropLeftOffset, cropRightOffset,
cropTopOffset, cropBottomOffset);
if (cropArea_ == nullptr)
{
ERROR_LOG("acldvppCreateRoiConfig cropArea_ failed");
return FAILED;
}

uint32_t pasteWidth = rbHorz_ - ltHorz_;
uint32_t pasteHeight = rbVert_ - ltVert_;
// 设置抠图区域
float rx = (float)originalImageWidth_ / (float)pasteWidth;
float ry = (float)originalImageHeight_ / (float)pasteHeight;

int dx = 0;
int dy = 0;
float r = 0.0f;
if (rx > ry)
{
dx = 0;
r = rx;
dy = (pasteHeight - originalImageHeight_ / r) / 2;
}
else
{
dy = 0;
r = ry;
dx = (pasteWidth - originalImageWidth_ / r) / 2;
}

// 偶数
uint32_t pasteLeftOffset = 0;
uint32_t pasteTopOffset = 0;
// 奇数
uint32_t pasteRightOffset = pasteWidth - 2 * dx;
uint32_t pasteBottomOffset = pasteHeight - 2 * dy;
// 创建贴图区域
pasteArea_ = acldvppCreateRoiConfig(pasteLeftOffset, pasteRightOffset,
pasteTopOffset, pasteBottomOffset);
if (pasteArea_ == nullptr)
{
ERROR_LOG("acldvppCreateRoiConfig pasteArea_ failed");
return FAILED;
}

// TODO:使用DVPP进行图片缩放,输入参数为(dvppChannelDesc_, vpcInputDesc_, vpcOutputDesc_, cropArea_, pasteArea_, stream_)
aclError ret = acldvppVpcCropAndPasteAsync(dvppChannelDesc_, vpcInputDesc_, vpcOutputDesc_, cropArea_, pasteArea_, stream_);
if (ret != ACL_SUCCESS)
{
ERROR_LOG("acldvppVpcCropAndPasteAsync failed, ret = %d", ret);
return FAILED;
}
// DONE
ret = aclrtSynchronizeStream(stream_);
if (ret != ACL_SUCCESS)
{
ERROR_LOG("crop and paste aclrtSynchronizeStream failed, ret = %d", ret);
return FAILED;
}

resizedImage.width = pasteWidth - 2 * dx;
resizedImage.height = pasteHeight - 2 * dy;
resizedImage.alignWidth = ALIGN_UP16(pasteWidth);
resizedImage.alignHeight = ALIGN_UP2(pasteHeight);
resizedImage.size = vpcOutBufferSize_;
resizedImage.data = SHARED_PTR_DVPP_BUF(vpcOutBufferDev_);

DestroyCropAndPasteResource();

return SUCCESS;
}

创建模型输入并传入Device内存,在Device上创建第二个输入

该步骤主要是完成Host到Device的数据传输

1
2
3
4
5
const float inputParam[5] = {(float)blackHair, (float)blondHair, (float)brownHair, (float)male, (float)young};
uint32_t inputParamSize = sizeof(inputParam);
void *inputParamBuffer = CopyDataToDevice((void *)inputParam, inputParamSize, ACL_MEMCPY_HOST_TO_DEVICE);

Result ret = modelProcess.CreateInput(resizedImage.data.get(), resizedImage.size, inputParamBuffer, inputParamSize);

模型推理

1
2
3
4
5
6
7
8
9
10
11
12
13
14
Result ModelProcess::Execute()
{
// TODO:进行模型推理
aclError ret = aclmdlExecute(modelId_, input_, output_);
if (ret != ACL_SUCCESS)
{
ERROR_LOG("execute model failed, modelId is %u, errorCode is %d", modelId_, static_cast<int32_t>(ret));
return FAILED;
}
// DONE

INFO_LOG("model execute success");
return SUCCESS;
}

数据后处理

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
// 获取输出,该模型输出只有一个,直接获取即可
aclmdlDataset *inferenceOutput = modelProcess.GetModelOutputData();
aclDataBuffer *dataBuffer = aclmdlGetDatasetBuffer(inferenceOutput, 0);
void *dataBufferDev = aclGetDataBufferAddr(dataBuffer);
size_t bufferSize = aclGetDataBufferSizeV2(dataBuffer);
void *data = nullptr;
// TODO:将dataBufferDev拷贝到data中,准备进行后处理
aclrtMallocHost(&data, bufferSize);
aclrtMemcpy(data, bufferSize, dataBufferDev, bufferSize, ACL_MEMCPY_DEVICE_TO_HOST);
// Done

// 输出为RGB,每个channel的size为bufferSize/3
float *detectData;
int picSize = bufferSize / 4;
detectData = (float *)data;
int channelSize = outWidth * outHeight;
int channel = 3;
float NORMALIZE = 127.5;
uint8_t img[channel][channelSize];

// 将输出数据进行恢复,-1-1的值恢复到0-255
for (int i = 0; i < picSize; i++)
{
img[i / channelSize][i % channelSize] = int32_t((detectData[i] + 1.0) * NORMALIZE);
}

// 输出数据为NCHW格式,转换为NHWC,使用opencv进行输出
cv::Mat imageMat = cv::Mat::zeros(outWidth, outHeight, CV_8UC3);
for (int i = 0; i < outHeight; i++)
{
for (int j = 0; j < outWidth; j++)
{
imageMat.at<Vec3b>(i, j)[0] = img[2][i * outWidth + j];
imageMat.at<Vec3b>(i, j)[1] = img[1][i * outWidth + j];
imageMat.at<Vec3b>(i, j)[2] = img[0][i * outWidth + j];
}
}
// 设置输出图片名
stringstream sstream;
sstream.str("");
sstream << "./output/result_" << index << ".jpg";
// 输出图片
cv::imwrite(sstream.str(), imageMat);

模型以及Host资源释放

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
// 销毁模型输入
void ModelProcess::DestroyInput()
{
if (input_ == nullptr)
{
return;
}

for (size_t i = 0; i < aclmdlGetDatasetNumBuffers(input_); ++i)
{
aclDataBuffer *dataBuffer = aclmdlGetDatasetBuffer(input_, i);
void *data = aclGetDataBufferAddr(dataBuffer);
(void)aclrtFree(data);
(void)aclDestroyDataBuffer(dataBuffer);
}
(void)aclmdlDestroyDataset(input_);
input_ = nullptr;
INFO_LOG("destroy model input success");
}

// 销毁模型输出
void ModelProcess::DestroyOutput()
{
if (output_ == nullptr)
{
return;
}

for (size_t i = 0; i < aclmdlGetDatasetNumBuffers(output_); ++i)
{
aclDataBuffer *dataBuffer = aclmdlGetDatasetBuffer(output_, i);
void *data = aclGetDataBufferAddr(dataBuffer);
(void)aclrtFree(data);
(void)aclDestroyDataBuffer(dataBuffer);
}

(void)aclmdlDestroyDataset(output_);
output_ = nullptr;
INFO_LOG("destroy model output success");
}

// 释放host
(void)aclrtFreeHost(data);
data = nullptr;

进程资源释放

1
2
3
4
5
6
7
// Stream -> Context -> Device
aclrtDestroyStream(stream);
aclrtDestroyContext(context);
aclrtResetDevice(deviceId);

// ACL
ret = aclFinalize();