xilinx-k26-starterkit-20221:~/kv260_median/dpu/vitis_ai_dpu_median_yolo$ sudo ./median_demo_yolov4 ../compiled/yolov4_leaky_416_tf.prototxt ../compiled/yolov4_leaky_416_tf.xmodel test2.jpg image
../compiled/yolov4_leaky_416_tf.prototxt ../compiled/yolov4_leaky_416_tf.xmodel test2.jpgModel Initialize Done
phys_addr = 40000000
in_img_total_bytes = 1441792
tvmonitor 0.985391 151.825 451.825 50.2466 335.823
tvmonitor 0.599225 0.870854 197.449 150.53 287.931
mouse 0.939509 462.855 526.316 313.673 346.846
mouse 0.877212 624.63 663.122 234.817 254.938
mouse 0.318674 621.84 666.77 227.559 259.926
keyboard 0.998828 277.435 500.512 314.605 444.413
cell phone 0.332537 354.317 476.452 415.798 503.778
// median_demo_yolo4.cpp
// 2022/11/23
// Combined demo_yolo4.cpp and median_pf.cpp.
// https://github.com/lp6m/vitis_ai_dpu_yolo/blob/master/demo_yolov4.cpp
// https://marsee101.blog.fc2.com/blog-entry-5747.html
#include <glog/logging.h>
#include <google/protobuf/text_format.h>
#include <cmath>
#include <iostream>
#include <numeric>
#include <opencv2/core.hpp>
#include <opencv2/highgui.hpp>
#include <opencv2/imgproc.hpp>
#include <vitis/ai/dpu_task.hpp>
#include <vitis/ai/nnpp/yolov3.hpp>
#include <fstream>
#include <map>
#include <vector>
#include <math.h>
#include <stdio.h>
#include <stdint.h>
#include <stdlib.h>
#include <fcntl.h>
#include <sys/mman.h>
#include <unistd.h>
#define BLOCK_SIZE 4096
#define MIDEAIN_REG_ADDR 0x80020000
#define AXI_DMA_REG_ADDR 0x80010000
#define IMAGE_WIDTH 800
#define IMAGE_HEIGHT 600
#define MAT_IMGAGE_BUF (IMAGE_WIDTH*IMAGE_HEIGHT*3)
#define MM2S_CONTROL_REG 0x00
#define MM2S_STATUS_REG (0x4 >> 2)
#define MM2S_START_ADDR (0x18 >> 2)
#define MM2S_LENGTH_REG (0x28 >> 2)
#define S2MM_CONTROL_REG (0x30 >> 2)
#define S2MM_STATUS_REG (0x34 >> 2)
#define S2MM_DESTINATION_ADDR (0x48 >> 2)
#define S2MM_LENGTH_REG (0x58 >> 2)
// bits 1 - idle
#define MM2S_IDLE_MASK 0x2
#define S2MM_IDLE_MASK 0x2
#define MEDIAN_CONTROL 0x00
#define MEDIAN_FUNCTION_R (0x18 >> 2)
#define MEDIAN_ROW_SIZE (0x20 >> 2)
#define MEDIAN_COL_SIZE (0x28 >> 2)
volatile uint32_t *reg;
using namespace std;
using namespace cv;
// The parameters of yolov3_voc, each value could be set as actual needs.
// Such format could be refer to the prototxts in /etc/dpu_model_param.d.conf/.
const string readFile(const char *filename){
ifstream ifs(filename);
return string(istreambuf_iterator<char>(ifs),
istreambuf_iterator<char>());
}
class YoloRunner{
public:
unique_ptr<vitis::ai::DpuTask> task;
vitis::ai::proto::DpuModelParam modelconfig;
cv::Size model_input_size;
vector<vitis::ai::library::InputTensor> input_tensor;
struct bbox{
int label;
float xmin;
float ymin;
float width;
float height;
float score;
bbox(vitis::ai::YOLOv3Result::BoundingBox yolobbox, float img_width, float img_height){
this->label = yolobbox.label;
this->score = yolobbox.score;
// does not clamp here
this->xmin = yolobbox.x * img_width;
this->ymin = yolobbox.y * img_height;
this->width = yolobbox.width * img_width;
this->height = yolobbox.height * img_height;
}
};
public: YoloRunner(const char* modelconfig_path, const char* modelfile_path){
const string config_str = readFile(modelconfig_path);
auto ok = google::protobuf::TextFormat::ParseFromString(config_str, &(this->modelconfig));
if (!ok) {
cerr << "Set parameters failed!" << endl;
abort();
}
this->task = vitis::ai::DpuTask::create(modelfile_path);
this->input_tensor = task->getInputTensor(0u);
int width = this->input_tensor[0].width;
int height = this->input_tensor[0].height;
this->model_input_size = cv::Size(width, height);
this->task->setMeanScaleBGR({0.0f, 0.0f, 0.0f},
{0.00390625f, 0.00390625f, 0.00390625f});
}
private: cv::Mat Preprocess(cv::Mat img){
cv::Mat resized_img;
cv::resize(img, resized_img, this->model_input_size);
return resized_img;
}
public: vector<bbox> Run(cv::Mat img){
cv::Mat resized_img = this->Preprocess(img);
vector<int> input_cols = {img.cols};
vector<int> input_rows = {img.rows};
vector<cv::Mat> inputs = {resized_img};
task->setImageRGB(inputs);
task->run(0);
auto output_tensor = task->getOutputTensor(0u);
auto results = vitis::ai::yolov3_post_process(
input_tensor, output_tensor, this->modelconfig, input_cols, input_rows);
auto result = results[0]; //batch_size is 1
vector<bbox> bboxes;
for(auto& yolobbox: result.bboxes){
bboxes.push_back(bbox(yolobbox, img.cols, img.rows));
}
return bboxes;
}
};
std::string get_basename(std::string& path) {
int l = path.find_last_of('/')+1;
int r = path.find_last_of('.');
return path.substr(l, r-l);
}
map<string, string> bbox_to_map(YoloRunner::bbox bbox, int frame_id){
map<string, string> res;
res["frame_id"] = to_string(frame_id);
res["prob"] = to_string(bbox.score);
res["x"] = to_string(bbox.xmin);
res["y"] = to_string(bbox.ymin);
res["width"] = to_string(bbox.width);
res["height"] = to_string(bbox.height);
return res;
}
int main(int argc, char* argv[]) {
int fd;
volatile uint32_t *median_reg, *axi_dma_reg;
volatile uint8_t *pict_buf;
uint32_t phy_addr;
uint32_t phy_addr_base;
int addr, wd;
uint32_t write_data;
cv::Mat in_img, median_img;
int fd_udmabuf;
u_int32_t fd_paddr;
unsigned char attr[1024];
unsigned long phys_addr;
if (argc != 5) {
cerr << "usage ./a.out config(.prototxt) modelfile(.xmodel) image(.jpg) image" << endl;
}
char* configfile = argv[1];
char* modelfile = argv[2];
string img_or_video_file = string(argv[3]);
cout << configfile << " " << modelfile << " " << img_or_video_file;
auto runner = YoloRunner(configfile, modelfile);
cout << "Model Initialize Done" << endl;
std::string img_or_video_mode = std::string(argv[4]);
if (img_or_video_mode == "image") {
in_img = cv::imread(img_or_video_file);
median_img.create(cv::Size(in_img.cols, in_img.rows), CV_8UC3);
fd = open("/dev/mem", O_RDWR | O_SYNC);
if (fd == -1){
fprintf(stderr, "/dev/mem open error\n");
exit(-1);
}
// median_filter registers
median_reg = (uint32_t *)mmap(NULL, BLOCK_SIZE,
PROT_READ | PROT_WRITE, MAP_SHARED,
fd, MIDEAIN_REG_ADDR );
if ((int64_t)median_reg == -1){
fprintf(stderr,"/dev/mem map error for median_filter registers\n");
exit(-1);
}
// axi_dma registers
axi_dma_reg = (uint32_t *)mmap(NULL, BLOCK_SIZE,
PROT_READ | PROT_WRITE, MAP_SHARED,
fd, AXI_DMA_REG_ADDR );
if ((int64_t)axi_dma_reg == -1){
fprintf(stderr,"/dev/mem map error for axi_dma registers\n");
exit(-1);
}
// udmabuf0
fd_udmabuf = open("/dev/udmabuf0", O_RDWR | O_SYNC); // frame_buffer, The chache is disabled.
if (fd_udmabuf == -1){
fprintf(stderr, "/dev/udmabuf0 open errorn");
exit(-1);
}
// phys_addr of udmabuf0
fd_paddr = open("/sys/class/u-dma-buf/udmabuf0/phys_addr", O_RDONLY);
if (fd_paddr == -1){
fprintf(stderr, "/sys/class/u-dma-buf/udmabuf0/phys_addr open errorn");
exit(-1);
}
read(fd_paddr, (void *)attr, 1024);
sscanf((const char *)attr, "%lx", &phys_addr);
close(fd_paddr);
printf("phys_addr = %x\n", (unsigned int)phys_addr);
uint32_t total_bytes = in_img.total()*in_img.channels();
uint32_t in_img_total_bytes = (in_img.total()*in_img.channels()+4096) & 0xfffff000; // 4k byte boundary
printf("in_img_total_bytes = %d\n", in_img_total_bytes);
pict_buf = (volatile uint8_t *)mmap(NULL, in_img_total_bytes*2, PROT_READ|PROT_WRITE, MAP_SHARED, fd_udmabuf, 0);
if (pict_buf == MAP_FAILED){
fprintf(stderr, "org_mat mmap error\n");
exit(-1);
}
// Copy Mat data from in_img to org_mat
uint8_t *in_img_data = in_img.data;
for(int i=0; i<total_bytes; i++){
pict_buf[i] = in_img_data[i];
}
// Halting Run DMA
axi_dma_reg[MM2S_CONTROL_REG] = 1; // MM2S DMA Controll Reg. Run
axi_dma_reg[S2MM_CONTROL_REG] = 1; // S2MM DMA Control Reg. Run
uint32_t median_mat_addr = (uint32_t)phys_addr+in_img_total_bytes;
uint32_t org_mat_addr = (uint32_t)phys_addr;
// axi dma settings
axi_dma_reg[S2MM_DESTINATION_ADDR] = median_mat_addr;
axi_dma_reg[MM2S_START_ADDR] = org_mat_addr;
axi_dma_reg[S2MM_LENGTH_REG] = total_bytes;
axi_dma_reg[MM2S_LENGTH_REG] = total_bytes;
// median filter start
median_reg[MEDIAN_COL_SIZE] = in_img.cols;
median_reg[MEDIAN_ROW_SIZE] = in_img.rows;
median_reg[MEDIAN_FUNCTION_R] = 3; // median filter for AXI DMA
median_reg[MEDIAN_CONTROL] = 1; // ap_start
// DMA completion detection
uint32_t mm2s_status_reg = axi_dma_reg[MM2S_STATUS_REG] & MM2S_IDLE_MASK;
while(mm2s_status_reg != MM2S_IDLE_MASK){
mm2s_status_reg = axi_dma_reg[MM2S_STATUS_REG] & MM2S_IDLE_MASK;
}
uint32_t s2mm_status_reg = axi_dma_reg[S2MM_STATUS_REG] & S2MM_IDLE_MASK;
while(s2mm_status_reg != S2MM_IDLE_MASK){
s2mm_status_reg = axi_dma_reg[S2MM_STATUS_REG] & S2MM_IDLE_MASK;
}
// Copy median image data from median_mat to megian_img
uint8_t *median_img_data = median_img.data;
for(int i=0; i<total_bytes; i++){
median_img_data[i] = pict_buf[in_img_total_bytes+i];
}
cv::Mat img;
resize(median_img, img, cv::Size(), 768.0/median_img.cols, 576.0/median_img.rows);
vector<YoloRunner::bbox> bboxes = runner.Run(median_img);
string label_names[] = {"person", "bicycle", "car", "motorbike", "aeroplane", "bus", "train", "truck", "boat", "traffic light", "fire hydrant", "stop sign", "parking meter", "bench", "bird", "cat", "dog", "horse", "sheep", "cow", "elephant", "bear", "zebra", "giraffe", "backpack", "umbrella", "handbag", "tie", "suitcase", "frisbee", "skis", "snowboard", "sports ball", "kite", "baseball bat", "baseball glove", "skateboard", "surfboard", "tennis racket", "bottle", "wine glass", "cup", "fork", "knife", "spoon", "bowl", "banana", "apple", "sandwich", "orange", "broccoli", "carrot", "hot dog", "pizza", "donut", "cake", "chair", "sofa", "pottedplant", "bed", "diningtable", "toilet", "tvmonitor", "laptop", "mouse", "remote", "keyboard", "cell phone", "microwave", "oven", "toaster", "sink", "refrigerator", "book", "clock", "vase", "scissors", "teddy bear", "hair drier", "toothbrush"};
for (auto& box : bboxes) {
int label = box.label;
float confidence = box.score;
float xmin = max(0.0f, box.xmin);
float ymin = max(0.0f, box.ymin);
float xmax = min(box.xmin + box.width, (float)img.cols-1.0f);
float ymax = min(box.ymin + box.height, (float)img.rows-1.0f);
cout << label_names[box.label] << " " << box.score << " " << xmin << " " << xmax << " " << ymin << " " << ymax << endl;
rectangle(img, Point(xmin, ymin), Point(xmax, ymax),
Scalar(0, 255, 0), 3, 1, 0);
}
imwrite("result.jpg", img);
} else {
cerr << "unknown mode :" << img_or_video_mode << endl;
}
return 0;
}
#
# Copyright 2019 Xilinx Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
# 2022/11/23 : Changed demo_yolov4 to median_demo_yolov4. by marsee
result=0 && pkg-config --list-all | grep opencv4 && result=1
if [ $result -eq 1 ]; then
OPENCV_FLAGS=$(pkg-config --cflags --libs-only-L opencv4)
else
OPENCV_FLAGS=$(pkg-config --cflags --libs-only-L opencv)
fi
CXX=${CXX:-g++}
$CXX -std=c++17 -O3 -I. -o median_demo_yolov4 median_demo_yolov4.cpp -lglog -lvitis_ai_library-xnnpp -lvitis_ai_library-model_config -lprotobuf -lvitis_ai_library-dpu_task ${OPENCV_FLAGS} -lopencv_core -lopencv_video -lopencv_videoio -lopencv_imgproc -lopencv_imgcodecs -lopencv_highgui
xilinx-k26-starterkit-20221:~/kv260_median/dpu/vitis_ai_dpu_yolo$ ./demo_yolov4 ../compiled/yolov4_leaky_416_tf.prototxt ../compiled/yolov4_leaky_416_tf.xmodel dog.jpg image
../compiled/yolov4_leaky_416_tf.prototxt ../compiled/yolov4_leaky_416_tf.xmodel dog.jpgModel Initialize Done
bicycle 0.951898 132.472 550.138 140.873 414.652
car 0.763319 458.46 700.775 71.8596 174.588
truck 0.558414 472.915 687.069 72.4816 169.532
dog 0.988311 133.963 300.746 210 548.74
name: "yolov4"
kernel {
mean: 0.0
mean: 0.0
mean: 0.0
scale: 0.00390625
scale: 0.00390625
scale: 0.00390625
}
model_type : YOLOv3
yolo_v3_param {
num_classes: 80
anchorCnt: 3
layer_name: "109"
layer_name: "101"
layer_name: "93"
conf_threshold: 0.3
nms_threshold: 0.6
biases: 10
biases: 13
biases: 16
biases: 30
biases: 33
biases: 23
biases: 30
biases: 61
biases: 62
biases: 45
biases: 59
biases: 119
biases: 116
biases: 90
biases: 156
biases: 198
biases: 373
biases: 326
test_mAP: false
}
日 | 月 | 火 | 水 | 木 | 金 | 土 |
---|---|---|---|---|---|---|
- | - | - | - | 1 | 2 | 3 |
4 | 5 | 6 | 7 | 8 | 9 | 10 |
11 | 12 | 13 | 14 | 15 | 16 | 17 |
18 | 19 | 20 | 21 | 22 | 23 | 24 |
25 | 26 | 27 | 28 | 29 | 30 | - |