[pybind11] -- python C/C++ 扩展编译
1. 前言
在之前的pybind11系列实践中,开发流程大致是这样的:
第一步: 首先在C/C++ IDE中编写C/C++函数,然后采用pybind11封装为python可调用的包装函数, 之后采用C/C++编译器生成.pyd文件
第二步:将生成的.pyd文件复制到python工程中,之后作为python module import导入使用
存在的问题
不同操作系统下直接调用生成的pyd可能会出错,不能跨平台调用
在上述过程中,pyd动态链接库的生成是在本地PC上,但是如果想在不同的操作系统、硬件平台上调用之前生成的pyd,显然是会出错的。比如在windows上编译生成了一个python扩展.pyd, 但是Ubuntu系统或者树莓派上想调用这个python扩展显然就不行了。
解决方案
为了使得C/C++创建的python扩展可以跨平台使用,那么最简单的办法就是直接发布源码, 然后在该操作系统、硬件平台上编译生成python扩展。
本节内容利用python setuptools 方式实现
2. 开发环境
ubuntu16.04
Anaconda3, with python 3.6.7
pybind11
opencv3.4
3. project1
创建一个project1的文件下,然后,在其中新建如下的文件
.
├── example.cpp
├── setup.py
└── test.py
example.cpp
#include<pybind11/pybind11.h>
#include<pybind11/numpy.h>
#include<fstream>
#include<iostream>
namespace py = pybind11;
/*
*/
py::array_t<float> calcMul(py::array_t<float>& input1, py::array_t<float>& input2) {
// read inputs arrays buffer_info
py::buffer_info buf1 = input1.request();
py::buffer_info buf2 = input2.request();
if (buf1.size != buf2.size)
{
throw std::runtime_error("Input shapes must match");
}
// allocate the output buffer
py::array_t<double> result = py::array_t<double>(buf1.size);
}
class Matrix
{
public:
Matrix() {};
Matrix(int rows, int cols) {
this->m_rows = rows;
this->m_cols = cols;
m_data = new float[rows*cols];
}
~Matrix() {};
private:
int m_rows;
int m_cols;
float* m_data;
public:
float* data() { return m_data; };
int rows() { return m_rows; };
int cols() { return m_cols; };
};
void save_2d_numpy_array(py::array_t<float, py::array::c_style> a, std::string file_name) {
std::ofstream out;
out.open(file_name, std::ios::out);
std::cout << a.ndim() << std::endl;
for (int i = 0; i < a.ndim(); i++)
{
std::cout << a.shape()[i] << std::endl;
}
for (int i = 0; i < a.shape()[0]; i++)
{
for (int j = 0; j < a.shape()[1]; j++)
{
if (j == a.shape()[1]-1)
{
//访问读取,索引 numpy.ndarray 中的元素
out << a.at(i, j)<< std::endl;
}
else {
out << a.at(i, j) << " ";
}
}
}
}
//
//py::array_t<unsigned char, py::array::c_style> rgb_to_gray(py::array_t<unsigned char, py::array::c_style>& a) {
//
// py::array_t<unsigned char, py::array::c_style> dst = py::array_t<unsigned char, py::array::c_style>(a.shape()[0] * a.shape()[1]);
// //指针访问numpy矩阵
// unsigned char* p = (unsigned char*)dst.ptr();
//
// for (int i = 0; i < a.shape()[0]; i++)
// {
// for (int j = 0; j < a.shape()[1]; j++)
// {
// auto var = a.data(i, j);
// auto R = var[0];
// auto G = var[1];
// auto B = var[2];
//
// //RGB to gray
// auto gray = (R * 30 + G * 59 + B * 11 + 50) / 100;
//
// std::cout << static_cast<int>(R) << " " << static_cast<int>(G) << " " << static_cast<int>(B)<< std::endl;
//
// //p[i*a.shape()[1] + j] = static_cast<unsigned char>(gray);
//
// }
// }
//}
PYBIND11_MODULE(numpy_demo, m) {
m.doc() = "Simple numpy demo";
py::class_<Matrix>(m,"Matrix",py::buffer_protocol())
.def_buffer([](Matrix& mm)->py::buffer_info {
return py::buffer_info(
mm.data(), //Pointer to buffer, 数据指针
sizeof(float), //Size of one scalar, 每个元素大小(byte)
py::format_descriptor<float>::format(), //python struct-style foramt descriptor
2, //Number of dims, 维度
{mm.rows(), mm.cols()}, //strides (in bytes)
{sizeof(float) * mm.cols(),sizeof(float)}
);
});
m.def("save_2d_numpy_array", &save_2d_numpy_array);
//m.def("rgb_to_gray", &rgb_to_gray);
}
setup.py
from setuptools import setup
from setuptools import Extension
example_module = Extension(name='numpy_demo', # 模块名称
sources=['example.cpp'], # 源码
include_dirs=[r'/home/data/CM/10_device/pybind11/include']
)
setup(ext_modules=[example_module])
注意,这里r’/home/data/CM/10_device/pybind11/include’的路径为pybind11,头文件的位置
- git clone https://github.com/pybind/pybind11
- 修改上面include路径到自己的路径就ok
test.py
import numpy as np
import numpy_demo as numpy_demo
# help(numpy_demo)
mat1 = numpy_demo.save_2d_numpy_array(np.zeros(shape=[10,10], dtype=np.float32), './data.dat')
print(mat1)
编译c++拓展python setup.py build_ext --inplace
注意,编译的python和待会测试test.py时的python要保持一致
结果
4. project2
上面c++的拓展,只是使用了pybind11,那如果我们c++中使用了opencv,或者其他的库,那我们在python中还能使用c++的拓展吗?
新建一个project2文件,在文件夹下新建如下的文件:
.
├── lena.jpg
├── main.cpp
├── mat_warper.cpp
├── mat_warper.h
├── setup.py
└── test.py
main.cpp
#include<iostream>
#include<vector>
#include<opencv2/opencv.hpp>
#include<pybind11/pybind11.h>
#include<pybind11/numpy.h>
#include<pybind11/stl.h>
#include"mat_warper.h"
namespace py = pybind11;
py::array_t<unsigned char> test_rgb_to_gray(py::array_t<unsigned char>& input) {
cv::Mat img_rgb = numpy_uint8_3c_to_cv_mat(input);
cv::Mat dst;
cv::cvtColor(img_rgb, dst, cv::COLOR_RGB2GRAY);
return cv_mat_uint8_1c_to_numpy(dst);
}
py::array_t<unsigned char> test_gray_canny(py::array_t<unsigned char>& input) {
cv::Mat src = numpy_uint8_1c_to_cv_mat(input);
cv::Mat dst;
cv::Canny(src, dst, 30, 60);
return cv_mat_uint8_1c_to_numpy(dst);
}
/*
@return Python list
*/
py::list test_pyramid_image(py::array_t<unsigned char>& input) {
cv::Mat src = numpy_uint8_1c_to_cv_mat(input);
std::vector<cv::Mat> dst;
cv::buildPyramid(src, dst, 4);
py::list out;
for (int i = 0; i < dst.size(); i++)
{
out.append<py::array_t<unsigned char>>(cv_mat_uint8_1c_to_numpy(dst.at(i)));
}
return out;
}
PYBIND11_MODULE(cv_demo1, m) {
m.doc() = "Simple opencv demo";
m.def("test_rgb_to_gray1", &test_rgb_to_gray);
m.def("test_gray_canny1", &test_gray_canny);
m.def("test_pyramid_image1", &test_pyramid_image);
}
mat_warper.cpp
#include"mat_warper.h"
#include <pybind11/numpy.h>
cv::Mat numpy_uint8_1c_to_cv_mat(py::array_t<unsigned char>& input) {
if (input.ndim() != 2)
throw std::runtime_error("1-channel image must be 2 dims ");
py::buffer_info buf = input.request();
cv::Mat mat(buf.shape[0], buf.shape[1], CV_8UC1, (unsigned char*)buf.ptr);
return mat;
}
cv::Mat numpy_uint8_3c_to_cv_mat(py::array_t<unsigned char>& input) {
if (input.ndim() != 3)
throw std::runtime_error("3-channel image must be 3 dims ");
py::buffer_info buf = input.request();
cv::Mat mat(buf.shape[0], buf.shape[1], CV_8UC3, (unsigned char*)buf.ptr);
return mat;
}
/*
C++ Mat ->numpy
*/
py::array_t<unsigned char> cv_mat_uint8_1c_to_numpy(cv::Mat& input) {
py::array_t<unsigned char> dst = py::array_t<unsigned char>({ input.rows,input.cols }, input.data);
return dst;
}
py::array_t<unsigned char> cv_mat_uint8_3c_to_numpy(cv::Mat& input) {
py::array_t<unsigned char> dst = py::array_t<unsigned char>({ input.rows,input.cols,3}, input.data);
return dst;
}
//PYBIND11_MODULE(cv_mat_warper, m) {
//
// m.doc() = "OpenCV Mat -> Numpy.ndarray warper";
//
// m.def("numpy_uint8_1c_to_cv_mat", &numpy_uint8_1c_to_cv_mat);
// m.def("numpy_uint8_1c_to_cv_mat", &numpy_uint8_1c_to_cv_mat);
//
//
//}
mat_warper.h
#ifndef MAT_WARPER_H_
#include<opencv2/opencv.hpp>
#include<pybind11/pybind11.h>
#include<pybind11/numpy.h>
namespace py = pybind11;
cv::Mat numpy_uint8_1c_to_cv_mat(py::array_t<unsigned char>& input);
cv::Mat numpy_uint8_3c_to_cv_mat(py::array_t<unsigned char>& input);
py::array_t<unsigned char> cv_mat_uint8_1c_to_numpy(cv::Mat & input);
py::array_t<unsigned char> cv_mat_uint8_3c_to_numpy(cv::Mat & input);
#endif // !MAT_WARPER_H_
setup.py
from setuptools import Extension
from setuptools import setup
__version__ = '0.0.1'
# 扩展模块
ext_module = Extension(
# 模块名称
name='cv_demo1',
# 源码
sources=[r'mat_warper.cpp', r'main.cpp'],
# 包含头文件
include_dirs=[r'/usr/local/include',
r'/home/data/CM/10_device/pybind11/include' ],
# 库目录
library_dirs=[r'/usr/local/lib'],
# 链接库文件
libraries=[r'opencv_core', r'opencv_imgproc'],
language='c++'
)
setup(
name='cv_demo1',
version=__version__,
author_email='xxxx@qq.com',
description='A simaple demo',
ext_modules=[ext_module],
install_requires=['numpy']
)
注意
- include_dirs:指定pybind11,和opencv include的路径
- library_dirs:指定使用的动态库,静态库的路径
- libraries:指定使用的库名称,原文章是将opencv编译成了opencv_world库的
test.py
import cv_demo1 as cv_demo
import numpy as np
import cv2
import matplotlib.pyplot as plt
# help(cv_demo)
image = cv2.imread('./lena.jpg', cv2.IMREAD_GRAYSCALE)
# canny
img_canny = cv_demo.test_gray_canny1(image)
plt.figure('canny')
plt.imshow(img_canny, cmap=plt.gray())
# pyramid
imgs_pyramid = cv_demo.test_pyramid_image1(image)
plt.figure('pyramid')
for i in range(1, len(imgs_pyramid)):
plt.subplot(2, 2, i)
plt.imshow(imgs_pyramid[i])
# rgb to gray
plt.figure('rgb->gray')
img_gray = cv_demo.test_rgb_to_gray1(cv2.imread('./lena.jpg'))
plt.imshow(img_gray)
plt.show()
编译
cd project2
python setup.py build_ext --inplace
打印信息
python setup.py build_ext --inplace
running build_ext
building 'cv_demo1' extension
creating build
creating build/temp.linux-x86_64-3.7
gcc -pthread -B /home/data/miniconda3/compiler_compat -Wl,--sysroot=/ -Wsign-compare -DNDEBUG -g -fwrapv -O3 -Wall -Wstrict-prototypes -fPIC -I/usr/local/include -I/home/data/CM/10_device/pybind11/include -I/home/data/miniconda3/include/python3.7m -c mat_warper.cpp -o build/temp.linux-x86_64-3.7/mat_warper.o
cc1plus: warning: command line option ‘-Wstrict-prototypes’ is valid for C/ObjC but not for C++
gcc -pthread -B /home/data/miniconda3/compiler_compat -Wl,--sysroot=/ -Wsign-compare -DNDEBUG -g -fwrapv -O3 -Wall -Wstrict-prototypes -fPIC -I/usr/local/include -I/home/data/CM/10_device/pybind11/include -I/home/data/miniconda3/include/python3.7m -c main.cpp -o build/temp.linux-x86_64-3.7/main.o
cc1plus: warning: command line option ‘-Wstrict-prototypes’ is valid for C/ObjC but not for C++
main.cpp: In function ‘pybind11::list test_pyramid_image(pybind11::array_t<unsigned char>&)’:
main.cpp:38:23: warning: comparison between signed and unsigned integer expressions [-Wsign-compare]
for (int i = 0; i < dst.size(); i++)
~~^~~~~~~~~~~~
creating build/lib.linux-x86_64-3.7
g++ -pthread -shared -B /home/data/miniconda3/compiler_compat -L/home/data/miniconda3/lib -Wl,-rpath=/home/data/miniconda3/lib -Wl,--no-as-needed -Wl,--sysroot=/ build/temp.linux-x86_64-3.7/mat_warper.o build/temp.linux-x86_64-3.7/main.o -L/usr/local/lib -lopencv_core -lopencv_imgproc -o build/lib.linux-x86_64-3.7/cv_demo1.cpython-37m-x86_64-linux-gnu.so
copying build/lib.linux-x86_64-3.7/cv_demo1.cpython-37m-x86_64-linux-gnu.so ->
会多一个build文件夹和so动态库文件
测试python test.py
图片
上一篇: 基于 pybind11 用 c++ 编写的 Python 调用 CV 算法的实现 - Next (Linux+Cmake)
下一篇: Github 项目 | 用于逼真全景图像的强化学习人工智能平台 -- Matterport3DSimulator
推荐阅读
-
LeetCode] 动态编程 - 95.动态编程 - 95.不同的二叉搜索树 II(附完整 Python/C++ 代码) - 基本思想
-
华为 OD 机测试 - RSA 加密算法(Python/JS/C/C++ 2024 E 卷 100 分) - V.Python 算法源代码
-
为 Windows 10 安装 MinGW 和配置 C/C++ 编译环境的 VS 代码
-
华为 OD 机测试 - Excel 单元格值统计(Python/JS/C/C++ 2024 年 E 卷 200 分) - VIII、C 算法源代码
-
华为 OD 机测试 - 间隔交织问题 - 贪婪算法(Python/JS/C/C++ 2024 年 E 卷 200 分) - IX、C++ 算法源代码
-
Python 和 C++ 混淆矩阵 地理 医学物理学 视觉语言模型和算法模型评估工具
-
华为 OD 机测试 - 最长回声字符串 - 贪婪算法(Python/JS/C/C++ 2024 年 E 卷 100 分) - IV.测试案例
-
[LeetCode] 动态编程 - 5.最长回声子串(含完整 Python/C++ 代码) - 前言
-
c++开发编译后的curl(安卓版)
-
OpenCV-based Object Tracking in C++/Python (使用C++/Python进行基于OpenCV的目标追踪)