Using GitHub Action to Build Python Wheel Package for Dynamsoft C++ Barcode SDK

Xiao Ling - May 27 '22 - - Dev Community

Implementing a Python extension in C or C++ is easy, but building the native module into a Python wheel package for different operating systems and different Python versions is a nightmare. A CPython wheel package is Python-version-dependent, therefore, to make a CPython module work universally for Python 3.x, you have to build wheels for Python 3.6, Python 3.7, Python 3.8, Python 3.9, and Python 3.10 per operating system. The maximum amount of packages equals to Python versions (3.6, 3.7, 3.8, 3.9, 3.10) * operating systems (Windows, Linux, macOS) * CPU architectures (x64, aarch64). To simplify the process, we can utilize GitHub Actions. In this article, we take Dynamsoft C/C++ Barcode SDK as an example. You will see how to build a CPython extension that links to external C/C++ libraries (*.dll, *.so, *.dylib), and how to automate the process of building and publishing the Python wheel package with GitHub Actions.

Requirements

Dynamsoft C/C++ Barcode SDK v9.0

Building CPython Extension Project with Scikit-build

If you have read Python development guide, you may know that distutils.core.Extension is the most widely used Python extension builder. However, distutils cannot sequentially build the extension and package the generated library with the package folder when running the pip wheel command for creating a wheel package.

Scikit-build is an alternative to distutils. It extends distutils functions with CMake build. To get started with scikit-build, we can visit scikit-build-sample-projects.

For our Python barcode and QR code extension project, the setup.py file is as follows:

from skbuild import setup
import io

packages = ['barcodeQrSDK']

setup (name = 'barcode-qr-code-sdk',
            version = '9.0.3',
            description = 'Barcode and QR code scanning SDK for Python',
            packages=packages,
            include_package_data=False,
          )
Enter fullscreen mode Exit fullscreen mode

As you can see, the setup.py file is pretty simple comparing to the setup.py file used with disutils. It only contains the package folder. To trigger extension build, we create a CMakeLists.txt along with the setup.py file.

cmake_minimum_required(VERSION 3.4...3.22)

project(barcodeQrSDK)

find_package(PythonExtensions REQUIRED)

if(CMAKE_HOST_UNIX)
    if(CMAKE_HOST_APPLE)
        SET(CMAKE_CXX_FLAGS "-std=c++11 -O3 -Wl,-rpath,@loader_path")
        SET(CMAKE_INSTALL_RPATH "@loader_path")
    else()
        SET(CMAKE_CXX_FLAGS "-std=c++11 -O3 -Wl,-rpath=$ORIGIN")
        SET(CMAKE_INSTALL_RPATH "$ORIGIN")
    endif()
    SET(CMAKE_INSTALL_RPATH_USE_LINK_PATH TRUE)
endif()

MESSAGE( STATUS "CPU architecture ${CMAKE_SYSTEM_PROCESSOR}" )
if(CMAKE_HOST_WIN32)
    link_directories("${PROJECT_SOURCE_DIR}/lib/win/") 
elseif(CMAKE_SYSTEM_NAME STREQUAL "Linux")
    if (CMAKE_SYSTEM_PROCESSOR STREQUAL x86_64)
        MESSAGE( STATUS "Link directory: ${PROJECT_SOURCE_DIR}/lib/linux/" )
        link_directories("${PROJECT_SOURCE_DIR}/lib/linux/")
    elseif(CMAKE_SYSTEM_PROCESSOR STREQUAL armv7l)
        MESSAGE( STATUS "Link directory: ${PROJECT_SOURCE_DIR}/lib/arm32/" )
        link_directories("${PROJECT_SOURCE_DIR}/lib/arm32/") 
    elseif(CMAKE_SYSTEM_PROCESSOR STREQUAL aarch64) 
        MESSAGE( STATUS "Link directory: ${PROJECT_SOURCE_DIR}/lib/aarch64/" )
        link_directories("${PROJECT_SOURCE_DIR}/lib/aarch64/") 
    endif()
elseif(CMAKE_HOST_APPLE)
    MESSAGE( STATUS "Link directory: ${PROJECT_SOURCE_DIR}/lib/macos/" )
    link_directories("${PROJECT_SOURCE_DIR}/lib/macos/") 
endif()
include_directories("${PROJECT_BINARY_DIR}" "${PROJECT_SOURCE_DIR}/include/")

add_library(${PROJECT_NAME} MODULE src/barcodeQrSDK.cpp)
if(CMAKE_HOST_WIN32)
    if(CMAKE_CXX_COMPILER_ID STREQUAL "GNU")
        target_link_libraries (${PROJECT_NAME} "DynamsoftBarcodeReaderx64")
    else()
        target_link_libraries (${PROJECT_NAME} "DBRx64")
    endif()
else()
    target_link_libraries (${PROJECT_NAME} "DynamsoftBarcodeReader" pthread)
endif()

if(CMAKE_HOST_WIN32)
    add_custom_command(TARGET ${PROJECT_NAME} POST_BUILD 
    COMMAND ${CMAKE_COMMAND} -E copy_directory
    "${PROJECT_SOURCE_DIR}/lib/win/"      
    $<TARGET_FILE_DIR:${PROJECT_NAME}>)
endif()

python_extension_module(barcodeQrSDK)
install(TARGETS barcodeQrSDK LIBRARY DESTINATION barcodeQrSDK)

if(CMAKE_HOST_WIN32)
    install (DIRECTORY  "${PROJECT_SOURCE_DIR}/lib/win/" DESTINATION barcodeQrSDK)
elseif(CMAKE_SYSTEM_NAME STREQUAL "Linux")
    if (CMAKE_SYSTEM_PROCESSOR STREQUAL x86_64)
        install (DIRECTORY  "${PROJECT_SOURCE_DIR}/lib/linux/" DESTINATION barcodeQrSDK)
    elseif(CMAKE_SYSTEM_PROCESSOR STREQUAL armv7l OR ARM32_BUILD)
        install (DIRECTORY  "${PROJECT_SOURCE_DIR}/lib/arm32/" DESTINATION barcodeQrSDK)
    elseif(CMAKE_SYSTEM_PROCESSOR STREQUAL aarch64) 
        install (DIRECTORY  "${PROJECT_SOURCE_DIR}/lib/aarch64/" DESTINATION barcodeQrSDK)
    endif()
elseif(CMAKE_HOST_APPLE)
    install (DIRECTORY  "${PROJECT_SOURCE_DIR}/lib/macos/" DESTINATION barcodeQrSDK)
endif()
Enter fullscreen mode Exit fullscreen mode

What we do in the CMakeLists.txt file:

  1. Set build arguments. The rpath is critical for finding dependent shared libraries on Linux and macOS.
  2. Set the directories of header files and libraries.
  3. Build the extension module.
  4. Link external dynamic libraries.
  5. Copy the Python module and dependent libraries to the barcodeQrSDK package folder.

A __init__.py file is also required in the barcodeQrSDK package folder.

from .barcodeQrSDK import * 
__version__ = version
Enter fullscreen mode Exit fullscreen mode

Run python setup.py develop command to test the extension module. If there is no error, we can create the wheel package:

pip wheel .
Enter fullscreen mode Exit fullscreen mode

Once the wheel package is built successfully, its folder structure is like this:

Python wheel folder

Python wheel folder

Creating Multiple Wheel Packages with GitHub Actions

As we have mentioned above, it drives us crazy to create multiple wheel packages for each version of Python and different platforms. Fortunately, GitHub Actions can relieve us a lot of work.

Here are the steps to build and publish multiple wheel packages:

  1. Go to the repository homepage and click Actions to create a new workflow.

  2. Click set up a workflow yourself to create a custom workflow. We can refer to the examples provided by cibuildwheel.

    name: Build and upload to PyPI

on: [push, pull_request]

jobs:
build_wheels:
    name: Build wheels on ${{ matrix.os }}
    runs-on: ${{ matrix.os }}
    strategy:
    matrix:
        os: [windows-2019, macos-10.15]

    steps:
    - uses: actions/checkout@v2

    - name: Build wheels
        uses: pypa/cibuildwheel@v2.5.0
        env:
        CIBW_ARCHS_WINDOWS: AMD64
        CIBW_ARCHS_MACOS: x86_64
        CIBW_ARCHS_LINUX: "x86_64 aarch64"
        CIBW_SKIP: "pp* *-win32 *-manylinux_i686"

    - uses: actions/upload-artifact@v2
        with:
        path: ./wheelhouse/*.whl

build_sdist:
    name: Build source distribution
    runs-on: ubuntu-latest
    steps:
    - uses: actions/checkout@v2

    - name: Build sdist
        run: pipx run build --sdist

    - uses: actions/upload-artifact@v2
        with:
        path: dist/*.tar.gz

upload_pypi:
    needs: [build_wheels, build_sdist]
    runs-on: ubuntu-latest
    # upload to PyPI on every tag starting with 'v'
    if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v')
    # alternatively, to publish when a GitHub Release is created, use the following rule:
    # if: github.event_name == 'release' && github.event.action == 'published'
    steps:
    - uses: actions/download-artifact@v2
        with:
        name: artifact
        path: dist

    - uses: pypa/gh-action-pypi-publish@v1.4.2
        with:
        user: __token__
        password: ${{ secrets.pypi_password }}
        skip_existing: true

    Since our extension is 64-bit only, we can skip 32-bit builds by setting CIBW_SKIP: "pp* *-win32 *-manylinux_i686". We can also specify the OS architectures:

    CIBW_ARCHS_WINDOWS: AMD64
CIBW_ARCHS_MACOS: x86_64
CIBW_ARCHS_LINUX: "x86_64 aarch64"

  3. Go to Settings > Secrets > Actions to create a repository secret for publishing the wheel packages to pypi.org.

    GitHub repository secret

  4. After finishing the workflows, we can download the artifact that contains the generate wheel packages.

    GitHub action for Python wheel

    Besides, the wheel packages are available for download on pypi.org.

    Python barcode and QR code SDK

Source Code

https://github.com/yushulx/python-barcode-qrcode-sdk

Image
S202DS4
Image
Study System Cell Models as well as the Successful Energy Conductivities of It Aerogel.
Image
Throughout Situ Functionalization regarding Cellulose with Zinc Pyrithione regarding Anti-microbial Applications.
Image
STP30NF10
Image
7 Simple Changes That'll Make A Big Difference In Your 2 Seater Leather Sofa Chesterfield
Image
Any time Familial The loss of hearing Means Hereditary Heterogeneity: One Situation Statement.
Image
PME264NE6100M
Image
Dive into Jackson for JSON in Java: Understanding JsonNode, ArrayNode, and ObjectMapper

java, json, softwareengineering, developer
Image
METASTATIC CHORIOCARCINOMA Introducing AS RENAL COLIC And also Epidermis Patch -- Scenario REPORT.
Image
How to Query CSV Files with SQLite

database, sqlite, csv, sql
Image
Zendesk Data Integration in ETL Using SSIS: Source, Lookup & Destination Components

zendesk, ssis, sqlserver
Image
SC26C92C1A
Image
Exercise within idiopathic inflamed myopathies: a couple of involvement proposals depending on books evaluation.
Image
The actual Spanish Musician Red washington signifiant Falla and His Sight: The Musical Mind.
Image
Top 6 dApps on Polygon Blockchain in 2024 Features, Usecases and Comparison
Image
Chicago Chromosome Good B-lymphoblastic Leukemia in a Infant.
Image
Cat Flap Fitters
Image
NM27C256Q-200
Image
Discover the Best in Pakistan Online Shopping Clothes for Every Style
. . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . .
Terabox Video Player