Pybind11 Shared Library Visibility Issues

We are trying to compile some c++ code into a shared library with pybind11. This shared library does not call python functions, but is instead called from a python script. Then we want to create an entrypoint C++ executable that uses this shared library to test the c++ code. It does not interface with pybind or the python side in any way.

When we do so, our shared library compiles successfully. However, in the executable target, we get a variety of linker errors specifying 'undefined reference to' our constructors in the shared library.

After reading the pybind documentation, we believe this is because the pybind11_add_module function forces cmake to compile the shared library with a 'hidden' visibility flag. We subsequently try to expose the relevant functions in our c++ shared library so that our executable can access it (using __attribute__((visibility("default")))). This resolves the undefined references to our constructors, but results in the errors provided below. Note that these errors occur when compiling our executable. The shared library still compiles succsesfully.

[build] ../pipeline_manager.cpython-38-x86_64-linux-gnu.so: undefined reference to `PyTuple_SetItem'
[build] ../pipeline_manager.cpython-38-x86_64-linux-gnu.so: undefined reference to `PyObject_Repr'
[build] ../pipeline_manager.cpython-38-x86_64-linux-gnu.so: undefined reference to `PyInstanceMethod_Type'
[build] ../pipeline_manager.cpython-38-x86_64-linux-gnu.so: undefined reference to `PyExc_ValueError'
[build] ../pipeline_manager.cpython-38-x86_64-linux-gnu.so: undefined reference to `PyLong_FromSsize_t'
[build] ../pipeline_manager.cpython-38-x86_64-linux-gnu.so: undefined reference to `PyDict_GetItemString'
[build] ../pipeline_manager.cpython-38-x86_64-linux-gnu.so: undefined reference to `_Py_TrueStruct'
[build] ../pipeline_manager.cpython-38-x86_64-linux-gnu.so: undefined reference to `PyExc_IndexError'
[build] ../pipeline_manager.cpython-38-x86_64-linux-gnu.so: undefined reference to `PyErr_NormalizeException'
[build] ../pipeline_manager.cpython-38-x86_64-linux-gnu.so: undefined reference to `PyInstanceMethod_New'
[build] ../pipeline_manager.cpython-38-x86_64-linux-gnu.so: undefined reference to `PyEval_AcquireThread'
[build] ../pipeline_manager.cpython-38-x86_64-linux-gnu.so: undefined reference to `PyObject_Str'
[build] ../pipeline_manager.cpython-38-x86_64-linux-gnu.so: undefined reference to `PyThreadState_DeleteCurrent'
[build] ../pipeline_manager.cpython-38-x86_64-linux-gnu.so: undefined reference to `PyGILState_GetThisThreadState'
[build] ../pipeline_manager.cpython-38-x86_64-linux-gnu.so: undefined reference to `PyObject_GetAttrString'
[build] ../pipeline_manager.cpython-38-x86_64-linux-gnu.so: undefined reference to `PyCapsule_Type'
[build] ../pipeline_manager.cpython-38-x86_64-linux-gnu.so: undefined reference to `PyModule_Type'
[build] ../pipeline_manager.cpython-38-x86_64-linux-gnu.so: undefined reference to `PyMem_Free'
[build] ../pipeline_manager.cpython-38-x86_64-linux-gnu.so: undefined reference to `PyErr_Restore'
[build] ../pipeline_manager.cpython-38-x86_64-linux-gnu.so: undefined reference to `PyType_IsSubtype'
[build] ../pipeline_manager.cpython-38-x86_64-linux-gnu.so: undefined reference to `PyModule_AddObject'
[build] ../pipeline_manager.cpython-38-x86_64-linux-gnu.so: undefined reference to `PyErr_WarnEx'
[build] ../pipeline_manager.cpython-38-x86_64-linux-gnu.so: undefined reference to `PyCapsule_SetPointer'
[build] ../pipeline_manager.cpython-38-x86_64-linux-gnu.so: undefined reference to `PyTuple_New'
[build] ../pipeline_manager.cpython-38-x86_64-linux-gnu.so: undefined reference to `PyObject_SetAttr'
[build] ../pipeline_manager.cpython-38-x86_64-linux-gnu.so: undefined reference to `PyObject_IsInstance'
[build] ../pipeline_manager.cpython-38-x86_64-linux-gnu.so: undefined reference to `PyEval_RestoreThread'
[build] ../pipeline_manager.cpython-38-x86_64-linux-gnu.so: undefined reference to `_Py_NoneStruct'
[build] ../pipeline_manager.cpython-38-x86_64-linux-gnu.so: undefined reference to `PyException_SetTraceback'
[build] ../pipeline_manager.cpython-38-x86_64-linux-gnu.so: undefined reference to `PyUnicode_FromFormat'
[build] ../pipeline_manager.cpython-38-x86_64-linux-gnu.so: undefined reference to `PyList_Append'
[build] ../pipeline_manager.cpython-38-x86_64-linux-gnu.so: undefined reference to `PyExc_MemoryError'
[build] ../pipeline_manager.cpython-38-x86_64-linux-gnu.so: undefined reference to `PyType_Type'
[build] ../pipeline_manager.cpython-38-x86_64-linux-gnu.so: undefined reference to `PyDict_Next'
[build] ../pipeline_manager.cpython-38-x86_64-linux-gnu.so: undefined reference to `PyList_Size'
[build] ../pipeline_manager.cpython-38-x86_64-linux-gnu.so: undefined reference to `PyTuple_Size'
[build] ../pipeline_manager.cpython-38-x86_64-linux-gnu.so: undefined reference to `PyBuffer_Release'
[build] ../pipeline_manager.cpython-38-x86_64-linux-gnu.so: undefined reference to `PyErr_Format'
[build] ../pipeline_manager.cpython-38-x86_64-linux-gnu.so: undefined reference to `PyObject_CallObject'
[build] ../pipeline_manager.cpython-38-x86_64-linux-gnu.so: undefined reference to `PyFloat_FromDouble'
[build] ../pipeline_manager.cpython-38-x86_64-linux-gnu.so: undefined reference to `PyUnicode_DecodeUTF8'
[build] ../pipeline_manager.cpython-38-x86_64-linux-gnu.so: undefined reference to `_Py_Dealloc'
[build] ../pipeline_manager.cpython-38-x86_64-linux-gnu.so: undefined reference to `PyCFunction_Type'
[build] ../pipeline_manager.cpython-38-x86_64-linux-gnu.so: undefined reference to `PyExc_OverflowError'
[build] ../pipeline_manager.cpython-38-x86_64-linux-gnu.so: undefined reference to `PyCFunction_NewEx'
[build] ../pipeline_manager.cpython-38-x86_64-linux-gnu.so: undefined reference to `PyList_New'
[build] ../pipeline_manager.cpython-38-x86_64-linux-gnu.so: undefined reference to `PyProperty_Type'
[build] ../pipeline_manager.cpython-38-x86_64-linux-gnu.so: undefined reference to `_PyObject_GetDictPtr'
[build] ../pipeline_manager.cpython-38-x86_64-linux-gnu.so: undefined reference to `PyUnicode_FromString'
[build] ../pipeline_manager.cpython-38-x86_64-linux-gnu.so: undefined reference to `Py_GetVersion'
[build] ../pipeline_manager.cpython-38-x86_64-linux-gnu.so: undefined reference to `PyCapsule_SetContext'
[build] ../pipeline_manager.cpython-38-x86_64-linux-gnu.so: undefined reference to `PyFrame_GetLineNumber'
[build] ../pipeline_manager.cpython-38-x86_64-linux-gnu.so: undefined reference to `PyThread_tss_get'
[build] ../pipeline_manager.cpython-38-x86_64-linux-gnu.so: undefined reference to `PyBytes_Size'
[build] ../pipeline_manager.cpython-38-x86_64-linux-gnu.so: undefined reference to `PySequence_Check'
[build] ../pipeline_manager.cpython-38-x86_64-linux-gnu.so: undefined reference to `PyList_GetItem'
... [additional reference errors removed for brevity]
[build] collect2: error: ld returned 1 exit status

We have tried a variety of commands regarding the visibility of the shared library compilation, but always end up with one or both of the errors described above. Find our CMAKEFILES below:

Main CMakeLists.txt that compiles the shared library (with irrelevant commands for gstreamer, cuda, etc... removed for brevity):

cmake_minimum_required(VERSION 3.5)
project(visual_processing_node)

# Default to C99
if(NOT CMAKE_C_STANDARD)
  set(CMAKE_C_STANDARD 99)
endif()

# Default to C++14
if(NOT CMAKE_CXX_STANDARD)
  set(CMAKE_CXX_STANDARD 14)
endif()

set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -pthread")

find_package(Threads REQUIRED)

if(CMAKE_COMPILER_IS_GNUCXX OR CMAKE_CXX_COMPILER_ID MATCHES "Clang")
  add_compile_options(-Wall -Wextra -Wpedantic)
endif()
    
# Fetch pybind11
include(FetchContent)
FetchContent_Declare(
    pybind11
    GIT_REPOSITORY https://github.com/pybind/pybind11.git
    GIT_TAG        v2.6.2
    GIT_SHALLOW    TRUE
)
FetchContent_MakeAvailable(pybind11)
pybind11_add_module(pipeline_manager SHARED src/deepstream_pipeline.cpp     src/deepstream_pipeline_helpers.cpp)

target_include_directories(
  pipeline_manager
  PUBLIC $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/include>
         $<INSTALL_INTERFACE:include>
          ${GSTREAMER_INCLUDE_DIRS} ${GSTREAMER_APP_INCLUDE_DIRS}
          ${GLIB_INCLUDE_DIRS} ${GLOG_INCLUDE_DIRS} "${DeepStream_DIR}/lib")

target_link_libraries(
  pipeline_manager
  PUBLIC
  ${GSTREAMER_LIBRARIES}
  ${GLIB_LIBRARIES}
  ${GLIB_GIO_LIBRARIES}
  ${GLIB_GOBJECT_LIBRARIES}
  ${GLOG_LIBRARIES}
  ${GSTREAMER_APP_LIBRARIES}
  "/opt/nvidia/deepstream/deepstream-6.0/lib/libnvdsgst_meta.so"
  "/opt/nvidia/deepstream/deepstream-6.0/lib/libnvds_meta.so"
  gstrtspserver-1.0
  Threads::Threads)

add_subdirectory (tests)

Second CMakeLists.txt file inside 'tests' directory that compiles the executable.

add_executable (inferenceTest inference_test.cpp)
target_link_libraries (inferenceTest pipeline_manager
                        pybind11::module)
target_include_directories(inferenceTest PUBLIC
                       "${PROJECT_BINARY_DIR}")