pybind11 type as function parameters in library yielding compile error in application linking against that library

Viewed 75

I'm not sure where I'm going wrong, but in any case, here's what I'm seeing:

Lets say I have a library that's defined something like this:

# minimal-lib/CMakeLists.txt

cmake_minimum_required(VERSION 3.14)

project(minimallib
    LANGUAGES CXX
    VERSION 1.0.0
    DESCRIPTION "minimal library")
include(GNUInstallDirs)

set(CMAKE_INCLUDE_CURRENT_DIR ON)
set(CMAKE_CXX_STANDARD 20)
set(CMAKE_CXX_STANDARD_REQUIRED ON)
set(CMAKE_CPP_COMPILER clang++-13)

# get rid of the pybind11 warning spam
include(CheckCXXCompilerFlag)
check_cxx_compiler_flag(-Wno-attributes HAVE_ATTRIBUTES)
if (HAVE_ATTRIBUTES)
    add_compile_options(-Wno-attributes)
endif()


IF(CMAKE_INSTALL_PREFIX_INITIALIZED_TO_DEFAULT)
SET(CMAKE_INSTALL_PREFIX /opt/experiment CACHE PATH "comment" FORCE)
ENDIF(CMAKE_INSTALL_PREFIX_INITIALIZED_TO_DEFAULT)

set(
    PSOURCES
        src/fun.cpp
)
set(
    PHEADERS
        include/fun.hpp
)

find_package(pybind11 REQUIRED)

add_library(${PROJECT_NAME} SHARED
    ${PSOURCES}
    ${PHEADERS}
)

set_target_properties(${PROJECT_NAME}
    PROPERTIES
        VERSION 1.0.0
        SOVERSION 1.0.0
        LINKER_LANGUAGE CXX
        MAP_IMPORTED_CONFIG_COVERAGE "DEBUG"
)
target_include_directories(${PROJECT_NAME}
    PUBLIC
        $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/include>
        $<INSTALL_INTERFACE:include>
        PRIVATE src)
target_link_libraries(${PROJECT_NAME} PUBLIC pybind11::embed)

install(
    TARGETS ${PROJECT_NAME}
    EXPORT ${PROJECT_NAME}Config
    LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR}
    ARCHIVE DESTINATION ${CMAKE_INSTALL_BINDIR}
)
install(
    DIRECTORY include/
    DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/${PROJECT_NAME}
)
export(
    TARGETS ${PROJECT_NAME}
    FILE ${PROJECT_NAME}Config.cmake
    NAMESPACE ${PROJECT_NAME}::
    FILE "${CMAKE_CURRENT_BINARY_DIR}/${PROJECT_NAME}Config.cmake"
)
install(
    EXPORT ${PROJECT_NAME}Config
    DESTINATION "${CMAKE_INSTALL_DATADIR}/${PROJECT_NAME}/cmake"
)
//minimal-lib/include/fun.hpp

#ifndef FUN_HPP
#define FUN_HPP

#include <pybind11/pybind11.h>
#include <optional>
#include <string>

namespace minimallib
{
void write(
        const std::optional<std::string> &file, 
        const pybind11::args &args = pybind11::none(), 
        const pybind11::kwargs &kwargs = pybind11::none());
}//namespace minimallib
#endif //FUN_HPP
//minimal-lib/src/fun.cpp

#include <iostream>
#include "fun.hpp"

void minimallib::write(
        const std::optional<std::string> &file, 
        const pybind11::args &args, 
        const pybind11::kwargs &kwargs)
{
    std::cout << "I should do something more useful." << std::endl;
}

and an application project that uses that library that looks something like this:

# rms/CMakeLists.txt
cmake_minimum_required(VERSION 3.16.0)

project(minimalapp
    LANGUAGES CXX
    VERSION 1.0.0)

include(GNUInstallDirs)
set(CMAKE_CXX_STANDARD 20)
set(CMAKE_CXX_STANDARD_REQUIRED ON)
set(CMAKE_INCLUDE_CURRENT_DIR ON)
IF(CMAKE_INSTALL_PREFIX_INITIALIZED_TO_DEFAULT)
  SET(CMAKE_INSTALL_PREFIX /opt/experiment CACHE PATH "comment" FORCE)
ENDIF(CMAKE_INSTALL_PREFIX_INITIALIZED_TO_DEFAULT)

list(APPEND CMAKE_PREFIX_PATH "/opt/experiment/")

find_package(pybind11 REQUIRED)
find_package(minimallib REQUIRED)

set(
    PSOURCES
        src/main.cpp
)

set(
    MAIN_HEADERS

)

add_executable(${PROJECT_NAME}
    ${PHEADERS}
    ${PSOURCES}
)

set_target_properties(${PROJECT_NAME}
    PROPERTIES
        VERSION 1.0.0
        MAP_IMPORTED_CONFIG_COVERAGE "DEBUG")

set_target_properties(${PROJECT_NAME} PROPERTIES LINKER_LANGUAGE CXX)

target_include_directories(${PROJECT_NAME}
    PUBLIC
        $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/include>
        $<INSTALL_INTERFACE:include>
    PRIVATE
        ${CMAKE_CURRENT_SOURCE_DIR}/src)

target_link_libraries(${PROJECT_NAME} PRIVATE pybind11::embed)
target_link_libraries(${PROJECT_NAME} PRIVATE minimallib)


install(TARGETS ${PROJECT_NAME}
    EXPORT ${PROJECT_NAME}Targets
    RUNTIME DESTINATION bin)
#include <minimallib/fun.hpp>

int main()
{
    minimallib::write("foobar.txt");
    return 0;
}

The library compiles and links without complaints. In any case, linking the application using that library fails with an error like this:

FAILED: minimalapp-1.0.0 
: && /usr/bin/c++   CMakeFiles/minimalapp.dir/src/main.cpp.o -o minimalapp-1.0.0  -Wl,-rpath,/opt/experiment/lib:  /opt/experiment/lib/libminimallib.so.1.0.0  /usr/lib/x86_64-linux-gnu/libpython3.9.so && :
/usr/bin/ld: CMakeFiles/minimalapp.dir/src/main.cpp.o: in function `main':
main.cpp:(.text+0x1bc): undefined reference to `minimallib::write(std::optional<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > const&, pybind11::args const&, pybind11::kwargs const&)'
collect2: error: ld returned 1 exit status
ninja: build stopped: subcommand failed.

If I remove the pybind11::kwargs and pybind11::args from the signature in the function in the library, the application that uses it compiles without issue.

I'm somewhat ignorant to looking at the symbols included in the *.so file, but I took a look anyway and here's what I observed. Case 1) pybind11::args and pybind11::kwargs included in signature: can see the relevant symbol with objdump -t but not with objdump -T.

0000000000006259 l     F .text  0000000000000046              _ZN10minimallib5writeERKSt8optionalINSt7__cxx1112basic_stringIcSt11char_traitsIcESaIcEEEERKN8pybind114argsERKNSA_6kwargsE

Case 2) not including those: can see the relevant symbol with both objdump -t and objdump -T. I note that it's also marked (g)lobal instead of (l)ocal, and marked (D)ynamic instead of ( )normal

0000000000006259 g    DF .text  000000000000003e  Base        _ZN10minimallib5writeERKSt8optionalINSt7__cxx1112basic_stringIcSt11char_traitsIcESaIcEEEE

Does this mean that the relevant symbol is invisible to the application trying to link to the library, or am I chasing my tail?

Does anyone have any insight or ideas that might help me fix this?

1 Answers

I've found a solution, though I still don't know why the behavior changed the way it did with the addition of a pair of parameters to a function ... I can only imagine that the library defining the types sets the visibility to hidden or something...

For posterity, for a library that doesn't need to exist outside GCC/Clang, can append this after the namespace name __attribute__ ((visibility("default"))) or can prepend #pragma GCC visibility push(default) and postpend #pragma GCC visibility pop this for larger blocks where you need everything visible. (I effectively boxed in all of my include files with this ... probably crude, but was effective)

Related