Using cppimport inside custom Python Package

I'm trying to use cppimport inside one of my files in a custom Python module. When I download the module from https://test.pypi.org/, cppimport does not find my C++ file which was packaged with all the other files. My setup.py file is below. Is there something I am missing inside this?

from setuptools.extension import Extension
import numpy as np
from Cython.Build import cythonize

# with open("README.md", "r", encoding="utf-8") as f:
#     long_description = f.read()

long_description = ''

install_requires = [
    'numpy',
    'pandas',
    'cppimport',
    'gensim',
    'stop_words',
    'numba',
    'cython',
    'tqdm'
    ]

# name is what you pip install
# name doesn't have to be the name of the python code that people will import, usually they are the same
# sometimes they are different

# py_modules: list of the actual python code modules, this is the code that we want to distribute.
# this is what people import, not what they pip install

setup(
    name='',
    version='0.0.9',
    author="All of Us",
    description='',
    url="https://github.com/",
    long_description=long_description,
    license = "LICENSE.txt",
    ext_modules = cythonize("src/cython_gibbs.pyx"),
    include_dirs=[np.get_include()],
    py_modules=["lda", "numba_gibbs"],  # packages=["lda", "numba_gibbs"]
    include_package_data=True,
    install_requires = install_requires,
    package_dir={'': 'src'}
)