How to reduce dynamic library loading time?

user_10133579 0
Viewed 429

I have a question about reducing the dynamic linking time.

I've got a binary which links with 189 shared libraries. I profiled the execution with perf tool and the dynamic linker (ld.so) takes up 40% of the runtime of my program, approximately 90 milliseconds of time.

Is there any way to optimize dynamic library loading time, or am I forced to use static linking?

I run this program a lot of times.

dynamic-linking ld shared-libraries linux
2 Answers
  1. Avoid C++ constructors in libraries as they cause delays for running constructors themselves, for resolving symbols and loading code from hard drive
  2. Minimize number of exported symbols by compiling libraries with -fvisibility=hidden (and selectively using __attribute__((visibility("default"))) for symbols that you do want to export)
  3. Use --as-needed to minimize library dependencies
  4. Make sure that lazy binding is enabled (i.e. LD_BIND_NOW is not set and library is not compiled with -Wl,-z,now)
  5. Finally you can use the Prelink tool to precompute symbol offsets statically

Library load time can be measured by exporting LD_DEBUG=statistics

user_21705270

The loading time depends on several factors: the time needed to transfer the libraries from media storage to memory, the number of symbols you are linking with... Here are some suggestions:

  1. If you use dlopen(), you can set the lazy binding flag (RTLD_LAZY)
  2. Make sure to have a ld.so.cache up to date using ldconfig (and update ld.so.conf if necessary) to improve the library search time. Using strace may help to see how many attempts are done to find the library files. Check also LD_LIBRARY_PATH environment variable.
  3. Make sure that the LD_BIND_NOW environment variable is not set to make the dynamic linker (ld.so) run in lazy mode

Tips

  • As the some part of the libraries (especially the readonly sections like code) are shared between the running processes. You can arrange to have a background process dynamically linked with the libraries to save some dynamic linking time for any new created processes using the same libraries as most of them will be already loaded in memory.
  • You can launch your program under strace to check if the libraries are found directly at their right place or if the dynamic linker is doing several tries to find them. This usually means that LD_LIBRARY_PATH environment variable is wrongly set or should not be set at all. This may also mean that the linker's command line is badly set a build time.
    For example, here is a strace result of the startup of /bin/ls executable. Libraries like libm or libpam are found at their right place but the libc requires several tries in several directories to be finally found in /lib:
[root@sa415m ~]# strace /bin/ls
execve("/bin/ls", ["/bin/ls"], [/* 17 vars */]) = 0
brk(NULL)                               = 0x1429000
uname({sysname="Linux", nodename="sa415m", ...}) = 0
mmap2(NULL, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0xb6f16000
access("/etc/ld.so.preload", R_OK)      = -1 ENOENT (No such file or directory)
openat(AT_FDCWD, "/etc/ld.so.cache", O_RDONLY|O_CLOEXEC) = 3
fstat64(3, {st_mode=S_IFREG|0644, st_size=57138, ...}) = 0
mmap2(NULL, 57138, PROT_READ, MAP_PRIVATE, 3, 0) = 0xb6f08000
close(3)                                = 0
openat(AT_FDCWD, "/lib/libm.so.6", O_RDONLY|O_CLOEXEC) = 3
read(3, "\177ELF\1\1\1\0\0\0\0\0\0\0\0\0\3\0(\0\1\0\0\0\320l\0\0004\0\0\0"..., 512) = 512
fstat64(3, {st_mode=S_IFREG|0555, st_size=476584, ...}) = 0
mmap2(NULL, 540784, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0xb6e63000
mprotect(0xb6ed7000, 61440, PROT_NONE)  = 0
mmap2(0xb6ee6000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x73000) = 0xb6ee6000
close(3)                                = 0
openat(AT_FDCWD, "/lib/libpam.so.0", O_RDONLY|O_CLOEXEC) = 3
read(3, "\177ELF\1\1\1\0\0\0\0\0\0\0\0\0\3\0(\0\1\0\0\0\30\34\0\0004\0\0\0"..., 512) = 512
fstat64(3, {st_mode=S_IFREG|0644, st_size=42672, ...}) = 0
mmap2(NULL, 106852, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0xb6e48000
mprotect(0xb6e52000, 61440, PROT_NONE)  = 0
mmap2(0xb6e61000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x9000) = 0xb6e61000
close(3)                                = 0
openat(AT_FDCWD, "/lib/libpam_misc.so.0", O_RDONLY|O_CLOEXEC) = 3
read(3, "\177ELF\1\1\1\0\0\0\0\0\0\0\0\0\3\0(\0\1\0\0\0\\\n\0\0004\0\0\0"..., 512) = 512
fstat64(3, {st_mode=S_IFREG|0644, st_size=9684, ...}) = 0
mmap2(NULL, 73936, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0xb6e35000
mprotect(0xb6e37000, 61440, PROT_NONE)  = 0
mmap2(0xb6e46000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x1000) = 0xb6e46000
close(3)                                = 0
openat(AT_FDCWD, "/lib/libresolv.so.2", O_RDONLY|O_CLOEXEC) = 3
read(3, "\177ELF\1\1\1\0\0\0\0\0\0\0\0\0\3\0(\0\1\0\0\0\250#\0\0004\0\0\0"..., 512) = 512
fstat64(3, {st_mode=S_IFREG|0555, st_size=67284, ...}) = 0
mmap2(NULL, 141272, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0xb6e12000
mprotect(0xb6e22000, 61440, PROT_NONE)  = 0
mmap2(0xb6e31000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0xf000) = 0xb6e31000
mmap2(0xb6e33000, 6104, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0xb6e33000
close(3)                                = 0
openat(AT_FDCWD, "/lib/libselinux.so.1", O_RDONLY|O_CLOEXEC) = 3
read(3, "\177ELF\1\1\1\0\0\0\0\0\0\0\0\0\3\0(\0\1\0\0\0\200Q\0\0004\0\0\0"..., 512) = 512
fstat64(3, {st_mode=S_IFREG|0644, st_size=116992, ...}) = 0
mmap2(NULL, 187996, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0xb6de4000
mprotect(0xb6e00000, 61440, PROT_NONE)  = 0
mmap2(0xb6e0f000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x1b000) = 0xb6e0f000
mmap2(0xb6e11000, 3676, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0xb6e11000
close(3)                                = 0
openat(AT_FDCWD, "/lib/tls/v7l/neon/vfp/libc.so.6", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory)
stat64("/lib/tls/v7l/neon/vfp", 0xbe941508) = -1 ENOENT (No such file or directory)
openat(AT_FDCWD, "/lib/tls/v7l/neon/libc.so.6", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory)
stat64("/lib/tls/v7l/neon", 0xbe941508) = -1 ENOENT (No such file or directory)
openat(AT_FDCWD, "/lib/tls/v7l/vfp/libc.so.6", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory)
stat64("/lib/tls/v7l/vfp", 0xbe941508)  = -1 ENOENT (No such file or directory)
openat(AT_FDCWD, "/lib/tls/v7l/libc.so.6", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory)
stat64("/lib/tls/v7l", 0xbe941508)      = -1 ENOENT (No such file or directory)
openat(AT_FDCWD, "/lib/tls/neon/vfp/libc.so.6", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory)
stat64("/lib/tls/neon/vfp", 0xbe941508) = -1 ENOENT (No such file or directory)
openat(AT_FDCWD, "/lib/tls/neon/libc.so.6", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory)
stat64("/lib/tls/neon", 0xbe941508)     = -1 ENOENT (No such file or directory)
openat(AT_FDCWD, "/lib/tls/vfp/libc.so.6", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory)
stat64("/lib/tls/vfp", 0xbe941508)      = -1 ENOENT (No such file or directory)
openat(AT_FDCWD, "/lib/tls/libc.so.6", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory)
stat64("/lib/tls", 0xbe941508)          = -1 ENOENT (No such file or directory)
openat(AT_FDCWD, "/lib/v7l/neon/vfp/libc.so.6", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory)
stat64("/lib/v7l/neon/vfp", 0xbe941508) = -1 ENOENT (No such file or directory)
openat(AT_FDCWD, "/lib/v7l/neon/libc.so.6", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory)
stat64("/lib/v7l/neon", 0xbe941508)     = -1 ENOENT (No such file or directory)
openat(AT_FDCWD, "/lib/v7l/vfp/libc.so.6", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory)
stat64("/lib/v7l/vfp", 0xbe941508)      = -1 ENOENT (No such file or directory)
openat(AT_FDCWD, "/lib/v7l/libc.so.6", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory)
stat64("/lib/v7l", 0xbe941508)          = -1 ENOENT (No such file or directory)
openat(AT_FDCWD, "/lib/neon/vfp/libc.so.6", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory)
stat64("/lib/neon/vfp", 0xbe941508)     = -1 ENOENT (No such file or directory)
openat(AT_FDCWD, "/lib/neon/libc.so.6", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory)
stat64("/lib/neon", 0xbe941508)         = -1 ENOENT (No such file or directory)
openat(AT_FDCWD, "/lib/vfp/libc.so.6", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory)
stat64("/lib/vfp", 0xbe941508)          = -1 ENOENT (No such file or directory)
openat(AT_FDCWD, "/lib/libc.so.6", O_RDONLY|O_CLOEXEC) = 3
read(3, "\177ELF\1\1\1\3\0\0\0\0\0\0\0\0\3\0(\0\1\0\0\0Lr\1\0004\0\0\0"..., 512) = 512
[...]
user_143937390
Related