Asd service is running but appears hung

asd service is running but appears hung

Problem Description

The Aerospike daemon is running when viewed via ps however, aql commands fail with errors in the following form:

bash-4.2# aql
Seed:         127.0.0.1
User:         None
Config File:  /etc/aerospike/astools.conf /root/.aerospike/astools.conf
2020-07-03 10:23:25 WARN Failed to connect to seed 127.0.0.1 3000. AEROSPIKE_ERR_CONNECTION Socket write error: 111, 127.0.0.1:3000
Error -10: Failed to connect

Checking with netstat shows that port 3000 is not open for the node. No aerospike.log is being written, either directly or via journald

When running asd with strace only the following lines are printed:

-bash-4.2# strace asd --config-file /etc/aerospike/aerospike.conf --fgdaemon
execve("/usr/bin/asd", ["asd", "--config-file", "/etc/aerospike/aerospike.conf", "--fgdaemon"], [/* 31 vars */]) = 0
brk(NULL)                               = 0x29da000
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f5e0c6e9000
access("/etc/ld.so.preload", R_OK)      = 0
open("/etc/ld.so.preload", O_RDONLY|O_CLOEXEC) = 3
fstat(3, {st_mode=S_IFREG|0644, st_size=42, ...}) = 0
mmap(NULL, 42, PROT_READ|PROT_WRITE, MAP_PRIVATE, 3, 0) = 0x7f5e0c6e8000
close(3)                                = 0
open("/etc/ld.so.cache", O_RDONLY|O_CLOEXEC) = 3
fstat(3, {st_mode=S_IFREG|0644, st_size=130172, ...}) = 0
mmap(NULL, 130172, PROT_READ, MAP_PRIVATE, 3, 0) = 0x7f5e0c6c8000
close(3)                                = 0
open("/lib64/tls/x86_64/libdbus-compact.so.1", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory)
stat("/lib64/tls/x86_64", 0x7ffef3ec2fc0) = -1 ENOENT (No such file or directory)
open("/lib64/tls/libdbus-compact.so.1", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory)
stat("/lib64/tls", {st_mode=S_IFDIR|0555, st_size=6, ...}) = 0
open("/lib64/x86_64/libdbus-compact.so.1", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory)
stat("/lib64/x86_64", 0x7ffef3ec2fc0)   = -1 ENOENT (No such file or directory)
open("/lib64/libdbus-compact.so.1", O_RDONLY|O_CLOEXEC) = 3
read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\360\22\0\0\0\0\0\0"..., 832) = 832
fstat(3, {st_mode=S_IFREG|S_ISUID|0755, st_size=18080, ...}) = 0
mmap(NULL, 2113344, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7f5e0c2c5000
mprotect(0x7f5e0c2c9000, 2093056, PROT_NONE) = 0
mmap(0x7f5e0c4c8000, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x3000) = 0x7f5e0c4c8000
close(3)                                = 0
open("/lib64/tls/libdbus-compact.so.2", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory)
open("/lib64/libdbus-compact.so.2", O_RDONLY|O_CLOEXEC) = 3
read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\300\n\0\0\0\0\0\0"..., 832) = 832
fstat(3, {st_mode=S_IFREG|S_ISUID|0755, st_size=61870, ...}) = 0
mmap(NULL, 2103200, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7f5e0c0c3000
mprotect(0x7f5e0c0c5000, 2093056, PROT_NONE) = 0
mmap(0x7f5e0c2c4000, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x1000) = 0x7f5e0c2c4000
close(3)                                = 0
munmap(0x7f5e0c6e8000, 42)              = 0
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f5e0c6e8000
open("/lib64/libcrypto.so.10", O_RDONLY|O_CLOEXEC) = 3
read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\0\320\6\0\0\0\0\0"..., 832) = 832
fstat(3, {st_mode=S_IFREG|0755, st_size=2513000, ...}) = 0
mmap(NULL, 4588328, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7f5e0bc62000
mprotect(0x7f5e0be96000, 2097152, PROT_NONE) = 0
mmap(0x7f5e0c096000, 167936, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x234000) = 0x7f5e0c096000
mmap(0x7f5e0c0bf000, 13096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0x7f5e0c0bf000
close(3)                                = 0
open("/lib64/libpthread.so.0", O_RDONLY|O_CLOEXEC) = 3
read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\0m\0\0\0\0\0\0"..., 832) = 832
fstat(3, {st_mode=S_IFREG|0755, st_size=144792, ...}) = 0
mmap(NULL, 2208904, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7f5e0ba46000
mprotect(0x7f5e0ba5d000, 2093056, PROT_NONE) = 0
mmap(0x7f5e0bc5c000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x16000) = 0x7f5e0bc5c000
mmap(0x7f5e0bc5e000, 13448, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0x7f5e0bc5e000
close(3)                                = 0
open("/lib64/librt.so.1", O_RDONLY|O_CLOEXEC) = 3
read(3, "\177ELF\2\1\1\3\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0P\"\0\0\0\0\0\0"..., 832) = 832
fstat(3, {st_mode=S_IFREG|0755, st_size=44448, ...}) = 0
mmap(NULL, 2128952, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7f5e0b83e000
mprotect(0x7f5e0b845000, 2093056, PROT_NONE) = 0
mmap(0x7f5e0ba44000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x6000) = 0x7f5e0ba44000
close(3)                                = 0
open("/lib64/libdl.so.2", O_RDONLY|O_CLOEXEC) = 3
read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0`\16\0\0\0\0\0\0"..., 832) = 832
fstat(3, {st_mode=S_IFREG|0755, st_size=19776, ...}) = 0
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f5e0c6c7000
mmap(NULL, 2109744, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7f5e0b63a000
mprotect(0x7f5e0b63c000, 2097152, PROT_NONE) = 0
mmap(0x7f5e0b83c000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x2000) = 0x7f5e0b83c000
close(3)                                = 0
open("/lib64/libz.so.1", O_RDONLY|O_CLOEXEC) = 3
read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\20!\0\0\0\0\0\0"..., 832) = 832
fstat(3, {st_mode=S_IFREG|0755, st_size=90248, ...}) = 0
mmap(NULL, 2183272, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7f5e0b424000
mprotect(0x7f5e0b439000, 2093056, PROT_NONE) = 0
mmap(0x7f5e0b638000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x14000) = 0x7f5e0b638000
close(3)                                = 0
open("/lib64/libm.so.6", O_RDONLY|O_CLOEXEC) = 3
read(3, "\177ELF\2\1\1\3\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0pS\0\0\0\0\0\0"..., 832) = 832
fstat(3, {st_mode=S_IFREG|0755, st_size=1139680, ...}) = 0
mmap(NULL, 3150136, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7f5e0b122000
mprotect(0x7f5e0b223000, 2093056, PROT_NONE) = 0
mmap(0x7f5e0b422000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x100000) = 0x7f5e0b422000
close(3)                                = 0
open("/lib64/libstdc++.so.6", O_RDONLY|O_CLOEXEC) = 3
read(3, "\177ELF\2\1\1\3\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\20\265\5\0\0\0\0\0"..., 832) = 832
fstat(3, {st_mode=S_IFREG|0755, st_size=995840, ...}) = 0
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f5e0c6c6000
mmap(NULL, 3175456, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7f5e0ae1a000
mprotect(0x7f5e0af03000, 2097152, PROT_NONE) = 0
mmap(0x7f5e0b103000, 40960, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0xe9000) = 0x7f5e0b103000
mmap(0x7f5e0b10d000, 82976, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0x7f5e0b10d000
close(3)                                = 0
open("/lib64/libgcc_s.so.1", O_RDONLY|O_CLOEXEC) = 3
read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\360*\0\0\0\0\0\0"..., 832) = 832
fstat(3, {st_mode=S_IFREG|0755, st_size=88720, ...}) = 0
mmap(NULL, 2184192, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7f5e0ac04000
mprotect(0x7f5e0ac19000, 2093056, PROT_NONE) = 0
mmap(0x7f5e0ae18000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x14000) = 0x7f5e0ae18000
close(3)                                = 0
open("/lib64/libc.so.6", O_RDONLY|O_CLOEXEC) = 3
read(3, "\177ELF\2\1\1\3\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\20\35\2\0\0\0\0\0"..., 832) = 832
fstat(3, {st_mode=S_IFREG|0755, st_size=2127336, ...}) = 0
mmap(NULL, 3940800, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7f5e0a841000
mprotect(0x7f5e0a9f9000, 2097152, PROT_NONE) = 0
mmap(0x7f5e0abf9000, 24576, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x1b8000) = 0x7f5e0abf9000
mmap(0x7f5e0abff000, 16832, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0x7f5e0abff000
close(3)                                = 0
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f5e0c6c5000
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f5e0c6c4000
mmap(NULL, 24576, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f5e0c6be000
arch_prctl(ARCH_SET_FS, 0x7f5e0c6c2840) = 0
mprotect(0x7f5e0abf9000, 16384, PROT_READ) = 0
mprotect(0x7f5e0ae18000, 4096, PROT_READ) = 0
mprotect(0x7f5e0b422000, 4096, PROT_READ) = 0
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f5e0c6bd000
mprotect(0x7f5e0b103000, 32768, PROT_READ) = 0
mprotect(0x7f5e0b638000, 4096, PROT_READ) = 0
mprotect(0x7f5e0b83c000, 4096, PROT_READ) = 0
mprotect(0x7f5e0bc5c000, 4096, PROT_READ) = 0
mprotect(0x7f5e0ba44000, 4096, PROT_READ) = 0
mprotect(0x7f5e0c096000, 114688, PROT_READ) = 0
mprotect(0x8cb000, 8192, PROT_READ)     = 0
mprotect(0x7f5e0c6ea000, 4096, PROT_READ) = 0
munmap(0x7f5e0c6c8000, 130172)          = 0
set_tid_address(0x7f5e0c6c2b10)         = 1253
set_robust_list(0x7f5e0c6c2b20, 24)     = 0
rt_sigaction(SIGRTMIN, {0x7f5e0ba4c7e0, [], SA_RESTORER|SA_SIGINFO, 0x7f5e0ba555e0}, NULL, 8) = 0
rt_sigaction(SIGRT_1, {0x7f5e0ba4c870, [], SA_RESTORER|SA_RESTART|SA_SIGINFO, 0x7f5e0ba555e0}, NULL, 8) = 0
rt_sigprocmask(SIG_UNBLOCK, [RTMIN RT_1], NULL, 8) = 0
getrlimit(RLIMIT_STACK, {rlim_cur=8192*1024, rlim_max=RLIM64_INFINITY}) = 0
readlink("/etc/jem_malloc.conf", 0x7ffef3ec35f0, 4096) = -1 ENOENT (No such file or directory)
open("/proc/sys/vm/overcommit_memory", O_RDONLY) = 3
futex(0x7f5e0b83d0d0, FUTEX_WAKE_PRIVATE, 2147483647) = 0
futex(0x8e1dc0, FUTEX_WAIT_PRIVATE, 2, NULL

Explanation

An upgrade or new package installation has caused linux dependencies to install d-bus packages (desktop bus). These are fine for desktop (GUI) software, but certain versions of libdbus-compact.so.1 and libdbus-compact.so.2, which may end up in ld-preload, are causing applications which use jemalloc to hang.

Solution

The solution to this requires the removal of said 2 packages which is done by editing /etc/ld.so.preload. If these are the only 2 files mentioned in /etc/ld.so.preload the whole file can be removed. Otherwise only those entries should be removed.

Notes

While the removal of the preload of libdbus should not have any negative impact on other applications running on the server, certain GUI application may no longer be able to properly function. If GUI applications are impacted, the OS vendor should be consulted.

Keywords

LIBDBUS ASD HUNG LD.SO.PRELOAD JEMALLOC NO LOG

Timestamp

July 2020

© 2015 Copyright Aerospike, Inc. | All rights reserved. Creators of the Aerospike Database.