FAILED ASSERTION (drv_ssd): (drv_ssd.c:3075) unable to open device /dev/nvme2n1p1: Operation not permitted

When aerospike is started by systemd, it fails to start with below error. But when I start the process manually as root from the shell, it successfully opens the device.

Mar 26 2020 17:47:11 GMT: FAILED ASSERTION (drv_ssd): (drv_ssd.c:3075) unable to open device /dev/nvme2n1p1: Operation not permitted
Mar 26 2020 17:47:11 GMT: WARNING (as): (signal.c:213) SIGUSR1 received, aborting Aerospike Community Edition build 4.8.0.8 os ubuntu18.04
Mar 26 2020 17:47:11 GMT: WARNING (as): (fault.c:1178) stacktrace: registers: rax 0000000000000000 rbx 00007fff47b48a60 rcx 00007f2595d76727 rdx 0000000000000000 rsi 00007fff47b488f0 rdi 0000000000000002 rbp 00007fff47b48f50 rsp 00007fff47b488f0 r8 0000000000000000 r9 00007fff47b488f0 r10 0000000000000008 r11 0000000000000246 r12 0000000000000012 r13 0000000000000000 r14 0000000000000084 r15 00005609437c7e60 rip 00007f2595d76727
Mar 26 2020 17:47:11 GMT: WARNING (as): (fault.c:1192) stacktrace: found 11 frames: 0x174344 0xbef58 0x7f2595d76890 0x7f2595d76727 0x171bdb 0x147d42 0x1487ea 0x14b360 0x5176e 0x7f2594a2cb97 0x5216a offset 0x56093d1e8000
Mar 26 2020 17:47:11 GMT: WARNING (as): (fault.c:1200) stacktrace: frame 0: /usr/bin/asd(cf_fault_print_signal_context+0x138) [0x56093d35c344]
Mar 26 2020 17:47:11 GMT: WARNING (as): (fault.c:1200) stacktrace: frame 1: /usr/bin/asd(as_sig_handle_usr1+0x5c) [0x56093d2a6f58]
Mar 26 2020 17:47:11 GMT: WARNING (as): (fault.c:1200) stacktrace: frame 2: /lib/x86_64-linux-gnu/libpthread.so.0(+0x12890) [0x7f2595d76890]
Mar 26 2020 17:47:11 GMT: WARNING (as): (fault.c:1200) stacktrace: frame 3: /lib/x86_64-linux-gnu/libpthread.so.0(raise+0xc7) [0x7f2595d76727]
Mar 26 2020 17:47:11 GMT: WARNING (as): (fault.c:1200) stacktrace: frame 4: /usr/bin/asd(cf_fault_event+0x27a) [0x56093d359bdb]
Mar 26 2020 17:47:11 GMT: WARNING (as): (fault.c:1200) stacktrace: frame 5: /usr/bin/asd(ssd_init_devices+0x3b6) [0x56093d32fd42]
Mar 26 2020 17:47:11 GMT: WARNING (as): (fault.c:1200) stacktrace: frame 6: /usr/bin/asd(as_storage_init_ssd+0x3f) [0x56093d3307ea]
Mar 26 2020 17:47:11 GMT: WARNING (as): (fault.c:1200) stacktrace: frame 7: /usr/bin/asd(as_storage_init+0x37) [0x56093d333360]

Aerospike database configuration file for use with systemd.

service {
   paxos-single-replica-limit 1 # Number of nodes where the replica count is automatically reduced to 1.
   proto-fd-max 15000
   #proto-fd-idle-ms 15000 # 15sec, default is 1 minute
   user root
   group root
}

logging {
    file /var/log/aerospike/aerospike.log {
        context any info
    }
}

network {
    service {
        address any
        port 3000
    }

    heartbeat {
        mode multicast
        multicast-group 239.1.99.224
        port 9918

        # To use unicast-mesh heartbeats, remove the 3 lines above, and see
        # aerospike_mesh.conf for alternative.

        interval 150
        timeout 10
    }

    fabric {
        port 3001
    }

    info {
        port 3003
    }
}

namespace transaction {
    replication-factor 1
    memory-size 64G
    default-ttl 0d # 0 to never expire/evict.
    storage-engine memory
}

namespace data {
    replication-factor 1
    memory-size 64G
    default-ttl 0d # 0 to never expire/evict.
    storage-engine device {
        max-write-cache 20G
        write-block-size 4M
        device  /dev/nvme2n1p1
        device  /dev/nvme2n1p2
        device  /dev/nvme2n1p3
        device  /dev/nvme2n1p4
    }
}
systemctl cat aerospike
# /usr/lib/systemd/system/aerospike.service
[Unit]
Description=Aerospike Server
After=network.target
Wants=network.target

[Service]
LimitNOFILE=100000
TimeoutSec=600
User=root
Group=root
EnvironmentFile=/etc/sysconfig/aerospike
PermissionsStartOnly=True
ExecStartPre=/usr/bin/asd-systemd-helper
ExecStart=/usr/bin/asd $ASD_OPTIONS --config-file $ASD_CONFIG_FILE --fgdaemon
ExecStartPre=-/bin/systemctl start aerospike_telemetry
ExecStopPost=-/bin/systemctl stop aerospike_telemetry

[Install]
WantedBy=multi-user.target
# /etc/systemd/system/aerospike.service.d/aerospike.conf
#

any ideas? anything I can look for or try out?

Curious what permissions are for set that device.

ls -al  /dev/nvme2n1p1

expecting something like this:

ubuntu-bionic:~$ ls -al /dev/sdc
brw-rw---- 1 root disk 8, 32 Mar 27 10:18 /dev/sdc

$ getent group disk
disk:x:6:


#ls -al /dev/nvme* brw-rw---- 1 root root 259, 9 Mar 25 22:44 /dev/nvme2n1p1

getent group disk

disk:x:6:

May be SeLinux related. Try the lsattr and chattr commands.

#lsattr /dev/nvme2n1p1 lsattr: Operation not supported While reading flags on /dev/nvme2n1p1

weird issue. May be try to explicitly add “root” user to disk group similar to a non-root case… Please see: https://www.aerospike.com/docs/operations/configure/non_root/index.html#configure-ssd-resources-used-by-namespaces