cat /proc/mounts
overlay / overlay rw,relatime,lowerdir=/var/lib/docker/overlay2/l/K2UPOYT3YRIUMYWTSUN4I4MHXI:/var/lib/docker/overlay2/l/7ARKPNYSF3ZFDLDNYF57RFYK3I:/var/lib/docker/overlay2/l/AN4NVMZW226HTYNTBGW77SRQCZ:/var/lib/docker/overlay2/l/LFD23XSMNJ66MYOXG7KEOXNUP7:/var/lib/docker/overlay2/l/4AE5Q5KKR5NXZHGU2RT53NQEQP,upperdir=/var/lib/docker/overlay2/d7754fdf0031e6eeba4458b06441057c3ed3a4c212e37c68307b09a5f38a5f24/diff,workdir=/var/lib/docker/overlay2/d7754fdf0031e6eeba4458b06441057c3ed3a4c212e37c68307b09a5f38a5f24/work 0 0
proc /proc proc rw,nosuid,nodev,noexec,relatime 0 0
tmpfs /dev tmpfs rw,nosuid,mode=755 0 0
devpts /dev/pts devpts rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=666 0 0
sysfs /sys sysfs ro,nosuid,nodev,noexec,relatime 0 0
tmpfs /sys/fs/cgroup tmpfs ro,nosuid,nodev,noexec,relatime,mode=755 0 0
cgroup /sys/fs/cgroup/systemd cgroup ro,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd 0 0
cgroup /sys/fs/cgroup/devices cgroup ro,nosuid,nodev,noexec,relatime,devices 0 0
cgroup /sys/fs/cgroup/net_cls,net_prio cgroup ro,nosuid,nodev,noexec,relatime,net_cls,net_prio 0 0
cgroup /sys/fs/cgroup/cpu,cpuacct cgroup ro,nosuid,nodev,noexec,relatime,cpu,cpuacct 0 0
cgroup /sys/fs/cgroup/blkio cgroup ro,nosuid,nodev,noexec,relatime,blkio 0 0
cgroup /sys/fs/cgroup/cpuset cgroup ro,nosuid,nodev,noexec,relatime,cpuset 0 0
cgroup /sys/fs/cgroup/freezer cgroup ro,nosuid,nodev,noexec,relatime,freezer 0 0
cgroup /sys/fs/cgroup/perf_event cgroup ro,nosuid,nodev,noexec,relatime,perf_event 0 0
cgroup /sys/fs/cgroup/hugetlb cgroup ro,nosuid,nodev,noexec,relatime,hugetlb 0 0
cgroup /sys/fs/cgroup/memory cgroup ro,nosuid,nodev,noexec,relatime,memory 0 0
cgroup /sys/fs/cgroup/pids cgroup ro,nosuid,nodev,noexec,relatime,pids 0 0
cgroup /sys/fs/cgroup/rdma cgroup ro,nosuid,nodev,noexec,relatime,rdma 0 0
mqueue /dev/mqueue mqueue rw,nosuid,nodev,noexec,relatime 0 0
/dev/sda1 /etc/aerospike ext4 ro,relatime,commit=30,data=ordered 0 0
/dev/sda1 /dev/termination-log ext4 rw,relatime,commit=30,data=ordered 0 0
/dev/sda1 /etc/resolv.conf ext4 rw,nosuid,nodev,relatime,commit=30,data=ordered 0 0
/dev/sda1 /etc/hostname ext4 rw,nosuid,nodev,relatime,commit=30,data=ordered 0 0
/dev/sda1 /etc/hosts ext4 rw,relatime,commit=30,data=ordered 0 0
shm /dev/shm tmpfs rw,nosuid,nodev,noexec,relatime,size=65536k 0 0
/dev/sdc /opt/aerospike/data ext4 rw,relatime,data=ordered 0 0
tmpfs /run/secrets/kubernetes.io/serviceaccount tmpfs ro,relatime 0 0
proc /proc/bus proc ro,nosuid,nodev,noexec,relatime 0 0
proc /proc/fs proc ro,nosuid,nodev,noexec,relatime 0 0
proc /proc/irq proc ro,nosuid,nodev,noexec,relatime 0 0
proc /proc/sys proc ro,nosuid,nodev,noexec,relatime 0 0
proc /proc/sysrq-trigger proc ro,nosuid,nodev,noexec,relatime 0 0
tmpfs /proc/kcore tmpfs rw,nosuid,mode=755 0 0
tmpfs /proc/timer_list tmpfs rw,nosuid,mode=755 0 0
tmpfs /sys/firmware tmpfs ro,relatime 0 0
ls -l /dev
total 0
lrwxrwxrwx 1 root root 11 Mar 22 14:52 core -> /proc/kcore
lrwxrwxrwx 1 root root 13 Mar 22 14:52 fd -> /proc/self/fd
crw-rw-rw- 1 root root 1, 7 Mar 22 14:52 full
drwxrwxrwt 2 root root 40 Mar 22 14:51 mqueue
crw-rw-rw- 1 root root 1, 3 Mar 22 14:52 null
lrwxrwxrwx 1 root root 8 Mar 22 14:52 ptmx -> pts/ptmx
drwxr-xr-x 2 root root 0 Mar 22 14:52 pts
crw-rw-rw- 1 root root 1, 8 Mar 22 14:52 random
drwxrwxrwt 2 root root 40 Mar 22 14:51 shm
lrwxrwxrwx 1 root root 15 Mar 22 14:52 stderr -> /proc/self/fd/2
lrwxrwxrwx 1 root root 15 Mar 22 14:52 stdin -> /proc/self/fd/0
lrwxrwxrwx 1 root root 15 Mar 22 14:52 stdout -> /proc/self/fd/1
-rw-rw-rw- 1 root root 0 Mar 22 14:52 termination-log
crw-rw-rw- 1 root root 5, 0 Mar 22 14:52 tty
crw-rw-rw- 1 root root 1, 9 Mar 22 14:52 urandom
crw-rw-rw- 1 root root 1, 5 Mar 22 14:52 zero
I’m not sure that memory issue is related to aerospike version, I had same issue when I used version 3 (And I used ubuntu node instead of container-optimized os node). Actually raw device is not supported on Kubernetes GKE (1.12 version), but I hope 1.13 will be release soon to be able to change this config.
Yes, htop output was runned from the node. I have runned the command from container too, and the result is the same (so container views all node memory).
Memory is high but memory looks stable, priority for me is performance so that might be fine if it does not explode more. With this percentage of memory usage it’s a bit difficult to scale cluster using Kubernetes metrics.
Notice that I didn’t set Memory limit on Kubernetes, because if the memory exceeds this limit, the container will be restarted. Without this limit, I guess the container is not aware of how much memory it has. Question is (I’m really not an expert on all these topics ), if I set the limit, will the “page cache” you are talking about be aware of it? I read article about memory limits on Kubernetes, it can help:
Thanks for help, I will try with direct-files and I’ll let you know!