Unexpected stop-writes

Operations Configuration
1

Version E-5.7.0.9

I have got the error Error 8,1,0,30000,0,0,BB9B692C56B1FAC 127.0.0.1 3000: Server memory error. Replication factor 1

namespace linkage {
        replication-factor 1
        memory-size 64G
        default-ttl 0
        set txs_local {
           disable-eviction true
        }
        storage-engine pmem {
                file /var/sda1/aerospike/linkage
                file /var/sda1/aerospike/linkage1
                file /var/sda1/aerospike/linkage2
                file /var/sda1/aerospike/linkage3
                filesize 32G
        }
}

aerospike log shows

WARNING (nsup): (nsup.c:878) {linkage} breached stop-writes limit (device-avail-pct), memory sz:9672149504 (9672149504 + 0 + 0 + 0) limit:61847529062, disk avail-pct:4

But thats strange total devices size 128G, and its far enough from 61847529062 limit.

asinfo -v "namespace/linkage" -l 

ns_cluster_size=1
effective_replication_factor=1
objects=151127337
tombstones=0
xdr_tombstones=0
xdr_bin_cemeteries=0
master_objects=151127337
master_tombstones=0
prole_objects=0
prole_tombstones=0
non_replica_objects=0
non_replica_tombstones=0
unreplicated_records=0
dead_partitions=0
unavailable_partitions=0
clock_skew_stop_writes=false
stop_writes=false
hwm_breached=false
current_time=384783036
non_expirable_objects=0
expired_objects=0
evicted_objects=0
evict_ttl=0
evict_void_time=0
smd_evict_void_time=0
nsup_cycle_duration=0
truncate_lut=0
truncated_records=0
sindex_gc_cleaned=0
memory_used_bytes=9672149568
memory_used_data_bytes=0
memory_used_index_bytes=9672149568
memory_used_set_index_bytes=0
memory_used_sindex_bytes=0
memory_free_pct=85
xmem_id=1
available_bin_names=65523
pmem_total_bytes=137438953472
pmem_used_bytes=78809382992
pmem_free_pct=42
pmem_available_pct=5
storage-engine.file[0].used_bytes=19692956624
storage-engine.file[0].free_wblocks=239
storage-engine.file[0].write_q=0
storage-engine.file[0].writes=137276
storage-engine.file[0].defrag_q=0
storage-engine.file[0].defrag_reads=225503
storage-engine.file[0].defrag_writes=99957
storage-engine.file[0].shadow_write_q=0
storage-engine.file[0].age=0
storage-engine.file[1].used_bytes=19726104208
storage-engine.file[1].free_wblocks=213
storage-engine.file[1].write_q=0
storage-engine.file[1].writes=140758
storage-engine.file[1].defrag_q=0
storage-engine.file[1].defrag_reads=229119
storage-engine.file[1].defrag_writes=101109
storage-engine.file[1].shadow_write_q=0
storage-engine.file[1].age=0
storage-engine.file[2].used_bytes=19708299520
storage-engine.file[2].free_wblocks=238
storage-engine.file[2].write_q=0
storage-engine.file[2].writes=146347
storage-engine.file[2].defrag_q=0
storage-engine.file[2].defrag_reads=237079
storage-engine.file[2].defrag_writes=104286
storage-engine.file[2].shadow_write_q=0
storage-engine.file[2].age=0
storage-engine.file[3].used_bytes=19682022640
storage-engine.file[3].free_wblocks=216
storage-engine.file[3].write_q=0
storage-engine.file[3].writes=138257
storage-engine.file[3].defrag_q=0
storage-engine.file[3].defrag_reads=225222
storage-engine.file[3].defrag_writes=99515
storage-engine.file[3].shadow_write_q=0
storage-engine.file[3].age=0
record_proto_uncompressed_pct=0.000
record_proto_compression_ratio=1.000
scan_proto_uncompressed_pct=0.000
scan_proto_compression_ratio=1.000
query_proto_uncompressed_pct=0.000
query_proto_compression_ratio=1.000
pending_quiesce=false
effective_is_quiesced=false
nodes_quiesced=0
effective_prefer_uniform_balance=true
migrate_tx_partitions_imbalance=0
migrate_tx_instances=0
migrate_rx_instances=0
migrate_tx_partitions_active=0
migrate_rx_partitions_active=0
migrate_tx_partitions_initial=0
migrate_tx_partitions_remaining=0
migrate_tx_partitions_lead_remaining=0
migrate_rx_partitions_initial=0
migrate_rx_partitions_remaining=0
migrate_records_skipped=0
migrate_records_transmitted=0
migrate_record_retransmits=0
migrate_record_receives=0
migrate_signals_active=0
migrate_signals_remaining=0
appeals_tx_active=0
appeals_rx_active=0
appeals_tx_remaining=0
appeals_records_exonerated=0
client_tsvc_error=0
client_tsvc_timeout=0
client_proxy_complete=0
client_proxy_error=0
client_proxy_timeout=0
client_read_success=586234958
client_read_error=0
client_read_timeout=0
client_read_not_found=0
client_read_filtered_out=0
client_write_success=813541266
client_write_error=1
client_write_timeout=0
client_write_filtered_out=0
xdr_client_write_success=0
xdr_client_write_error=0
xdr_client_write_timeout=0
client_delete_success=0
client_delete_error=0
client_delete_timeout=0
client_delete_not_found=0
client_delete_filtered_out=0
xdr_client_delete_success=0
xdr_client_delete_error=0
xdr_client_delete_timeout=0
xdr_client_delete_not_found=0
client_udf_complete=0
client_udf_error=0
client_udf_timeout=0
client_udf_filtered_out=0
client_lang_read_success=0
client_lang_write_success=0
client_lang_delete_success=0
client_lang_error=0
from_proxy_tsvc_error=0
from_proxy_tsvc_timeout=0
from_proxy_read_success=0
from_proxy_read_error=0
from_proxy_read_timeout=0
from_proxy_read_not_found=0
from_proxy_read_filtered_out=0
from_proxy_write_success=0
from_proxy_write_error=0
from_proxy_write_timeout=0
from_proxy_write_filtered_out=0
xdr_from_proxy_write_success=0
xdr_from_proxy_write_error=0
xdr_from_proxy_write_timeout=0
from_proxy_delete_success=0
from_proxy_delete_error=0
from_proxy_delete_timeout=0
from_proxy_delete_not_found=0
from_proxy_delete_filtered_out=0
xdr_from_proxy_delete_success=0
xdr_from_proxy_delete_error=0
xdr_from_proxy_delete_timeout=0
xdr_from_proxy_delete_not_found=0
from_proxy_udf_complete=0
from_proxy_udf_error=0
from_proxy_udf_timeout=0
from_proxy_udf_filtered_out=0
from_proxy_lang_read_success=0
from_proxy_lang_write_success=0
from_proxy_lang_delete_success=0
from_proxy_lang_error=0
batch_sub_tsvc_error=0
batch_sub_tsvc_timeout=0
batch_sub_proxy_complete=0
batch_sub_proxy_error=0
batch_sub_proxy_timeout=0
batch_sub_read_success=0
batch_sub_read_error=0
batch_sub_read_timeout=0
batch_sub_read_not_found=0
batch_sub_read_filtered_out=0
from_proxy_batch_sub_tsvc_error=0
from_proxy_batch_sub_tsvc_timeout=0
from_proxy_batch_sub_read_success=0
from_proxy_batch_sub_read_error=0
from_proxy_batch_sub_read_timeout=0
from_proxy_batch_sub_read_not_found=0
from_proxy_batch_sub_read_filtered_out=0
udf_sub_tsvc_error=0
udf_sub_tsvc_timeout=0
udf_sub_udf_complete=0
udf_sub_udf_error=0
udf_sub_udf_timeout=0
udf_sub_udf_filtered_out=0
udf_sub_lang_read_success=0
udf_sub_lang_write_success=0
udf_sub_lang_delete_success=0
udf_sub_lang_error=0
ops_sub_tsvc_error=0
ops_sub_tsvc_timeout=0
ops_sub_write_success=0
ops_sub_write_error=0
ops_sub_write_timeout=0
ops_sub_write_filtered_out=0
dup_res_ask=0
dup_res_respond_read=0
dup_res_respond_no_read=0
retransmit_all_read_dup_res=0
retransmit_all_write_dup_res=0
retransmit_all_write_repl_write=0
retransmit_all_delete_dup_res=0
retransmit_all_delete_repl_write=0
retransmit_all_udf_dup_res=0
retransmit_all_udf_repl_write=0
retransmit_all_batch_sub_dup_res=0
retransmit_udf_sub_dup_res=0
retransmit_udf_sub_repl_write=0
retransmit_ops_sub_dup_res=0
retransmit_ops_sub_repl_write=0
scan_basic_complete=1
scan_basic_error=0
scan_basic_abort=0
scan_aggr_complete=0
scan_aggr_error=0
scan_aggr_abort=0
scan_udf_bg_complete=0
scan_udf_bg_error=0
scan_udf_bg_abort=0
scan_ops_bg_complete=0
scan_ops_bg_error=0
scan_ops_bg_abort=0
query_reqs=0
query_fail=0
query_false_positives=0
query_short_queue_full=0
query_long_queue_full=0
query_short_reqs=0
query_long_reqs=0
query_basic_complete=0
query_basic_error=0
query_basic_abort=0
query_basic_avg_rec_count=0
query_aggr_complete=0
query_aggr_error=0
query_aggr_abort=0
query_aggr_avg_rec_count=0
query_udf_bg_complete=0
query_udf_bg_error=0
query_udf_bg_abort=0
query_ops_bg_complete=0
query_ops_bg_error=0
query_ops_bg_abort=0
geo_region_query_reqs=0
geo_region_query_cells=0
geo_region_query_points=0
geo_region_query_falsepos=0
re_repl_success=0
re_repl_error=0
re_repl_timeout=0
fail_xdr_forbidden=0
fail_key_busy=0
fail_generation=0
fail_record_too_big=0
fail_client_lost_conflict=0
fail_xdr_lost_conflict=0
deleted_last_bin=0
replication-factor=1
memory-size=68719476736
default-ttl=0
allow-ttl-without-nsup=false
background-scan-max-rps=10000
conflict-resolution-policy=generation
conflict-resolve-writes=false
data-in-index=false
disable-cold-start-eviction=false
disable-write-dup-res=false
disallow-null-setname=false
enable-benchmarks-batch-sub=false
enable-benchmarks-ops-sub=false
enable-benchmarks-read=false
enable-benchmarks-udf=false
enable-benchmarks-udf-sub=false
enable-benchmarks-write=false
enable-hist-proxy=false
evict-hist-buckets=10000
evict-tenths-pct=5
high-water-disk-pct=0
high-water-memory-pct=0
ignore-migrate-fill-delay=false
index-stage-size=1073741824
index-type=shmem
max-record-size=0
migrate-order=5
migrate-retransmit-ms=5000
migrate-sleep=1
nsup-hist-period=3600
nsup-period=0
nsup-threads=1
partition-tree-sprigs=256
prefer-uniform-balance=true
rack-id=0
read-consistency-level-override=off
reject-non-xdr-writes=false
reject-xdr-writes=false
single-bin=false
single-scan-threads=4
stop-writes-pct=90
strong-consistency=false
strong-consistency-allow-expunge=false
tomb-raider-eligible-age=86400
tomb-raider-period=86400
transaction-pending-limit=20
truncate-threads=4
write-commit-level-override=off
xdr-bin-tombstone-ttl=86400
xdr-tomb-raider-period=120
xdr-tomb-raider-threads=1
storage-engine=pmem
storage-engine.file[0]=/var/sda1/aerospike/linkage
storage-engine.file[1]=/var/sda1/aerospike/linkage1
storage-engine.file[2]=/var/sda1/aerospike/linkage2
storage-engine.file[3]=/var/sda1/aerospike/linkage3
storage-engine.filesize=34359738368
storage-engine.commit-to-device=false
storage-engine.compression=none
storage-engine.compression-level=0
storage-engine.defrag-lwm-pct=50
storage-engine.defrag-queue-min=0
storage-engine.defrag-sleep=1000
storage-engine.defrag-startup-minimum=0
storage-engine.direct-files=false
storage-engine.disable-odsync=false
storage-engine.enable-benchmarks-storage=false
storage-engine.encryption-key-file=null
storage-engine.encryption-old-key-file=null
storage-engine.flush-max-ms=1000
storage-engine.max-write-cache=67108864
storage-engine.min-avail-pct=5
storage-engine.serialize-tomb-raider=false
storage-engine.tomb-raider-sleep=1000
sindex.num-partitions=32
geo2dsphere-within.strict=true
geo2dsphere-within.min-level=1
geo2dsphere-within.max-level=20
geo2dsphere-within.max-cells=12
geo2dsphere-within.level-mod=1
geo2dsphere-within.earth-radius-meters=6371000

I dont realize where this limit specified.

2

min-available-pct is configuration default of 5% - you should not set it any lower. defrag-lwm-pct is 50% by default. write-blocks in PMEM are 8MB. You have blocks that are partially full (>50%) due to record updates so can’t be coalesced by defrag and you are running out of entirely free blocks. Less than 5% remain and therefore you have hit stop writes. Your cluster has hit its sizing limit. You can temporarily, dynamically, increase defrag-lwm-pct, gradually, in steps of 1% to see if it gets you out of stop-writes but that is not the solution. Increasing it will make write-amplification worse. You need to add more capacity.

pmem_total_bytes=137438953472
pmem_used_bytes=78809382992
pmem_free_pct=42
pmem_available_pct=5