Why Aerospike server is very slow


#1

Latency: reads:18:12:32-GMT,ops/sec,>1ms,>8ms,>64ms;18:12:42,1421.0,12.62,12.61,12.59;writes_master:18:12:32-GMT,ops/sec,>1ms,>8ms,>64ms;18:12:42,1411.0,100.00,100.00,100.00;proxy:18:12:32-GMT,ops/sec,>1ms,>8ms,>64ms;18:12:42,0.0,0.00,0.00,0.00;writes_reply:18:12:32-GMT,ops/sec,>1ms,>8ms,>64ms;18:12:42,473.2,100.00,100.00,100.00;udf:18:12:32-GMT,ops/sec,>1ms,>8ms,>64ms;18:12:42,0.0,0.00,0.00,0.00;query:18:12:32-GMT,ops/sec,>1ms,>8ms,>64ms;18:12:42,0.0,0.00,0.00,0.00; 10.1.10.34:3000 returned : reads:18:12:28-GMT,ops/sec,>1ms,>8ms,>64ms;18:12:38,1448.9,4.38,4.33,4.13;writes_master:18:12:28-GMT,ops/sec,>1ms,>8ms,>64ms;18:12:38,1505.6,100.00,100.00,100.00;proxy:18:12:28-GMT,ops/sec,>1ms,>8ms,>64ms;18:12:38,0.0,0.00,0.00,0.00;writes_reply:18:12:28-GMT,ops/sec,>1ms,>8ms,>64ms;18:12:38,155.8,100.00,100.00,100.00;udf:18:12:28-GMT,ops/sec,>1ms,>8ms,>64ms;18:12:38,0.0,0.00,0.00,0.00;query:18:12:28-GMT,ops/sec,>1ms,>8ms,>64ms;18:12:38,0.0,0.00,0.00,0.00;

Aerospike server statistics: cluster_size=2;cluster_key=EFDCB4DE2C32CBD3;cluster_integrity=true;objects=636564946;sub-records=0;total-bytes-disk=3159997022208;used-bytes-disk=723444536448;free-pct-disk=77;total-bytes-memory=193273528320;used-bytes-memory=40740156544;data-used-bytes-memory=0;index-used-bytes-memory=40740156544;sindex-used-bytes-memory=0;free-pct-memory=78;stat_read_reqs=56553119;stat_read_reqs_xdr=35358;stat_read_success=53198419;stat_read_errs_notfound=3238264;stat_read_errs_other=0;stat_write_reqs=53445348;stat_write_reqs_xdr=0;stat_write_success=38632304;stat_write_errs=16;stat_xdr_pipe_writes=94431412;stat_xdr_pipe_miss=0;stat_delete_success=14993;stat_rw_timeout=14928145;udf_read_reqs=0;udf_read_success=0;udf_read_errs_other=0;udf_write_reqs=0;udf_write_success=0;udf_write_err_others=0;udf_delete_reqs=0;udf_delete_success=0;udf_delete_err_others=0;udf_lua_errs=0;udf_scan_rec_reqs=0;udf_query_rec_reqs=0;udf_replica_writes=0;stat_proxy_reqs=1;stat_proxy_reqs_xdr=0;stat_proxy_success=1;stat_proxy_errs=0;stat_ldt_proxy=0;stat_cluster_key_trans_to_proxy_retry=0;stat_cluster_key_transaction_reenqueue=0;stat_slow_trans_queue_push=0;stat_slow_trans_queue_pop=0;stat_slow_trans_queue_batch_pop=0;stat_cluster_key_regular_processed=0;stat_cluster_key_prole_retry=0;stat_cluster_key_err_ack_dup_trans_reenqueue=0;stat_cluster_key_partition_transaction_queue_count=0;stat_cluster_key_err_ack_rw_trans_reenqueue=0;stat_expired_objects=0;stat_evicted_objects=0;stat_deleted_set_objects=0;stat_evicted_set_objects=0;stat_evicted_objects_time=0;stat_zero_bin_records=0;stat_nsup_deletes_not_shipped=0;err_tsvc_requests=16;err_out_of_space=0;err_duplicate_proxy_request=0;err_rw_request_not_found=4653;err_rw_pending_limit=0;err_rw_cant_put_unique=0;fabric_msgs_sent=215158617;fabric_msgs_rcvd=215161002;paxos_principal=BB958AC25F0A840;migrate_msgs_sent=9557387;migrate_msgs_recv=9573830;migrate_progress_send=2007;migrate_progress_recv=1;migrate_num_incoming_accepted=8214;migrate_num_incoming_refused=0;queue=0;transactions=123719481;reaped_fds=1;tscan_initiate=0;tscan_pending=0;tscan_succeeded=0;tscan_aborted=0;batch_initiate=192625;batch_queue=0;batch_tree_count=0;batch_timeout=0;batch_errors=0;info_queue=0;delete_queue=0;proxy_in_progress=0;proxy_initiate=1;proxy_action=8527;proxy_retry=0;proxy_retry_q_full=0;proxy_unproxy=0;proxy_retry_same_dest=0;proxy_retry_new_dest=0;write_master=98908904;write_prole=97978714;read_dup_prole=108769888;rw_err_dup_internal=0;rw_err_dup_cluster_key=0;rw_err_dup_send=0;rw_err_write_internal=0;rw_err_write_cluster_key=0;rw_err_write_send=0;rw_err_ack_internal=0;rw_err_ack_nomatch=14926460;rw_err_ack_badnode=0;client_connections=2393;waiting_transactions=268;tree_count=0;record_refs=430529;record_locks=0;migrate_tx_objs=2007;migrate_rx_objs=1;ongoing_write_reqs=1372;err_storage_queue_full=0;partition_actual=4174;partition_replica=4018;partition_desync=0;partition_absent=0;partition_object_count=636564946;partition_ref_count=11573;system_free_mem_pct=98;sindex_ucgarbage_found=0;sindex_gc_locktimedout=0;sindex_gc_inactivity_dur=0;sindex_gc_activity_dur=0;sindex_gc_list_creation_time=0;sindex_gc_list_deletion_time=0;sindex_gc_objects_validated=0;sindex_gc_garbage_found=0;sindex_gc_garbage_cleaned=0;system_swapping=false;err_replica_null_node=0;err_replica_non_null_node=0;err_sync_copy_null_node=0;err_sync_copy_null_master=0;storage_defrag_corrupt_record=0;err_write_fail_prole_unknown=0;err_write_fail_prole_generation=0;err_write_fail_unknown=0;err_write_fail_key_exists=0;err_write_fail_generation=0;err_write_fail_generation_xdr=0;err_write_fail_bin_exists=0;err_write_fail_parameter=0;err_write_fail_incompatible_type=0;err_write_fail_noxdr=0;err_write_fail_prole_delete=0;err_write_fail_not_found=0;err_write_fail_key_mismatch=0;stat_duplicate_operation=56748952;uptime=60380;stat_write_errs_notfound=16;stat_write_errs_other=0;heartbeat_received_self=0;heartbeat_received_foreign=755494;query_reqs=0;query_success=0;query_fail=0;query_abort=0;query_avg_rec_count=0;query_short_queue_full=0;query_long_queue_full=0;query_short_running=0;query_long_running=0;query_tracked=0;query_agg=0;query_agg_success=0;query_agg_err=0;query_agg_abort=0;query_agg_avg_rec_count=0;query_lookups=0;query_lookup_success=0;query_lookup_err=0;query_lookup_abort=0;query_lookup_avg_rec_count=0

Aerospike Config: transaction-queues=32;transaction-threads-per-queue=4;transaction-duplicate-threads=0;transaction-pending-limit=20;migrate-threads=1;migrate-xmit-priority=40;migrate-xmit-sleep=500;migrate-read-priority=10;migrate-read-sleep=500;migrate-xmit-hwm=10;migrate-xmit-lwm=5;migrate-max-num-incoming=256;migrate-rx-lifetime-ms=60000;proto-fd-max=15000;proto-fd-idle-ms=300000;transaction-retry-ms=1000;transaction-max-ms=1000;transaction-repeatable-read=false;dump-message-above-size=134217728;ticker-interval=10;microbenchmarks=false;storage-benchmarks=false;ldt-benchmarks=false;scan-priority=200;scan-sleep=1;batch-threads=4;batch-max-requests=5000;batch-priority=200;nsup-delete-sleep=0;nsup-period=120;nsup-startup-evict=true;paxos-retransmit-period=5;paxos-single-replica-limit=1;paxos-max-cluster-size=32;paxos-protocol=v3;paxos-recovery-policy=manual;write-duplicate-resolution-disable=false;respond-client-on-master-completion=false;replication-fire-and-forget=false;info-threads=16;allow-inline-transactions=true;use-queue-per-device=false;snub-nodes=false;fb-health-msg-per-burst=0;fb-health-msg-timeout=200;fb-health-good-pct=50;fb-health-bad-pct=0;auto-dun=false;auto-undun=false;prole-extra-ttl=0;max-msgs-per-type=-1;pidfile=/var/run/aerospike/asd.pid;memory-accounting=false;udf-runtime-gmax-memory=18446744073709551615;udf-runtime-max-memory=18446744073709551615;sindex-populator-scan-priority=3;sindex-data-max-memory=18446744073709551615;query-threads=6;query-worker-threads=15;query-priority=10;query-in-transaction-thread=0;query-req-in-query-thread=0;query-req-max-inflight=100;query-bufpool-size=256;query-batch-size=100;query-sleep=1;query-job-tracking=false;query-short-q-max-size=500;query-long-q-max-size=500;query-rec-count-bound=4294967295;query-threshold=10;query-untracked-time=1000000;service-address=0.0.0.0;service-port=3000;mesh-seed-address-port=10.1.10.33:3002;access-address=10.1.10.34;reuse-address=true;fabric-port=3001;network-info-port=3003;enable-fastpath=true;heartbeat-mode=mesh;heartbeat-protocol=v2;heartbeat-address=10.1.10.34;heartbeat-port=3002;heartbeat-interval=150;heartbeat-timeout=20;enable-security=false;privilege-refresh-period=300;report-authentication-sinks=0;report-sys-admin-sinks=0;report-user-admin-sinks=0;report-violation-sinks=0;syslog-local=-1;xdr-delete-shipping-enabled=true;xdr-nsup-deletes-enabled=false;enable-xdr=true;stop-writes-noxdr=false;reads-hist-track-back=1800;reads-hist-track-slice=10;reads-hist-track-thresholds=1,8,64;writes_master-hist-track-back=1800;writes_master-hist-track-slice=10;writes_master-hist-track-thresholds=1,8,64;proxy-hist-track-back=1800;proxy-hist-track-slice=10;proxy-hist-track-thresholds=1,8,64;writes_reply-hist-track-back=1800;writes_reply-hist-track-slice=10;writes_reply-hist-track-thresholds=1,8,64;udf-hist-track-back=1800;udf-hist-track-slice=10;udf-hist-track-thresholds=1,8,64;query-hist-track-back=1800;query-hist-track-slice=10;query-hist-track-thresholds=1,8,64;query_rec_count-hist-track-back=1800;query_rec_count-hist-track-slice=10;query_rec_count-hist-track-thresholds=1,8,64


#2

Hi,

What kind of setup is it? Where is the server running? What is the type of hardware that you are using? Which machine are you running your client on?