Client timeout - Error 9 in nodejs client after 100 sequential puts

#1

Hello, I’m moving data from mysql to aerospike I used php client and nodejs client With php client all work fine with nodejs client, after ~100 inserts, the put method return error code 9

This are my writepolicy settings socketTimeout: 500, totalTimeout: 1000, key: Aerospike.policy.key.SEND, exists: Aerospike.policy.exists.CREATE_OR_REPLACE

This is the error: Client timeout: iterations=2 lastNode=51

What I’m doing wrong? thanks

#2

Could you share you code, Aerospike config, and hardware description?

Regarding the code, a common problem for new users is that they create a client per request, the client should typically have the same lifetime as the App.

#3

Thanks @kporter hardware: 1 node Intel Xeon E3-1270v6 - 32GB DDR4 ECC 2400 hd 500gb nvme intel The server don’t have work load actually, so the only work are this write operations

I’m running from command line, like a CLI script

'use strict'
const mysql = require('mysql')
const mysqlconn = mysql.createConnection({
    host: '-',
    user: '-',
    password: '-',
    database: '-'
})
// writePolicy.sendKey = true
//https://www.aerospike.com/apidocs/nodejs/module-aerospike_maps.html
const Aerospike = require('aerospike')
const op = Aerospike.operations
const maps = Aerospike.maps
const predexp = Aerospike.predexp

let offset = process.argv[3]
let inicial = process.argv[2]

let d = new Date();
const bloquesize = 1000
const defaultNamespace = 'dmp'
const defaultSet = 'europe'
const ts = d.getFullYear() + ((d.getMonth() < 9) ? '0' : '') + (d.getMonth() + 1) + ((d.getDate() < 10) ? '0' : '') + d.getDate()
const defaults = {
    socketTimeout: 0,
    totalTimeout: 0,
    maxRetries: 3,
    key: Aerospike.policy.key.SEND,
    exists: Aerospike.policy.exists.CREATE_OR_REPLACE
}

const config = {
    hosts: '-',
    modlua: {
        systemPath: '/opt/aerospike/udf/sys/',
        userPath: '/codigo/udf'
    },
    maxConnsPerNode: 1024,
    policies: {
        write: new Aerospike.WritePolicy(defaults)
    }
}



function assertOk(error, message) {
    if (error) {
        console.error('ERROR - %s: %s [%s]\n%s', message, error.message, error.code, error.stack)
        throw error
    }
}

function insertadmp(client, argumentos) {
    return new Promise(function(resolve, reject) {
        const key = new Aerospike.Key(defaultNamespace, defaultSet, argumentos.dmpID)
        let points = 0;
        let ops = [
            op.write('lastupdated', ts),
            maps.put('fuentes', 'tap', ts)
        ]
        argumentos.segmento.split(',').forEach(function(currentValue, index, arr) {
            ops.push(maps.increment('segszeo', currentValue, 1))
            points++
        })
        ops.push(op.incr('points', points))
        ops.push(maps.size('segszeo'))
        let results = client.operate(key, ops, (error, record) => {
            if (error) {
                reject(error)
            } else {
                client.operate(key, [op.write('nsegmentos', record.bins.segszeo)], (error, record) => {
                    if (error) {
                        reject(error)
                    } else {
                        resolve(record)
                    }
                })
            }



        })
    })
}


function getLastRecord(inicial, offset) {
    return new Promise(function(resolve, reject) {
        mysqlconn.query('Select dmpID,segmento from aaSegs where dmpID like "' + inicial + '%" order by dmpID asc LIMIT ' + offset + ',' + bloquesize, function(err, rows, fields) {
            if (err) {
                return reject(err);
            }
            resolve(rows);
        });
    });
}

getLastRecord(inicial, offset).then(resultados => {
    Aerospike.connect(config).then(client => {
        for (var i = 0; i < resultados.length; i++) {
            insertadmp(client, resultados[i]).then(function(resultado) {
                console.info('inserta ok %s', resultado.key.key)
            }).catch(function(error) {
                console.error('err inserta')
            })
        }

    })

})
#4

Could you also include the version of your client, Aerospike Server, and the Aerospike Server config (/etc/aerospike/aerospike.conf).

#5
# Aerospike database configuration file for use with systemd.
service {
	paxos-single-replica-limit 1 # Number of nodes where the replica count is automatically reduced to 1.
	proto-fd-max 100000
}
logging {
	console {
		context any info
	}
}
network {
	service {
		address any
		port 3000
	}
	heartbeat {
		mode multicast
		multicast-group 239.1.99.222
		port 9918
		# To use unicast-mesh heartbeats, remove the 3 lines above, and see
		# aerospike_mesh.conf for alternative.
		interval 150
		timeout 10
	}
	fabric {
		port 3001
	}
	info {
		port 3003
	}
}
 
namespace dmp {
	replication-factor 1
	memory-size 12G
	default-ttl 0 # 30 days, use 0 to never expire/evict.
	stop-writes-pct 90
	high-water-disk-pct 90 
	storage-engine device {
	device /dev/nvme1n1
	write-block-size 128K
	         post-write-queue 2048
                max-write-cache 512M
		# file /opt/aerospike/data/dmp.dat
		# filesize 16G
		# data-in-memory true # Store data in memory in addition to file.
	}
	}
#6

One thing to note about your example is that you are setting up a default WritePolicy but then you are actually calling client.operate(), which uses the OperatePolicy, so you defaults are not getting applied.

Since the number of operations in the command depends on the number of segments, roughly (order of magnitude) how many segments does one entry have on average? 10s? 100s? 1000s?

Do you expect there to be much latency between the client and server or are they in the same local network?

#7

Hi, thanks for your attention Based in the documentation operate and write policies have different scopes https://www.aerospike.com/apidocs/nodejs/policies_operate_policy.js.html Operate don’t have timeouts settings

number of items in the map: less 15, with an average of 6

Are on same dataserver I solved temporary adding a 5ms wait between each insert, but it isn’t a solution

#8

OperatePolicy extends BasePolicy, which is where socketTimeout, totalTimeout and maxRetries properties are defined: https://www.aerospike.com/apidocs/nodejs/OperatePolicy.html#totalTimeout. But in any case, that doesn’t explain the timeout error.

#9

Hello Kevin, some notice about this issue? Thanks

#10

A set of operations will only perform one disk read and, if it has a write op, asynchronously add to the write load. The only difference between a get/put is the operations which I would the disk read to dominate - so I don’t believe this number of ops would be an issue.

Could you provide output of sar -A.