Difference between revisions of "KeystonePerformance"
m |
(Add KeystonePerformance tests for UUID vs PKI, SQL vs LDAP) |
||
Line 22: | Line 22: | ||
=== Improvement with multi-core keystone service === | === Improvement with multi-core keystone service === | ||
− | === Compare PKI vs UUID === | + | === Compare PKI vs UUID, SQL vs LDAP === |
+ | ==== Methodology ==== | ||
+ | In this story we will be measuring keystone performance for those sequencial tests: | ||
+ | * User creation | ||
+ | * User token generation | ||
+ | * User token validation | ||
− | === | + | ===== Test setup ===== |
+ | Devstack on ubuntu server 13.10, with most services disabled (beside keystone, mysql and ldap): | ||
+ | <pre> | ||
+ | for service in g-api g-reg n-api n-crt n-obj n-cpu n-net n-cond \ | ||
+ | cinder c-sch c-api c-vol n-sch n-novnc n-xvnc n-cauth \ | ||
+ | horizon rabbit tempest; do | ||
+ | echo disable_service $service >> localrc | ||
+ | done | ||
+ | </pre> | ||
+ | |||
+ | Added at the end of localrc: | ||
+ | <pre> | ||
+ | ADMIN_PASSWORD=nomoresecrete | ||
+ | SERVICE_TOKEN=ADMIN | ||
+ | OFFLINE=True | ||
+ | |||
+ | KEYSTONE_TOKEN_FORMAT=UUID | ||
+ | KEYSTONE_IDENTITY_BACKEND=ldap | ||
+ | |||
+ | enable_service ldap | ||
+ | KEYSTONE_CLEAR_LDAP=yes | ||
+ | LDAP_PASSWORD=ldappass | ||
+ | </pre> | ||
+ | |||
+ | ===== Test scripts ===== | ||
+ | User creation (perf-create-users.sh): | ||
+ | <pre> | ||
+ | #!/bin/sh | ||
+ | |||
+ | export OS_SERVICE_TOKEN=ADMIN | ||
+ | export OS_SERVICE_ENDPOINT=http://localhost:35357/v2.0 | ||
+ | for i in $(seq 100); do | ||
+ | keystone user-create --name perf_testuser_$(printf "%04d" $i) \ | ||
+ | --tenant demo --pass demopass 2>&1 > /dev/null | ||
+ | if [ "$?" != 0 ]; then | ||
+ | echo "User creation failed" | ||
+ | break | ||
+ | fi | ||
+ | done | ||
+ | </pre> | ||
+ | |||
+ | Generate user's tokens list (perf-gen-tokens.sh): | ||
+ | <pre> | ||
+ | #!/bin/sh | ||
+ | |||
+ | get_id () { | ||
+ | echo `"$@" | awk '/ id / { print $4 }'` | ||
+ | } | ||
+ | |||
+ | unset OS_SERVICE_TOKEN OS_SERVICE_ENDPOINT | ||
+ | export OS_AUTH_URL=http://localhost:5000/v2.0 | ||
+ | export OS_TENANT_NAME=demo | ||
+ | export OS_PASSWORD=demopass | ||
+ | echo -n > /tmp/tokens_list.txt | ||
+ | for i in $(seq 100); do | ||
+ | export OS_USERNAME=perf_testuser_$(printf "%04d" $i)· | ||
+ | get_id keystone token-get >> /tmp/tokens_list.txt | ||
+ | if [ "$?" != 0 ]; then | ||
+ | echo "Token get failed" | ||
+ | break | ||
+ | fi | ||
+ | done | ||
+ | </pre> | ||
+ | |||
+ | Token validation (perf-validate-tokens.sh): | ||
+ | <pre> | ||
+ | #!/bin/sh | ||
+ | |||
+ | get_id () { | ||
+ | echo `"$@" | awk '/ id / { print $4 }'` | ||
+ | } | ||
+ | unset OS_SERVICE_TOKEN OS_SERVICE_ENDPOINT | ||
+ | export OS_AUTH_URL=http://localhost:5000/v2.0 | ||
+ | export OS_TENANT_NAME=demo | ||
+ | export OS_PASSWORD=nomoresecrete | ||
+ | export OS_USERNAME=admin | ||
+ | ADMIN_TOKEN=$(get_id keystone token-get) | ||
+ | while read USER_TOKEN; do | ||
+ | curl -H "X-Auth-Token:${ADMIN_TOKEN}" http://0.0.0.0:5000/v2.0/tokens/${USER_TOKEN} \ | ||
+ | 2>&1 | grep "issued_at" > /dev/null | ||
+ | if [ "$?" != 0 ]; then | ||
+ | echo "Token validation failed" | ||
+ | break | ||
+ | fi | ||
+ | done < /tmp/tokens_list.txt | ||
+ | </pre> | ||
+ | |||
+ | ==== Performance data acquisition ==== | ||
+ | |||
+ | This script will write real time (userland + kernelland) in /tmp/perf-test_name-TOKEN_FORMAT-BACKEND: | ||
+ | <pre> | ||
+ | #!/bin/sh | ||
+ | |||
+ | cd ~/devstack | ||
+ | for TOKEN_FORMAT in UUID PKI; do | ||
+ | for BACKEND in ldap sql; do | ||
+ | echo "== ${TOKEN_FORMAT} - ${BACKEND} ==" | ||
+ | echo "[+] Unstack // stack..." | ||
+ | ./unstack.sh | ||
+ | |||
+ | sed -e "s/^KEYSTONE_TOKEN_FORMAT=.*/KEYSTONE_TOKEN_FORMAT=${TOKEN_FORMAT}/" \ | ||
+ | -e "s/^KEYSTONE_IDENTITY_BACKEND=.*/KEYSTONE_IDENTITY_BACKEND=${BACKEND}/" \ | ||
+ | -i localrc | ||
+ | if [ "${BACKEND}" = "ldap" ]; then | ||
+ | sed -e "s/^disable_service ldap$/enable_service ldap/" -i localrc | ||
+ | else | ||
+ | sed -e "s/^enable_service ldap$/disable_service ldap/" -i localrc | ||
+ | fi | ||
+ | ./stack.sh 2>&1 > /dev/null | ||
+ | if [ "$?" != 0 ]; then | ||
+ | echo "Stack.sh failed" | ||
+ | exit 1 | ||
+ | fi | ||
+ | |||
+ | echo "[+] Create users..." | ||
+ | PERF_OUTPUT="/tmp/perf-create-users_${TOKEN_FORMAT}_${BACKEND}" | ||
+ | /usr/bin/time -o ${PERF_OUTPUT} -f "%e" -a ~/bin/perf-create-users.sh | ||
+ | echo -n "-> "; tail -n 1 ${PERF_OUTPUT} | ||
+ | sleep 1 | ||
+ | |||
+ | echo "[+] Generate tokens..." | ||
+ | PERF_OUTPUT="/tmp/perf-gen-tokens_${TOKEN_FORMAT}_${BACKEND}" | ||
+ | /usr/bin/time -o ${PERF_OUTPUT} -f "%e" -a ~/bin/perf-gen-tokens.sh | ||
+ | echo -n "-> "; tail -n 1 ${PERF_OUTPUT} | ||
+ | sleep 1 | ||
+ | |||
+ | echo "[+] Validate tokens..." | ||
+ | PERF_OUTPUT="/tmp/perf-validate-tokens_${TOKEN_FORMAT}_${BACKEND}" | ||
+ | /usr/bin/time -o ${PERF_OUTPUT} -f "%e" -a ~/bin/perf-validate-tokens.sh | ||
+ | echo -n "-> "; tail -n 1 ${PERF_OUTPUT} | ||
+ | sleep 1 | ||
+ | done | ||
+ | done | ||
+ | </pre> | ||
+ | |||
+ | ==== Performances results ==== | ||
+ | After running the data acquisition several time, here is the average observed time: | ||
+ | {| class="wikitable" | ||
+ | |- | ||
+ | | || LDAP || SQL | ||
+ | |- | ||
+ | | UUID || || | ||
+ | |- | ||
+ | | PKI || || | ||
+ | |} |
Revision as of 15:54, 16 December 2013
Contents
Keystone Performance
This is to track the performance work related to Keystone.
Work Items
Identify CPU, Disk, Memory, Database bottlenecks
Test #1, Create users in parallel and look for CPU, disk or memory bottleneck.
Methodology
- Install RDO Havana Stable [1] on a bare metal.
- Create one instance of m1.medium flavor and other of type m1.large, so that we can have different CPU and memory config
- Install Keystone Manually (from RDO release) on both of the above created instances
- Using python multiprocessing module create users in parallel using keystoneclient.v2_0 module on each one of them.
- key.users.create(<user>, "test", "test@test.com") where key = client.Client( .... )
- Collect the CPU, Disk, Memory and Database related stats while user creation is in progress.
Effect of caching - memcached
Effect of expired tokens
Overhead of syncing revocation list
Improvement with multi-core keystone service
Compare PKI vs UUID, SQL vs LDAP
Methodology
In this story we will be measuring keystone performance for those sequencial tests:
- User creation
- User token generation
- User token validation
Test setup
Devstack on ubuntu server 13.10, with most services disabled (beside keystone, mysql and ldap):
for service in g-api g-reg n-api n-crt n-obj n-cpu n-net n-cond \ cinder c-sch c-api c-vol n-sch n-novnc n-xvnc n-cauth \ horizon rabbit tempest; do echo disable_service $service >> localrc done
Added at the end of localrc:
ADMIN_PASSWORD=nomoresecrete SERVICE_TOKEN=ADMIN OFFLINE=True KEYSTONE_TOKEN_FORMAT=UUID KEYSTONE_IDENTITY_BACKEND=ldap enable_service ldap KEYSTONE_CLEAR_LDAP=yes LDAP_PASSWORD=ldappass
Test scripts
User creation (perf-create-users.sh):
#!/bin/sh export OS_SERVICE_TOKEN=ADMIN export OS_SERVICE_ENDPOINT=http://localhost:35357/v2.0 for i in $(seq 100); do keystone user-create --name perf_testuser_$(printf "%04d" $i) \ --tenant demo --pass demopass 2>&1 > /dev/null if [ "$?" != 0 ]; then echo "User creation failed" break fi done
Generate user's tokens list (perf-gen-tokens.sh):
#!/bin/sh get_id () { echo `"$@" | awk '/ id / { print $4 }'` } unset OS_SERVICE_TOKEN OS_SERVICE_ENDPOINT export OS_AUTH_URL=http://localhost:5000/v2.0 export OS_TENANT_NAME=demo export OS_PASSWORD=demopass echo -n > /tmp/tokens_list.txt for i in $(seq 100); do export OS_USERNAME=perf_testuser_$(printf "%04d" $i)· get_id keystone token-get >> /tmp/tokens_list.txt if [ "$?" != 0 ]; then echo "Token get failed" break fi done
Token validation (perf-validate-tokens.sh):
#!/bin/sh get_id () { echo `"$@" | awk '/ id / { print $4 }'` } unset OS_SERVICE_TOKEN OS_SERVICE_ENDPOINT export OS_AUTH_URL=http://localhost:5000/v2.0 export OS_TENANT_NAME=demo export OS_PASSWORD=nomoresecrete export OS_USERNAME=admin ADMIN_TOKEN=$(get_id keystone token-get) while read USER_TOKEN; do curl -H "X-Auth-Token:${ADMIN_TOKEN}" http://0.0.0.0:5000/v2.0/tokens/${USER_TOKEN} \ 2>&1 | grep "issued_at" > /dev/null if [ "$?" != 0 ]; then echo "Token validation failed" break fi done < /tmp/tokens_list.txt
Performance data acquisition
This script will write real time (userland + kernelland) in /tmp/perf-test_name-TOKEN_FORMAT-BACKEND:
#!/bin/sh cd ~/devstack for TOKEN_FORMAT in UUID PKI; do for BACKEND in ldap sql; do echo "== ${TOKEN_FORMAT} - ${BACKEND} ==" echo "[+] Unstack // stack..." ./unstack.sh sed -e "s/^KEYSTONE_TOKEN_FORMAT=.*/KEYSTONE_TOKEN_FORMAT=${TOKEN_FORMAT}/" \ -e "s/^KEYSTONE_IDENTITY_BACKEND=.*/KEYSTONE_IDENTITY_BACKEND=${BACKEND}/" \ -i localrc if [ "${BACKEND}" = "ldap" ]; then sed -e "s/^disable_service ldap$/enable_service ldap/" -i localrc else sed -e "s/^enable_service ldap$/disable_service ldap/" -i localrc fi ./stack.sh 2>&1 > /dev/null if [ "$?" != 0 ]; then echo "Stack.sh failed" exit 1 fi echo "[+] Create users..." PERF_OUTPUT="/tmp/perf-create-users_${TOKEN_FORMAT}_${BACKEND}" /usr/bin/time -o ${PERF_OUTPUT} -f "%e" -a ~/bin/perf-create-users.sh echo -n "-> "; tail -n 1 ${PERF_OUTPUT} sleep 1 echo "[+] Generate tokens..." PERF_OUTPUT="/tmp/perf-gen-tokens_${TOKEN_FORMAT}_${BACKEND}" /usr/bin/time -o ${PERF_OUTPUT} -f "%e" -a ~/bin/perf-gen-tokens.sh echo -n "-> "; tail -n 1 ${PERF_OUTPUT} sleep 1 echo "[+] Validate tokens..." PERF_OUTPUT="/tmp/perf-validate-tokens_${TOKEN_FORMAT}_${BACKEND}" /usr/bin/time -o ${PERF_OUTPUT} -f "%e" -a ~/bin/perf-validate-tokens.sh echo -n "-> "; tail -n 1 ${PERF_OUTPUT} sleep 1 done done
Performances results
After running the data acquisition several time, here is the average observed time:
LDAP | SQL | |
UUID | ||
PKI |