| #!/bin/bash |
| # |
| # Copyright (c) 2015, 2018 Oracle and/or its affiliates. All rights reserved. |
| # |
| # This program and the accompanying materials are made available under the |
| # terms of the Eclipse Public License v. 2.0, which is available at |
| # http://www.eclipse.org/legal/epl-2.0. |
| # |
| # This Source Code may also be made available under the following Secondary |
| # Licenses when the conditions for such availability set forth in the |
| # Eclipse Public License v. 2.0 are satisfied: GNU General Public License, |
| # version 2 with the GNU Classpath Exception, which is available at |
| # https://www.gnu.org/software/classpath/license.html. |
| # |
| # SPDX-License-Identifier: EPL-2.0 OR GPL-2.0 WITH Classpath-exception-2.0 |
| # |
| |
| APP_LIST=(mbw-text-plain \ |
| mbw-json-moxy mbw-json-jackson \ |
| mbw-xml-moxy mbw-xml-jaxb \ |
| mbw-custom-provider \ |
| mbw-kryo \ |
| param-srl \ |
| filter-global filter-name filter-dynamic \ |
| interceptor-global interceptor-name interceptor-dynamic \ |
| proxy-injection) |
| |
| WARM_UP_SECONDS=300 |
| WAIT_FOR_APP_STARTUP_SEC=5 |
| WAIT_FOR_APP_RUNNING_SEC=60 |
| CHECK_RUNNER_INTERVAL=5 |
| CHECK_TERM_INTERVAL=10 |
| |
| JMX_URI_TEMPLATE="service:jmx:rmi:///jndi/rmi://SERVER_MACHINE:11112/jmxrmi" |
| SAMPLES=30 |
| |
| #uncomment for debug purposes: |
| #WARM_UP_SECONDS=3 |
| #WAIT_FOR_APP_STARTUP_SEC=2 |
| #WAIT_FOR_APP_RUNNING_SEC=1 |
| #CHECK_RUNNER_INTERVAL=1 |
| #CHECK_TERM_INTERVAL=2 |
| #SAMPLES=3 |
| #/debug |
| |
| MODULES_TO_BUILD="" |
| for app in ${APP_LIST[*]}; do |
| MODULES_TO_BUILD="$MODULES_TO_BUILD,tests/performance/test-cases/$app" |
| done |
| MODULES_TO_BUILD=`echo $MODULES_TO_BUILD|sed -e's/,//'` |
| |
| SERVER_LIST=() |
| CLIENT_LIST_ALL=() |
| |
| |
| function waitForGroupStatus() { |
| echo "########### Waiting for group status: $*" |
| RUNNER_ID=$1 |
| shift |
| GROUP_ID=$1 |
| shift |
| STATUS=$1 |
| |
| echo "#`date`" > $STATUS_DIR/.runner.$RUNNER_ID.waiting.$GROUP_ID.$STATUS |
| FILE="$STATUS_DIR/.group.$GROUP_ID.running.$RUNNER_ID.$STATUS" |
| |
| rm -f $STATUS_DIR/.group.$GROUP_ID.running.$RUNNER_ID.* |
| |
| available=false |
| while [ "$available" != true ]; do |
| if [ -e "$FILE" ]; then |
| available=true |
| else |
| sleep 1 |
| fi |
| done |
| } |
| |
| function releaseRunnerAndGroup() { |
| echo "########### Release Runner and Group: $*" |
| RUNNER_ID=$1 |
| shift |
| GROUP_ID=$1 |
| |
| echo "#`date`" > $STATUS_DIR/.runner.$RUNNER_ID.available |
| rm -f $STATUS_DIR/.group.$GROUP_ID.running.$RUNNER_ID.* $STATUS_DIR/.runner.$RUNNER_ID.running |
| } |
| |
| function checkWaitingRunners() { |
| echo "########### Check Waiting Runners" |
| for waiting_file in `ls $STATUS_DIR/.runner.*.waiting.*.lock 2> /dev/null`; do |
| filename_array=(`basename $waiting_file | tr "." " "`) |
| |
| runner_id=${filename_array[1]} |
| group_id=${filename_array[3]} |
| status=${filename_array[4]} |
| |
| _files=($STATUS_DIR/.group.$group_id.running.*) |
| if [ ! -f "${_files}" ]; then |
| echo "#`date`" > "$STATUS_DIR/.group.$group_id.running.$runner_id.lock" |
| rm $waiting_file |
| fi |
| done |
| |
| for waiting_file in `ls $STATUS_DIR/.runner.*.waiting.*.open 2> /dev/null`; do |
| filename_array=(`basename $waiting_file | tr "." " "`) |
| |
| runner_id=${filename_array[1]} |
| group_id=${filename_array[3]} |
| status=${filename_array[4]} |
| |
| _files=($STATUS_DIR/.group.$group_id.running.*.lock) |
| if [ ! -f "${_files}" ]; then |
| echo "#`date`" > "$STATUS_DIR/.group.$group_id.running.$runner_id.open" |
| rm $waiting_file |
| fi |
| done |
| } |
| |
| function createMachineFiles { |
| echo "########### Creating machine files in $STATUS_DIR for: $*" |
| RUNNER_ID=$1 |
| shift |
| GROUP_ID=$1 |
| shift |
| SERVER_MACHINE=$1 |
| shift |
| CLIENT_LIST=($@) |
| |
| echo ${GROUP_ID} > $STATUS_DIR/.runner.$RUNNER_ID.group |
| echo "#`date`" > $STATUS_DIR/.runner.$RUNNER_ID.available |
| echo ${SERVER_MACHINE} > $STATUS_DIR/.runner.$RUNNER_ID.server |
| echo ${CLIENT_LIST[@]} > $STATUS_DIR/.runner.$RUNNER_ID.clients |
| |
| SERVER_LIST=("${SERVER_LIST[@]}" "$SERVER_MACHINE") |
| CLIENT_LIST_ALL=("${CLIENT_LIST_ALL[@]}" "${CLIENT_LIST[@]}") |
| } |
| |
| function waitForTerminator { |
| echo "########### Waiting for finish" |
| # wait for the last round to finish |
| terminated=false |
| while [ "$terminated" != true ]; do |
| checkWaitingRunners |
| _files=($STATUS_DIR/.runner.*.running) |
| if [ ! -f "${_files}" ]; then |
| terminated=true |
| fi |
| if [ "$terminated" != true ]; then |
| echo "########### Terminated tests: $terminated, waiting $CHECK_TERM_INTERVAL sec..." |
| sleep $CHECK_TERM_INTERVAL |
| fi |
| done |
| |
| echo "DONE!" |
| |
| wait |
| sleep 4 |
| wait |
| } |
| |
| function testLoop { |
| # Following is the main measurement loop |
| # MEASUREMENT_DATA is the input data in the following format: |
| # application directory name|command line to generate load on client machines|JMX URI for the application|MBean name|output filename |
| |
| echo "########### Let's test it, reading from $MEASUREMENT_DATA file" |
| |
| TOTAL_COUNT=`cat $MEASUREMENT_DATA | grep -v "^#" | grep "^." | wc -l | sed -e 's/ *//g'` |
| INDEX=0 |
| cat $MEASUREMENT_DATA | grep -v "^#" | grep "^." | while IFS="\|" read app ab_cmdline app_class agent_param mbean filename |
| do |
| INDEX=`expr $INDEX + 1` |
| echo "========================================= DATA [$INDEX/$TOTAL_COUNT] ==============================================" |
| echo "app = $app" |
| echo "app_class = $app_class" |
| echo "ab_cmdline= $ab_cmdline" |
| |
| spawned=false |
| while [ "$spawned" != true ]; do |
| for runner_file in `ls $STATUS_DIR/.runner.*.available 2> /dev/null`; do |
| if [ "$spawned" != true ]; then |
| filename_array=(`basename $runner_file | tr "." " "`) |
| actual_runner=${filename_array[1]} |
| echo "#`date`" > $STATUS_DIR/.runner.$actual_runner.running |
| rm $runner_file |
| |
| SERVER_MACHINE=`cat $STATUS_DIR/.runner.$actual_runner.server` |
| APP_CONTEXT=$app |
| CLIENT_LIST=(`cat $STATUS_DIR/.runner.$actual_runner.clients`) |
| ab_cmdline=`echo $ab_cmdline | sed -e"s/SERVER_MACHINE/$SERVER_MACHINE/" | sed -e"s/SERVER_PORT/$SERVER_PORT/" | sed -e"s/APP_CONTEXT/$APP_CONTEXT/"` |
| JMX_URI=`echo $JMX_URI_TEMPLATE | sed -e"s/SERVER_MACHINE/$SERVER_MACHINE/"` |
| group_id=`cat $STATUS_DIR/.runner.$actual_runner.group` |
| |
| echo "#[$INDEX/$TOTAL_COUNT]" >> $STATUS_DIR/.runner.$actual_runner.running |
| echo "$app" >> $STATUS_DIR/.runner.$actual_runner.running |
| echo "$app_class" >> $STATUS_DIR/.runner.$actual_runner.running |
| echo "$ab_cmdline" >> $STATUS_DIR/.runner.$actual_runner.running |
| echo "$filename" >> $STATUS_DIR/.runner.$actual_runner.running |
| echo "$mbean" >> $STATUS_DIR/.runner.$actual_runner.running |
| echo "$agent_param" >> $STATUS_DIR/.runner.$actual_runner.running |
| |
| spawned=true |
| singleTest & |
| fi |
| done |
| checkWaitingRunners |
| if [ "$spawned" != true ]; then |
| sleep $CHECK_RUNNER_INTERVAL |
| fi |
| done |
| done |
| } |
| |
| function removeOldCapturedData { |
| rm -f $WORKSPACE/*.properties |
| } |
| |
| function retrieveJmxClient { |
| echo "########### Retrieving JMX client" |
| scp jerseyrobot@${SERVER_LIST[0]}:jmxclient.jar . |
| } |
| |
| function prepareClients { |
| echo "########### Copy client files to each client" |
| for CLIENT_MACHINE in ${CLIENT_LIST_ALL[@]}; do |
| echo "... copy client files to ${CLIENT_MACHINE}" |
| scp $WORKSPACE/jersey/tests/performance/etc/client/* jerseyrobot@${CLIENT_MACHINE}:. & |
| done |
| |
| wait |
| } |
| |
| function buildTestAppOnServers { |
| echo "########### Building test applications on each server" |
| # git fetch jersey on the server machine and build all apps there: |
| for SERVER_MACHINE in ${SERVER_LIST[@]}; do |
| ssh -n jerseyrobot@${SERVER_MACHINE} '(cd $HOME/workspace/jersey && '$GIT_FETCH_COMMAND' && mvn -pl '$MODULES_TO_BUILD' -am -Dskip.tests=true clean install)' & |
| done |
| |
| wait |
| } |
| |
| function cleanupHudsonSlave { |
| rm -f $STATUS_DIR/.runner.* $STATUS_DIR/.group.* |
| } |
| |
| function cleanupServer { |
| echo "########### Kill java processes on server $1" |
| ssh -n jerseyrobot@$1 'if ! test -e `ps h o pid -Cjava`; then kill -s TERM `ps h o pid -Cjava` ; fi' |
| } |
| |
| function cleanupServers { |
| echo "########### Kill all java processes on each server" |
| for SERVER_MACHINE in ${SERVER_LIST[@]}; do |
| cleanupServer ${SERVER_MACHINE} |
| done |
| } |
| |
| |
| trap "cleanupHudsonSlave; cleanupServers" EXIT SIGTERM SIGINT |
| |
| #uncomment for debug purposes: |
| #trap 'echo "[$BASH_SOURCE:$LINENO] $BASH_COMMAND" >> .debug; tail -10 .debug > .debug.swap; mv .debug.swap .debug' DEBUG |