pacemaker/extra/resources/docker-wrapper

#!/bin/bash
#
# Copyright (c) 2015 David Vossel <davidvossel@gmail.com>
#                    All Rights Reserved.
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of version 2 of the GNU General Public License as
# published by the Free Software Foundation.
#
# This program is distributed in the hope that it would be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
#
# Further, this software is distributed without any warranty that it is
# free of the rightful claim of any third person regarding infringement
# or the like.  Any license provided herein, whether implied or
# otherwise, applies only to this software file.  Patent licenses, if
# any, provided herein do not apply to combinations of this program with
# other software, or any other product whatsoever.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write the Free Software Foundation,
# Inc., 59 Temple Place - Suite 330, Boston MA 02111-1307, USA.
#

#######################################################################
# Initialization:

: ${OCF_FUNCTIONS_DIR=${OCF_ROOT}/lib/heartbeat}
. ${OCF_FUNCTIONS_DIR}/ocf-shellfuncs

#######################################################################

CONF_PREFIX="pcmk_docker"

meta_data() {
	cat <<END
<?xml version="1.0"?>
<!DOCTYPE resource-agent SYSTEM "ra-api-1.dtd">
<resource-agent name="docker-wrapper" version="1.0">
<version>1.0</version>

<longdesc lang="en">
Docker technology wrapper for pacemaker remote.
</longdesc>
<shortdesc lang="en">docker wrapper</shortdesc>

<parameters>
<parameter name="${CONF_PREFIX}_image" required="1" >
<longdesc lang="en">
Docker image to run resources within
</longdesc>
<shortdesc lang="en">docker image</shortdesc>
<content type="string" default="" />
</parameter>

<parameter name="${CONF_PREFIX}_privileged" >
<longdesc lang="en">
Give resources within container access to cluster resources
such as the CIB and the ability to manage cluster attributes.

NOTE: Do not confuse this with the docker run command's
'--priviledged' option which gives a container permission
to access system devices. To toggle the docker run option,
 set --priviledged=true as part of the ${CONF_PREFIX}_run_opts
arguments. The ${CONF_PREFIX}_privileged option only pertains
to whether or not the container has access to the cluster's
CIB or not. Some multistate resources need to be able to write
values to the cib, which would require enabling ${CONF_PREFIX}_privileged
</longdesc>
<shortdesc lang="en">is privileged</shortdesc>
<content type="string" default="false" />
</parameter>

<parameter name="${CONF_PREFIX}_run_opts" required="0" unique="0">
<longdesc lang="en">
Add options to be appended to the 'docker run' command which is used
when creating the container during the start action. This option allows
users to do things such as setting a custom entry point and injecting
environment variables into the newly created container. Note the '-d' 
option is supplied regardless of this value to force containers to run 
in the background.

NOTE: Do not explicitly specify the --name argument in the run_opts. This
agent will set --name using the resource's instance name

</longdesc>
<shortdesc lang="en">run options</shortdesc>
<content type="string"/>
</parameter>

<parameter name="${CONF_PREFIX}_reuse" required="0" unique="0">
<longdesc lang="en">
Allow the container to be reused after stopping the container. By default
containers are removed after stop. With the reuse option containers
will persist after the container stops.
</longdesc>
<shortdesc lang="en">reuse container</shortdesc>
<content type="boolean"/>
</parameter>

</parameters>

<actions>
<action name="start"        timeout="400" />
<action name="stop"         timeout="120" />
<action name="monitor"      timeout="60" interval="30" depth="0"/>
<action name="reload"       timeout="120" />
<action name="promote"      timeout="120s" />
<action name="demote"       timeout="120s" />
<action name="notify"       timeout="120" />
<action name="validate-all" timeout="30" />
<action name="meta-data"    timeout="10" />
</actions>
</resource-agent>
END
}

#######################################################################


CLIENT="/usr/libexec/pacemaker/lrmd_internal_ctl"
DOCKER_AGENT="/usr/lib/ocf/resource.d/heartbeat/docker"
KEY_VAL_STR=""
PROVIDER=$OCF_RESKEY_CRM_meta_provider
CLASS=$OCF_RESKEY_CRM_meta_class
TYPE=$OCF_RESKEY_CRM_meta_type

CONTAINER=$OCF_RESKEY_CRM_meta_isolation_instance
if [ -z "$CONTAINER" ]; then
	CONTAINER=$OCF_RESOURCE_INSTANCE
fi

RSC_STATE_DIR="${HA_RSCTMP}/docker-wrapper/${CONTAINER}-data/"
RSC_STATE_FILE="$RSC_STATE_DIR/$OCF_RESOURCE_INSTANCE.state" 
CONNECTION_FAILURE=0
HOST_LOG_DIR="${HA_RSCTMP}/docker-wrapper/${CONTAINER}-logs"
HOST_LOG_FILE="$HOST_LOG_DIR/pacemaker.log"
GUEST_LOG_DIR="/var/log/pcmk"
GUEST_LOG_FILE="$GUEST_LOG_DIR/pacemaker.log"

pcmk_docker_wrapper_usage() {
	cat <<END
usage: $0 {start|stop|monitor|reload|promote|demote|notify|validate-all|meta-data}

Expects to have a fully populated OCF RA-compliant environment set.
END
}

write_state_file()
{
	if ! [ -f "$RSC_STATE_FILE" ]; then
		[ -d "$RSC_STATE_DIR" ] || mkdir -p $RSC_STATE_DIR
		echo "$OCF_RESOURCE_INSTANCE" > $RSC_STATE_FILE
	fi
}

clear_state_file()
{
	if [ -f "$RSC_STATE_FILE" ]; then
		rm -f $RSC_STATE_FILE
	fi
}

clear_state_dir()
{
	[ -d "$RSC_STATE_DIR" ] || return 0

	rm -rf $RSC_STATE_DIR
}

num_active_resources()
{
	local count

	[ -d "$RSC_STATE_DIR" ] || return 0

	count="$(ls $RSC_STATE_DIR | wc -w)"
	if [ $? -ne 0 ] || [ -z "$count" ]; then
		return 0
	fi
	return $count
}

random_port()
{
	local port=$(python -c 'import socket; s=socket.socket(); s.bind(("localhost", 0)); print(s.getsockname()[1]); s.close()')
	if [ $? -eq 0 ] && [ -n "$port" ]; then
		echo "$port"
	fi
}

get_active_port()
{
	PORT="$(docker port $CONTAINER 3121 | awk -F: '{ print $2 }')"
}

# separate docker args from ocf resource args.
separate_args()
{
	local env key value

	# write out arguments to key value string for ocf agent
	while read -r line;
	do
		key="$(echo $line | awk -F= '{print $1}' |  sed 's/^OCF_RESKEY_//g')"
		val="$(echo $line | awk -F= '{print $2}')"
		KEY_VAL_STR="$KEY_VAL_STR -k '$key' -v '$val'"
	done < <(printenv | grep "^OCF.*" | grep -v "^OCF_RESKEY_${CONF_PREFIX}_.*")

	# sanitize args for DOCKER agent's consumption
	while read -r line;
	do
		env="$(echo $line | awk -F= '{print $1}')"
		val="$(echo $line | awk -F= '{print $2}')"
		key="$(echo "$env" | sed "s/^OCF_RESKEY_${CONF_PREFIX}/OCF_RESKEY/g")"
		export $key="$val"
	done < <(printenv | grep "^OCF_RESKEY_${CONF_PREFIX}_.*")

	if ocf_is_true $OCF_RESKEY_privileged ; then
		export OCF_RESKEY_run_cmd="/usr/sbin/pacemaker_remoted"
		# on start set random port to run_opts
		# write port to state file... or potentially get from ps? maybe docker info or inspect as well? 

	else 
		export OCF_RESKEY_run_cmd="/usr/libexec/pacemaker/lrmd"
	fi
	export OCF_RESKEY_name="$CONTAINER"
}

monitor_container()
{
	local rc

	$DOCKER_AGENT monitor
	rc=$?
	if [ $rc -ne $OCF_SUCCESS ]; then
		clear_state_dir
		return $rc
	fi

	poke_remote
	rc=$?
	if [ $rc -ne $OCF_SUCCESS ]; then
		# container is up without an active daemon. this is bad
		ocf_log err "Container, $CONTAINER, is active without a responsive pacemaker_remote instance"
		CONNECTION_FAILURE=1
		return $OCF_ERR_GENERIC
	fi
	CONNECTION_FAILURE=0

	return $rc
}

pcmk_docker_wrapper_monitor() {
	local rc

	monitor_container
	rc=$?
	if [ $rc -ne $OCF_SUCCESS ]; then
		return $rc
	fi
	
	client_action "monitor"
	rc=$?
	if [ $rc -eq $OCF_SUCCESS ] || [ $rc -eq $OCF_RUNNING_MASTER ]; then
		write_state_file
	else
		clear_state_file
	fi

	return $rc
}

pcmk_docker_wrapper_generic_action()
{
	local rc

	monitor_container
	rc=$?
	if [ $? -ne $OCF_SUCCESS ]; then
		return $rc
	fi

	client_action "$1"
}

client_action()
{
	local action=$1
	local agent_type="-T $TYPE -C $CLASS"
	local rc=0

	if [ -n "$PROVIDER" ]; then
		agent_type="$agent_type -P $PROVIDER"
	fi

	if ocf_is_true $OCF_RESKEY_privileged ; then
		if [ -z "$PORT" ]; then
			get_active_port
		fi
		export PCMK_logfile=$HOST_LOG_FILE
		ocf_log info "$CLIENT -c 'exec' -S '127.0.0.1' -p '$PORT' -a '$action' -r '$OCF_RESOURCE_INSTANCE' -n '$CONTAINER' '$agent_type' $KEY_VAL_STR "
		eval $CLIENT -c 'exec' -S '127.0.0.1' -p '$PORT' -a '$action' -r '$OCF_RESOURCE_INSTANCE' -n '$CONTAINER' '$agent_type' $KEY_VAL_STR
	else 
		export PCMK_logfile=$GUEST_LOG_FILE
		ocf_log info "$CLIENT -c \"exec\" -a $action -r \"$OCF_RESOURCE_INSTANCE\" $agent_type $KEY_VAL_STR"
		echo "$CLIENT -c \"exec\" -a $action -r \"$OCF_RESOURCE_INSTANCE\" $agent_type $KEY_VAL_STR " | nsenter --target $(docker inspect --format {{.State.Pid}} ${CONTAINER}) --mount --uts --ipc --net --pid
	fi
	rc=$?

	ocf_log debug "Client action $action with result $rc"
	return $rc
}

poke_remote()
{
	# verifies daemon in container is active
	if ocf_is_true $OCF_RESKEY_privileged ; then
		get_active_port
		ocf_log info "Attempting to contect $CONTAINER on port $PORT"
		$CLIENT -c "poke" -S "127.0.0.1" -p $PORT -n $CONTAINER
	fi
	# no op for non privileged containers since we handed the
	# client monitor action as the monitor_cmd for the docker agent
}

start_container()
{
	local rc

	monitor_container
	rc=$?
	if [ $rc -eq $OCF_SUCCESS ]; then
		return $rc
	fi

	mkdir -p $HOST_LOG_DIR
	export OCF_RESKEY_run_opts="-e PCMK_logfile=$GUEST_LOG_FILE $OCF_RESKEY_run_opts"
	export OCF_RESKEY_run_opts="-v $HOST_LOG_DIR:$GUEST_LOG_DIR $OCF_RESKEY_run_opts"
	if ocf_is_true $OCF_RESKEY_privileged ; then
		if ! [ -f "/etc/pacemaker/authkey" ]; then
			# generate an authkey if it doesn't exist. 
			mkdir -p /etc/pacemaker/
			dd if=/dev/urandom of=/etc/pacemaker/authkey bs=4096 count=1 > /dev/null 2>&1
			chmod 600 /etc/pacemaker/authkey
		fi

		PORT=$(random_port)
		if [ -z "$PORT" ]; then
			ocf_exit_reason "Unable to assign random port for pacemaker remote"
			return $OCF_ERR_GENERIC
		fi
		export OCF_RESKEY_run_opts="-p 127.0.0.1:${PORT}:3121 $OCF_RESKEY_run_opts"
		export OCF_RESKEY_run_opts="-v /etc/pacemaker/authkey:/etc/pacemaker/authkey $OCF_RESKEY_run_opts"
		ocf_log debug "using privileged mode: run_opts=$OCF_RESKEY_run_opts"
	else 
		export OCF_RESKEY_monitor_cmd="$CLIENT -c poke"
	fi

	$DOCKER_AGENT start
	rc=$?
	if [ $rc -ne $OCF_SUCCESS ]; then

		docker ps > /dev/null 2>&1
		if [ $? -ne 0 ]; then
			ocf_exit_reason "docker daemon is inactive."
		fi
		return $rc
	fi

	monitor_container
}

pcmk_docker_wrapper_start() {
	local rc

	start_container
	rc=$?
	if [ $rc -ne $OCF_SUCCESS ]; then
		return $rc
	fi

	client_action "start"
	rc=$?
	if [ $? -ne "$OCF_SUCCESS" ]; then
		ocf_exit_reason "Failed to start agent within container"
		return $rc
	fi

	pcmk_docker_wrapper_monitor
	rc=$?
	if [ $rc -eq $OCF_SUCCESS ]; then
		ocf_log notice "$OCF_RESOURCE_INSTANCE started successfully. Container's logfile can be found at $HOST_LOG_FILE"
	fi

	return $rc
}

stop_container()
{
	local rc
	local count

	num_active_resources
	count=$?
	if [ $count -ne 0 ]; then
		ocf_log err "Failed to stop agent within container. Killing container $CONTAINER with $count active resources"
	fi

	$DOCKER_AGENT "stop"
	rc=$?
	if [ $rc -ne $OCF_SUCCESS ]; then
		ocf_exit_reason "Docker container failed to stop"
		return $rc
	fi
	clear_state_dir
	return $rc
}

stop_resource()
{
	local rc

	client_action "stop"
	rc=$?
	if [ $? -ne "$OCF_SUCCESS" ]; then
		export OCF_RESKEY_force_stop="true"
		kill_now=1
	else 
		clear_state_file
	fi
}

pcmk_docker_wrapper_stop() {
	local rc
	local kill_now=0
	local all_stopped=0

	pcmk_docker_wrapper_monitor
	rc=$?
	if [ $rc -eq $OCF_NOT_RUNNING ]; then
		rc=$OCF_SUCCESS
		num_active_resources
		if [ $? -eq 0 ]; then
			# stop container if no more resources are running
			ocf_log info "Gracefully stopping container $CONTAINER because no resources are left running."
			stop_container
			rc=$?
		fi
		return $rc
	fi

	# if we can't talk to the remote daemon but the container is
	# active, we have to force kill the container.
	if [ $CONNECTION_FAILURE -eq 1 ]; then
		export OCF_RESKEY_force_kill="true"
		stop_container
		return $?
	fi


	# If we've gotten this far, the container is up, and we
	# need to gracefully stop a resource within the container.
	client_action "stop"
	rc=$?
	if [ $? -ne "$OCF_SUCCESS" ]; then
		export OCF_RESKEY_force_stop="true"
		# force kill the container if we fail to stop a resource.
		stop_container
		rc=$?
	else 
		clear_state_file
		num_active_resources
		if [ $? -eq 0 ]; then
			# stop container if no more resources are running
			ocf_log info "Gracefully stopping container $CONTAINER because last resource has stopped"
			stop_container
			rc=$?
		fi
	fi

	return $rc
}

pcmk_docker_wrapper_validate() {
	check_binary docker

	if [ -z "$CLASS" ] || [ -z "$TYPE" ]; then
		ocf_exit_reason "Update pacemaker to a version that supports container wrappers."
		return $OCF_ERR_CONFIGURED
	fi

	if ! [ -f "$DOCKER_AGENT" ]; then
		ocf_exit_reason "Requires $DOCKER_AGENT to be installed. update the resource-agents package"
		return $OCF_ERR_INSTALLED
	fi
	$DOCKER_AGENT validate-all
	return $?
}

case $__OCF_ACTION in
meta-data)	meta_data
		exit $OCF_SUCCESS
		;;
usage|help)	pcmk_docker_wrapper_usage
		exit $OCF_SUCCESS
		;;
esac

separate_args
pcmk_docker_wrapper_validate
rc=$?
if [ $rc -ne 0 ]; then
	case $__OCF_ACTION in
		stop) exit $OCF_SUCCESS;;
		monitor) exit $OCF_NOT_RUNNING;;
	*) exit $rc;;
	esac
fi

case $__OCF_ACTION in
	start)		pcmk_docker_wrapper_start;;
	stop)		pcmk_docker_wrapper_stop;;
	monitor|status)	pcmk_docker_wrapper_monitor;;
	reload|promote|demote|notify)
			pcmk_docker_wrapper_generic_action $__OCF_ACTION;;
	validate-all)	pcmk_docker_wrapper_validate;;
	*)		pcmk_docker_wrapper_usage
		exit $OCF_ERR_UNIMPLEMENTED
		;;
esac
rc=$?
ocf_log debug "Docker-wrapper ${OCF_RESOURCE_INSTANCE} $__OCF_ACTION : $rc"
exit $rc