pacemaker/cts/lxc_autogen.sh.in

#!/bin/bash

containers="2"
download=0
share_configs=0
# different than default libvirt network in case this is run nested in a KVM instance
addr="192.168.123.1"
restore=0
restore_pcmk=0
restore_all=0
generate=0
key_gen=0
cib=0
anywhere=0
add_master=0
verify=0
working_dir="@CRM_CONFIG_CTS@/lxc"
run_dirs="/run /var/run /usr/var/run"

SSH_CMD_OPTS="
	-o StrictHostKeyChecking=no
	-o ConnectTimeout=30
	-o BatchMode=yes
	-l root
	-T
"
# must be on one line b/c used inside quotes
SSH_RSYNC_OPTS="-o UserKnownHostsFile=/dev/null -o BatchMode=yes -o StrictHostKeyChecking=no"

function helptext() {
	echo "lxc_autogen.sh - A tool for generating libvirt lxc containers for testing purposes."
	echo ""
	echo "Usage: lxc-autogen [options]"
	echo ""
	echo "Options:"
	echo "-g, --generate         Generate libvirt lxc environment in the directory this script is run from."
	echo "-k, --key-gen          Generate pacemaker remote key only."
	echo "-r, --restore-libvirt  Restore the default network, and libvirt config to before this script ran."
	echo "-p, --restore-cib      Remove cib entries this script generated."
	echo "-R, --restore-all      Restore both libvirt and cib plus clean working directory. This will leave libvirt xml files though so rsc can be stopped properly."
	echo ""
	echo "-A, --allow-anywhere   Allow the containers to live anywhere in the cluster"
	echo "-a, --add-cib          Add remote-node entries for each lxc instance into the cib"
	echo "-m, --add-master       Add master resource shared between remote-nodes"
	echo "-d, --download-agent   Download and install the latest VirtualDomain agent."
	echo "-s, --share-configs    Synchronize on all known cluster nodes"
	echo "-c, --containers       Specify the number of containers to generate, defaults to $containers. Used with -g"
	echo "-n, --network          What network to override default libvirt network to. Example: -n 192.168.123.1. Used with -g"
	echo "-v, --verify           Verify environment is capable of running lxc"
	echo ""
	exit $1
}

while true ; do
	case "$1" in
	--help|-h|-\?) helptext 0;;
	-c|--containers) containers="$2"; shift; shift;;
	-d|--download-agent) download=1; shift;;
	-s|--share-configs) share_configs=1; shift;;
	-n|--network) addr="$2"; shift; shift;;
	-r|--restore-libvirt) restore=1; shift;;
	-p|--restore-cib) restore_pcmk=1; shift;;
	-R|--restore-all)
		restore_all=1
		restore=1
		restore_pcmk=1
		shift;;
	-g|--generate) generate=1; key_gen=1; shift;;
	-k|--key-gen) key_gen=1; shift;;
	-a|--add-cib) cib=1; shift;;
	-A|--allow-anywhere) anywhere=1; shift;;
	-m|--add-master) add_master=1; shift;;
	-v|--verify) verify=1; shift;;
	"") break;;
	*) helptext 1;;
	esac
done

if [ $verify -eq 1 ]; then
	# verify virsh tool is available and that 
	# we can connect to lxc driver.
	virsh -c lxc:/// list --all > /dev/null 2>&1
	if [ $? -ne 0 ]; then
		echo "Could not connect 'virsh -c lxc:///' check that libvirt lxc driver is installed"
		# yum install -y libvirt-daemon-driver-lxc libvirt-daemon-lxc libvirt-login-shell
		exit 1
	fi


	cat /etc/selinux/config  | grep -e "SELINUX.*=.*permissive" -e "SELINUX.*=.*enforcing" > /dev/null 2>&1
	if [ $? -ne 0 ]; then
		echo "/etc/selinux/config must have SELINUX set to permissive or enforcing mode."
		exit 1
	fi

	ps x > /tmp/lxc-autogen-libvirt-test.txt
	grep "libvirtd" /tmp/lxc-autogen-libvirt-test.txt
	if [ $? -ne 0 ]; then
		rm -f /tmp/lxc-autogen-libvirt-test.txt
		echo "libvirtd isn't up."
		exit 1
	fi
	rm -f /tmp/lxc-autogen-libvirt-test.txt

	which rsync > /dev/null 2>&1
	if [ $? -ne 0 ]; then
		echo "rsync is required"
	fi

	which pacemaker_remoted > /dev/null 2>&1
	if [ $? -ne 0 ]; then
		echo "pacemaker_remoted is required"
	fi
fi

#strip last digits off addr
addr=$(echo $addr | awk -F. '{print $1"."$2"."$3}')

this_node()
{
	crm_node -n
}

other_nodes()
{
	crm_node -l | awk "\$2 != \"$(this_node)\" {print \$2}"
}

make_directory()
{
	# argument must be full path
	DIR="$1"

	mkdir -p "$DIR"
	if [ $share_configs -eq 1 ]; then
		for node in $(other_nodes); do
			ssh $SSH_CMD_OPTS $node mkdir -p "$DIR"
		done
	fi
}

sync_file()
{
	TARGET="$1"

	if [ $share_configs -eq 1 ]; then
		for node in $(other_nodes); do
			rsync -ave "ssh $SSH_RSYNC_OPTS" "$TARGET" "${node}:${TARGET}"
		done
	fi
}

download_agent()
{
	wget https://raw.github.com/ClusterLabs/resource-agents/master/heartbeat/VirtualDomain
	chmod 755 VirtualDomain
	mv -f VirtualDomain /usr/lib/ocf/resource.d/heartbeat/VirtualDomain
	sync_file /usr/lib/ocf/resource.d/heartbeat/VirtualDomain
}

set_network()
{
	rm -f cur_network.xml
	cat << END >> cur_network.xml
<network>
  <name>default</name>
  <uuid>41ebdb84-7134-1111-a136-91f0f1119225</uuid>
  <forward mode='nat'/>
  <bridge name='virbr0' stp='on' delay='0' />
  <mac address='52:54:00:A8:12:35'/>
  <ip address='$addr.1' netmask='255.255.255.0'>
    <dhcp>
      <range start='$addr.2' end='$addr.254' />
    </dhcp>
  </ip>
</network>
END
	sync_file ${working_dir}/cur_network.xml
}

distribute_configs()
{
	for node in $(other_nodes); do
		rsync -ave "ssh $SSH_RSYNC_OPTS" ${working_dir}/lxc*.xml ${node}:${working_dir}
		rsync -ave "ssh $SSH_RSYNC_OPTS" ${working_dir}/lxc*-filesystem ${node}:${working_dir}
	done
}

start_network()
{
	NODE="$1"

	ssh $SSH_CMD_OPTS $NODE <<-EOF
	cd $working_dir
	virsh net-info default >/dev/null 2>&1
	if [ \$? -eq 0 ]; then
		if [ ! -f restore_default.xml ]; then
			virsh net-dumpxml default > restore_default.xml
		fi
		virsh net-destroy default
		virsh net-undefine default
	fi
	virsh net-define cur_network.xml
	virsh net-start default
	virsh net-autostart default
EOF
}

start_network_all()
{
	start_network $(this_node)
	if [ $share_configs -eq 1 ]; then
		for node in $(other_nodes); do
			start_network $node
		done
	fi
}

add_hosts_entry()
{
	IP="$1"
	HNAME="$2"

	echo $IP $HNAME >>/etc/hosts
	if [ $share_configs -eq 1 ]; then
		for node in $(other_nodes); do
			ssh $SSH_CMD_OPTS $node "echo $IP $HNAME >>/etc/hosts"
		done
	fi
}

generate_key()
{
	if [ ! -e /etc/pacemaker/authkey ]; then
		make_directory /etc/pacemaker
		dd if=/dev/urandom of=/etc/pacemaker/authkey bs=4096 count=1
		sync_file /etc/pacemaker/authkey
	fi
}

generate()
{
	set_network

	# Generate libvirt domains in xml
	for (( c=1; c <= $containers; c++ ))
	do
		# Clean any previous definition
		rm -rf lxc$c.xml lxc$c-filesystem

		# Create a basic filesystem with run directories
		for dir in $run_dirs; do
			mkdir -p lxc$c-filesystem/$dir
		done

		# Create libvirt definition
		suffix=$((10 + $c))
		prefix=$(echo $addr | awk -F. '{print $1"."$2}')
		subnet=$(echo $addr | awk -F. '{print $3}')
		while [ $suffix -gt 255 ]; do
		    subnet=$(($subnet + 1))
		    suffix=$(($subnet - 255))
		done
		cip=$prefix.$subnet.$suffix

		cat << END >> lxc$c.xml
<domain type='lxc'>
  <name>lxc$c</name>
  <memory unit='KiB'>200704</memory>
  <os>
    <type>exe</type>
    <init>$working_dir/lxc$c-filesystem/launch-helper</init>
  </os>
  <devices>
    <console type='pty'/>
    <filesystem type='ram'>
        <source usage='150528'/>
        <target dir='/dev/shm'/>
    </filesystem>
END
		for dir in $run_dirs; do
			cat << END >> lxc$c.xml
    <filesystem type='mount'>
      <source dir='$working_dir/lxc$c-filesystem${dir}'/>
      <target dir='$dir'/>
    </filesystem>
END
		done
		cat << END >> lxc$c.xml
    <interface type='network'>
      <mac address='52:54:$(($RANDOM % 9))$(($RANDOM % 9)):$(($RANDOM % 9))$(($RANDOM % 9)):$(($RANDOM % 9))$(($RANDOM % 9)):$(($RANDOM % 9))$(($RANDOM % 9))'/>
      <source network='default'/>
    </interface>
  </devices>
</domain>
END

		# Create CIB definition
		rm -f container$c.cib
		cat << END >> container$c.cib
      <primitive class="ocf" id="container$c" provider="heartbeat" type="VirtualDomain">
        <instance_attributes id="container$c-instance_attributes">
          <nvpair id="container$c-instance_attributes-force_stop" name="force_stop" value="true"/>
          <nvpair id="container$c-instance_attributes-hypervisor" name="hypervisor" value="lxc:///"/>
          <nvpair id="container$c-instance_attributes-config" name="config" value="$working_dir/lxc$c.xml"/>
        </instance_attributes>
        <utilization id="container$c-utilization">
          <nvpair id="container$c-utilization-cpu" name="cpu" value="1"/>
          <nvpair id="container$c-utilization-hv_memory" name="hv_memory" value="100"/>
        </utilization>
        <meta_attributes id="container$c-meta_attributes">
          <nvpair id="container$c-meta_attributes-remote-node" name="remote-node" value="lxc$c"/>
        </meta_attributes>
      </primitive>
END

		# Create container init
		rm -f lxc$c-filesystem/launch-helper
		cat << END >> lxc$c-filesystem/launch-helper
#!/bin/bash
ip -f inet addr add $cip/24 dev eth0
ip link set eth0 up
ip route add default via $addr.1
hostname lxc$c
df > $working_dir/lxc$c-filesystem/disk_usage.txt
export PCMK_debugfile=/var/log/pacemaker_remote_lxc$c.log
/usr/sbin/pacemaker_remoted
END
		chmod 711 lxc$c-filesystem/launch-helper

		add_hosts_entry $cip lxc$c
	done

	# Create CIB fragment for a master-slave resource
	rm -f lxc-ms.cib
	cat << END >> lxc-ms.cib
      <master id="lxc-ms-master">
        <primitive class="ocf" id="lxc-ms" provider="pacemaker" type="Stateful">
          <instance_attributes id="lxc-ms-instance_attributes"/>
          <operations>
            <op id="lxc-ms-monitor-interval-10s" interval="10s" name="monitor"/>
          </operations>
        </primitive>
        <meta_attributes id="lxc-ms-meta_attributes">
          <nvpair id="lxc-ms-meta_attributes-master-max" name="master-max" value="1"/>
          <nvpair id="lxc-ms-meta_attributes-clone-max" name="clone-max" value="$containers"/>
        </meta_attributes>
      </master>
END

}

apply_cib_master()
{
	cibadmin -Q > cur.cib
	export CIB_file=cur.cib

	cibadmin -o resources -Mc -x lxc-ms.cib
	for tmp in $(ls lxc*.xml | sed -e 's/\.xml//g'); do
		echo "<rsc_location id=\"lxc-ms-location-${tmp}\" node=\"${tmp}\" rsc=\"lxc-ms-master\" score=\"INFINITY\"/>" > tmp_constraint
		cibadmin -o constraints -Mc -x tmp_constraint
	done
	# Make sure the version changes even if the content doesn't
	cibadmin -B
	unset CIB_file

	cibadmin --replace -o configuration --xml-file cur.cib
	rm -f cur.cib
}

apply_cib_entries()
{
	cibadmin -Q > cur.cib
	export CIB_file=cur.cib
	for tmp in $(ls container*.cib); do
		cibadmin -o resources -Mc -x $tmp

		remote_node=$(cat ${tmp} | grep remote-node | sed -n -e 's/^.*value=\"\(.*\)\".*/\1/p')
		if [ $anywhere -eq 0 ]; then
			tmp=$(echo $tmp | sed -e 's/\.cib//g')
			crm_resource -M -r $tmp -H $(this_node)
		fi
		echo "<rsc_location id=\"lxc-ping-location-${remote_node}\" node=\"${remote_node}\" rsc=\"Connectivity\" score=\"-INFINITY\"/>" > tmp_constraint
		# it's fine if applying this constraint fails. it's just to help with cts
		# when the connectivity resources are in use. those resources fail the remote-nodes.
		cibadmin -o constraints -Mc -x tmp_constraint > /dev/null 2>&1

		for rsc in $(crm_resource -l | grep rsc_ ); do
			echo "<rsc_location id=\"lxc-${rsc}-location-${remote_node}\" node=\"${remote_node}\" rsc=\"${rsc}\" score=\"-INFINITY\"/>" > tmp_constraint
			cibadmin -o constraints -Mc -x tmp_constraint > /dev/null 2>&1
		done

		rm -f tmp_constraint
	done

	# Make sure the version changes even if the content doesn't
	cibadmin -B

	unset CIB_file

	cibadmin --replace -o configuration --xml-file cur.cib
	rm -f cur.cib
}

restore_cib()
{
	cibadmin -Q > cur.cib
	export CIB_file=cur.cib

	for tmp in $(ls lxc*.xml | sed -e 's/\.xml//g'); do
		echo "<rsc_location id=\"lxc-ms-location-${tmp}\" node=\"${tmp}\" rsc=\"lxc-ms-master\" score=\"INFINITY\"/>" > tmp_constraint
		cibadmin -o constraints -D -x tmp_constraint
		echo "<rsc_location id=\"lxc-ping-location-${tmp}\" node=\"${tmp}\" rsc=\"Connectivity\" score=\"-INFINITY\"/>" > tmp_constraint
		cibadmin -o constraints -D -x tmp_constraint

		for rsc in $(crm_resource -l | grep rsc_ ); do
			echo "<rsc_location id=\"lxc-${rsc}-location-${tmp}\" node=\"${tmp}\" rsc=\"${rsc}\" score=\"-INFINITY\"/>" > tmp_constraint
			cibadmin -o constraints -D -x tmp_constraint
		done
		rm -f tmp_constraint
	done
	cibadmin -o resources -D -x lxc-ms.cib

	for tmp in $(ls container*.cib); do
		tmp=$(echo $tmp | sed -e 's/\.cib//g')
		crm_resource -U -r $tmp -H $(this_node)
		crm_resource -D -r $tmp -t primitive
	done
	# Make sure the version changes even if the content doesn't
	cibadmin -B
	unset CIB_file

	cibadmin --replace -o configuration --xml-file cur.cib
	rm -f  cur.cib 

	# Allow the cluster to stabilize before continuing
	crm_resource --wait

	# Purge nodes from caches and CIB status section
	for tmp in $(ls lxc*.xml | sed -e 's/\.xml//g'); do
		crm_node --force --remove $tmp
	done
}

restore_network()
{
	NODE="$1"

	ssh $SSH_CMD_OPTS $NODE <<-EOF
	cd $working_dir
	for tmp in \$(ls lxc*.xml | sed -e 's/\.xml//g'); do
		virsh -c lxc:/// destroy \$tmp >/dev/null 2>&1
		virsh -c lxc:/// undefine \$tmp >/dev/null 2>&1
		sed -i.bak "/...\....\....\..* \${tmp}/d" /etc/hosts
	done
	virsh net-destroy default >/dev/null 2>&1
	virsh net-undefine default >/dev/null 2>&1
	if [ -f restore_default.xml ]; then
		virsh net-define restore_default.xml
		virsh net-start default
		rm restore_default.xml
	fi
EOF
	echo "Containers destroyed and default network restored on $NODE"
}

restore_libvirt()
{
	restore_network $(this_node)
	if [ $share_configs -eq 1 ]; then
		for node in $(other_nodes); do
			restore_network $node
		done
	fi
}

restore_files()
{
	ls | grep -v "lxc.\.xml" | xargs rm -rf
	if [ $share_configs -eq 1 ]; then
		for node in $(other_nodes); do
			ssh $SSH_CMD_OPTS $node rm -rf \
				$working_dir/lxc*-filesystem \
				$working_dir/cur_network.xml
		done
	fi
}

make_directory $working_dir
cd $working_dir

# Generate files as requested
if [ $download -eq 1 ]; then
	download_agent
fi
if [ $key_gen -eq 1 ]; then
	generate_key
fi
if [ $generate -eq 1 ]; then
	generate
fi
if [ $share_configs -eq 1 ]; then
	distribute_configs
fi
if [ $generate -eq 1 ]; then
	start_network_all
fi

# Update cluster as requested
if [ $cib -eq 1 ]; then
	apply_cib_entries
fi
if [ $add_master -eq 1 ]; then
	apply_cib_master
fi

# Restore original state as requested
if [ $restore_pcmk -eq 1 ]; then
	restore_cib
fi
if [ $restore -eq 1 ]; then
	restore_libvirt
fi
if [ $restore_all -eq 1 ]; then
	restore_files
fi