pacemaker/extra/cluster-init

#!/bin/bash

accept_defaults=0
do_raw=0
ETCHOSTS=0
CMAN=0
do_heartbeat=0
plugin_ver=-1
pcmk_ver=11
nodelist=0
limit=0

pkgs="corosync xinetd nmap abrt-cli fence-agents perl-TimeDate gdb"

transport="multicast"
inaddr_any="no"

INSTALL=
cs_conf=
fence_conf=
rpm_repo=
distro=

dsh_group=0
if [ ! -z $cluster_name ]; then
    cluster=$cluster_name
else
    cluster=dummy0
fi

#  Corosync/OpenAIS Settings
cs_port=666

# Settings that work great on nXX
join=60
#token=3000
consensus=1500

# Official settings
join=2000
token=5000
consensus=2500

# Testing
join=1000
consensus=7500
do_debug=off

function ip_for_node() {
    ping -c 1 $1 | grep "bytes from" | head -n 1 | sed -e 's/.*bytes from//' -e 's/: icmp.*//'  | awk  '{print $NF}' | sed 's:(::' | sed 's:)::'
#    if [ $do_raw = 1 ]; then
#	echo $1
#    else
#	#host $1 | grep "has address" | head -n 1 | awk '{print $NF}' | sed 's:(::' | sed 's:)::'
#    fi
}
function id_for_node() {
    ip_for_node $* | tr '.' ' ' | awk '{print $4}'
}
function name_for_node() {
    echo $1 | awk -F. '{print $1}'
}

function helptext() {
    echo "cluster-init - Configure cluster communication for the different infrastructures supported by Pacemaker"
    echo ""
    echo "-g, --group         Specify the group to operate on/with"
    echo "-w, --host          Specify a host to operate on/with.  May be specified multiple times"
    echo "-r, --raw-ip        Supplied nodes were listed as their IP addresses"
    echo ""
    echo "-h, --heartbeat     configure for heartbeat"
    echo "-c, --corosync      configure for corosync"
    echo "-C, --nodelist      configure for corosync with a node list"
    echo "-o, --openais       configure for openais"
    echo "--cman              configure for cman"
    echo "-p, --plugin version"
    echo "-u, --unicast       configure point-to-point communication instead of multicast (corosync only)"
    echo ""
    echo "-I, --install       Install packages"
    echo "-R, --repo name     Setup and update/install Pacemaker from the named clusterlabs.org repo"
    echo "                    Known values: rpm, rpm-test, rpm-next, rpm-test-next, rpm-test-rhel"
    echo "-D, --distro        The distro within the --repo.  Defaults to fedora-15"
    echo ""
    echo "-d, --debug         Enable debug logging for the cluster"
    echo "-10                 install stable-1.0 packages, implies: -p 0 -R rpm-test -I"
    echo "--hosts             Copy /etc/hosts from the test master to the nodes"
    echo "-e, --extra list    Whitespace separated list of extra packages to install"
    echo "-l, --limit N       Use the first N hosts from the named group" 
    echo "                    Extra packages to install"
    exit $1
}

host_input=""
while true; do
    case "$1" in
	-g) cluster=$2;
	    shift; shift;;
	-w|--host)
	    for h in $2; do
		host_input="$host_input -w $h";
	    done
	    shift; shift;;
	-w) host_input="$host_input -w $2"
	    shift; shift;;
	-r|--raw-ip) do_raw=1;       shift;;

	-D) distro=$2; shift; shift;;
	-d|--debug) do_debug=on; shift;;

	-R|--repo) rpm_repo=$2; shift; shift;;
	-I|--install) INSTALL=Yes; shift;;
	--hosts) ETCHOSTS=1; shift;;

	cman|--cman) CTYPE=cman; shift;;
	-h|--heartbeat) CTYPE=heartbeat; shift;;
	-c|--corosync) CTYPE=corosync; shift;;
	-C|--nodelist) CTYPE=corosync; nodelist=1; shift;;
	-o|--openais) CTYPE=openais; shift;;
	--plugin|-p) CTYPE=plugin; plugin_ver=$2; shift; shift;;
	-u|--unicast) nodelist=1; transport=udpu; inaddr_any="yes"; shift;;
	-e|--extra) pkgs="$pkgs $2"; shift; shift;;
	-t|--test)  rpm_repo=rpm-test-next; pkgs="$pkgs valgrind"; shift;;
	-l|--limit) limit=$2; shift; shift;;

	r*[0-9])
	    rhel=`echo $1 | sed -e s/rhel// -e s/-// -e s/r//`
	    distro="rhel-$rhel";
	    pkgs="$pkgs qarsh-server";
	    case $rhel in
		6) CTYPE=cman;;
		7) CTYPE=corosync;;
	    esac
	    shift
	    ;;

	f*[0-9][0-9])
	    distro="fedora-`echo $1 | sed -e s/fedora// -e s/-// -e s/f//`";
	    CTYPE=corosync;
	    shift
	    ;;
	p0|10) pcmk_ver=10; plugin_ver=0; rpm_repo="rpm-test"; install=1; shift;;

	-y|--yes|--defaults) accept_defaults=1; shift;;
	-x) set -x; shift;;
	-\?|--help) helptext 0; shift;;
	"") break;;
	*) echo "unknown option: $1"; exit 1;;
    esac
done

if [ ! -z $cluster ]; then
    host_input="-g $cluster"
    # use the last digit present in the variable (if any)
    dsh_group=`echo $cluster | sed 's/[^0-9][^0-9]*//g;s/.*\([0-9]\)$/\1/'`
fi

if [ -z $dsh_group ]; then
    dsh_group=1
fi

if [ x = "x$host_input" -a x = "x$cluster" ]; then
    if [ -d $HOME/.dsh/group ]; then
        read -p "Please specify a dsh group you'd like to configure as a cluster? [] " -t 60 cluster
    else
        read -p "Please specify a whitespace delimetered list of nodes you'd like to configure as a cluster? [] " -t 60 host_list

        for h in $2; do
	    host_input="$host_input -w $h";
	done
    fi
fi

if [ -z "$host_input" ]; then
    echo "You didn't specify any nodes or groups to configure"
    exit 1
fi

if [ $limit -gt 0 ]; then
    echo "Using only the first $limit hosts in $cluster group"
    host_list=`cluster-helper --list bullet $host_input | head -n $limit | tr '\n*' '  '`
else
    host_list=`cluster-helper --list short $host_input`
fi
num_hosts=`echo $host_list | wc -w`

if [ $num_hosts -gt 9 ]; then
    cs_port=66
fi

for h in $host_list; do
    ping -c 1 -q $h
    if [ $? != 0 ]; then
	echo "Using long names..."
	host_list=`cluster-helper --list long $host_input`
	break
    fi
done

if [ -z $CTYPE ]; then
    echo ""
    read -p "Where should Pacemaker obtain membership and quorum from? [corosync] (corosync, cman, plugin, openais, heartbeat) " -t 60 CTYPE
fi

case $CTYPE in
    cman) CMAN=1; cs_conf=/etc/cluster/cluster.conf;;
    corosync) cs_conf=/etc/corosync/corosync.conf;;
    heartbeat) do_heartbeat=1;;
    openais) cs_conf=/etc/ais/openais.conf;;
    plugin)
	cs_conf=/etc/corosync/corosync.conf
	if [ -z $plugin_ver ]; then
	    echo ""
	    read -p "Should corosync start Pacemaker? [No]" -t 60 PTYPE
	    case $PTYPE in
		[Yy][Ee][Ss]|[Yy]) plugin_ver=0;;
		*) plugin_ver=1;;
	    esac
	fi
	;;
esac

function get_defaults()
{
    if [ -z $SSH ]; then
	SSH="No"
    fi

    if [ -z $SELINUX ]; then
	SELINUX="No"
    fi

    if [ -z $IPTABLES ]; then
	IPTABLES="Yes"
    fi

    if [ -z $DOMAIN ]; then
	DOMAIN="No"
    fi
    if [ -z $INSTALL ]; then
	INSTALL="Yes"
    fi
    if [ -z $DATE ]; then
	DATE="No"
    fi
}

get_defaults
if [ $accept_defaults = 0 ]; then
    echo ""
    read -p "Shall I install an ssh key to cluster nodes? [$SSH] " -t 60 SSH
    echo ""
    echo "SELinux prevent many things, including password-less ssh logins"
    read -p "Shall I disable selinux? [$SELINUX] " -t 60 SELINUX
    echo ""
    echo "Incorrectly configured firewalls will prevent corosync from starting up"
    read -p "Shall I disable iptables? [$IPTABLES] " -t 60 IPTABLES
    if [ $CMAN = 1 ]; then
	echo ""
	echo "Without a default domain, cman probably wont start because it can't look up the IP for hosts"
	read -p "Shall I set one? [No] (domain.name) " -t 60 DOMAIN
    elif [ $pcmk_ver = 10 ]; then
	echo ""
	echo "Without a default domain, external/ssh fencing probably wont work because it can't find its peers"
	read -p "Shall I set one? [No] (domain.name) " -t 60 DOMAIN
    fi

    echo ""
    read -p "Shall I install/update the relevant packages? [$INSTALL] "  -t 60 INSTALL

    case $INSTALL in
	[Yy][Ee][Ss]|[Yy]|"")
	    if [ -z $rpm_repo ]; then
		echo ""
		read -p "Would you like to install packages from ClusterLabs.org? [No] (rpm, rpm-next, rpm-test-next) "  -t 60 rpm_repo
	    fi

	    if [ ! -z $rpm_repo ]; then
		if [ -z $distro ]; then
		    distro=fedora-18
		    echo ""
		    read -p "Which distro are you installing for? [$distro] (eg. fedora-17, rhel-6) "  -t 60 distro
		fi
	    fi
	    ;;
    esac

    echo ""
    read -p "Shall I sync the date/time? [$DATE] "  -t 60 DATE
fi
get_defaults

echo ""
echo "Detecting possible fencing options"
if [ -e /etc/cluster/fence_xvm.key ]; then
    echo "* Found fence_xvm"
    fence_conf=/etc/cluster/fence_xvm.key
    pkgs="$pkgs fence-virt"
fi

if [ ! -z ${OS_AUTH_URL} ]; then
    echo "* Found openstack credentials"
    fence_conf=/sbin/fence_openstack
    pkgs="$pkgs python-novaclient"
fi
echo ""
echo "Beginning cluster configuration"
echo ""

case $SSH in
    [Yy][Ee][Ss]|[Yy])
	for host in $host_list; do
	    echo "Installing our ssh key on ${host}"
	    ssh-copy-id root@${host} >/dev/null 2>&1
	    # Fix selinux labeling
	    ssh -l root ${host} -- restorecon -R -v .
	done
	;;
esac

case $DATE in
    [Yy][Ee][Ss]|[Yy])
	for host in $host_list; do
	    echo "Setting time on ${host}"
	    scp /etc/localtime root@${host}:/etc
	    now=`date +%s`
	    ssh -l root ${host} -- date -s @$now
	    echo ""
	done
	;;
esac

REPO=
if [ ! -z $rpm_repo ]; then
    REPO=$rpm_repo/$distro
fi

if [ $do_heartbeat = 1 ]; then
    pkgs="$pkgs heartbeat"
fi
if [ $CMAN = 1 ]; then
    pkgs="$pkgs cman"
fi


init=`mktemp`
cat<<-END>$init
verbose=0
pkgs="$pkgs"

lhost=\`uname -n\`
lshort=\`echo \$lhost | awk -F. '{print \$1}'\`

log() {
    printf "%-10s  \$*\n" "\$lshort:" 1>&2
}

debug() {
    if [ \$verbose -gt 0 ]; then
	log "Debug: \$*"
    fi
}

info() {
    log "\$*"
}

warning() {
    log "WARN: \$*"
}

fatal() {
    log "ERROR: \$*"
    exit 1
}

case $SELINUX in
    [Yy][Ee][Ss]|[Yy])
	sed -i.sed "s/enforcing/disabled/g" /etc/selinux/config
	;;
esac

case $IPTABLES in
    [Yy][Ee][Ss]|[Yy]|"")
	service iptables stop
	chkconfig iptables off
	service firewalld stop
	chkconfig firewalld off
	;;
esac

case $DOMAIN in
    [Nn][Oo]|"")
	;;
    *.*)
	if
            ! grep domain /etc/resolv.conf
        then
	    sed -i.sed "s/nameserver/domain\ $DOMAIN\\\nnameserver/g" /etc/resolv.conf
	fi
	;;
    *) echo "Unknown domain: $DOMAIN";;
esac

case $INSTALL in
    [Yy][Ee][Ss]|[Yy]|"")

	if [ ! -z $REPO ]; then
	    info Configuring Clusterlabs repo: $REPO
	    yum install -y wget
	    rm -f /etc/yum.repos.d/clusterlabs.repo
            wget -O /etc/yum.repos.d/clusterlabs.repo http://www.clusterlabs.org/$REPO/clusterlabs.repo &>/dev/null
	    yum clean all
	fi

	info Installing cluster software
	if [ $pcmk_ver = 10 ]; then
	    yum install -y $pkgs at
	    service atd start
	    systemctl enable atd.service

	    yum install -y "pacemaker < 1.1"
	else
	    yum install -y $pkgs pacemaker
	fi
	;;
esac

info "Configuring services"
chkconfig xinetd on
service xinetd start &>/dev/null

chkconfig corosync off &> /dev/null
chkconfig heartbeat off &> /dev/null
mkdir -p /etc/cluster

info "Turning on core files"
grep -q "unlimited" /etc/bashrc
if [ $? = 1 ]; then
    sed -i.sed "s/bashrc/bashrc\\\nulimit\ -c\ unlimited/g" /etc/bashrc
fi

if [ $CMAN = 1 ]; then
    grep -q "logger" /etc/init.d/cman
    if [ $? = 1 ]; then
	info "Turning on CMAN init-script logging"
	sed -i.sed "s/function=/logger\ -p\ daemon.info\ cman\ init\ \\\$1\\\nfunction=/g" /etc/init.d/cman
    fi

    sed -i.sed "s/.*CMAN_QUORUM_TIMEOUT=.*/CMAN_QUORUM_TIMEOUT=0/g" /etc/sysconfig/cman
fi
END

function create_hb_config() {
    cat <<-END >/tmp/lha.$$
auth 1
1 crc
END
    printf "%-10s  Installing authkeys\n" ${host}
    scp -q /tmp/lha.$$ root@${host}:/etc/ha.d/authkeys
    ssh -l root ${host} -- chmod 600 /etc/ha.d/authkeys

    cat <<-END >/tmp/lha.$$
traditional_compression off
compression		bz2

realtime	yes
conn_logd_time	120
coredumps	true
udpport		$cs_port$dsh_group

bcast		eth0
autojoin	any

logfacility	daemon
crm		respawn

# 8-node version
debug		0
keepalive	1
warntime	6
deadtime	10
initdead	15
END
    printf "%-10s  Installing ha.cf\n" ${host}
    scp -q /tmp/lha.$$ root@${host}:/etc/ha.d/ha.cf
}

function patch_cs_config() {
    test $num_hosts != 2
    two_node=$?

    priority="info"
    if [ $do_debug = 1 ]; then
	priority="debug"
    fi

    ssh -l root ${host} -- sed -i.sed "s/.*mcastaddr:.*/mcastaddr:\ 226.94.1.1/g" $cs_conf
    ssh -l root ${host} -- sed -i.sed "s/.*mcastport:.*/mcastport:\ $cs_port$dsh_group/g" $cs_conf
    ssh -l root ${host} -- sed -i.sed "s/.*bindnetaddr:.*/bindnetaddr:\ $ip/g" $cs_conf
    ssh -l root ${host} -- sed -i.sed "s/.*syslog_facility:.*/syslog_facility:\ daemon/g" $cs_conf
    ssh -l root ${host} -- sed -i.sed "s/.*logfile_priority:.*/logfile_priority:\ $priority/g" $cs_conf

    if [ ! -z $token ]; then
	ssh -l root ${host} -- sed -i.sed "s/.*token:.*/token:\ $token/g" $cs_conf
    fi
    if [ ! -z $consensus ]; then
	ssh -l root ${host} -- sed -i.sed "s/.*consensus:.*/consensus:\ $consensus/g" $cs_conf
    fi
    if [ ! -z $join ]; then
	ssh -l root ${host} -- sed -i.sed "s/^join:.*/join:\ $join/g" $cs_conf
	ssh -l root ${host} -- sed -i.sed "s/\\\Wjoin:.*/join:\ $join/g" $cs_conf
    fi

    if [ $plugin_ver -ge 0 ]; then
	ssh -l root ${host} -- grep -q "name: pacemaker" $cs_conf 2>&1 > /dev/null
	if [ $? = 0 ]; then
	    ssh -l root ${host} -- sed -i.sed "s/.*ver:.*/ver:\ $plugin_ver/g" $cs_conf
	else
	    printf "%-10s  Wrong quorum provider: installing $cs_conf for plugin v${plugin_ver} instead\n" ${host}
	    create_cs_config
	fi

    elif [ $CMAN = 1 ]; then
	ssh -l root ${host} -- grep -q "quorum_cman" $cs_conf 2>&1 > /dev/null
	if [ $? = 0 ]; then
	    ssh -l root ${host} -- sed -i.sed "s/\\\Wexpected_votes:.*/expected_votes:\ $num_hosts/g" $cs_conf
	    ssh -l root ${host} -- sed -i.sed "s/\\\Wtwo_node:.*/two_node:\ $two_node/g" $cs_conf
	else
	    printf "%-10s  Wrong quorum provider: installing $cs_conf for cman instead\n" ${host}
	    create_cs_config
	fi
    else
	ssh -l root ${host} -- grep -q "corosync_votequorum" $cs_conf 2>&1 > /dev/null
	if [ $? = 0 ]; then
	    ssh -l root ${host} -- sed -i.sed "s/\\\Wexpected_votes:.*/expected_votes:\ $num_hosts/g" $cs_conf
	    ssh -l root ${host} -- sed -i.sed "s/\\\Wtwo_node:.*/two_node:\ $two_node/g" $cs_conf
	else
	    printf "%-10s  Wrong quorum provider: installing $cs_conf for corosync instead\n" ${host}
	    create_cs_config
	fi
    fi
}

function create_cs_config() {
    cs_tmp=/tmp/cs_conf.$$
    test $num_hosts != 2
    two_node=$?

    # Base config
    priority="info"
    if [ $do_debug = 1 ]; then
	priority="debug"
    fi

    cat <<-END >$cs_tmp
# Please read the corosync.conf.5 manual page
totem {
        version: 2

        # cypto_cipher and crypto_hash: Used for mutual node authentication.
        # If you choose to enable this, then do remember to create a shared
        # secret with "corosync-keygen".
        crypto_cipher: none
        crypto_hash: none

        # Assign a fixed node id
        nodeid:         $id

        # Disable encryption
        secauth:        off

	transport:      $transport
	inaddr_any:     $inaddr_any

        # interface: define at least one interface to communicate
        # over. If you define more than one interface stanza, you must
        # also set rrp_mode.
        interface {
                # Rings must be consecutively numbered, starting at 0.
                ringnumber: 0

                # This is normally the *network* address of the
                # interface to bind to. This ensures that you can use
                # identical instances of this configuration file
                # across all your cluster nodes, without having to
                # modify this option.
                bindnetaddr: $ip

                # However, if you have multiple physical network
                # interfaces configured for the same subnet, then the
                # network address alone is not sufficient to identify
                # the interface Corosync should bind to. In that case,
                # configure the *host* address of the interface
                # instead:
                # bindnetaddr: 192.168.1.1
                # When selecting a multicast address, consider RFC
                # 2365 (which, among other things, specifies that
                # 239.255.x.x addresses are left to the discretion of
                # the network administrator). Do not reuse multicast
                # addresses across multiple Corosync clusters sharing
                # the same network.

                # Corosync uses the port you specify here for UDP
                # messaging, and also the immediately preceding
                # port. Thus if you set this to 5405, Corosync sends
                # messages over UDP ports 5405 and 5404.
                mcastport: $cs_port$dsh_group

                # Time-to-live for cluster communication packets. The
                # number of hops (routers) that this ring will allow
                # itself to pass. Note that multicast routing must be
                # specifically enabled on most network routers.
                ttl: 1
                mcastaddr: 226.94.1.1
        }
}

logging {
        debug: off
        fileline: off
        to_syslog: yes
        to_stderr: no
        syslog_facility: daemon
        timestamp: on
        to_logfile: yes
        logfile: /var/log/corosync.log
        logfile_priority: $priority
}

amf {
        mode: disabled
}
END

    # Corosync Variant

    if [ $plugin_ver -ge 0 ]; then
	cat <<-END >>$cs_tmp
service {
        name: pacemaker
        ver:  $plugin_ver
}
END
    elif [ $CMAN = 1 ]; then
	cat <<-END >>$cs_tmp
cluster {
        name: $cluster

        clusternodes {
END

	for peer in $host_list; do
	    p_name=`name_for_node $peer`
	    p_id=`id_for_node $peer`

	    cat <<-END >>$cs_tmp
                clusternode {
                        votes: 1
                        nodeid: $p_id
                        name: $p_name
                }
END
	done
	cat <<-END >>$cs_tmp
        }
        cman {
                expected_votes: $num_hosts
                cluster_id: $dsh_group
                nodename: $cs_short_host
                two_node: $two_node
                max_queued: 10
        }
}

service {
        name: corosync_cman
        ver: 0
}

quorum {
        provider: quorum_cman
}
END
    elif [ $nodelist = 1 ]; then
	cat <<-END >>$cs_tmp
nodelist {
END

	for peer in $host_list; do
	    p_name=`name_for_node $peer`
	    p_id=`id_for_node $peer`
	    p_ip=`ip_for_node $peer`

	    cat <<-END >>$cs_tmp
    node {
        ring0_addr: $p_ip
        nodeid: $p_id
        quorum_votes: 1
        name: $peer
    }
END
	done
	cat <<-END >>$cs_tmp
}

quorum {
   provider: corosync_votequorum
   expected_votes: $num_hosts
   votes: 1
   two_node: $two_node
   wait_for_all: 0
   last_man_standing: 0
   auto_tie_breaker: 0
}
END
    else
	cat <<-END >>$cs_tmp
quorum {
   provider: corosync_votequorum
   expected_votes: $num_hosts
   votes: 1
   two_node: $two_node
   wait_for_all: 0
   last_man_standing: 0
   auto_tie_breaker: 0
}
END
    fi
    scp -q $cs_tmp root@${host}:$cs_conf
    rm -f $cs_tmp
}

function create_cman_config() {
    cs_tmp=/tmp/cs_conf.$$
    cat <<-END >$cs_tmp
<?xml version="1.0"?>
<cluster config_version="1" name="$cluster">
  <logging debug="$do_debug"/>
  <clusternodes>
END

    lpc=1
    for h in $host_list; do
	short_h=`name_for_node $h`
	cat <<-END >>$cs_tmp
    <clusternode name="$short_h" nodeid="$lpc">
      <fence>
        <method name="pcmk-redirect">
          <device name="pcmk" port="$short_h"/>
        </method>
      </fence>
    </clusternode>
END
	lpc=`expr $lpc + 1`
    done

    test $num_hosts != 2
    two_node=$?
    extra=""

    if [ $two_node = 1 ]; then
	extra="\n  <cman two_node=\"1\" expected_votes=\"1\"/>"
    fi
    cat <<-END >>$cs_tmp
  </clusternodes>
  <fencedevices>
    <fencedevice name="pcmk" agent="fence_pcmk"/>
  </fencedevices>$extra
</cluster>
END
    scp -q $cs_tmp root@${host}:$cs_conf
    rm -f $cs_tmp
}

for host in $host_list; do
    echo ""
    echo ""
    echo "* Configuring $host"

    cs_short_host=`name_for_node $host`
    ip=`ip_for_node $host`
    id=`id_for_node $host`

    echo $ip | grep -qis NXDOMAIN
    if [ $? = 0 ]; then
	echo "Couldn't find resolve $host to an IP address"
	exit 1
    fi

    if [ `uname -n` = $host ]; then
	bash $init
    else
	cat $init | ssh -l root -T $host -- "cat > $init; bash $init"
    fi

    if [ "x$fence_conf" != x ]; then
	if [ -e $fence_conf ]; then
	    scp $fence_conf root@${host}:$fence_conf
	fi
    fi

    if [ $ETCHOSTS = 1 ]; then
	scp /etc/hosts root@${host}:/etc/hosts
    fi

    if [ $pcmk_ver = 10 ]; then
	scp /etc/hosts root@${host}:/etc/hosts
	scp ~/.ssh/id_dsa.suse root@${host}:.ssh/id_dsa
	scp ~/.ssh/known_hosts root@${host}:.ssh/known_hosts
    fi

    if [ $do_heartbeat = 1 ]; then
	create_hb_config ${host}

    elif [ $CMAN = 1 ]; then
	create_cman_config
        printf "%-10s  Installed $cs_conf\n" ${host}:

    else
	ssh -l root ${host} -- grep -q "token:" $cs_conf 2>&1 > /dev/null
	new_config=$?
	new_config=1

	if [ $new_config = 0 ]; then
            printf "%-10s  Updating $cs_conf\n" ${host}:
	    patch_cs_config

	else
            printf "%-10s  Installing $cs_conf\n" ${host}:
	    create_cs_config
	fi
    fi
done