#!/usr/bin/ksh
#
# Copyright (c) 2004 by Sun Microsystems, Inc.
# All rights reserved.
#
#ident  "@(#)reconfig.cluster 1.8     04/06/10 SMI"
#
# NWS DataServices within SunCluster reconfiguration script.
#
# Description:
#
# This script is called from /usr/cluster/lib/sc/run_reserve at
# appropriate times to start and stop the NWS DataServices as SunCluster
# disk device groups are brought online or taken offline.
#
# SNDR configuration requires that a resource group to be configured.
# 1. The resource group name should be same as device group name with -rg
#    added. e.g. if device group name is abc-dg then resource group name
#    would be abc-dg-stor-rg. 
# 2. The device group name is used as a cluster tag in the 11th field of
#    the NWS configurations, listed using dscfg -l command. This 11th field
#    is checked for validity of the SNDR device group in cluster.
# 3. It should have 2 resources in it, unless one of the resource types is the
#    SUNW.GeoCtlAVS. One of type SUNW.LogicalHostname and either SUNW.HAStorage
#    or SUNW.HAStoragePlus types. Resource type versioning is ignored.
#    HAStorage type resource, should have ServicePaths property set to
#    device group name. HAStoragePlus type resource, should have either the
#    FilesystemMountPoints pointing to a files system associated with the
#    device group name, or GlobalDevicePaths property set to device group name.
#    LogicalHostname type resource should have a failoverIP address in it and
#    it will be used by SNDR to communicate with the secondary side.
#
# As SNDR requires that the LogicalHost (failover) IP address which is a
# part of resource group for SNDR, to be hosted on the same node where the 
# device group is, it tries to move the resource group also alongwith the
# device group, in become_primary case of run_reserve script. While
# in primary_to_secondary case, it will try to kill the switchover function
# if it is still running in background, after stopping NWS data services.
# 
# Usage:
#
# /usr/opt/SUNWesm/cluster/sbin/reconfig { start | stop } diskgroup
#
# Configuration:
#
# Scripts to be run should have been symlinked into $NWS_START_DIR and
# $NWS_STOP_DIR.  Note that the scripts are processed in lexical order,
# and that unlike /etc/rc?.d/ there is no leading S or K character.
#
# Exit status:
#
# 0 - success
# 1 - error
#

#
# Global variables
#

# this program
typeset -r ARGV0=$(basename $0)

# directory full of start scripts
typeset -r NWS_START_DIR=/usr/opt/SUNWesm/cluster/lib/start

# directory full of stop scripts
typeset -r NWS_STOP_DIR=/usr/opt/SUNWesm/cluster/lib/stop

# the syslog facility to use.
# - conceptually this should be based on the output of
#   "scha_cluster_get -O SYSLOG_FACILITY", but that won't work early
#   during boot.
typeset -r SYSLOG_FACILITY=daemon

PATH=$PATH:/usr/cluster/bin:/etc:/usr/opt/SUNWesm/sbin:/usr/opt/SUNWscm/sbin/

# Variables for retrying scswitch of Resource group for SNDR
retry_num=3
retry_interval=20
rgname=
rgstat=
skip_resource=0

# Since the switchover of the resource group is called in background,    
# the stop action of the reconfig script will kill the background switchover
# if it is running. Since we are stopping the NWS services on the node, there
# is no need to switch the resource group, so  it is killed.
# The pid of the process is kept in file /var/run/scnws/$dg.pid.
# Input:  dg - device group
# Output: Nothing, kills the process

function kill_scswitch
{
        dg=$1
        if [ -f /var/run/scnws/$dg.pid ]
        then
                for i in `cat /var/run/scnws/$dg.pid`
                do
                        pid=$i
                        kill -9 $pid
                done
                rm -f /var/run/scnws/$dg.pid
        fi
}

# Get the status of the resource group on this node, using scha commands.
# Input: resource group - $1
# Output: Status

function get_rgstat
{
	rg=$1
	rgstat=`scha_resourcegroup_get -O RG_STATE -G $rg`
}

# This function is called in background from do_scswitch function, to
# switch the resource group to this node, which is becoming primary for
# the diskgroup. If the status of resource group is Offline, it will use
# scswitch command to switch the resource group to this node. If it has
# become Online, cleanup pid file. If it is Pending, the resource group
# is in the state of becoming online, so wait for sometime to become Online..
# scswitch may fail, so the function retries $retry_num times, waiting for
# $retry_interval seconds.
# Input: resource group - $1, Diskgroup/Diskset - $2
# Output: 0 - success, 1 - failure

function switchfunc
{
        rg=$1
        dg=$2
	how_many=0
	sleep 2
	while [ $how_many != $retry_num ]
	do
		get_rgstat $rg
		case "$rgstat" in
		"ONLINE")
		 	rm -f /var/run/scnws/$dg.pid
			return 0
			;;

		"OFFLINE")
			scswitch -z -g $rg -h $(hostname)
			retval=$?
			if [ $retval != 0 ]
			then
				logger -p ${SYSLOG_FACILITY}.notice \
				-t "NWS.[$ARGV0]" `gettext "retrying scswitch of resource group"`
				sleep $retry_interval
				how_many=$(($how_many + 1))
			fi
			;;

		"PENDING_ONLINE")
			sleep $retry_interval
			how_many=$(($how_many + 1))
			;;

		*)
			logger -p ${SYSLOG_FACILITY}.notice \
			-t "NWS.[$ARGV0]" `gettext "Improper resource group status for SNDR"` "$rgstat"
		 	rm -f /var/run/scnws/$dg.pid
			return 1
			;;	
		esac
	done
	logger -p ${SYSLOG_FACILITY}.err \
	-t "NWS.[$ARGV0]" "Did not switch resource group for SNDR. System Administrator intervention required"
 	rm -f /var/run/scnws/$dg.pid
	return 1
}


# This function calls switchfunc function in background, to switch the 
# resource group for SNDR. It validates the diskgroup/diskset is configured 
# for SNDR, checks if the resource group is in Managed state etc.
# If it detects a mis-configuration, it will disable SNDR for the
# device group being processed. This is to prevent cluster hangs and panics.
#  
# The ServicePaths extension property of HAStorage type resource or the
# GlobalDevicePaths extension property of HAStoragePlus, both of which
# specify the device group, serve as a link or mapping to retrieve the 
# resource group associated with the SNDR configured device group.
# Switchfunc is called in the background to avoid the deadlock situation arising
# out of switchover of resource group from within device group switchover.
#
# In run_reserve context, we are doing the device group switchover, trying to
# bring it online on the node. Device group is not completely switched online,
# until the calling script run_reserve returns. In the process, we are calling
# the associated SNDR resource group switchover using scswitch command. 
# Resource group switchover will trigger the switchover of device group also. 
#
# If resource group switchover is called in foreground, before the device 
# group has become online, then it will result in switching the device group 
# again, resulting in deadlock. Resource group can not become online until 
# the device group is online and the device group can not become online until the 
# script returns, causing this circular dependency resulting in deadlock. 
#
# Calling the resource group switch in background allows current run_reserve
# script to return immediately, allowing device group to become online.
# If the device group is already online on the node, then the resource group 
# does not cause the device group switchover again.
#
# Input: Device group dg - $1
# Output: 0 - success
#	  1 - either dg not applicable for SNDR or error
#	  2 - SNDR mis-configuration

function do_scswitch
{
        if [ ! -x /usr/opt/SUNWscm/sbin/dscfg \
		-o ! -x /usr/cluster/bin/scha_resource_get \
		-o ! -x /usr/cluster/bin/scha_resourcegroup_get ]
        then
                return 1
        fi

# Check if this device group is configured for SNDR

	dg=$1
	found=0
	mkdir -p /var/run/scnws/
	rm -f /var/run/scnws/nws.sndr.$$

# From the NWS configuration output, filter out lines related to SNDR only
# and check if 11th field of the line matches the device group, dg, which 
# is used as a tag in clustering.

	dscfg -l | grep "^sndr:" > /var/run/scnws/nws.sndr.$$
	if [ $? != 0 ]
	then
# There is no SNDR in config or some error
		return 1
	fi
	if [ ! -s /var/run/scnws/nws.sndr.$$ ]
	then
# There is no SNDR in config
		return 1
	fi
	while read f1 f2 f3 f4 f5 f6 f7 f8 f9 f10 f11 f12
	do
		if [ "$dg" = "$f11" ]
		then
			found=1
			break
		fi
	done < /var/run/scnws/nws.sndr.$$
	rm -f /var/run/scnws/nws.sndr.$$
	if [ $found == 0 ]
	then
# There is no device group configured in cluster for SNDR with this cluster tag
        	return 1
	fi

	/usr/bin/pgrep -u 0 -P 1 -x rgmd >/dev/null 2>&1
	if [ $? != 0 ]
	then
		logger -p ${SYSLOG_FACILITY}.notice \
		-t "NWS.[$ARGV0]" `gettext "rgmd daemon is not running"`
		return 1
	fi

# hard coded rg name from dg
	rgname="$dg-stor-rg"
	scha_resourcegroup_get -O rg_description -G $rgname > /dev/null
	if [ $? != 0 ]
	then
		logger -p ${SYSLOG_FACILITY}.notice \
		-t "NWS.[$ARGV0]" `gettext "Resource group <$rgname> for SNDR is not configured"`
       		return 2 
	fi

# Check the state of resource group

	get_rgstat $rgname
	if [ -z "$rgstat" \
		-o "$rgstat" = "UNMANAGED" -o "$rgstat" = "ERROR_STOP_FAILED" ]
	then
		logger -p ${SYSLOG_FACILITY}.notice \
		-t "NWS.[$ARGV0]" \
		`gettext "Improper SNDR resource group state"` "$rgstat"
        	return 2 
	fi

# Check whether resources are of proper type and they are enabled

	rs_list=`scha_resourcegroup_get -O resource_list -G $rgname`
	if [ -z "$rs_list" ]
	then
		logger -p ${SYSLOG_FACILITY}.notice \
		-t "NWS.[$ARGV0]" \
		`gettext "No resources in SNDR resource group <$rgname>"`
		return 2 
	fi
	for rs in $rs_list
	do
		rs_type=`scha_resource_get -O type -R $rs -G $rgname  | cut -d':' -f1`
		case "$rs_type" in
		SUNW.LogicalHostname | SUNW.HAStorage | SUNW.HAStoragePlus | SUNW.GeoCtlAVS)
			if [ "$rs_type" = "SUNW.GeoCtlAVS" ]		
			then
				skip_resource=$(($skip_resource + 1))
			else
				rs_enb=`scha_resource_get -O ON_OFF_SWITCH -R $rs -G $rgname`
				if [ "$rs_enb" = "DISABLED" ]
				then
					logger -p ${SYSLOG_FACILITY}.notice \
					-t "NWS.[$ARGV0]" `gettext "Disabled resource <$rs> in resource group <$rgname> for SNDR"`
					if [ "$rs_type" = "SUNW.LogicalHostname" ]
					then
						logger -p ${SYSLOG_FACILITY}.notice \
						-t "NWS.[$ARGV0]" `gettext "Continuing without Logical Host"`
					else
						return 2
					fi
				fi
			fi
			;;
		*)
			logger -p ${SYSLOG_FACILITY}.notice \
			-t "NWS.[$ARGV0]" `gettext "Incorrect type of resource configured in resource group <$rgname> for SNDR"`
			return 2 
			;;
		esac
	done

# Check if num of resources in the group are two, skipping those resources that are of type SUNW.GeoCtlAVS

    numrs=`scha_resourcegroup_get -O resource_list -G $rgname | wc -l`
    numrs=$(($numrs - $skip_resource))
    if [ $numrs != 2 ]
    then
	logger -p ${SYSLOG_FACILITY}.warn \
        -t "NWS.[$ARGV0]" \
        `gettext "Resources in SNDR resource group <$rgname> \
                     are not equal to 2"`
        return 2
    fi


# Invoke switchfunc to switch the resource group. 

	switchfunc $rgname $dg &
	pid=$!
	mkdir -p /var/run/scnws/
	rm -f /var/run/scnws/$dg.pid
	echo $pid > /var/run/scnws/$dg.pid

	return 0
}


#
# Functions
#

usage()
{
	logger -p ${SYSLOG_FACILITY}.err \
	    -t "NWS.[$ARGV0]" "usage: $ARGV0 { start | stop } diskgroup"
	exit 1
}


# Input: arg1) $NWS_START_DIR - location of NWS scripts
#	 arg2) start / stop
#	 arg3 ) device group - $2
#	 arg4) sndr_ena / sndr_dis
# Output: Nothing. Log error if seen

process_dir()
{
	typeset dir=$1
	typeset arg1=$2
	typeset dg=$3
	typeset arg2=$4
	typeset RDC=$dir/10rdc

	if [[ -d $dir ]]
	then
		for f in $dir/*
		do
			# process scripts in the directories in lexical order
			# note - no leading S or K unlike /etc/rc?.d/

			if [ -s $f ] && [ $arg2 != "sndr_dis" ]   
			then
				# run script and pipe output through
				# logger into syslog

				/usr/bin/ksh $f $arg1 $dg 2>&1 |
				    logger -p ${SYSLOG_FACILITY}.notice \
					-t "NWS.[${ARGV0}:$(basename $f)]"
			else
			# SNDR misconfigured - prevent start
                            if [ -s $f ] && [ $f != $RDC ] 
                            then
                                # run script and pipe output through
                                # logger into syslog
                                /usr/bin/ksh $f $arg1 $dg 2>&1 |
                                    logger -p ${SYSLOG_FACILITY}.notice \
                                        -t "NWS.[${ARGV0}:$(basename $f)]"
			    fi
			fi
		done
	else
		logger -p ${SYSLOG_FACILITY}.err \
		    -t "NWS.[$ARGV0]" "no directory: $dir"
	fi
}


#
# main
#

if [ $# -ne 2 ]
then
	usage
	# not reached
fi


case "$1" in
start)
	logger -p ${SYSLOG_FACILITY}.notice -t "NWS.[$ARGV0]" "starting: $ARGV0 $*"
	do_scswitch $2
	retval=$?
	if [ $retval == 2 ]
	then
		logger -p ${SYSLOG_FACILITY}.err \
		    -t "NWS.[$ARGV0]" "**FATAL ERROR** SNDR is mis-configured and DISABLED for devicegroup <"$2"> " 
		# Disable SNDR 
		process_dir $NWS_START_DIR start "$2" sndr_dis
	else
		process_dir $NWS_START_DIR start "$2" sndr_ena
	fi
	;;
stop)
	logger -p ${SYSLOG_FACILITY}.notice -t "NWS.[$ARGV0]" "stopping: $ARGV0 $*"
	process_dir $NWS_STOP_DIR stop "$2" sndr_ena
	kill_scswitch $2
	;;

*)
	usage
	# not reached
	;;
esac

logger -p ${SYSLOG_FACILITY}.notice -t "NWS.[$ARGV0]" "completed: $ARGV0 $*"

exit 0
