#!/bin/ksh
#
#pragma ident "@(#)loghostreconfig.sh 1.35 01/08/05"
#
# Copyright (c) 1997-1999 by Sun Microsystems, Inc.
# All rights reserved.
#
#

# loghostreconfig - Energizer Cluster Reconfiguration Programs for
#                   logical hosts.
#
# Input:
#               All environment variables like "currnodes"/
#               "allnodes"/"localnodeid", CURRSTEP
#
# Action:       Run Reconfiguration Programs based on current
#               "cmmstepN"|"cmmabort"|"cmmstart"|"cmmreturn".
#
# Output:       Return 0 if success.
#               Return 1 if failure
#               Return 200 if reconfiguration program result is to
#               be ignored.
 
# should be used only the reconf_framework.
pre="SUNWcluster.reconf.loghost"

RECONF_SCRIPTS=${RECONF_SCRIPTS:-/opt/SUNWcluster/etc/reconf/scripts}
CLUSTERBIN=${CLUSTERBIN:-/opt/SUNWcluster/bin/}
CLUSTERVAR=${CLUSTERVAR:-/var/opt/SUNWcluster/}
CLUSTERETC=${CLUSTERETC:-/etc/opt/SUNWcluster/}

DEFAULT_TIMEOUT=180
DEFAULT_WAIT=5
DEFAULT_RETRIES=35
RETRIES=5

INCLUDE=.

# Required for failure fencing code
${INCLUDE} ${RECONF_SCRIPTS}/reconf_ener.disks

#
# set the number of retries for CCD operations.
#
function set_retries_for_ccd_operation
{
  typeset timeout

  set +a

  action=$1

  # For stopnode there is no timeout and hence we will fall back
  # to default timeout since only this node might be in transition
  # stopnode.
  if [[ ${action} != "stopnode" ]]; then
  	timeout=$(${CLUSTERBIN}/cdbmatch cmm.transition.${action}.timeout \
			 ${cdbfile})
  fi

  if [[ "${timeout}" = "" ]]; then
	timeout=${DEFAULT_TIMEOUT}
  fi

  # retries  is equal to timeout / wait_period which is fixed to 5 seconds

  RETRIES=$(expr ${timeout} / ${DEFAULT_WAIT})

  if [[ ${RETRIES} -lt ${DEFAULT_RETRIES} ]]; then
	RETRIES=${DEFAULT_RETRIES}
  fi

}


########################################################################
#
# function is_member i_curr_nodes i_nodename
#
# i_curr_nodes	- list of space separated node names which are in the
#		  cluster membership
#
# i_nodename	- The nodename to be checked for in the current membership.
#
# Returns 0 (success) if nodename is in the current membership, otherwise
# returns 1 (failure).

function is_member
{
typeset i_curr_nodes n
typeset i_nodename

i_curr_nodes=$1
i_nodename=$2

for n in ${i_curr_nodes}; do
  if [[ "${n}" = "${i_nodename}" ]]; then
    return 0
  fi
done
return 1
}

########################################################################
#
# function first_backup_node i_curr_nodes i_nodelist o_fb_node
#
# i_curr_nodes	- list of space separated node names that are in the
#		  current membership.
#
# i_nodelist	- list of space separated node names, usually the nodes
#		  on which a logical host can be mastered.
#
# o_fb_node	- output parameter which contains the name of the first
#		  backup node in i_nodelist
#
# The first backup node is the first node in the ordered list i_nodelist
# which is in the cluster membership and is not the first node in the
# ordered list.

function first_backup_node
{
typeset i_curr_nodes m
typeset i_nodelist nodelist_array n

i_curr_nodes=$1
i_nodelist=$2
set -A nodelist_array ${i_nodelist}

let n=1
while (( n < ${#nodelist_array[*]} )); do
  for m in ${i_curr_nodes}; do
    if [[ "${nodelist_array[n]}" = "${m}" ]]; then
      eval $3=${nodelist_array[n]}
      return
    fi
  done
  let n=n+1
done

eval $3=""
}

########################################################################
#
# function is_backup i_nodelist i_nodename
#
# i_nodelist	- space separated list of node names
#
# i_nodename	- given node name
#
# Returns 0 if i_nodename is in i_nodelist and is not the first node in
# i_nodelist. Otherwise returns 1 (failure).

function is_backup
{
typeset i_nodelist
typeset nodelist_array n
typeset i_nodename
i_nodelist=$1
i_nodename=$2
set -A nodelist_array ${i_nodelist}
let n=1

while (( n < ${#nodelist_array[*]} )); do
  if [[ "${nodelist_array[n]}" = "${i_nodename}" ]]; then
    return 0
  fi
  let n=n+1
done
return 1
}
########################################################################
#
# function exec_fault_methods method-type
#
# method_type - "fm_start" or "fm_stop"
#
# Calls the appropriate fault monitor method for all registered data
# services.

function exec_fault_methods
{
typeset method method_type
typeset ds_state dslist dsrow ds
typeset timeout default_timeout
typeset all_loghosts lm_loghosts nm_loghosts
typeset loghost_rows loghost_name
typeset cm_row cm
typeset ccdfile

method_type=$1
default_timeout=30

ccdfile=$(${CLUSTERBIN}/ccdadm ${clustname} -w)
dslist=$(${CLUSTERBIN}/hareg | tr '\t' ':')
for dsrow in ${dslist}; do
  ds=${dsrow%:*}
  ds_state=${dsrow#*:}
  
  method=$(${CLUSTERBIN}/hareg -q ${ds} -M ${method_type})
  if [[ -n "${method}" && -x "${method}" ]]; then

    # find out all the logical hosts offering this data service
    # split this list into mastered and not-mastered lists

    all_loghosts=""
    lm_loghosts=""
    nm_loghosts=""
    loghost_rows=$(${CLUSTERBIN}/scccd -f ${ccdfile} ${clustname} LOGHOST_DS \
		query dsname ${ds})


    
    for r in ${loghost_rows}; do
      loghost_name=${r%:*}
      loghost_name=${loghost_name#*:}
      
      if [[ -z "${all_loghosts}" ]]; then
	all_loghosts=${loghost_name}
      else
	all_loghosts="${all_loghosts},${loghost_name}"
      fi

      #
      # retrieve the current master of the logical host
      # only if we are in start transition, for stop methods
      # we do not use current master.
      #

     if [[ "${method_type}" = "start" ||   "${method_type}" = "start_net" || \
        "${method_type}" = "fm_init" || "${method_type}" = "fm_start" ]]; then
      
      # retrieve the current master of the logical host
      cm_row=$(${CLUSTERBIN}/scccd -f ${ccdfile} ${clustname} LOGHOST_CM \
		query lname ${loghost_name})
      cm=${cm_row#*:*:}

      if [[ -n "${cm}" && "${cm}" = "${localhostname}" ]]; then
	if [[ -z "${lm_loghosts}" ]]; then
	  lm_loghosts=${loghost_name}
	else
	  lm_loghosts="${lm_loghosts},${loghost_name}"
	fi
      else
	if [[ -z "${nm_loghosts}" ]]; then
	  nm_loghosts=${loghost_name}
	else
	  nm_loghosts="${nm_loghosts},${loghost_name}"
	fi
      fi
    fi
    done

    timeout=$(${CLUSTERBIN}/hareg -q ${ds} -T ${method_type})
    if [[ -z "${timeout}" ]]; then
      timeout=${default_timeout}
    fi

    #
    # Call fm_start methods only if data-service is on state.
    # 
    #
    if [[ "${method_type}" = "stop" || "${method_type}" = "stop_net" || \
        "${method_type}" = "abort" || "${method_type}" = "abort_net" || \
        "${method_type}" = "fm_stop" ]]; then

	# No need to check if ds_state is on or off and use
	# always all_loghosts. For stop/abort programs, run
        # in real time. Force timeout to be short for the stop methods.
	# Change the priority to the minimum (100) so that it does interfere
	# with other time-critical clustering software such as heartbeat.

      	/usr/bin/priocntl -c RT -p 0 -e ${CLUSTERBIN}/timed_run 10 ${method} "" "${all_loghosts}" \
		${timeout} || retval=$?
    else
	# we handle only start methods.
	# call them only if the data service state is ON.
	#
    	if [[ "${ds_state}" = "on" ]]; then
		CONF_FILE=/etc/opt/SUNWcluster/conf/dflt_sched_class
		DFLT_CLASS=TS
		if [ -f ${CONF_FILE} ] ; then
			DFLT_CLASS=`cat ${CONF_FILE}`
		fi
		/usr/bin/priocntl -c ${DFLT_CLASS} -e ${CLUSTERBIN}/timed_run ${timeout} \
			${method} "${lm_loghosts}" "${nm_loghosts}" ${timeout} \
			retval=$?
	fi
    fi
  fi
done
}

########################################################################
#
# function loghoststop_cmd
#
# This function is executed during the stop and abort transitions. It
# removes all the logical hosts being mastered by this node from the
# cluster configuration database.

function loghoststop_cmd
{
typeset r rows
typeset lhost
typeset ccdfile
typeset release_failed
typeset some_other_failure
if [[ "${CURRSTEP}" = "cmmabort" ]]; then

    ccdfile=$(${CLUSTERBIN}/ccdadm ${clustname} -w)

    if [[ -z ${ccdfile} && "${CLNODEUP}" -ne 0 ]]; then
	log_info "${pre}.4100" "Could not get a valid static ccd file."
	exit 1
    elif [[ -z ${ccdfile} && "${CLNODEUP}" -eq 0 ]]; then
	# no valid ccd file and this is the first attempt to join.
	# in this case don't do anything since scccd will return
	# an error if the -f option is supplied without a filename.
	# and since we have not been a cluster member yet we have
	# nothing mastered so in turn nothing to give up.
	return 0
    fi
    
    # This is the abort case. We call loghost_sync directly and simulate
    # its expected arguments since we want to give up the mastered logical
    # hosts without updating the CCD
    
    numnodes=$(print ${allnodes} | wc -w)
    rows=$(${CLUSTERBIN}/scccd -f ${ccdfile}  ${clustname} LOGHOST_CM query \
		"curr_master" "${localhostname}")

	# stop all fault monitors - abort case
	exec_fault_methods fm_stop

    for r in ${rows}; do
      lhost=${r%:*}
      lhost=${lhost#*:}

      /bin/rm -rf ${CLUSTERVAR}/${lhost}.abort
      /bin/rm -rf ${CLUSTERVAR}/loghost.${lhost}
     ( ${CLUSTERBIN}/loghost_sync CCDSYNC_PRE_REMOVE "${r}" ${ccdfile} \
                "${numnodes}" "${currnodes}" "${localnodeid}" || \
	echo $? > ${CLUSTERVAR}/${lhost}.abort ) &
   done

   wait

   for r in ${rows}; do
      lhost=${r%:*}
      lhost=${lhost#*:}

      if ( [ -f ${CLUSTERVAR}/${lhost}.abort ] || [ -f ${CLUSTERVAR}/loghost.${lhost} ] ) \
		then
	log_info "${pre}.4010" "Give up of logical host ${lhost} failed."
	/bin/rm -rf ${CLUSTERVAR}/${lhost}.abort
	exit 1
      fi 
   done

   # following lines added by bugid_4411476
   #
   # Now check for those logicalhosts that may have timed out in step11
   # Need to make sure the diskgroups are not imported and mounted
   if [ -f ${CLUSTERVAR}/loghost.dg ]; then
      #
      # A logical host did timeout at step11, therefore make a call
      # to loghost_sync to make sure the dg's are deported
      #
      row=$(<${CLUSTERVAR}/loghost.dg)
      lname=${row%:*:*:*}
      state=${row%:*}
      state=${row%:*}
      r="LOGHOST_CM:${lname}:${localhostname}"
      log_info "${pre}.5020" "Incomplete take over of ${lname} state ${state}"
      #
      # Fake a call to loghost_sync to do the work of getting rid of the dg's
      # XXX - verify that ${lname}.abort is correct XXX
      ( ${CLUSTERBIN}/loghost_sync CCDSYNC_PRE_REMOVE "${r}" ${ccdfile} \
	"${numnodes}" "${currnodes}" "${localnodeid}" || \
	echo $? > ${CLUSTERVAR}/${lname}.abort )
      #
      # If the status file exists then the loghost_sync above timed
      # out for a reason or failed to do something correctly therefore
      # this node needs to be aborted
      #
      if [ -f ${CLUSTERVAR}/loghost.dg ]; then
	 log_info "${pre}.5020" "Failed to remove ${lname} - aborting node"
	 rm -f ${CLUSTERVAR}/loghost.dg
	 # Kill off clustd
	 clpid=`ps -e|grep clustd|nawk '{ print $1 }'`
	 kill -9 ${clpid}
      fi
    fi
    # end of added lines by bugid_4411476

  # return now for cmmabort case
  return 0
fi

# continue on for stopnode case

# stop all fault monitors - stop case
exec_fault_methods fm_stop

#
# There is a Possibility that other nodes are also executing
# the pdbadmin stopnode, in which case the LOGHOST_CM query
# might return NULL (since we are accessing dynamic CCD)
# and we would continue with stop and then abort
# XXX:
#   1. First get the LOGHOST_CM queries using static CCD.
#   (This should not do harm since LOGHOST_CM can be modified only
#   if someone else is doing a takeover).
#		OR
#   2. Increase the timeout when querying the LOGHOST_CM
#		OR
#   3. First Query using static CCD.
# 	If there are entries keep qurying until query is retrieved.
#   while :
#    static-query;
#    if no entries break;
#    else
#    query-dynamic ccd
#	if rows="" then
#	  continue;
#    else
#        break;
#

rows=$(${CLUSTERBIN}/scccd -r ${RETRIES} -w 5 ${clustname} LOGHOST_CM query \
		"curr_master" "${localhostname}")

# First clear out any leftover files created by loghost_sync PRE_REMOVE
# on SDS diskset release failure..
rm -f ${CLUSTERVAR}/release_failed*
release_failed=false
# flag indicating some LOGHOST_CM failure not due to SDS diskset release
some_other_failure=false

for r in ${rows}; do
	lhost=${r%:*}
	lhost=${lhost#*:}

	# Note: this logic is duplicated in scadmin.
	${CLUSTERBIN}/scccd -r ${RETRIES} -w 5 ${clustname} LOGHOST_CM remove \
	    "lname" "${lhost}"

	if [[ $? -ne 0 ]]; then
		log_info "${pre}.4010" \
		    "Give up of logical host ${lhost} failed."

		if [[ "${vm}" = "sds" ]]; then
			# This file was created by loghost_sync PRE_REMOVE
			if [ -f ${CLUSTERVAR}/release_failed ]; then
				# LOGHOST_CM remove failed due to
				# SDS diskset release failure.
				# Set flag to be used outside this loop.
				release_failed=true
				rm ${CLUSTERVAR}/release_failed.${lhost}
			else
				some_other_failure=true
			fi
		fi
		row=$(${CLUSTERBIN}/scccd ${clustname} LOGHOST_CM \
		    query lname ${lhost})
		cm=${row#*:*:}
		if [[ -z "${cm}" ]]; then
			# row successfully removed; add it back
			${CLUSTERBIN}/scccd -r ${RETRIES} -w 5 \
			    ${clustname} LOGHOST_CM add \
			    "lname:curr_master" \
			    "${lhost}:${localhostname}"
 
			if [[ $? -ne 0 ]]; then
 
				# Abort the node out of the
				# cluster to prevent leaving the
				# logical host in an inconsistent
				# state.

				/bin/rm -f ${DOINGSTOPFLAG}
				${CLUSTERBIN}/clustm abort ${clustname} this
			fi
		fi
		# Preserve CVM, VxVM behavior and exit on the first failure
		# of any type.
		if [[ "${vm}" != "sds" ]]; then
			/bin/rm -f ${DOINGSTOPFLAG}
			# This 'exit 1' does not result in the node rebooting.
			exit 1
		fi
	fi
done

# If the release of any DiskSuite diskset failed, abort the node.
# scadmin will reboot the node when the abort transition finishes.
if [[ "${vm}" = "sds" ]]; then
	if [[ "${release_failed}" = "true" ]]; then
		# The giveup of some logical host failed due to
		# SDS diskset release failure.
		/bin/rm -f ${DOINGSTOPFLAG}
		# The exit 1 and the existence of
		# ${CLUSTERVAR}/release_failed will tell scadmin to
		# reboot the node.  We need to reboot to force the
		# release of the diskset.
		${CLUSTERBIN}/clustm abort ${clustname} this
		exit 1
	fi
	if [[ "${some_other_failure}" = "true" ]]; then
		/bin/rm -f ${DOINGSTOPFLAG}
		# This does not result in the node rebooting.
		exit 1
	fi
fi
}

########################################################################
#
# function loghoststep1_cmd
#
# This function executes step 1 of the logical host reconfiguration. All hosts
# participating in the reconfiguration give up logical hosts that they are
# supposed to give up in preparation for the takeover during the next step.

function loghoststep1_cmd
{
typeset r rows
typeset nodelist dglist m_mode
typeset lhost lhostrow
typeset nodelist_array
typeset primary_node

if [[ "${vm}" = "sds" ]]; then
	# First remove any leftover files created by loghost_sync
	# PRE_ADDU indicating SDS diskset release failed.
	rm -f ${CLUSTERVAR}/release_failed*
fi

# retrieve all logical hosts for which this node is the current master

rows=$(${CLUSTERBIN}/scccd -r ${RETRIES} -w 5 ${clustname} LOGHOST_CM query \
		"curr_master" "${localhostname}")

for r in ${rows}; do
  lhost=${r%:*}
  lhost=${lhost#*:}

  # retrieve information about the logical host

  lhostrow=$(${CLUSTERBIN}/scccd -r ${RETRIES} -w 5 ${clustname} LOGHOST query \
		"lname" "${lhost}")
  dglist=${lhostrow%:*:*}
  dglist=${dglist#*:*:*:}
  dglist=$(print ${dglist} | tr ',' ' ')

  nodelist=${lhostrow%:*:*:*}
  nodelist=${nodelist#*:*:}
  nodelist=$(print ${nodelist} | tr ',' ' ')
  set -A nodelist_array ${nodelist}
  primary_node=${nodelist_array[0]}

  m_mode=${lhostrow#*:*:*:*:*:}

  # If I own this logical host and I am just
  # joining the cluster, remove this LOGHOST_CM entry. This row will
  # be added in the subsequent step, but it serves to avoid confusing the
  # fault monitors. This row must have been left around from the last time
  # this node aborted or panicked.

  if [[  "${CLNODEUP}" -eq 0 ]]; then
    ${CLUSTERBIN}/scccd -r ${RETRIES} -w 5 ${clustname} LOGHOST_CM remove "lname"\
		"${lhost}"
    #
    # No Action Needs to taken since it should always succeed, since this
    # is a left over from previous cluster.
    #
  else
    if is_backup "${nodelist}" ${localhostname}; then

      # If the primary node is in the cluster membership and manual mode is
      # off, give up the logical host as the primary will take it over in
      # the next step. If manual mode is on, we should not give up the
      # logical host, but we should just release any reservations in case
      # the primary takes over at a later point in time.

      if is_member "${curr_members}" ${primary_node} && \
	  is_primary_joining ${primary_node} ; then
	if [[ "${m_mode}" -eq 0 ]]; then
	  ${CLUSTERBIN}/scccd -r ${RETRIES} -w 5 ${clustname} LOGHOST_CM remove \
		 "lname" "${lhost}"
	  if [[ $? -ne 0 ]]; then

	    # If the LOGHOST_CM remove failed due to SDS diskset release
	    # failure, we won't worry about it.  We'll let the node
	    # continue to master that logical host.

	    # Recovery:
	    # If removal of CCD ROW fails, we add it here so that
	    # this nodes will remaster the node. Add a Message in the console
	    # Log so that user knows about it and does not complain
	    # that  in m_mode = 0, the logical host is not being moved.
	    # If Add fails again then it would be better to exit.

	    ${CLUSTERBIN}/scccd -r ${RETRIES} -w 5 ${clustname} LOGHOST_CM add \
		"lname:curr_master" "${lhost}:${localhostname}"
	    if [[ $? -ne 0 ]]; then
	     log_info "${pre}.4010" "Give up of logical host ${lhost} failed."
	     #
	     # return transition failure which will cause cmm to abort the
	     # node out of the cluster
	     #
	     exit 1
	    else 
	     log_info "${pre}.1400" "Could not give up logical host ${lhost}, ${localhostname} will still master it"
	    fi
	  fi
	  continue
	else
	  # Autoswitchback mode is off.  Keep the logical host,
	  # releasing reservations on non-SDS disks.
	  if [[ "${vm}" != "sds" ]]; then
	    failure_fencing release private ${dglist}
	  fi
	fi
      else
	# The primary node of this logical host is not in the cluster
	# membership.
	# 
	# VxVm/CVM disks: reserve the diskgroups if no other node that
	# can master this logical host is in the membership.  Otherwise,
	# release the reservations.

	if [[ "${vm}" != "sds" ]]; then
	  # VxVM/CVM diskgroups
	  if anyone_else_present "${nodelist}" "${curr_members}" \
	    ${localhostname}; then
	    failure_fencing release private ${dglist}
	  else
	    failure_fencing reserve private ${dglist}
	  fi
	fi
      fi

      # we may need to mount the filesystem if there are any additions.
      # Hence call scnfs

      ${CLUSTERBIN}/scnfs ${clustname} mount ${lhost}
    fi
  fi
done
# now that all logical host have given away, calling FM_INIT
# so that it gets called on the logical hosts that are mastered
# on this node.(after this point we have the list of logical
# hosts that will remain mastered on this node)
# For logical hosts that will be taken over, loghost_sync will
# take care of it.
exec_fault_methods fm_init

}


set_loghost_mstate()
{
    typeset logname value

    logname=$1
    value=$2

    mstate=$(${CLUSTERBIN}/scccd ${clustname} LOGHOST_MSTATE query lname \
	${logname})
    if (( $? == 0 )); then
    	mode=${mstate#*:*:}
    	if [[ -n ${mode} && ${mode} != ${value} ]]; then

        	${CLUSTERBIN}/scccd ${clustname} LOGHOST_MSTATE remove lname \
		    ${logname}

        	if [[ $? -ne 0 ]]; then
			log_info "${pre}.3340" \
			    "Cannot remove LOGHOST_MSTATE for ${logname}."
		fi

        	${CLUSTERBIN}/scccd ${clustname} LOGHOST_MSTATE add \
		    "lname:mmode" "${logname}:${value}"

        	if [[ $? -ne 0 ]]; then
			log_info "${pre}.3341" \
		"Cannot set LOGHOST_MSTATE of ${logname} to ${value}."
        	fi
    	fi
    else
	log_info "${pre}.3342" "Cannot get current state for ${logname}."
    fi
}


########################################################################
#
# function loghoststep2_cmd
#
# This function executes step2 of the logical host reconfiguration. In
# this step, all nodes that are supposed to take over or master logical
# hosts will do so. The correctness of the cluster reconfiguration
# protocols should ensure that all logical hosts that are going to be
# taken over in this step should have been given up by their corresponding
# masters before this step.

function loghoststep2_cmd
{
typeset r rows
typeset nodelist dglist lhost
typeset nodelist_array primary_node fb_node
typeset mstate
typeset ccdfile
typeset readonly

if [[ "${vm}" = "sds" ]]; then
	# First remove any leftover files created by loghost_sync
	# PRE_ADDU indicating SDS diskset was taken read-only.
	rm -f ${CLUSTERVAR}/readonly*
fi

# retrieve all logical hosts in the cluster

ccdfile=$(${CLUSTERBIN}/ccdadm ${clustname} -w)

rows=$(${CLUSTERBIN}/scccd -f ${ccdfile}  ${clustname} LOGHOST query "lname" "")

for r in ${rows}; do
  lhost=${r%:*:*:*:*}
  lhost=${lhost#*:}

  dglist=${r%:*:*}
  dglist=${dglist#*:*:*:}
  dglist=$(print ${dglist} | tr ',' ' ')

  nodelist=${r%:*:*:*}
  nodelist=${nodelist#*:*:}
  nodelist=$(print ${nodelist} | tr ',' ' ')
  set -A nodelist_array ${nodelist}
  primary_node=${nodelist_array[0]}

  #
  # If the Logical Host is in maintenance mode then we should not
  # touch it.
  #
  mstate=$(${CLUSTERBIN}/scccd -f ${ccdfile} ${clustname} LOGHOST_MSTATE \
		query "lname" ${lhost})
  mstate=${mstate#*:*:}
  if [[ "${mstate}" -ne "1" ]]; then
		continue;
  fi



  # get the first backup node for this logical host

  fb_node=""
  first_backup_node "${curr_members}" "${nodelist}" fb_node

  # all processing for this step applies only if the local host is either
  # the primary or the first backup node for this logical host

  if [[ "${primary_node}" = "${localhostname}" || \
        "${fb_node}" = "${localhostname}" ]]; then

    # retrieve current master of the logical host
    curr_master=$(${CLUSTERBIN}/scccd -f ${ccdfile} ${clustname} LOGHOST_CM \
			query "lname" "${lhost}")

    # if there is no current master, the primary or the first backup (if
    # primary is not in the cluster membership) should take over this
    # logical host.

    if [[ -z "${curr_master}" ]]; then
      if ( [[ "${primary_node}" = "${localhostname}" ]] ) || \
         ( [[ "${fb_node}" = "${localhostname}" ]] && \
	    ! is_member "${curr_members}" ${primary_node} ) ; then

	${CLUSTERBIN}/scccd -r ${RETRIES} -w 5 ${clustname} LOGHOST_CM add \
		"lname:curr_master"  "${lhost}:${localhostname}"

	if [[ $? -ne 0 ]]; then
	  # LOGHOST_CM add could have failed because SDS diskset was
	  # taken read-only.  Check for existence of file indicating this
	  # happened.  This file would exist only in SDS configurations.

	  readonly=false
	  if [[ -f ${CLUSTERVAR}/readonly.${lhost} ]]; then
		# Set readonly flag to true so we
		# can put logical host into maint mode
		# below.  Remove the temp file created
		# by loghost_sync PRE_ADDU.
		readonly=true
		rm -f ${CLUSTERVAR}/readonly.${lhost}
	  fi

	  # Clean up the mess caused by LOGHOST_CM add failure

	  row=$(${CLUSTERBIN}/scccd ${clustname} LOGHOST_CM query lname \
		${lhost})
	  cm=${row#*:*:}
	  if [[ -n "${cm}" ]]; then
		# row exists, so remove it
	  	${CLUSTERBIN}/scccd -r ${RETRIES} -w 5 ${clustname} LOGHOST_CM \
		    remove lname "${lhost}"
	  	if [[ $? -ne 0 ]]; then
		    log_info "${pre}.4200" \
"Unable to to restore consistent data service status while taking over logical host ${lhost}"
		    exit 1
	  	fi
	  fi
	  log_info "${pre}.4020" "Take over of logical host ${lhost} failed."

	  if [[ "${readonly}" = "true" ]]; then
		# This code would be executed only in SDS configurations.
		log_info "${pre}.3343" "Putting ${lhost} into maintenance mode."
		# Set to maintenance mode.
		set_loghost_mstate ${lhost} 0
	  fi
	fi
      fi
      continue
    fi

    # If this host owns the logical host and and is already a cluster
    # member, then we just need to release the reservations on the disks if
    # any other host that can master this logical host is in the membership.
    # Otherwise, we reserve the disks.

    curr_master=${curr_master#*:*:}
    if [[ "${curr_master}" = "${localhostname}" ]]; then
      if [[ "${vm}" != "sds" ]]; then
	if anyone_else_present "${nodelist}" "${curr_members}" ${localhostname}; then
	  failure_fencing release private ${dglist}
	else
	  failure_fencing reserve private ${dglist}
	fi
      fi
    else

      # some other host is the current master. If the current master is not in
      # the cluster membership, then the primary or the first backup (if the
      # primary is not in the membership) should take over this logical host
      # after first removing the row
      # the loghost_sync will take care of calling fm_init.

      if ! is_member "${curr_members}" ${curr_master} ; then
	if ( [[ "${primary_node}" = "${localhostname}" ]] ) || \
	   ( [[ "${fb_node}" = "${localhostname}" ]] && \
	        ! is_member "${curr_members}" ${primary_node} ) ; then

	    ${CLUSTERBIN}/scccd -r ${RETRIES} -w 5 ${clustname} LOGHOST_CM remove \
			   lname "${lhost}"

	    if [[ $? -ne 0 ]]; then
		log_info "${pre}.5001" "Unable to giveup logical host ${lhost}, ignoring logical host ${lhost}"
		continue;
	    fi

	    ${CLUSTERBIN}/scccd -r ${RETRIES} -w 5 ${clustname} LOGHOST_CM add \
		"lname:curr_master" "${lhost}:${localhostname}"

	    if [[ $? -ne 0 ]]; then
		# LOGHOST_CM add could have failed
		# because SDS diskset was taken
		# read-only.  Check for existence of
		# file indicating this happened.
		# This file would only exist in SDS
		# configurations.
		readonly=false
		if [[ -f ${CLUSTERVAR}/readonly.${lhost} ]]; then
		    rm -f ${CLUSTERVAR}/readonly.${lhost}
		    readonly=true
		fi

		#
		# Clean up the mess caused by LOGHOST_CM add failure.
		#
                row=$(${CLUSTERBIN}/scccd ${clustname} LOGHOST_CM query \
		    lname ${lhost})
		cm=${row#*:*:}
		if [[ -n "${cm}" ]]; then
		    # row exists, so remove it

		    ${CLUSTERBIN}/scccd -r ${RETRIES} -w 5 ${clustname} \
			LOGHOST_CM remove  lname "${lhost}"

		    if [[ $? -ne 0 ]]; then
			log_info "${pre}.4200" \
			"Unable to to restore consistent data service status while taking over logical host ${lhost}"
			exit 1
		    fi
		fi

		log_info "${pre}.4020" "Take over of logical host ${lhost} failed."
		if [[ "${readonly}" = "true" ]]; then
		    # This code would be executed only in SDS configs.
		    log_info "${pre}.3344" \
			"Putting ${lhost} into maintenance mode."
		    # Set to maint mode.
		    set_loghost_mstate ${lhost} 0
		fi
	    fi
	fi
      fi
    fi
  fi
done
}



function check_lnode_format
{
    typeset node fmt_present

#
# make sure only one node tries to add the format
#
    for node in ${currnodes}; do
	if [ "${node}" != "${localnodeid}" ]; then
		return 0
	else
		break
	fi
    done

#
# check if format already exists in the CCD
#
    ccdfile=$(${CLUSTERBIN}/ccdadm ${clustname} -w)
    fmt_present=$(${CLUSTERBIN}/scccd -f ${ccdfile} ${clustname} LOCKNODE \
	query "" "" )
    if (( $? == 8 )); then
	${CLUSTERBIN}/scccd ${clustname} LOCKNODE add_format lnode ccd_nofreeze
	if [[ $? != 0 ]]; then
		exit 1
	fi
    fi

    return 0
}

function check_cnode_format
{
    typeset node fmt_present

#
# make sure only one node tries to add the format
#
    for node in ${currnodes}; do
	if [ "${node}" != "${localnodeid}" ]; then
		return 0
	else
		break
	fi
    done

#
# check if format already exists in the CCD
#
    ccdfile=$(${CLUSTERBIN}/ccdadm ${clustname} -w)
    fmt_present=$(${CLUSTERBIN}/scccd -f ${ccdfile} ${clustname} CLUSTM \
	query "" "")
    if (( $? == 8 )); then
	${CLUSTERBIN}/scccd ${clustname} CLUSTM add_format curr_mem \
		ccd_nofreeze
	if [[ $? != 0 ]]; then
		exit 1
	fi
    fi

    return 0
}

function clear_membership
{
  typeset apps
  typeset rows
  typeset row

#
# make sure only one node tries to add the format
#
    for node in ${currnodes}; do
        if [ "${node}" != "${localnodeid}" ]; then
                return 0
        else
                break
        fi
    done
  
  rows=$(${CLUSTERBIN}/scccd ${clustname} CLUSTM query curr_mem '')
  if [[ -n ${rows} ]]; then
    for row in ${rows}; do
	apps=${row#*:}
  	${CLUSTERBIN}/scccd ${clustname} CLUSTM remove curr_mem "${apps}"
    done
  fi
}

function add_membership
{
  typeset rows
  typeset row
  typeset apps
  typeset currmembers

#
# make sure only one node tries this
#
    for node in ${currnodes}; do
        if [ "${node}" != "${localnodeid}" ]; then
                return 0
        else
                break
        fi
    done

  rows=$(${CLUSTERBIN}/scccd ${clustname} CLUSTM query curr_mem '')
  if [[ -n ${rows} ]]; then
    for row in ${rows}; do
	apps=${row#*:}
      	${CLUSTERBIN}/scccd ${clustname} CLUSTM remove curr_mem ${apps}
    done
  fi
  currmembers=$(print ${curr_members} | tr ' ' ',')
  ${CLUSTERBIN}/scccd ${clustname} CLUSTM add curr_mem "${currmembers}"
}

function is_primary_joining
{
	typeset primary_node
	typeset prev_mem
	typeset ccdfile

	primary_node=$1

	ccdfile=$(${CLUSTERBIN}/ccdadm ${clustname} -w)
	prev_mem=$(${CLUSTERBIN}/scccd -f ${ccdfile} ${clustname} CLUSTM query curr_mem '')
	prev_mem=${prev_mem#*:}

	if print ${prev_mem} | /bin/grep -i ${primary_node} >/dev/null 2>&1
	then
		return 1
	else
		return 0
	fi

}

function clear_ccd_lock
{
	${CLUSTERBIN}/scccd ${clustname} LOCKNODE remove lnode ${clustname}
}


localhostname=$(${CLUSTERBIN}/cdbmatch cluster.node.${localnodeid}.hostname \
			${cdbfile})


curr_members=""
for n in ${currnodes}; do
  nodename=$(${CLUSTERBIN}/cdbmatch cluster.node.${n}.hostname ${cdbfile})
  curr_members="${curr_members} ${nodename}"
done

# determine which volume manager is in use
CVM=3
VxVM=4
SDS=5
pdbapps=$(enmatch cluster.pdbapps)
vm=$(${CLUSTERBIN}/appmatch ${pdbapps} ${CVM})
if [[ "${vm}" = "1" ]]; then
	vm=cvm
else
	vm=$(${CLUSTERBIN}/appmatch ${pdbapps} ${VxVM})
	if [[ "${vm}" = "1" ]]; then
		vm=vxvm
	else
		vm=$(${CLUSTERBIN}/appmatch ${pdbapps} ${SDS})
		if [[ "${vm}" = "1" ]]; then
			vm=sds
		else
			log_info "${pre}.4340" \
	 		   "Volume manager is not properly configured."
			exit 1
		fi
	fi
fi

case ${CURRSTEP} in
  cmmabort | stopnode)
	set_retries_for_ccd_operation stopnode;
	loghoststop_cmd ;;
  cmmstep4)
	check_lnode_format;
	clear_ccd_lock;
	exec_fault_methods fm_stop ;;
  cmmstep10)
	# first parameter should match the step number;
	set_retries_for_ccd_operation step10
	loghoststep1_cmd ;;
  cmmstep11)
	# first parameter should match the step number.
	set_retries_for_ccd_operation step11
	loghoststep2_cmd ;;
  cmmstep12)
	exec_fault_methods fm_start
	check_cnode_format
	add_membership ;;
  *) ;;
esac

exit 0
