#!/sbin/sh -
# $Id: vxrelocd.sh,v 1.29.65.6.2.1 2003/02/14 17:02:06 rohit Exp $
#ident "$Source: /project/unixvm-cvs/src/sol/cmd/vxvm/support/vxrelocd.sh,v $"

# Copyright (c) 2001 VERITAS Software Corporation.  ALL RIGHTS RESERVED.
# UNPUBLISHED -- RIGHTS RESERVED UNDER THE COPYRIGHT
# LAWS OF THE UNITED STATES.  USE OF A COPYRIGHT NOTICE
# IS PRECAUTIONARY ONLY AND DOES NOT IMPLY PUBLICATION
# OR DISCLOSURE.
# 
# THIS SOFTWARE CONTAINS CONFIDENTIAL INFORMATION AND
# TRADE SECRETS OF VERITAS SOFTWARE.  USE, DISCLOSURE,
# OR REPRODUCTION IS PROHIBITED WITHOUT THE PRIOR
# EXPRESS WRITTEN PERMISSION OF VERITAS SOFTWARE.
# 
#               RESTRICTED RIGHTS LEGEND
# USE, DUPLICATION, OR DISCLOSURE BY THE GOVERNMENT IS
# SUBJECT TO RESTRICTIONS AS SET FORTH IN SUBPARAGRAPH
# (C) (1) (ii) OF THE RIGHTS IN TECHNICAL DATA AND
# COMPUTER SOFTWARE CLAUSE AT DFARS 252.227-7013.
#               VERITAS SOFTWARE
# 1600 PLYMOUTH STREET, MOUNTAIN VIEW, CA 94043


__VXVM_CONNECT_WAIT=WAIT
export __VXVM_CONNECT_WAIT

#
# Need to force C locale setting so that
# english text is used during processing.
#
ENV_LANG=$LANG
ENV_LC_ALL=$LC_ALL

#
# I18N:
# As part of requirement for non-english environment,
# the mail message will contain both english and
# non-english text in the email body.
#
current_lang=`/usr/bin/locale | fgrep LC_MESSAGES | awk 'BEGIN { FS = "=" } { print $2 }'`
if [ "$current_lang" = "\"C\"" ]
then
        english_only_email=1
else
        english_only_email=0
fi

LANG=C
export LANG
LC_ALL=C
export LC_ALL

: ${VOLROOT_DIR:=$__VXVM_ROOT_DIR}
. ${VOL_SCRIPTS_LIB:-$VOLROOT_DIR/usr/lib/vxvm/lib}/vxcommon

prog=vxrelocd

#
# Need to make sure that hot relocation functionality
# is disabled if "hot relocation" feature is turned on
# by license.
#
# Note: Per PM, hot-relocation is a form of mirroring. So we
#  will check for "mirror" instead.)
#
# The license restriction will be enforced in two places:
#   1. Prevent vxrelocd daemon from starting (this code)
#   2. THe actual command to relocate plex data will return an error.
#         i.e. vxassist -r -g <diskgrp> move vol !<disk1> <disk2>
#                will return a "invalid license" error
#
VXLICTEST=/sbin/vxlictest

if [ -x $VXLICTEST ]
then
	$VXLICTEST -n "VERITAS Volume Manager" -f "VxVM" -sup
	general_lic_avail=$?
	$VXLICTEST -n "VERITAS Volume Manager" -f "PHOTON" -sup
	photon_lic_avail=$?
	$VXLICTEST -n "VERITAS Volume Manager" -f "SSA" -sup
	ssa_lic_avail=$?

	if [ $general_lic_avail -eq 1 ] && [ $photon_lic_avail -eq 1 ] && [ $ssa_lic_avail -eq 1 ] 
	then
		egettxt >&2 "\
No valid licenses found. VM Full license(s) is needed to enable hot-relocation feature.
Will terminate the hot-relocation daemon\n" vxvmshm:1733
		exit 1
	fi
else
	export VXLICTEST; egettxt >&2 "\
Unable to find license. ($VXLICTEST is missing). Will terminate the hot-relocation daemon\n" vxvmshm:1756
	exit 1

fi



# Try to use I18N-compliant mailer problem if possible
if [ -f /usr/SUNWale/bin/mailx ]
then
	MAILER=/usr/SUNWale/bin/mailx
else
	MAILER=mailx
fi



set_to_env_locale()
{
	LANG=$ENV_LANG; export LANG
	LC_ALL=$ENV_LC_ALL; export LC_ALL
}

set_to_c_locale()
{
	LANG=C; export LANG
	LC_ALL=C; export LC_ALL
}

usage()
{
	set_to_env_locale

	export prog; egettxt >&2 \
"Usage: $prog [-o vxrecover_arg] [-O old_version] [-s save_max] [mail-address ...]" \
	vxvmshm:956
	exit 1
}

# Find all dead volumes that have storage on a speficic DM
deadvols()
{
	dmname=$1
	dgname=$2
	shift 2

	vxprint -g $dgname -nv -e 'v_read_pol!=V_RAID &&
			!(any aslist.pl_kstate=ENABLED) &&
			(any aslist.aslist.sd_dmname="'$dmname'")'
}

deadraids()
{
	dmname=$1
	dgname=$2
	shift 2

	vxprint -g $dgname -p -F"%vname" \
		-e '(any aslist.sd_dmname="'$dmname'") &&
		    pl_layout=RAID && assoc.v_unusable'	
}

# Find all affected volumes that have storage on a specific DM but
# are not completely bashed
affvols()
{
	dmname=$1
	dgname=$2
	shift 2

	vxprint -g $dgname -nv -e '(v_read_pol!=V_RAID) && 
			(any aslist.pl_kstate=ENABLED) && 
			(any aslist.aslist.sd_dmname="'$dmname'")'

	vxprint -g $dgname -p -F'%vname' \
		-e '(any aslist.sd_dmname="'$dmname'") &&
		    pl_layout=RAID && !assoc.v_unusable'
}

o_args=
check_cnt=2
oldhot=
savemax=

while getopts :vs:O:o: c
do
	case $c in
	s)	savemax=$OPTARG;;
	O)	oldhot=$OPTARG;;
	o)	o_args=$OPTARG;;
	v)	verbose="on";;
	?)	usage;;
	esac
done

case "$oldhot" in
	"1"* )		oldhot=pre3.1;;
	"2"* )		oldhot=pre3.1;;
	"3.0"* )	oldhot=pre3.1;;
esac

# maximum number of config files to be kept per disk group
[ "X$savemax" = "X" ] && savemax=32
[ X`echo $savemax | awk '/^[0-9]+$/ {print $1}'` = "X" ] && usage

shift `expr $OPTIND - 1`
[ $# -eq 0 ] && set root

recipients="$@"

shift $#

# set the da_name for root disk
set_OS_variables

# check c#t#d# format
hr_checkctd()
{
	echo $1 | grep "_" 2>/dev/null 1>&2
	if [ $? -eq 0 ]
	then
		return 0
	fi

	if 	[ "X`echo $1 | \
			awk '/^m?c[0-9]+t[0-9]+d[0-9]+s[0-9]+$/ {print $1}'`" != "X" ] ||
		[ "X`echo $1 | \
			awk '/^m?c[0-9]+t[0-9]+d[0-9]+$/ {print $1}'`" != "X" ] ||
		[ "X`echo $1 | \
			awk '/^m?c[0-9]+[tds][0-9]+$/ {print $1}'`" != "X" ] ||
		[ "X`echo $1 | \
			awk '/^m?c[0-9]+$/ {print $1}'`" != "X" ]
	then
		return 0
	else
		return 1
	fi
}

# display da_name in c#t#d#s# format
hr_da2ctds()
{
	# check for the ctds format
	ctdpat=$1
	hr_checkctd "$ctdpat"
    if [ $? -eq 0 ]; then
        echo $ctdpat
        return
    fi

	# get CTD info for EMC disks
	if [ -s /etc/powermt ]; then
		emcdevno=`echo $ctdpat | tr -d [a-z] | tr -d [A-Z]`
		activedas=`/etc/powermt display dev=$emcdevno 2> /dev/null | \
						awk '$3=="Active" || $3=="Standby" {print $4}'`
		if [ "X$activedas" != "X" ]; then
			for emcda in $activedas; do
				hr_checkctd "$emcda"
    			if [ $? -eq 0 ]; then
					vxdisk list $emcda > /dev/null 2>&1
					if [ $? -eq 0 ] ; then
						echo $emcda
						return
					fi      
				fi
			done
		fi
	fi

	# get controller info
	ctlr_num=c`vxdisk getctlr $ctdpat 2> /dev/null `
	if [ $? -eq 0 ] && [ "X$ctlr_num" != "Xc-1" ] ; then
		hr_checkctd "$ctlr_num"
		if [ $? -eq 0 ]; then
			echo $ctlr_num
			return
		fi
	fi

	# if got here, da_name format is different from cNtNdN format.
	# relocation still work but disk selection based on CTD not supported.
    echo "-"
}

hr_add_avail_disks()
{
	devpat=$1
	> $hr_tmp2
	> $hr_tmp3
	
	# arranging dms, putting active dms first
	echo $devpat | grep "_" 2>/dev/null 1>&2
	if [ $? -eq 0 ]
	then
		cat $hr_tmp1 | egrep "^(.+)?${devpat}[0-9]+ " |\
					 awk '{print $2}' |\
		while read a; do
			for b in $voldms ; do
				if [ "X$a" = "X$b" ]; then
					echo $a >> $hr_tmp2
				else
					echo $a >> $hr_tmp3
				fi
			done
		done
	else
		cat $hr_tmp1 | egrep "^m?${devpat}([A-Za-z]+|-|$| |	)" |\
							awk '{print $2}' |\
		while read a; do
			for b in $voldms ; do
				if [ "X$a" = "X$b" ]; then
					echo $a >> $hr_tmp2
				else
					echo $a >> $hr_tmp3
				fi
			done
		done
	fi

	cat $hr_tmp2 $hr_tmp3 | sed -e '/^$/d' | \
	while read cdm; do
		[ X`cat $hr_availdms | awk '$1=="'$cdm'" {print $1}'` != "X" ] &&
			continue
		vxdisk -g $failed_dg check $cdm > /dev/null 2>&1
		[ $? -eq 0 ] && echo $cdm >> $hr_availdms
	done
}

# get list of disks that are marked as spares/free for hot-relocation,
# sorted by the minimal distance to the failed dm

hr_list_avail_disks()
{
	spareflag=$1
	freeuseflag=$2
	hr_availdms=$3

	> $hr_tmp1
	> $hr_tmp2
	> $hr_availdms

	if [ "X`vxprint -g $failed_dg -dF %nohotuse $failed_dm`" = "X-" ]; then
		[ "X$freeuseflag" = "Xoff" ] && freeuseflag="-"
	fi

	vxprint -g $failed_dg -Qqd -F \
		'%name %daname %spare %nohotuse %nodarec %failing %reserve' | \
		awk '$3=="'$spareflag'" && $4=="'$freeuseflag'" && $5=="off" && \
			$7=="off" && \
			$6=="off" && $1!="'$failed_dm'" {print $2 " " $1}' > \
			$hr_tmp2 2> /dev/null
	[ ! -s $hr_tmp2 ] && return

	cat $hr_tmp2 | \
	while read da dm; do
		echo `hr_da2ctds $da` $dm >> $hr_tmp1
	done

	failed_da=`vxprint -g $failed_dg -Qqd -F '%last_da_name' $failed_dm `
	failed_da=`hr_da2ctds $failed_da | sed 's/mc/c/g' `
	voldms=`vxprint -g $failed_dg -Qqse 'assoc.assoc.v_name="'$failed_v'"' \
				-F %dmname 2> /dev/null`

	# check for enclosure based names
	echo $failed_da | grep "_" 2>/dev/null 1>&2
	if [ $? -eq 0 ]
	then
		enclosure=`vxdmpadm getsubpaths dmpnodename=$failed_da |\
					 tail +3 | head -1 | awk '{ print $6}'`_
		hr_add_avail_disks $enclosure
		hr_add_avail_disks "_"
		hr_add_avail_disks ""
		return
	fi

	# get disks with same controller, target, and devs
	hr_add_avail_disks `echo $failed_da | awk '{FS="s"; print $1}'`

	# get disks with same controller, target
	hr_add_avail_disks `echo $failed_da | awk '{FS="d"; print $1}'`

	# get disks with same controller
	hr_add_avail_disks `echo $failed_da | awk '{FS="t"; print $1}'`

	# get everything else
	hr_add_avail_disks ""
}

# go through the list and find disks with enough free space
# space for root and swap should be contig

hr_select_spacedm()
{
	ifaillen=$1
	rootlen=$2
	swaplen=$3
	dmlist=$4
	spacedms=$5

	> $spacedms
	cat $dmlist | \
	while read fldm; do
		[ X`cat $spacedms | awk '$1=="'$fldm'" {print $1}'` != "X" ] &&
			continue
		freespace=`vxdg -g $failed_dg -qa free $fldm 2> /dev/null | \
						awk '{sum+=$5} END {printf "%d\n", 0+sum}'`
		[ "X$freespace" = "X" ] && continue
		if [ $freespace -ge $ifaillen ]; then
			allchunks=`vxdg -aqg $failed_dg free $fldm | \
							awk '{printf "%d\n", 0+$5}' | sort -nr `
			chunk1=`echo $allchunks | awk '{print $1}'`
			chunk2=`echo $allchunks | awk '{print $2}'`
			if	[ `expr $rootlen + $swaplen` -le $chunk1 ] || \
				[ $rootlen -le $chunk1 -a $swaplen -le $chunk2 ] || \
				[ $rootlen -le $chunk2 -a $swaplen -le $chunk1 ]; then
				echo $fldm >> $spacedms
			fi
		fi
	done
}

# find a replacement dm where failed SDs can go without any conflicts with
# the existing SDs on it.

hr_get_cleardm()
{
	hr_failvp=$1
	spacedms=$2

	cat $spacedms | \
	while read newdm ; do
		> $hr_tmp2
		# list existing volumes and plexes whose SDs are on a dm
		> $hr_newvp
		vxprint -g $failed_dg -Qqse 'sd_dmname="'$newdm'" ' \
				-F '%v_name %pl_name' 2> /dev/null | \
				sed -e '/^$/d' | sort | uniq > $hr_newvp

		# check for overlap/conflict between a failed plex and a
		# plex that is existing on a disk intended for relocation

		cat  $hr_newvp | \
		while read newdmv newdmp; do

			# skip if not associated.
			[ "X$newdmv" = "X-" ] && continue

			cat $hr_failvp | \
			while read failv failp ; do

				# if layered vols, get the top common plex and its layout.
				nlayerv=$newdmv
				nlayerp=""
				while [ "X$nlayerv" != "X" ]; do
					nlayerp="$nlayerp "`vxprint -g $failed_dg -Qse \
										'sd_dmname="'$nlayerv'"' -F %pl_name`
					nlayerv=`vxprint -g $failed_dg -Qse \
										'sd_dmname="'$nlayerv'"' -F %v_name`
				done

				faillyp=""
				newlyp=""
				faillylayout=""
				flayerv=$failv
				while [ "X$flayerv" != "X" ]; do
					flayerp=`vxprint -g $failed_dg -Qse \
										'sd_dmname="'$flayerv'"' -F %pl_name`
					for cplex in $nlayerp; do
						if [ "$cplex" = "$flayerp" ]; then
							faillylayout=`vxprint -g $failed_dg \
											-F '%layout' "$cplex" `
							newlyp=$cplex
							faillyp=$cplex
							break 2
						fi
					done
					flayerv=`vxprint -g $failed_dg -Qse \
										'sd_dmname="'$flayerv'"' -F %v_name`
				done

				# if sds are part of the same volume, check for overlap.
				if [ "$newdmv" = "$failv" ] || \
					[ "X$newlyp" = "X$faillyp" -a "X$newlyp" != "X" ]; then

					faillayout=`vxprint -g $failed_dg -F '%layout' "$failp" `

					# 0 = plex is not a DRL log, 1 = yes.
					newlog=X`vxprint -g $failed_dg -Qpe 'pl_name="'$newdmp'" &&
							aslist.sd_is_log' -F %name | \
							wc -l | awk '{print $1}' `
					faillog=X`vxprint -g $failed_dg -Qpe 'pl_name="'$failp'" &&
							aslist.sd_is_log' -F %name | \
							wc -l | awk '{print $1}' `

					# a failed sd can go to a dm if there is no conflict with
					# another sd on that dm and all are false:
					# 1. sds of the same plex, plex!=concat
					# 2. sds of a different plex of the same volume and
					#		!(one plex is a DRL log)
					# 3. sds of a different plex and volume, same non-CONCAT
					# 		super-plex and both sub-plexes are not DRL logs

	 				if  [ "$newdmp" = "$failp" -a \
							"$faillayout" != "CONCAT" ] || \
						[ "$newdmp" != "$failp" -a \
							"$newlog" = "$faillog" -a \
							"$newdmv" = "$failv" ] || \
						[ "$newdmv" != "$failv" -a \
							"$faillylayout" != "CONCAT" -a \
							"$newlog" = "X0" -a \
							"$faillog" = "X0" ]; then
						echo $newdmp > $hr_tmp2
						break
 					fi
				fi
			done
			[ -s $hr_tmp2 ] && break
		done
		if [ ! -s $hr_tmp2 ]; then
			echo "$newdm"
			break
		fi
	done
}

# saving the current config to a file before attempting to hot-relocate

hr_save_config()
{
	[ $savemax -eq 0 ] && return

	# one save per trip
	[ -f $failed_dir/$failed_dg.$failed_file ] && return

	[ -d $failed_dir ] || mkdir -p $failed_dir

	touch $failed_dir/$failed_dg.$failed_file > $hr_tmp4 2>&1

	if [ $? -ne 0 ]; then
		echo "\
\\nVERITAS Volume Manager failed to save the current configuration in:
$failed_dir/$failed_dg.$failed_file\\n" > /tmp/mail_vxrelocd.$$
cat $hr_tmp4 >> /tmp/mail_vxrelocd.$$
		if [ $english_only_email -eq 0 ]
		then
			echo "\n===========================================\n" >> /tmp/mail_vxrelocd.$$
			set_to_env_locale
			(export hr_tmp4 failed_dir failed_dg failed_file ; egettxt "\
\\nVERITAS Volume Manager failed to save the current configuration in:
$failed_dir/$failed_dg.$failed_file
\\n" vxvmshm:976
cat $hr_tmp4
			) >> /tmp/mail_vxrelocd.$$
			set_to_c_locale
		fi
		$MAILER -s "$_ms" "$recipients" < /tmp/mail_vxrelocd.$$
		rm /tmp/mail_vxrelocd.$$
		return
	else
		echo "\
\\nVERITAS Volume Manager is preparing to relocate for diskgroup $failed_dg.
Saving the current configuration in:
$failed_dir/$failed_dg.$failed_file
\\n" > /tmp/mail_vxrelocd.$$
		if [ $english_only_email -eq 0 ]
		then
			echo "\n===========================================\n" >> /tmp/mail_vxrelocd.$$
			set_to_env_locale
			(export failed_dir failed_dg failed_file ; egettxt "\
\\nVERITAS Volume Manager is preparing to relocate for diskgroup $failed_dg.
Saving the current configuration in:
$failed_dir/$failed_dg.$failed_file
\\n" vxvmshm:977
			) >> /tmp/mail_vxrelocd.$$
			set_to_c_locale
		fi
		$MAILER -s "$_ms" "$recipients" < /tmp/mail_vxrelocd.$$
		rm /tmp/mail_vxrelocd.$$
	fi

	set_to_env_locale
	echo "#vxprint -g $failed_dg -ht" > \
			$failed_dir/$failed_dg.$failed_file 2>&1

	vxprint -g $failed_dg -ht    >> $failed_dir/$failed_dg.$failed_file 2>&1

	echo "#vxprint -g $failed_dg -mpvsh" >> \
			$failed_dir/$failed_dg.$failed_file 2>&1
	vxprint -g $failed_dg -mpvsh >> $failed_dir/$failed_dg.$failed_file 2>&1
	set_to_c_locale

	# remove the oldest and keep only the most recent
	savecount=`ls -l $failed_dir/$failed_dg.* 2> /dev/null | wc -l`
	if [ $savecount -gt $savemax ]; then
		rm -f `ls -lrt $failed_dir/$failed_dg.* | \
				sed -n '1p' | awk '{print $NF}' `
	fi
}

# save target dm into history file
hr_put_hist()
{
	histdm=$1
	[ "X$histdm" = "X" ] && return
	findres=`cat $hr_hist | \
			awk '$1=="'$failed_dm'" && $2=="'$histdm'" && \
				$3=="'$failed_dg'" {print $1}' `
	[ "X$findres" = "X" ] && echo $failed_dm $histdm $failed_dg >> $hr_hist
}

# unrelocate flags don't exist in diskgroup versions before 70.
hr_set_unreloc_flag()
{
	dgversion=`vxdg list $1 | grep "^version" | awk '{print $2}'`
	if [ -z "$dgversion" ] || [ "$dgversion" -lt "70" ]; then
		eval "${4}="
	else
		eval "${4}=\"${2} ${3}\""
	fi
}

# if the moved volume is a swap and is on top of a dump device,
# reconfig the device.
hr_conf_dumpdev()
{
	_ckdg=$1
	_ckvol=$2
	_fdumpdm=$3
	_ndumpdm=$4
	[ -n "$_ckdg" -a -n "$_ckvol" -a -n "$_fdumpdm" -a -n "$_ndumpdm" ] ||
		return
	# reconfig dump device only if volume is a swap
	[ "X`vxprint -g $_ckdg -F %use_type $_ckvol`" = "Xswap" ] || return

	# get current dump partition
	if [ -x /usr/sbin/dumpadm ]; then
		# nothing to do if savecore is off
		[ "X`dumpadm | grep \"Savecore enabled:\" | \
			awk '{print $3}'`" != "Xyes" ] && return
		_curr_dump_part=`dumpadm | grep "Dump device:" | awk '{print $3}'`
	else
		_curr_dump_part=`echo "dumpfile+0x10/s" |
							adb -k /dev/ksyms /dev/mem |
							grep "^dumpfile" | awk '{print $2}'`
	fi
	[ -n "$_curr_dump_part" -a "X$_curr_dump_part" != "Xnone" ] || return
	_orig_dumpdev=`subdisk_to_partition  $_ckdg $_ckvol $_fdumpdm`

	# skip if the underlying partition is different from the dump device
	if [ -n "$_orig_dumpdev" ] && [ "$_orig_dumpdev" != "$_curr_dump_part" ]
	then
		return
	fi

	# get current dump path
	dogi_path_to_slice $_curr_dump_part _curr_dump_slice
	dogi_name_is_slice $_curr_dump_slice || return
	dogi_slice_to_device $_curr_dump_slice _curr_dump_device
	dogi_device_blkpath $_curr_dump_device _curr_dump_blkpath
	[ -n "$_curr_dump_blkpath" ] || return

	# get block path of the failed disk
	_orig_dump_slice=`vxprint -g $_ckdg -F %last_da_name $_fdumpdm`
	dogi_name_is_slice $_orig_dump_slice || return
	dogi_slice_to_device $_orig_dump_slice _orig_dump_device
	dogi_device_blkpath $_orig_dump_device _orig_dump_blkpath
	[ -n "$_orig_dump_blkpath" ] || return
	[ "$_curr_dump_blkpath" = "$_orig_dump_blkpath" ] || return

	_new_dumpdev=`subdisk_to_partition  $_ckdg $_ckvol $_ndumpdm`
	[ -n "$_new_dumpdev" ] || return

	if [ ! -x /usr/sbin/dumpadm ]; then
		echo "\
\\nVolume $_ckvol of disk group $_ckdg was moved from disk $_fdumpdm
to disk $_ndumpdm. A dump device $_curr_dump_part
was found on disk $_fdumpdm. A reboot is needed to reconfigure
the device.\\n" > /tmp/mail_vxrelocd.$$
		if [ $english_only_email -eq 0 ]
		then
			echo "\n========================================\n" >> /tmp/mail_vxrelocd.$$
			set_to_env_locale
			(export _ckvol _curr_dump_part _fdumpdm _ndumpdm _ckdg ; egettxt "\
\\nVolume $_ckvol of disk group $_ckdg was moved from disk $_fdumpdm
to disk $_ndumpdm. A dump device $_curr_dump_part
was found on disk $_fdumpdm. A reboot is needed to reconfigure the device.
\\n" vxvmshm:1476) >> /tmp/mail_vxrelocd.$$
			set_to_c_locale
		fi
		$MAILER -s "$rootdm_ms" "$recipients" < /tmp/mail_vxrelocd.$$
		rm /tmp/mail_vxrelocd.$$
		return
	fi

	# set dump device to new partition
	dumpadm -d $_new_dumpdev > /dev/null 2>&1

	if [ $? -eq 0 ]; then
		echo "\
\\nVolume $_ckvol of disk group $_ckdg was moved from disk $_fdumpdm
to disk $_ndumpdm. A dump device $_curr_dump_part
was found on disk $_fdumpdm and was reconfigured
to $_new_dumpdev.
\\n" > /tmp/mail_vxrelocd.$$
		if [ $english_only_email -eq 0 ]
		then
			echo "\n========================================\n" >> /tmp/mail_vxrelocd.$$
			set_to_env_locale
			(export _ckvol _fdumpdm _ndumpdm _ckdg _curr_dump_part _new_dumpdev
			egettxt "\
\\nVolume $_ckvol of disk group $_ckdg was moved from disk $_fdumpdm
to disk $_ndumpdm. A dump device $_curr_dump_part
was found on disk $_fdumpdm and was reconfigured
to $_new_dumpdev.
\\n" vxvmshm:1268
			) >> /tmp/mail_vxrelocd.$$
			set_to_c_locale
		fi
	else
		echo "\
\\nVolume $_ckvol of disk group $_ckdg was moved from disk $_fdumpdm
to disk $_ndumpdm. A dump device $_curr_dump_part
was found on disk $_fdumpdm. Attempt to reconfigure
the device to $_new_dumpdev failed.
\\n" > /tmp/mail_vxrelocd.$$
		if [ $english_only_email -eq 0 ]
		then
			echo "\n========================================\n" >> /tmp/mail_vxrelocd.$$
			set_to_env_locale
			(export _ckvol _fdumpdm _ndumpdm _ckdg _curr_dump_part _new_dumpdev
			egettxt "\
\\nVolume $_ckvol of disk group $_ckdg was moved from disk $_fdumpdm
to disk $_ndumpdm. A dump device $_curr_dump_part
was found on disk $_fdumpdm. Attempt to reconfigure
the device to $_new_dumpdev failed.
\\n" vxvmshm:1270
			) >> /tmp/mail_vxrelocd.$$
			set_to_c_locale
		fi
	fi
	$MAILER -s "$rootdm_ms" "$recipients" < /tmp/mail_vxrelocd.$$
	rm /tmp/mail_vxrelocd.$$

}

# If an error occurs on the rootdisk, another disk will be selected and
# all vols will be moved to the new disk. If the entired rootdisk can't be
# moved then nothing will be done.

hr_move_rootdm()
{
	rootdm_ms="VxVM relocation of rootdisk on host `uname -n`"

	# If there is a disabled vol, then nothing will be moved.
	failrootvs=""
	for a in `vxprint -g $failed_dg -ve \
				'aslist.aslist.sd_dmname="'$failed_dm'"' -F %name `; do
		goodpl_name=`vxprint -g $failed_dg -pe 'pl_state!="LOG"
					&& pl_v_name=="'$a'" && pl_kstate==ENABLED
					&& pl_state !~ /SNAP/ && !pl_log' -F %name`
		if [ "X$goodpl_name" = "X" ] ||
			[ X`vxprint -g $failed_dg -ve 'v_name="'$a'" && \
					v_kstate!=ENABLED' -F %name` = "X$a" ]; then
			failrootvs="$failrootvs $a"
		fi
	done


	if [ "X$failrootvs" != "X" ]; then
		echo "\
\\nRootdisk $failed_dm of disk group $failed_dg experienced a failure,
but no relocation will be done because its volumes:
$failrootvs
have no suitable mirror from which to recover data.
\\n"  > /tmp/mail_vxrelocd.$$
		if [ $english_only_email -eq 0 ]
		then
			echo "\n========================================\n" >> /tmp/mail_vxrelocd.$$
			set_to_env_locale
			(export failed_dm failed_dg failrootvs ; egettxt "\
\\nRootdisk $failed_dm of disk group $failed_dg experienced a failure,
but no relocation will be done because its volumes:
$failrootvs
have no suitable mirror from which to recover data.
\\n" vxvmshm:972
			) >> /tmp/mail_vxrelocd.$$
			set_to_c_locale
		fi
		$MAILER -s "$rootdm_ms" "$recipients"  < /tmp/mail_vxrelocd.$$
		rm /tmp/mail_vxrelocd.$$
		return 11
	fi

	# get list of plexes to move with root and swap vols first so to get
	# the contiguous allocs, larger plex first

	> $hr_failvp
	rootlen=`vxprint -g $failed_dg -Qpe 'aslist.sd_dmname="'$failed_dm'" &&
				assoc.v_use_type="root"' -F '%len %v_name %pl_name' | \
				awk '{print $2 " " $3 | " sort | uniq >> '$hr_failvp'";
						sum+=$1} END {printf "%d", 0+sum}' `
	swaplen=`vxprint -g $failed_dg -Qpe 'aslist.sd_dmname="'$failed_dm'" &&
				assoc.v_use_type="swap"' -F '%len %v_name %pl_name' | \
				awk '{print $2 " " $3 | " sort | uniq >> '$hr_failvp'";
						sum+=$1} END {printf "%d", 0+sum}' `
	[ "X$swaplen" = "X" ] && swaplen=0
	[ "X$rootlen" = "X" ] && rootlen=0

	if [ $swaplen -gt $rootlen ]; then
		cat $hr_failvp > $hr_tmp1
		cat $hr_tmp1 | grep -n '.*' | sort -n -r | \
				sed 's/^.*://' > $hr_failvp
	fi

	# list of log plexes to be moved.
	logslen=`vxprint -g $failed_dg -se 'sd_dmname="'$failed_dm'" &&
					(assoc.pl_state="LOG" || assoc.pl_state="BADLOG" ||
					assoc.pl_state="SNAPDONE" || assoc.pl_state="SNAPATT" ||
					sd_is_log) ' -F '%len %v_name %pl_name' | \
					awk '{print $2 " " $3 | " sort | uniq >> '$hr_failvp'"
							sum+=$1} END {printf "%d", 0+sum}' `
	[ "X$logslen" = "X" ] && logslen=0

	# list all other plexes
	movelen=`vxprint -g $failed_dg -se 'sd_dmname="'$failed_dm'" &&
					assoc.pl_state!="LOG" && assoc.pl_state!="BADLOG" &&
					assoc.pl_state!="SNAPDONE" && assoc.pl_state!="SNAPATT" &&
					!sd_is_log && assoc.assoc.v_use_type!="root" &&
					assoc.assoc.v_use_type!="swap" ' \
					-F '%len %v_name %pl_name' | \
					awk '{print $2 " " $3 | " sort | uniq >> '$hr_failvp'"
							sum+=$1} END {printf "%d", 0+sum}' `
	[ "X$movelen" = "X" ] && movelen=0
	movelen=`expr $movelen + $rootlen + $swaplen + $logslen`
	[ $movelen -eq 0 ] && return 0

	# get list of spares and disks with nohotuse flags not set
	hr_list_avail_disks on  off $hr_sparedms

	# check if spare=only flag is set from the attribute defaults
	if [ -s $VOL_DEFAULTS_FILE ]; then
		dsparefl=`cat $VOL_DEFAULTS_FILE | awk '{FS="#"; print $1}' | \
						awk ' {for (i=1; i<=NF; i++)
							if ($i ~ /^spare=/) { svar=$i }
							} END {print svar} ' `
	fi

	# non-spares can be used if not marked with nohotuse flag
	if [ "X$dsparefl" != "Xspare=only" ]; then
		hr_list_avail_disks off off $hr_nofuodms
		cat $hr_nofuodms >> $hr_sparedms
	fi

	# select disks with enough space
	hr_select_spacedm $movelen $rootlen $swaplen $hr_sparedms $hr_fitdms

	# select disk with no conflicts
	cleardm=`hr_get_cleardm "$hr_failvp" "$hr_fitdms" `

	if [ "X$cleardm" = "X" ]; then
		echo "\
\\nRootdisk $failed_dm of disk group $failed_dg experienced a failure,
but there is no suitable disk found to relocate
all associated subdisks.
\\n" > /tmp/mail_vxrelocd.$$
		if [ $english_only_email -eq 0 ]
		then
			echo "\n========================================\n" >> /tmp/mail_vxrelocd.$$
			set_to_env_locale
			(export failed_dm failed_dg; egettxt "\
\\nRootdisk $failed_dm of disk group $failed_dg experienced a failure,
but there is no suitable disk found to relocate
all associated subdisks.
\\n" vxvmshm:973
			) >> /tmp/mail_vxrelocd.$$ 
			set_to_c_locale
		fi
		$MAILER -s "$rootdm_ms" "$recipients"  < /tmp/mail_vxrelocd.$$
		rm /tmp/mail_vxrelocd.$$
		return 12
	else
		echo "\
\\nRootdisk $failed_dm of disk group $failed_dg experienced a failure,
hot-relocation is attempting to move all associated subdisks to $cleardm
\\n" > /tmp/mail_vxrelocd.$$
		if [ $english_only_email -eq 0 ]
		then
			echo "\n========================================\n" >> /tmp/mail_vxrelocd.$$
			set_to_env_locale
			(export failed_dm failed_dg cleardm ; egettxt "\
\\nRootdisk $failed_dm of disk group $failed_dg experienced a failure,
hot-relocation is attempting to move all associated subdisks to $cleardm
\\n" vxvmshm:975
			) >> /tmp/mail_vxrelocd.$$ 
			set_to_c_locale
		fi
		$MAILER -s "$rootdm_ms" "$recipients"  < /tmp/mail_vxrelocd.$$
		rm /tmp/mail_vxrelocd.$$
	fi

	# save current config
	hr_save_config

	> $hr_rtdone
	> $hr_rtfail

	cat $hr_failvp | \
	while read movev movep ; do
		# skip if not associated or moved already.
		[ "X$movev" = "X-" ] && continue
		if [ X`vxprint -g $failed_dg -pe 'aslist.sd_dmname="'$failed_dm'" &&
				pl_name="'$movep'" ' -F %name ` = "X" ]; then
			continue
		fi

		relc_ret=0
		plusetype=`vxprint -g $failed_dg -p -F %use_type $movep`

		# remirror root and swap
		if [ "X$plusetype" = "Xroot" -o "X$plusetype" = "Xswap" ]; then

			hr_set_unreloc_flag "$failed_dg" "-l" "$movep" hr_unreloc_flag

			vxassist -g $failed_dg -o nofmr $hr_unreloc_flag mirror \
			      $movev layout=nospan,contig $cleardm 2> /dev/null

			move_ret=$?
			if [ $move_ret -eq 0 ] ; then
				/etc/vx/bin/vxbootsetup
				# reconfig dump device
				[ "X`vxprint -g $failed_dg -F %use_type $movev`" = "Xswap" ] &&
					hr_conf_dumpdev $failed_dg $movev $failed_dm $cleardm
				vxplex -g $failed_dg -v $movev dis $movep
				vxedit -g $failed_dg -rf rm $movep
				echo "-" $movev $failed_dm $failed_dg >> $hr_rootlog

				echo "\
\\nRemirrored $movev volume plex $movep because of error on disk $failed_dm.
\\n" > /tmp/mail_vxrelocd.$$

				if [ $english_only_email -eq 0 ]
				then
					echo "\n========================================\n" >> /tmp/mail_vxrelocd.$$
					set_to_env_locale
					(export movev movep failed_dm; egettxt "\
\\nRemirrored $movev volume plex $movep because of error on disk $failed_dm.
\\n" vxvmshm:971
					) >> /tmp/mail_vxrelocd.$$ 
					set_to_c_locale
				fi
				$MAILER -s "$_ms" "$recipients"  < /tmp/mail_vxrelocd.$$
				rm /tmp/mail_vxrelocd.$$
			fi

		# DRL/raid logs, or snap plexes
		elif [ X$movep = X`vxprint -Qp -g $failed_dg -e '
							pl_name=="'$movep'" &&
							(aslist->sd_is_log ||
							pl_state="SNAPDONE" || pl_state="SNAPATT" ||
							pl_state=="LOG" || pl_state=="BADLOG") ' \
							-F %name` ]; then
			logtype=""
			if [ X$movep = X`vxprint -Qp -g $failed_dg -e '
						pl_name=="'$movep'" &&
						(pl_state=="LOG" || pl_state=="BADLOG") ' \
						-F %name` ]; then
				logtype=raid
			elif [ X$movep = X`vxprint -Qp -g $failed_dg -e '
						pl_name=="'$movep'" && aslist->sd_is_log' \
						-F %name` ]; then
				logtype=drl
			else
				logtype=`echo \`vxprint -Qp -g $failed_dg -F %state $movep\`|
								sed -e "/^SNAP/!d"`
			fi
			[ "X$logtype" = "X" ] && logtype="-"

			move_ret=1
			hr_addlog_snapstart $failed_dg $movev $movep \
								"$failed_dm" "$cleardm" ""
			if [ $? -eq 0 ] ; then
				move_ret=0
				echo $movep $failed_dg $logtype >> $hr_rootlog
			fi

		# mirror volume plex
		elif [ X$movep = X`vxprint -Qp -g $failed_dg -e '
							pl_name=="'$movep'" &&
							(!aslist->sd_is_log &&
							pl_state!="LOG" && pl_state!="BADLOG") ' \
							-F %name` ]; then

			# move all the sds, even the good ones

            # relocate failed sds

			relc_ret=0
			relcsds=`vxprint -Qs -g $failed_dg -e 'assoc.pl_name="'$movep'" &&
					assoc.assoc.v_name="'$movev'" &&
					sd_dmname="'$failed_dm'" &&
					(sd_relocate || sd_nodevice || sd_kdetach)' -F %name`
			if [ "X$relcsds" != "X" ]; then

				vxassist -r -g $failed_dg move $movev \
						!$failed_dm $cleardm > $hr_tmp1 2> /dev/null
				relc_ret=$?

				if  [ $relc_ret -eq 0 ] && [ -s $hr_tmp1 ]; then
					cat $hr_tmp1 | sed -e "s/^/$movev /" > $hr_tmp2
					cat $hr_tmp2 >> $resultfile
					echo $failed_dg $movev >> $recoverlist
					echo "-" $movev $failed_dm $failed_dg >> $hr_rootlog
				fi
			fi

			# move good sds

			move_ret=0
			goodsds=`vxprint -Qs -g $failed_dg -e 'assoc.pl_name="'$movep'" &&
					assoc.assoc.v_name="'$movev'" &&
					sd_dmname="'$failed_dm'" &&
					(!sd_relocate && !sd_nodevice && !sd_kdetach)' -F %name`
			if [ "X$goodsds" != "X" ]; then

				hr_set_unreloc_flag "$failed_dg" "-m" "" hr_unreloc_flag

				vxassist $hr_unreloc_flag -g $failed_dg move $movev \
						!$failed_dm $cleardm 2>/dev/null
				move_ret=$?
			fi
		fi

		if [ $move_ret -eq 0 -a $relc_ret -eq 0 ]; then

			if [ X`cat $hr_rtdone | awk '$1=="'$movev'" {print $1}'` = "X" ]
			then
				echo $movev >> $hr_rtdone
				hr_put_hist $cleardm
			fi
		else
			if [ X`cat $hr_rtfail | awk '$1=="'$movev'" {print $1}'` = "X" ]
			then
				echo $movev >> $hr_rtfail
			fi
		fi

	done

	# create partitions for everything else
	/etc/vx/bin/vxbootsetup

	(echo "\
\\nRootdisk $failed_dm of disk group $failed_dg experienced a failure,
hot-relocation attempted to move/remirror all the associated
subdisks to $cleardm
Subdisks from the following volumes got moved/remirrored:
\\n"
cat $hr_rtdone

	if [ -s $hr_rtfail ]; then
		echo "\
\\nError: Subdisks from the following volumes failed to move/remirror:
\\n"
		cat $hr_rtfail
	fi
	) > /tmp/mail_vxrelocd.$$ 

	if [ $english_only_email -eq 0 ]
	then
		echo "\n========================================\n" >> /tmp/mail_vxrelocd.$$
		set_to_env_locale
		(export failed_dm failed_dg cleardm hr_rtdone hr_rtfail ; egettxt "\
\\nRootdisk $failed_dm of disk group $failed_dg experienced a failure,
hot-relocation attempted to move/remirror all the associated
subdisks to $cleardm
Subdisks from the following volumes got moved/remirrored:
\\n" vxvmshm:974
		cat $hr_rtdone

		if [ -s $hr_rtfail ]; then
			egettxt "\
\\nError: Subdisks from the following volumes failed to move/remirror:
\\n" vxvmshm:970
			cat $hr_rtfail
		fi
		) >> /tmp/mail_vxrelocd.$$ 
		set_to_c_locale
	fi
       	$MAILER -s "$rootdm_ms" "$recipients"  < /tmp/mail_vxrelocd.$$
	rm /tmp/mail_vxrelocd.$$

	[ -s $hr_rtfail ] && return 13

	return 0
}

# create a new if plex is a log or a snapdone
hr_addlog_snapstart()
{
	_fdiskg=$1
	_fvol=$2
	_fplex=$3
	_fdm=$4
	_tdm=$5
	_sopt=$6

	_nfdm="!$_fdm"
	[ "X$_fdm" = "X" ] && _nfdm=""

	# if failed plex is a log
	if [ X$_fplex = X`vxprint -Qp -g $_fdiskg -e ' pl_name=="'$_fplex'" &&
					(aslist->sd_is_log ||
					pl_state=="LOG" || pl_state=="BADLOG") ' -F %name` ]
	then
		hr_set_unreloc_flag "$_fdiskg" "-l" "$_fplex" hr_unreloc_flag

		vxassist -g $_fdiskg $hr_unreloc_flag addlog $_fvol \
					$_sopt $_nfdm $_tdm > $vxassist_res 2>&1
		alret=$?

	# if failed plex is a snapdone
	elif [ X$_fplex = X`vxprint -Qp -g $_fdiskg -e ' pl_name=="'$_fplex'" &&
				(pl_state=="SNAPDONE" || pl_state=="SNAPATT") ' -F %name` ]
	then

		vxassist -g $_fdiskg snapstart $_fvol \
					$_sopt $_nfdm $_tdm > $vxassist_res 2>&1
		alret=$?
	fi

	grep ERROR $vxassist_res
	if [ $? -ne 0 -a $alret -eq 0 ] ; then
		vxplex -g $_fdiskg -o rm dis $_fplex
		hr_put_hist $_tdm
		return 0
	fi

	return 1
}

# The actual vxassist calls to relocate
hr_vxassist()
{
	targetdm=$1
	spareopt=$2

	# create new if plex is a log or a snap
	if [ X$failed_pl = X`vxprint -Qp -g $failed_dg -e '
			pl_name=="'$failed_pl'" &&
			(pl_state="SNAPDONE" || pl_state="SNAPATT" || aslist->sd_is_log ||
			pl_state=="LOG" || pl_state=="BADLOG")' -F %name` ]; then

		hr_addlog_snapstart $failed_dg $failed_v $failed_pl \
							"$failed_dm" "$targetdm" "$spareopt"
		return $?
	fi

	# not a log or a snap

	vxassist -r -g $failed_dg move $failed_v \
					$spareopt !$failed_dm $targetdm > $hr_tmp1 2> /dev/null

	if [ $? -eq 0 ] && [ -s $hr_tmp1 ]; then
		cat $hr_tmp1 | sed -e "s/^/$failed_v /" > $hr_tmp2
		cat $hr_tmp2 >> $resultfile
		hr_put_hist $targetdm
		return 0
	fi

	return 1
}

# attempt to hot-relocate with the provided pool of dms
hr_try_pool()
{
	diskpool=$1

	# select disks with enough space
	hr_select_spacedm $faillen 0 0 $diskpool $hr_fitdms

	# select a disk with no conflicts
	cleardm=`hr_get_cleardm "$hr_failvp" "$hr_fitdms" `

	# save current configuration
	hr_save_config

	if [ "X$cleardm" != "X" ]; then
		hr_vxassist $cleardm ""
		[ $? -eq 0 ] && return 0
	fi

	# a single disk cannot be found, try all disks in the pool.
	cat $diskpool | \
	while read trydm; do
		[ "X$trydm" = "X$cleardm" ] && continue
		hr_vxassist $trydm ""
		[ $? -eq 0 ] && return 0
	done

	# disregarding sd co-location
		a=`cat $diskpool | sed -n '1p' `
		spareflag=`vxprint -g $failed_dg -Qqd -F %spare $a`
		if [ "X$spareflag" = "Xon" ] ; then
			spareopt="spare=yes spare=only"
		else
			spareopt="spare=yes"
			hr_list_avail_disks off  on $hr_tmp4
			if [ -s $hr_tmp4 ] ; then
				for a in `cat $hr_tmp4` ; do
 					spareopt="$spareopt !$a"
 				done
			fi
		fi
		hr_vxassist "" "$spareopt"
		vxassist_ret=$?

	return $vxassist_ret
}

# an error occured on a disk that is not a rootdisk. Try to move first to 
# spare space and then free space.
hr_move_nonrootdm()
{
	# Sum all failed subdisks and list their volumes
	> $hr_failvp
	faillen=`vxprint -g $failed_dg -Qqse 'sd_dmname="'$failed_dm'"  &&
					(assoc.pl_kstate==DETACHED ||
					assoc.pl_nodarec || assoc.pl_recover ||
					sd_kdetach || sd_nodevice || sd_relocate) ' \
					-F '%len %v_name %pl_name' | \
					awk '{print $2 " " $3 >> "'$hr_failvp'" ; sum+=$1}
						END {printf "%d", 0+sum}' `
	sort < $hr_failvp | sed -e '/^$/d' > $hr_tmp1
	uniq < $hr_tmp1 > $hr_failvp
	[ "X$faillen" = "X" -o $faillen -eq 0 ] && return 0

	# get list of spares and try to use those

	hr_list_avail_disks on  off $hr_sparedms
	> $hr_nofuodms

	if [ -s $hr_sparedms ]; then
		# get list of spares from history
	    > $hr_tmp1
		histdm=`cat $hr_hist | awk '$1=="'$failed_dm'" && \
									$3=="'$failed_dg'" {print $2}'`
		if [ "X$histdm" != "X" ]; then
			vxprint -g $failed_dg -Qqd -F '%name %spare' $histdm |\
				awk '$2=="on" {print $1 >> "'$hr_tmp1'" }'
		fi
		cat $hr_sparedms >> $hr_tmp1
		cat $hr_tmp1 > $hr_sparedms

		hr_try_pool $hr_sparedms
		[ $? -eq 0 ] && return 0
	fi

	# spare space can't be used, get list of free disks not
	# marked with nohotuse.

	# check if spare=only flag is set as default
	if [ -s $VOL_DEFAULTS_FILE ]; then
		dsparefl=`cat $VOL_DEFAULTS_FILE | awk '{FS="#"; print $1}' | \
						awk ' {for (i=1; i<=NF; i++)
							if ($i ~ /^spare=/) { svar=$i }
							} END {print svar} ' `
	fi

	# non-spares can be used if not marked with nohotuse flag
	if [ "X$dsparefl" != "Xspare=only" ]; then
		hr_list_avail_disks off  off $hr_nofuodms

		if [ -s $hr_nofuodms ]; then
			# get list of non-spares from history
			> $hr_tmp1
			histdm=`cat $hr_hist | awk '$1=="'$failed_dm'" && \
										$3=="'$failed_dg'" {print $2}'`
			if [ "X$histdm" != "X" ]; then
				vxprint -g $failed_dg -Qqd -F '%name %spare' $histdm |\
					awk '$2=="off" {print $1 >> "'$hr_tmp1'" }'
			fi
			cat $hr_nofuodms >> $hr_tmp1
			cat $hr_tmp1 > $hr_nofuodms

			hr_try_pool $hr_nofuodms
			[ $? -eq 0 ] && return 0
		fi
	fi

	return 1
}

# hr_hot_relocate(failed_dm,failed_dg,failed_v)
# if root volume is on the failed disk, then all on that disk will be moved.
# else, only the failed sds will be moved.
# ret: errors larger than 10 indicates an error occurred
# while relocating the rootdisk.
# kept in the rootlog: successful log and data relocation,
# status of disk relocation, and whether notification was sent.

hr_hot_relocate()
{

	failed_dm=$1
	failed_dg=$2
	failed_v=$3

	> $vxassist_res

	# skip if already done.
	[ X`vxprint -g $failed_dg -ve 'aslist.aslist.sd_dmname="'$failed_dm'" &&
		v_name="'$failed_v'" ' -F %v_name ` = "X" ] && return 0

	# if the failed disk was the rootdisk, this volume was moved. if not
	# the rootdisk was partially moved (error #13), check log if the volume
	# was successfully relocated.

	rootdm_ret=`cat $hr_rootlog | awk '$1=="--" && $2=="'$failed_dm'" && \
										$3=="'$failed_dg'"	{ print $4 }'`
	if [ "X$rootdm_ret" != "X" ] ; then
		if [ $rootdm_ret -eq 13 ]; then
			relocres=`cat $hr_rootlog | \
				awk '$1=="-" && $2=="'$failed_v'" && $3=="'$failed_dm'" && \
							$4=="'$failed_dg'"	{ print $2 }'`
			[ "X$relocres" != "X" ] && return 0
		fi
		return $rootdm_ret
	fi

	rootplex=`vxprint -g $failed_dg -Qqpe 'assoc.v_use_type="root" && \
							aslist.sd_dmname="'$failed_dm'" ' -F %name`

	if [ "X$rootplex" != "X" ]; then
		hr_move_rootdm
		rootdm_ret=$?
		echo "--" $failed_dm $failed_dg $rootdm_ret >> $hr_rootlog

		# if there was an error while relocating the rootdisk,
		# check if the failed plex was relocated successfully
		if [ $rootdm_ret -eq 13 ]; then
			logtype=`cat $hr_rootlog | \
				awk '$1=="'$failed_pl'" && $2=="'$failed_dg'" { print $3 }'`
			relocres=`cat $hr_rootlog | \
				awk '$1=="-" && $2=="'$failed_v'" && $3=="'$failed_dm'" && \
							$4=="'$failed_dg'"	{ print $2 }'`
			if [ "X$logtype" != "X" ] || [ "X$relocres" != "X" ]; then
				return 0
			fi
		fi
		return $rootdm_ret

	fi

	# not rootdisk, move failed sds only
	hr_move_nonrootdm
	return $?
}

# add a new log plex
hr_create_new_plex()
{
	> $vxassist_res
	goodpl_name=`vxprint -Qp -g $dg_name -e 'pl_state!="LOG" 
					&& pl_state !~ /SNAP/
					&& pl_v_name=="'$v_name'" && pl_kstate==ENABLED &&
					!pl_log' -F %name`
	[ "X$goodpl_name" = "X" ] && return 1

	# using the previous version of hot-relocation
	if [ "X$oldhot" = "Xpre3.1" ]; then
		hr_addlog_snapstart $dg_name $v_name $pl_name "" "" "spare=yes"
		return $?
	fi

	# use the new version
	failed_pl=$pl_name
	logdms=`vxprint -g $dg_name -se 'assoc.pl_name="'$failed_pl'"' -F %dmname`

	# check if part of the log plex is on the rootdisk
	logonroot=""
	for failed_dm in $logdms; do
		rootplex=`vxprint -g $dg_name -Qqpe 'assoc.v_use_type="root" && \
						aslist.sd_dmname="'$failed_dm'" ' -F %name`
		[ "X$rootplex" != "X" ]	&& logonroot=$failed_dm
	done

	failed_dm=`echo $logdms | awk '{print $1}' `

	# set the failed dm to rootdisk if any part of the log plex is on it
	[ "X$logonroot" != "X" ] && failed_dm=$logonroot

	hr_hot_relocate $failed_dm $dg_name $v_name
	return $?
}

#
# try_reloc(dm, dg, volume)  attempt hot-relocation of subdisks
#
#
try_reloc()
{
	[ -n "$verbose" ] && echo "connect_spare $*" >&2

	dm_name=$1
	dg_name=$2
	v_name=$3

	_ms="Attempting VxVM relocation on host `uname -n`"

	# using the previous version of hot-relocation?
	if [ "X$oldhot" = "Xpre3.1" ]; then
		# If it is the root volume, do not relocate, create another
		# mirror
		if [ X$v_name = X`vxprint -AQve '(v_use_type="root" ||
				v_use_type="swap") && v_name="'$v_name'"' -F %name` ]; then
			_old_dms=`vxprint -g $dg_name -se '
							assoc.assoc.v_name="'$v_name'"' -F %dm_name `

			plexname=`vxprint -qQp -g $dg_name -e '(pl_kstate==DETACHED ||
						pl_nodarec) && pl_volume="'$v_name'"' -F %name`

			hr_set_unreloc_flag "$dg_name" "-l" "$plexname" hr_unreloc_flag

			vxassist -g $dg_name -o nofmr $hr_unreloc_flag mirror \
					$v_name spare=yes layout=nospan,contig
			vxassist_ret=$?
			if [ $vxassist_ret -eq 0 ] ; then

				# run vxbootsetup to create hard partitions for
				# swap and root
				/etc/vx/bin/vxbootsetup
				# reconfig dump device
				if [ "X`vxprint -g $dg_name -F %use_type $v_name`" = "Xswap" ]
				then
					_new_dms=`vxprint -g $dg_name -se '
								assoc.assoc.v_name="'$v_name'"' -F %dm_name `
					swplap=
					_new_swap_dm=
					for iswpdm in $_new_dms; do
						swplap=`echo $_old_dms | awk ' {for (i=1; i<=NF; i++)
											if ($i=="'$iswpdm'") {print $i}}'`
						[ "X$swplap" = "X" ] && _new_swap_dm=$iswpdm
					done
					[ -n "$_new_swap_dm" ] &&
					hr_conf_dumpdev  $dg_name $v_name $dm_name $_new_swap_dm
				fi

				vxplex -g $dg_name -v $v_name dis $plexname 
				vxedit -g $dg_name -rf rm $plexname
				echo "\
\\nRemirrored $v_name volume plex $plexname because of error on disk $dm_name.
\\n" > /tmp/mail_vxrelocd.$$
				if [ $english_only_email -eq 0 ]
				then
					echo "\n========================================\n" >> /tmp/mail_vxrelocd.$$
					set_to_env_locale
					(export v_name plexname dm_name; egettxt "\
\\nRemirrored $v_name volume plex $plexname because of error on disk $dm_name.
\\n" vxvmshm:584) >> /tmp/mail_vxrelocd.$$
					set_to_c_locale
				fi
				$MAILER -s "$_ms" "$recipients" < /tmp/mail_vxrelocd.$$
				rm /tmp/mail_vxrelocd.$$
				return 0
   	        fi
		else

			vxassist -r -g $dg_name move $v_name !$dm_name \
				spare=yes > $hr_tmp1
			vxassist_ret=$?
			if [ $vxassist_ret -eq 0 ]; then
				if [ -s $hr_tmp1 ] ; then
					cat $hr_tmp1 | sed -e "s/^/$v_name /" > $hr_tmp2
					cat $hr_tmp2 >> $resultfile
				fi
				return 0
			fi
		fi
	else
		# relocate the failed sds with the new version of hot-relocation
		failed_pl="-"
		hr_hot_relocate $dm_name $dg_name $v_name
		vxassist_ret=$?

		[ $vxassist_ret -eq 0 ] && return 0

		# If errors while moving from the rootdisk, check if the notice below
		# was sent for this disk.
		if [ $vxassist_ret -gt 10 ]; then
			[ X`cat $hr_rootlog | awk '$1=="---" && $2=="'$failed_dm'" && \
				$3=="'$failed_dg'" {print $2}'` != "X" ] &&
					return $vxassist_ret
			echo "---" $failed_dm $failed_dg >> $hr_rootlog
		fi
	fi

	# If we've gotten here, relocation was not successful,
	# so send mail
   	(
		echo "\
Relocation was not successful for subdisks on disk $dm_name in
volume $v_name in disk group $dg_name.  No replacement was made and the
disk is still unusable."

		avols=`affvols $dm_name $dg_name`
		dvols=`deadvols $dm_name $dg_name`
		draid=`deadraids $dm_name $dg_name`
		alldead="${dvols}${dvols:+ }${draid}"

		if [ ! -z "$avols" ] 
		then
			echo "\
The following volumes have storage on ${dm_name}:

$avols

These volumes are still usable, but the the redundancy of
those volumes is reduced. Any RAID-5 volumes with storage on 
the failed disk may become unusable in the face of further 
failures."
		fi

		if [ ! -z "$dvols" ]
		then
			 echo "\
The following volumes:

$dvols

have data on $dm_name but have no other usable mirrors on other
disks. These volumes are now unusable and the data on them is
unavailable. These volumes must have their data restored." 

		fi

		if [ ! -z "$draid" ]
		then
			echo "\

The following RAID-5 volumes:

$draid

have storage on $dm_name and have experienced other failures. These
RAID-5 volumes are now unusable and the data on them is unavailable.
These RAID-5 volumes must have their data restored." 
		fi
	) > /tmp/mail_vxrelocd.$$ 

	if [ $english_only_email -eq 0 ]
	then
		echo "\n========================================\n" >> /tmp/mail_vxrelocd.$$
		set_to_env_locale
   		(
		export v_name dm_name dg_name; egettxt \
"Relocation was not successful for subdisks on disk $dm_name in
volume $v_name in disk group $dg_name.  No replacement was made and the
disk is still unusable." vxvmshm:585

		avols=`affvols $dm_name $dg_name`
		dvols=`deadvols $dm_name $dg_name`
		draid=`deadraids $dm_name $dg_name`
		alldead="${dvols}${dvols:+ }${draid}"

		if [ ! -z "$avols" ] 
		then
			export avols dm_name; egettxt "
The following volumes have storage on ${dm_name}:

$avols

These volumes are still usable, but the the redundancy of
those volumes is reduced. Any RAID-5 volumes with storage on 
the failed disk may become unusable in the face of further 
failures." vxvmshm:535
		fi

		if [ ! -z "$dvols" ]
		then
			 export dvols dm_name; egettxt "
The following volumes:

$dvols

have data on $dm_name but have no other usable mirrors on other
disks. These volumes are now unusable and the data on them is
unavailable. These volumes must have their data restored." vxvmshm:587

		fi

		if [ ! -z "$draid" ]
		then
			export draid dm_name; egettxt "

The following RAID-5 volumes:

$draid

have storage on $dm_name and have experienced other failures. These
RAID-5 volumes are now unusable and the data on them is unavailable.
These RAID-5 volumes must have their data restored." \
vxvmshm:625
		fi
		) >> /tmp/mail_vxrelocd.$$ 
		set_to_c_locale
	fi
	$MAILER -s "$_ms" "$recipients" < /tmp/mail_vxrelocd.$$
	rm /tmp/mail_vxrelocd.$$

	return $vxassist_ret
}

# search_relocate_subdisk - search for subdisk marked for relocation based on a plex name

search_relocate_subdisk()
{
        DGNAME=$1
        PLNAME=$2
        vxprint -Qs -g $DGNAME -e '(sd_relocate || sd_nodevice || sd_kdetach)
                && sd_pl_name="'$PLNAME'"' -F %name >> $failsd_list
        for SDNAME in `vxprint -Qs -g $DGNAME -e 'sd_pl_name="'$PLNAME'"' -F %name`
        do
                DMNAME=`vxprint -g $DGNAME -m $SDNAME | grep "dm_name=" | awk -F= '{print $2}'`
                for PLNAME in `vxprint -Qp -g $DGNAME -e 'pl_v_name="'$DMNAME'"' -F %name`
                do
                        search_relocate_subdisk $DGNAME $PLNAME
                done
        done
}

hr_cleanup()
{
	rm -f $failsd_list $faildisks $plexlist $reloc_dm_list
	rm -f $raidlist $vxassist_res $vxrecover_res $hr_newvp
	rm -f $hr_failvp $hr_sparedms $hr_nofuodms $hr_fitdms
	rm -f $hr_tmp1 $hr_tmp2 $hr_rootlog $hr_rtdone $hr_rtfail
	rm -f $hr_tmp3 $hr_tmp4 $tmpfile $tmpfile2
}

# Checkdetach - Look for failed objects and send the administrator mail

checkdetach()
{
	hr_hist=$hrhistfile				# list of source-target-group relocations
	hr_newvp=/tmp/hreloc-2t$$		# vols and plexes on a good dm
	hr_failvp=/tmp/hreloc-3t$$		# vols and plexes on a failed dm
	hr_tmp1=/tmp/hreloc-4t$$
	hr_sparedms=/tmp/hreloc-5t$$	# spare dms
	hr_nofuodms=/tmp/hreloc-6t$$	# !nohotuse dms
	hr_fitdms=/tmp/hreloc-7t$$		# dms with enough space
	hr_tmp2=/tmp/hreloc-8t$$
	hr_rootlog=/tmp/hreloc-9t$$		# hist of activities on rootdisk
	hr_rtdone=/tmp/hreloc-bt$$		# list of sucessful moves from rootdisk
	hr_rtfail=/tmp/hreloc-ct$$		# list of failed moves from rootdisk
	hr_tmp3=/tmp/hreloc-dt$$
	hr_tmp4=/tmp/hreloc-et$$

	# snapshot of configuration before hot-relocation is attempted.
	failed_dir=$VOL_SAVECONFIG_DIR
	failed_file=`date '+%y%m%d_%H'``date '+%M'``date '+%S'`.mpvsh

	faildisks=/tmp/hs-4$$		# list of failed disks
	plexlist=/tmp/hs-5$$		# list of detached plexes
	reloc_dm_list=/tmp/hs-6$$	# list of dm's from which to relocate
	raidlist=/tmp/hs-7$$		# list of failed raid disks
	recoverlist=$hrrecoverfile	# list of volumes to recover
	failsd_list=/tmp/sd-1$$         # list of failed subdisks
	vxassist_res=/tmp/hs-vxas$$     # vxassist output
	vxrecover_res=/tmp/hs-rec$$     # vxrecover otuput
	tmpfile=/tmp/hs-9$$
	tmpfile2=/tmp/hs-t$$
	resultfile=$hrresultfile
	
	rm -f $faildisks
	[ ! -f $hr_hist ] && > $hr_hist
	[ ! -f $resultfile ] && > $resultfile
	> $hr_rootlog

	trap 'hr_cleanup ' 1 2 3 15
 
	_ms="Volume Manager failures on host `uname -n`"

	vxprint -AQdF '%name %nodarec %dgname' 2> /dev/null | \
		awk '$2=="on" {print " " $1 " " $3}' > $faildisks
	d=`awk '{print " " $1}' < $faildisks`

	# Create a list of failed raid subdisks and their dg and volume
	# Then check each volume.  If it is not detached we can relocate
	# the failed subdisk, so add it to the list to relocate.  Otherwise,
	# mail an error message.

	rm -f $reloc_dm_list $raidlist
	vxprint -AQse 'assoc->pl_layout=RAID  
		&& (sd_kdetach || sd_nodevice || sd_relocate)
		&& assoc->assoc->v_kstate=ENABLED' -F '%name %dg_name'\
		| sed -e '/^$/d' >$tmpfile
	cat $tmpfile | \
	while read sd_name dg_name
	do
		dm_name=`vxprint -g $dg_name -F "%dm_name" $sd_name`
		v_name=`vxprint -g $dg_name -F "%v_name" $sd_name`
		echo $dm_name $dg_name $v_name >>$reloc_dm_list
	done
	
	  # create a list of disabled raid volumes
	vxprint -AQse 'assoc->pl_layout=RAID
		&& (sd_kdetach || sd_nodevice || sd_relocate)
		&& assoc->assoc->v_kstate!=ENABLED' \
		-F %v_name | sort | uniq | sed -e '/^$/d' > $tmpfile
	 cat $tmpfile | \
         while read vol_name
         do
	         echo "\
\\nRaid volume $vol_name experienced a failure, but no relocation
will be done as the volume has been detached or disabled 
\\n" > /tmp/mail_vxrelocd.$$
		if [ $english_only_email -eq 0 ]
		then
			echo "\n========================================\n" >> /tmp/mail_vxrelocd.$$
			set_to_env_locale
	         	(export vol_name; egettxt "\
\\nRaid volume $vol_name experienced a failure, but no relocation
will be done as the volume has been detached or disabled 
\\n" vxvmshm:588
                	) >> /tmp/mail_vxrelocd.$$ 
			set_to_c_locale
		fi
		$MAILER -s "$_ms" "$recipients" < /tmp/mail_vxrelocd.$$
		rm /tmp/mail_vxrelocd.$$
         done

	# create a list of detached plexes (or plexes with no da record)
	# and their dg and volume
	vxprint -AQpe 'pl_kstate==DETACHED || pl_nodarec || pl_recover' \
		-F '%name %dgname %vname' 2>/dev/null | \
		sed -e '/^$/d' > $plexlist

	# set failing flag on dm if plex is log
	cat $plexlist | \
 	while read pl_name dg_name v_name; do
	    [ X$v_name = "X-" ] && continue

		islogplex=`vxprint -Qp -g $dg_name -e 'pl_name=="'$pl_name'"
					&& (aslist->sd_is_log || pl_state="BADLOG" ||
				     pl_state="LOG")' -F %name`
		if [ "X$islogplex" = "X$pl_name" ]; then
			a=`vxprint -g $dg_name -se 'assoc.pl_name="'$pl_name'"' \
						-F %dmname 2> /dev/null `
			for b in $a; do
				[ X`vxprint -g $dg_name -Qqd -F %nodarec $b` != "Xon" ] &&
					vxedit -g $dg_name set failing=on $b
			done
		fi
	done

	# create a list of bad plexes then send mail about failed disks
	# and bad plexes
	p=`vxprint -AQpe '(pl_kstate==DETACHED || pl_recover ||
		pl_nodarec) && (pl_state!="LOG" && pl_state!="BADLOG" &&
		!aslist->sd_is_log)' -F %name 2>/dev/null`
	
	lp=`vxprint -AQpe '(pl_kstate==DETACHED || pl_nodarec || pl_recover)
		&& (pl_state=="LOG" || pl_state=="BADLOG" || aslist->sd_is_log)' -F %name`

	failing_d=`vxprint -AQdF '%name %failing' | awk '$2=="on" {print $1}'`

	if [ ! -z "$d" ] || [ ! -z "$p" ]  || [ ! -z "$lp" ] || [ ! -z "$failing_d" ]
	then
		(echo "\
Failures have been detected by the VERITAS Volume Manager:"
		[ -z "$d" ] || \
			d="$d" echo "\\nfailed disks:\\n$d" 
		[ -z "$p" ] || \
			p="$p" echo "\\nfailed plexes:\\n$p" 
		[ -z "$lp" ] || \
			lp="$lp" echo "\\nfailed log plexes:\\n$lp" 
		[ -z "$failing_d" ] || \
			echo "\\nfailing disks:\\n$failing_d" 
		[ -n "$d" ] && echo "\

The Volume Manager will attempt to find spare disks, relocate failed
subdisks and then recover the data in the failed plexes. 
") > /tmp/mail_vxrelocd.$$
		
		if [ $english_only_email -eq 0 ]
		then
			echo "\n========================================\n" >> /tmp/mail_vxrelocd.$$
			set_to_env_locale
			(egettxt "\
Failures have been detected by the VERITAS Volume Manager:" \
vxvmshm:191
			[ -z "$d" ] || \
				d="$d" egettxt "\\nfailed disks:\\n$d" vxvmshm:520
			[ -z "$p" ] || \
				p="$p" egettxt "\\nfailed plexes:\\n$p" vxvmshm:521
			[ -z "$lp" ] || \
				lp="$lp" egettxt "\\nfailed log plexes:\\n$lp" vxvmshm:590
			[ -z "$failing_d" ] || \
				export failing_d; egettxt "\\nfailing disks:\\n$failing_d" vxvmshm:591
			[ -n "$d" ] && egettxt "\

The Volume Manager will attempt to find spare disks, relocate failed
subdisks and then recover the data in the failed plexes. 
" vxvmshm:624
			) >> /tmp/mail_vxrelocd.$$
			set_to_c_locale
		fi
	 	$MAILER -s "$_ms" "$recipients" < /tmp/mail_vxrelocd.$$
		rm /tmp/mail_vxrelocd.$$
	fi

	# check mirror vol plexes
	cat $plexlist | \
 	while read pl_name dg_name v_name; do
	    [ X$v_name = "X-" ] && continue

		if [ X$pl_name = X`vxprint -Qp -g $dg_name -e 'pl_name=="'$pl_name'" 
				&& (!aslist->sd_is_log && pl_state!="BADLOG"
				&& pl_state!="SNAPDONE" && pl_state!="SNAPATT"
			    && pl_state!="LOG")' -F %name` ] ; then

			goodpl_name=`vxprint -Qp -g $dg_name -e 'pl_state!="LOG" 
					&& pl_state !~ /SNAP/
					&& pl_v_name=="'$v_name'" && pl_kstate==ENABLED &&
					!pl_log' -F %name`
			rm $failsd_list > /dev/null 2>&1
			search_relocate_subdisk $dg_name $pl_name

			# A disk failure will not mark the subdisks with a relocate
			# but it is disabled, we should move them too.
			cat $failsd_list | \
			while read failsd_name
			do

			# If a failed plex has a good mirror, save the name of the
		    	# subdisk, dg, and volume, so it can be relocated.

			    if [ ! -z "$goodpl_name" ] 
			    then 

				dm_name=`vxprint -g $dg_name -F '%dm_name' $failsd_name`
				echo $dm_name $dg_name $v_name >>$reloc_dm_list
				offset=`vxprint -g $dg_name -F '%dev_offset' $failsd_name`
				len=`vxprint -g $dg_name -F '%len' $failsd_name`
				da_name=`vxprint -g $dg_name -F %last_da_name $dm_name`

				( echo "\
\\nAttempting to relocate subdisk $failsd_name from plex $pl_name.
Dev_offset $offset length $len dm_name $dm_name da_name $da_name.
The available plex $goodpl_name will be used recover the data. 
\\n"
				) > /tmp/mail_vxrelocd.$$ 

				if [ $english_only_email -eq 0 ]
				then
					echo "\n========================================\n" >> /tmp/mail_vxrelocd.$$
					set_to_env_locale
					(export failsd_name pl_name goodpl_name offset len dm_name da_name; egettxt "\
\\nAttempting to relocate subdisk $failsd_name from plex $pl_name.
Dev_offset $offset length $len dm_name $dm_name da_name $da_name.
The available plex $goodpl_name will be used recover the data. 
\\n" vxvmshm:597
					) >> /tmp/mail_vxrelocd.$$ 

					set_to_c_locale
				fi
        			$MAILER -s "$_ms" "$recipients"  < /tmp/mail_vxrelocd.$$
				rm /tmp/mail_vxrelocd.$$

			    else
				( echo "\
\\nUnable to relocate failed subdisk from plex $pl_name because no
suitable mirror was found from which to recover data. 
\\n"
				) > /tmp/mail_vxrelocd.$$ 
				if [ $english_only_email -eq 0 ]
				then
					echo "\n========================================\n" >> /tmp/mail_vxrelocd.$$
					set_to_env_locale
					(export pl_name; egettxt "\
\\nUnable to relocate failed subdisk from plex $pl_name because no
suitable mirror was found from which to recover data. 
\\n" vxvmshm:598
					) >> /tmp/mail_vxrelocd.$$ 
					set_to_c_locale
				fi
				$MAILER -s "$_ms" "$recipients"  < /tmp/mail_vxrelocd.$$
				rm /tmp/mail_vxrelocd.$$


			    fi 
			done
		fi
	done

	# check logs

	cat $plexlist | \
 	while read pl_name dg_name v_name
	do
	    # skip this plex if it isn't associated
	    [ X$v_name = "X-" ] && continue

		# check if plex was on the rootdisk's log; if yes, it was relocated.
		logtype=""
		if [ X`vxprint -g $dg_name -pe 'pl_name="'$pl_name'" ' \
				-F %name ` = "X" ]; then
			logtype=`cat $hr_rootlog | \
					awk '$1=="'$pl_name'" && $2=="'$dg_name'" { print $3 }'`
		fi

		# snap plexes cannot be moved and get relocated like log plexes
		snapstate=`vxprint -Qp -g $dg_name -F %state $pl_name 2> /dev/null`
		if [ "X$snapstate" = "XSNAPDONE" -o "X$snapstate" = "XSNAPATT" ] ||
			[ "X$logtype" = "XSNAPDONE" -o "X$logtype" = "XSNAPATT" ]; then

			[ "X$snapstate" = "X" ] && snapstate=$logtype
			[ "X$logtype" = "X" ] && hr_create_new_plex

			if [ $? -eq 0 ] || [ "X$logtype" != "X" ]; then 
				(echo "\
\\nA $snapstate plex $pl_name in volume $v_name failed.
A new plex for volume $v_name was created.
\\n"
				) > /tmp/mail_vxrelocd.$$ 

				if [ $english_only_email -eq 0 ]
				then
					echo "\n========================================\n" >> /tmp/mail_vxrelocd.$$
					set_to_env_locale
					(export snapstate pl_name v_name; egettxt "\
\\nA $snapstate plex $pl_name in volume $v_name failed.
A new plex for volume $v_name was created.
\\n" vxvmshm:969
					) >> /tmp/mail_vxrelocd.$$ 
					set_to_c_locale
				fi
				$MAILER -s "$_ms" "$recipients"  < /tmp/mail_vxrelocd.$$
				rm /tmp/mail_vxrelocd.$$
			else 
				(echo "\
\\nA $snapstate plex $pl_name failed, 
creation of new $snapstate plex for volume $v_name failed.
\\n"
				cat $vxassist_res
				) > /tmp/mail_vxrelocd.$$ 
				if [ $english_only_email -eq 0 ]
				then
					echo "\n========================================\n" >> /tmp/mail_vxrelocd.$$
					set_to_env_locale
					(export vxassist_res snapstate pl_name v_name; egettxt "\
\\nA $snapstate plex $pl_name failed, 
creation of new $snapstate plex for volume $v_name failed.
\\n" vxvmshm:968
					cat $vxassist_res
					) >> /tmp/mail_vxrelocd.$$ 
					set_to_c_locale
				fi
				$MAILER -s "$_ms" "$recipients"  < /tmp/mail_vxrelocd.$$
				rm /tmp/mail_vxrelocd.$$

			fi 
		
	    # Is this a raid log plex

            elif [ X$pl_name = X`vxprint -Qp -g $dg_name -e '
				(pl_state=="LOG" || pl_state=="BADLOG") &&
				pl_name=="'$pl_name'"' -F %name` ] || [ "X$logtype" = "Xraid" ]
	    then 

		[ "X$logtype" = "X" ] && hr_create_new_plex

		if [ $? -eq 0 ] || [ "X$logtype" != "X" ]; then 

			(echo "\
\\nRaid log $pl_name in volume $v_name failed.
A new raid log for volume $v_name was created.
\\n"
			) > /tmp/mail_vxrelocd.$$ 

			if [ $english_only_email -eq 0 ]
			then
				echo "\n========================================\n" >> /tmp/mail_vxrelocd.$$
				set_to_env_locale
				(export pl_name v_name; egettxt "\
\\nRaid log $pl_name in volume $v_name failed.
A new raid log for volume $v_name was created.
\\n" vxvmshm:592
				) >> /tmp/mail_vxrelocd.$$ 
				set_to_c_locale
			fi
			$MAILER -s "$_ms" "$recipients"  < /tmp/mail_vxrelocd.$$
			rm /tmp/mail_vxrelocd.$$

		else 
			(echo "\
\\nRaid log $pl_name failed, 
creation of new raid log for volume $v_name failed.
\\n"
			cat $vxassist_res
			) > /tmp/mail_vxrelocd.$$ 

			if [ $english_only_email -eq 0 ]
			then
				echo "\n========================================\n" >> /tmp/mail_vxrelocd.$$
				set_to_env_locale
				(export vxassist_res pl_name v_name; egettxt "\
\\nRaid log $pl_name failed, 
creation of new raid log for volume $v_name failed.
\\n" vxvmshm:594
				cat $vxassist_res
				) >> /tmp/mail_vxrelocd.$$ 
				set_to_c_locale
			fi
			$MAILER -s "$_ms" "$recipients"  < /tmp/mail_vxrelocd.$$
			rm /tmp/mail_vxrelocd.$$
		fi 

		            # is this a DRL plex 
	    elif [ X$pl_name = X`vxprint -Qp -g $dg_name -e '
				pl_name=="'$pl_name'" &&
				aslist->sd_is_log' -F %name` ] || [ "X$logtype" = "Xdrl" ]
		then 		

		[ "X$logtype" = "X" ] && hr_create_new_plex

		if [ $? -eq 0 ] || [ "X$logtype" != "X" ]; then 
			(echo "\
\\nRemoved detached DRL plex $pl_name, 
a new DRL log for volume $v_name was created.
\\n"
			) > /tmp/mail_vxrelocd.$$ 
			if [ $english_only_email -eq 0 ]
			then
				echo "\n========================================\n" >> /tmp/mail_vxrelocd.$$
				set_to_env_locale
				(export pl_name v_name; egettxt "\
\\nRemoved detached DRL plex $pl_name, 
a new DRL log for volume $v_name was created.
\\n" vxvmshm:595
				) >> /tmp/mail_vxrelocd.$$ 
				set_to_c_locale
			fi
			$MAILER -s "$_ms" "$recipients"  < /tmp/mail_vxrelocd.$$
			rm /tmp/mail_vxrelocd.$$

                 else 
			(echo "\
\\nDRL $pl_name detached, Creating new DRL for volume $v_name failed.
\\n"; cat $vxassist_res
			) > /tmp/mail_vxrelocd.$$ 
			if [ $english_only_email -eq 0 ]
			then
				echo "\n========================================\n" >> /tmp/mail_vxrelocd.$$
				set_to_env_locale
				(export vxassist_res pl_name v_name; egettxt "\
\\nDRL $pl_name detached, Creating new DRL for volume $v_name failed.
\\n" vxvmshm:596; cat $vxassist_res
				) >> /tmp/mail_vxrelocd.$$ 
				set_to_c_locale
			fi
			$MAILER -s "$_ms" "$recipients"  < /tmp/mail_vxrelocd.$$
			rm /tmp/mail_vxrelocd.$$

                 fi

	    fi  
		
	done

		# attempt relocation of all dms in the list
		if [ -s $reloc_dm_list ]
		then
			# remove dups if multiple subdisks on dm failed
			sort <$reloc_dm_list >$tmpfile
			uniq <$tmpfile >$reloc_dm_list
			cat $reloc_dm_list | \
			while read dm_name dg_name v_name
			do
				vxdg list | grep $dg_name|grep shared >/dev/null
				if [ $? -eq 0 ]; then
					# Only master will start the relocation
					vxdctl -c mode | grep SLAVE > /dev/null
					if [ $? -eq 0 ]; then
						(echo "\
hot-relocation operation for shared disk group $dg_name will be performed on master node"
						) > /tmp/mail_vxrelocd.$$ 
						if [ $english_only_email -eq 0 ]
						then
							echo "\n========================================\n" >> /tmp/mail_vxrelocd.$$
							set_to_env_locale
							(export dg_name; egettxt "\
hot-relocation operation for shared disk group $dg_name will be performed on master node" vxvmshm:863
							) >> /tmp/mail_vxrelocd.$$ 
							set_to_c_locale
						fi
						$MAILER -s "$_ms" "$recipients"  < /tmp/mail_vxrelocd.$$
						rm /tmp/mail_vxrelocd.$$

						continue
					fi
				fi

				# if relocation works then add volume to
				# list of volumes to recover
				try_reloc $dm_name $dg_name $v_name"$@"
				[ $? -eq 0 ] && echo $dg_name $v_name >> $recoverlist

				if [ -s $resultfile ]
				then
					(
					cat $resultfile | \
					while read r_v_name a b sd_name c newsd_name
					do
						echo "\
Volume $r_v_name Subdisk $sd_name relocated to $newsd_name,
but not yet recovered.\\n"
					done
					) > /tmp/mail_vxrelocd.$$ 
					if [ $english_only_email -eq 0 ]
					then
						echo "\n========================================\n" >> /tmp/mail_vxrelocd.$$
						set_to_env_locale
						(
						cat $resultfile | \
						while read r_v_name a b sd_name c newsd_name
						do
							export sd_name newsd_name r_v_name; egettxt "\
Volume $r_v_name Subdisk $sd_name relocated to $newsd_name,
but not yet recovered.\\n" vxvmshm:962
						done
						) >> /tmp/mail_vxrelocd.$$ 
						set_to_c_locale
					fi
					$MAILER -s "$_ms" "$recipients"  < /tmp/mail_vxrelocd.$$
					rm /tmp/mail_vxrelocd.$$

				fi
				> $resultfile
			done
		fi
                if [ -s $recoverlist ]
                then
			sort <$recoverlist >$tmpfile
			uniq <$tmpfile >$recoverlist
                        cat $recoverlist | \
                        while read dg_name v_name
                        do

		# recover snapdis plexes.
		snapplex=`vxprint -g $dg_name -pe ' assoc.v_name="'$v_name'" &&
						(pl_state="SNAPDIS" || pl_state="SNAPTMP") &&
						pl_kstate!=ENABLED' -F %name `
		snapret=0
		if [ "X$snapplex" != "X" ]; then
			for a in $snapplex; do
				vxplex -g $dg_name dis $a
				vxplex -g $dg_name snapstart $v_name $a > $vxrecover_res 2>&1
				snapret=$?
				grep ERROR $vxrecover_res
				[ $? -eq 0 ] && snapret=1
			done
		fi

                                if [ -n "$o_args" ]
				then
				    vxrecover -g $dg_name -o $o_args -s $v_name   > $vxrecover_res 2>&1 
				    ret=$?
				    grep ERROR $vxrecover_res
				else
				    vxrecover -g $dg_name -s $v_name  > $vxrecover_res 2>&1 
				    ret=$?
				    grep ERROR $vxrecover_res
				fi

				if [ $ret -ne 0 -o $? -ne 1 -o $snapret -ne 0 ]
				then
					(echo "\
Failure recovering $v_name in disk group $dg_name.\n"
					cat $vxrecover_res
					) > /tmp/mail_vxrelocd.$$ 
					if [ $english_only_email -eq 0 ]
					then
						echo "\n========================================\n" >> /tmp/mail_vxrelocd.$$
						set_to_env_locale
						(export dg_name v_name vxrecover_res; egettxt "\
Failure recovering $v_name in disk group $dg_name.\n" vxvmshm:600
						cat $vxrecover_res
						) >> /tmp/mail_vxrelocd.$$ 
						set_to_c_locale
					fi
					$MAILER -s "$_ms" "$recipients"  < /tmp/mail_vxrelocd.$$
					rm /tmp/mail_vxrelocd.$$

				else
					(echo "\
Recovery complete for volume $v_name in disk group $dg_name.\n"
					) > /tmp/mail_vxrelocd.$$ 
					if [ $english_only_email -eq 0 ]
					then
						echo "\n========================================\n" >> /tmp/mail_vxrelocd.$$
						set_to_env_locale
						(export dg_name v_name vxrecover_res; egettxt "\
Recovery complete for volume $v_name in disk group $dg_name.\n" vxvmshm:601
						) >> /tmp/mail_vxrelocd.$$ 
						set_to_c_locale
					fi
					$MAILER -s "$_ms" "$recipients"  < /tmp/mail_vxrelocd.$$
					rm /tmp/mail_vxrelocd.$$
                                fi
                        done
		fi

		rm -f $hr_hist $recoverlist $resultfile
		hr_cleanup
}
 
checkfailing()
{
	failing_d1=`vxprint -AQdF '%name %failing %nodarec' | 
			awk '$2=="on" && $3=="off" {print $1}'`
	if [ -n "$failing_d1" ]
	then
		for i in $failing_d1
		do
			vxdisk check $i 2>/dev/null
		done

		checkfailingcnt=`expr $checkfailingcnt + 1`

		failing_d1=`vxprint -AQdF '%name %failing %nodarec' | 
				awk '$2=="on" && $3=="off" {print $1}'`
		if [ -n "$failing_d1" ] &&  [ $checkfailingcnt -lt 4 ]; then
			docheckfailing=yes
		else
			docheckfailing=no
		fi
	fi
}

# Get events from vold, waiting for 15 seconds of quiescence before
# checking for failed objects.
 
docheck=yes
docheckfailing=yes
checkfailingcnt=0


vxnotify -f -w 15 | while read code more
do
	[ -z "$verbose" ] || echo "Code: $code"
	case $code in
	waiting)    if [ "$docheck" = yes ]
			then
				checkdetach "$@"
				docheck=no
			fi
			if [ "$docheckfailing" = yes ]
			then
				checkfailing "$@"
			fi;;
	log-detach|detach|relocate|more|connected)
		checkfailingcnt=0
		docheckfailing=yes
                docheck=yes;;
	esac

done
