#!/usr/bin/ksh
# CDDL HEADER START
#
# The contents of this file are subject to the terms of the
# Common Development and Distribution License (the "License").
# You may not use this file except in compliance with the License.
#
# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
# or http://www.opensolaris.org/os/licensing.
# See the License for the specific language governing permissions
# and limitations under the License.
#
# When distributing Covered Code, include this CDDL HEADER in each
# file and include the License file at usr/src/OPENSOLARIS.LICENSE.
# If applicable, add the following below this CDDL HEADER, with the
# fields enclosed by brackets "[]" replaced with your own identifying
# information: Portions Copyright [yyyy] [name of copyright owner]
#
# CDDL HEADER END
#
#
# Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
# Use is subject to license terms.
#
# NWS DataServices within SunCluster reconfiguration script.
#
# Description:
#
# This script is called from /usr/cluster/lib/sc/run_reserve at
# appropriate times to start and stop the NWS DataServices as SunCluster
# disk device groups are brought online or taken offline.
#
# SNDR configuration requires that a resource group to be configured.
# 1. The resource group name should be same as device group name with -stor-rg
#    added. e.g. if device group name is abc-dg then resource group name
#    would be abc-dg-stor-rg. 
# 2. It should have 2 resources in it, unless one of the resource types is the
#    SUNW.GeoCtlAVS. One of type SUNW.LogicalHostname and either SUNW.HAStorage
#    or SUNW.HAStoragePlus types. Resource type versioning is ignored.
#    HAStorage type resource, should have ServicePaths property set to
#    device group name. HAStoragePlus type resource, should have either the
#    FilesystemMountPoints pointing to a files system associated with the
#    device group name, or GlobalDevicePaths property set to device group name.
#    LogicalHostname type resource should have a failoverIP address in it and
#    it will be used by SNDR to communicate with the secondary side.
#
# As SNDR requires that the LogicalHost (failover) IP address which is a
# part of resource group for SNDR, to be hosted on the same node where the 
# device group is, it tries to move the resource group also alongwith the
# device group, in become_primary case of run_reserve script. While
# in primary_to_secondary case, it will try to kill the switchover function
# if it is still running in background, after stopping NWS data services.
# 
# Usage:
#
# /usr/cluster/sbin/dscfg_reconfigure { start | stop } diskgroup
#
# Configuration:
#
# Scripts to be run should have been symlinked into $NWS_START_DIR and
# $NWS_STOP_DIR.  Note that the scripts are processed in lexical order,
# and that unlike /etc/rc?.d/ there is no leading S or K character.
#
# Exit status:
#
# 0 - success
# 1 - error
#

#
# Global variables
#

# this program
typeset -r ARGV0=$(basename $0)

# directory full of start scripts
typeset -r NWS_START_DIR=/usr/cluster/lib/dscfg/start

# directory full of stop scripts
typeset -r NWS_STOP_DIR=/usr/cluster/lib/dscfg/stop

# the syslog facility to use.
# - conceptually this should be based on the output of
#   "scha_cluster_get -O SYSLOG_FACILITY", but that won't work early
#   during boot.
typeset -r SYSLOG_FACILITY=daemon

setvar PATH = "$PATH:/usr/cluster/bin:/etc"

# Variables for retrying scswitch of Resource group for SNDR
setvar retry_num = '12'
setvar retry_interval = '10'
setvar rgname = ''
setvar rgstat = ''
setvar skip_resource = '0'
setvar count_LogicalHostname = '0'
setvar count_HAStoragePlus = '0'

# Since the switchover of the resource group is called in background,    
# the stop action of the reconfig script will kill the background switchover
# if it is running. Since we are stopping the NWS services on the node, there
# is no need to switch the resource group, so  it is killed.
# The pid of the process is kept in file /var/run/scnws/$dg.pid.
# Input:  dg - device group
# Output: Nothing, kills the process

proc kill_scswitch
{
        setvar dg = "$1"
        if test -f /var/run/scnws/$dg.pid
        {
                for i in [$(cat /var/run/scnws/$dg.pid)]
                {
                        setvar pid = "$i"
                        kill -9 $pid
                }
                rm -f /var/run/scnws/$dg.pid
        }
}

# Get the status of the resource group on this node, using scha commands.
# Input: resource group - $1
# Output: Status

proc get_rgstat
{
	setvar rg = "$1"
	setvar rgstat = $(scha_resourcegroup_get -O RG_STATE -G $rg)
}

# This function is called in background from do_scswitch function, to
# switch the resource group to this node, which is becoming primary for
# the diskgroup. If the status of resource group is Offline, it will use
# scswitch command to switch the resource group to this node. If it has
# become Online, cleanup pid file. If it is Pending, the resource group
# is in the state of becoming online, so wait for sometime to become Online..
# scswitch may fail, so the function retries $retry_num times, waiting for
# $retry_interval seconds.
# Input: resource group - $1, Diskgroup/Diskset - $2
# Output: 0 - success, 1 - failure

proc switchfunc
{
        setvar rg = "$1"
        setvar dg = "$2"
	setvar how_many = '0'
	sleep 2
	while [ $how_many != $retry_num ]
	{
		get_rgstat $rg
		match $rgstat {
		with "ONLINE"
		 	rm -f /var/run/scnws/$dg.pid
			return 0
			

		with "OFFLINE"
			logger -p ${SYSLOG_FACILITY}.notice \
			-t "NWS.[$ARGV0]" $(gettext "scswitch of resource group) $rg

			scswitch -z -g $rg -h $(hostname)
			setvar retval = "$Status"
			if test $retval != 0
			{
				sleep $retry_interval
				setvar how_many = $(($how_many + 1))
			}
			

		with "PENDING_ONLINE"
			logger -p ${SYSLOG_FACILITY}.notice \
			-t "NWS.[$ARGV0]" $(gettext "pending online of resource group) $rg
			sleep $retry_interval
			setvar how_many = $(($how_many + 1))
			

		with *
			logger -p ${SYSLOG_FACILITY}.notice \
			-t "NWS.[$ARGV0]" $(gettext "Improper resource group status for Remote Mirror) $rgstat
		 	rm -f /var/run/scnws/$dg.pid
			return 1
				
		}
	}
	logger -p ${SYSLOG_FACILITY}.err \
	-t "NWS.[$ARGV0]" "Did not switch resource group for Remote Mirror. System Administrator intervention required"
 	rm -f /var/run/scnws/$dg.pid
	return 1
}


# This function calls switchfunc function in background, to switch the 
# resource group for SNDR. It validates the diskgroup/diskset is configured 
# for SNDR, checks if the resource group is in Managed state etc.
# If it detects a mis-configuration, it will disable SNDR for the
# device group being processed. This is to prevent cluster hangs and panics.
#  
# The ServicePaths extension property of HAStorage type resource or the
# GlobalDevicePaths extension property of HAStoragePlus, both of which
# specify the device group, serve as a link or mapping to retrieve the 
# resource group associated with the SNDR configured device group.
# Switchfunc is called in the background to avoid the deadlock situation arising
# out of switchover of resource group from within device group switchover.
#
# In run_reserve context, we are doing the device group switchover, trying to
# bring it online on the node. Device group is not completely switched online,
# until the calling script run_reserve returns. In the process, we are calling
# the associated SNDR resource group switchover using scswitch command. 
# Resource group switchover will trigger the switchover of device group also. 
#
# If resource group switchover is called in foreground, before the device 
# group has become online, then it will result in switching the device group 
# again, resulting in deadlock. Resource group can not become online until 
# the device group is online and the device group can not become online until the 
# script returns, causing this circular dependency resulting in deadlock. 
#
# Calling the resource group switch in background allows current run_reserve
# script to return immediately, allowing device group to become online.
# If the device group is already online on the node, then the resource group 
# does not cause the device group switchover again.
#
# Input: Device group dg - $1
# Output: 0 - success
#	  1 - either dg not applicable for SNDR or error
#	  2 - SNDR mis-configuration

proc do_scswitch
{
	setvar dg = "$1"

        if test ! -x /usr/cluster/bin/scha_resource_get \
		-o ! -x /usr/cluster/bin/scha_resourcegroup_get
        {
                return 1
        }

# hard coded rg name from dg
	setvar rgname = ""$dg-stor-rg""
	scha_resourcegroup_get -O rg_description -G $rgname > /dev/null
	if test $Status != 0
	{
# There is no device group configured in cluster for SNDR with this cluster tag
		return 1
	}

# Check the state of resource group

	get_rgstat $rgname
	if test -z $rgstat \
		-o $rgstat = "UNMANAGED" -o $rgstat = "ERROR_STOP_FAILED"
	{
		logger -p ${SYSLOG_FACILITY}.notice \
		-t "NWS.[$ARGV0]" \
		$(gettext "Improper Remote Mirror resource group state) $rgstat
        	return 2 
	}

# Check whether resources are of proper type and they are enabled

	setvar rs_list = $(scha_resourcegroup_get -O resource_list -G $rgname)
	if test -z $rs_list
	{
		logger -p ${SYSLOG_FACILITY}.notice \
		-t "NWS.[$ARGV0]" \
		$(gettext "No resources in Remote Mirror resource group <$rgname>)
		return 2 
	}
	for rs in [$rs_list]
	{
		setvar rs_type = $(scha_resource_get -O type -R $rs -G $rgname  | cut -d':' -f1)
		match $rs_type {
		with SUNW.LogicalHostname
			setvar rs_enb = $(scha_resource_get -O ON_OFF_SWITCH -R $rs -G $rgname)
			if test $rs_enb = "ENABLED"
			{
			setvar count_LogicalHostname = $(($count_LogicalHostname + 1))
			}
			
		with SUNW.HAStoragePlus
			setvar rs_enb = $(scha_resource_get -O ON_OFF_SWITCH -R $rs -G $rgname)
			if test $rs_enb = "ENABLED"
			{
			setvar count_HAStoragePlus = $(($count_HAStoragePlus + 1))
			}
			
		}
	}
	if test $count_LogicalHostname -lt 1
	{
		logger -p ${SYSLOG_FACILITY}.notice \
		-t "NWS.[$ARGV0]" $(gettext "Missing Enabled Logical Host in resource group <$rgname> for Remote Mirror)
		return 2
	} elif test $count_LogicalHostname -gt 1
        {
		logger -p ${SYSLOG_FACILITY}.notice \
		-t "NWS.[$ARGV0]" $(gettext "Too Many Enabled Logical Host in resource group <$rgname> for Remote Mirror)
		return 2
	}

	if test $count_HAStoragePlus -lt 1
	{
		logger -p ${SYSLOG_FACILITY}.notice \
		-t "NWS.[$ARGV0]" $(gettext "Missing Enabled HAStoragePlus in resource group <$rgname> for Remote Mirror)
		return 2
	} elif test $count_HAStoragePlus -gt 1
	{
		logger -p ${SYSLOG_FACILITY}.notice \
		-t "NWS.[$ARGV0]" $(gettext "Too Many Enabled HAStoragePlus in resource group <$rgname> for Remote Mirror)
		return 2
	}

# Invoke switchfunc to switch the resource group. 

	switchfunc $rgname $dg &
	setvar pid = "$BgPid"
	mkdir -p /var/run/scnws/
	rm -f /var/run/scnws/$dg.pid
	echo $pid > /var/run/scnws/$dg.pid

	return 0
}


#
# Functions
#

proc usage
{
	logger -p ${SYSLOG_FACILITY}.err \
	    -t "NWS.[$ARGV0]" "usage: $ARGV0 { start | stop } diskgroup"
	exit 1
}


# Input: arg1) $NWS_START_DIR - location of NWS scripts
#	 arg2) start / stop
#	 arg3 ) device group - $2
#	 arg4) sndr_ena / sndr_dis
# Output: Nothing. Log error if seen

proc process_dir
{
	typeset dir=$1
	typeset arg1=$2
	typeset dg=$3
	typeset arg2=$4
	typeset RDC=$dir/10rdc

	if [[ -d $dir ]]
	{
		for f in [$dir/*]
		{
			# process scripts in the directories in lexical order
			# note - no leading S or K unlike /etc/rc?.d/

			if test -s $f && test $arg2 != "sndr_dis"   
			{
				# run script and pipe output through
				# logger into syslog

				/usr/bin/ksh $f $arg1 $dg 2>&1 |
				    logger -p ${SYSLOG_FACILITY}.notice \
					-t "NWS.[${ARGV0}:$(basename $f)]"
			} else {
			# SNDR misconfigured - prevent start
                            if test -s $f && test $f != $RDC 
                            {
                                # run script and pipe output through
                                # logger into syslog
                                /usr/bin/ksh $f $arg1 $dg 2>&1 |
                                    logger -p ${SYSLOG_FACILITY}.notice \
                                        -t "NWS.[${ARGV0}:$(basename $f)]"
			    }
			}
		}
	} else {
		logger -p ${SYSLOG_FACILITY}.err \
		    -t "NWS.[$ARGV0]" "no directory: $dir"
	}
}


#
# main
#

if test $Argc -ne 2
{
	usage
	# not reached
}


match $1 {
with start
	logger -p ${SYSLOG_FACILITY}.notice -t "NWS.[$ARGV0]" "starting: $ARGV0 $ifsjoin(ARGV)"
	do_scswitch $2
	setvar retval = "$Status"
	if test $retval == 2
	{
		logger -p ${SYSLOG_FACILITY}.err \
		    -t "NWS.[$ARGV0]" "**FATAL ERROR** Remote Mirror is mis-configured and DISABLED for devicegroup <"$2"> " 
		# Disable SNDR 
		process_dir $NWS_START_DIR start $2 sndr_dis
	} else {
		process_dir $NWS_START_DIR start $2 sndr_ena
	}
	
with stop
	logger -p ${SYSLOG_FACILITY}.notice -t "NWS.[$ARGV0]" "stopping: $ARGV0 $ifsjoin(ARGV)"
	process_dir $NWS_STOP_DIR stop $2 sndr_ena
	kill_scswitch $2
	

with *
	usage
	# not reached
	
}

logger -p ${SYSLOG_FACILITY}.notice -t "NWS.[$ARGV0]" "completed: $ARGV0 $ifsjoin(ARGV)"

exit 0