resource MySQLData {
	on PeanutBrain01 {
		local /dev/ada1
		remote 192.168.69.42
	}
	on PeanutBrain02 {
		local /dev/ada1
		remote 192.168.69.41
	}
}

resource FilesData {
	on PeanutBrain01 {
		local /dev/ada2
		remote 192.168.69.42
	}
	on PeanutBrain02 {
		local /dev/ada2
		remote 192.168.69.41
	}
}

[mysqld]
bind-address                    = 0.0.0.0

NAME       PROPERTY  VALUE               SOURCE
FilesData  sharenfs  maproot=root:wheel  local

notify 30 {
	match "system" "CARP";
	match "subsystem" "1@em0";
	match "type" "MASTER";
	action "/usr/local/sbin/carp-hast-switch primary";
};

notify 30 {
	match "system" "CARP";
	match "subsystem" "1@em0";
	match "type" "BACKUP";
	action "/usr/local/sbin/carp-hast-switch secondary";
};

#!/bin/sh

# The names of the HAST resources, as listed in /etc/hast.conf
resources="MySQLData FilesData"
# Resource mountpoints
resource_mountpoints="/var/db/mysql /FilesData"
# Supported file system types: UFS, ZFS
resource_filesystems="UFS ZFS"
# Service types: mysql nfs
resource_services="mysql nfs"

# Delay in mounting HAST resource after becoming primary
delay=3

# logging
log="local0.debug"
name="carp-hast"

# end of user configurable stuff

case "$1" in
	primary)
        logger -p $log -t $name "Switching to primary provider for - ${resources} -."
        sleep ${delay}

        # -- SERVICE MANAGEMENT --
        logger -p $log -t $name ">> Stopping services."
        resource_counter=1
        for resource in ${resources}; do
            resource_service=`echo $resource_services | cut -d\  -f$resource_counter`
            case "${resource_service}" in
            mysql)
                logger -p $log -t $name "Service MySQL detected for resource - ${resource} -."
                logger -p $log -t $name "Stoping MySQL service for resource - ${resource} -."
                service mysql-server stop
                logger -p $log -t $name "Done for resource ${resource}"
                ;;
            nfs)
                logger -p $log -t $name "Service NFS detected for resource - ${resource} -."
                logger -p $log -t $name "Disabling NFS-ZFS share for resource - ${resource} -."
                zfs set sharenfs="off" FilesData
                logger -p $log -t $name "Done for resource ${resource}"
                logger -p $log -t $name "Stopping NFS service for resource - ${resource} -."
                service nfsd stop
                logger -p $log -t $name "Done for resource ${resource}"
                ;;
            *)
                logger -p local0.error -t $name "ERROR: Unknown service: ${resource_filesystem}, exiting."
                exit 1
                ;;
            esac
            let resource_counter=$resource_counter+1
        done

        # -- HAST ROLE MANAGEMENT --
        logger -p $log -t $name ">> Managing disks."
        for resource in ${resources}; do
            # When primary HAST node is inaccesible secondary node stops hastd secondary processes automatically
            # Wait 30s for any "hastd secondary" processes to stop
            num=0
            logger -p $log -t $name "Waitting for secondary process of resource - ${resource} - to die."
            while $( pgrep -lf "hastd: ${resource} \(secondary\)" > /dev/null 2>&1 ); do
                let num=$num+1
                sleep 1
                if [ $num -gt 29 ]; then
                    logger -p $log -t $name "ERROR: Secondary process for resource - ${resource} - is still running after 30 seconds, exiting."
                    exit
                fi
            done
            logger -p $log -t $name "Secondary process for resource - ${resource} - dead successfully."

            # Switch role for resource
            logger -p $log -t $name "Switching resource - ${resource} - to primary."
            hastctl role primary ${resource}
            if [ $? -ne 0 ]; then
                logger -p $log -t $name "ERROR: Unable to change role to primary for resource - ${resource} -."
                exit 1
            fi
            logger -p $log -t $name "Role for HAST resource - ${resource} - switched to primary."
        done

        # -- WAIT FOR HAST DEVICE CREATION --
        logger -p $log -t $name ">> Waitting for hast devices."
        for resource in ${resources}; do
            num=0
            logger -p $log -t $name "Waitting for hast device of resource - ${resource} -."
            while [ ! -c "/dev/hast/${resource}" ]; do
                let num=$num+1
                sleep 1
                if [ $num -gt 29 ]; then
                    logger -p $log -t $name "ERROR: GEOM provider /dev/hast/${resource} did not appear, exiting."
                    exit
                fi
            done
            logger -p $log -t $name "Device /dev/hast/${resource} appeared for resource - ${resource} -."
        done

        # -- FILESYSTEM MANAGEMENT --
        logger -p $log -t $name ">> Managing filesystems."
        resource_counter=1
        for resource in ${resources}; do
            resource_mountpoint=`echo $resource_mountpoints | cut -d\  -f$resource_counter`
            resource_filesystem=`echo $resource_filesystems | cut -d\  -f$resource_counter`
            case "${resource_filesystem}" in
            UFS)
                logger -p $log -t $name "UFS filesystem detected in resource - ${resource} -."
                mkdir -p ${resource_mountpoint} 2>/dev/null
                logger -p $log -t $name "Checking /dev/hast/${resource} of resource - ${resource} -."
                fsck -p -y -t ufs /dev/hast/${resource}
                logger -p $log -t $name "Mounting /dev/hast/${resource} in ${resource_mountpoint}."
                out=`mount /dev/hast/${resource} ${resource_mountpoint} 2>&1`
                if [ $? -ne 0 ]; then
                    logger -p local0.error -t hast "ERROR: UFS mount - ${resource} - failed: ${out}."
                    exit 1
                fi
                logger -p local0.debug -t $name "UFS mount - ${resource} - mounted successfully."
                ;;
            ZFS)
                logger -p $log -t $name "ZFS filesystem detected in resource - ${resource} -."
                logger -p $log -t $name "Importing ZFS pool of resource - ${resource} -."
                out=`zpool import -f "${resource}" 2>&1`
                if [ $? -ne 0 ]; then
                    logger -p local0.error -t $name "ERROR: ZFS pool import for resource - ${resource} - failed: ${out}."
                    exit 1
                fi
                logger -p local0.debug -t $name "ZFS pool for resource - ${resource} - imported successfully."
                ;;
            *)
                logger -p local0.error -t $name "ERROR: Unknown filesystem: ${resource_filesystem}, exiting."
                exit 1
                ;;
            esac
            let resource_counter=$resource_counter+1
        done

        # -- SERVICE MANAGEMENT --
        logger -p $log -t $name ">> Starting services."
        resource_counter=1
        for resource in ${resources}; do
            logger -p $log -t $name "Starting service for resource - ${resource} -."
            resource_service=`echo $resource_services | cut -d\  -f$resource_counter`
            case "${resource_service}" in
            mysql)
                logger -p $log -t $name "Service MySQL detected for resource - ${resource} -."
                logger -p $log -t $name "Starting MySQL service for resource - ${resource} -."
                service mysql-server start
                logger -p $log -t $name "Done for resource . ${resource} -."
                ;;
            nfs)
                logger -p $log -t $name "Service NFS detected for resource - ${resource} -."
                logger -p $log -t $name "Starting NFS service for resource - ${resource} -."
                service nfsd start
                logger -p $log -t $name "Done for resource - ${resource} -."
                logger -p $log -t $name "Enabling NFS-ZFS share for resource - ${resource} -."
                zfs set sharenfs="maproot=root:wheel" FilesData
                logger -p $log -t $name "Done for resource - ${resource} -."
                ;;
            *)
                logger -p local0.error -t $name "ERROR: Unknown service: ${resource_filesystem}, exiting."
                exit 1
                ;;
            esac
            let resource_counter=$resource_counter+1
        done
    ;;

    secondary)
        logger -p $log -t $name "Switching to secondary provider for - ${resources} -."

        # -- SERVICE MANAGEMENT --
        logger -p $log -t $name ">> Stopping services."
        resource_counter=1
        for resource in ${resources}; do
            resource_service=`echo $resource_services | cut -d\  -f$resource_counter`
            logger -p $log -t $name "Stopping services for resource - ${resource} -."
            case "${resource_service}" in
            mysql)
                logger -p $log -t $name "Service MySQL detected for resource - ${resource} -."
                logger -p $log -t $name "Stopping MySQL service for resource - ${resource} -."
                service mysql-server stop
                logger -p $log -t $name "Done for resource - ${resource} -."
                ;;
            nfs)
                logger -p $log -t $name "Service NFS detected for resource - ${resource} -."
                logger -p $log -t $name "Disabling NFS-ZFS share for resource - ${resource} -."
                zfs set sharenfs="off" FilesData
                logger -p $log -t $name "Done for resource - ${resource} -."
                logger -p $log -t $name "Restarting NFS service for resource - ${resource} -."
                service nfsd restart
                logger -p $log -t $name "Done for resource - ${resource} -."
                ;;
            *)
                logger -p local0.error -t $name "ERROR: Unknown service: ${resource_service}, exiting."
                exit 1
                ;;
            esac
            let resource_counter=$resource_counter+1
        done

        # -- FILESYSTEM MANAGEMENT --
        logger -p $log -t $name ">> Managing filesystems."
        resource_counter=1
        for resource in ${resources}; do
            resource_mountpoint=`echo $resource_mountpoints | cut -d\  -f$resource_counter`
            resource_filesystem=`echo $resource_filesystems | cut -d\  -f$resource_counter`
            case "${resource_filesystem}" in
            UFS)
                logger -p $log -t $name "UFS filesystem detected in resource - ${resource} -."
                if ! mount | grep -q "^/dev/hast/${resource} on "
                then
                else
                    logger -p $log -t $name "Umounting - ${resource} -."
                    umount -f ${resource_mountpoint}
                    logger -p $log -t $name "Done."
                fi
                sleep $delay
                ;;
            ZFS)
                logger -p $log -t $name "ZFS filesystem detected in resources - ${resource} -."
                if ! mount | grep -q "^${resource} on ${resource_mountpoint}"
                then
                else
                    logger -p $log -t $name "Umounting - ${resource} -."
                    zfs umount ${resource}
                    logger -p $log -t $name "Done."
                    logger -p $log -t $name "Exporting ZFS pool of resources - ${resource} -."
                    out=`zpool export -f "${resource}" 2>&1`
                    if [ $? -ne 0 ]; then
                        logger -p local0.error -t $name "ERROR: ZFS pool export for resource - ${resource} - failed: ${out}."
                        exit 1
                    fi
                    logger -p local0.error -t $name "ZFS pool for resource - ${resource} - exported successfully."
                fi
                sleep $delay
                ;;
            *)
                logger -p local0.error -t $name "ERROR: Unknown filesystem: ${resource_filesystem}, exiting."
                exit 1
                ;;
            esac
            let resource_counter=$resource_counter+1
        done

        # -- HAST ROLE MANAGEMENT --
        logger -p $log -t $name ">> Managing resources."
        resource_counter=1
        for resource in ${resources}; do
            logger -p $log -t $name "Switching resource - ${resource} - to secondary."
            hastctl role secondary ${resource} 2>&1
            if [ $? -ne 0 ]; then
                logger -p $log -t $name "ERROR: Unable to switch resource - ${resource} - to secondary role."
                exit 1
            fi
            logger -p $log -t $name "Role for resource - ${resource} - switched to secondary successfully."
            let resource_counter=$resource_counter+1
        done
    ;;
esac

Name	Status	 Role		Components
MySQLData	-        init           /dev/ada1	192.168.69.42

Name	Status	 Role		Components
FilesData	-        init           /dev/ada2	192.168.69.42

Name	Status	 Role		Components
MySQLData	complete secondary      /dev/ada1	192.168.69.42

Name	Status	 Role		Components
FilesData	complete secondary      /dev/ada2	192.168.69.42