#!/bin/ksh
#
# DISCLAIMER
# ----------
# This material contains programming source code for your consideration.
# These examples have not been thoroughly tested under all conditions.
# IBM, therefore, cannot guarantee or imply reliability, serviceability,
# or function of these programs.
# ALL PROGRAMS CONTAINED HEREIN ARE PROVIDED TO YOU "AS IS", WITHOUT ANY
# WARRANTIES (EXPRESS OR IMPLIED) OR SUPPORT WHATSOEVER, INCLUDING BUT
# NOT LIMITED TO ANY IMPLIED WARRANTIES OF MERCHANTABILITY OR FITNESS
# FOR A PARTICULAR PURPOSE.
#
# +=================================================================+
# |Shell script to automate daily backup of  Flowmark/Objectstore DB|
# +=================================================================+
#  THIS VERSION IS INTENDED FOR FM 2.3 FIXPACK1 ON AIX 4.x
#
#  Scenario:
#  -----------
#  Flowmark RTSRV and Objectstore server are running on same machine.
#  Each day you have a time window where you shut down the FM system and
#  backup the database.
#  7 backup images (history) are kept (one for each day).
#
#  Invocation:
#  -------------
#  see usage() below
#
#  Assumption:- run as root
#  -----------
#             - all FlowMark users are logged off
#             - all activity implementations on execution clients
#               have ended when the script is invoked
#             - Flowmark server and Objectstore server are running
#             - only one RTSRV is running on the machine
#             - only one FM database needs to be backed up
#             - run every day at same time by using 'at' or 'cron'
#             - running FM 2.3 Fixpack 1 or later, otherwise oscompact
#               should not run
#
#  Steps:
#  --------   - verify parameters
#             - delete last weeks backup and archive for the day (e.g.
#               on Monday delete last Monday's backup and archive logs)
#             - shutdown FM server components if running
#             - ossrvchkpt the database server
#             - osverifydb the database
#               optionally:
#               - oscompact the database
#               - osverify the compacted database again
#             - osbackup the FM database
#             - close osarchiv
#             - backup operational databases (e.g. DB2)
#                                              ==> ADD YOUR CODE HERE!
#             - shutdown osserver and cache manager
#               optionally
#             - osserver -ReallocateLog  to keep log from growing
#             - start osserver
#             - start archiving
#             - start FM server components  ==> ADAPT TO YOUR needs
#             - create a script restore.ksh which allows to restore DB
#
# Peter Bendel (BED at SDFVM1)

usage ()
{
echo "                                                             " >&2
echo "Usage:                                                       " >&2
echo "----------                                                   " >&2
echo $0 "-d <database> -f <backup-directory>"                        >&2
echo "                                                             " >&2
echo "e.g. " $0 "-d /var/exm/db/EXMDB -f /backups/FM               " >&2
echo "where                                                        " >&2
echo "<database> is the fully qualified filename of the FlowMark DB" >&2
echo "<backup-directory> is located on a hard disk different from  " >&2
echo "                   the database's hard disk                  " >&2
echo "                 Use fully qualified path without trailing /." >&2
echo "                                                             " >&2
}


# check if run as root user
# -------------------------
user=`whoami`
if [ $user != "root" ]; then
   echo "Error: Only root is allowed to execute this script." >&2
   exit 1
fi

# check input parms and whether database-name provided exists
# -----------------------------------------------------------
if [ $# != 4 ] || [ $1 != "-d" ]  || [ $3 != "-f" ]
then
    usage
    exit 3
fi

if [ ! -f $2 ]
then
    usage
    echo Error: $2 is not a valid database name.  >&2
    exit 4
fi

if [ ! -d $4 ]
then
    usage
    echo Error: $4 is not a valid directory.  >&2
    exit 4
fi

# CONSTANTS
# ---------
FMBASE=/usr/lpp/exm
#OS_ROOTDIR=/usr/lpp/ostore/cset
HOST=`hostname`
TODAY=`date | cut -f1 -d" "`
BACKUPDIR=$4"/"${TODAY}
DATABASE=$2

# Restart the Objectstore server
# ----------------------------------------
start_server ()
{
  echo Starting Objectstore server
  j=1
  ${OS_ROOTDIR}/lib/osserver 2>>${BACKUPDIR}/restart.log \
                              >>${BACKUPDIR}/restart.log
  while [[ $? != 0 && $j < 9 ]]
  do
      echo "Waiting for 5 seconds ...."
      sleep 5
      echo " ... now retry"
      let j=j+1
      ${OS_ROOTDIR}/lib/osserver 2>>${BACKUPDIR}/restart.log \
                                  >>${BACKUPDIR}/restart.log
  done
  if [ $? != 0 ]
  then
      cat ${BACKUPDIR}/restart.log >&2
      echo "Error: could not restart Objectstore server." >&2
      exit 13
  fi
  echo "Objectstore server started successfully."
}

# Reallocate the Objectstore server log file
# ----------------------------------------
reallocate_server ()
{
  echo Reallocating Objectstore server log
  j=1
  ${OS_ROOTDIR}/lib/osserver -ReallocateLog \
                              2>>${BACKUPDIR}/reallocate.log \
                              >>${BACKUPDIR}/reallocate.log
  while [[ $? != 0 && $j < 9 ]]
  do
      echo "Waiting for 5 seconds ...."
      sleep 5
      echo " ... now retry"
      let j=j+1
      ${OS_ROOTDIR}/lib/osserver -ReallocateLog \
                                  2>>${BACKUPDIR}/reallocate.log \
                                  >>${BACKUPDIR}/reallocate.log
  done
  if [ $? != 0 ]
  then
      cat ${BACKUPDIR}/reallocate.log >&2
      echo "Error: could not reallocate Objectstore log." >&2
      exit 13
  fi
  echo "Objectstore log file reallocated successfully."
}

# erasing last weeks backup
# -----------------------------------------------------------
echo "Erasing last weeks backup in directory " ${BACKUPDIR}
rm -r ${BACKUPDIR}
mkdir ${BACKUPDIR}


# shutdown flowmark server components if running
# -----------------------------------------------------------
echo "Shutting down FlowMark server components:"
${FMBASE}/bin/exmpshtd
echo "Waiting for 5 seconds ...."
sleep 5

# make sure FlowMark server is no longer running
# ------------------------------------------
j=1
processes=`ps -ef | grep exmp | wc -l`
while [[ $processes -gt 1 && $j < 9 ]]
do
    echo "FlowMark still running...Waiting another 20 seconds ...."
    sleep 20
    echo " ... now retry"
    let j=j+1
    processes=`ps -ef | grep exmp | grep -v exmpisrv | wc -l`
done

processes=`ps -ef | grep exmp | wc -l`

if [ $processes -gt 1 ]; then
   echo "Flowmark server still running. "
   echo "Using brute force to shut it down. "
   ${FMBASE}/adm/exmclean kill
   echo "Waiting for 5 seconds ...."
   sleep 5
fi
processes=`ps -ef | grep exmp | wc -l`
if [ $processes -gt 1 ]; then
   echo "Flowmark server cannot be shut down. " >&2
   exit 5
fi
${FMBASE}/adm/exmclean
echo "FlowMark server components were shut down successfully."

# checkpoint the active Objectstore server
# ----------------------------------------
echo Checkpointing Objectstore server on host $HOST
${OS_ROOTDIR}/bin/ossvrchkpt $HOST
if [ $? != 0 ]
then
    echo "Error: Objectstore server on host " $HOST >&2
    echo "cannot be checkpointed". >&2
    exit 5
fi

echo Sleeping for 30 seconds
sleep 30

# verify that the database is still valid (osverifydb)
# ----------------------------------------
# use osverifydb and redirect output
echo "Verifying Objectstore database " ${DATABASE}
${OS_ROOTDIR}/bin/osverifydb ${DATABASE} >${BACKUPDIR}"/"osverify.out
if [ $? != 0 ]
then
    echo "Error: osverifydb command failed." >&2
    exit 6
fi

# check output if it contains errors
foundlines=`egrep "contains a pointer|illegal|err_" \
                     ${BACKUPDIR}"/"osverify.out | wc -l`
if [ $foundlines -gt 0 ]; then
   echo "Database verification found errors! " >&2
   echo "Please contact the FlowMark service organization. " >&2
   exit 7
fi
echo "Found " ${foundlines} " line(s) containing errors."

# compact the database ONLY IF RUNNING OBJECSTORE 4.0.2.B.2
# ----------------------------------------
# check Objectstore version
version=`${OS_ROOTDIR}/bin/osversion | grep "4\.0\.2\.B\.2" | wc -l`
if [ $version -ne 1 ]; then
   echo "No need to compact database." >&2
else
  echo "Objectstore version is 4.0.2.B.2."
  # no need to save database before compact because old state can
  # be restored from yesterday's osbackup and osarchiv log
  echo "Compacting database " ${DATABASE}
  ${OS_ROOTDIR}/bin/oscompact -dbs_to_compact ${DATABASE}
  if [ $? != 0 ]
  then
      echo "Error: oscompact command failed..." >&2
      echo "... processing continues." >&2
  else
      echo "oscompact completed successfully."
  fi

fi

# verify that the database is still valid (osverifydb)
# ----------------------------------------
# use osverifydb and redirect output
echo "Verifying Objectstore database after compact" ${DATABASE}
${OS_ROOTDIR}/bin/osverifydb ${DATABASE} >${BACKUPDIR}"/"osverify2.out
if [ $? != 0 ]
then
    echo "Error: osverifydb command failed." >&2
    exit 7
fi
# check output if it contains errors
foundlines=`egrep "contains a pointer|illegal|err_" \
               ${BACKUPDIR}"/"osverify2.out | wc -l`
if [ $foundlines -gt 0 ]
then
   echo "Database verification found errors after compact! " >&2
   echo "Please contact the FlowMark service organization. " >&2
   exit 8
fi
echo "Found " ${foundlines} " line(s) containing errors."

# do a full backup
# -----------------------------------------------------------
echo "Creating a full backup in directory " ${BACKUPDIR}
${OS_ROOTDIR}/bin/osbackup -i ${BACKUPDIR}"/"recordfile \
                           -f ${BACKUPDIR}"/EXMDB.BACKUP" \
                           ${DATABASE}
if [ $? != 0 ]
then
    echo "Error: osbackup command failed." >&2
    exit 9
fi

# close yesterday's osarchiv
# -----------------------------------------------------------
echo "Closing yesterday's osarchiv logging"

# by killing the 'sleep' process osarchiv is ended gracefully
# (closes last archive log file)
#
# create a list of processes, filter out the correct sleep process and
# kill it (using a "strange" sleep time to uniquely identify the sleep process)
#
ps -e -F"%p %a" | grep "sleep 21[0]321" | cut -f1 -d" " | xargs kill -9

echo Sleeping for 30 seconds
sleep 30

# make sure yesterday's osarchiv has ended
# -----------------------------------------------------------
j=1
processes=`ps -ef | grep osarchiv | wc -l`
while [[ $processes -gt 1 && $j < 9 ]]
do
    echo "osarchiv still running...Waiting another 20 seconds ...."
    sleep 20
    echo " ... now retry"
    let j=j+1
    processes=`ps -ef | grep osarchiv | wc -l`
done
if [ $processes -gt 1 ]; then
 echo "osarchiv still running. "
 echo "Using brute force to shut it down. "
 ps -e -F"%p %a" | grep "osarchiv -[i]" | cut -f1 -d" " | xargs kill -9
 echo "Waiting for 5 seconds ...."
 sleep 5
fi
processes=`ps -ef | grep osarchiv | wc -l`
if [ $processes -gt 1 ]; then
   echo "osarchiv cannot be shut down. " >&2
   exit 9
fi

# ADD YOUR CODE HERE TO BACKUP OTHER (OUTSIDE FLOWMARK) DATA
# ----------------------------------------------------------
# (E.G. DB2 DATABASES)
# ----------------------------------------------------------

# shutdown osserver and cache mgr
#==================================
echo Shutting down Objectstore cachemgr on host $HOST
${OS_ROOTDIR}/bin/oscmshtd $HOST
if [ $? != 0 ]
then
    echo "Error: Objectstore cache mgr on host " $HOST \
         "cannot be shut down". >&2
    exit 10
fi

echo Shutting down Objectstore server on host $HOST
${OS_ROOTDIR}/bin/ossvrshtd -f $HOST
if [ $? != 0 ]
then
    echo "Error: Objectstore server on host " $HOST \
         "cannot be shut down". >&2
    exit 11
fi

# - osserver -ReallocateLog  to keep log from growing
#=====================================================
reallocate_server

# Restart the Objectstore server
# ----------------------------------------
start_server

# Restart osarchiv archiving
# ----------------------------------------
echo "Restarting osarchiv."

#Starting osarchiv in background on unix:
#
# osarchiv is interactive and reads from stdin
# this is the reason for the sleep command which runs forever and keeps
# osarchiv alive
# osarchiv output is redirected into a file
# (using a "strange" sleep time to uniquely identify the sleep process)
#
sleep 210321 | ${OS_ROOTDIR}/bin/osarchiv -i 60 -a ${BACKUPDIR}"/recordfile" \
                        -d ${BACKUPDIR} \
                        ${DATABASE} \
                        > ${BACKUPDIR}"/osarchiv.msgs" 2>&1 &
if [ $? != 0 ]
then
    echo "Error: osarchiv cannot be started." >&2
    exit 14
else
    echo "Archive logs are placed in " ${BACKUPDIR}
    echo "osarchiv messages are reported into " \
          ${BACKUPDIR}"/osarchiv.msgs"
fi



# start FlowMark server components
# -----------------------------------------------------------
# add your own components here, this script only starts runtime server
echo "Starting FlowMark runtime server"
${FMBASE}/bin/exmpesrv -f /usr/lpp/exm/bin/exmpzcfg.prf &

# TBD notification server, bundle server, delivery server,
# whatever you need

# create a restore procedure
#------------------------------
echo '#!/bin/ksh' >${BACKUPDIR}"/restore.ksh"
echo '# 1. shut down a running osarchiv process' >>${BACKUPDIR}"/restore.ksh"
echo 'ps -e -F"%p %a" | grep "sleep 21[0]321" | cut -f1 -d" " | xargs kill -9' >>${BACKUPDIR}"/restore.ksh"
echo '# 2. restore from full backup' >>${BACKUPDIR}"/restore.ksh"
echo '/bin/yes no | ${OS_ROOTDIR}/bin/osrestore -f ' ${BACKUPDIR}"/EXMDB.BACKUP" >>${BACKUPDIR}"/restore.ksh"
echo '# 3. recover from archive logs' >>${BACKUPDIR}"/restore.ksh"
echo 'ls ' ${BACKUPDIR}"/9*" '>' ${BACKUPDIR}"/osrecovr.lst" >>${BACKUPDIR}"/restore.ksh"
echo '/bin/yes yes | ${OS_ROOTDIR}/bin/osrecovr -F ' ${BACKUPDIR}"/osrecovr.lst" >>${BACKUPDIR}"/restore.ksh"
chmod 700 ${BACKUPDIR}"/restore.ksh"

echo "To restore from today's backup invoke " ${BACKUPDIR}"/restore.ksh"
echo It does the following
cat ${BACKUPDIR}"/restore.ksh"
