#!/bin/sh
# * ************************************************************************ *
# *            IDBLOG COLLECTOR SCRIPT                                       *
# *  Filename   :                                                            *
# *      idblog_collector.sh                                                 *
# *  Usage      :                                                            *
# *      $ sudo  sh idblog_collector.sh -h                                   *
# *  Description:                                                            *
# *      This script collects all necessary logs for ScaleArc investigation. *
# * ************************************************************************ *
# * *************************************************************** *
# *  STATIC VARIABLES                                               *
# * *************************************************************** *
readonly VERSION="5.4"
readonly C_RED=$'\033[31;1m'
readonly C_GREEN=$'\033[32;1m'
readonly C_BLUE=$'\033[34;1m'
readonly C_WHITE=$'\033[37;1;40m'
readonly C_RESET=$'\033[0m'

SA_MAJOR_VER=""
SA_MINOR_VER=""

dry="off"

# * *************************************************************** *
# *  OS COMMANDS                                                    *
# * *************************************************************** *
# Abort the script if any of OS commands are not available in this system
# make sure that bin/echo exists...
if [ -f /bin/echo ]; then
    readonly ECHO="/bin/echo";
else
    exit 1;
fi

# make sure that bin/awk exists...
if [ -f /bin/awk ]; then
    readonly AWK="/bin/awk";
else
    exit 1;
fi

# make sure that bin/cat exists...
if [ -f /bin/cat ]; then
    readonly CAT="/bin/cat";
else
    exit 1;
fi

# make sure that /bin/date exists...
if [ -f /bin/date ]; then
    readonly DATE="/bin/date";
else
    ${ECHO} "/bin/date is not available on this machine, aborting the script...";
    exit 1;
fi

# make sure that /bin/df exists...
if [ -f /bin/df ]; then
    readonly DF="/bin/df -P";
else
    ${ECHO} "/bin/df is not available on this machine, aborting the script...";
    exit 1;
fi

# make sure that /bin/find exists...
if [ -f /bin/find ]; then
    readonly FIND="/bin/find";
else
    ${ECHO} "/bin/find is not available on this machine, aborting the script...";
    exit 1;
fi

# make sure that /bin/grep exists...
if [ -f /bin/grep ]; then
    readonly GREP="/bin/grep";
else
    ${ECHO} "/bin/grep is not available on this machine, aborting the script...";
    exit 1;
fi

# make sure that /bin/ls exists...
if [ -f /bin/ls ]; then
    readonly LS="/bin/ls";
else
    ${ECHO} "/bin/ls is not available on this machine, aborting the script...";
    exit 1;
fi

# make sure that /bin/mkdir exists...
if [ -f /bin/mkdir ]; then
    readonly MKDIR="/bin/mkdir";
else
    ${ECHO} "/bin/mkdir is not available on this machine, aborting the script...";
    exit 1;
fi

# make sure that /bin/netstat exists....
if [ -f /bin/netstat  ]; then
    readonly NETSTAT="/bin/netstat";
else
    ${ECHO} "/bin/netstat is not available on this machine, aborting the script...";
    exit 1;
fi

# make sure that /bin/ps exists...
if [ -f /bin/ps ]; then
    readonly PS="/bin/ps";
else
    ${ECHO} "/bin/ps is not available on this machine, aborting the script...";
    exit 1;
fi

# make sure that /bin/uname exists...
if [ -f /bin/uname ]; then
    readonly UNAME="/bin/uname";
else
    ${ECHO} "/bin/uname is not available on this machine, aborting the script...";
    exit 1;
fi

# make sure that /bin/rm exists...
if [ -f /bin/rm ]; then
    readonly RM="/bin/rm";
else
    ${ECHO} "/bin/rm is not available on this machine, aborting the script...";
    exit 1;
fi

# make sure that /bin/rpm exists...
if [ -f /bin/rpm ]; then
    readonly RPM="/bin/rpm";
else
    ${ECHO} "/bin/rpm is not available on this machine, aborting the script...";
    exit 1;
fi

# make sure that /bin/sed exists...
if [ -f /bin/sed ]; then
    readonly SED="/bin/sed";
else
    ${ECHO} "/bin/sed is not available on this machine, aborting the script...";
    exit 1;
fi

# make sure that /bin/sleep exists...
if [ -f /bin/sleep ]; then
    readonly SLEEP="/bin/sleep";
else
    ${ECHO} "/bin/sleep is not available on this machine, aborting the script...";
    exit 1;
fi

# make sure that /bin/sync exists...
if [ -f /bin/sync ]; then
    readonly SYNC="/bin/sync";
else
    ${ECHO} "/bin/sync is not available on this machine, aborting the script...";
    exit 1;
fi

# make sure that /bin/tar exists...
if [ -f /bin/tar ]; then
    readonly TAR="/bin/tar";
else
    ${ECHO} "/bin/tar is not available on this machine, aborting the script...";
    exit 1;
fi

# make sure that /bin/touch exists...
if [ -f /bin/touch ]; then
    readonly TOUCH="/bin/touch";
else
    ${ECHO} "/bin/touch is not available on this machine, aborting the script...";
    exit 1;
fi

# make sure that /bin/hostname exists...
if [ -f /bin/hostname ]; then
    readonly SA_HOSTNAME="/bin/hostname";
else
    ${ECHO} "/bin/hostname is not available on this machine, aborting the script...";
    exit 1;
fi

# make sure that /sbin/ifconfig exists...
if [ -f /sbin/ifconfig ]; then
    readonly IFCONFIG="/sbin/ifconfig";
else
    ${ECHO} "/sbin/ifconfig is not available on this machine, aborting the script...";
    exit 1;
fi

# make sure that /sbin/ip exists...
if [ -f /sbin/ip ]; then
    readonly IP="/sbin/ip";
else
    ${ECHO} "/sbin/ip is not available on this machine, aborting the script...";
    exit 1;
fi

# make sure that /usr/bin/free exists...
if [ -f /usr/bin/free ]; then
    readonly FREE="/usr/bin/free";
else
    ${ECHO} "/usr/bin/free is not available on this machine, aborting the script...";
    exit 1;
fi

# make sure that /usr/bin/sqlite3 exists...
if [ -f /usr/bin/sqlite3 ]; then
    readonly SQLITE="/usr/bin/sqlite3";
else
    ${ECHO} "/usr/bin/sqlite3 is not available on this machine, aborting the script...";
    exit 1;
fi

# make sure that /usr/bin/printf exists...
if [ -f /usr/bin/printf ]; then
    readonly PRINTF="/usr/bin/printf";
else
    ${ECHO} "/usr/bin/printf is not available on this machine, aborting the script...";
    exit 1;
fi

# make sure that /usr/bin/tee exists...
if [ -f /usr/bin/tee ]; then
    readonly TEE="/usr/bin/tee";
else
    ${ECHO} "/usr/bin/tee is not available on this machine, aborting the script...";
    exit 1;
fi

# make sure that /usr/bin/top exists...
if [ -f /usr/bin/top ]; then
    readonly TOP="/usr/bin/top";
else
    ${ECHO} "/usr/bin/top is not available on this machine, aborting the script...";
    exit 1;
fi

# make sure that /usr/bin/wc exists...
if [ -f /usr/bin/wc ]; then
    readonly WC="/usr/bin/wc";
else
    ${ECHO} "/usr/bin/wc is not available on this machine, aborting the script...";
    exit 1;
fi

# make sure that /usr/bin/whoami exists...
if [ -f /usr/bin/whoami ]; then
    readonly WHOAMI="/usr/bin/whoami";
else
    ${ECHO} "/usr/bin/whoami is not available on this machine, aborting the script...";
    exit 1;
fi

# make sure that /opt/idb/bin/CliProto exists...
if [ -f /opt/idb/bin/CliProto ]; then
    readonly CLIPROTO="/opt/idb/bin/CliProto";
else
    ${ECHO} "/opt/idb/bin/CliProto is not available on this machine, aborting the script...";
    exit 1;
fi

# make sure that /sbin/chkconfig exists...
if [ -f /sbin/chkconfig ]; then
    readonly CHKCONFIG="/sbin/chkconfig";
else
    ${ECHO} "/sbin/chkconfig is not available on this machine, aborting the script...";
    exit 1;
fi

# make sure that /bin/cut exists...
if [ -f /bin/cut ]; then
    readonly CUT="/bin/cut";
else
    ${ECHO} "/bin/cut is not available on this machine, aborting the script...";
    exit 1;
fi

# * *************************************************************** *
# *  DEFINE FUNCTIONS                                               *
# * *************************************************************** *
function abort (){
    errorLogWriter "Aborting the script...";
    exit 1;
};

function getScaleArcVersion (){
    logWriter "Collecting ScaleArc software version"
    SA_MAJOR_VER="$(${CAT} /etc/issue | ${AWK} -F '-' '{print $2;}' | ${AWK} -F '.' '{print $1;}')"
    SA_MINOR_VER="$(${CAT} /etc/issue | ${AWK} -F '-' '{print $2;}' | ${AWK} -F '.' '{print $2;}')"
    logWriter "Major version: ${SA_MAJOR_VER}, Minor version: ${SA_MINOR_VER}"
    displayWhiteText "Major version: ${SA_MAJOR_VER}, Minor version: ${SA_MINOR_VER}"
};

function collectCliProtoCalls (){
    if [ ! $# -eq 2 ]; then
        ${PRINTF} "${C_RED}%s${C_RESET}\n" "[INTERNAL ERROR] Incorrect usage of collectCliProtoCalls function"
        abort;
    fi

    _cid="$1"
    _lb_sock_file="$2"

    logWriter "The script is going to collect CliProto calls against the cluster ${_cid}"

    writeTitle "call fd_status ${_cid}";
    debugLogWriter "CliProto -s ${_lb_sock_file} call fd_status"
    ${CLIPROTO} -s ${_lb_sock_file} call fd_status                  >> ${DIRECTORY}/${LOGFILE} 

    writeTitle "get_server_info ${_cid}";
    debugLogWriter "CliProto -s ${_lb_sock_file} get_server_info"
    ${CLIPROTO} -s ${_lb_sock_file} get_server_info ${_cid}         >> ${DIRECTORY}/${LOGFILE} 

    writeTitle "get_thread_instances ${_cid}";
    debugLogWriter "CliProto -s ${_lb_sock_file} get_thread_instances ${_cid} 0"
    ${CLIPROTO} -s ${_lb_sock_file} get_thread_instances ${_cid} 0  >> ${DIRECTORY}/${LOGFILE} 

    writeTitle "show_stat_status ${_cid}";
    debugLogWriter "CliProto -s ${_lb_sock_file} show_stat_status"
    ${CLIPROTO} -s ${_lb_sock_file} show_stat_status                >> ${DIRECTORY}/${LOGFILE} 

    writeTitle "get_perf_stats ${_cid}";
    debugLogWriter "CliProto -s ${_lb_sock_file} get_perf_stats"
    ${CLIPROTO} -s ${_lb_sock_file} get_perf_stats                  >> ${DIRECTORY}/${LOGFILE} 

    writeTitle "get_thread_queue_info ${_cid}";
    debugLogWriter "CliProto -s ${_lb_sock_file} get_thread_queue_info ${_cid}"
    ${CLIPROTO} -s ${_lb_sock_file} get_thread_queue_info ${_cid}   >> ${DIRECTORY}/${LOGFILE} 

    logWriter "The script successfully completed collecting all CliProto calls against the cluster ${_cid}."
};

function collectGCore (){
    if [ ! $# -eq 2 ]; then
        ${PRINTF} "${C_RED}%s${C_RESET}\n" "[INTERNAL ERROR] Incorrect usage of collectGCore function"
        abort;
    fi

    _pid="$1"
    _cmd="$2"

    writeTitle "PSTACK for ${_cmd}";
    logWriter "The script is going to collect pstack for ${_cmd}";
    pstack ${_pid} 1>>${DIRECTORY}/${LOGFILE} 2>>${DIRECTORY}/${ERRORLOGFILE};
    logWriter "PSTACK collections for ${_cmd} has been completed successfully.";

    writeTitle "gCore for ${_cmd}";
    logWriter "The script is going to collect gCore for ${_cmd}.";
    logWriter "This might take up to 3-5min."
    gcore ${_pid} 1>>${DIRECTORY}/${LOGFILE} 2>>${DIRECTORY}/${ERRORLOGFILE};

    # The script is going to compress the gCore file
    if [ -f ${DIRECTORY}/core.${_pid} ]; then
        logWriter "The script successfully collected gCore for ${_pid}"
        logWriter "Now, the script is going to tar the collected gCore"
        if [ "bzip" == "off" ]; then
            debugLogWriter "${TAR} cvfz ${DIRECTORY}/core.${_pid}.tz ${DIRECTORY}/core.${_pid}"
            ${TAR} cvfz ${DIRECTORY}/core.${_pid}.tz ${DIRECTORY}/core.${_pid} 1>>${DIRECTORY}/${LOGFILE} 2>>${DIRECTORY}/${ERRORLOGFILE};
        elif [ "bzip" == "on" ]; then
            debugLogWriter "${TAR} cvfj ${DIRECTORY}/core.${_pid}.bz2 ${DIRECTORY}/core.${_pid}"
            ${TAR} cvfj ${DIRECTORY}/core.${_pid}.bz2 ${DIRECTORY}/core.${_pid} 1>>${DIRECTORY}/${LOGFILE} 2>>${DIRECTORY}/${ERRORLOGFILE};
        fi
        # The script is going to remove the original gCore file
        if [[ -f ${DIRECTORY}/core.${_pid}.tz || -f ${DIRECTORY}/core.${_pid}.bz2 ]]; then
            logWriter "The script successfully compress the gCore for ${_pid}"
            logWriter "Now, the script is going to remove the original file..."
            debugLogWriter "# rm ${DIRECTORY}/core.${_pid}"
            ${RM} -rf ${DIRECTORY}/core.${_pid}

            # The script checks if such files were successfully removed
            if [ ! -f ${DIRECTORY}/core.${_pid} ]; then
                logWriter "The script successfully removed the original file. Continue."
            else
                errorLogWriter "The script failed to remove the original file, namely ${DIRECTORY}/core.${_pid}"
                errorLogWriter "Please manually remove such files after this script completes."
            fi
        else
            errorLogWriter "The script fails to compress gCore file, namely ${DIRECTORY}/core.${_pid}.tz"
            errorLogWriter "Please be aware of that this tar would includes uncompressed gCore files."
            logWriter "The script is skipping compressing a gCore for ${_pid}..."
        fi
    else
        errorLogWriter "The script failed to collect gCore for ${_pid}"
        errorLogWriter "Please be aware of that gCore for ${_pid} is not included in the tar file."
        logWriter "The script is skipping collecting a gCore for ${_pid}..."
    fi

    logWriter "gCore collection for ${_cmd} has been completed successfully."
};

function collectMemoryStats (){
    if [ ! $# -eq 2 ]; then
        ${PRINTF} "${C_RED}%s${C_RESET}\n" "[INTERNAL ERROR] Incorrect usage of collectMemoryStats function"
        abort;
    fi

    _cid="$1"
    _lb_sock_file="$2"

    logWriter "The script is going to enable memory dump for CID:${_cid}"
    logWriter "The script is going to collect memory dump 1 minute apart for 3 times"
    debugLogWriter "${CLIPROTO} -s ${_lb_sock_file} call mem_stat_on"
    ${CLIPROTO} -s ${_lb_sock_file} call mem_stat_on 1>>${DIRECTORY}/${LOGFILE} 2>>${DIRECTORY}/${ERRORLOGFILE}

    logWriter "Collecting memory dump for 1st time"
    debugLogWriter "${CLIPROTO} -s ${_lb_sock_file} call mem_stat_dump"
    ${DATE} >> ${DIRECTORY}/mem_stat_dump_cid${_cid}_1st.log
    ${CLIPROTO} -s ${_lb_sock_file} call mem_stat_dump >> ${DIRECTORY}/mem_stat_dump_cid${_cid}_1st.log
    logWriter "The script will sleep for 1 min..."
    ${SLEEP} 60
    logWriter "Collecting memory dump for 2nd time"
    debugLogWriter "${CLIPROTO} -s ${_lb_sock_file} call mem_stat_dump"
    ${DATE} >> ${DIRECTORY}/mem_stat_dump_cid${_cid}_2nd.log
    ${CLIPROTO} -s ${_lb_sock_file} call mem_stat_dump >> ${DIRECTORY}/mem_stat_dump_cid${_cid}_2nd.log
    logWriter "The script will sleep for 1 min..."
    ${SLEEP} 60

    logWriter "The script is going to disable memory dump for CID:${_cid}"
    debugLogWriter "${CLIPROTO} -s ${_lb_sock_file} call mem_stat_off"
    ${DATE} >> ${DIRECTORY}/mem_stat_dump_cid${_cid}_end.log
    ${CLIPROTO} -s ${_lb_sock_file} call mem_stat_off >> ${DIRECTORY}/mem_stat_dump_cid${_cid}_end.log 
};

function errorLogWriter () {
    local _message="$1";
    local _timestamp="$(getTimestamps)";

    if [ ! $# -eq 1 ]; then
        ${PRINTF} "${C_RED}%s${C_RESET}\n" "[INTERNAL ERROR] Incorrect usage of errorLogWriter function"
        abort;
    fi

    displayText   "${_timestamp}" "ERROR" "${_message}"
    writeLog      "${_timestamp}" "ERROR" "${_message}"
    writeErrorLog "${_timestamp}" "ERROR" "${_message}"
};

function debugLogWriter (){
    local _message="$1";
    local _timestamp="$(getTimestamps)";

    if [ ! $# -eq 1 ]; then
        ${PRINTF} "${C_RED}%s${C_RESET}\n" "[INTERNAL ERROR] Incorrect usage of debugLogWriter function"
        abort;
    fi

    # The script is going to write debug log on standard output if debug log flag is ON
    if [ "${debug}" == "on" ]; then
        displayText   "${_timestamp}" "DEBUG" "${_message}"
    fi
    writeLog      "${_timestamp}" "DEBUG" "${_message}"
};

function displayText () {
    local _timestamp="$1"
    local _type="$2"
    local _message="$3"
    local _color="${C_RESET}"
    local _hostname="$(${SA_HOSTNAME})"

    if [ ! $# -eq 3 ]; then
        ${PRINTF} "${C_RED}%s${C_RESET}\n" "[INTERNAL ERROR] Incorrect usage of display function"
        abort;
    fi

    if [ "${_type}" == "_LOG_" ]; then
        displayWhiteText "${_timestamp}::${_hostname}::${_type}::${_message}"
    elif [ "${_type}" == "ERROR" ]; then
        displayRedText   "${_timestamp}::${_hostname}::${_type}::${_message}"
    elif [ "${_type}" == "DEBUG" ]; then
        displayBlueText  "${_timestamp}::${_hostname}::${_type}::${_message}"
    else
        displayRedText "[INTERNAL ERROR] Incorrect usage of display function"
    fi
};

function displayBlueText () {
    local _message=$1

    if [ ! $# -eq 1 ]; then
        ${PRINTF} "${C_RED}%s${C_RESET}\n" "[INTERNAL ERROR] Incorrect usage of displayBlueText function"
        abort;
    fi

    ${PRINTF} "${C_BLUE}%s${C_RESET}\n" "${_message}"
}

function displayGreenText () {
    local _message=$1

    if [ ! $# -eq 1 ]; then
        ${PRINTF} "${C_RED}%s${C_RESET}\n" "[INTERNAL ERROR] Incorrect usage of displayGreenText function"
        abort;
    fi

    ${PRINTF} "${C_GREEN}%s${C_RESET}\n" "${_message}"
}

function displayRedText () {
    local _message=$1

    if [ ! $# -eq 1 ]; then
        ${PRINTF} "${C_RED}%s${C_RESET}\n" "[INTERNAL ERROR] Incorrect usage of displayRedText function"
        abort;
    fi

    ${PRINTF} "${C_RED}%s${C_RESET}\n" "${_message}"
}

function displayWhiteText () {
    local _message=$1

    if [ ! $# -eq 1 ]; then
        ${PRINTF} "${C_RED}%s${C_RESET}\n" "[INTERNAL ERROR] Incorrect usage of displayWhiteText function"
        abort;
    fi

    ${PRINTF} "${C_WHITE}%s${C_RESET}\n" "${_message}"
}

function getTimestamp ()
{
    local _timestamp=$(${DATE} '+%Y%m%d%H%M%S');
    echo "${_timestamp}";
};

function getTimestamps ()
{
    local _timestamp="$(${DATE} '+%Y/%m/%d %H:%M:%S')";
    echo "${_timestamp}";
};

function logWriter (){
    local _message="$1";
    local _timestamp="$(getTimestamps)";

    if [ ! $# -eq 1 ]; then
        ${PRINTF} "${C_RED}%s${C_RESET}\n" "[INTERNAL ERROR] Incorrect usage of logWriter function"
        abort;
    fi

    displayText "${_timestamp}" "_LOG_" "${_message}"
    writeLog    "${_timestamp}" "_LOG_" "${_message}"
};

function success() {
    logWriter "The script successfully completes."
    logWriter "The end of script..."
    exit 0
};

function usage() {
    ${CAT} <<__EOS
Version:
    v${VERSION}
Usage:
    idb_collector.sh [ -x=CID | --cluster=CID ] [ -c | --core ] [ -d=YYYYMMDD(HH) | --date=YYYYMMDD(HH) ]
    [ -h=DAY | --history=DAY ] [ -m | --memory ] [ -v | --verbose ] [ -j | --bzip2 ] [ -h | --help ]
    [ -q | --quiet ] [ --dry-run ]

Description:
    n/a No option is specified (default behavior).
        idb_collector only collects standard logs for further investigation.
        This operation will not affect traffic on an active ScaleArc except when core file collection is specified.

        Standard logs includes...
        - OS logs (syslog, heartbeat-debug.log, rsync.log, ha.cf, etc...)
        - ScaleArc configuration (/system/*, a copy of license file, external API script, etc...)
        - ScaleArc logs (/currentlogs)

    -x=CID | --cluster=CID
        This option is used to enable cluster specific log collection.
        idblog_collector only collects logs for specified cluster.
    -c | --core
        This option is used to enable core collection.
        idb_collector collects GCORE, which WILL AFFECT current traffic,
        In order to specify this option, CID option (-x or --cluster) is required.
    -d=YYYYMMDD(HH) | --date=YYYYMMDD(HH)
        This option is used to set the date for log collection execution.
        Hour (HH) is optional. Multiple occurrences are permitted.

        Additional logs include...
        - Query Logs
        - Alert Logs
        - Generic Logs
        - DB Health Log
        - UI Log

    -h=DAY | --history=DAY
        This option is used to collect additional historical stats database files for clusters
        stretching back <DAY> days from today. [Default = 1]
    -m | --memory
        This option is used to collect memory leak information.
        The script collects a memory dump 3 times each 1 minute apart for each cluster.
    -v | --verbose
        This option is used to enable verbose mode.
        idb_collector displays debug outputs along with standard outputs.
    -j | --bzip2
        This option indicates the desire to use the bzip2 compression filter rather than the default gzip.
    -q | --quiet
        This option suppresses all STDOUT messages. When called from another script / process,
        this option should be used.
    --dry-run
        This option will complete the normal execution of the script but skip the actual collection of the logs.
    -h | --help
        Display this usage
__EOS

  exit 0;
};

function waitUser () {
    logWriter "Do you like to proceed? ([y]/n) [Default will be selected after 5 seconds] "
    while :
    do
        read -t 5 -n 1 _res
        if [ "${_res}" = "Y" -o "${_res}" = "y" -o "${_res}" = "" ]; then
            logWriter "Proceeding...";
            break
        elif [ "${_res}" = "N" -o "${_res}" = "n" ]; then
            errorLogWriter "You selected NOT to proceed.";
            abort
        fi
        logWriter "Invalid argument, choose Y/y or N/n."
        logWriter "Do you like to proceed?(Y/n)"
    done;
};

function writeLog () {
    local _timestamp="$1"
    local _type="$2"
    local _message="$3"
    local _color="${C_RESET}"
    local _hostname="$(${SA_HOSTNAME})"

    if [ ! $# -eq 3 ]; then
        ${PRINTF} "${C_RED}%s${C_RESET}\n" "[INTERNAL ERROR] Incorrect usage of writeLog function"
        abort;
    fi

    if [ "${_type}" == "_LOG_" ]; then
        _color="${C_WHITE}"
    elif [ "${_type}" == "ERROR" ]; then
        _color="${C_RED}"
    elif [ "${_type}" == "DEBUG" ]; then
        _color="${C_BLUE}"
    else
        ${PRINTF} "${C_RED}%s${C_RESET}\n" "[INTERNAL ERROR] Incorrect usage of writeLog function"
        _color="${C_WHITE}"
    fi

    # The script logs the message
    if [ -f ${DIRECTORY}/${LOGFILE} ]; then
        ${PRINTF} "${_color}%s::%s::%s::%s${C_RESET}\n" "${_timestamp}" "${_hostname}" "${_type}" "${_message}" >> ${DIRECTORY}/${LOGFILE}
    else
        displayRedText "MESSAGE::${_message}"
        displayRedText "This message failed to be written in ${DIRECTORY}/${LOGFILE}."
    fi

}

function writeErrorLog () {
    local _timestamp="$1"
    local _type="$2"
    local _message="$3"
    local _color="${C_RESET}"
    local _hostname="$(${SA_HOSTNAME})"

    if [ ! $# -eq 3 ]; then
        ${PRINTF} "${C_RED}%s${C_RESET}\n" "[INTERNAL ERROR] Incorrect usage of writeErrorLog function"
        abort;
    fi

    if [ "${_type}" == "LOG" ]; then
        _color="${C_WHITE}"
    elif [ "${_type}" == "ERROR" ]; then
        _color="${C_RED}"
    elif [ "${_type}" == "DEBUG" ]; then
        _color="${C_BLUE}"
    else
        ${PRINTF} "${C_RED}%s${C_RESET}\n" "[INTERNAL ERROR] Incorrect usage of writeErrorLog function"
        _color="${C_WHITE}"
    fi

    # The script logs the message
    if [ -f ${DIRECTORY}/${ERRORLOGFILE} ]; then
        ${PRINTF} "${_color}%s::%s::%-5s::%s${C_RESET}\n" "${_timestamp}" "${_hostname}" "${_type}" "${_message}" >> ${DIRECTORY}/${ERRORLOGFILE}
    else
        displayRedText "This message failed to be written in ${DIRECTORY}/${LOGFILE}."
    fi
};

function writeTitle () {
    if [ ! $# -eq 1 ]; then
        ${ECHO} "[INTERNAL ERROR] Incorrect usage of writeTitle function";
        abort;
    fi
    local _message="$1"
    ${PRINTF} "# * ***************************************************** *\n"             >> ${DIRECTORY}/${LOGFILE} 
    ${PRINTF} "# *  %-52s *\n"                                              "${_message}" >> ${DIRECTORY}/${LOGFILE} 
    ${PRINTF} "# * ***************************************************** *\n"             >> ${DIRECTORY}/${LOGFILE} 
};

if [[ "$1" == "-h" || "$1" == "--help" ]]; then
    usage
fi

# * *************************************************************** *
# *  GLOBAL VARIABLE                                                *
# * *************************************************************** *
# Verbose mode flag is OFF by default
debug="off"

# cluster collection flag is OFF by default
clusters="off"
cluster_id=0

# gCore collection flag is OFF by default
gcore="off"

# Query Log collection mode flag is OFF by default
logs="off"
userdate=()

# Memory leak investigation flag is OFF by default
memory="off"

# The script is going to collect 2 day-worth historical stats data by default
hstats="off"
hdays=1

# Use BZIP2 compression instead of GZIP
bzip="off"

# * *************************************************************** *
# *  VARIABLE DECLARATION                                           *
# * *************************************************************** *
displayWhiteText "The script is initializing...."

# Declare the timestamp
readonly TIMESTAMP=$(getTimestamp);
displayBlueText "SET TIMESTAMP=${TIMESTAMP}";

# Declare the directory that the script generates logs
if [ -d /data/logs ]; then
    readonly DIRECTORY=/data/logs/log_${TIMESTAMP}
    displayBlueText "SET DIRECTORY=${DIRECTORY}";
else
    displayRedText "/data/logs does not exist";
    abort;
fi

# Create a log directory
if [ ! -d ${DIRECTORY} ]; then
    displayBlueText "# mkdir ${DIRECTORY}";
    ${MKDIR} ${DIRECTORY};
else
    displayRedText "Directory ${DIRECTORY} already exists";
    displayRedText "Skipping creating a ${DIRECTORY}";
fi

# Declare the logfile
readonly LOGFILE=idblog_collector.log
displayBlueText "SET LOGFILE=${DIRECTORY}/${LOGFILE}";

# Create a logfile
displayBlueText "# touch ${DIRECTORY}/${LOGFILE}"
${TOUCH} ${DIRECTORY}/${LOGFILE};
if [ ! -f ${DIRECTORY}/${LOGFILE} ]; then
    displayRedText "Failed to create a logfile ${LOGFILE}";
    abort;
fi

# Declare the error logfile
readonly ERRORLOGFILE=idblog_collector.err
displayBlueText "SET ERRORLOGFILE=${DIRECTORY}/${ERRORLOGFILE}";

# Create an error log file
displayBlueText "# touch ${DIRECTORY}/${ERRORLOGFILE}"
${TOUCH} ${DIRECTORY}/${ERRORLOGFILE};
if [ ! -f ${DIRECTORY}/${ERRORLOGFILE} ]; then
    displayRedText "Failed to create an error logfile ${ERRORLOGFILE}";
    abort;
fi

# Get ScaleArc software version
getScaleArcVersion

displayGreenText "The script initialization successfully completed."
# * *************************************************************** *
# *  SCRIPT HEADER                                                  *
# * *************************************************************** *
logWriter "The idblog_collector.sh has been initiated"
logWriter "Version: ${VERSION}"
logWriter "All standard outputs are written into ${DIRECTORY}/${LOGFILE}..."
logWriter "Any errors observed during the execution are logged into ${DIRECTORY}/${ERRORLOGFILE}..."

# * *************************************************************** *
# *  PRE-CHECK                                                      *
# * *************************************************************** *
# Abort the script if the executor is not root user
if [ ! "$(${WHOAMI})" == "root" ]; then
    errorLogWriter "Please execute this script as root user";
    abort;
else
    debugLogWriter "Root user check is successful";
fi

# * *************************************************************** *
# *  Collect Arguments                                              *
# * *************************************************************** *

logWriter "The script is going to evaluate provided argument(s)..."
for i in "$@"
do
  case ${i} in
    -x=*|--cid=*|--cluster=*)
        clusters="on"
        logWriter "Cluster collection mode is enabled"
        cluster_id="${i#*=}"
        shift
        ;;
    -c|--core)
        gcore="on";
        logWriter "Core collection mode is enabled"
        shift
        ;;
    -d=*|--date=*)
        logs="on";
        userdate+=("${i#*=}")
        shift
        ;;
    -h=*|--history=*)
        hstats="on"
        hdays="${i#*=}"
        shift
        ;;
    -m|--memory)
        memory="on"
        logWriter "Memory leak investigation is enabled"
        shift
        ;;
    -v|--verbose)
        debug="on"
        logWriter "Verbose mode is enabled"
        shift
        ;;
    -j|--bzip2)
        bzip="on"
        logWriter "BZIP2 compression selected"
        shift
        ;;
    --dry-run)
	    dry="on"
	    logWriter "Dry Run enabled. No files will be collected"
	    shift
	    ;;
	-q|--quiet)
	    logWriter "Quiet execution enabled. All stdout has been redirected to /dev/null"
	    exec > /dev/null
	    shift
	    ;;
    -h|--help)
        usage
        exit 0
        ;;
    * )
        errorLogWriter "$i : Invalid Argument(s)"
        usage
        exit 1
        ;;
  esac
done
logWriter "The script completed evaluating arguments successfully."

# Disable cluster collection mode if specified CID is wrong
if [ "${clusters}" == "on" ]; then
    if ! [[ "${cluster_id}" =~ ^[0-9]+$ ]]; then
        errorLogWriter "Non-numeric characters were specified as date, ${cluster_id}."
        errorLogWriter "The script is going to disable cluster collection mode"
        debugLogWriter "RESET clusters=off"
        clusters="off"
        debugLogWriter "RESET cluster_id=0"
        cluster_id=0
    elif [ -z "$(${LS} /tmp/lb_sock_${cluster_id})" ]; then
        ls -al /tmp/lb_sock_${cluster_id}
        errorLogWriter "The specified CID does not exist, /tmp/lb_sock_${cluster_id}."
        errorLogWriter "The script is going to disable cluster collection mode"
        debugLogWriter "RESET clusters=off"
        clusters="off"
        debugLogWriter "RESET cluster_id=0"
        cluster_id=0
    else
        if [ "${cluster_id}" -gt 0 ]; then
            logWriter "User chosen to collect only logs from CID=${cluster_id}"
        else
            errorLogWriter "Please specify cluster ID greater than 0 for -x|--cluster option."
            errorLogWriter "The script is going to disable cluster collection mode"
            debugLogWriter "RESET clusters=off"
            clusters="off"
            debugLogWriter "RESET cluster_id=0"
            cluster_id=0
        fi
    fi
fi

# Reset HDAYS=1 if specified date was wrong format
if [ "${hstats}" == "on" ]; then
    if ! [[ "${hdays}" =~ ^[0-9]+$ ]]; then
        errorLogWriter "Non-numeric characters were specified as date, ${hdays}."
        errorLogWriter "The script is going to ignore this value."
        debugLogWriter "RESET hdays=1"
        hdays=1
    else
        if [ "${hdays}" -gt 1 ]; then
            logWriter "Historical Stats collection is extended for ${hdays} days"
        else
            errorLogWriter "Please specify more than 1 day for historical log collection"
            debugLogWriter "RESET hdays=1"
            hdays=1
        fi
    fi
fi

# Skip query log collection if specified date was wrong format
if [ "${logs}" == "on" ]; then
    logWriter "The script is going to evaluate the provided date value(s)"
    for d in "${userdate[@]}"
    do

        debugLogWriter "Provided Date -> ${d}";

        # Abort the script if the provided timestamp contains non-numeric character(s)
        if ! [[ "${d}" =~ ^[0-9]+$ ]]; then
            errorLogWriter "The provided timestamp ${d} contains non-numeric character(s)"

            # Return the failure
            _rst=0

            # Disable query log collection
            debugLogWriter "SET logs=off"
            logs="off"
        fi

        # Abort the script if the target date directory does not exist under /logs
        _dat=$(${ECHO} "${d:0:8}")
        debugLogWriter "# ${LS} -d \"/data/logs/${_dat}\""
        if [ ! -d $(${LS} -d "/data/logs/${_dat}") ]; then
            errorLogWriter "ScaleArc does not contain any directory named /data/logs/${_dat}";
            abort
        fi

        # Abort the script if the target directory does not contain any query logs
        debugLogWriter "# ${FIND} /data/logs/${_dat} -type f -name idb.log*${d}* | ${WC} -l"
        if [ $(${FIND} /data/logs/${_dat} -type f -name idb.log*${d}* | ${WC} -l) -eq 0 ]; then
            errorLogWriter "ScaleArc does not contain any query logs under /data/logs/${_dat} directory"

            waitUser
        fi

        # Abort the script if the target directory does not contain any alert logs
        debugLogWriter "# ${FIND} /data/logs/${_dat} -type f -name idb.alert*${d}* | ${WC} -l"
        if [ $(${FIND} /data/logs/${_dat} -type f -name idb.alert*${d}* | ${WC} -l) -eq 0 ]; then
            errorLogWriter "ScaleArc does not contain any alert logs under /data/logs/${_dat} directory"

            waitUser
        fi

        # Abort the script if the target directory does not contain any health logs
        debugLogWriter "# ${FIND} /data/logs/${_dat} -type f -name idb.health*${d}* | ${WC} -l"
        if [ $(${FIND} /data/logs/${_dat} -type f -name idb.health*${d}* | ${WC} -l) -eq 0 ]; then
            errorLogWriter "ScaleArc does not contain any new-type DB health logs under /data/logs/${_dat} directory"

            waitUser
        fi

        # Abort the script if the target directory does not contain any dbhealth logs
        debugLogWriter "# ${FIND} /data/logs/${_dat} -type f -name idb.dbhealth*${d}* | ${WC} -l"
        if [ $(${FIND} /data/logs/${_dat} -type f -name idb.dbhealth*${d}* | ${WC} -l) -eq 0 ]; then
            errorLogWriter "ScaleArc does not contain any old-type DB health logs under /data/logs/${_dat} directory"

            waitUser
        fi

        # Abort the script if the target directory does not contain any genlog logs
        debugLogWriter "# ${FIND} /data/logs/${_dat} -type f -name idb.genlog*${d}* | ${WC} -l"
        if [ $(${FIND} /data/logs/${_dat} -type f -name idb.genlog*${d}* | ${WC} -l) -eq 0 ]; then
            errorLogWriter "ScaleArc does not contain any DB genlog logs under /data/logs/${_dat} directory"

            waitUser
        fi

        # Abort the script if the target directory does not contain any failover logs
        debugLogWriter "# ${FIND} /data/logs/${_dat} -type f -name failover*${d}* | ${WC} -l"
        if [ $(${FIND} /data/logs/${_dat} -type f -name failover*${d}* | ${WC} -l) -eq 0 ]; then
            errorLogWriter "ScaleArc does not contain any DB failover logs under /data/logs/${_dat} directory"

            waitUser
        fi

        # Display the message if the provided date was correct format
        if [ "${logs}" == "on" ]; then
            logWriter "The script is going to collect query logs generated on ${d}"
        fi
    done
    logWriter "The script successfully completes evaluating the provided date."
else
    # The script now requests user input to acknowledge that the script is not going to collect query logs for over past 2HR if --date option is not specified
    logWriter "You DID NOT specify --date option."
    logWriter "This script will collect /logs/currentlogs "

    waitUser
fi

# Abort the script if available disk space is lowered than requirement (2G)
if [ $(${DF} /logs | ${GREP} / | ${AWK} '{print $4}') -lt 2097152 ]; then
    errorLogWriter "/logs does not contain enough space (2G) to execute this script";
    abort;
fi

# Display a warning message and abort the script if the user won't agrees all
# conditions
logWriter "This script is going to collect necessary information for support investigation.";
if [ "${gcore}" == "on" -a "${clusters}" == "on" ]; then
    logWriter ""
    logWriter "You specified gCore option (-c)."
    logWriter ""
    logWriter "The script is going to collect gCore for all idblb processes running on this system.";
    logWriter "Please be aware of that collecting gCore *WILL AFFECT* all queries running through the target cluster.";
    logWriter "Your cluster will be put under pause state and the cluster won't process any queries during the time (up to 3~5min).";
elif [ "${gcore}" == "on" -a "${clusters}" == "off" ]; then
    errorLogWriter "You must specify CID (-x or --cluster) to speficy gCore option (-c or --core)."
    abort
else
    logWriter "Please be aware of that this script does not have any influence on traffic.";
fi

# Request user input for proceeding the script
waitUser

# * *************************************************************** *
# *  OS INFORMATION                                                 *
# * *************************************************************** *
logWriter "The script is going to collect information related to OS"
writeTitle "OS INFO"
debugLogWriter "# hostname"
${SA_HOSTNAME} >> ${DIRECTORY}/${LOGFILE}
debugLogWriter "# uname -a"
${UNAME} -a >> ${DIRECTORY}/${LOGFILE}
debugLogWriter "# cat /boot/grub/grub.conf"
${CAT} /boot/grub/grub.conf >> ${DIRECTORY}/${LOGFILE}

writeTitle "Resource Info: CPU"
debugLogWriter "# cat /proc/cpuinfo"
${CAT} /proc/cpuinfo >> ${DIRECTORY}/${LOGFILE}

writeTitle "Resource Info: USER LIMIT"
debugLogWriter "# ulimit -a"
ulimit -a >> ${DIRECTORY}/${LOGFILE}

writeTitle "Resource Info: MEMORY"
debugLogWriter "# free -k"
${FREE} -k >> ${DIRECTORY}/${LOGFILE}
debugLogWriter "# cat /proc/meminfo"
${CAT} /proc/meminfo    >> ${DIRECTORY}/${LOGFILE}

writeTitle "Resource Info: df"
debugLogWriter "# df -h"
${DF} -h >> ${DIRECTORY}/${LOGFILE}

writeTitle "Resource Info: rpm"
debugLogWriter "# rpm -qa"
${RPM} -qa >> ${DIRECTORY}/${LOGFILE}

writeTitle "Resource Info: SQLite3 Version"
debugLogWriter "# sqlite3 --version"
${SQLITE} --version >> ${DIRECTORY}/${LOGFILE}

writeTitle "PS AUX";
debugLogWriter "# ps aux";
${PS} aux >> ${DIRECTORY}/${LOGFILE}

writeTitle "TOP"
debugLogWriter "# cat ~/.toprc"
${CAT} > ~/.toprc <<EOF
RCfile for "top with windows"        # shameless braggin'
Id:a, Mode_altscr=0, Mode_irixps=1, Delay_time=3.000, Curwin=0
Def    fieldscur=AEHIOQTWKNMbcdfgJplrsuvyzX
    winflags=62777, sortindx=10, maxtasks=0
    summclr=1, msgsclr=1, headclr=3, taskclr=1
Job    fieldscur=ABcefgjlrstuvyzMKNHIWOPQDX
    winflags=62777, sortindx=0, maxtasks=0
    summclr=6, msgsclr=6, headclr=7, taskclr=6
Mem    fieldscur=ANOPQRSTUVbcdefgjlmyzWHIKX
    winflags=62777, sortindx=13, maxtasks=0
    summclr=5, msgsclr=5, headclr=4, taskclr=5
Usr    fieldscur=ABDECGfhijlopqrstuvyzMKNWX
    winflags=62777, sortindx=4, maxtasks=0
    summclr=3, msgsclr=3, headclr=2, taskclr=3
EOF
debugLogWriter "# top -Hb -n1 -c";
${TOP} -Hb -n1 -c >> ${DIRECTORY}/${LOGFILE}
debugLogWriter "# rm ~/.toprc"
${RM} -f ~/.toprc

writeTitle "CHKCONFIG";
debugLogWriter "# chkconfig";
${CHKCONFIG} >> ${DIRECTORY}/${LOGFILE}

writeTitle "NETSTAT";
debugLogWriter "# netstat -lanpt";
${NETSTAT} -lanpt >> ${DIRECTORY}/${LOGFILE}
debugLogWriter "# netstat -s";
${NETSTAT} -s >> ${DIRECTORY}/${LOGFILE}

writeTitle "NETWORK INFO";
debugLogWriter "# ifconfig";
${IFCONFIG} >> ${DIRECTORY}/${LOGFILE}
debugLogWriter "# ip addr";
${IP} addr >> ${DIRECTORY}/${LOGFILE}
debugLogWriter "# cat /etc/hosts"
${CAT} /etc/hosts >> ${DIRECTORY}/${LOGFILE}
debugLogWriter "# cat /etc/ntp_scalearc.conf"
${CAT} /etc/ntp_scalearc.conf >> ${DIRECTORY}/${LOGFILE}
debugLogWriter "# cat /etc/ntp.conf"
${CAT} /etc/ntp.conf >> ${DIRECTORY}/${LOGFILE}
debugLogWriter "# cat /etc/sysconfig/network"
${CAT} /etc/sysconfig/network >> ${DIRECTORY}/${LOGFILE}
debugLogWriter "# grep -He \"^\" /etc/sysconfig/network-scripts/ifcfg-*"
${GREP} -He "^" /etc/sysconfig/network-scripts/ifcfg-* >> ${DIRECTORY}/${LOGFILE}
debugLogWriter "# cat /etc/resolv.conf"
${CAT} /etc/resolv.conf >> ${DIRECTORY}/${LOGFILE}
debugLogWriter "# cat /proc/net/softnet_stat"
${CAT} /proc/net/softnet_stat >> ${DIRECTORY}/${LOGFILE}
debugLogWriter "# cat /proc/interrupts"
${CAT} /proc/interrupts >> ${DIRECTORY}/${LOGFILE}

writeTitle "GIT INFO";
debugLogWriter "# grep ^ /opt/idb/git_sha/*.txt"
${GREP} -s ^ /opt/idb/git_sha/*.txt >> ${DIRECTORY}/${LOGFILE}

writeTitle "NETWORK INFO for MSSQL"
if [ -f /etc/freetds.conf ]; then
  ${CAT} /etc/freetds.conf >> ${DIRECTORY}/${LOGFILE}
fi

writeTitle "SCALEARC LOG DIRECTORY INFO"
debugLogWriter "# ls -d /data/logs/*"
${LS} -d /data/logs/* >> ${DIRECTORY}/${LOGFILE}

logWriter "The script successfully completed collecting information related to OS"
# * *************************************************************** *
# *  CliProto CALLS                                                 *
# * *************************************************************** *
logWriter "The script is going to collect CliProto calls..."
for _lb_sock_file in $(${LS} /tmp/lb_sock_*);
do
    _cid=$(${ECHO} "${_lb_sock_file}" | ${SED} -e 's/.*lb_sock_//');

    # Collect all CliProto calls if idblb process associated with lb_sock file exists
    if [ $(${PS} aux | grep ${_lb_sock_file} | grep -v grep | grep -v idbudpsvc | wc -l) -gt 0 ]; then
        if [ "${clusters}" == "off" ]; then
            collectCliProtoCalls "${_cid}" "${_lb_sock_file}"
        elif [ "${clusters}" == "on" -a "${cluster_id}" == "${_cid}" ]; then
            collectCliProtoCalls "${_cid}" "${_lb_sock_file}"
        else
          logWriter "The cluster collection mode is ON."
          logWriter "The script is going to skip collecting CliPtoto calls for CID=${_cid}"
        fi
    else
        # Display an error message if idblb process associated with lb_sock file does not exist
        errorLogWriter "The idblb process associated with ${_lb_sock_file} does not exist"
        errorLogWriter "The script is skipping collecting CliProto calls for CID:${_cid}..."
    fi
done;
logWriter "The script successfully completes collecting CliProto calls against all clusters"

# * *************************************************************** *
# *  Memory Leak Investigation                                      *
# * *************************************************************** *
if [ "${memory}" == "on" ]; then
    logWriter "The script is going to collect memory information"
    for _lb_sock_file in $(${LS} /tmp/lb_sock_*);
    do
        debugLogWriter "SET LB_SOCK_FILE=${_lb_sock_file}"

        _cid=$(${ECHO} "${_lb_sock_file}" | ${SED} -e 's/.*lb_sock_//');
        debugLogWriter "SET CID=${_cid}"

        if [ $(${PS} aux | grep ${_lb_sock_file} | grep -v grep | wc -l) -gt 0 ]; then
            if [ "${clusters}" == "off" ]; then
                collectMemoryStats "${_cid}" "${_lb_sock_file}"
            elif [ "${clusters}" == "on" -a "${cluster_id}" == "${_cid}" ]; then
                collectMemoryStats "${_cid}" "${_lb_sock_file}"
            else
              logWriter "The cluster collection mode is ON."
              logWriter "The script is going to skip collecting memory stats for CID=${_cid}"
            fi
        else
            errorLogWriter "idblb process associated with ${_lb_sock_file} does not exist"
            errorLogWriter "The script is skipping collecting memory dump for CID:${_cid}..."
        fi
    done;

    logWriter "The script is going to flush OS reclaimable memory"
    debugLogWriter "# ${FREE}"
    ${FREE} 1>>${DIRECTORY}/${LOGFILE} 2>>${DIRECTORY}/${ERRORLOGFILE}

    debugLogWriter "# ${SYNC}"
    ${SYNC} 1>>${DIRECTORY}/${LOGFILE} 2>>${DIRECTORY}/${ERRORLOGFILE}

    debugLogWriter "# ${ECHO} 3 > /proc/sys/vm/drop_caches"
    ${ECHO} 3 > /proc/sys/vm/drop_caches

    debugLogWriter "# ${FREE}"
    ${FREE} 1>>${DIRECTORY}/${LOGFILE} 2>>${DIRECTORY}/${ERRORLOGFILE}

    logWriter "The script successfully completes collecting memory information"
fi

# * *************************************************************** *
# *  PROC INFORMATION                                               *
# * *************************************************************** *
writeTitle "PROC INFORMATION"
logWriter "The script is going to collect proc of all clusters...";

for _pid in $(${PS} -eo pid,cmd | ${GREP} [i]dblb | ${AWK} '{print $1}')
do
    debugLogWriter "SET PID=${_pid}";

    _cmd=$(${PS} -eo pid,cmd | ${GREP} ${_pid} | ${GREP} -v grep | ${AWK} '{print $2}');
    debugLogWriter "SET CMD=${_cmd}";

    logWriter "The script is going to collect proc information for ${_cmd} (pid:${_pid})"
    debugLogWriter "# ${CAT} /proc/${_pid}/smaps     >> ${DIRECTORY}/${LOGFILE}"
    ${CAT} /proc/${_pid}/smaps     >> ${DIRECTORY}/${LOGFILE} 
    debugLogWriter "# ${CAT} /proc/${_pid}/stack     >> ${DIRECTORY}/${LOGFILE}"
    ${CAT} /proc/${_pid}/stack     >> ${DIRECTORY}/${LOGFILE} 
    debugLogWriter "# ${CAT} /proc/${_pid}/maps      >> ${DIRECTORY}/${LOGFILE}"
    ${CAT} /proc/${_pid}/maps      >> ${DIRECTORY}/${LOGFILE} 
    debugLogWriter "# ${CAT} /proc/${_pid}/numa_maps >> ${DIRECTORY}/${LOGFILE}" 
    ${CAT} /proc/${_pid}/numa_maps >> ${DIRECTORY}/${LOGFILE} 
done;

logWriter "The script successfully completes collecting all proc for all clusters"
# * *************************************************************** *
# *  gCORE COLLECTION                                               *
# * *************************************************************** *
if [ "${gcore}" == "on" ]; then
    logWriter "The script is going to collect gCore of all clusters..."

    readonly CURRDIR=$(pwd)
    debugLogWriter "SET CURRDIR=${CURRDIR}"

    cd ${DIRECTORY}
    debugLogWriter "# cd ${DIRECTORY}"

    for _pid in $(${PS} -eo pid,cmd | ${GREP} [i]dblb | ${AWK} '{print $1}')
    do
        debugLogWriter "SET PID=${_pid}"
        if [ "${clusters}" == "off" ]; then
            errorLogWriter "Cluster ID (-x | --cluster) must be specified in order to collect gCore"
        elif [ "${clusters}" == "on"  ]; then
            _cmd=$(${PS} -eo pid,cmd | ${GREP} ${_pid} | ${GREP} -v grep | ${AWK} '{print $2}');
            debugLogWriter "SET CMD=${_cmd}"
            if [ "${_cmd}" == "idblb_mysql_${cluster_id}" -o "${_cmd}" == "idblb_mssql_${cluster_id}" -o "${_cmd}" == "idblb_oracle_${cluster_id}" ]; then
                collectGCore "${_pid}" "${_cmd}"
            else
                logWriter "The script is going to skip collecting gCore for ${_cmd}."
                logWriter "The script only collects gCore for CID=${cluster_id}"
            fi
        else
            errorLogWriter "[INTERNAL ERROR] Unexpected value for CLUSTER=${clusters} in gCore collection."
            abort
        fi
    done;

    cd ${CURRDIR};
    debugLogWriter "# cd ${CURRDIR}";
    logWriter "The script successfully completes collecting all gCore for all clusters"
fi

# * *************************************************************** *
# *  FILE LIST CREATION                                             *
# * *************************************************************** *

logWriter "The script is going to generate the list of files to collect"
files=""
file="/etc/hosts"
files+=${file}
${ECHO} "+ ${file}" >> ${DIRECTORY}/${LOGFILE}
file=" /etc/centos-release"
files+=${file}
${ECHO} "+ ${file}" >> ${DIRECTORY}/${LOGFILE}
# Only collect Heartbeat related files if ScaleArc version is 3.10 or lower
if [[ "${SA_MAJOR_VER}" == "3" && "${SA_MINOR_VER}" -lt 11 ]]; then
  file=" /etc/ha*"
  files+=${file}
  ${ECHO} "+ ${file}" >> ${DIRECTORY}/${LOGFILE}
  file=" /var/log/hb*"
  files+=${file}
  ${ECHO} "+ ${file}" >> ${DIRECTORY}/${LOGFILE}
  file=" /var/log/heartbeat*"
  files+=${file}
  ${ECHO} "+ ${file}" >> ${DIRECTORY}/${LOGFILE}
fi
if [[ -d "/etc/cluster/" && "$(${LS} -A /etc/cluster/)" ]]; then
  file=" /etc/cluster/*"
  files+=${file}
  ${ECHO} "+ ${file}" >> ${DIRECTORY}/${LOGFILE}
fi
file=" /etc/localtime"
files+=${file}
${ECHO} "+ ${file}" >> ${DIRECTORY}/${LOGFILE}
file=" /etc/sysconfig/clock"
files+=${file}
${ECHO} "+ ${file}" >> ${DIRECTORY}/${LOGFILE}
file=" /var/log/sa/sa*"
files+=${file}
${ECHO} "+ ${file}" >> ${DIRECTORY}/${LOGFILE}
file=" /var/log/rsyncd*"
files+=${file}
${ECHO} "+ ${file}" >> ${DIRECTORY}/${LOGFILE}
file=" /var/log/cron*"
files+=${file}
${ECHO} "+ ${file}" >> ${DIRECTORY}/${LOGFILE}
file=" /var/log/dmesg*"
files+=${file}
${ECHO} "+ ${file}" >> ${DIRECTORY}/${LOGFILE}
file=" /var/log/messages*"
files+=${file}
${ECHO} "+ ${file}" >> ${DIRECTORY}/${LOGFILE}
file=" /var/log/watchdog*"
files+=${file}
${ECHO} "+ ${file}" >> ${DIRECTORY}/${LOGFILE}
file=" /system/*"
files+=${file}
${ECHO} "+ ${file}" >> ${DIRECTORY}/${LOGFILE}
file=" /opt/idb/license/idb_license.lic"
files+=${file}
${ECHO} "+ ${file}" >> ${DIRECTORY}/${LOGFILE}
file=" /opt/idb/lbgui/gui/app/php/utils/external_scripts/*"
files+=${file}
${ECHO} "+ ${file}" >> ${DIRECTORY}/${LOGFILE}
file=" /opt/idb/bin/idblb*"
files+=${file}
${ECHO} "+ ${file}" >> ${DIRECTORY}/${LOGFILE}
file=" /opt/idb/conf/*"
files+=${file}
${ECHO} "+ ${file}" >> ${DIRECTORY}/${LOGFILE}
file=" /data/logs/ha*"
files+=${file}
${ECHO} "+ ${file}" >> ${DIRECTORY}/${LOGFILE}
file=" /data/logs/*.log"
files+=${file}
${ECHO} "+ ${file}" >> ${DIRECTORY}/${LOGFILE}
file=" /data/logs/*.txt"
files+=${file}
${ECHO} "+ ${file}" >> ${DIRECTORY}/${LOGFILE}
file=" /data/logs/*.sqlite"
files+=${file}
${ECHO} "+ ${file}" >> ${DIRECTORY}/${LOGFILE}
file=" /data/logs/services/*"
files+=${file}
${ECHO} "+ ${file}" >> ${DIRECTORY}/${LOGFILE}
file=" /data/logs/currentlogs/*"
files+=${file}
${ECHO} "+ ${file}" >> ${DIRECTORY}/${LOGFILE}
if [[ -d "/var/log/samba/" && "$(${LS} -A /var/log/samba/)" ]]; then
  file=" /var/log/samba/*"
  files+=${file}
  ${ECHO} "+ ${file}" >> ${DIRECTORY}/${LOGFILE}
fi
# 3.11 HA files
if [[ -d "/var/log/cluster/" && "$(${LS} -A /var/log/cluster/)" ]]; then
  file=" /var/log/cluster/*"
  files+=${file}
  ${ECHO} "+ ${file}" >> ${DIRECTORY}/${LOGFILE}
fi

# Kerberos files, if they exist
if [[ -e "/etc/krb5.conf" ]]; then
  file=" /etc/krb5.conf"
  files+=${file}
  ${ECHO} "+ ${file}" >> ${DIRECTORY}/${LOGFILE}
  # Collect log file defined in /etc/krb5.conf
  krb_log="${GREP} /var/log /etc/krb5.conf | ${CUT} -d: -f2"
  if [[ -e "${krb_log}" ]]; then
    files+=${krb_log}
    ${ECHO} "+ ${krb_log}" >> ${DIRECTORY}/${LOGFILE}
  fi
fi
if [[ -d "/data/logs/kerberos/" && "$(${LS} -A /data/logs/kerberos/)" ]]; then
  file=" /data/logs/kerberos/*"
  files+=${file}
  ${ECHO} "+ ${file}" >> ${DIRECTORY}/${LOGFILE}
fi

# CPE-401 Add traffic analyszer logs
if [[ -d "/data/logs/traffic_analyzer/" ]]; then
  file=" /data/logs/traffic_analyzer/*"
  files+=${file}
  ${ECHO} "+ ${file}" > ${DIRECTORY}/${LOGFILE}
fi

# CPE-420
if [[ -e "/var/log/secure" ]]; then
  file=" /var/log/secure*"
  files+=${file}
  ${ECHO} "+ ${file}" >> ${DIRECTORY}/${LOGFILE}
fi

# CPE-419
if [[ -e "/var/log/pacemaker.log" ]]; then
  file=" /var/log/pacemaker.log*"
  files+=${file}
  ${ECHO} "+ ${file}" >> ${DIRECTORY}/${LOGFILE}
fi

# Add Historical Stats logs
if [ "${clusters}" == "off" ]; then
    debugLogWriter "# ${FIND} /data/ -type f -mtime -${hdays} -name 'lbstats_historical*'"
    file=" $(${FIND} /data/ -type f -mtime -${hdays} -name 'lbstats_historical*')"
elif [ "${clusters}" == "on" ]; then
    debugLogWriter "# ${FIND} /data/ -type f -mtime -${hdays} -name 'lbstats_historical*' | ${GREP} cid_${cluster_id}"
    file=" $(${FIND} /data/ -type f -mtime -${hdays} -name 'lbstats_historical*' | ${GREP} cid_${cluster_id})"
else
    errorLogWriter "[INTERNAL ERROR] Unexpected value of CLUSTERS=${clusters} in historical stats collection"
    abort
fi
files+=${file}
${ECHO} "+ ${file}" >> ${DIRECTORY}/${LOGFILE}

# Add query log in the list if the date was provided
if [ "${logs}" == "on" ]; then
    for d in "${userdate[@]}"
        do
        _yyyymmdd=${d:0:8}
        logWriter "The script is going to add query logs as of ${d} located under /data/logs/${_yyyymmdd} onto the collect log list."
        debugLogWriter "SET _yyyymmdd=${_yyyymmdd}"
        if [ "${clusters}" == "on" ]; then
            debugLogWriter "# ${FIND} /data/logs/${_yyyymmdd} -type f -name '*${d}*' | ${GREP} cid_${cluster_id}"
            file=" $(${FIND} /data/logs/${_yyyymmdd} -type f -name \*${d}\* | ${GREP} cid_${cluster_id})"
            debugLogWriter "# ${FIND} /data/logs/${_yyyymmdd} -type f -name '*${d}*' | ${GREP} -v cid_"
            file=" $(${FIND} /data/logs/${_yyyymmdd} -type f -name \*${d}\* | ${GREP} -v cid_)"

        elif [ "${clusters}" == "off" ]; then
            debugLogWriter "# ${FIND} /data/logs/${_yyyymmdd} -type f -name '*${d}*'"
            file=" $(${FIND} /data/logs/${_yyyymmdd} -type f -name \*${d}\*)"
        else
            errorLogWriter "[INTERNAL ERROR] Unexpected error for CLUSTERS=${clusters} in query log collection"
            abort
        fi

        files+=${file}
        ${ECHO} "+ ${file}" >> ${DIRECTORY}/${LOGFILE}
	file=" /data/logs/${_yyyymmdd}/idb_main.log"
        files+=${file}
        ${ECHO} "+ ${file}" >> ${DIRECTORY}/${LOGFILE}
        # CPE-421
        if [[ -e "/data/logs/${_yyyymmdd}/log_parser.txt" ]]; then
	   	file=" /data/logs/${_yyyymmdd}/log_parser.txt"
       		files+=${file}
        	${ECHO} "+ ${file}" >> ${DIRECTORY}/${LOGFILE}
	fi
    done
fi

# Add generated logs
file=" $(${LS} ${DIRECTORY}/*)"
files+=${file}
${ECHO} "+ ${file}" >> ${DIRECTORY}/${LOGFILE}

# Display all the log files that the script is going to collect
logWriter "The script successfully completes generating the file list"
writeTitle "COLLECTED LOG LIST"
${ECHO} "${files}" >> ${DIRECTORY}/${LOGFILE}

# The script starts compressing files
writeTitle "COMPRESSING FILES"
logWriter "The script is going to compress all collected files"
logWriter "Please be aware of that this operation might take a while...";

if [ ${dry} == "off" ]; then
	if [ ${bzip} == "off" ]; then
	    debugLogWriter "${TAR} cvfz ${DIRECTORY}/$(${SA_HOSTNAME})_${TIMESTAMP}.tz files..."
		${TAR} cvfz ${DIRECTORY}/$(${SA_HOSTNAME})_${TIMESTAMP}.tz --exclude='cluster_creation' --exclude='cluster_command_idblb.log' ${files} | ${TEE} -a ${DIRECTORY}/${LOGFILE}
		debugLogWriter "${TAR} exit code: ${ECHO} $?"
	elif [ ${bzip} == "on" ]; then
	    debugLogWriter "${TAR} cvfj ${DIRECTORY}/$(${SA_HOSTNAME})_${TIMESTAMP}.bz2 files..."
		${TAR} cvfj ${DIRECTORY}/$(${SA_HOSTNAME})_${TIMESTAMP}.bz2 --exclude='cluster_creation' --exclude='cluster_command_idblb.log' ${files} | ${TEE} -a ${DIRECTORY}/${LOGFILE}
		debugLogWriter "${TAR} exit code: ${ECHO} $?"

	fi
	logWriter "Taring of the files has been completed";

	# Verify that the logs were collected
	if [[ -f ${DIRECTORY}/$(${SA_HOSTNAME})_${TIMESTAMP}.tz || -f ${DIRECTORY}/$(${SA_HOSTNAME})_${TIMESTAMP}.bz2 ]]; then
	    logWriter "All files were successfully compressed"
	    logWriter "The script complete successfully.";
	    if [ ${bzip} == "off" ]; then
	        displayGreenText "Please provide ${DIRECTORY}/$(${SA_HOSTNAME})_${TIMESTAMP}.tz to support for further analysis.";
	    elif [ ${bzip} == "on" ]; then
	        displayGreenText "Please provide ${DIRECTORY}/$(${SA_HOSTNAME})_${TIMESTAMP}.bz2 to support for further analysis.";
	    fi
	else
	    errorLogWriter "An unknown error occurred while compressing files....";
	    if [ ${bzip} == "off" ]; then
	        errorLogWriter "TAR file name: ${DIRECTORY}/$(${SA_HOSTNAME})_${TIMESTAMP}.tz";
	    elif [ ${bzip} == "on" ]; then
	        errorLogWriter "BZ2 file name: ${DIRECTORY}/$(${SA_HOSTNAME})_${TIMESTAMP}.bz2";
	    fi
	    errorLogWriter "Please manually compress files.";
	fi
fi

# The script ends successfully
success