#!/usr/bin/env python
"""
####################
##
## idb_healthcheck.py - perform automated healtcheck of ScaleArc system based upon
##                      https://support.scalearc.com/kb/articles/2724
##
## Created on: 2016-03-07 by Mark Tripod
##
####################
"""

import ast
import pwd
import grp
import datetime
from datetime import date, timedelta
import getopt
import glob
import json
import logging
import os
import pprint
import random
import re
import math
import sqlite3 as sqlite
import string
import subprocess
import sys
import stat
import time
from ConfigParser import SafeConfigParser
from distutils.version import LooseVersion

import MySQLdb
import pyodbc
from idb.cluster_util import PasswordUtils

# GLOBAL Options Settings
DEFAULTS = {}
RECOMMENDATIONS = {}
FOOTER_STR = "\n"
SARS = {}
VERSION = '0.99.6.2b'
DEBUG = True
VERBOSE = True
QUIET = False
MINIMAL_TESTS = False
IGNORE_CFG_FILE = False
CONFIG_FILE = 'idb_healthcheck.cfg'
OUTPUT_FILE = 'idb_healthcheck-report.txt'
OUTPUT_BASE = 'idb_healthcheck-report.txt'
OUTPUT_DATA_FILE = 'idb_healthcheck-data.txt'
VERSION_URL = 'https://s3.amazonaws.com/scalarcidb/HealthCheck/version'
HC_DOWNLOAD = 'https://s3.amazonaws.com/scalarcidb/HealthCheck/idb_healthcheck.py'
IP_ADDRESSES = []
INPUT_FILE = ''
TAR_FILE_LIST = []
PRINT_REPORT = True
RUN_RECOMMENDATIONS = True
MAX_ITERATIONS = 5
MAX_INTERVAL = 300
TABLE_WIDTH = 128
GLOBAL_LB_SQLITE_FILE = '/system/lb.sqlite'
GLOBAL_OUTPUT_SQLITE_FILE = '/system/healthcheck.sqlite'
GLOBAL_SERVER_CONNECTIONS = 0
MAX_RETRY = 10
TMPL_PATH = '/opt/idb/conf/'
SQLITE_PATH = '/system/'
NOW = datetime.datetime.now()
GB = 1073741824
TESTS_RUN = 0
ips = {}
APIKEY = None
NO_COLOR = False

# Set variable defaults. These will be overridden by command line parameters or configuration file settings
DEFAULTS['ENVIRONMENT'] = 'Production'
DEFAULTS['GA_VERSION'] = '3.10.1.2'
DEFAULTS['LOG_FILEPATH'] = '/tmp/'
DEFAULTS['RSYNC_CHECK'] = False
DEFAULTS['VERBOSE'] = True
DEFAULTS['DEBUG'] = True
DEFAULTS['QUIET'] = False
DEFAULTS['PRINT_REPORT'] = True
DEFAULTS['SYSTEM_CHECK'] = False
DEFAULTS['CHECK_STORAGE_THROUGHPUT'] = False

CPU_DEFAULTS = {}

LEVEL1_HEADINGS = ['Cluster Landing Page', 'Cluster Settings', 'Logs Settings']
LEVEL2_HEADINGS = {'Cluster Landing Page': ['Users & DBs', 'Auto Failover', 'Database Server Settings'],
                   'Cluster Settings': ['Cluster Tab', 'Client Tab', 'Server Tab', 'ScaleArc Tab', 'Debug Tab'],
                   'Logs Settings': ['Main', 'Logs Backup']}

LB_CLUSTERS = {'MYSQL': ["Cluster ID", "Cluster Name", "IP Address", "Port", "Backend Port", "Backend IP",
                         "Authentication Offload", "Data Cache", "Root Connection", "Status", "Update Time",
                         "Read/Write Split", "LB Type", "Inbound Interface", "Outbound Interface", "Firewall Status",
                         "SQL Injection", "ACL Status", "PreData Cache", "Quota Cache", "Authentication",
                         "CMD Sync Port",
                         "Cache Sync Port", "Max Client Limit", "Max Client Limit_org", "comsetoption_replay",
                         "SSL Enabled", "Client SSL Enabled", "Client Verify SSL", "Server Verify SSL",
                         "Start Cluster After Setup", "Is Cluster Started"],
               'MSSQL': ["Cluster ID", "Cluster Name", "IP Address", "Port", "Backend Port", "Backend IP", "Auth Cache",
                         "Data Cache", "Root Connection", "Status", "Update Time", "Read/Write Split", "LB Type",
                         "In Interface", "Out Interface", "Firewall Status", "SQL Injection", "ACL Status",
                         "PreData Cache", "Quota Cache", "Instance Name", "Authentication", "CMD Sync Port",
                         "Cache Sync Port", "Max Client Limit", "Max Client Limit Org", "comsetoption_replay",
                         "AlwaysOn",
                         "VNN Server", "VNN Port", "Availability Group ID", "Availability Group Name", "SSL Enabled",
                         "Client SSL Enabled", "Client Verify SSL", "Server Verify SSL", "Start Cluster After Setup",
                         "Is Cluster Started"],
               'ORACLE': ["Cluster ID", "Cluster Name", "IP Address", "Port", "Backend Port", "Backend IP",
                          "Auth Cache",
                          "Data Cache", "Root Connection", "Status", "Update Time", "Read/Write Split", "LB Type",
                          "In Interface", "Out Interface", "Firewall Status", "SQL Injection", "ACL Status",
                          "PreData Cache", "Quota Cache", "Authentication", "Cluster Name", "CMD Sync Port",
                          "Cache Sync Port", "Max Client Limit", "Max Client Limit Org", "AlwaysOn", "VNN Server",
                          "VNN Port", "comsetoption_replay", "SID Route Status", "SID Route Default Server",
                          "SSL Enabled",
                          "Start Cluster After Setup", "Is Cluster Started"]}

TODAY = time.strftime('%Y%m%d')
YESTERDAY = date.today() - timedelta(1)
YESTERDAY = YESTERDAY.strftime('%Y%m%d')
FILE_STATS = {'/data/': {'group': 'apache', 'owner': 'root', 'permissions': 16895},
              '/data/cache': {'group': 'root', 'owner': 'root', 'permissions': 16895},
              '/data/cache/ramdisk': {'group': 'root',
                                      'owner': 'root',
                                      'permissions': 17407},
              '/data/cache/ramdisk/lbstats.sqlite': {'group': 'apache',
                                                     'owner': 'root',
                                                     'permissions': 33277},
              '/data/cache/report.txt': {'group': 'root',
                                         'owner': 'root',
                                         'permissions': 33279},
              '/data/logs': {'group': 'root', 'owner': 'root', 'permissions': 16895},
              '/data/logs/' + YESTERDAY: {'group': 'apache',
                                          'owner': 'apache',
                                          'permissions': 16877},
              '/data/logs/' + TODAY: {'group': 'root',
                                      'owner': 'root',
                                      'permissions': 16877},
              '/data/logs/access.log': {'group': 'apache',
                                        'owner': 'apache',
                                        'permissions': 33188},
              '/data/logs/currentlogs': {'group': 'apache',
                                         'owner': 'root',
                                         'permissions': 16895},
              '/data/logs/currentlogs/' + TODAY: {'group': 'root',
                                                  'owner': 'root',
                                                  'permissions': 16877},
              '/data/logs/currentlogs/' + TODAY + '/idb_main.log': {'group': 'root',
                                                                    'owner': 'root',
                                                                    'permissions': 33188},
              '/data/logs/grub_reboots.conf': {'group': 'root',
                                               'owner': 'root',
                                               'permissions': 33279},
              '/data/logs/idb.log': {'group': 'root',
                                     'owner': 'root',
                                     'permissions': 33188},
              '/data/logs/installer.log': {'group': 'root',
                                           'owner': 'root',
                                           'permissions': 33188},
              '/data/logs/log.sqlite': {'group': 'root',
                                        'owner': 'root',
                                        'permissions': 33204},
              '/data/logs/old_grub.conf': {'group': 'root',
                                           'owner': 'root',
                                           'permissions': 33279},
              '/data/logs/schema_upgrade.log': {'group': 'root',
                                                'owner': 'root',
                                                'permissions': 33188},
              '/data/logs/services': {'group': 'apache',
                                      'owner': 'root',
                                      'permissions': 16895},
              '/data/logs/services/alert_engine.log': {'group': 'root',
                                                       'owner': 'root',
                                                       'permissions': 33188},
              '/data/logs/services/analytics.log': {'group': 'root',
                                                    'owner': 'root',
                                                    'permissions': 33188},
              '/data/logs/services/cluster_monitor.log': {'group': 'root',
                                                          'owner': 'root',
                                                          'permissions': 33188},
              '/data/logs/services/continuous_monitor.log': {'group': 'root',
                                                             'owner': 'root',
                                                             'permissions': 33206},
              '/data/logs/services/crondelete.log': {'group': 'root',
                                                     'owner': 'root',
                                                     'permissions': 33188},
              '/data/logs/services/db_monitor.log': {'group': 'root',
                                                     'owner': 'root',
                                                     'permissions': 33188},
              '/data/logs/services/dummy': {'group': 'root',
                                            'owner': 'root',
                                            'permissions': 33188},
              '/data/logs/services/error.log': {'group': 'root',
                                                'owner': 'root',
                                                'permissions': 33188},
              '/data/logs/services/idb_analytics.log': {'group': 'root',
                                                        'owner': 'root',
                                                        'permissions': 33152},
              '/data/logs/services/idb_snmp.log': {'group': 'root',
                                                   'owner': 'root',
                                                   'permissions': 33188},
              '/data/logs/services/license_monitor.log': {'group': 'root',
                                                          'owner': 'root',
                                                          'permissions': 33188},
              '/data/logs/services/logs_backup.log': {'group': 'root',
                                                      'owner': 'root',
                                                      'permissions': 33188},
              '/data/logs/services/monitor_idb.log': {'group': 'root',
                                                      'owner': 'root',
                                                      'permissions': 33152},
              '/data/logs/services/move_currentlogs.log': {'group': 'root',
                                                           'owner': 'root',
                                                           'permissions': 33188},
              '/data/logs/services/reverse_tunnel.log': {'group': 'root',
                                                         'owner': 'root',
                                                         'permissions': 33188},
              '/data/logs/services/socket_client.log': {'group': 'root',
                                                        'owner': 'root',
                                                        'permissions': 33206},
              '/data/logs/services/system_monitor.log': {'group': 'root',
                                                         'owner': 'root',
                                                         'permissions': 33188},
              '/data/logs/services/watchdog.log': {'group': 'root',
                                                   'owner': 'root',
                                                   'permissions': 33188},
              '/data/logs/system_reboot.log': {'group': 'root',
                                               'owner': 'root',
                                               'permissions': 33188},
              '/data/logs/tmp_messages.log': {'group': 'root',
                                              'owner': 'root',
                                              'permissions': 33188},
              '/data/lost+found': {'group': 'root', 'owner': 'root', 'permissions': 16832},
              '/system/': {'group': 'apache', 'owner': 'root', 'permissions': 17917},
              '/system/cluster_lbstats.sqlite': {'group': 'apache',
                                                 'owner': 'root',
                                                 'permissions': 33261},
              '/system/cpucores.txt': {'group': 'apache',
                                       'owner': 'root',
                                       'permissions': 33277},
              '/system/cpudistribution.sqlite': {'group': 'apache',
                                                 'owner': 'root',
                                                 'permissions': 33204},
              '/system/idb_software_version': {'group': 'apache',
                                               'owner': 'root',
                                               'permissions': 33188},
              '/system/iscloud.txt': {'group': 'apache',
                                      'owner': 'root',
                                      'permissions': 33277},
              '/system/lb.sqlite': {'group': 'apache',
                                    'owner': 'root',
                                    'permissions': 33204},
              '/system/lb_events.sqlite': {'group': 'apache',
                                           'owner': 'root',
                                           'permissions': 33204},
              '/system/lbipid.txt': {'group': 'apache',
                                     'owner': 'root',
                                     'permissions': 33206},
              '/system/lbsession.sqlite': {'group': 'apache',
                                           'owner': 'root',
                                           'permissions': 33204},
              '/system/lbstats.sqlite': {'group': 'apache',
                                         'owner': 'root',
                                         'permissions': 33277},
              '/system/lbstats.sqlite.ram': {'group': 'apache',
                                             'owner': 'root',
                                             'permissions': 33277},
              '/system/monitor_idb.file': {'group': 'apache',
                                           'owner': 'root',
                                           'permissions': 33188},
              '/system/ntp_scalearc.conf': {'group': 'apache',
                                            'owner': 'root',
                                            'permissions': 33261},
              '/system/remotersyncip': {'group': 'apache',
                                        'owner': 'root',
                                        'permissions': 33204},
              '/system/resolv.conf': {'group': 'apache',
                                      'owner': 'root',
                                      'permissions': 33188},
              '/system/scalearc_static.sqlite': {'group': 'apache',
                                                 'owner': 'root',
                                                 'permissions': 33261},
              '/system/tid': {'group': 'apache', 'owner': 'root', 'permissions': 33188}}

# Check if run by root
if not os.geteuid() == 0:
    sys.exit("Please run {0} as root.\n Example: # sudo {1}\n".format(os.path.basename(__file__),
                                                                      os.path.basename(__file__)))

if os.access(os.getcwd(), os.X_OK) is False:
    sys.exit("The current working directory does not provide the ability to execute. Please resolve this issue "
             + "and execute the script again or move to a different filesystem location that allows the ability "
             + "to execute.\n")

if os.access(os.getcwd(), os.W_OK) is False:
    sys.exit(
        "The current working directory is not read/write. Please resolve this issue and " +
        "execute the script again or move the script to a different filesystem with read/write ability.\n")

# Initialize primary value storage dictionary
sa_dict = {'Alerts': [], 'System Messages': [], 'CPU Core Usage': {}, 'CPU Core Threads': [], 'Interface IPs': {},
           'Virtual IPs': {}, 'NTP Servers': [], 'Cluster Configurations': {}, 'Log Settings': {},
           'Server Configurations': {}, 'DNS Name Servers': []}

RECOMMENDATIONS['VERSION'] = []
RECOMMENDATIONS['MEMORY'] = []
RECOMMENDATIONS['DATABASE'] = []
RECOMMENDATIONS['HEARTBEAT'] = {}
RECOMMENDATIONS['HEARTBEAT']['HA DEAD TIME'] = 5000
RECOMMENDATIONS['HEARTBEAT']['HA INIT DEAD TIME'] = 30000
RECOMMENDATIONS['HEARTBEAT']['HA KEEP ALIVE'] = 1000
RECOMMENDATIONS['HEARTBEAT']['HA WARN TIME'] = 2500
RECOMMENDATIONS['RSYNC'] = {}
RECOMMENDATIONS['ALERTING'] = []
RECOMMENDATIONS['SYSTEM LOGS'] = []
RECOMMENDATIONS['DISK LAYOUT'] = []
RECOMMENDATIONS['CPU'] = []
RECOMMENDATIONS['NETWORK INTERRUPTS'] = []
RECOMMENDATIONS['SYSTEM MEMORY'] = {}
RECOMMENDATIONS['SYSTEM MEMORY']['minimum'] = {}
RECOMMENDATIONS['SYSTEM MEMORY']['minimum']['ENTERPRISE'] = 8
RECOMMENDATIONS['SYSTEM MEMORY']['minimum']['PLATINUM'] = 8
RECOMMENDATIONS['SYSTEM MEMORY']['4'] = {}
RECOMMENDATIONS['SYSTEM MEMORY']['4']['ENTERPRISE'] = 16
RECOMMENDATIONS['SYSTEM MEMORY']['4']['PLATINUM'] = 24
RECOMMENDATIONS['SYSTEM MEMORY']['8'] = {}
RECOMMENDATIONS['SYSTEM MEMORY']['8']['ENTERPRISE'] = 16
RECOMMENDATIONS['SYSTEM MEMORY']['8']['PLATINUM'] = 40
RECOMMENDATIONS['SYSTEM MEMORY']['10'] = {}
RECOMMENDATIONS['SYSTEM MEMORY']['10']['ENTERPRISE'] = 20
RECOMMENDATIONS['SYSTEM MEMORY']['10']['PLATINUM'] = 48
RECOMMENDATIONS['SYSTEM MEMORY']['12'] = {}
RECOMMENDATIONS['SYSTEM MEMORY']['12']['ENTERPRISE'] = 24
RECOMMENDATIONS['SYSTEM MEMORY']['12']['PLATINUM'] = 54
RECOMMENDATIONS['SYSTEM MEMORY']['16'] = {}
RECOMMENDATIONS['SYSTEM MEMORY']['16']['ENTERPRISE'] = 32
RECOMMENDATIONS['SYSTEM MEMORY']['16']['PLATINUM'] = 80
RECOMMENDATIONS['SYSTEM MEMORY']['20'] = {}
RECOMMENDATIONS['SYSTEM MEMORY']['20']['ENTERPRISE'] = 40
RECOMMENDATIONS['SYSTEM MEMORY']['20']['PLATINUM'] = 96
RECOMMENDATIONS['SYSTEM MEMORY']['24'] = {}
RECOMMENDATIONS['SYSTEM MEMORY']['24']['ENTERPRISE'] = 40
RECOMMENDATIONS['SYSTEM MEMORY']['24']['PLATINUM'] = 112
RECOMMENDATIONS['SYSTEM MEMORY']['32'] = {}
RECOMMENDATIONS['SYSTEM MEMORY']['32']['ENTERPRISE'] = 48
RECOMMENDATIONS['SYSTEM MEMORY']['32']['PLATINUM'] = 128
RECOMMENDATIONS['SWAP'] = {}
RECOMMENDATIONS['SWAP']['8'] = ''  # system memory * 2
RECOMMENDATIONS['SWAP']['64'] = ''  # 16+ GB
RECOMMENDATIONS['SWAP']['256'] = ''  # 32+ GB

CLUSTER_CONFIG_ITEMS = {}
TESTS = {}


# noinspection PyClassHasNoInit
class bcolors:
    HEADER = '\033[95m'
    OKBLUE = '\033[94m'
    OKGREEN = '\033[92m'
    WARNING = '\033[93m'
    FAIL = '\033[91m'
    ENDC = '\033[0m'
    BOLD = '\033[1m'
    UNDERLINE = '\033[4m'


NOW = NOW.strftime("%Y-%m-%d_%H:%M:%S")

LOG_FILENAME = DEFAULTS['LOG_FILEPATH'] + os.path.basename(__file__) + "-" + NOW + ".log"

hc_log = logging.getLogger('idb_healthcheck')
hc_log.setLevel(logging.DEBUG)
fh = logging.FileHandler(LOG_FILENAME)
fh.setLevel(logging.DEBUG)
formatter = logging.Formatter('%(asctime)s  [ %(levelname)s ]::%(lineno)s  %(message)s', '%m/%d/%Y %I:%M:%S %p')
fh.setFormatter(formatter)
hc_log.addHandler(fh)

hc_log.info("Health Check execution started")
hc_log.info("Version: " + bcolors.OKGREEN + "{0}".format(VERSION) + bcolors.ENDC)
hc_log.info("####################################")


#####
## function definitions
####


def version():
    """
    Display the current version
    :return:
    """
    sys.stdout.write("{0} \nVersion: {1}\n\n".format(os.path.basename(__file__), VERSION))
    return 0


def usage():
    """
    Display the usage information
    :return:
    """
    if NO_COLOR:
        sys.stdout.write("\nUsage: {0}".format(os.path.basename(__file__)))
        sys.stdout.write(" [ -c<config file> -i<input file> -o<output file> -p -q -d -v -h -V ]")
        print '''

          [ -c<path to file> | --config=<path to file> ]   : Configuration file for idb_healthcheck.py. Default: ./idb_healthcheck.cfg
          [ -o<path to file> | --output=<path to file> ]   : Name of file to write report results
          [ -i<path to file> | --input=<path to file> ]    : Name of file to read previous test data from. Useful for generating new report from existing data.
          [ -p | --report ]                                : Output formatted report [Default]
          [ -h | --help ]                                  : Prints this help message
          [ -v | --verbose ]                               : Output extra information during execution [Default]
          [ -V | --version ]                               : Display script version information
          [ -d | --debug ]                                 : Output DEBUG level information during execution
          [ -q | --quiet ]                                 : Suppress all terminal output
          [ -m | --minimal ]                               : Minimal health check tests. Skip all log file and SAR data inspection
          [ --no-color ]                                   : Disable color output to terminal (for machine parsing of output)
        '''
    else:
        sys.stdout.write("\nUsage: {0}".format(bcolors.WARNING + os.path.basename(__file__) + bcolors.ENDC))
        sys.stdout.write(" \033[94m[ -c<config file> -i<input file> -o<output file> -p -q -d -v -h -V ]\033[0m")
        print '''

        [ -c<path to file> | --config=<path to file> ]   : Configuration file for idb_healthcheck.py. Default: ./idb_healthcheck.cfg
        [ -o<path to file> | --output=<path to file> ]   : Name of file to write report results
        [ -i<path to file> | --input=<path to file> ]    : Name of file to read previous test data from. Useful for generating new report from existing data.
        [ -p | --report ]                                : Output formatted report [\033[92mDefault\033[0m]
        [ -h | --help ]                                  : Prints this help message
        [ -v | --verbose ]                               : Output extra information during execution [\033[92mDefault\033[0m]
        [ -V | --version ]                               : Display script version information
        [ -d | --debug ]                                 : Output DEBUG level informaton during execution
        [ -q | --quiet ]                                 : Suppress all terminal output
        [ -m | --minimal ]                               : Minimal health check tests. Skip all log file and SAR data inspection
        [ --no-color ]                                   : Disable color output to terminal (for machine parsing of output)
      '''

    '''
    TODO: Not currently implemented
    [ -f | --format ]                  : Output format selection (JSON, table, PDF)

  '''
    sys.exit(9)


def check_version():
    """
    Check if there is a newer version of the script available. If so, prompt for update.
    :return:
    """
    hc_version = '0.01'
    hc_log.info("Checking latest version of idb_healthcheck.py...")
    hc_log.info("  hc_version is set to {0}".format(hc_version))
    hc_log.info("  VERSION is set to {0}".format(VERSION))
    hc_log.info(
        "  Executing system_call for /usr/bin/curl -k -X GET https://s3.amazonaws.com/scalarcidb/HealthCheck/version --connect-timeout 5 --max-time 10...")
    try:
        hc_version = system_call(
            '/usr/bin/curl -k -X GET https://s3.amazonaws.com/scalarcidb/HealthCheck/version --connect-timeout 5 --max-time 10 2> /dev/null')
        hc_log.debug("    Return value from curl call: '{0}'".format(hc_version))
        hc_version = hc_version.replace("\n", "")
        if hc_version == '':
            hc_log.warning(
                "    Unable to determine the current production version. Skipping idb_healthcheck.py check_version().")
            return 0
    except Exception as e:
        hc_log.error("  Exception caught while retrieving current idb_healthcheck.py version from S3.")
        return -1

    if LooseVersion(hc_version) > VERSION:
        hc_log.info("  Newer version available. Current: {0}, Available: {1}".format(VERSION, hc_version))
        sys.stderr.write("\nThere is a newer version of the idb_healthcheck.py script available.\n")
        decision = ''
        while decision not in ['y', 'Y', 'n', 'N']:
            decision = raw_input("Would you like to download it now? (y/n)")
        if decision.lower() == 'y':
            hc_log.info("  Update selected. Downloading...")
            sys.stderr.write("\nDownloading idb_healthcheck.py version {0}...".format(hc_version))
            result = system_call("/usr/bin/curl -k -X GET {0} -o idb_healthcheck.py 2> /dev/null".format(HC_DOWNLOAD))
            if result == '':
                sys.stderr.write(
                    "\nThe latest version ({0}) of idb_healthcheck.py has been downloaded.".format(hc_version))
                sys.stderr.write("\nPlease launch the idb_healtcheck.py script again.\n")
                sys.exit(0)
            else:
                sys.stderr.write("\nidb_healthcheck.py download failed.\n")
                sys.exit(0)
        else:
            hc_log.info("  Update refused.")
    return 0


def parse_config():
    """
    Read and parse the configuration file defined in the CONFIG_FILE global variable
    :return:
    """

    global DEFAULTS
    global CPU_DEFAULTS
    global RECOMMENDATIONS
    hc_log.info("Parsing configuration file ({0})...".format(CONFIG_FILE))

    if os.path.isfile(CONFIG_FILE) is False:
        hc_log.info("Configuration file {0} not found. Using internal defaults for configuration options.".format(
            bcolors.OKGREEN + CONFIG_FILE + bcolors.ENDC))
        return 0

    config = SafeConfigParser()
    config.read(CONFIG_FILE)
    if config.has_section('IDB_HEALTHCHECK'):
        hc_log.info("Reading IDB_HEALTHCHECK section of {0}".format(CONFIG_FILE))
        for name, value in config.items('IDB_HEALTHCHECK'):
            #            name = name.upper()
            name = name.title()
            value = value.rstrip()
            # TODO: Check for expected VERBOSE an DEBUG name parameters as well as values to set the correct BOOL types
            hc_log.info("  Setting '{0}' to '{1}'".format(name, value))
            DEFAULTS[name] = value
    else:
        hc_log.info(
            "IDB_HEALTHCHECK section is missing from configuration file ({0}). Internal defaults in use.".format(
                CONFIG_FILE))

    if config.has_section('CPU_THREAD_MAPPING'):
        hc_log.info("Reading CPU_THREAD_MAPPING section of {0}".format(CONFIG_FILE))
        for name, value in config.items('CPU_THREAD_MAPPING'):
            name = name.upper()
            CPU_DEFAULTS[name] = value

    if config.has_section('RECOMMENDATIONS'):
        hc_log.info("Reading RECOMMENDATIONS section of {0}".format(CONFIG_FILE))
        for name, value in config.items('RECOMMENDATIONS'):
            name = name.upper()
            RECOMMENDATIONS[name] = value
    TAR_FILE_LIST.append(CONFIG_FILE)
    return 0


def parse_input_file():
    """
    Read a previously generated idb_healthcheck-data.txt file and load data into
    dictionaries for duplicate report generation
    :return:
    """
    global sa_dict
    global CLUSTER_CONFIG_ITEMS
    hc_log.info("Parsing input file {0}".format(bcolors.OKGREEN + INPUT_FILE + bcolors.ENDC))
    try:
        input_file = open(INPUT_FILE, "r")
        input_str = ''
        hc_log.info("  Reading data from {0}".format(INPUT_FILE))
        for line in input_file:
            if line in ['\n', '\r\n']:
                break
            else:
                input_str += line
    except Exception as e:
        hc_log.info(bcolors.FAIL + "Exception caught while attempting to open {0}: {1}".format(INPUT_FILE,
                                                                                               str(e)) + bcolors.ENDC)
        hc_log.info("Ending execution.")
        sys.exit(-1)

    hc_log.info("  Loading data into dictionary...")
    try:
        sa_dict = ast.literal_eval(input_str)
    except Exception as e:
        hc_log.info(bcolors.FAIL + "  Invalid data in {0}. Aborting execution.".format(INPUT_FILE) + bcolors.ENDC)
        sys.stderr.write(
            "Error loading string into dictionary: {0}\n".format(str(bcolors.FAIL + str(e) + bcolors.ENDC)))
        sys.stderr.write(
            "\n{0} contains invalid data. Please provide an idb_healthcheck-data.txt or equivalent file format.\n\n".format(
                INPUT_FILE))
        hc_log.info("Ending execution.")
        exit()
    try:
        CLUSTER_CONFIG_ITEMS = ast.literal_eval(sa_dict['Cluster Configuration Items'])
    except Exception as e:
        hc_log.info(bcolors.FAIL +
                    "Unable to load cluster configuration settings. The cluster settings section of the report will be omitted." +
                    bcolors.ENDC)
        sys.stderr.write(bcolors.FAIL +
                         "Unable to load cluster configuration settings. The cluster settings section of the report will be omitted.\n" +
                         bcolors.ENDC)
        sys.stderr.write("Exception: {0}\n".format(bcolors.FAIL + str(e) + bcolors.ENDC))
    hc_log.info("  Data load complete.")
    return 0


def print_table():
    """
    Print the first table in the output report
    All items listed in the _table_items list are output
    :return:
    """
    hc_log.info("Printing output table...")

    _table_items = ['Current Production Version',
                    'Current Staging Version',
                    'Current GA Version',
                    'CentOS Version',
                    'Kernel Version',
                    'License Expiration',
                    'Platform',
                    'System Memory',
                    'Swap Size',
                    'Licensed CPU Cores',
                    'Storage Devices',
                    'Database Type',
                    'Database Version',
                    'HA Keep Alive',
                    'HA Warn Time',
                    'HA Dead Time',
                    'HA Init Dead Time',
                    'Heartbeat Process CPU',
                    'Data Mount',
                    'Data Size',
                    'Data Used',
                    'Network Interrupts',
                    'CPU Core Usage',
                    'NTP Servers',
                    'UUID',
                    'Host ID'
                    ]

    for width in range(1, TABLE_WIDTH):
        sys.stdout.write("-")
    sys.stdout.write("\n|  {0:<122} |".format(os.path.basename(__file__) + " version " + VERSION))
    sys.stdout.write(
        "\n|  Report generated: {0:<104} |".format(datetime.datetime.now().strftime("%A, %B %d %Y %I:%M%p")))
    if INPUT_FILE != '':
        sys.stdout.write("\n|  Input file: {0:<110} |\n".format(INPUT_FILE))
    else:
        sys.stdout.write("\n")
    for width in range(1, TABLE_WIDTH):
        sys.stdout.write("-")

    for i in _table_items:
        if i is 'Storage Devices':
            devices = sa_dict[i].keys()
            c = 0
            for device in devices:
                if c > 0:
                    col1 = ''
                else:
                    col1 = i
                sys.stdout.write(
                    "\n| {0:>60} | {1:<30} {2:<20} {3:<8} |".format(col1, device,
                                                                    str(sa_dict[i][device]['Mount Point']),
                                                                    str(sa_dict[i][device]['Type'])))
                c += 1
        elif i is 'CPU Core Usage':
            cores = sorted(sa_dict[i].keys())
            c = 0
            for core in cores:
                if c > 0:
                    col1 = ''
                else:
                    col1 = i
                usage_values = sa_dict[i][core]
                v1 = 0
                for usage_value in usage_values:
                    if v1 == 0:
                        sys.stdout.write("\n| {0:>60} | {1:<10} {2:<49} |".format(col1, core, usage_value))
                        v1 += 1
                    else:
                        sys.stdout.write("\n| {0:>60} | {1:<10} {2:<49} |".format("", "", usage_value))
                        # sys.stdout.write( "\n| {0:>60} | {1:<10} {2:<49} |".format( col1, core, str(dict[i][core]) ) )
                c += 1
        elif i is 'Network Interrupts':
            nis = sorted(sa_dict[i].keys())
            c = 0
            for ni in nis:
                if c > 0:
                    col1 = ''
                else:
                    col1 = i
                sys.stdout.write("\n| {0:>60} | {1:<30} {2:<29} |".format(col1, ni, str(sa_dict[i][ni])))
                c += 1
        else:
            if (re.search(r'\bused\b|\bsize\b|\bmemory\b', i, re.IGNORECASE)) is not None:
                if sa_dict[i] == 'Unknown':
                    _value_str = sa_dict[i]
                else:
                    _value = float(sa_dict[i])
                    _value_str = sizeof_format(_value)
                sys.stdout.write("\n| {0:>60} | {1:<60} |".format(i, _value_str))
            elif (re.search(r'\binterval\b', i, re.IGNORECASE)) is not None:
                _value_str = str(sa_dict[i]) + " seconds"
                sys.stdout.write("\n| {0:>60} | {1:<60} |".format(i, _value_str))
            else:
                sys.stdout.write("\n| {0:>60} | {1:<60} |".format(i, str(sa_dict[i])))

    sys.stdout.write("\n")
    for width in range(1, TABLE_WIDTH):
        sys.stdout.write("-")
    width = TABLE_WIDTH - 5
    sys.stdout.write("\n| {0:<{width}} |".format("Cluster Configuration Settings", width=width))

    _cluster_names = CLUSTER_CONFIG_ITEMS.keys()
    for _cluster_name in _cluster_names:
        sys.stdout.write("\n|   Cluster: {0:<112} |".format(_cluster_name))
        sys.stdout.write("\n|{0:<58} | {1:^30} | {2:^30} |".format('', 'Current Setting', "Proposed New Setting"))
        for _l1_heading in LEVEL1_HEADINGS:
            sys.stdout.write("\n|      {0:<118} |".format(_l1_heading.upper()))
            _l2_headings = LEVEL2_HEADINGS[_l1_heading]
            for _l2_heading in _l2_headings:
                sys.stdout.write("\n|            {0:_<112} |".format(_l2_heading))
                _items = CLUSTER_CONFIG_ITEMS[_cluster_name][_l1_heading][_l2_heading].keys()
                for item in _items:
                    value = CLUSTER_CONFIG_ITEMS[_cluster_name][_l1_heading][_l2_heading][item]
                    recommended_value = ''

                    if item == 'Authentication Offload' and value == 'False':
                        recommended_value = bcolors.OKGREEN + 'On' + bcolors.ENDC
                    elif item == 'Auto Failover':
                        recommended_value = bcolors.OKGREEN + 'Enabled' + bcolors.ENDC
                    elif item == 'Idle Server Connection Timeout' and value != 300:
                        recommended_value = bcolors.OKGREEN + '300 seconds' + bcolors.ENDC
                    elif item == 'Logging Level' and value != 'Extended':
                        recommended_value = bcolors.OKGREEN + 'Extended' + bcolors.ENDC

                    if _l2_heading == 'Auto Failover' and item == 'Enabled' and value == 'False':
                        recommended_value = bcolors.OKGREEN + 'On' + bcolors.ENDC

                    if value == '0':
                        value = "Off"
                    elif value == '1':
                        value = "On"

                    # Adjust formatting of output to account for hidden characters for color coded output
                    if recommended_value == '':
                        sys.stdout.write(
                            "\n|        {0:>50} | {1:^30} | {2:^30} |".format(item, value, recommended_value))
                    else:
                        sys.stdout.write(
                            "\n|        {0:>50} | {1:^30} | {2:^39} |".format(item, value, recommended_value))

    sys.stdout.write("\n")
    for width in range(1, TABLE_WIDTH):
        sys.stdout.write("-")
    sys.stdout.write("\n| {0:<123} |".format("Alerts"))
    sys.stdout.write("\n| {0:<123} |".format(""))

    a_count = 1
    alerts = sa_dict['Alerts']
    for alert in alerts:
        count_str = str(a_count) + ") "  # Number each of the alerts in the output report
        alert_str = alert
        if len(alert_str) > 119:
            loop_count = 0
            start_pos = 0
            end_pos = 118
            tmp_end_pos = 0
            while len(alert_str) > 119:
                loop_count += 1
                if loop_count > 5:
                    break
                tmp_alert_str = alert[start_pos:end_pos]
                word_search = re.search(r"(.*)\s\S+", tmp_alert_str, re.IGNORECASE)
                if word_search:
                    char_count = len(word_search.group(1))
                    if tmp_end_pos == 0:
                        end_pos = char_count
                    else:
                        end_pos = start_pos + char_count
                else:
                    break
                alert_str = alert[
                            start_pos:end_pos]  # reset the alert string to the proper length for word boundary match
                start_pos = end_pos
                if start_pos > 0:
                    end_pos = start_pos + 118
                tmp_end_pos = end_pos
                if loop_count == 1:
                    sys.stdout.write("\n| {0:>4} {1:<118} |".format(count_str, alert_str))
                else:
                    sys.stdout.write("\n| {0:>4} {1:<118} |".format("", alert_str))
                alert_str = alert[start_pos:]
                sys.stderr.flush()
            sys.stdout.write("\n| {0:>4} {1:<118} |".format("", alert_str))
        else:
            sys.stdout.write("\n| {0:>4} {1:<118} |".format(count_str, alert))
        a_count += 1

    sys.stdout.write("\n")
    for width in range(1, TABLE_WIDTH):
        sys.stdout.write("-")
    sys.stdout.write("\n\n")
    return 0


def write_data_file():
    """
    Store global dictionary to idb_healthcheck-data.txt
    :return:
    """
    hc_log.info("Writing {0} file...".format(OUTPUT_DATA_FILE))
    try:
        sys.stdout = open(OUTPUT_DATA_FILE, "w")
        if DEBUG is True:
            pprint.pprint(sa_dict, width=120)
        print("\n\n")
        if DEBUG is True:
            pprint.pprint(TESTS, width=120)
        print("\n\n")
        sys.stdout.close()
        hc_log.info("  Completed!")
    except Exception as e:
        hc_log.error(bcolors.FAIL + "Exception caught while writing {0}".format(OUTPUT_DATA_FILE) + bcolors.ENDC)
        hc_log.error("[{0}]".format(str(e)))
    sys.stdout = sys.__stdout__
    TAR_FILE_LIST.append(OUTPUT_DATA_FILE)
    return 0


def print_report():
    """
    Generate and output the final report
    :return:
    """
    global OUTPUT_FILE
    global OUTPUT_BASE
    global FOOTER_STR
    hc_log.info("Printing report of collected data...")
    if os.path.isfile(OUTPUT_FILE) is True:
        hc_log.info("{0} file already exists.".format(OUTPUT_FILE))
        _count = 1
        while _count > 0:
            OUTPUT_FILE = OUTPUT_BASE + "_" + str(_count)
            if os.path.isfile(OUTPUT_FILE) is True:
                _count += 1
            else:
                break
    if NO_COLOR:
        FOOTER_STR += "Report output file: {0}\n".format(OUTPUT_FILE)
    else:
        FOOTER_STR += "Report output file: {0}\n".format(bcolors.OKBLUE + OUTPUT_FILE + bcolors.ENDC)
    hc_log.info("  Output file is {0}".format(bcolors.OKBLUE + OUTPUT_FILE + bcolors.ENDC))
    TAR_FILE_LIST.append(OUTPUT_FILE)

    try:
        sys.stdout = open(OUTPUT_FILE, "w")
        print_table()
        sys.stdout.close()
        sys.stdout = sys.__stdout__
        hc_log.info("Report output completed.")
    except Exception as e:
        hc_log.error(bcolors.FAIL + "Exception caught while writing report." + bcolors.ENDC)
        hc_log.error(bcolors.FAIL + str(e) + bcolors.ENDC)
    return 0


def tar_output_files():
    """
    Create a single gzipped tar file with containing the report, data, log, configuration, and sqlite files
    :return:
    """
    hc_log.info("Entering tar_output_files...")
    global FOOTER_STR
    OUTPUT_TAR_FILE = 'idb_healthcheck_results'
    hc_log.info("  Selecting primary IP address...")
    result = system_call(
        "/sbin/ifconfig | /bin/grep -Eo 'inet (addr:)?([0-9]*\.){3}[0-9]*' | /bin/grep -Eo '([0-9]*\.){3}[0-9]*' " +
        "| /bin/grep -v '127.0.0.1' | /usr/bin/head -1")
    ip_match = re.match(r"^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}", result)
    if ip_match:
        suffix = result.replace("\n", "")
    else:
        suffix = _id_generator()
    hc_log.info("  Collecting output files for tar creation...")
    OUTPUT_TAR_FILE = OUTPUT_TAR_FILE + "_" + suffix + '.tz'
    hc_log.info("  Tar file ouput = {0}".format(bcolors.OKBLUE + OUTPUT_TAR_FILE + bcolors.ENDC))
    system_call("/bin/tar cvfz {0} {1} 2>/dev/null".format(OUTPUT_TAR_FILE, str(" ".join(TAR_FILE_LIST))))
    try:
        if os.stat(OUTPUT_TAR_FILE).st_size > 0:
            hc_log.info(bcolors.OKGREEN + "TAR file created successfully." + bcolors.ENDC)
            if NO_COLOR:
                FOOTER_STR += "Please provide the {0} file to ScaleArc support via SFTP transfer to upload.scalearc.com\n".format(
                    OUTPUT_TAR_FILE)
            else:
                FOOTER_STR += "Please provide the {0} file to ScaleArc support via SFTP transfer to upload.scalearc.com\n".format(
                    bcolors.OKGREEN + OUTPUT_TAR_FILE + bcolors.ENDC)
        else:
            hc_log.warning(bcolors.WARNING + "TAR file created, however it is empty." + bcolors.ENDC)
    except OSError:
        hc_log.error(bcolors.FAIL + "TAR file creation failed." + bcolors.ENDC)
    return 0


def _id_generator(size=6, chars=string.ascii_uppercase + string.digits):
    """
    Generate a random ID string
    :param size: The length of the string to generate
    :param chars: Which characters to use in the generated string
    :return: Random string
    """
    return ''.join(random.choice(chars) for _ in range(size))


def time_string(seconds):
    """
    Convert seconds into human readable string
    :param seconds: integer value of seconds to convert
    :return: hours, minutes, and seconds string
    """
    m, s = divmod(seconds, 60)
    h, m = divmod(m, 60)

    time_str = ''
    if h > 0:
        time_str += "{0:d} hours, ".format(int(h))
    if m > 0:
        time_str += "{0:d} minutes and ".format(int(m))
    time_str += "{0:.2f} seconds".format(float(s))
    return time_str


def sizeof_format(num, suffix='B'):
    """
    Convert Bytes into human readable string value
    :param num: integer (bytes)
    :param suffix: type of value value passed in 'num', default is 'B' (bytes)
    :return: condensed string with proper suffix
    """
    for unit in ['', 'K', 'M', 'G', 'T', 'P', 'E', 'Z']:
        if abs(num) < 1024.0:
            return "{0:3.1f} {1}{2}".format(num, unit, suffix)
        num /= 1024.0
    return "{0:.1f} {1}{2}".format(num, 'Y', suffix)


def system_call(command):
    """
    wrapper for operating system call
    :param command: command to run
    :return: command output
    """
    try:
        p = subprocess.Popen([command], stdout=subprocess.PIPE, shell=True)
        return p.stdout.read()
    except OSError as e:
        if e.errno == os.errno.ENOENT:
            hc_log.error(bcolors.FAIL + "Exception caught in system_call: {0}".format(e.strerr) + bcolors.ENDC)
            return ''
    except Exception as e:
        hc_log.error(bcolors.FAIL + "Exception caught in system_call function: {0}".format(e.strerr) + bcolors.ENDC)
        return ''


def get_mysql_version(server, user, password, database="mysql"):
    """
    Query the MySQL server version from the database server
    :param server: IP address or name of the database server
    :param user: database username
    :param password: decrpyted password for database user
    :param database: database to query, DEFAULT: mysql
    :return: DB version string or "Not Available"
    """
    hc_log.info("Getting MySQL software version for {0}...".format(server))
    try:
        hc_log.info("  Connecting to {0}...".format(server))
        db = MySQLdb.connect(host=server, user=user, passwd=password, db=database)
        c = db.cursor()
        hc_log.info("  Executing 'SELECT VERSION()'...")
        c.execute("SELECT VERSION()")
        rows = c.fetchall()
        db.close()
        db_version = str(rows[0])
        hc_log.info("  Result: {0}".format(db_version))
        db_version = db_version.replace("-log", "")
        return db_version
    except Exception as e:
        hc_log.error(
            bcolors.FAIL + "  Unable to determine database software version for {0}".format(server) + bcolors.ENDC)
        hc_log.error(bcolors.FAIL + str(e) + bcolors.ENDC)
        return "Not Available"


def get_mssql_version(server, user, password, port="1433"):
    """
    Query the MS SQL Server version from the database server
    :param server: IP address or name of the database server
    :param user: database username
    :param password: decrypted password for database user
    :param port: port number of MS SQL instance
    :return: database version string or "Not Available"
    """
    password_mask = '************'
    hc_log.info("Getting SQL Server software version for {0}...".format(server))
    hc_log.info("  [Parameters: server {0}, port {1}, user {2}".format(server, port, user))
    conn_str = 'DRIVER={FreeTDS};SERVER=' + server + ';PORT=' + str(port) + ';UID=' + user + ';PWD=' + password
    conn_str_mask = 'DRIVER={FreeTDS};SERVER=' + server + ';PORT=' + str(
        port) + ';UID=' + user + ';PWD=' + password_mask
    conn_str += ';TDS_VERSION=8.0;timeout=60;Trusted_Connection=no;UseNTLMv2=yes;'
    conn_str_mask += ';TDS_VERSION=8.0;timeout=60;Trusted_Connection=no;UseNTLMv2=yes;'
    hc_log.info("  Connecting to {0} using {1}...".format(server, conn_str_mask))
    try:
        db = pyodbc.connect(conn_str)
        c = db.cursor()
        hc_log.info("  Executing 'SELECT CAST(@@VERSION AS TEXT)'...")
        c.execute("SELECT CAST(@@VERSION AS TEXT)")
        rows = c.fetchall()
        db.close()
        v_str = str(rows[0])
        hc_log.info("  Result: {0}".format(v_str))
        fields = v_str.split("\\n\\t")
        pre = fields[0].replace("('", "")
        post = fields[3].replace("\\n', )", "")
        db_version = pre + post
        return db_version
    except Exception as e:
        hc_log.error(
            bcolors.FAIL + "  Unable to determine database software version for {0}".format(server) + bcolors.ENDC)
        hc_log.error(bcolors.FAIL + str(e) + bcolors.ENDC)
        return "Not Available"


def get_cluster_root_user(cid):
    """
    Get the cluster root database username and password
    :param cid: ScaleArc cluster ID (integer)
    :return: tuple of username and password, or empty string for each on failure
    """
    hc_log.info("Getting database admin user for cluster {0}".format(cid))
    sql_conn = sqlite.connect("/system/lb_{0}.sqlite".format(cid))
    sql_conn.text_factory = str
    c = sql_conn.cursor()
    hc_log.info("  Executing 'SELECT username,encpassword from lb_users where status=1 LIMIT 1")
    try:
        c.execute("SELECT username,encpassword FROM lb_users WHERE status=1 LIMIT 1")
        rows = c.fetchall()
        sql_conn.close()
    except Exception as e:
        hc_log.info(bcolors.FAIL + "  Unable to retrieve root user for cluster {0}".format(cid) + bcolors.ENDC)
        hc_log.info(str(e))
        return "Not Found", "Not Found"
    hc_log.info("  Result: {0}".format(str(rows)))
    if rows:
        username, encpasswd = rows[0]
        password = PasswordUtils.decrypt(encpasswd)
        # password = encpasswd
        return username, password
    else:
        return "Not Found", "Not Found"


def _get_active_clusters():
    hc_log.info("Getting list of active clusters...")
    active_clusters = []
    try:
        conn = sqlite.connect('/system/lb.sqlite')
        conn.text_factory = str
        conn.row_factory = sqlite.Row
        c = conn.cursor()
        hc_log.info("  Executing 'SELECT config_path FROM lb_clusters_summary WHERE status=1'...")
        c.execute("SELECT config_path FROM lb_clusters_summary WHERE status=1")
        active_clusters_list = c.fetchall()
        for active_cluster in active_clusters_list:
            active_clusters.append(active_cluster[0])
    except Exception as e:
        hc_log.error(bcolors.FAIL + "  Exception caught while gathering list of active clusters." + bcolors.ENDC)
        hc_log.error(bcolors.FAIL + str(e) + bcolors.ENDC)
    return active_clusters


def get_apikey():
    """
    Get the API key for ScaleArc from /system/lb.sqlite
    :return: tuple of status, API key or error
    """
    hc_log.info("Getting ScaleArc API key...")
    try:
        hc_log.info("  Connecting to SQLite file /system/lb.sqlite")
        conn = sqlite.connect('/system/lb.sqlite')
        conn.text_factory = str
        c = conn.cursor()
        hc_log.info("  Executing 'SELECT apikey FROM lb_network'...")
        c.execute("SELECT apikey FROM lb_network")
        results = c.fetchall()
        conn.close()
        hc_log.info("  Result: {0}".format(str(results)))
        for result in results[0]:
            if result is None:
                return "Fail", "No apikey defined."
            else:
                return "Success", result
    except Exception as e:
        hc_log.info(bcolors.FAIL + str(e) + bcolors.ENDC)
        return "Fail", str(e)


def get_idb_file_list(idb_file_type):
    """
    Gather a list of the requested ScaleArc log file type for the most recent 30 days
    :param idb_file_type: ScaleArc log type (log, alert, etc..)
    :return: list of files
    """
    hc_log.info("Gathering list of idb.{0} log files from /data/logs/currentlogs/...".format(idb_file_type))
    file_list = []
    for root, dirs, files in os.walk("/data/logs/currentlogs"):
        for f in files:
            if f.startswith("idb.{0}.".format(idb_file_type)):
                file_list.append(os.path.join(root, f))
    hc_log.info(
        "Gathering list of idb.{0} log files from last 30 days /data/logs/<DATE>/cid_*/...".format(idb_file_type))
    dirs = sorted(glob.glob('/data/logs/20[0-9][0-9][0-9][0-9][0-9][0-9]'), reverse=True)
    dirs = dirs[0:30]
    for d in dirs:
        file_list += glob.glob("{0}/cid_*/idb.{1}.*".format(d, idb_file_type))
    hc_log.info("  Total files found: {0}".format(len(file_list)))
    return file_list


def get_sar_file_list():
    """
    Collect list of /var/log/sa/sar files for the last three days
    :return: list of absolute files paths
    """
    global SARS
    hc_log.info("Gathering list of most recent sar files...")
    hc_log.info("  Executing /bin/ls -rt /var/log/sa/ | /bin/grep 'sa[0-3]' | /usr/bin/tail -3 ...")
    result = system_call("/bin/ls -rt /var/log/sa/ | /bin/grep 'sa[0-3]' | /usr/bin/tail -3")
    sar_list = []
    if result != '':
        files = result.splitlines()
        for f in files:
            f = '/var/log/sa/' + f
            sar_list.append(f)
            hc_log.info("    Getting date for {0} ...".format(f))
            result = system_call("/bin/ls -l --time-style=+%Y%b%d {0} | /bin/awk '{{print $6;}}'".format(f))
            hc_log.info("      Result: {0}".format(result))
            if result != '':
                result = result.replace("\n", "")
                SARS[f] = result
    else:
        hc_log.info(bcolors.FAIL + "    Error retrieving list of most recent sar files. [Result] {0}".format(
            result) + bcolors.ENDC)
    return sar_list


def check_system_file_handles():
    """
    Check the system file handle usage
    :return:
    """
    hc_log.info("Checking system file handles...")
    stats = {}
    result = system_call("/bin/cat /proc/sys/fs/file-nr")
    if result != '':
        values = result.split()
        stats['allocated'] = values[0]
        stats['used'] = values[1]
        stats['available'] = values[2]

        if stats['available'] < 10000:
            sa_dict['Alerts'].append(
                "Available system file handles is EXTREMELY low ({0}). ".format(stats['available'])
                + "Please contact ScaleArc support immediately to diagnose.")
        elif stats['available'] < 20000:
            sa_dict['Alerts'].append(
                "Available systel file handles is low ({0}). ".format(stats['available'])
                + "Please contact ScaleArc support to diagnose.")
    else:
        hc_log.info(bcolors.FAIL + "Unable to gather current system file handle information." + bcolors.ENDC)

    TESTS['System File Handle Usage'] = json.dumps(stats, ensure_ascii=False)
    return 0


def get_sysctl_info():
    """
    Collect the sysctl -a output
    :return:
    """
    hc_log.info("Collecting sysctl -a output...")
    stats = {}
    result = system_call("/sbin/sysctl -a")
    if result == '':
        hc_log.info(bcolors.FAIL + "  Unable to collect sysctl -a output" + bcolors.ENDC)
        return 1
    else:
        rows = result.splitlines()
        for row in rows:
            keypair = row.split('=')
            stats[keypair[0]] = keypair[1]

    TESTS['sysctl -a'] = json.dumps(stats, ensure_ascii=False)
    return 0


def get_lsmod_info():
    """
    Collect lsmod output
    :return:
    """
    hc_log.info("Collecting lsmod output...")
    stats = {}
    result = system_call("/sbin/lsmod")
    if result == '':
        hc_log.info(bcolors.FAIL + "  Unable to collect lsmod output" + bcolors.ENDC)
        return 1
    else:
        rows = result.splitlines()
        for row in rows:
            keypair = row.split()
            stats['Module'] = keypair[0]
            stats['Size'] = keypair[1]
            stats['Used by'] = keypair[2]

    TESTS['lsmod'] = json.dumps(stats, ensure_ascii=False)
    return 0


def check_sar_cpu(files):
    """
    Check the CPU usage statistics within the list of sar files provided
    :param files: list of sar files to check
    :return:
    """
    hc_log.info("Checking CPU averages from most recent SAR files...")
    stats = {}
    for f in files:
        hc_log.info("  Executing /usr/bin/sar -P ALL -f {0} | /bin/grep Average: ...".format(f))
        result = system_call("/usr/bin/sar -P ALL -f {0} | /bin/grep Average:".format(f))
        file_date = SARS[f]
        stats[file_date] = {}
        if result != '':
            # 12:00:04 AM     CPU     %user     %nice   %system   %iowait    %steal     %idle
            rows = result.splitlines()
            for row in rows:
                cols = row.split()
                stats[file_date][cols[1]] = {}
                stats[file_date][cols[1]]['User'] = cols[2]
                stats[file_date][cols[1]]['Nice'] = cols[3]
                stats[file_date][cols[1]]['System'] = cols[4]
                stats[file_date][cols[1]]['IO Wait'] = cols[5]
                stats[file_date][cols[1]]['Steal'] = cols[6]
                stats[file_date][cols[1]]['Idle'] = cols[7]
                if float(cols[2]) > 30:
                    sa_dict['Alerts'].append(
                        "User space CPU core {0} usage exceeded 30% on {1}. [Value: {2}]".format(cols[1], f,
                                                                                                 cols[2]))
                if float(cols[4]) > 30:
                    sa_dict['Alerts'].append(
                        "System CPU core {0} usage exceeded 30% on {1}. [Value {2}]".format(cols[1], f, cols[4]))
                if float(cols[5]) > 30:
                    sa_dict['Alerts'].append(
                        "IO wait on CPU core {0} exceeded 30% on {1}. [Value {2}]".format(cols[1], f, cols[5]))
                if float(cols[7]) < 60:
                    sa_dict['Alerts'].append(
                        "Idle time on CPU core {0} dropped below 60% on {1}. [Value {2}]".format(cols[1], f,
                                                                                                 cols[7]))
        else:
            hc_log.info(bcolors.FAIL
                        + "    Unable to collect CPU core averages from SAR files. [Result] {0}".format(result)
                        + bcolors.ENDC)
        TESTS['CPU Averages'] = json.dumps(stats, ensure_ascii=False)
    return ()


def check_sar_load(files):
    """
    Check the system load averages within the list of sar files passed
    :param files: list of sar files
    :return:
    """
    hc_log.info("Checking system load averages from most recent SAR files...")
    stats = {}
    for f in files:
        hc_log.info("  Executing /usr/bin/sar -q -f {0} | /bin/grep Average: ...".format(f))
        result = system_call("/usr/bin/sar -q -f {0} | /bin/grep Average:".format(f))
        file_date = SARS[f]
        stats[file_date] = {}
        if result != '':
            # Average:      runq-sz  plist-sz   ldavg-1   ldavg-5  ldavg-15
            cols = result.split()
            stats[file_date]['Task Queue Length'] = cols[1]
            stats[file_date]['Number if Tasks'] = cols[2]
            stats[file_date]['1 Minute Avg'] = cols[3]
            stats[file_date]['5 Minute Avg'] = cols[4]
            stats[file_date]['15 Minute Avg'] = cols[5]
            if float(cols[3]) > 9:
                sa_dict['Alerts'].append("1 minute load average of {0:f}% exceeded 9 on {1}.".format(cols[3], f))
            if float(cols[4]) > 8:
                sa_dict['Alerts'].append("5 minute load average of {0:f}% exceeded 8 on {1}.".format(cols[4], f))
            if float(cols[5]) > 6:
                sa_dict['Alerts'].append("15 minute load average of {0:f}% exceeded 6 on {1}.".format(cols[5], f))
        else:
            hc_log.info(bcolors.FAIL +
                        "    Unable to collect memory usage averages from SAR files. [Result] {0}".format(result) +
                        bcolors.ENDC)
        TESTS['Load Averages'] = json.dumps(stats, ensure_ascii=False)
    return ()


def check_sar_swap(files):
    """
    Check the swap usage values within the list of sar files provided
    :param files: list of sar files
    :return:
    """
    hc_log.info("Checking swap usage averages from most recent SAR files...")
    stats = {}
    for f in files:
        hc_log.info("  Executing /usr/bin/sar -S -f {0} | /bin/grep Average: ...".format(f))
        result = system_call("/usr/bin/sar -S -f {0} | /bin/grep Average:".format(f))
        file_date = SARS[f]
        stats[file_date] = {}
        if result != '':
            # Average:    kbswpfree kbswpused  %swpused  kbswpcad   %swpcad
            cols = result.split()
            stats[file_date]['KB Free'] = cols[1]
            stats[file_date]['KB Used'] = cols[2]
            stats[file_date]['Percent Used'] = cols[3]
            stats[file_date]['KB Cache'] = cols[4]
            stats[file_date]['Percent Cache'] = cols[5]
            if float(cols[3]) > 70:
                sa_dict['Alerts'].append("Percent swap usage of {0:f}% exceeded 70% on {1}.".format(cols[3], f))
        else:
            hc_log.info(bcolors.FAIL +
                        "    Unable to collect memory usage averages from SAR files. [Result] {0}".format(result) +
                        bcolors.ENDC)
        TESTS['Swap Averages'] = json.dumps(stats, ensure_ascii=False)
    return ()


def check_sar_memory(files):
    """
    Check the memory usage statistics within the provided list of sar files
    :param files: list of sar files
    :return:
    """
    hc_log.info("Checking memory averages from most recent SAR files...")
    stats = {}
    for f in files:
        hc_log.info("  Executing /usr/bin/sar -r -f {0} | /bin/grep Average: ...".format(f))
        result = system_call("/usr/bin/sar -r -f {0} | /bin/grep Average:".format(f))
        file_date = SARS[f]
        stats[file_date] = {}
        if result != '':
            # Average:    kbmemfree kbmemused  %memused kbbuffers  kbcached  kbcommit   %commit
            cols = result.split()
            stats[file_date]['KB Free'] = cols[1]
            stats[file_date]['KB Used'] = cols[2]
            stats[file_date]['Percent Used'] = cols[3]
            stats[file_date]['KB Buffers'] = cols[4]
            stats[file_date]['KB Cached'] = cols[5]
            stats[file_date]['KB Commit'] = cols[6]
            stats[file_date]['Percent Commit'] = cols[7]
            if float(cols[3]) > 70:
                sa_dict['Alerts'].append(
                    "Percent memory usage of {0} percent exceeded 70% on {1}.".format(cols[3], f))
        else:
            hc_log.info(bcolors.FAIL +
                        "    Unable to collect memory usage averages from SAR files. [Result] {0}".format(result) +
                        bcolors.ENDC)
        TESTS['Memory Averages'] = json.dumps(stats, ensure_ascii=False)
    return ()


def check_sar_device(files):
    """
    Check the hard drive device statistics within each sar file in the provided list
    :param files: list of sar files
    :return:
    """
    hc_log.info("Checking device averages from most recent SAR files...")
    stats = {}
    for f in files:
        hc_log.info("  Executing /usr/bin/sar -d -p -f {0} | /bin/grep Average: ...".format(f))
        result = system_call("/usr/bin/sar -d -p -f {0} | /bin/grep Average:".format(f))
        file_date = SARS[f]
        stats[file_date] = {}
        if result != '':
            rows = result.splitlines()
            for row in rows:
                # 12:00:01 AM       DEV       tps  rd_sec/s  wr_sec/s  avgrq-sz  avgqu-sz     await     svctm     %util
                cols = row.split()
                stats[file_date][cols[1]] = {}
                stats[file_date][cols[1]]['TPS'] = cols[2]
                stats[file_date][cols[1]]['RdPS'] = cols[3]
                stats[file_date][cols[1]]['WrPS'] = cols[4]
                stats[file_date][cols[1]]['Average Request Size'] = cols[5]
                stats[file_date][cols[1]]['Average Q Length'] = cols[6]
                stats[file_date][cols[1]]['Average Time'] = cols[7]
                stats[file_date][cols[1]]['Average Service Time'] = cols[8]
                stats[file_date][cols[1]]['Percent Utilization'] = cols[9]
                if float(cols[9]) > 70:
                    sa_dict['Alerts'].append(
                        "Percent device CPU usage of {0:f}% exceeded 70% on {1}.".format(cols[9], cols[1]))
        else:
            hc_log.info(bcolors.FAIL +
                        "    Unable to collect device usage averages from SAR files. [Result] {0}".format(result) +
                        bcolors.ENDC)
        TESTS['Device Averages'] = json.dumps(stats, ensure_ascii=False)
    return ()


def check_heartbeat_messages():
    """
    Scan the heartbeat logs for error messages
    :return:
    """
    sa_dict['Heartbeat Messages'] = []
    hc_log.info("Checking if /var/log/hb.log exists...")
    if os.path.isfile("/var/log/hb.log"):
        hc_log.info("  hb.log file exists.")
    else:
        hc_log.warning("  hb.log file does not exist. Skipping message check.")
        return ()
    hc_log.info("Checking if /var/log/hb.log is empty...")
    if os.stat("/var/log/hb.log").st_size == 0:
        hc_log.info("  /var/log/hb.log is empty!")
        sa_dict['Alerts'].append(
            "/var/log/hb.log file is empty. Please investigate. A restart of the heartbeat process may be necessary.")
    else:
        hc_log.info("Checking for heartbeat messages in /var/log/hb.log...")
        result = system_call("/bin/grep -iE 'WARN:|CRIT:|ERROR:' /var/log/hb.log")
        if not result:
            hc_log.info(bcolors.OKGREEN + "  No heartbeat errors found in /var/log/hb.log" + bcolors.ENDC)
            sa_dict['Heartbeat Messages'].append('None in /var/log/hb.log')
        else:
            sa_dict['Alerts'].append(
                "Error messages found in /var/log/hb.log. Please investigate the 'Heartbeat Messages' " +
                "test result data within the healthcheck.sqlite file.")
            error_msgs = result.splitlines()
            error_count = len(error_msgs)
            sa_dict['Heartbeat Messages'].append(error_msgs)
            hc_log.info(bcolors.FAIL +
                        "  Heartbeat errors ({0}) found in /var/log/hb.log".format(error_count) + bcolors.ENDC)

    hc_log.info("Checking for heartbeat messages in /var/log/heartbeat-debug.log...")
    result = system_call("/bin/grep -iE 'WARN:|CRIT:|ERROR:' /var/log/heartbeat-debug.log")
    if not result:
        hc_log.info(bcolors.OKGREEN +
                    "  No heartbeat errors found in /var/log/heartbeat-debug.log" + bcolors.ENDC)
        sa_dict['Heartbeat Messages'].append('None in /var/log/heartbeat-debug.log')
    else:
        sa_dict['Alerts'].append(
            "Error messages found in /var/log/heartbeart-debug.log. Please investigate the " +
            "'Heartbeat Messages' test result data within sqlite file.")
        error_msgs = result.splitlines()
        error_count = len(error_msgs)
        sa_dict['Heartbeat Messages'].append(error_msgs)
        hc_log.info(bcolors.FAIL +
                    "  Heartbeat errors ({0})found in /var/log/heartbeat-debug.log".format(error_count) + bcolors.ENDC)

    TESTS["Heartbeat Messages"] = json.dumps(sa_dict['Heartbeat Messages'], ensure_ascii=False)
    return 0


def check_rsync_messages():
    """
    Scan /var/log/messages for rsync errors
    :return:
    """
    sa_dict['Rsync Messages'] = []
    hc_log.info("Checking for RSYNC messages in /var/log/message...")
    result = system_call("/bin/grep -i -e rsync /var/log/messages* | grep -i -e unexpected")
    if not result:
        hc_log.info(bcolors.OKGREEN + "  No RSYNC errors found" + bcolors.ENDC)
        sa_dict['Rsync Messages'].append('None')
    else:
        sa_dict['Alerts'].append(
            "RSYNC messages found in /var/log/messages. Please investigate the 'Rsync Messages' " +
            "test result data within sqlite file.")
        error_msgs = result.splitlines()
        error_count = len(error_msgs)
        sa_dict['Rsync Messages'].append(error_msgs)
        hc_log.info(bcolors.FAIL + "  RSYNC errors ({0}) found".format(error_count) + bcolors.ENDC)
    TESTS["Rsync Messages"] = json.dumps(sa_dict['Rsync Messages'], ensure_ascii=False)
    return 0


def check_system_messages():
    """
    Scan /var/log/messages for general errors
    :return:
    """
    sa_dict['System Messages'] = []
    hc_log.info("Checking for failure messages in /var/log/message...")
    result = system_call("/bin/grep -iE 'alert|fail|denied|segfault|segmentation|rejected|oops|warn' /var/log/messages")
    if not result:
        hc_log.info(bcolors.OKGREEN + "  No system errors found" + bcolors.ENDC)
        sa_dict['System Messages'].append('None')
    else:
        sa_dict['Alerts'].append("SYSTEM messages found in /var/log/messages. Please investigate the System Messages "
                                 + "test result data within sqlite file.")
        error_msgs = result.splitlines()
        error_count = len(error_msgs)
        sa_dict['System Messages'].append(error_msgs)
        hc_log.info(bcolors.FAIL + "  System errors ({0}) found".format(error_count) + bcolors.ENDC)
    TESTS["System Messages"] = json.dumps(sa_dict['System Messages'], ensure_ascii=False)
    return 0


def check_idb_log_messages(file_list):
    """
    Scan the list of ScaleArc logs provided for errors
    :param file_list: ScaleArc log file list
    :return:
    """
    sa_dict.setdefault('ScaleArc Log Messages', {})
    hc_log.info("Checking for error messages in idb log files...")
    FOUND = False
    for f in file_list:
        hc_log.info("  processing {0}...".format(f))
        FOUND = False
        hc_log.info("    Executing /bin/grep -iE 'error|authentication failure|not support' {0}...".format(f))
        result = system_call("/bin/grep -iE 'error|authentication failure|not support' {0}".format(f))
        if result != '':
            FOUND = True
            sa_dict['ScaleArc Log Messages'].setdefault(f, dict())
            hc_log.warning(
                bcolors.WARNING
                + "  {0} errors found in {1}".format(result.count('\n'), f)
                + bcolors.ENDC
            )
            result_msgs = result.splitlines()
            delimiter = ', '
            field_number = 4
            for result_msg in result_msgs:
                try:
                    if '#!#' in result_msg:
                        delimiter = '#!#'
                        field_number = 6
                    fields = result_msg.split(delimiter)
                    # normalized_msg = " ".join(fields[1:])
                    normalized_msg = fields[field_number]
                    normalized_msg = normalized_msg.replace("'", "")
                    normalized_msg = re.sub('\( \d+ / \d+ \) : Reason', ': Reason', normalized_msg)
                    sa_dict['ScaleArc Log Messages'][f][normalized_msg] = sa_dict['ScaleArc Log Messages'][
                                                                              f].setdefault(normalized_msg, 0) + 1
                except Exception as e:
                    hc_log.error(bcolors.FAIL
                                 + "Exception caught while parsing log file record ({0}".format(result_msg)
                                 + bcolors.ENDC)
                    hc_log.warning(bcolors.WARNING + str(e) + bcolors.ENDC)
        else:
            hc_log.info(bcolors.OKGREEN + "  {0} errors found in {1}".format(len(result), f) + bcolors.ENDC)
            FOUND = False

    if FOUND:
        sa_dict['Alerts'].append("Messages found in ScaleArc log files. Please investigate the 'ScaleArc Log Messages' "
                                 + "test result data within sqlite file.")
    try:
        TESTS["ScaleArc Log Messages"] = json.dumps(sa_dict['ScaleArc Log Messages'], ensure_ascii=False)
    except Exception as e:
        hc_log.info(bcolors.FAIL
                    + "  Exception caught loading ScaleArc Log Messages into TESTS dictionary! "
                    + "Messages have not been stored!" + bcolors.ENDC)
        hc_log.info(str(e))
    return 0


def check_data_partition():
    '''
    Gather information for the /data filesystem
    :return:
    '''
    hc_log.info("Checking for /data partition size and mount point...")
    hc_log.info("  Executing /bin/df -P --block-size=1 /data ...")
    result = system_call("/bin/df -P --block-size=1 /data")
    hc_log.info("    result: {0}".format(result))
    lines = result.splitlines()
    data_volume = {}
    try:
        cols = lines[1].split()
        sa_dict['Data Device'] = cols[0]
        data_size = cols[1]
        sa_dict['Data Size'] = data_size
        sa_dict['Data Used'] = cols[2]
        sa_dict['Data Available'] = cols[3]
        percent_used = cols[4]
        data_mount = cols[5]
        sa_dict['Data Mount'] = data_mount
        data_volume['Device'] = sa_dict['Data Device']
        data_volume['Size'] = sa_dict['Data Size']
        data_volume['Used'] = sa_dict['Data Used']
        data_volume['Available'] = sa_dict['Data Available']
        data_volume['Mount Point'] = sa_dict['Data Mount']
        if data_mount == '/':
            hc_log.info(
                bcolors.WARNING + "  [WARNING] /data has not been allocated its own partition. " +
                "Please refer to https://support.scalearc.com/kb/articles/1651-info for recommended " +
                "system configuration. " + bcolors.ENDC)
            sa_dict['Alerts'].append(
                "[WARNING] /data has not been allocated its own partition. Please refer to " +
                "https://support.scalearc.com/kb/articles/1651-info for recommended system configuration. ")

        if data_size < 500000000000:
            hc_log.info(
                bcolors.WARNING + "  [WARNING] /data is less than 500GB in size. Please refer to " +
                "https://support.scalearc.com/kb/articles/1651-info for proper sizing." + bcolors.ENDC)
            sa_dict['Alerts'].append(
                "[WARNING] /data is less than 200GB in size. Please refer to " +
                "https://support.scalearc.com/kb/articles/1651-info for proper sizing.")
        percent_used = int(percent_used.strip(' \t\n\r%'))
        if percent_used > 85:
            hc_log.info(
                bcolors.WARNING + "  [WARNING] /data is over 85% used. Additional log space should be " +
                "allocated." + bcolors.ENDC)
            sa_dict['Alerts'].append("[WARNING] /data is over 85% used. Additional log space should be allocated.")

    except Exception as e:
        hc_log.info(bcolors.FAIL + "  Unable to parse result. [{0}]".format(str(e)) + bcolors.ENDC)
        hc_log.info(bcolors.WARNING + "  Skipping /data volume analysis. Please check manually." + bcolors.ENDC)
        sa_dict['Alerts'].append("  Skipping /data volume analysis. Please check manually.")

    TESTS["Data Volume"] = json.dumps(data_volume, ensure_ascii=False)
    return 0


def get_storage_devices():
    '''
    Collect information about storage devices
    :return:
    '''
    hc_log.info("Gathering storage device information...")
    hc_log.info("  Executing /bin/df -PhT | /bin/grep -v tmpfs | /bin/grep dev | /bin/awk \'{print $1, $2, $7;}\'")
    result = system_call("/bin/df -PhT | /bin/grep -v tmpfs | /bin/grep dev | /bin/awk \'{print $1, $2, $7;}\'")
    hc_log.info("     result: {0}".format(result))
    sa_dict['Storage Devices'] = {}
    try:
        results = result.splitlines()
        results = results[1:]
        for r in results:
            device, fstype, mount_point = r.split()
            sa_dict['Storage Devices'][device] = {'Type': fstype, 'Mount Point': mount_point}
    except Exception as e:
        hc_log.info(bcolors.FAIL +
                    "  Unable to parse output of /bin/df -PhT | /bin/grep -v tmpfs | /bin/grep dev | " +
                    "/bin/awk \'{print $1, $2, $7;}\'. [{0}]".format(e) + bcolors.ENDC)
        hc_log.info(bcolors.WARNING + "  Skipping storage device analysis. Please check manually." + bcolors.ENDC)
        sa_dict['Alerts'].append("  Skipping storage device analysis. Please check manually.")

    TESTS["Storage Devices"] = json.dumps(sa_dict['Storage Devices'], ensure_ascii=False)
    return 0


def get_network_interrupts():
    '''
    Gather network interrupt CPU affinity information
    :return:
    '''
    hc_log.info("Gathering network interrupts...")
    hc_log.info("/bin/cat /proc/interrupts | /bin/grep -e eth | /bin/awk \'{$NF=$(NF-1)=\"\"; print $0}\'")
    result = system_call("/bin/cat /proc/interrupts | /bin/grep -e eth | /bin/awk \'{$NF=$(NF-1)=\"\"; print $0}\'")
    if not result:
        hc_log.info(bcolors.FAIL + "  Unable to determine CPU mapping for network interrupts" + bcolors.ENDC)
        sa_dict['Network Interrupts'] = {'irq': 'Error'}
    else:
        hc_log.info(r'    Output of /bin/cat /proc/interrupts | /bin/grep -e eth | /bin/awk \'{$NF=$(NF-1)=""; ' +
                    "print $0}}\'\n {0}".format(result))
        fields = result.split(':')
        temp_dict = {'irq': fields[0]}
        irq_cpus = fields[1].rstrip()
        core_count = 0
        for cpu in irq_cpus.split():
            cpu_string = "CPU" + str(core_count)
            temp_dict[cpu_string] = cpu
            core_count += 1
        sa_dict['Network Interrupts'] = temp_dict
    TESTS["Network Interrupts"] = json.dumps(sa_dict['Network Interrupts'], ensure_ascii=False)
    return 0


def check_email_alerts():
    '''
    Check if the email alerts feature is configured for each defined cluster
    :return:
    '''
    hc_log.info("Checking email alerts configurations within defined clusters...")

    hc_log.info("  Connecting to SQLite file /system/lb.sqlite")
    conn = sqlite.connect('/system/lb.sqlite')
    conn.text_factory = str
    c = conn.cursor()
    # cluster_id|type|status|config_path
    # 1|MYSQL|1|/system/lb_1.sqlite
    hc_log.info("    Executing 'SELECT config_path from lb_clusters_summary WHERE status == 1")
    try:
        c.execute("SELECT config_path FROM lb_clusters_summary WHERE status == 1")
        rows = c.fetchall()
        conn.close()
    except Exception as e:
        hc_log.info(
            bcolors.FAIL + "    Unable to retrieve active cluster information from /system/lb.sqlite [{0}]".format(
                str(e)) + bcolors.ENDC)
        return 1
    hc_log.info("    RESULT: {0}".format(rows))
    email_alerts = {}
    if rows:
        for lb_cluster_file in rows:
            hc_log.info("    lb_cluster file is: {0}".format(lb_cluster_file[0]))
            conn2 = sqlite.connect(lb_cluster_file[0])
            conn2.text_factory = str
            c2 = conn2.cursor()
            hc_log.info("    SELECT clustername FROM lb_clusters")
            c2.execute("SELECT clustername FROM lb_clusters")
            row = c2.fetchone()
            if row is None:
                hc_log.info(bcolors.WARNING + "      Unable to retrieve cluster name." + bcolors.ENDC)
                continue
            else:
                _cluster_name = row[0]
                hc_log.info("      Cluster name: {0}".format(_cluster_name))
                email_alerts[_cluster_name] = {}
            hc_log.info("    Executing 'SELECT emailids FROM lb_sendalert WHERE status = 1'")
            c2.execute("SELECT emailids FROM lb_sendalert WHERE status = 1")
            row = c2.fetchone()
            if row is None:
                hc_log.info("    Unable to retrieve alert email addresses from {0}.".format(lb_cluster_file))
                if DEBUG:
                    hc_log.info("Setting Email Alerts to " + bcolors.OKBLUE + "False" + bcolors.ENDC)
                sa_dict['Cluster Configurations'][_cluster_name]['Email Alerts'] = False
                email_alerts[_cluster_name]['Email Alerts'] = False
            else:
                hc_log.info("      [OUTPUT] {0}".format(row))
                hc_log.info("    Setting Email Alerts to True.")
                sa_dict['Cluster Configurations'][_cluster_name]['Email Alerts'] = True
                sa_dict['Cluster Configurations'][_cluster_name]['Email Alerts IDs'] = list(row)
                email_alerts[_cluster_name]['Email Alerts'] = True
                email_alerts[_cluster_name]['Email Alerts IDs'] = list(row)
                hc_log.info("  {0} added to email alerts IDs list.".format(row))
            conn2.close()
    else:
        hc_log.info(
            "[/system/lb.sqlite] SELECT config_path from lb_clusters_summary WHERE status != 9 returned no results.")
        hc_log.info(bcolors.WARNING + "No active clusters found in /system/lb.sqlite file." + bcolors.ENDC)
    TESTS["Email Alerts"] = json.dumps(email_alerts, ensure_ascii=False)
    return 0


def check_ntp():
    '''
    Collect NTP configuration and verify that the process is running with the ScaleArc defined configuration
    :return:
    '''
    sa_dict['NTP Servers'] = []
    hc_log.info("Checking NTP configuration...")
    if os.path.isfile('/etc/ntp_scalearc.conf') is True:
        hc_log.info("  Executing /bin/grep '^server' /etc/ntp_scalearc.conf ...")
        results = system_call("/bin/grep '^server' /etc/ntp_scalearc.conf")
        if results != '':
            hc_log.info("  NTP Server(s): {0}".format(results))
            results = results.splitlines()
            for result in results:
                aa = re.search(r'^server (.*?)\s', result)
                if aa:
                    ntp_server = aa.group(1)
                    hc_log.info("    server extract: {0}".format(ntp_server))
                    sa_dict['NTP Servers'].append(ntp_server)
                else:
                    hc_log.info(bcolors.FAIL + "    failied to extract ntp server from string." + bcolors.ENDC)
        else:
            hc_log.info("  No NTP servers found in /etc/ntp_scalearc.conf")
            sa_dict['NTP Servers'].append('None')
            sa_dict['Alerts'].append("No NTP servers configured.")
    else:
        hc_log.info("  No ScaleArc NTP configuration file found.")
        sa_dict['NTP Servers'].append('None')
        sa_dict['Alerts'].append("No ScaleArc NTP configuration file could be found (/etc/ntp_scalearc.conf).")
        return 1

    # TODO: Check if NTP process with ScaleArc configuration file is running
    hc_log.info("  /sbin/pidof ntpd")
    ntpd_pid = system_call("/sbin/pidof ntpd")
    if ntpd_pid == '':
        hc_log.info(bcolors.WARNING + "  NTPD is not running." + bcolors.ENDC)
        sa_dict['Alerts'].append("NTPD is not running.")
    else:
        ntpd_pid.replace("\n", "")
        hc_log.info(
            bcolors.OKGREEN + "  NTPD is running. Executing /bin/ps --no-headers {0}".format(ntpd_pid) + bcolors.ENDC)
        ps_result = system_call("/bin/ps --no-headers {0}".format(ntpd_pid))
        hc_log.info("    result: {0}".format(ps_result.replace("\n", "")))
        hc_log.info("  Executing /usr/sbin/ntpq -p to determine sync state.")
        ntpd_sync_state = system_call("/usr/sbin/ntpq -p")
        if ntpd_sync_state != '':
            hc_log.info("    ntpd_sync_state: {0}".format(ntpd_sync_state.replace("\n", "")))
        else:
            hc_log.info(bcolors.FAIL + "  Failed to retrieve NTP client sync information." + bcolors.ENDC)
            sa_dict['Alerts'].append(
                "Failed to retrieve NTP client sync information. Please check NTP is configured properly and running.")

    # TODO: If HA node exists check time difference between nodes

    TESTS["NTP Servers"] = json.dumps(sa_dict['NTP Servers'], ensure_ascii=False)
    return 0


def get_heartbeat_config():
    '''
    Collect heartbeart configuration parameters
    :return:
    '''
    hc_log.info("Checking HA configuration parameters...")
    ha = {}

    def _convert_unit(value):
        unit_search = re.search('(\d+)ms', value, re.IGNORECASE)
        if unit_search:
            value = unit_search.group(1)
        else:
            value = int(value) * 1000
        return value

    hc_log.info('  /bin/grep keepalive /etc/ha.d/ha.cf')
    result = system_call('/bin/grep keepalive /etc/ha.d/ha.cf')
    if result != '':
        cols = result.split()
        ha_keepalive = cols[1]
        ha_keepalive = _convert_unit(ha_keepalive)
        sa_dict['HA Keep Alive'] = ha_keepalive
        ha['Keep Alive'] = ha_keepalive
    else:
        hc_log.info(bcolors.WARNING + "  Keepalive setting missing from /etc/ha.d/ha.cf" + bcolors.ENDC)

    hc_log.info('  /bin/grep deadtime /etc/ha.d/ha.cf')
    result = system_call('/bin/grep deadtime /etc/ha.d/ha.cf')
    if result != '':
        cols = result.split()
        ha_deadtime = cols[1]
        # Convery to ms if no unit defined
        ha_deadtime = _convert_unit(ha_deadtime)
        sa_dict['HA Dead Time'] = ha_deadtime
        ha['Dead Time'] = ha_deadtime
    else:
        hc_log.info(bcolors.WARNING + "    Deadtime setting missing from /etc/ha.d/ha.cf" + bcolors.ENDC)

    hc_log.info('  /bin/grep warntime /etc/ha.d/ha.cf')
    result = system_call('/bin/grep warntime /etc/ha.d/ha.cf')
    if result != '':
        cols = result.split()
        ha_warntime = cols[1]
        ha_warntime = _convert_unit(ha_warntime)
        sa_dict['HA Warn Time'] = ha_warntime
        ha['Warn Time'] = ha_warntime
    else:
        hc_log.info(bcolors.WARNING + "    Warntime setting missing from /etc/ha.d/ha.cf" + bcolors.ENDC)

    hc_log.info('  /bin/grep initdead /etc/ha.d/ha.cf')
    result = system_call('/bin/grep initdead /etc/ha.d/ha.cf')
    if result != '':
        cols = result.split()
        ha_initdead = cols[1]
        ha_initdead = _convert_unit(ha_initdead)
        sa_dict['HA Init Dead Time'] = ha_initdead
        ha['Init Dead Time'] = ha_initdead
    else:
        hc_log.info(bcolors.WARNING + "Initdeadtime setting missing from /etc/ha.d/ha.cf" + bcolors.ENDC)
    TESTS["HA"] = json.dumps(ha, ensure_ascii=False)
    return 0


def check_memory():
    '''
    Gather system memory information
    :return:
    '''
    hc_log.info("Gathering system memory information...")
    hc_log.info("  Executing /usr/bin/free -o -b ...")
    _result = system_call('/usr/bin/free -o -b')
    hc_log.info("    result: {0}".format(_result))
    memory = {}
    if _result != '':
        try:
            _lines = _result.splitlines()
            _cols = _lines[1].split()
            sa_dict['System Memory'] = _cols[1]
            sa_dict['System Memory Used'] = _cols[2]
            sa_dict['System Memory Free'] = _cols[3]
            _sys_mem_percent_used = int(_cols[2]) / int(_cols[1])
            if _sys_mem_percent_used > 65:
                sa_dict['Alerts'].append("System memory is over 65% used. Please consider allocating more memory.")

            _cols = _lines[2].split()
            # _swap_mem = _cols[1]
            sa_dict['Swap Memory'] = _cols[1]
            sa_dict['Swap Memory Used'] = _cols[2]
            sa_dict['Swap Memory Free'] = _cols[3]
            _swap_mem_percent_used = int(_cols[2]) / int(_cols[1])
            if _swap_mem_percent_used > 65:
                sa_dict['Alerts'].append("Swap space is over 65% used. Please consider resizing.")

        except Exception as e:
            hc_log.info(bcolors.FAIL + "Error collecting system memory [{0}].".format(e) + bcolors.ENDC)
            sa_dict['System Memory'] = "Unknown"
            sa_dict['Swap Memory'] = "Unknown"
        memory['System'] = sa_dict['System Memory']
        memory['Swap'] = sa_dict['Swap Memory']

    TESTS["System Memory"] = json.dumps(memory, ensure_ascii=False)
    return 0


def get_swap():
    '''
    Collect system swap information
    :return:
    '''
    # TODO: Separate swap information collection from swap size recommendation
    hc_log.info("Gathering system swap space information...")
    hc_log.info('  Executing /bin/cat /proc/swaps ...')
    _result = system_call('/bin/cat /proc/swaps')
    hc_log.info("    result: {0}".format(_result))
    if _result != '':
        try:
            _lines = _result.splitlines()
            _cols = _lines[1].split()
            sa_dict['Swap Device'] = _cols[0]
            sa_dict['Swap Type'] = _cols[1]
            sa_dict['Swap Size'] = int(_cols[2]) * 1024
            sa_dict['Swap Used'] = _cols[3]

        except Exception as e:
            hc_log.info(bcolors.FAIL + "    Error collecting swap information. [{0}]".format(e) + bcolors.ENDC)
            sa_dict['Swap Device'] = "None"
            sa_dict['Swap Type'] = "None"
            sa_dict['Swap Size'] = 0
            sa_dict['Swap Used'] = 0
    return 0


def check_swap():
    '''
    Check swap information against recommendations
    :return:
    '''
    hc_log.info("Checking system swap space against recommendations")
    if sa_dict['System Memory'] == 'Unknown':
        hc_log.info(bcolors.WARNING + "System memory size is 'unknown', therefore skipping swap check." + bcolors.ENDC)
        return 0
    rec_swap = 4 * GB
    if int(sa_dict['System Memory']) <= (8 * GB):
        rec_swap = 2 * int(sa_dict['System Memory'])
    elif int(sa_dict['System Memory']) <= (64 * GB):
        rec_swap = 16 * GB
    elif int(sa_dict['System Memory']) > (64 * GB):
        rec_swap = 32 * GB
    # round the swap space value up to the nearest integer before comparison
    if math.ceil(sa_dict['Swap Size']) < rec_swap:
        sa_dict['Alerts'].append(
            "Swap size of {0} is less than the recommended value of {1} for a ScaleArc system with {2} system memory.".format(
                sizeof_format(int(math.ceil(sa_dict['Swap Size']))), sizeof_format(int(rec_swap)),
                sizeof_format(int(sa_dict['System Memory']))))
    swap = {'Device': sa_dict['Swap Device'], 'Type': sa_dict['Swap Type'], 'Size': sa_dict['Swap Size'],
            'Used': sa_dict['Swap Used']}
    TESTS["Swap Space"] = json.dumps(swap, ensure_ascii=False)
    return 0


def get_cpu_usage():
    '''
    Collect CPU usage statistics
    :return:
    '''
    hc_log.info("Gathering CPU usage information...")
    count = 0

    # Get current CPU core usage stats
    iterations = int(CPU_DEFAULTS.get('ITERATIONS', 1))

    if re.match(r"^\d\s*$", str(iterations)):
        hc_log.info("  CPU Core utilization iteration count from config file [{0}] is valid.".format(iterations))
    else:
        hc_log.info(bcolors.OKBLUE +
                    "  CPU Core utilization iteration count from config file [{0}] is not valid. Setting to {1}.".format(
                        iterations, MAX_ITERATIONS) + bcolors.ENDC)
        # interval = MAX_INTERVAL

    if iterations > 5:
        hc_log.info(bcolors.OKBLUE +
                    "CPU core utilization iteration count [{0}] is greater than the maximum allowed value [{1}]. " +
                    "Resetting iteration count to {2}.".format(iterations, MAX_ITERATIONS,
                                                               MAX_ITERATIONS) + bcolors.ENDC)
        iterations = MAX_ITERATIONS

    interval = int(CPU_DEFAULTS.get('INTERVAL', 30))

    if re.match(r"^\d{1,3}$", str(interval)):
        hc_log.info("  CPU Core utilization interval value from config file [{0}] is valid.".format(interval))
    elif re.match(r"^\d+m\s*$", interval):
        hc_log.info(
            "  CPU Core utilization interval value from config file [{0}] is in minutes rather than seconds. " +
            "Converting to seconds.".format(interval))
        interval = int(interval * 60)  # Convert from minutes to seconds
    else:
        hc_log.info(bcolors.OKBLUE +
                    "  CPU Core utilization interval value from config file [{0}] is not valid. Setting to {1}.".format(
                        interval, MAX_INTERVAL) + bcolors.ENDC)
        interval = MAX_INTERVAL

    if interval > 300:
        hc_log.info(bcolors.OKBLUE +
                    "  CPU core utilization interval value [{0}] is greater than the maximum allowed value [{1}]. " +
                    "Resetting interval duration to {2} seconds.".format(interval, MAX_INTERVAL,
                                                                         MAX_INTERVAL) + bcolors.ENDC)
        interval = MAX_INTERVAL

    sa_dict['CPU Core Usage Interval'] = interval
    sa_dict['CPU Core Usage Iterations'] = iterations

    if iterations is 1:
        hc_log.info("  Gathering CPU core utilization stats for {0} iteration.".format(iterations))
    else:
        hc_log.info(
            "  Gathering CPU core utilization stats for {0} iterations with {1} pause between each.".format(iterations,
                                                                                                            interval))

    while iterations > 0:
        hc_log.info("Iteration: {0}".format(iterations))
        sys.stdout.flush()

        hc_log.info('  Executing /usr/bin/mpstat -P ALL | /usr/bin/tail -n +5 | awk \'{print $3\":\"$4;}\' ...')
        _result = system_call('/usr/bin/mpstat -P ALL | /usr/bin/tail -n +5 | awk \'{print $3\":\"$4;}\'')
        hc_log.info("    result: {0}".format(_result))
        if _result != '':
            try:
                result = _result.splitlines()
                for line in result:
                    hc_log.info("    mpstat output line[{0}]: {1}".format(count, line))
                    _cols = line.split(":")
                    if _cols[0] is not "\n":
                        hc_log.info("    mpstat output {0} : {1}".format(_cols[0], _cols[1]))
                    value = _cols[1]
                    sa_dict['CPU Core Usage'].setdefault(_cols[0], []).append(value)
                    if float(value) > 70:
                        sa_dict['Alert'].append(
                            "CPU core {0} usage exceeds 70%. Current usage is {1}%. Please investigate ScaleArc " +
                            "thread distribution.".format(_cols[0], value))
                    count += 1
            except Exception as e:
                hc_log.info(
                    bcolors.FAIL + "Error collecting current CPU core utilization. [{0}]".format(e) + bcolors.ENDC)

        if iterations > 1:
            for i in xrange(int(interval), 0, -1):
                sys.stdout.write("\rIteration: {0:3d} Sleeping for {1:3d} seconds.".format(iterations, i))
                sys.stdout.flush()
                time.sleep(1)
        elif iterations == 1:
            sys.stdout.flush()
        else:
            sys.stdout.write("\rIteration: {0:3d}.".format(iterations))
            sys.stdout.flush()
        iterations = int(iterations) - 1
    return 0


def get_cpu_thread_mapping():
    '''
    Collect the ScaleArc thread to CPU core mapping information
    :return:
    '''
    hc_log.info("Connecting to /system/cpudistribution.sqlite ")
    conn = sqlite.connect('/system/cpudistribution.sqlite')
    conn.text_factory = str
    c = conn.cursor()
    try:
        # clusterid|threadid|cpuid|spawn|updatetime|threadgroup|pid|cloneid|multiply_flag
        hc_log.info("  SELECT * FROM cpu_distribution")
        for row in c.execute("SELECT * FROM cpu_distribution"):
            sa_dict['CPU Core Threads'].append(row)
        conn.close()
    except Exception as e:
        hc_log.info(
            bcolors.FAIL + "Exception caught while collecting ScaleArc thread to CPU core mapping information." +
            str(e) + bcolors.ENDC)
    return 0


def get_heartbeat_cpu():
    '''
    Check which CPU the heartbeat process is running on
    :return:
    '''
    # Get CPU that heartbeat process runs on
    hc_log.info("Getting heartbeat process CPU information...")
    hc_log.info(
        '  Executing /bin/ps -eF | /bin/grep -e \'heartbeat: master control process\' | ' +
        '/bin/grep -v grep | /bin/awk \'{print $4;}\' ...')
    result = system_call(
        '/bin/ps -eF | /bin/grep -e \'heartbeat: master control process\' | /bin/grep -v grep | ' +
        '/bin/awk \'{print $4;}\' ')
    try:
        hc_log.info("    result: {0}".format(result))
        sa_dict['Heartbeat Process CPU'] = result.rstrip()
    except Exception as e:
        hc_log.info(
            bcolors.FAIL + "    Unable to determine heartbeat process information. [{0}]".format(e) + bcolors.ENDC)
        sa_dict['Heartbeat Process CPU'] = "Unknown"

    TESTS['Heartbeat Process CPU'] = json.dumps(sa_dict['Heartbeat Process CPU'], ensure_ascii=False)
    return 0


def get_hw_basics():
    '''
    Get the hardware platform information
    :return:
    '''
    hc_log.info("Collecting hardware information...")
    hc_log.info("  Getting System Manufacturer from dmidecode")
    try:
        system = system_call("/usr/sbin/dmidecode -s system-manufacturer")
        hc_log.info("    output: {0}".format(system))
        system = system.replace("\n", "")
    except Exception as e:
        hc_log.info(bcolors.FAIL + "   Unable to determine system manufacturer. [{0}]".format(e) + bcolors.ENDC)
        system = "Unknown"

    hc_log.info("  Getting Chassis Manufacturer from dmidecode")
    try:
        chassis = system_call("/usr/sbin/dmidecode -s chassis-manufacturer")
        hc_log.info("    output: {0}".format(chassis))
        chassis = chassis.replace("\n", "")
    except Exception as e:
        hc_log.info(bcolors.FAIL + "   Unable to determine chassis manufacturer. [{0}]".format(e) + bcolors.ENDC)
        chassis = "Unknown"

    sa_dict['System Manufacturer'] = system
    sa_dict['Chassis Manufacturer'] = chassis
    sa_dict['Platform'] = system + " (" + chassis + ")"
    platform = {'System': system, 'Chassis': chassis}
    TESTS["Platform Information"] = json.dumps(platform, ensure_ascii=False)
    return 0


def get_idb_basics():
    '''
    Gather ScaleArc software version and license information
    :return:
    '''
    hc_log.info("Gathering ScaleArc software and license information...")
    hc_log.info("  SQLite file is {0}".format(GLOBAL_LB_SQLITE_FILE))
    conn = sqlite.connect(GLOBAL_LB_SQLITE_FILE)
    conn.text_factory = str
    c = conn.cursor()
    hc_log.info(
        "  SELECT rpm_list,status FROM idb_software_version WHERE Id = (SELECT MAX(Id) FROM idb_software_version)")
    c.execute("SELECT rpm_list,status FROM idb_software_version WHERE Id = (SELECT MAX(Id) FROM idb_software_version)")

    idb = {}
    try:
        row = c.fetchone()
        if row != '':
            rpm_list = row[0].split(",")
            package = [s for s in rpm_list if "installer" in s][0]
            # status = row[1]
            _product, _version, _build = package.split("-")
            # _build = re.sub(".x86_64", "", _build)
            sa_dict['Current Production Version'] = _version
        else:
            hc_log.info(bcolors.WARNING + "    Error fetching latest software information from {0}".format(
                GLOBAL_LB_SQLITE_FILE) +
                        bcolors.ENDC)
            sa_dict['Current Production Version'] = "Unknown"
    except Exception as e:
        hc_log.info(bcolors.FAIL +
                    "  [ERROR]: Unable to retrieve ScaleArc Software version from {0}. [{1}]".format(
                        GLOBAL_LB_SQLITE_FILE,
                        e) +
                    bcolors.ENDC)
        sa_dict['Current Production Version'] = "Unknown"

    hc_log.info("    Current Production Version: {0}".format(sa_dict['Current Production Version']))
    hc_log.info("  SELECT maxclusters,maxmemory,status,maxcores,edition,expirydate,database_type FROM lb_license")
    c.execute("SELECT maxclusters,maxmemory,status,maxcores,edition,expirydate,database_type FROM lb_license")
    try:
        row = c.fetchone()
        if row != '':
            # maxclusters = row[0]
            # maxmemory = row[1]
            # status = row[2]
            maxcores = row[3]
            edition = row[4]
            lic_exp_date = row[5]
            db_type = row[6]
            sa_dict['Current Staging Version'] = ''
            sa_dict['Current GA Version'] = DEFAULTS['GA_VERSION']
            sa_dict['License Expiration'] = lic_exp_date
            sa_dict['License Edition'] = edition.upper()
            sa_dict['Licensed CPU Cores'] = maxcores
            sa_dict['Database Type'] = db_type.upper()
            sa_dict['Database Version'] = 'Unknown'

        else:
            hc_log.info(bcolors.WARNING +
                        "  Unable to retrieve ScaleArc License information from {0}".format(GLOBAL_LB_SQLITE_FILE) +
                        bcolors.ENDC)
    except Exception as e:
        hc_log.info(bcolors.FAIL +
                    " [ERROR]: Unable to retrieve ScaleArc license details from {0}. [{1}]".format(
                        GLOBAL_LB_SQLITE_FILE, e) +
                    bcolors.ENDC)
        sa_dict['Current Staging Version'] = ''
        sa_dict['Current GA Version'] = DEFAULTS['GA_VERSION']
        sa_dict['License Expiration'] = "Unknown"
        sa_dict['License Edition'] = "Unknown"
        sa_dict['Licensed CPU Cores'] = "Unknown"
        sa_dict['Database Type'] = "Unknown"
        sa_dict['Database Version'] = "Unknown"
        idb['License Expiration'] = sa_dict['License Expiration']
        idb['License Edition'] = sa_dict['License Edition']
        idb['Licensed CPU Cores'] = sa_dict['Licensed CPU Cores']
        idb['Database Type'] = sa_dict['Database Type']
        idb['Installed Version'] = sa_dict['Current Production Version']
        idb['Database Version'] = sa_dict['Database Version']
        TESTS["ScaleArc Information"] = json.dumps(idb, ensure_ascii=False)
    return 0


def check_idb_basics():
    '''
    Check current ScaleArc software version against current GA version
    :return:
    '''
    hc_log.info("Checking ScaleArc software and license information...")
    try:
        if LooseVersion(sa_dict['Current GA Version']) > LooseVersion(sa_dict['Current Production Version']):
            hc_log.info(bcolors.OKBLUE +
                        "Current ScaleArc GA software {0} is higher than the currently installed {1}. " +
                        "Please consider upgrading.".format(
                            sa_dict['Current GA Version'], sa_dict['Current Production Version']) + bcolors.ENDC)
            sa_dict['Alerts'].append(
                "Current ScaleArc GA software {0} is higher than the currently installed {1}. " +
                "Please consider upgrading.".format(
                    sa_dict['Current GA Version'], sa_dict['Current Production Version']))
    except Exception as e:
        hc_log.info(
            bcolors.FAIL + "Exception caught during ScaleArc software version check. [{0}]".format(e) + bcolors.ENDC)
    return 0


def check_license_expiration():
    '''
    Check if ScaleArc license expiration is within the next three months
    :return:
    '''
    hc_log.info("Checking license expiration...")
    months = {"jan": 1, "feb": 2, "mar": 3, "apr": 4, "may": 5, "jun": 6, "jul": 7, "aug": 8, "sep": 9, "oct": 10,
              "nov": 11, "dec": 12}
    days_per_month = [0, 31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31]
    if sa_dict['License Expiration'] == "Unknown":
        return -1
    mon, day, year = sa_dict['License Expiration'].split('/')
    mon = months[mon.lower()]
    license_expiration = datetime.datetime(int(year), int(mon), int(day))
    today = datetime.datetime.now()
    exp_year = today.year
    exp_mon = int(today.month) + 3
    exp_day = int(today.day)
    if exp_mon > 12:
        exp_mon -= 12
        exp_year += 1
    if exp_day > days_per_month[int(exp_mon)]:
        exp_day = exp_day - days_per_month[int(exp_mon)]
        exp_mon += 1
    try:
        three_mon_rel = datetime.datetime(int(exp_year), int(exp_mon), int(exp_day))
        hc_log.info(bcolors.OKBLUE
                    + "License Expiration Date: {0}, Three Month Delta: {1}".format(license_expiration, three_mon_rel)
                    + bcolors.ENDC)
        if three_mon_rel > license_expiration:
            hc_log.info(bcolors.WARNING + "  License expiration date is within the next three months." + bcolors.ENDC)
            sa_dict['Alerts'].append(
                "ScaleArc license expiration date ({0})is within the next three months.".format(
                    sa_dict['License Expiration']))
    except Exception as e:
        hc_log.error(bcolors.FAIL + "Exception caught while checking license expiration. Skipping..." + bcolors.ENDC)
        hc_log.error(bcolors.FAIL + "Exception {0}".format(str(e))  + bcolors.ENDC)
        hc_log.debug(bcolors.FAIL + "days_per_month[{0}] : {1}".format(int(exp_mon), days_per_month[int(exp_mon)]) + bcolors.ENDC)
        hc_log.debug(bcolors.FAIL + "exp year: {0}, exp month: {1}, exp day: {2}".format(exp_year, exp_mon, exp_day) + bcolors.ENDC) 
    return 0


def get_network_interfaces():
    '''
    Collect list of connected network interfaces. Default to a single list item of 'eth0'
    :return:
    '''
    hc_log.info("Collecting list of attached network interfaces...")
    result = system_call("/bin/grep -iE 'eth|bond|em' /proc/net/dev | /bin/awk '{print $1;}' | /bin/sed 's/://'")
    hc_log.info(
        "  result of /bin/grep -iE 'eth|bond|em' /proc/net/dev | /bin/awk '{{print $1;}}' | /bin/sed 's/://' is {0}".format(
            result))
    interfaces = result.splitlines()
    if len(result) == 0:
        hc_log.info(bcolors.WARNING + "  No network interfaces found. Defaulting to 'eth0'." + bcolors.ENDC)
        interfaces = ['eth0']
    return interfaces


def get_interface_ips(interfaces):
    '''
    Retrieve the IP addresses assigned to each network interface
    :param interfaces: list of network interfaces
    :return:
    '''
    global ips
    global IP_ADDRESSES
    hc_log.info("Collecting primary IP addresses for network interfaces...")
    for interface in interfaces:
        result = system_call(
            "/sbin/ip addr show dev {0} | /bin/grep inet | /bin/grep {0} | /usr/bin/head -1 | ".format(
                interface) + "/bin/awk '{print $2;}'")
        hc_log.info(
            "  result of /sbin/ip addr show dev {0} | /bin/grep inet | /bin/grep {0} | /usr/bin/head -1 | ".format(
                interface) +
            "/bin/awk '{{print $2;}}': {0}".format(result))
        result = result.replace("\n", "")
        if len(result) == 0:
            hc_log.info(bcolors.OKBLUE + "Interface {0} IP address is unassigned".format(interface) + bcolors.ENDC)
            sa_dict['Interface IPs'][interface] = 'Unassigned'
        else:
            sa_dict['Interface IPs'][interface] = result
        IP_ADDRESSES.append(result)
    ips['Interfaces'] = sa_dict['Interface IPs']
    return 0


def get_virtual_ips():
    '''
    Gather informaton for any virtual IP addresses that may be configured
    :return:
    '''
    global ips
    global IP_ADDRESSES
    hc_log.info("Collecting virtual IP addresses...")
    try:
        hc_log.info("  Checking for /etc/ha.d/haresources ...")
        if os.path.isfile('/etc/ha.d/haresources') is True:
            hc_log.info("  Executing /bin/cat /etc/ha.d/haresources ...")
            result = system_call("/bin/cat /etc/ha.d/haresources")
            hc_log.info("    Result: {0}".format(result))
            cols = result.split()
            ip_match = re.compile(r'^(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}/\d{1,2})/(.*)')
            found = False
            for col in cols:
                result = ip_match.match(col)
                if result:
                    found = True
                    sa_dict['Virtual IPs'].setdefault(result.group(2), [])
                    sa_dict['Virtual IPs'][result.group(2)].append(result.group(1))
                    IP_ADDRESSES.append(result.group(1))
            if found is False:
                hc_log.info(bcolors.OKBLUE + "  No virtual IP addresses found." + bcolors.ENDC)
                sa_dict['Virtual IPs']['eth0'] = 'None'
            hc_log.info("  Virtual IP addresses: {0}".format(str(sa_dict['Virtual IPs'])))
        else:
            hc_log.info(bcolors.WARNING + "  /etc/ha.d/haresources not found." + bcolors.ENDC)
    except Exception as e:
        hc_log.info(bcolors.FAIL + "    Unable to gather virtual IP address(es). [{0}]".format(e) + bcolors.ENDC)
        sa_dict['Virtual IPs']['eth0'] = 'Unknown'

    ips['Virtual'] = sa_dict['Virtual IPs']
    return 0


def get_ha_peer_ip():
    '''
    Collect the HA peer node IP address if defined
    :return:
    '''
    global ips
    hc_log.info("  Checking /system/remotersyncip ...")
    if os.path.isfile('/system/remotersyncip') is True:
        hc_log.info("  Executing /bin/cat /system/remotersyncip ...")
        try:
            result = system_call("/bin/cat /system/remotersyncip")
            result = result.replace("\n", "")
            hc_log.info("  HA node IP address: {0}".format(result))
            aa = re.match(r"^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}", result)
            if result == '':
                hc_log.info("  Stand alone device. No HA peer node defined.")
                sa_dict['HA Node IP'] = 'Standalone'
            elif aa:
                sa_dict['HA Node IP'] = result
            else:
                hc_log.info("  HA node IP address ({0}) is invalid.".format(result))
                sa_dict['HA Node IP'] = 'Undetermined'
                sa_dict['Alerts'].append("/system/remotersyncip file exists, however the contents " +
                                         "did not match the expected value. Please investigate for proper HA operation.")
        except Exception as e:
            hc_log.info(bcolors.FAIL +
                        "    Exception caught while dumping /systen/remotersyncip file. [{0}]".format(e) + bcolors.ENDC)
            sa_dict['HA Node IP'] = "Undetermined"
    else:
        sa_dict['HA Node IP'] = 'Standalone'
    ips['HA Node'] = sa_dict['HA Node IP']
    TESTS["IP Information"] = json.dumps(ips, ensure_ascii=False)
    return 0


def check_virtual_netmasks():
    '''
    Verify that the subnet mask configured for each virtual IP address is the same as the
    subnet mask for the primary IP address of the associated network interface
    :return:
    '''
    hc_log.info("Checking virtual IP address subnet masks...")
    interfaces = sa_dict['Interface IPs'].keys()
    for interface in interfaces:
        interface_ip = sa_dict['Interface IPs'][interface]
        if interface_ip == 'None' or interface_ip == 'Unknown':
            continue
        try:
            int_ip, int_mask = interface_ip.split('/')
            hc_log.info("  interface: {0}, ip address: {1}, netmask: {2}".format(interface, int_ip, int_mask))
            if interface in sa_dict['Virtual IPs'].keys():
                virtual_ips = sa_dict['Virtual IPs'][interface]
                for virtual_ip in virtual_ips:
                    virt_ip, virt_mask = virtual_ip.split('/')
                    if int(int_mask) != int(virt_mask):
                        sa_dict['Alerts'].append(
                            "The netmask for virtual IP address {0} on interface {1} ".format(virt_ip, interface) +
                            "does not match the netmask for the primary IP address [{0}].".format(int_ip) +
                            "This configuration will not work properly. Please remove {0} ".format(virt_ip) +
                            "and re-create with the proper netmask [/{0}]".format(int_mask))
            else:
                hc_log.info("  Skipping interface {0}".format(interface))
                continue
        except Exception as e:
            hc_log.info(bcolors.FAIL + "  Exception caught while verifying virtual IP address netmasks." +
                        " [{0}]".format(str(e)) + bcolors.ENDC)
    return 0


def check_dns_entries():
    hc_log.info("Checking forward and reverse DNS for defined IP addresses...")
    try:
        for ip in IP_ADDRESSES:
            ip_search = re.search('(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})/\d', ip, re.IGNORECASE)
            if ip_search:
                ip = ip_search.group(1)
                result = system_call("/usr/bin/host {0}".format(ip))
                ptr_record = re.search('name pointer (.*)$', result, re.IGNORECASE)
                if ptr_record:
                    ptr_record = ptr_record.group(1)
                    result = system_call("/usr/bin/host {0}".format(ptr_record))
                    name_record = re.search('has address (.*)$', result, re.IGNORECASE)
                    if name_record:
                        name_record = name_record.group(1)
                        if name_record == ip:
                            hc_log.info(bcolors.OKGREEN
                                        + "  DNS records for {0} match (A: {1}, PTR: {2})".format(ip, name_record,
                                                                                                  ptr_record)
                                        + bcolors.ENDC)
                        else:
                            hc_log.warning(bcolors.WARNING
                                           + "  DNS records for {0} do not match (A: {1}, PTR: {2})".format(ip,
                                                                                                            name_record,
                                                                                                            ptr_record)
                                           + bcolors.ENDC)
                            sa_dict['Alerts'].append(
                                "DNS records for {0} do not match (A: {1}, PTR: {2})".format(ip, name_record,
                                                                                             ptr_record))
                    else:
                        hc_log.error(
                            bcolors.FAIL + "  No DNS A record found for {0}".format(name_record) + bcolors.ENDC)
                else:
                    hc_log.error(bcolors.FAIL + "  No DNS PTR record found for {0}".format(ip) + bcolors.ENDC)
            else:
                hc_log.error(
                    bcolors.FAIL + "  Unable to extract IP address string from {0}".format(ip) + bcolors.ENDC)
    except Exception as e:
        hc_log.error(
            bcolors.FAIL + " Exception caught while checking DNS entries for configured IP addresses. [{0}]".format(e) +
            bcolors.ENDC)
        sa_dict['Alerts'].append("DNS check for configured IPs skipped due to exception. [{0}]".format(e))
    return 0


def run_recommendations():
    '''
    Compare collected information to recommended values and configuration combinations
    :return:
    '''
    hc_log.info("Running ScaleArc Recommendations engine...")

    # Check licensed core count against hardware count
    hc_log.info("Getting CPU core count...")
    hc_log.info("  Executing /usr/bin/nproc --all ...")
    try:
        core_count = system_call("/usr/bin/nproc --all")
        # core_count = core_count.splitlines()[0]
        core_count = core_count.replace("\n", '')
        hc_log.info("    Result: {0}".format(core_count))
        if (int(core_count) - int(sa_dict['Licensed CPU Cores'])) < 1:
            hc_log.info(bcolors.WARNING +
                        "  Insufficient CPU cores left in reserve for OS. Hardware cores available: {0}, ".format(
                            core_count) +
                        "ScaleArc Licensed cores: {0}".format(sa_dict['Licensed CPU Cores']) + bcolors.ENDC)
            sa_dict['Alerts'].append(
                "Insufficient CPU cores left in reserve for OS. Hardware cores available: {0}, ".format(core_count) +
                "ScaleArc Licensed cores: {0}".format(sa_dict['Licensed CPU Cores']))
    except Exception as e:
        hc_log.info(bcolors.FAIL + "    Unable to determine system CPU core count. [{0}]".format(e) + bcolors.ENDC)
        sa_dict['Alerts'].append(
            "System CPU core count failed. ScaleArc license check for reserved OS capacity skipped.")

    # Check system memory
    try:
        _lic_core = str(sa_dict['Licensed CPU Cores'])
        if _lic_core == 'Unknown':
            raise ValueError('Licensed CPU count unknown')
        _lic_edition = str(sa_dict['License Edition'])
        hc_log.info("_lic_core: {0}, _lic_edition: {1}".format(_lic_core, _lic_edition))
        GB = 1073741824
        core_key = int(_lic_core)
        rec_cores = RECOMMENDATIONS['SYSTEM MEMORY'].keys()
        key_match = False
        while key_match is False:
            hc_log.info("  _lic_core: {0}, core_key: {1}".format(_lic_core, core_key))
            if int(_lic_core) < 4:
                core_key = 'minimum'
            if any(str(core_key) in s for s in rec_cores):
                key_match = True
            elif core_key > 32:
                hc_log.warning(bcolors.WARNING
                               + "  Unable to match ScaleArc CPU core license value "
                               + "[{0}] with a recommendation dictionary key.".format(_lic_core)
                               + bcolors.ENDC)
                hc_log.warning("  Breaking loop")
                break
            else:
                core_key += 1

        minimum_memory = RECOMMENDATIONS['SYSTEM MEMORY'][str(core_key)][_lic_edition]
        minimum_memory = GB * int(minimum_memory)
        # Provide 500MB leeway in memory comparison
        if (float(sa_dict['System Memory']) + float(GB / 2)) < float(minimum_memory):
            hc_log.info(bcolors.WARNING +
                        "  System memory is less than the recommended minimum for currently installed license. " +
                        "The recommended minimum system memory for {0} CPU cores with {1} ScaleArc edition is ".format(
                            _lic_core, _lic_edition) +
                        "{0}GB. The current system memory is {1}.".format(
                            RECOMMENDATIONS['SYSTEM MEMORY'][str(core_key)][_lic_edition],
                            sizeof_format(int(sa_dict['System Memory']))) + bcolors.ENDC)
            sa_dict['Alerts'].append(
                "  System memory is less than the recommended minimum for currently installed license. " +
                "The recommended minimum system memory for {0} CPU cores with {1} ScaleArc edition is ".format(
                    _lic_core, _lic_edition) +
                "{0}GB. The current system memory is {1}.".format(
                    RECOMMENDATIONS['SYSTEM MEMORY'][str(core_key)][_lic_edition],
                    sizeof_format(int(sa_dict['System Memory']))))
    except Exception as e:
        hc_log.info(
            bcolors.FAIL + "  Unable to compare available CPU core count to license authorization. [{0}]".format(e) +
            bcolors.ENDC)
        sa_dict['Alerts'].append("Unable to compare available CPU core count to license authorization. [{0}]".format(e))

    # Check HA Settings
    try:
        for key in ['HA Dead Time', 'HA Init Dead Time', 'HA Keep Alive', 'HA Warn Time']:
            cfg_value = sa_dict[key]
            if cfg_value == 'N/A':
                raise ValueError('HA settings are unknown')
            rec_value = RECOMMENDATIONS['HEARTBEAT'][key.upper()]
            if int(cfg_value) < int(rec_value):
                hc_log.info(bcolors.WARNING + "{0} is set to {1}. The recommended value is {2}.".format(key, cfg_value,
                                                                                                        rec_value) +
                            bcolors.ENDC)
                sa_dict['Alerts'].append(
                    "{0} is set to {1}ms. The recommended value is {2}ms, but should be customized for ".format(key,
                                                                                                                cfg_value,
                                                                                                                rec_value) + "your particular network environment and failover needs.")
    except Exception as e:
        hc_log.info("  Unable to complete HA settings recommendations. [{0}]".format(e))
        sa_dict['Alerts'].append("Unable to complete HA settings recommendations. [{0}]".format(e))

    # Check email alert settings
    try:
        _clusters = sa_dict['Cluster Configurations'].keys()
        for cluster in _clusters:
            alerts = sa_dict['Cluster Configurations'][cluster]['Email Alerts']
            if bool(alerts) is False:
                hc_log.info(bcolors.WARNING + "  Email alerts for cluster {0} is not configured.".format(
                    cluster) + bcolors.ENDC)
                sa_dict['Alerts'].append("Email alerts for cluster {0} is not configured.".format(cluster))
    except Exception as e:
        hc_log.info(
            bcolors.FAIL + "  Unable to complete email alert settings recommendations. [{0}]".format(e) + bcolors.ENDC)
        sa_dict['Alerts'].append("  Unable to complete email alert settings recommendations. [{0}]".format(e))

    # Check Network Interrupt
    interrupts = []
    try:
        cpus = sa_dict['Network Interrupts'].keys()
        for cpu in cpus:
            if str(cpu) == 'irq':
                continue
            value = sa_dict['Network Interrupts'][cpu]
            if int(value) != 0:
                interrupts.append(cpu)
        item_count = len(interrupts)
        if item_count > 1:
            hc_log.info(
                bcolors.OKGREEN + "  Network Interrupts are spread across multiple CPUs. [{0}]".format(interrupts) +
                bcolors.ENDC)
        else:
            hc_log.info(
                bcolors.WARNING + "  Network Interrupts are handled by only one CPU ({0}).".format(interrupts[0]) +
                bcolors.ENDC)
            sa_dict['Alerts'].append(
                "Network Interrupts are handled by only one CPU ({0}). If the network interface supports ".format(
                    interrupts[0]) +
                "interrupt balancing. Please consider using /opt/idb/utils/set_irq_affinity to rebalance IRQ " +
                "processing for eth0.")
    except Exception as e:
        hc_log.info(bcolors.FAIL + " Unable to retreive network interface interrupt list from master internal " +
                    "dictionary. [{0}]".format(e) + bcolors.ENDC)

    # Check CPU Mapping
    try:
        _host_ip = sa_dict['Interface IPs']['eth0']
        _host_ip_bare = re.search('(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})/\d+', _host_ip, re.IGNORECASE)
        if _host_ip_bare:
            _host_ip = _host_ip_bare.group(1)
        _cpu_cores = sa_dict['CPU Core Usage'].keys()
        avgs = []
        for core in sorted(_cpu_cores):
            usages = sa_dict['CPU Core Usage'][core]
            # _interval = dict['CPU Core Usage Interval']
            if len(usages) > 1:
                _iterations = sa_dict['CPU Core Usage Iterations']
            else:
                _iterations = 1

            _peak = float(0.0)
            _avg = float(0.0)
            for usage in usages:
                if usage > _peak:
                    _peak = usage
                _avg += float(usage)
            _avg = _avg / _iterations
            avgs.append(_avg)
            if float(_peak) > int(80):
                hc_log.info(bcolors.FAIL +
                            "  Peak CPU{0} usage is extremely high [{1}]. Please investigate CPU distribution https://{2}/#system/cpu".format(
                                core, _peak, _host_ip) + bcolors.ENDC)
                sa_dict['Alerts'].append(
                    "  Peak CPU{0} usage is extremely high [{1}]. Please investigate CPU distribution https://{2}/#system/cpu".format(
                        core, _peak, _host_ip))
            elif float(_avg) > int(50):
                hc_log.info(bcolors.WARNING +
                            "  Average CPU{0} usage is high [{1}]. Please investigate CPU distribution https://{2}/#system/cpu".format(
                                core, _avg, _host_ip) + bcolors.ENDC)
                sa_dict['Alerts'].append(
                    "Average CPU{0} usage is high [{1}]. Please investigate CPU distribution https://{2}/#system/cpu".format(
                        core, _avg, _host_ip))

        if float(variance(avgs)) > int(20):
            hc_log.info(bcolors.WARNING +
                        "  Variance of CPU core usage averages is greater than 20. Please investigate CPU distribution https://{0}/#system/cpu".format(
                            _host_ip) + bcolors.ENDC)
            sa_dict['Alerts'].append(
                "Variance of CPU core usage averages is greater than 20. Please investigate CPU distribution https://{0}/#system/cpu".format(
                    _host_ip))
    except Exception as e:
        hc_log.info("  Unable to complete CPU core distribution analysis. [{0}]".format(e))

    return 0


def variance(lst):
    '''
    Calculate the variance between the list values provided
    :param lst: list of numerical values
    :return: variance value
    '''
    num_items = len(lst)
    mean = sum(lst) / num_items
    differences = [x - mean for x in lst]
    sq_differences = [d ** 2 for d in differences]
    ssd = sum(sq_differences)
    variance = ssd / num_items
    return variance


def quiet_stdout():
    '''
    redirect STDOUT to /dev/null
    :return:
    '''
    hc_log.info(bcolors.OKBLUE + "Setting quiet output. STDOUT has been redirected to /dev/null" + bcolors.ENDC)
    sys.stdout = open('/dev/null', 'w')
    return 0


def restore_stdout():
    '''
    Restore STDOUT to standard output device
    :return:
    '''
    hc_log.info(bcolors.OKBLUE + "Restoring STDOUT output." + bcolors.ENDC)
    sys.stdout = sys.__stdout__
    return 0


def check_storage_througput():
    # use dd if=/dev/zero of=/data/logs/ddfile.txt bs=1024 count=194394 to check storage performance.
    # Could take several minutes to complete
    hc_log.info("Storage throughput testing (NOT CURRENTLY IMPLEMENTED)...")
    return 0


def get_cluster_settings():
    # Collect all of the active cluster settings
    # select cluster_id,type,config_path from lb_clusters_summary where status != 9;
    global GLOBAL_SERVER_CONNECTIONS
    LOGLEVEL = ['Minimal', 'Extended', 'Verbose']

    def _get_server_settings(cid_name, db_type):
        """
        Get cluster server details from sqlite file
        :param db_type: database type (MYSQL, MSSQL, ORACLE)
        :return:
        """
        global GLOBAL_SERVER_CONNECTIONS
        db_type = db_type.upper()
        db = conn2.cursor()
        hc_log.info("    DB Type: {0}".format(db_type))
        hc_log.info("    Executing 'SELECT * FROM lb_servers WHERE status=1'...")
        db.execute("SELECT * FROM lb_servers WHERE status=1")
        results = db.fetchall()
        hc_log.info("    Results: {0}".format(str(results)))
        for result in results:
            _server_ip = result["ipaddress"]
            GLOBAL_SERVER_CONNECTIONS += int(result["maxconn"])
            sa_dict["Server Configurations"][cid_name][_server_ip] = {}

            for column in result.keys():
                sa_dict["Server Configurations"][cid_name][_server_ip][column] = result[column]

            if db_type == 'MYSQL':
                hc_log.info("    Getting MySQL version from {0} using {1}".format(_server_ip, username))
                _version = get_mysql_version(_server_ip, username, password)
            elif db_type == 'MSSQL':
                hc_log.info(
                    "    Getting MSSQL version from {0}:{1} using {2}".format(_server_ip, result['port'], username))
                _version = get_mssql_version(_server_ip, username, password, result["port"])
            else:
                _version = 'Unknown'

            sa_dict['Server Configurations'][cid_name][_server_ip]["version"] = _version

        return 0

    hc_log.info("Gathering cluster configuration information...")

    # loglevel|dumpfreq|ftpip|ftpport|username|password|status|updatetime|
    # delorgfile|slowqueryfreq|clusterid|archive_enabled|archive_older_than
    LB_LOG = ["Log Level", "Interval", "FTP Server IP", "FTP Server Port", "Username", "Password", "Status",
              "Update Time", "Delete Original File", "Slow Query Frequency", "Cluster ID", "Enabled",
              "Archive Older Than"]

    LB_ADVSETTINGS = ["TCP Connection Linger", "Cluster ID", "qfailovertime", "Cache Compression Minimum Data Size",
                      "Server Idle Timeout", "Client Idle Timeout", "Unresponsive Server Timeout",
                      "Cache Cleanup Cycle", "maxpercache", "failsafestatus", "Max Replication Lag",
                      "Max Query Response Size for Cache", "QLLB", "Cache Empty Result Set", "Write Ignore",
                      "Ignore Replication Lag for LB", "Query Level Sticky Query", "Query Routing", "querytrans",
                      "Dynamic Query Caching", "Authentication Salt Reuse", "Connection Reset",
                      "Server Connection Timeout", "Server Greeting Timeout", "Max Client Authentication Timeout",
                      "Surge Queue Timeout", "Auto Failover", "Counter Module Status", "Counter Update Interval",
                      "No Lock", "Slow Query Termination", "Max Query Logging Size", "DB Down Retry",
                      "Prep/Exec Support", "Surge Queue Timeout", "Galera Cluster", "Enable Persistent Cache",
                      "Enable Compressed MySQL", "Passthru Enabled", "Query Level Sticky Query",
                      "Health Check Interval", "Read/Write Split Prep", "Query Based Connection Unstick",
                      "Connection Pooling", "Max Open Prepare", "Ignore Weightage", "Prepare-Exec Handling",
                      "Ignore Server Response Time", "Ignore Server Max Connections", "Maximum Query Packet Size"]

    hc_log.info("  Connecting to /system/lb.sqlite")
    conn = sqlite.connect('/system/lb.sqlite')
    conn.text_factory = str
    c = conn.cursor()
    hc_log.info("  SELECT * FROM lb_clusters_summary WHERE status=1")
    try:
        c.execute("SELECT * FROM lb_clusters_summary WHERE status=1")
        rows = c.fetchall()
    except Exception as e:
        hc_log.error(
            bcolors.FAIL + "  Unable to retrieve list of clusters from sqlite files: [{0}]".format(e) + bcolors.ENDC)
        return 1

    for row in rows:
        _cluster_id = row[0]
        _cluster_type = row[1].upper()
        # _cluster_status = row[2]
        _config_path = row[3]

        if os.path.isfile(_config_path) is False:
            hc_log.error(bcolors.FAIL + "    File not found: {0}".format(_config_path) + bcolors.ENDC)
            continue

        (username, password) = get_cluster_root_user(_cluster_id)

        hc_log.info("  Connecting to {0}".format(_config_path))
        conn2 = sqlite.connect(_config_path)
        conn2.text_factory = str
        conn2.row_factory = sqlite.Row
        c2 = conn2.cursor()
        hc_log.info("    SELECT * FROM lb_clusters")
        try:
            c2.execute("SELECT * FROM lb_clusters")
            row = c2.fetchone()
            hc_log.info("      RESULT: {0}".format(str(row)))
            _cluster_name = row[1]

            sa_dict['Cluster Configurations'][_cluster_name] = {}
            sa_dict['Log Settings'][_cluster_name] = {}
            sa_dict['Server Configurations'][_cluster_name] = {}
            CLUSTER_CONFIG_ITEMS[_cluster_name] = {}
            CLUSTER_CONFIG_ITEMS[_cluster_name]['Type'] = str(_cluster_type)

            for l1 in LEVEL1_HEADINGS:
                CLUSTER_CONFIG_ITEMS[_cluster_name][l1] = {}
                _headings = LEVEL2_HEADINGS[l1]
                for l2 in _headings:
                    CLUSTER_CONFIG_ITEMS[_cluster_name][l1][l2] = {}

            CLUSTER_CONFIG_ITEMS[_cluster_name]['Cluster Landing Page']['Users & DBs']['Authentication Offload'] = str(
                bool(row[6]))
            CLUSTER_CONFIG_ITEMS[_cluster_name]['Cluster Landing Page']['Auto Failover']['Enabled'] = str(bool(row[26]))
            CLUSTER_CONFIG_ITEMS[_cluster_name]['Cluster Settings']['Cluster Tab']['Read/Write Split'] = str(
                bool(row[11]))
            CLUSTER_CONFIG_ITEMS[_cluster_name]['Cluster Settings']['Client Tab']['Max Client Connections'] = row[24]

            for i in range(len(LB_CLUSTERS[_cluster_type])):
                sa_dict['Cluster Configurations'][_cluster_name][LB_CLUSTERS[_cluster_type][i]] = row[i]
        except Exception as e:
            hc_log.error(bcolors.FAIL + "  Unable to retrieve cluster configuration information from {0} [{1}]".format(
                _config_path, str(e)) + bcolors.ENDC)
            continue

        hc_log.info("  Checking for /opt/idb/utils/stop_analytics.{0} file".format(_cluster_id))
        if os.path.isfile("/opt/idb/utils/stop_analytics." + str(_cluster_id)):
            _cluster_analytics = 'Off'
        else:
            _cluster_analytics = 'On'

        hc_log.info("    SELECT * FROM lb_advsettings")
        try:
            c2.execute("SELECT * FROM lb_advsettings")
            row = c2.fetchone()
            hc_log.info("      RESULT: {0}".format(str(row)))
            for i in range(len(LB_ADVSETTINGS)):
                try:
                    sa_dict['Cluster Configurations'][_cluster_name][LB_ADVSETTINGS[i]] = row[i]
                except Exception as e:
                    hc_log.info(
                        "  Unable to store cluster configuration advanced settings for {0} in primary dictionary".format(
                            _cluster_name))
                    hc_log.info(str(e))

                if str(_cluster_type.upper()) == 'MYSQL':
                    CLUSTER_CONFIG_ITEMS[_cluster_name]['Cluster Settings']['ScaleArc Tab']['Surge Queue Timeout'] = \
                        row[34]
                elif str(_cluster_type.upper()) == 'MSSQL':
                    CLUSTER_CONFIG_ITEMS[_cluster_name]['Cluster Settings']['ScaleArc Tab']['Surge Queue Timeout'] = \
                        row[44]
                    CLUSTER_CONFIG_ITEMS[_cluster_name]['Cluster Settings']['ScaleArc Tab'][
                        'Advanced Replication Monitoring'] = row[32]
                    CLUSTER_CONFIG_ITEMS[_cluster_name]['Cluster Settings']['ScaleArc Tab'][
                        'Windows Authentication Offload'] = row[40]
                elif str(_cluster_type.upper()) == 'ORACLE':
                    CLUSTER_CONFIG_ITEMS[_cluster_name]['Cluster Settings']['ScaleArc Tab']['Surge Queue Timeout'] = \
                        row[47]
                else:
                    hc_log.info("  Unable to determine database type. [{0}]".format(str(_cluster_type.upper())))

                CLUSTER_CONFIG_ITEMS[_cluster_name]['Cluster Landing Page']['Database Server Settings'][
                    'Idle Connection Timeout'] = row[4]
                CLUSTER_CONFIG_ITEMS[_cluster_name]['Cluster Landing Page']['Database Server Settings'][
                    'Max Replication Lag'] = row[10]
                CLUSTER_CONFIG_ITEMS[_cluster_name]['Cluster Settings']['Cluster Tab'][
                    'Cluster Analytics'] = _cluster_analytics
                CLUSTER_CONFIG_ITEMS[_cluster_name]['Cluster Settings']['Client Tab'][
                    'Idle Client Connection Timeout'] = row[5]
                CLUSTER_CONFIG_ITEMS[_cluster_name]['Cluster Settings']['Server Tab'][
                    'Idle Server Connection Timeout'] = row[4]

                CLUSTER_CONFIG_ITEMS[_cluster_name]['Cluster Settings']['ScaleArc Tab']['Prepare Exec Handling'] = str(
                    bool(row[33]))
                CLUSTER_CONFIG_ITEMS[_cluster_name]['Cluster Settings']['ScaleArc Tab']['Write Ignore Rule'] = str(
                    bool(row[14]))
                CLUSTER_CONFIG_ITEMS[_cluster_name]['Cluster Settings']['ScaleArc Tab']['QLLB'] = str(bool(row[12]))
                CLUSTER_CONFIG_ITEMS[_cluster_name]['Cluster Settings']['ScaleArc Tab']['Query Level Sticky Count'] = \
                    row[39]
                CLUSTER_CONFIG_ITEMS[_cluster_name]['Cluster Settings']['Debug Tab']['Unresponsive Server Timeout'] = \
                    row[6]
        except Exception as e:
            hc_log.error(
                bcolors.FAIL + "  Unable to retrieve advanced cluster configuration information from {0} [{1}]".format(
                    _config_path, str(e)) + bcolors.ENDC)
            continue

        hc_log.info("    SELECT * FROM lb_log")
        try:
            c2.execute("SELECT * FROM lb_log")
            row = c2.fetchone()
            hc_log.info("      RESULT: {0}".format(str(row)))
            if row is None:
                hc_log.info(
                    "    SELECT * FROM lb_log within /system/lb.sqlite returned NO results. " +
                    "NO external og information defined!")
                for i in range(len(LB_LOG)):
                    sa_dict['Log Settings'][_cluster_name][LB_LOG[i]] = ""
            else:
                for i in range(len(LB_LOG)):
                    if i is 'Log Level':
                        sa_dict['Log Settings'][_cluster_name][LB_LOG[i]] = LOGLEVEL[int(row[i])]
                    else:
                        sa_dict['Log Settings'][_cluster_name][LB_LOG[i]] = row[i]

                CLUSTER_CONFIG_ITEMS[_cluster_name]['Logs Settings']['Main']['Logging Level'] = LOGLEVEL[int(row[0])]

                if row[2] == '':
                    CLUSTER_CONFIG_ITEMS[_cluster_name]['Logs Settings']['Logs Backup']['FTP Server IP'] = "<undefined>"
                else:
                    CLUSTER_CONFIG_ITEMS[_cluster_name]['Logs Settings']['Logs Backup']['FTP Server IP'] = row[2]

                if row[3] == '':
                    CLUSTER_CONFIG_ITEMS[_cluster_name]['Logs Settings']['Logs Backup'][
                        'FTP Server Port'] = "<undefined>"
                else:
                    CLUSTER_CONFIG_ITEMS[_cluster_name]['Logs Settings']['Logs Backup']['FTP Server Port'] = row[3]

                if row[4] == '':
                    CLUSTER_CONFIG_ITEMS[_cluster_name]['Logs Settings']['Logs Backup']['Username'] = "<undefined>"
                else:
                    CLUSTER_CONFIG_ITEMS[_cluster_name]['Logs Settings']['Logs Backup']['Username'] = row[4]
        except Exception as e:
            hc_log.error(
                bcolors.FAIL + "  Unable to retrieve cluster hc_log configuration information from {0} [{1}]".format(
                    _config_path, str(e)) + bcolors.ENDC)
            continue

        hc_log.info("    SELECT failover_type FROM lb_autofailover")
        try:
            c2.execute("SELECT failover_type FROM lb_autofailover")
            row = c2.fetchone()
            hc_log.info("      RESULT: {0}".format(str(row)))
            if str(row[0]) is '0':
                CLUSTER_CONFIG_ITEMS[_cluster_name]['Cluster Landing Page']['Auto Failover'][
                    'Failover Type'] = 'ScaleArc Based'
            elif str(row[0]) is '1':
                CLUSTER_CONFIG_ITEMS[_cluster_name]['Cluster Landing Page']['Auto Failover'][
                    'Failover Type'] = 'External API'
            else:
                hc_log.info("  Unexpected Auto Failover Type ({0})".format(str(row[0])))
        except Exception as e:
            hc_log.error(
                bcolors.FAIL + "  Unable to retrieve cluster failover type configuration information from {0} [{1}]".format(
                    _config_path, str(e)) + bcolors.ENDC)
            continue

        hc_log.info("    SELECT * FROM lb_servers WHERE status=1")
        try:
            _get_server_settings(_cluster_name, _cluster_type)

        except Exception as e:
            sys.stderr.write("\nException while getting server settings [{0}]\n".format(str(e)))
            hc_log.error(bcolors.FAIL +
                         "  Unable to retrieve cluster database server configuration information from {0}".format(
                             _config_path) +
                         bcolors.ENDC)
            continue
        conn2.close()

        hc_log.info("    Getting dbuserids...")
        dbuserids = _get_dbuserids(_config_path)
        hc_log.info("    Checking for orphaned write ignore rules...")
        check_orphaned_writeignore_rules(_config_path, dbuserids)
        hc_log.info("    Checking for orphaned cache patterns...")
        check_orphaned_cache_rules(_config_path, dbuserids)
    conn.close()
    TESTS["Cluster Configurations"] = json.dumps(CLUSTER_CONFIG_ITEMS, ensure_ascii=False)
    TESTS["Server Configurations"] = sa_dict["Server Configurations"]
    sa_dict['Cluster Configuration Items'] = json.dumps(CLUSTER_CONFIG_ITEMS, ensure_ascii=False)

    return 0


def check_global_server_connections():
    hc_log.info("Checking global server connection count...")
    if GLOBAL_SERVER_CONNECTIONS > 30000:
        sa_dict['Alerts'].append(
            "The total configured max connections of ({0}) for all database servers defined in the ".format(
                GLOBAL_SERVER_CONNECTIONS) +
            "ScaleArc configuration exceeds recommended limits. Please consider adjusting the configurations " +
            "or deploying additional ScaleArc appliances to distribute the client application load. ScaleArc " +
            "has a maximum of 95,000 file handles available. Sixty thousand (60,000) are reserved for client " +
            "connections. The remaining 35,000 are available to distribute across all active database servers.")
        hc_log.warning(
            bcolors.WARNING + "The aggregate server max connections is high: {0}".format(GLOBAL_SERVER_CONNECTIONS) +
            bcolors.ENDC)
    else:
        hc_log.info(bcolors.OKGREEN + "  Aggregate server connection count: {0}".format(
            GLOBAL_SERVER_CONNECTIONS) + bcolors.ENDC)
    return 0


def get_uuid():
    '''
    Generate system UUID from block device ID
    :return:
    '''
    hc_log.info("Getting UUID...")
    hc_log.info("  /bin/df / | /bin/grep '^/' | /bin/awk '{print $1;}'")
    device = system_call("/bin/df / | /bin/grep '^/' | /bin/awk '{print $1;}'")
    device = device.replace("\n", "")
    hc_log.info("  /sbin/blkid -o value {0}".format(device))
    result = system_call("/sbin/blkid -o value {0} | /bin/grep '-'".format(device))
    if result != '':
        blkid_results = result.replace("\n", "")
        hc_log.info(bcolors.OKBLUE + "    UUID: {0}".format(blkid_results) + bcolors.ENDC)
        uuid = blkid_results
    else:
        hc_log.error(bcolors.FAIL + "    Unable to retrieve root filesystem UUID." + bcolors.ENDC)
        uuid = ''
    sa_dict['UUID'] = uuid.replace("\n", "")
    return 0


def get_hostid():
    '''
    Generate Host ID
    :return:
    '''
    hc_log.info("Getting Host ID...")
    hc_log.info("  /usr/bin/hostid")
    result = system_call("/usr/bin/hostid")
    if result != '':
        hc_log.info(bcolors.OKBLUE + "    HostID: {0}".format(result) + bcolors.ENDC)
        hostid = result.splitlines()[0]
    else:
        hc_log.error(bcolors.FAIL + "  Unable to retrieve host ID." + bcolors.ENDC)
        hostid = ''
    sa_dict['Host ID'] = hostid.replace("\n", "")
    return hostid.replace("\n", "")


def store_results():
    # Write collected information out to SQLITE file
    hc_log.info("Writing collected data to {0}...".format(bcolors.OKBLUE + GLOBAL_OUTPUT_SQLITE_FILE + bcolors.ENDC))
    TAR_FILE_LIST.append(GLOBAL_OUTPUT_SQLITE_FILE)
    hc_log.info("  Connecting to {0}".format(GLOBAL_OUTPUT_SQLITE_FILE))
    conn = sqlite.connect(GLOBAL_OUTPUT_SQLITE_FILE)
    conn.text_factory = str
    conn.isolation_level = None
    c = conn.cursor()

    CREATE_TABLE_SQL = 'CREATE TABLE IF NOT EXISTS version( ID INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, ' + \
                       'LastUpdated TEXT NOT NULL, ResultID INTEGER NOT NULL, Version TEXT NOT NULL )'
    hc_log.info("    {0}".format(CREATE_TABLE_SQL))
    c.execute(CREATE_TABLE_SQL)

    CREATE_TABLE_SQL = 'CREATE TABLE IF NOT EXISTS raw_data( ID INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, ' + \
                       'LastUpdated TEXT NOT NULL, ResultID INTEGER NOT NULL, Data TEXT )'
    hc_log.info("    {0}".format(CREATE_TABLE_SQL))
    c.execute(CREATE_TABLE_SQL)

    CREATE_TABLE_SQL = 'CREATE TABLE IF NOT EXISTS results( ResultID INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, ' + \
                       'LastUpdated TEXT NOT NULL, UUID TEXT NOT NULL, HostID TEXT )'
    hc_log.info("    {0}".format(CREATE_TABLE_SQL))
    c.execute(CREATE_TABLE_SQL)

    CREATE_TABLE_SQL = 'CREATE TABLE IF NOT EXISTS tests( TestID INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, ' + \
                       'TestName TEXT NOT NULL, ResultID INTEGER NOT NULL, Data TEXT )'
    hc_log.info("    {0}".format(CREATE_TABLE_SQL))
    c.execute(CREATE_TABLE_SQL)

    CREATE_TABLE_SQL = 'CREATE TABLE IF NOT EXISTS alerts( AlertID INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, ' + \
                       'ResultID INTEGER NOT NULL, Message TEXT )'
    hc_log.info("    {0}".format(CREATE_TABLE_SQL))
    c.execute(CREATE_TABLE_SQL)

    INSERT_TABLE_SQL = 'INSERT INTO results( ResultID, LastUpdated, UUID, HostID ) VALUES ( NULL, '
    INSERT_TABLE_SQL += '"' + NOW + '", "'
    INSERT_TABLE_SQL += '{0}\", \"{1}\")'.format(sa_dict['UUID'], sa_dict["Host ID"])
    hc_log.info("    {0}".format(INSERT_TABLE_SQL))

    try:
        hc_log.info("  Writing healthcheck result data to sqlite file.")
        c.execute(INSERT_TABLE_SQL)
        conn.commit()
    except sqlite.Error as er:
        hc_log.info(bcolors.FAIL + "    Healthcheck result storage error: {0}".format(er) + bcolors.ENDC)

    c.execute("SELECT last_insert_rowid()")
    result_id = c.fetchone()[0]
    hc_log.info("  ResultID: {0}".format(result_id))

    INSERT_TABLE_SQL = 'INSERT INTO version( ID, LastUpdated, ResultID, Version ) VALUES ( NULL, '
    INSERT_TABLE_SQL = INSERT_TABLE_SQL + "\"" + NOW + "\", "
    INSERT_TABLE_SQL = INSERT_TABLE_SQL + str(result_id) + ", \""
    INSERT_TABLE_SQL = INSERT_TABLE_SQL + VERSION + "\")"
    hc_log.info("    {0}".format(INSERT_TABLE_SQL))

    try:
        hc_log.info("  Writing healthcheck version to sqlite file.")
        c.execute(INSERT_TABLE_SQL)
        conn.commit()
    except sqlite.Error as er:
        hc_log.error(bcolors.FAIL + "    Healthcheck version storage error: {0}".format(er) + bcolors.ENDC)
    try:
        hc_log.info("  Writing alert data to sqlite file.")
        for alert in sa_dict['Alerts']:
            alert = alert.replace("'", "")
            INSERT_TABLE_SQL = "INSERT INTO alerts( AlertID, ResultID, Message) VALUES ( NULL, {0:d}".format(result_id)
            INSERT_TABLE_SQL += ", '{0}')".format(alert)
            hc_log.info("    {0}".format(INSERT_TABLE_SQL))
            c.execute(INSERT_TABLE_SQL)
    except sqlite.Error as er:
        hc_log.error(bcolors.FAIL + "    Healthcheck alert data storage error: {0}".format(er) + bcolors.ENDC)

    hc_log.info("  Writing test data to sqlite file.")
    for test in TESTS.keys():
        INSERT_TABLE_SQL = "INSERT INTO tests( TestID, TestName, ResultID, Data ) VALUES ( NULL, '{0}'".format(test)
        test_str = str(TESTS[test])
        test_str = test_str.replace("'", '"')
        INSERT_TABLE_SQL += ", {0:d},'{1}')".format(result_id, test_str)
        hc_log.info("    {0}".format(INSERT_TABLE_SQL))
        try:
            c.execute(INSERT_TABLE_SQL)
        except sqlite.Error as er:
            hc_log.error(bcolors.FAIL + "    Healthcheck test data storage error: {0}".format(er) + bcolors.ENDC)
            continue

    conn.close()
    return 0


def get_dns_information():
    '''
    Collect DNS Server configuration information
    :return:
    '''
    hc_log.info("Gathering DNS configuration information.")

    _scalearc_resolver = '/system/resolv.conf'
    # _system_resolver = '/etc/resolv.conf'
    # Check if file exists
    if os.path.isfile(_scalearc_resolver) is False:
        hc_log.info(bcolors.FAIL + "  File not found: {0}".format(_scalearc_resolver) + bcolors.ENDC)
        return
    else:
        hc_log.info("  Collecting nameserver entires...")
        result = system_call("/bin/grep -i -e nameserver {0}".format(_scalearc_resolver))
        result = result.splitlines()
        for server in result:
            ns_ip = server.rsplit(None, 1)[-1]
            hc_log.info("    Name server: {0}".format(ns_ip))
            sa_dict['DNS Name Servers'].append(ns_ip)
        hc_log.info("  Collecting search domains...")
        result = system_call("/bin/grep -i -e search {0}".format(_scalearc_resolver))
        if result == '':
            result = 'None'
        search_domain = result.rsplit(None, 1)[-1]
        hc_log.info("    Search domains: {0}".format(search_domain))
        sa_dict['DNS Search Domains'] = search_domain
    dns = {'Servers': sa_dict['DNS Name Servers'], 'Search Domains': sa_dict['DNS Search Domains']}
    TESTS["DNS Information"] = json.dumps(dns, ensure_ascii=False)
    return 0


def get_centos_version():
    '''
    Collect the CentOS version
    :return:
    '''
    hc_log.info("Inspecting CentOS version...")
    hc_log.info("  Checking if /etc/centos-release exists...")
    if os.path.isfile('/etc/centos-release') is True:
        hc_log.info("    Collecting CentOS release version...")
        centos_version = system_call("/bin/cat /etc/centos-release")
        centos_version = centos_version.replace("\n", "")
        hc_log.info("      CentOS version is {0}".format(centos_version))
    else:
        hc_log.info("    /etc/centos-version not found. Checking /bin/rpm -q centos-release...")
        centos_version = system_call("/bin/rpm -q centos-release")
        centos_version = centos_version.replace("\n", "")

    sa_dict['CentOS Version'] = centos_version
    TESTS["CentOS Version"] = json.dumps(sa_dict['CentOS Version'], ensure_ascii=False)
    return 0


def get_kernel_version():
    '''
    Collect the linux kernel version
    :return:
    '''
    hc_log.info("Inspecting kernel version...")
    hc_log.info("  Executing /bin/uname -r to retrieve the kernel version...")
    kernel_version = system_call("/bin/uname -r")
    kernel_version = kernel_version.replace("\n", "")
    if kernel_version == '':
        hc_log.info("    Unable to determine kernel version.")
        kernel_version = "Unknown"
    else:
        hc_log.info("    result: {0}".format(kernel_version))
    sa_dict['Kernel Version'] = kernel_version
    TESTS["Kernel Version"] = json.dumps(sa_dict['Kernel Version'], ensure_ascii=False)
    return 0


def check_yum_auto_update():
    '''
    Check if auto update has been enabled for yum
    :return:
    '''
    hc_log.info("Checking if YUM auto update has been configured.")
    hc_log.info("  Executing /sbin/service yum-cron status...")
    yum_auto_update = system_call("/sbin/service yum-cron status 2>&1")
    yum_auto_update = yum_auto_update.replace("\n", "")
    os_updates = {}
    if yum_auto_update == "yum-cron: unrecognized service":
        # yum-cron not installed
        hc_log.info("    yum-cron not installed. [{0}]".format(yum_auto_update))
        os_updates['Installed'] = False
        os_updates['Running'] = False
    elif yum_auto_update == "Nightly yum update is disabled.":
        hc_log.info(
            bcolors.WARNING + "    yum-cron installed but not running. [{0}]".format(yum_auto_update) + bcolors.ENDC)
        sa_dict['Alerts'].append(
            "yum-cron is installed but not running. Please verify the auto software update policy and adjust configuration accordingly.")
        os_updates['Installed'] = True
        os_updates['Running'] = False
    elif yum_auto_update == "Nightly yum update is enabled.":
        hc_log.info(bcolors.FAIL + "    yum-cron installed and running. [{0}]".format(yum_auto_update) + bcolors.ENDC)
        sa_dict['Alerts'].append(
            "yum-cron is installed and running. Please verify the auto software update policy and adjust configuration accordingly.")
        os_updates['Installed'] = True
        os_updates['Running'] = True
    else:
        hc_log.info("    Unable to determine the status of yum-cron.")
    TESTS["Auto OS Updates"] = json.dumps(os_updates, ensure_ascii=False)
    return 0


def _verify_log_file(logfile):
    try:
        if os.stat(logfile).st_size > 0:
            hc_log.info(bcolors.OKGREEN + "Log file created successfully." + bcolors.ENDC)
            return True
        else:
            sys.stderr.write(bcolors.WARNING + "Log file created, however it is empty." + bcolors.ENDC + "\n")
            return False
    except OSError:
        sys.stderr.write(bcolors.FAIL + "Log file creation failed." + bcolors.ENDC + "\n")
    return False


def check_orphaned_writeignore_rules(dbfile, dbuserids):
    """
    Compare the configured write ignore rules against defined dbuserids
    :param dbfile: path to lb_<cid>.sqlite file
    :param dbuserids: dictionary of dbuserids
    :return:
    """
    hc_log.info("Checking for orphaned write ignore rules in sqlite...")
    hc_log.info("  Getting writeignore rules from {0}...".format(dbfile))
    sql_cmd = 'SELECT DISTINCT dbuserid FROM lb_writeignore WHERE status=1'
    hc_log.debug("  SQL cmd: {0}".format(sql_cmd))
    try:
        _conn = sqlite.connect(dbfile)
        _conn.text_factory = str
        _conn.row_factory = sqlite.Row
        _c = _conn.cursor()
        _c.execute(sql_cmd)
        wi_dbuserids = _c.fetchall()
        hc_log.debug("    Result: {0}".format(str(wi_dbuserids)))
        _conn.close()
    except Exception as e:
        hc_log.error("  Exception caught: [{0}]".format(str(e)))
        return -1

    dbuserids_list = dbuserids.keys()
    for wi_dbuserid in wi_dbuserids:
        if wi_dbuserid['dbuserid'] in dbuserids_list:
            pass
        else:
            hc_log.warning(bcolors.WARNING + "  orphaned write ignore rules exist for unknown dbuserid: {0}".format(
                wi_dbuserid['dbuserid']) +
                           bcolors.ENDC)
            # sa_dict['Alerts'].append("Write ignore rules exist for unknown dbuserid {0}".format(dbuserid))
    return 0


def check_orphaned_cache_rules(dbfile, dbuserids):
    hc_log.info("Checking for orphaned cache rules in sqlite...")
    hc_log.info("  Getting cache patterns from {0}...".format(dbfile))
    sql_cmd = 'SELECT DISTINCT dbuserid FROM lb_cachepattern WHERE status=1'
    hc_log.debug("  SQL cmd: {0}".format(sql_cmd))
    try:
        _conn = sqlite.connect(dbfile)
        _conn.text_factory = str
        _conn.row_factory = sqlite.Row
        _c = _conn.cursor()
        _c.execute(sql_cmd)
        cache_dbuserids = _c.fetchall()
        _conn.close()
    except Exception as e:
        hc_log.error("  Exception caught: [{0}]".format(str(e)))
        return -1

    dbuserids_list = dbuserids.keys()
    for cache_dbuserid in cache_dbuserids:
        if cache_dbuserid['dbuserid'] in dbuserids_list:
            pass
        else:
            hc_log.warning(bcolors.WARNING + "  orphaned cache rules exist for unknown dbuserid: {0}".format(
                cache_dbuserid['dbuserid']) +
                           bcolors.ENDC)
            # sa_dict['Alerts'].append("Write ignore rules exist for unknown dbuserid {0}".format(dbuserid))
    return 0


def check_storage_throughput():
    pass


def _get_dbuserids(dbfile):
    hc_log.info("Collecting dbuserids from sqlite...")
    sql_cmd = 'SELECT dbuserid,lb_users.username,lb_dbs.dbname FROM lb_dbusers LEFT JOIN lb_users ON '
    sql_cmd += 'lb_dbusers.userid = lb_users.userid LEFT JOIN lb_dbs ON lb_dbusers.dbid = lb_dbs.dbid '
    sql_cmd += 'WHERE lb_dbusers.status=1'
    conn = sqlite.connect(dbfile)
    conn.text_factory = str
    conn.row_factory = sqlite.Row
    c = conn.cursor()
    c.execute(sql_cmd)
    dbuserid_rows = c.fetchall()
    _dbuserids = {}
    for dbuserid_row in dbuserid_rows:
        dbuserid = dbuserid_row['dbuserid']
        _dbuserids[dbuserid] = {}
        _dbuserids[dbuserid]['username'] = dbuserid_row['username']
        _dbuserids[dbuserid]['dbname'] = dbuserid_row['dbname']
    conn.close()
    return _dbuserids


def check_fs_ownership_and_permissions():
    """
    Compare local filesystem files against internal reference for owner, group, and permissions
    :rtype: list, dictionary
    :return: list of missing files and dictionary of reference mismatches
    """
    stats = {}
    dirs = ['/data/', '/system/']

    def _store_fs_details(fs_obj, stat_obj):
        fs_obj = str(fs_obj)
        _user = pwd.getpwuid(stat_obj.st_uid)[0]
        _grp = grp.getgrgid(stat_obj.st_gid)[0]
        stats[fs_obj] = {}
        stats[fs_obj]['owner'] = _user
        stats[fs_obj]['group'] = _grp
        stats[fs_obj]['size'] = stat_obj.st_size
        stats[fs_obj]['permissions'] = stat_obj.st_mode
        return 0

    for d in dirs:
        for root, subdirs, files in os.walk(d):
            st = os.stat(root)
            _store_fs_details(root, st)
            for subdir in subdirs:
                dir_path = os.path.join(root, subdir)
                st = os.stat(dir_path)
                _store_fs_details(dir_path, st)
            for filename in files:
                file_path = os.path.join(root, filename)
                st = os.stat(file_path)
                _store_fs_details(file_path, st)

    missing_files = []
    permission_issues = {}
    paths = FILE_STATS.keys()
    for path in paths:
        if path in stats.keys():
            # hc_log.info("Required path {0} exists in local system".format(path))
            pass
        else:
            hc_log.warning(
                bcolors.WARNING + "Required path {0} is missing from local system".format(path) + bcolors.ENDC)
            missing_files.append(path)
            continue
        permission_issues[path] = {}
        items = FILE_STATS[path].keys()
        for item in items:
            value = FILE_STATS[path][item]
            if value == stats[path][item]:
                # hc_log.info(
                #    "{0} value of {1} for {2} on local system matches the reference value".format(
                #        item, stats[path][item], path))
                pass
            else:
                hc_log.warning(
                    bcolors.WARNING +
                    "{0} value of '{1}' for {2} on local system DOES NOT match the reference value [{3}]".format(
                        item, stats[path][item], path, value) + bcolors.ENDC)
                permission_issues[path][item] = {}
                if item == 'permissions':
                    chmod_bits = oct(stat.S_IMODE(stats[path][item]))
                    hc_log.warning(bcolors.OKBLUE + "    Recommend to chmod {0} for {1}".format(chmod_bits, path) +
                                   bcolors.ENDC)
                elif item == 'owner':
                    hc_log.warning(
                        bcolors.OKBLUE + "    Recommend to chown {0} for {1}".format(value, path) + bcolors.ENDC)
                elif item == 'group':
                    hc_log.warning(
                        bcolors.OKBLUE + "    Recommend to chgrp {0} for {1}".format(value, path) + bcolors.ENDC)
                permission_issues[path][item]['reference'] = value
                permission_issues[path][item]['actual'] = stats[path][item]
    return missing_files, permission_issues


# noinspection PyUnusedLocal,PyUnusedLocal
def main():
    '''
    Main program code
    :return:
    '''
    begin_ts = time.time()
    global CONFIG_FILE
    global OUTPUT_FILE
    global INPUT_FILE
    global OUTPUT_BASE
    global PRINT_REPORT
    global IGNORE_CFG_FILE
    global DEBUG
    global VERBOSE
    global QUIET
    global NOW
    global NO_COLOR
    global TESTS_RUN
    global MINIMAL_TESTS

    VERBOSE_CMD = False
    DEBUG_CMD = False

    try:
        opts, args = getopt.getopt(sys.argv[1:], "i:qdhc:vVo:pm",
                                   ["input=", "report", "quiet", "debug", "help", "config=", "verbose", "version",
                                    "output=", "minimal", "no-color"])
    except getopt.GetoptError as err:
        # print help information and exit:
        print str(err)  # will print something like "option -a not recognized"
        usage()
        sys.exit(2)

    for o, a in opts:
        if o in ("-v", "--verbose"):
            VERBOSE_CMD = True
        elif o in ("-d", "--debug"):
            DEBUG_CMD = True
            IGNORE_CFG_FILE = True
        elif o in ("-q", "--quiet"):
            QUIET = True
            quiet_stdout()
        elif o in ("-h", "--help"):
            usage()
            sys.exit()
        elif o in ("-V", "--version"):
            version()
            sys.exit()
        elif o in ("-c", "--config"):
            CONFIG_FILE = a
        elif o in ("-p", "--report"):
            PRINT_REPORT = True
        elif o in ("-i", "--input"):
            INPUT_FILE = a
        elif o == "--no-color":
            NO_COLOR = True
        elif o in ("-o", "--output"):
            OUTPUT_FILE = a
            OUTPUT_BASE = OUTPUT_FILE
        elif o in ("-m", "--minimal"):
            MINIMAL_TESTS = True
        else:
            usage()

    ''' check if a newer version of the script is available '''
    check_version()

    if QUIET is False:
        sys.stderr.write("ScaleArc Healthcheck execution started\n")
        if NO_COLOR:
            sys.stderr.write("Version: {0}\n\n".format(VERSION))
        else:
            sys.stderr.write("Version: " + bcolors.OKGREEN + "{0}".format(VERSION) + bcolors.ENDC + "\n\n")

    hc_log.info("Configuration file: {0}".format(CONFIG_FILE))
    if IGNORE_CFG_FILE is False:
        parse_config()

    if VERBOSE_CMD is True:
        VERBOSE = True
    elif bool(DEFAULTS['VERBOSE']) is not True:
        VERBOSE = False
    else:
        VERBOSE = True

    if DEBUG_CMD is True:
        DEBUG = True
    elif bool(DEFAULTS['DEBUG']) is not True:
        DEBUG = False
    else:
        DEBUG = True

    if DEBUG is True:
        hc_log.info("### Global variables DEBUG: {0} VERBOSE: {1}".format(DEBUG, VERBOSE))

    hc_log.info("Execution log being written to {0}".format(bcolors.WARNING + LOG_FILENAME + bcolors.ENDC))
    TAR_FILE_LIST.append(LOG_FILENAME)

    # If input file provided, read data and output report, then exit
    if INPUT_FILE != '':
        parse_input_file()
        print_report()
        hc_log.info("Execution completed.")
        exit()

    ####
    ## Run ScaleArc HealthCheck data gathering
    ####
    print "Generating UUID..."
    get_uuid()
    print "Collecting HostID..."
    get_hostid()

    print "Checking CentOS version..."
    get_centos_version()
    print "Checking kernel version..."
    get_kernel_version()
    print "Checking for OS auto-update..."
    check_yum_auto_update()
    print "Collecting hardware information..."
    get_hw_basics()
    print "Collecting ScaleArc base information..."
    get_idb_basics()
    print "Collecting sysctl settings..."
    get_sysctl_info()
    print "Collecting kernel module information..."
    get_lsmod_info()
    print "Collecting swap space information..."
    get_swap()
    print "Collecting ScaleArc thread to CPU core mapping information..."
    get_cpu_thread_mapping()

    # Merge defaults and configuration file settings into global dictionary
    sa_dict.update(DEFAULTS)

    print "Checking memory configuration..."
    check_memory()
    TESTS_RUN += 1
    print "Checking swap volume..."
    check_swap()
    TESTS_RUN += 1

    if LooseVersion(sa_dict['Current Production Version']) >= '3.11.0.2':
        sys.stderr.write("\n" + sys.argv[0] + " version " + VERSION + " does not support ScaleArc 3.11.0.2 and above\n")
        sys.stderr.write("for HA features.\n")
        sys.stderr.write(bcolors.OKBLUE + "Skipping high availability settings check!" + bcolors.ENDC + "\n")
        sys.stderr.flush()
        sa_dict['HA Keep Alive'] = 'N/A'
        sa_dict['HA Warn Time'] = 'N/A'
        sa_dict['HA Dead Time'] = 'N/A'
        sa_dict['HA Init Dead Time'] = 'N/A'
        sa_dict['Heartbeat Process CPU'] = 'N/A'
    else:
        if MINIMAL_TESTS is False:
            print "Checking heartbeat logs..."
            check_heartbeat_messages()
            TESTS_RUN += 1
        print "Checking high availability settings..."
        get_heartbeat_config()
        TESTS_RUN += 1
        print "Determining HA process CPU core assignment..."
        get_heartbeat_cpu()
        TESTS_RUN += 1

    print "Checking ScaleArc license expiration..."
    check_license_expiration()
    TESTS_RUN += 1
    sys.stderr.flush()
    print "Checking NTP configuration..."
    check_ntp()
    TESTS_RUN += 1
    print "Verifying DNS settings..."
    get_dns_information()
    TESTS_RUN += 1
    print "Gathering IP addresses..."
    interfaces = get_network_interfaces()
    if len(interfaces) > 0:
        get_interface_ips(interfaces)
    get_virtual_ips()
    get_ha_peer_ip()
    print "Checking virtual IP addresses netmasks..."
    check_virtual_netmasks()
    TESTS_RUN += 1
    if DEFAULTS['RSYNC_CHECK'] or MINIMAL_TESTS is False:
        print "Scanning for RSYNC error messages..."
        check_rsync_messages()
        TESTS_RUN += 1
    else:
        if NO_COLOR:
            print "Skipping scan for RSYNC error messages due to parameter setting"
        else:
            print bcolors.WARNING + "Skipping scan for RSYNC error messages due to parameter setting" + bcolors.ENDC
    if DEFAULTS['SYSTEM_CHECK'] or MINIMAL_TESTS is False:
        print "Scanning for system error messages..."
        check_system_messages()
        TESTS_RUN += 1
    else:
        if NO_COLOR:
            print "Skipping scan for system error messages due to parameter setting"
        else:
            print bcolors.WARNING + "Skipping scan for system error messages due to parameter setting" + bcolors.ENDC
    if MINIMAL_TESTS is False:
        print "Scanning for error messages in idb.alert files (if any)..."
        files = get_idb_file_list("alert")
        check_idb_log_messages(files)
        TESTS_RUN += 1
        print "Scanning for error messages in idb.error files (if any)..."
        files = get_idb_file_list("error")
        check_idb_log_messages(files)
        TESTS_RUN += 1
    print "Inspecting /logs (/data/logs) volume..."
    check_data_partition()
    TESTS_RUN += 1
    print "Inspecting storage devices..."
    get_storage_devices()
    TESTS_RUN += 1
    if DEFAULTS['CHECK_STORAGE_THROUGHPUT'] or MINIMAL_TESTS is False:
        print "Checking storage device throughput..."
        check_storage_throughput()
        TESTS_RUN += 1
    if MINIMAL_TESTS is False:
        print "Collecting CPU core to thread mapping..."
        get_cpu_usage()
        TESTS_RUN += 1
    print "Determinig network interface interrupt to CPU core mapping..."
    get_network_interrupts()
    TESTS_RUN += 1
    print "Collecting ScaleArc application cluster configuration settings..."
    get_cluster_settings()
    TESTS_RUN += 1
    print "Checking the aggregate number of server connections..."
    check_global_server_connections()
    TESTS_RUN += 1
    print "Determining if email alerts are enabled for configured ScaleArc application cluster(s)..."
    check_email_alerts()
    TESTS_RUN += 1

    if MINIMAL_TESTS is False:
        files = get_sar_file_list()
        if not files:
            hc_log.info("No recent SAR files found. Skipping SAR file analysis.")
        else:
            print "Inspecting historical CPU utilization statistics..."
            try:
                check_sar_cpu(files)
                TESTS_RUN += 1
            except Exception as e:
                hc_log.info(bcolors.FAIL + "Exception caught for check_sar_cpu(): {0}".format(str(e)) + bcolors.ENDC)
            print "Inspecting historical memory utilization statistics..."
            try:
                check_sar_memory(files)
                TESTS_RUN += 1
            except Exception as e:
                hc_log.info(bcolors.FAIL + "Exception caught for check_sar_memory(): {0}".format(str(e)) + bcolors.ENDC)
            print "Inspecting historical swap volume utilization statistics..."
            try:
                check_sar_swap(files)
                TESTS_RUN += 1
            except Exception as e:
                hc_log.info(bcolors.FAIL + "Exception caught for check_sar_swap(): {0}".format(str(e)) + bcolors.ENDC)
            print "Inspecting historical storage device utilization statistics..."
            try:
                check_sar_device(files)
                TESTS_RUN += 1
            except Exception as e:
                hc_log.info(bcolors.FAIL + "Exception caught for check_sar_device(): {0}".format(str(e)) + bcolors.ENDC)
            print "Inspecting historical system load statistics..."
            try:
                check_sar_load(files)
                TESTS_RUN += 1
            except Exception as e:
                hc_log.info(bcolors.FAIL + "Exception caught for check_sar_load(): {0}".format(str(e)) + bcolors.ENDC)
            print "Checking filesystem structure, ownership, and permissions..."
            try:
                missing_files, permission_errors = check_fs_ownership_and_permissions()
                TESTS_RUN += 1
            except Exception as e:
                hc_log.info(bcolors.FAIL + "Exception caught for check_fs_ownership_and_permissions(): {0}".format(
                    str(e)) + bcolors.ENDC)
            print "Checking for orphaned write ignore rules..."
            try:
                active_cids = _get_active_clusters()
                for cluster_path in active_cids:
                    dbuserids = _get_dbuserids(cluster_path)
                    check_orphaned_writeignore_rules(cluster_path, dbuserids)
                TESTS_RUN += 1
                try:
                    for cluster_path in active_cids:
                        dbuserids = _get_dbuserids(cluster_path)
                        check_orphaned_cache_rules(cluster_path, dbuserids)
                    TESTS_RUN += 1
                except Exception as e:
                    hc_log.info(bcolors.FAIL + "Exception caught for check_orphaned_cache_rules(): {0}".format(
                        str(e)) + bcolors.ENDC)
            except Exception as e:
                hc_log.info(bcolors.FAIL + "Exception caught for check_orphaned_writeignore_rules(): {0}".format(
                    str(e)) + bcolors.ENDC)
            print "Checking for orphaned cache rules..."

    hc_log.info("Output file is: {0}".format(bcolors.OKBLUE + OUTPUT_FILE + bcolors.ENDC))

    if sa_dict['Database Type'] == 'MSSQL':
        check_dns_entries()
        TESTS_RUN += 1

    if RUN_RECOMMENDATIONS:
        run_recommendations()

    store_results()
    print "\nHealth Check complete. Saving results..."

    print "Saving data file..."
    write_data_file()

    if PRINT_REPORT:
        if not QUIET:
            print "Generating Health Check report..."
        print_report()

    end_ts = time.time()

    if not QUIET:
        print "\nCreating compressed tar file for submission to ScaleArc..."
    tar_output_files()

    sys.stdout.write(FOOTER_STR)
    if _verify_log_file(LOG_FILENAME):
        if NO_COLOR:
            print("Execution log written to {0}\n".format(LOG_FILENAME))
        else:
            print("Execution log written to {0}\n".format(bcolors.OKBLUE + LOG_FILENAME + bcolors.ENDC))
    total_time = end_ts - begin_ts
    execution_time = time_string(total_time)
    if not QUIET:
        if NO_COLOR:
            print("{0} tests executed in {1}".format(TESTS_RUN, execution_time))
        else:
            print("{0} tests executed in {1}".format(TESTS_RUN, bcolors.WARNING + execution_time + bcolors.ENDC))
    hc_log.info("Total execution time: {0}".format(bcolors.WARNING + execution_time + bcolors.ENDC))
    hc_log.info("Execution ended.")
    return 0


####
## end functions
####

if __name__ == "__main__":
    main()