#!/bin/bash
#
# orion: the ION trouble hunter
#
# Copyright 2007-2024 Keith Josephson @ ION Computer Systems, Inc., Hauppauge, NY, USA

# 2024-10-08 15:15 | Keith Josephson | added drivemap to syssum, so we do not need to display it later.
# 2024-10-08 14:40 | Keith Josephson | if there is a progress file for drive bay mapping, display that
# 2023-08-25 09:55 | Keith Josephson | selview fails on FCP aka ION E1 & E2 servers
# 2023-08-25 09:00 | Keith Josephson | Revised Parsing of Power Supply Records for CFG
# 2023-05-09 13:20 | Keith Josephson | Replaced START_BLUE with Start_WhiteOnBlue; replaced read in color with echo in color, read normal
# 2023-05-09 13:10 | Keith Josephson | Include ionstress, memstress & nvmestress results
# 2022-08-17 12:50 | Keith Josephson | Made folder for TPM info a wildcard as the specific device name has changed.
# 2022-06-10 14:40 | Keith Josephson | Added support for sysfwupdt on M50CYP instead of flashupdt.
# 2022-06-09 13:15 | Keith Josephson | PCTs have been discontinued by Intel.  Not working after 2020-12-31 and not working on M50CYP.
# 2022-06-09 13:10 | Keith Josephson | Removing dmraid - no longer present.
# 2020-12-16 16:30 | Keith Josephson | Replaced old code for ARCH & DISTRO to get upmiutil|ipmitool with yum/apt-get code.
# 2020-12-16 16:05 | Keith Josephson | Replaced MegaCli/MegaCli64/CmdTool2 with storcli64
# 2020-02-14 11:25 | Keith Josephson | Remove redundant display of FRU data from ${CFG}
# 2020-02-13 | Keith Josephson | removed virt-inspector again...  Or is it OK when NOT PXE-booted?
# 2020-02-13 | Keith Josephson | provided alternate information from DMI and sensors for systems without IPMI; skip syscfg & flashupdt in that case also.
# 2019-04-01 | Keith Josephson | restored virt-inspector and review of SW manifest
# 2019-03-11 | Keith Josephson | removed virt-inspector2 and review of SW manifest because scorpion 6 does not support.  re-visit!
# 2019-01-23 | Keith Josephson | added mdadm RAID info after dmraid RAID info
# 2017-03-20 | Keith Josephson | filtered output of df -h to only include /dev/sd and /dev/hd points
# 2017-03-09 | Keith Josephson | replaced duplicate code for procs, dimms and I/O with syssum. -s used to also save another copy; -l to print it.
# 2017-03-09 | Keith Josephson | added -e to invocation of syssum to enable listing of empty dimm sockets by dimms
# 2017-01-13 | Keith Josephson | changed "RECORD to REPORT in SCORPION check at end
# 2017-01-13 | Keith Josephson | before grep of PCT at end, check both existance and non-zero size
# 2016-04-29 | Keith Josephson | so changed test of ipmiutil availability to "ipmiutil fru -bc"
# 2016-04-29 | Keith Josephson | sent DIMM "locator" to $CFG instead of screen, for every socket
# 2016-04-29 | Keith Josephson | removed DIMM "locator" for populated sockets only
# 2016-04-29 | Keith Josephson | "ipmiutil health" returns error on SMC systems, 
# 2016-04-29 | Keith Josephson | added "Unknown" to sensor readings to ignore
# 2016-04-29 | Keith Josephson | added rounding to MemTotal reporting
# 2015-06-12 | Keith Josephson | Cleaned up DIMM reporting
# 2015-05-26 | Keith Josephson | Report more FRU data
# 2015-04-08 | Keith Josephson | Added code to capture software manifest using virt-inspector2#

export ORION_VERSION="202410081515"
export ORION_COPYRIGHT="orion Copyright 2007-2024 Keith Josephson @ ION Computer Systems, Inc., Hauppauge, NY, USA"
export TIMESTAMP="`date --utc +%Y%m%d.%H%M`"
export HEADING="${TIMESTAMP}: orion [${ORION_VERSION}]\n${ORION_COPYRIGHT}"
export RETURNHERE=$PWD
FORMFEED="\f"
NUC=false

which lsb_release &>/dev/null
if [ $? -eq 0 ]; then
	OS=`lsb_release -d | sed -e 's/^.*:[ \t]*//'`
else
	# try to find a /etc/__release file.
	for i in `find /etc -maxdepth 1 -name *release -print | head -1`; do
		OS=`cat $i | tr -d "\n"`
	done
fi
if [ ${#OS} -eq 0 ]; then
	OS=`uname -r | tr -d "\n"`
fi

export OS

if [ -x /etc/init.d/ipmi ]; then
	IPMISERVICE="ipmi"
fi
if [ -x /etc/init.d/openipmi ]; then
	IPMISERVICE="openipmi"
fi

usage()
{
	echo "Usage:"
	echo -e "    \033[1;34morion\033[0m [-h]|[-p]|[-V]|[-p <folder>]|[-r <folder>] [sernum]"
	echo "where"
	echo -e "        \033[1;34m${0}\033[0m collects FRU, DMI, sensor, SEL and RAID data fon an ION system."
	echo -e "        -h    prints this message."
	echo -e "        -V    just prints the version of ${0}."
	echo -e "        -p <folder>   skips data collection and justs posts existing data from <folder>. (Use after fixing network.)"
	echo -e "        -n    skips IPMI and SELVIEW on NUC."
	echo -e "        -f <folder>   skips data collection and justs re-views existing data from <folder>."
	echo -e "        sernum   any other value here, at least 5 chars long will be used to identify results."
	echo -e "        ${0} checks DMI, then FRU for a valid looking serial number."
	if [[ NOSN == 1 ]] ; then
		echo -e "\033[1;31m$0 did not find a valid Serial Number on this system.\033[0m"
	else
		echo -e "\033[1;31m$0 would use Serial Number ${SERNUM} on this system.\033[0m"
	fi
}

# reset to screen default colors and clear.
echo -n @[0m | tr @ '\033'
clear

#############################################################
#  Where am I?
#############################################################
export IPMI="/tmp/ipmi.txt"
if [ ! -s "${IPMI}" ]; then
    ipmiutil health >/dev/null 2>&1
    if [ "$?" -eq "0" ]; then
        echo -n "true" >${IPMI}
    else
        ipmiutil health >/dev/null 2>&1
        echo $?
        echo -n "false" >${IPMI}
    fi
fi
export HASIPMI=$(cat ${IPMI})

# echo -n "DMI: "
export DMITMP="/tmp/${TIMESTAMP}.dmi.txt"
dmidecode >>${DMITMP}

# retrieve product name
export PRODUCTNAME="`grep -A6 'System Information' ${DMITMP} \
	| grep 'Product Name:' \
	| sed -e  's/.*: //'`"

# retrieve serial number from DMI
export SERNUM="`grep -A6 'System Information' ${DMITMP} \
	| grep 'Serial Number' \
	| sed -e  's/.*: //' \
	| sed -e 's/ //g'`"

# If DMI Serial Number is short, get it from FRU
# When would that happen?  Does it?
if [[ ${#SERNUM} < 5 ]]; then
	export SERNUM=`ifru -b \
		| grep "\[Baseboard" \
		| grep 'Product Serial Num' \
		| sed -e 's/^.* : //'`
fi

# echo "${PRODUCTNAME} #${SERNUM}" 

if [[ ${#SERNUM} < 5 ]]; then
	NOSN=1
else
	NOSN=0
fi
	
#############################################################
#  What am I supposed to do?  Parse command line
#############################################################

while [ "$#" -gt 0 ]
do
  case "$1" in
  -h)
	usage
	exit 0
	;;
  -V)
	echo "orion: ION trouble hunter"
	echo "Version ${ORION_VERSION}"
	echo "${ORION_COPYRIGHT}"
	exit 0
	;;
  -n)
	NUC=true
	shift
	;;
  -p)
	POSTONLY=
	shift || { usage; exit 1; }
	FOLDER="$1"
	shift
	;;
  -q)
	QA=1
	REASON="ION Quality Assurance"
	WHO="integration@ioncomputer.com"
	echo "$REASON: $WHO"
	shift
	;;

  -f)
	REREAD=1
	shift || { usage; exit 1; }
	FOLDER="$1"
	shift
	;;
  *)
	if [ $NOSN ]; then
		SERNUM="$1"
		NOSN=0
		echo "Using ${SERNUM} for the serial number of this system."
	elif [ "$1" == "${SERNUM}" ]; then
		echo "Serial Number given matches Serial Number found: ${1}"
	else
		echo "Serial Number given, ${1} does not match Serial Number found: ${SERNUM}"
		usage
		exit 1
	fi
	shift
	;;
  esac
done

if [ -z "$POSTONLY" ] && [ -z "$REREAD" ]; then
	# If Serial Number does not look valid, ask operator.
	SERNUMLEN=${#SERNUM}
	while (( SERNUMLEN < 5  || SERNMULEN > 8 )) ; do
		usage
		echo "${SERNUM} (${SERNUMLEN} characters) does not seem to be a valid product serial number. "
		read -p "Please enter the correct S/N: " SERNUM
		SERNUMLEN=${#SERNUM}
	done

	if [ -z "$FOLDER" ]; then
		# If Folder was defined on the command line for use of existing data, do not create a new one; otherwise
		if [ $QA ]; then
			export FOLDER="${SERNUM}.QA"
			rm -rf ${FOLDER}
		else
			export FOLDER="${SERNUM}.${TIMESTAMP}"
		fi
		mkdir $FOLDER
	fi
fi

. boardid -q

cd ${FOLDER}
# ${FOLDER} = (${SERNUM}.${TIMESTAMP}) name is also the basename of all files created in the folder.
export REPORT=${FOLDER}.orion.txt
	echo -e "${HEADING}">>${REPORT}
export BMC=${FOLDER}.bmc.txt
	echo -e "${HEADING}">>$BMC
	echo "BMC and IPMI Configuration" >>$BMC
	echo "#############################################################" >>$BMC
	echo >>$BMC
export CFG=${FOLDER}.cfg.txt
	echo -e "${HEADING}">>$CFG
	echo "Configuration of #${SERNUM}" >>$CFG
	echo "#############################################################" >>$CFG
	echo >>$CFG
export DMI=${FOLDER}.dmi.txt
	echo -e "${HEADING}">>$DMI
	echo "DMI data from #${SERNUM}" >>$DMI
	echo "#############################################################" >>$DMI
	echo >>$DMI
	cat ${DMITMP} >>$DMI
	rm ${DMITMP}
export FRU=${FOLDER}.fru.txt
	echo -e "${HEADING}">>$FRU
	echo "Field Replaceable Units in #${SERNUM}" >>$FRU
	echo "#############################################################" >>$FRU
	echo >>$FRU
export FW=${FOLDER}.fw.txt
	echo -e "${HEADING}">>${FW}
	echo "Firmware on #${SERNUM}" >>${FW}
	echo "######################" >>${FW}
	echo >>${FW}
export SW=${FOLDER}.sw.txt
	echo -e "${HEADING}">>${SW}
	echo "Software on #${SERNUM}" >>${SW}
	echo "######################" >>${SW}
	echo >>${FW}
export RAID=${FOLDER}.raid.txt
	echo -e "${HEADING}">>$RAID
	echo "RAID and disk info for #${SERNUM}" >>$RAID
	echo "#############################################################" >>$RAID
	echo >>$RAID
export SENSOR=${FOLDER}.sensor.txt
	echo -e "${HEADING}">>$SENSOR
	echo "Sensors in #${SERNUM}" >>$SENSOR
	echo "#############################################################" >>$SENSOR
	echo >>$SENSOR
export SEL=${FOLDER}.sel.txt
	echo -e "${HEADING}">>$SEL
	echo "System Event Log of #${SERNUM}" >>$SEL
	echo "#############################################################" >>$SEL
	echo >>$SEL
export INI=${FOLDER}.ini
export PCT=${FOLDER}.pct.txt
export MEMTEST=${FOLDER}.memtest.html
export LOG=${FOLDER}.log.txt
cd ${RETURNHERE}

if [ -z "$POSTONLY" ] && [ -z "$REREAD" ]; then
	cd $FOLDER

	#############################################################
	#  Save DMI data
	#############################################################
#	dmidecode >>$DMI
	
	grep -A1 'Base Board' $DMI | grep 'Manufacturer' | grep -Eq "Intel|Hitachi"
	export IBOARD=$?
	grep -A1 'Base Board' $DMI | grep 'Manufacturer' | grep -q 'Supermicro'
	export SBOARD=$?
	if [ $((SBOARD&IBOARD)) == 1 ]; then export EXEC='ex'; else export EXEC=''; fi
	

	#############################################################
	#  Start creating the report
	#############################################################
	
	echo "Welcome to @[1;34morion@[0m, ION's trouble hunter!" | tr @ '\033'
	echo "   ${ORION_COPYRIGHT}"
	echo "orion: ION's trouble hunter Version:${ORION_VERSION} collecting data at ${TIMESTAMP} [UTC]"
	echo "#$REPORT Version:${ORION_VERSION}" >${REPORT}
	echo "orion collecting data at ${TIMESTAMP} [UTC]" >> ${REPORT}
	echo `ls -oh $0 2>/dev/null | sed -e  's/.*K //' `>>${REPORT}
	echo "This is the system ${PRODUCTNAME} with serial number ${SERNUM}." | tee -a ${REPORT}
	echo

	if [ -z "$REASON" ]; then
		echo "First, so that we may better help you, please tell us why you are running orion."
		echo "Please be @[1;31mSPECIFIC@[0m, including details and a way to contact you." | tr @ '\033'
		echo "@[1;34m1@[0m. I am investigating a current failure indicator." | tr @ '\033'
		echo "@[1;34m2@[0m. I am investigating a failure or other issue." | tr @ '\033'
	
		echo "@[1;34m3@[0m. I am taking a snapshot of a healthy system as a baseline report." | tr @ '\033'
		echo "@[1;34m4@[0m. I am taking a snapshot after maintenance or an upgrade." | tr @ '\033'
		echo "@[1;34m5@[0m. I am experimenting with orion." | tr @ '\033'
		read -p "Type a number [1-5], or type another reason, and then press [Enter]: " REASON
		if [ "$REASON" == "1" ]; then
			echo -n " REASON: I am investigating a current failure indicator: " >>${REPORT}
			echo -n "Please @[1;36mdescribe@[0m the indication or symptoms: " | tr @ '\033'; read SYMPTOM
			while (( ${#SYMPTOM} < 10 )) ; do
				read -p "Please provide more details than just '$SYMPTOM': " SYMPTOM
			done
			echo "$SYMPTOM" >>${REPORT}
		elif [ "$REASON" == "2" ]; then
			echo -n " REASON: I am investigating a failure or other issue: " >>${REPORT}
			echo -n "Please @[1;36mdescribe@[0m the failure or symptoms: " | tr @ '\033'; read SYMPTOM
			while (( ${#SYMPTOM} < 10 )) ; do
				read -p "Please report more details than just '$SYMPTOM': " SYMPTOM
			done
			echo "$SYMPTOM" >>${REPORT}
		elif [ "$REASON" == "3" ]; then
			echo " REASON: I am taking a snapshot of a healthy system as a baseline report." >>${REPORT}
		elif [ "$REASON" == "4" ]; then
			echo -n " REASON: I am taking a snapshot after maintenance or an upgrade: " >>${REPORT}
			echo -n "Please @[1;36mdescribe@[0m what was changed or replaced: " | tr @ '\033'; read CHANGE
			while (( ${#CHANGE} < 10 )) ; do
				read -p "Please report more details than just '$CHANGE': " CHANGE
			done
			echo "$CHANGE" >>${REPORT}
		elif [ "$REASON" == "5" ]; then
			echo " REASON: I am experimenting with orion." >>${REPORT}
		else
			while (( ${#REASON} < 15 )) ; do
				read -p "Please provide more information than just '$REASON': " REASON
			done
			echo " REASON: ${REASON}" >>${REPORT}
		fi
	else
		echo " REASON: ${REASON}" >>${REPORT}
	fi
	
	if [ -z "$WHO" ]; then
		echo
		echo "Next, please tell us who you are: Your name and/or organization."
		echo -e "Please include your \033[1;36mphone\033[0m and/or \033[1;36memail\033[0m so that we may follow up with you."
		read -p "Who? " WHO
        if [ ${WHO} == "II" ]; then
            WHO=Integration@IONcomputer.com
            echo ${WHO}
        fi
		while (( ${#WHO} < 10 )) ; do
			read -p "Please provide more identifcation than just \"${WHO}\": " WHO
		done
	fi
	echo "CONTACT: ${WHO}" >>${REPORT}
	
	echo "Operating system is ${OS}" >>${REPORT}
	uname -snrm >>${REPORT}
	uptime  >>${REPORT}
	echo "Operating system is ${OS}" >>$CFG
	uname -snrm >>$CFG
	if [ -e /etc/ion ]; then 
		echo -n "ION " >>${REPORT}
		cat /etc/ion >>${REPORT}
		echo -n "ION " >>$CFG
		cat /etc/ion >>$CFG
	fi
	if [ $((SBOARD&IBOARD)) == 1 ]; then export UTE='it'; else export UTE=''; fi
	
	DISTRO=$(cat /etc/os-release | grep "PRETTY_NAME" | cut -d "=" -f 2)
	export ARCH=`uname -m`
	# echo "This looks like $DISTRO on $ARCH"
	
    if [ "${HASIPMI}" = true ]; then
	###############################################################################
	# Check for an IPMI package we can work with. If none found, try to get from ION and install.
	###############################################################################
	ipmiutil fru -bc >/dev/null 2>&1
	RETVAL=$?
	if [ $RETVAL -eq 0 ]; then
		export IPMIUTIL=1
		echo -n "ipmiutil is available.  "
	else
		export IPMIUTIL=0
		echo -n "ipmiutil is not available.  "
	fi
	ipmitool -V >/dev/null 2>&1
	RETVAL=$?
	if [ $RETVAL -eq 0 ]; then
		export IPMITOOL=1
		echo -n "ipmitool is available.  "
	else
		export IPMITOOL=0
		echo -n "ipmitool is not available.  "
	fi

	if  [ "${IPMIUTIL}" == "0" ] && [ "${IPMITOOL}" == "0" ]; then
	    if [ "$(echo "${DISTRO}" | grep -Eic "ubuntu|mint")" -ge "1" ]; then
		    apt-get install ipmitool ipmiutil
		else
    	    yum --assumeyes install ipmiutil ipmitool
		fi
	
		# check to see if it worked
		ipmiutil fru -bc >/dev/null 2>&1
		RETVAL=$?
		if [ $RETVAL -eq 0 ]; then
			export IPMIUTIL=1
			echo ipmiutil is now available
		else
			export IPMIUTIL=0
			echo ipmiutil is still not available
		fi
		ipmitool -V >/dev/null 2>&1
		RETVAL=$?
		if [ $RETVAL -eq 0 ]; then
			export IPMITOOL=1
			echo ipmitool is now available
		else
			export IPMITOOL=0
			echo ipmitool is still not available
		fi
	fi
	
	echo
	if  [ "${IPMIUTIL}" == "0" ] && [ "${IPMITOOL}" == "0" ]; then
		echo 'Neither IPMIUTIL nor IPMITOOL is available on this system.'
		echo 'I attempted to install both, but apparently failed.'
		echo 'Your network configuration may not allow internet access.'
		echo 'Please download and install ipmiutil and/or ipmitool for your OS.'
		echo "Neither IPMIUTIL nor IPMITOOL is available on this system." >>${REPORT}
		read -p "Hit [enter] to continue."
	else
		# ipmi collection possible
		if  [ ${IMPIUTIL} ]; then
			ipmiutil | head -1 | tee -a ${REPORT}
		fi
		if  [ ${IPMITOOL} -eq 1 ]; then
			ipmitool -V | tee -a ${REPORT}
		fi
	fi
	###############################################################################
	# End of IPMI Package work
	###############################################################################
    else
        export IPMIUTIL=0
        export IPMITOOL=0
    fi

    if [ "${NUC}" = true ]; then
        export IPMIUTIL=0
        export IPMITOOL=0
    fi

    # RMM4 will be 1, we think, if RMM4 is present and 0 otherwise
	RMM4=$(ipmitool raw 0x30 0x71 0x00 0x01 0x00 0x00 | grep -c "11 02")
    if [ "${RMM4}" == "1" ]; then
        echo "RMM4 installed" >>${BMC}
        echo "RMM4 installed" >>${CFG}
    fi
    # Check for presence of TPM
    # 2022-08-17 12:50 | Keith Josephson | Made folder for TPM info a wildcard as the specific device name has changed.
    # cat /sys/devices/platform/IFX0762\:00/tpm/tpm0/device/firmware_node/description 2>/dev/null  >>${CFG} 
    cat /sys/devices/platform/IFX*/tpm/tpm0/device/firmware_node/description 2>/dev/null  >>${CFG} 
	sleep 1
	echo "SEL: Reading System Event Log may take some time  ..."
	
	###############################################################################
	# The SEL can be very long, so only use one tool, with this priority:
	# 1. Intel selview has the most complete interpretation
	# 2. OLD Intel Selview has a complete interpretation
	# 3. ipmitool has a more readable format than ipmiutil
	# 4. ipmiutil may be on more targets
	###############################################################################

# 2023-08-25 09:55 | Keith Josephson | selview fails on FCP aka ION E1 & E2 servers

#	if  [ -x /usr/bin/selview/selview ] && [ "${NUC}" = false ]; then
	if  [ -x "/usr/bin/selview/selview" ] && [ "${HASIPMI}" = true ] && [ "${SERVERBOARD}" != "M50FCP2SBSTD" ]; then
		echo -n "|selview|text & hex|"
		rm *.sel 2>/dev/null
		TXTSEL=`echo "${FOLDER}_txt" | sed -e 's/\./_/g' -e 's/$/.sel/'`
		HEXSEL=`echo "${FOLDER}_hex" | sed -e 's/\./_/g' -e 's/$/.sel/'`
		/usr/bin/selview/selview /save ${TXTSEL}
		/usr/bin/selview/selview /save /hex ${HEXSEL}
		cat ${TXTSEL} >>${SEL}
		echo "<BR><A HRef=\"/orders/recordreader.cfm?System_UUID=#URL.System_UUID#&amp;RecordType=File&amp;TimeStamp=#URL.TimeStamp#&amp;FileName=${TXTSEL}\">${TXTSEL}</A>" >>${SEL}
		echo "<BR><A HRef=\"/orders/recordreader.cfm?System_UUID=#URL.System_UUID#&amp;RecordType=File&amp;TimeStamp=#URL.TimeStamp#&amp;FileName=${HEXSEL}\">${HEXSEL}</A>" >>${SEL}
	elif  [ -x "/usr/local/selview/Selview" ] && [ "${HASIPMI}" = true ] && [ "${SERVERBOARD}" != "M50FCP2SBSTD" ]; then
		echo -n "|SelView|"
		cd /usr/local/selview
		chmod -R 755 scripts 
		# selview cannot append and prompts for overwrite if file exists
		rm ${SEL} 2>/dev/null
		./Selview ${SEL} /save
		cat ${SEL} >>${RETURNHERE}/${FOLDER}/${SEL}
		cd ${RETURNHERE}/${FOLDER}
	elif  [ ${IPMITOOL} -eq 1 ]; then
		echo -n "|ipmitool|"
		ipmitool sel elist >>${SEL}
	elif  [ ${IPMIUTIL} -eq 1 ]; then
		echo -n "|ipmiutil|"
	    ipmiutil sel -e >>${SEL}
    else
        echo -n "No "
	fi
	echo "SEL                         done."
	sleep 3

	###############################################################################
	#  Now we can start collecting data
	###############################################################################
	echo "FRU"
	if  [ ${IPMIUTIL} -eq 1 ]; then
		ipmiutil fru 2>/dev/null >>$FRU
		let i=0

		while [ `grep -Ec . $FRU` -lt 30 ] && [ $i -lt 4 ] &&  ((IBOARD == 0)); do
#			echo "IBOARD is __${IBOARD}__"
			echo "FRU data retrieved was only `grep -Ec . $FRU` lines. That seems incomplete.  Trying again in 5 seconds."
			echo -e "${HEADING}">$FRU
			echo "Field Replaceable Units in #${SERNUM}" >>$FRU
			echo "#############################################################" >>$FRU
			echo >>$FRU
			sleep 5
			ipmiutil fru 2>/dev/null >>$FRU
			let i=i+1
		done
		if [ $i -eq 10 ]; then
			echo "Seems like we failed to retrieve complete FRU info.  Continuing."
		fi
	elif  [ ${IPMITOOL} -eq 1 ]; then
		ipmitool fru print 2>/dev/null >>$FRU
		echo  >>$FRU
		echo  >>$FRU
		echo  >>$FRU
		echo  >>$FRU
		echo  >>$FRU
	else
		: echo "${SERVERBOARDMFR} ${SERVERBOARD} | IPMI = ${HASIPMI}" >>$FRU
	fi
	if [ ! -e "/tmp/bmc.ini" ]; then
	    syscfg /s /tmp/bmc.ini /f
	fi
    grep --color=never "DHCPHostName" /tmp/bmc.ini | cut -d ";" -f 1 >>$FRU
	
	sleep 2
	echo "SENSORS"
    if [ "${HASIPMI}" = true ]; then
    	if  [ ${IPMIUTIL} -eq 1 ]; then
    		ipmiutil sensor >>$SENSOR
    		ipmiutil alarms -r >>$SENSOR
    	fi
    	if  [ ${IPMITOOL} -eq 1 ]; then
    		ipmitool sensor list >>$SENSOR
    	fi

    	#  These BMC commands seem to leave the BMC unable to respond for a while, so do them LAST.
    	sleep 2
    	echo "BMC"
    	if  [ ${IPMIUTIL} -eq 1 ]; then
    		ipmiutil health -fgh 2>/dev/null >>$BMC
    	fi
    	if  [ ${IPMITOOL} -eq 1 ]; then
    		ipmitool -V >>$BMC
    		if  [ ${IPMIUTIL} -ne 1 ]; then
    			# Only report bmc health with ipmitool if ipmiutil did not just do the same.
    			ipmitool mc info | head -8 >>$BMC
    			ipmitool chassis status >>$BMC
    			ipmitool chassis poh >>$BMC
    		fi
    		ipmitool session info active >>$BMC
    	fi
	else
        sensors >>$SENSOR
    fi
	###############################################################################
	# Build cfg file
	###############################################################################
	
	# System
	echo "DMI" >>$CFG
	grep -A5 'System Information' $DMI \
	    | grep -v 'Not ' \
	    | sed -e 's/[\t ]*//' -e 's/^.*://' \
	    | tr -d '\n' \
	    | sed -e 's/$/\n/' -e 's/ Information/:/' >>$CFG
	# Serverboard
	grep -A5 'Base Board Information' $DMI\
	    | grep -v 'Version'  \
	    | sed -e 's/^.*: //' \
	    | tr '\n' ' ' \
	    | sed -e 's/^[\t ]*//' -e 's/ $/\n/' -e 's/ Information/:/' >>$CFG
	# Chassis
	grep -A6 'Chassis Information' $DMI \
	    | grep -v 'Not Present' \
	    | grep -v 'Not Specified' \
	    | sed -e 's/[\t ]*//' -e 's/^.*://' \
	    | tr -d '\n' \
	    | sed -e 's/$/\n/' -e 's/ Information/:/' >>$CFG
	
	echo 'FRU' >>$CFG
	if [ ${IPMIUTIL} -eq 1 ]; then
	# System
		echo -n 'Product' >>$CFG
		egrep "Baseboard.*Product" $FRU | egrep -v 'Size|File|Name' | sed -e 's/^.*:/ | /' | tr -d "\n" >>$CFG
		echo >>$CFG
#		sed -n -e '/Mainboard FRU/,/successful/p' $FRU \
#			| grep -e '^Product' \
#			| egrep -v ': $|^$|Size|not present' \
#			| sed -e 's/^Product.*: //' \
#			| tr '\n' ' '\
#			| sed -e 's/$/\n\n/' >>$CFG
	elif [ ${IPMITOOL} -eq 1 ]; then
		echo -n 'Product | ' >>$CFG
		ipmitool fru print main | sed -e 's/^ //' | grep -e '^Product' >>$CFG
	else
        grep -A4 "System Information" /tmp/dmi.txt | tail -n +2 >>$CFG
		# echo >>$CFG
	fi
	
	# Baseboard FRU, if available
	if [ ${IPMIUTIL} -eq 1 ]; then
		echo -n 'BaseBoard' >>$CFG
		egrep "Baseboard.*Board" $FRU | egrep -v 'Size|File' | sed -e 's/^.*:/ | /' | tr -d "\n" >>$CFG
		echo >>$CFG
#		sed -n -e '/Mainboard FRU/,/successful/p' $FRU \
#			| grep -e '^Board' \
#			| egrep -v ': $|^$|Size|not present' \
#			| sed -e 's/^Board.*: //' \
#			| tr '\n' ' '\
#			| sed -e 's/$/\n/' >>$CFG
	elif [ ${IPMITOOL} -eq 1 ]; then
		echo -n 'BaseBoard | ' >>$CFG
		ipmitool fru print main | sed -e 's/^ //' | grep -e '^Board' >>$CFG
	else
		: echo "${SERVERBOARDMFR} ${SERVERBOARD} | IPMI = ${HASIPMI}" >>$CFG
	fi
	
	# Chassis
	if [ ${IPMIUTIL} -eq 1 ]; then
		echo -n 'Chassis' >>$CFG
		egrep -i "Baseboard.*Chassis" $FRU | egrep -v 'Size|File' | sed -e 's/^.*:/ |/' | tr -d "\n" >>$CFG
		echo >>$CFG
#		egrep -i "Baseboard.*Chassis" /mnt/ion/work/fru.txt | egrep -v 'Size|Multi|Asset|File' | sed -e 's/^.*:/ |/' | tr -d "\n";
#		sed -n -e '/Mainboard FRU/,/successful/p' $FRU \
#			| grep -e '^Chassis' \
#			| egrep -v ': $|^$|Size|not present' \
#			| sed -e 's/^Chassis.*: //' \
#			| tr '\n' ' '\
#			| sed -e 's/$/\n\n/' >>$CFG
		
	elif [ ${IPMITOOL} -eq 1 ]; then
		echo -n 'Chassis | ' >>$CFG
		ipmitool fru print main | sed -e 's/^ //' | grep -e '^Chassis' >>$CFG
	else
        grep -A6 "Chassis Information" /tmp/dmi.txt | tail -n +2 >>$CFG
		# echo >>$CFG
	fi
	
	# Power, if FRU data is available
	if [ ${IPMIUTIL} -eq 1 ]; then
#		echo -n 'FRU: Chassis FRUs' >>$CFG
#		egrep -A7 '^SDR' $FRU \
#			| egrep -v ': $|^$|Size|error|^Chassis|01 Bas' \
#			| sed -e 's/Serial Num.*: /: S\/N:/' -e 's/Part Num.*: /: P\/N:/' -e 's/^Product.*: //g'  \
#			-e 's/Board //'  -e 's/Number/\#/g' \
#			| tr '\n' ' '\
#			| sed -e 's/SDR/\nSDR/g' -e 's/[\t ]*:/:/g' -e 's/$/\n/' >>$CFG
#		echo "FRU | Manufacturer | Product | PartNumber | Version | SerialNumber" >>$CFG
		echo -n "PowerCage" >>$CFG
        egrep -i "PowerCage" $FRU | egrep -v 'Size|Multi|Asset|File' | sed -e 's/^.*:/ |/' | tr -d "\n"  >>$CFG
		echo  >>$CFG
		echo -n "PowerSupply1" >>$CFG
#		egrep -i "PowerSply.*02]" $FRU | egrep -v 'Size|Multi|Asset|File|OEM' | sed -e 's/^.*:/ |/' | tr -d "\n"  >>$CFG
        grep -B 10 "Power Supply Record" ${FRU} | head -10 | grep -Ev "Asset|File|Record|Part Number" | sed -e 's/^\[.*\] //' >>$CFG
		echo  >>$CFG
		echo -n "PowerSupply2" >>$CFG
#		egrep -i "PowerSply.*03]" $FRU | egrep -v 'Size|Multi|Asset|File|OEM' | sed -e 's/^.*:/ |/' | tr -d "\n"  >>$CFG
        grep -B 10 "Power Supply Record" ${FRU} | tail -10 | grep -Ev "Asset|File|Record|Part Number" | sed -e 's/^\[.*\] //' >>$CFG
		echo  >>$CFG
		PS3=`grep "PS3" $SENSOR | grep -ic "present"`
		if [ $PS3 -gt 0 ]; then
			echo "PowerSupply3 present." >>$CFG
		fi
		RPS=`grep "Redund" $SENSOR | grep -ic "Redundant"`
		if [ $RPS -gt 0 ]; then
			echo "PowerSupply is Redundant." >>$CFG
		fi
	elif [ ${IPMITOOL} -eq 1 ]; then
		echo -n 'Chassis Power' >>$CFG
		egrep -A5 'FRU Device Description : P' $FRU >>$CFG
		PS1=`grep "PS1" $SENSOR | grep -ic "0100"`
		if [ $PS1 -gt 0 ]; then
			echo "PowerSupply1 present." >>$CFG
		fi
		PS2=`grep "PS2" $SENSOR | grep -ic "0100"`
		if [ $PS2 -gt 0 ]; then
			echo "PowerSupply2 present." >>$CFG
		fi
		PS3=`grep "PS3" $SENSOR | grep -ic "0100"`
		if [ $PS3 -gt 0 ]; then
			echo "PowerSupply3 present." >>$CFG
		fi
		RPS=`grep "Redund" $SENSOR | grep -ic "0100"`
		if [ $RPS -gt 0 ]; then
			echo "PowerSupply is Redundant." >>$CFG
		fi
	else
        grep -A1 " Power " /tmp/dmi.txt | grep -Ev "^System|Group|--" | sed -e 'N;s/\n/ /' | sort -u  >>$CFG
		# echo >>$CFG
	fi
	echo  >>$CFG
	
	# Fans
	FANS_OK=`grep 'RPM' $SENSOR | egrep -ic "OK"`
	FANS_NOTOK=`grep 'RPM' $SENSOR | egrep -ivc "OK"`
	if  [ ${IPMIUTIL} -eq 1 ] && [ ${IPMITOOL} -eq 1 ]; then
		let FANS_OK=FANS_OK/2
		let FANS_NOTOK=FANS_NOTOK/2
	fi
	if [ ${FANS_NOTOK} -gt 0 ]; then
		echo "Fans: ${FANS_OK} OK, ${FANS_NOTOK} NOT OK" >>$CFG
	else
        grep -A1 " Fan" /tmp/dmi.txt | grep -Ev "\-\-" | sed -e 'N;s/\n/ /' | sort -u >>$CFG
		# echo "Fans: ${FANS_OK} OK" >>$CFG
	fi
	echo -e "${FORMFEED}" >>$CFG

 	###############################################################################
	# FRU PN & SN to cfg
	###############################################################################
	grep -E "Product Name|Part|Serial" $FRU >>$CFG
    echo -e "${FORMFEED}" >>$CFG

	###############################################################################
	# Processor and RAM info to cfg
	###############################################################################
	# syssum -n -s -e -l >>$CFG
	# syssum -n -s -e >>$CFG
	# 2019-05-20 15:30 | Keith Josephson | if not printed in last 36 hours, print SYStem SUMmary
	if [ -e "/mnt/img/progress/${SERNUM}.IONSUM" ]; then
        LASTPRINTED=$(cut -d " " -f 5 /mnt/img/progress/${SERNUM}.IONSUM | tr "." " ")
        LASTPRINTEDS=$(date -d "${LASTPRINTED}" +"%s")
        NOWS=$(date +"%s")
        let AGE=NOWS-LASTPRINTEDS
        HOURS36=129600
        if [ "${AGE}" -gt "${HOURS36}" ]; then
            syssum -n -s -e -l >>$CFG
        else
            syssum -n -s -e >>$CFG
        fi
	else
	    syssum -n -s -e -l >>$CFG
	fi

	IBA_count=`lspci 2>/dev/null | grep -c "Infini"`
	if [ "${IBA_count}" -gt "0" ]; then
		echo "InfiniBand"  >>$CFG
		lspci 2>/dev/null | grep "Infini" >>$CFG
		echo  >>$CFG
	fi

	echo -e "${FORMFEED}" >>$CFG
	if  [ ${IPMIUTIL} -eq 1 ]; then
		ipmiutil lan -rc -L1 >>$BMC
		ipmiutil lan -rc -L2 | grep -A45 'GetLanEntry' >>$BMC
		ipmiutil lan -rc -L3 | grep -A45 'GetLanEntry' >>$BMC
		echo -n "BMC LAN1: " >>$CFG 
		ipmiutil lan -L1 -r | egrep -io "IP address.*$|MAC addr.*$|DHCP|STATIC" | tr "\n" " " >>$CFG
		echo >>$CFG
		echo -n "BMC LAN2: " >>$CFG 
		ipmiutil lan -L2 -r | egrep -io "IP address.*$|MAC addr.*$|DHCP|STATIC" | tr "\n" " " >>$CFG
		echo >>$CFG
		echo -n "BMC LAN3: " >>$CFG 
		ipmiutil lan -L3 -r | egrep -io "IP address.*$|MAC addr.*$|DHCP|STATIC" | tr "\n" " " >>$CFG
		echo >>$CFG

	elif  [ ${IPMITOOL} -eq 1 ]; then
		ipmitool lan print 1 | egrep -v '=|Gateway|Cipher|MD5' >>$BMC
		ipmitool lan print 2 | egrep -v '=|Gateway|Cipher|MD5' >>$BMC
		ipmitool lan print 3 | egrep -v '=|Gateway|Cipher|MD5' >>$BMC
		ipmitool user list 1 >>$BMC
		ipmitool user list 2 >>$BMC
		ipmitool user list 3 >>$BMC
		echo -n "BMC LAN1: " >>$CFG 
		ipmitool lan print 1 | egrep -o "IP Address  .*$|MAC Addr.*$" | tr "\n" " " | sed -e 's/  */ /g' >>$CFG
		echo >>$CFG
		echo -n "BMC LAN2: " >>$CFG 
		ipmitool lan print 2 | egrep -o "IP Address  .*$|MAC Addr.*$" | tr "\n" " " | sed -e 's/  */ /g' >>$CFG
		echo >>$CFG
		echo -n "BMC LAN3: " >>$CFG 
		ipmitool lan print 3 | egrep -o "IP Address  .*$|MAC Addr.*$" | tr "\n" " " | sed -e 's/  */ /g' >>$CFG
		echo >>$CFG
	fi
	echo -e "${FORMFEED}" >>$CFG

	if [ -s ~/${SERNUM}.progress.txt ]; then
		echo "Integration Log for ${SERNUM}:"  >>$CFG
		# integration progress timestamps
		cat ~/${SERNUM}.progress.txt >>$CFG 
		echo -e "${FORMFEED}" >>$CFG
	fi

	###############################################################################
	# Get BIOS and FirmWare
	###############################################################################
	grep -A2 'BIOS Information' $DMI\
	    | tail -1 \
	    | sed -e 's/^.*Version/BIOS/' -e 's/tion/tion:/' >>${FW}
	dmidecode -s bios-version >>${FW}
	
	###############################################################################
	# If flashupdt or syscfg are available, then NO LONGER stop ipmi to use them.
	###############################################################################
	$EXEC$UTE 2>/dev/null
	if ( [ "${SERVERBOARD}" == "M50CYP2SBSTD" ] || [ "${SERVERBOARD}" == "M50CYP2SB1U" ] ) && [ "$PXE" != true ] && [ "$IBOARD" == "0" ] && [ "${HASIPMI}" = true ]; then
		echo 'FW: sysfwupdt'
		sysfwupdt -i >/dev/null
		if [[ $? -eq 0 ]]; then
			sleep 1
			sysfwupdt -i | grep -E "Vers|Code" | grep -v "Util" >>${FW}
			echo "######################" >>${FW}
			echo >>${FW}
		fi

		# cd ${RETURNHERE}/${FOLDER}
		stty sane
	elif [ `which flashupdt` ] && [ "$PXE" != true ] && [ "$IBOARD" == "0" ] && [ "${HASIPMI}" = true ]; then
		echo 'FW: flashupdt'
		flashupdt -i >/dev/null
		if [[ $? -eq 0 ]]; then
			sleep 1
			flashupdt -i | grep -E "Vers|Code" | grep -v "Util" >>${FW}
			echo "######################" >>${FW}
			echo >>${FW}
		fi

		# cd ${RETURNHERE}/${FOLDER}
		stty sane
	fi
    if [ -x /usr/bin/syscfg/syscfg ] && [ "$IBOARD" == "0" ] && [ "${HASIPMI}" = true ]; then
		echo 'FW: syscfg'
		/usr/bin/syscfg/syscfg -i | tail --lines=+5 | tr "\n" "~" | sed -e 's/~~/~/' | tr "~" "\n" >>${FW}
		echo -e "${FORMFEED}" >>$CFG
		echo "syscfg: BIOS & BMC FW settings"
		/usr/bin/syscfg/syscfg -bbosys >>$CFG
		/usr/bin/syscfg/syscfg -bbosys >>$RAID
		rm -f $INI 2>/dev/null
		/usr/bin/syscfg/syscfg -save ${INI} /b /f
		stty sane
	elif [ -s /usr/local/syscfg/syscfg ] && [ "$IBOARD" == "0" ] && [ "${HASIPMI}" = true ]; then
		# service tam stop
		# ipmi_stop
		# service $IPMISERVICE stop
		echo 'FW: syscfg'
		cd /usr/local/syscfg
		./syscfg -i | tail --lines=+5 >>${RETURNHERE}/${FOLDER}/${FW}
		echo -e "${FORMFEED}" >>$CFG
		echo "syscfg: BIOS & BMC FW settings"
		./syscfg -bbosys >>${RETURNHERE}/${FOLDER}/$CFG
		./syscfg -bbosys >>${RETURNHERE}/${FOLDER}/$RAID
		rm -f $INI 2>/dev/null
		./syscfg -save ${INI} /b /f
		mv $INI ${RETURNHERE}/${FOLDER}/$INI
		cd ${RETURNHERE}/${FOLDER}
		stty sane
	fi
	grep -A5 'OEM Strings' $DMI >>${FW}

	###############################################################################
	# Look for RAID Controllers.
	# Put available RAID info in both full and ORION reports.
	###############################################################################
	
    # 2024-10-08 14:40 | Keith Josephson | if there is a progress file for drive bay mapping, display that
    # 2024-10-08 15:15 | Keith Josephson | added this to syssum, so it should have already displayed.

#    if [ -s "/tmp/${SERNUM}.DriveMap" ]; then
#        more /tmp/${SERNUM}.DriveMap
#        echo -e "${FORMFEED}" >>$RAID
#    elif ${atION} && [ -s "/mnt/img/progress/${SERNUM}.DriveMap" ]; then
#        more /mnt/img/progress/${SERNUM}.DriveMap
#        echo -e "${FORMFEED}" >>$RAID
#	fi
	
	# First, list info on disks and partitions found
	which fsarchiver >/dev/null 2>&1
	if [[ $? -eq 0 ]]; then
        fsarchiver probe >>$RAID 2>&1
        echo -e "${FORMFEED}" >>$RAID
	fi

    echo "mdraid" >>$RAID
    MDLIST=$(mdadm --detail --scan | cut -d " " -f 2)
    if [ "${#MDLIST}" -eq "0" ]; then
        echo "none found" >>$RAID
    else
        for I in $MDLIST; do
            mdadm --detail ${I} >>$RAID
            echo "================" >>$RAID
            echo >>$RAID
        done
    fi
    
    # 2023-05-09 11:25 | Keith Josephson | 3ware has been gone for years; removing collection of 3ware data in RAID.

	# Areca
	export HAVE_ARECA_CLI=0
	export ARECA32=0
	export ARECA64=0
	export NUMareca=`lspci | grep -c 'Areca'`
	if [ $NUMareca -gt 0 ]; then
		echo "$NUMareca Areca controller(s) present."
		# make sure we have cli32 or cli64
		cli32 sys info >/dev/null 2>&1
		if [[ $? -eq 0 ]]; then
			export ARECA32=1
			export HAVE_ARECA_CLI=1
			export ARECA_CLI=cli32
		fi
		cli64 sys info >/dev/null 2>&1
		if [[ $? -eq 0 ]]; then
			export ARECA64=1
			export HAVE_ARECA_CLI=1
			export ARECA_CLI=cli64
		fi

		if [ $HAVE_ARECA_CLI -eq 0 ]; then
			# see if FalconStor has it hidden
			find /root -name cli64 -exec cp {} /usr/local/sbin \;
			chmod 700 /usr/local/sbin/cli64
			cli64 sys info >/dev/null 2>&1
			if [[ $? -eq 0 ]]; then
				echo "found cli64"
				export ARECA64=1
				export HAVE_ARECA_CLI=1
				export ARECA_CLI=cli64
			fi
		fi
	
		if [ $HAVE_ARECA_CLI -eq 0 ]; then
			# we have to go get it
			echo 'user anonymous orion@ioncomputer.com' >gettools
			echo 'bin' >>gettools
			echo 'cd tools' >>gettools
			echo 'lcd /usr/local/sbin' >>gettools
			if [ "$ARCH" == "i386" ] || [ "$ARCH" == "i586" ] || [ "$ARCH" == "i686" ]; then
				echo 'get cli32' >>gettools
				export ARECA_CLI=cli32
			elif [ "$ARCH" == "x86_64" ]; then
				echo 'get cli64' >>gettools
				export ARECA_CLI=cli64
			else
				echo "Cannot figure out architecture.  No areca cli."
			fi
			echo bye >>gettools
			echo -n "No Areca management CLI ($ARECA_CLI) was found on this system. Trying to obtain it..."
			ftp -i -n ftp.ioncomputer.com <gettools >/dev/null 2>&1
			rm gettools
			chmod 711 /usr/local/sbin/$ARECA_CLI
			# Check for success
			which $ARECA_CLI >/dev/null 2>&1
			if [[ $? -eq 0 ]]; then
				export HAVE_ARECA_CLI=1
				echo '  success.'
			else
				echo '  fail.'
			fi
		fi
		if [ $HAVE_ARECA_CLI -eq 1 ]; then
			for (( i=1 ; i <= $NUMareca ; i++ )) ; do  # for each Areca card
				$ARECA_CLI ctrl=$i set password=0000
				ARCSERNUM=`$ARECA_CLI ctrl=$i sys info | grep 'Serial' | sed -e 's/^.*: //'`
				ARCMODEL=`$ARECA_CLI ctrl=$i sys info | grep 'Name' | sed -e 's/^.*: //'`
				ARCHDR="Areca #${i}: ${ARCMODEL} S/N ${ARCSERNUM}"
				$ARECA_CLI sys info ctrl=$i | grep -Ev 'GuiErrMsg' >>$RAID
				if [ -s /mnt/img/progress/${SERNUM}.ARECA_FW_$i ]; then
					cat /mnt/img/progress/${SERNUM}.ARECA_FW_$i >>$RAID
				elif [ -d /mnt/img/progress ]; then
					echo "There is no record of FW update for Areca #${i}: ${ARCMODEL} S/N ${ARCSERNUM}" >>$RAID
				fi
				echo -e "${FORMFEED}" >>$RAID
				echo ${ARCHDR} >>$RAID
				$ARECA_CLI sys showcfg ctrl=$i | grep -Ev 'GuiErrMsg' >>$RAID
				echo -e "${FORMFEED}" >>$RAID
				echo ${ARCHDR} >>$RAID
				$ARECA_CLI hw info ctrl=$i | grep -Ev 'GuiErrMsg' >>$RAID
				$ARECA_CLI net info ctrl=$i | grep Local | grep -Ev 'GuiErrMsg' >>$RAID
				echo -e "${FORMFEED}" >>$RAID
				echo ${ARCHDR} >>$RAID
				$ARECA_CLI rsf info ctrl=$i | grep -Ev 'GuiErrMsg' >>$RAID
				$ARECA_CLI vsf info ctrl=$i | grep -Ev 'GuiErrMsg' >>$RAID
				echo -e "${FORMFEED}" >>$RAID
				echo ${ARCHDR} >>$RAID
#	On SAS controllers, this output does not include serial number or much else besides drive num and model
				$ARECA_CLI disk info ctrl=${i} >/tmp/areca_${i}_disks
				echo "It takes a little while to collect Areca disk info."
				echo -e "${FORMFEED}" >>$RAID
				echo ${ARCHDR} >>$RAID
				for D in `grep -Ev "N.A.|===|Model|GuiErrMsg" /tmp/areca_${i}_disks | sed -e 's/^ *//' -e 's/[ \t].*//'` ; do
					if [[ ${D} -gt 9 ]]; then
						FILL=""
					else
						FILL="0"
					fi
					echo "C${i} D${FILL}${D}: "
					echo -n "C${i} D${FILL}${D}: " >>$RAID
					$ARECA_CLI disk info ctrl=$i drv=$D \
						| grep -Ev '===|Info|GuiErrMsg|SMART|Count' \
						| sed -e 's/^.*: //' -e 's/(.*)//' -e 's/ *$//' -e 's/ C$/C/' \
						-e 's/Enclosure/E/'  -e 's/[Ll][Oo][Tt][ \t]/#/' \
						| tr '\n' ' ' >>$RAID
					grep -E "^[ ][ ]*${D}" /tmp/areca_${i}_disks \
						| sed -e 's/^.*GB  //' -e 's/ //g' >>$RAID
				done
				$ARECA_CLI event info ctrl=$i >>$RAID
				echo -e "${FORMFEED}" >>$RAID
			done
		else
			echo "${NUMareca} Areca RAID controllers were found but cli32/cli64 was not available." \
					| tee -a $RAID
			echo "Your network configuration may not allow internet access."
			echo "Please download a copy of $ARECA_CLI from ftp://ftp.ioncomputer.com/tools/"
			read -p "Hit [enter] to continue."
		fi
	fi
	
	# MegaRAID from LSI Logic or OEM by Intel
	# For now, we are assuming that MegaCLI is already here for MegaRAID systems
	let NUMmegaPCI=`lspci 2>/dev/null | grep -Ec "RAID bus controller: LSI|MegaRAID"`+`lsmod | grep -c "^megasr"`
	export NUMmegaPCI
	if [[ $NUMmegaPCI -gt 0 ]]; then
		MEGACLI=NO
		# skip next section - converting to storcli64
		if false; then
		which MegaCli64
		if [[ $? -eq 0 ]]; then
			MEGACLI=MegaCli64
		else
			which MegaCli
			if [[ $? -eq 0 ]]; then
				MEGACLI=MegaCli
			else
				which CmdTool2
				if [[ $? -eq 0 ]]; then
					MEGACLI=CmdTool2
				else
					echo "${NUMmegaPCI} MegaRAID controllers were found but MegaCli was not available." \
						| tee -a $RAID
				fi
			fi
		fi
		fi
        which storcli64
			if [[ $? -eq 0 ]]; then
			    MEGACLI="storcli64"
            else
                MEGACLI="NO"
            fi
		if [ "${MEGACLI}" != "NO" ]; then

			echo "$NUMmegaPCI MegaRAID controller(s) present."
			# ${MEGACLI} -adpcount >/dev/null 2>&1
			# export NUMmega=$?	# Command returns count
		    export NUMmega=$(storcli64 show ctrlcount | grep "Count" | tr -s " " | cut -d " " -f 4)
			echo "$NUMmega MegaRAID controllers reported by storcli64."
			for (( i=0 ; i < $NUMmega ; i++ )) ; do  # for each MegaRAID card
				echo MegaRAID \#${i} Adapter, RAID, Volume and Disk info >>$RAID
				# ${MEGACLI} -ShowSummary -a${i} 2>/dev/null >>$RAID
				storcli64 /c${i} show all 2>/dev/null >>$RAID
				if [[ $? -eq 0 ]]; then
					# Not all FW supports ShowSummary. If it does succeed, add a FF.
					echo -e "${FORMFEED}" >>$RAID
				fi
				# storcli64 /c0 show all includes everything from here.
				# ${MEGACLI} -AdpAllInfo -a${i} 2>/dev/null | head -25 >>$RAID
				echo -e "${FORMFEED}" >>$RAID
				# storcli64 /c0 show all includes everything from here.
				# ${MEGACLI} -ELF -ControllerFeatures -a${i} 2>/dev/null >>$RAID
				# storcli64 /c0 show all includes everything from here.
				# ${MEGACLI} -PdList -a${i} | egrep 'Enclosure D|Slot|Inquiry|Firmware state' | tr "\n" " " \
                #    | sed -e 's/Enclosure Device ID/\nEnc ID/g' -e 's/ST/ ST/g' -e 's/ Number//g' -e 's/Firmware //g' \
                #    | sed -e 's/[ \t][ \t]*/ /g' >>$RAID
				# echo "" >>$RAID
				# echo -e "${FORMFEED}" >>$RAID
				# storcli64 /c0 show all includes everything from here.
				# ${MEGACLI} -LDInfo -lAll -a${i} 2>/dev/null >>$RAID
				# echo -e "${FORMFEED}" >>$RAID
				# storcli64 /c0 show all includes everything from here.
				# ${MEGACLI} -ldpdinfo -a${i} | grep -iE "adapter|virtual|level|^size|^state|span|Type|PD:|enclosure|slot|inquiry|firmware" \
                #   | sed -e '/^Span D/d'  -e 's/ Device//' \
                #   | sed -e '/Info/N;s/\n/ /' -e '/Info/N;s/\n/ /' -e '/Info/N;s/\n/ /' -e '/Info/N;s/\n/ /' \
                #   | -e '/Info/N;s/\n/ /' -e '/Info/N;s/\n/ /' -e 's/ [ \t]*/ /g' \
                #   | | sed -e 's/Information //' -e 's/Firmware state: //' -e 's/ST/ ST/g' -e 's/ Number//' -e 's/Virt/---\nVirt/'\
                #   | -e 's/PD Type//' -e 's/ Type//' -e 's/ Inquiry Data//'  -e 's/- of/# of/' >>$RAID
				# echo -e "${FORMFEED}" >>$RAID
				# storcli64 /c0 show all includes everything from here.
				# ${MEGACLI} -AdpBbuCmd -a${i} 2>/dev/null >>$RAID
				# echo -e "${FORMFEED}" >>$RAID
				# echo "MegaRAID \#${i} Adapter Event Log Info" >>$RAID
				# ${MEGACLI} -AdpEventLog -GetEventLogInfo -a${i} >>$RAID
#				echo "See MegaRAID_${i}.log for logs and MegaRAID_${i}.txt for full Adapter, array and disk details" >>$RAID
				echo "<BR><A HRef=\"/orders/recordreader.cfm?System_UUID=#URL.System_UUID#&amp;RecordType=File&amp;TimeStamp=#URL.TimeStamp#&amp;FileName=${FOLDER}.MegaRAID_${i}.txt\">${FOLDER}.MegaRAID_${i}.txt: AdpAllInfo + LDPDInfo</A>" >>$RAID
				echo "<BR><A HRef=\"/orders/recordreader.cfm?System_UUID=#URL.System_UUID#&amp;RecordType=File&amp;TimeStamp=#URL.TimeStamp#&amp;FileName=${FOLDER}.MegaRAID_${i}.log\">${FOLDER}.MegaRAID_${i}.log: EventLog + EliLog</A>" >>$RAID
#				echo "<BR><A HRef=\"MegaRAID_${i}.log\">MegaRAID_${i}.log: EventLog + EliLog</A>" >>$RAID
				# ${MEGACLI} -LDPDInfo -a${i} 2>/dev/null >>${FOLDER}.MegaRAID_${i}.txt
				# ${MEGACLI} -AdpAllInfo -a${i} 2>/dev/null >>${FOLDER}.MegaRAID_${i}.txt
				# ${MEGACLI} -AdpEventLog -GetEvents -f ${FOLDER}.MegaRAID_${i}.log -a${i}
				# ${MEGACLI} -AdpAlILog -a${i} 2>/dev/null >>${FOLDER}.MegaRAID_${i}.log 
				storcli64 /c${i} show all 2>/dev/null >>${FOLDER}.MegaRAID_${i}.log
				storcli64 /c${i} show Alilog 2>/dev/null >>${FOLDER}.MegaRAID_${i}.log
				storcli64 /c${i} show termlog 2>/dev/null >>${FOLDER}.MegaRAID_${i}.log
				echo -e "${FORMFEED}" >>$RAID
			done
			echo "<BR><A HRef=\"/orders/recordreader.cfm?System_UUID=#URL.System_UUID#&amp;RecordType=File&amp;TimeStamp=#URL.TimeStamp#&amp;FileName=MegaSAS.log\">MegaSAS.log</A>" >>$RAID
		fi		
	fi		

    # 2023-05-09 11:30 | Keith Josephson | Have not used Adaptec RAID in years; removing collection of Adaptec data in RAID.

	#LSI MPT Adapters
	export NUMMPT=`lspci 2>/dev/null | grep -c 'Fusion-MPT'`
	if [[ $NUMMPT > 0 ]]; then
		which cfggen 2>/dev/null
		if [[ $? -eq 0 ]]; then
			modprobe mptctl
			cfggen LIST  >>$RAID
		fi
	fi
	# on Fox Cove, the MPT without key still says RAID bus...
	which cfggen 2>/dev/null
	if [[ $? -eq 0 ]]; then
	for (( i=0 ; i < $NUMMPT ; i++ )) ; do  # for each LSI MPT Adapter
		echo LSI MPT \#$i Controller and Enclosure info >>$RAID
		cfggen $i DISPLAY | grep -B1 -A5 'Controller information' >>$RAID
		echo -e "${FORMFEED}" >>$RAID
		cfggen $i DISPLAY | grep -B1 -A4 'Enclosure information' | grep -v 'Logical ID ' >>$RAID
		echo -e "${FORMFEED}" >>$RAID
		echo LSI MPT \#$i Volume info >>$RAID
		cfggen $i DISPLAY | grep -B1 -A1 'IR Volume information' >>$RAID
		cfggen $i DISPLAY | grep -A5 'IR volume' >>$RAID
		echo -e "${FORMFEED}" >>$RAID
		echo LSI MPT \#$i Physical Disk info >>$RAID
		cfggen $i DISPLAY | grep -A11 'Target on ID' >>$RAID
		echo -e "${FORMFEED}" >>$RAID
	done
	fi
	
#	if [[ $NUMmegaPCI -eq 0 ]]; then
		# look for simple disks
        echo Other Disks: >>$RAID
        grep Model /proc/scsi/scsi >>$RAID
        echo -e "${FORMFEED}" >>$RAID
        	
		echo "probing IDE disks" >>$RAID
		for I in `ls -B /dev/hd? 2>/dev/null` ; do 
			if [[ `smartctl -a $I 2>/dev/null | grep -c Device 2>/dev/null` > 0 ]]; then
				echo -en "$I:\t" >>$RAID
				smartctl -a $I | egrep 'Device:|Model:|Serial|Health' \
					>>$RAID 2>/dev/null
					echo >>$RAID
			fi
			parted -s ${I} print >>$RAID 2>/dev/null
		done
		echo "probing SCSI disks" >>$RAID
		for I in `ls -B /dev/sd? 2>/dev/null` ; do 
			echo -en "$I:\t" >>$RAID
			scsi_id -g -i -p 0x80 -s ${I/dev/block} >>$RAID 2>/dev/null
			smartctl -a $I | egrep 'Device:|Model:|Serial|Health|Wearout' >>$RAID 2>/dev/null
			parted -s ${I} print | grep -iv "unrecognised" >>$RAID 2>/dev/null
			echo
		done
#	fi

	df -h | grep -E "/dev/sd|/dev/hd|Filesystem" >>$RAID
	if [[ ${#EJECT} -eq 1 ]]; then
		if [ "$EJECT" = "Y" ] || [ "$EJECT" = "y" ] ; then
			echo "Optical disk ejected successfully." >>$RAID
		elif [ "$EJECT" = "N" ] || [ "$EJECT" = "n" ] ; then
			echo "Optical disk failed to eject." >>$RAID
		elif [ "$EJECT" = "M" ]; then
			echo "OS did not find an optical disk." >>$RAID
		fi
	else
		echo "Status of optical disk is unknown: $EJECT." >>$RAID
	fi

	if [ -n "${SCORPION}" ]; then
		# Probably in a QA run.  SCORPION is defined.
		if [ "${SCORPION}" = "Y" ] || [ "${SCORPION}" = "y" ] ; then
				echo -n "SCORPION created" >>${REPORT}
		elif [ "${SCORPION}" = "N" ] || [ "${SCORPION}" = "n" ] ; then
				echo "No record of SCORPION creation." >>${REPORT}
		else
				echo "Status of SCORPION is ${SCORPION}." >>${REPORT}
		fi
	fi
	
	# SW manifest collection
	# PARTLIST=`fdisk -l | grep "^/dev" | cut -f 1 -d " "`
	# scorpION6 does not include virt-inspector2, so skip?
	# DISKLIST=`fdisk -l | grep "^Disk" | grep ":" | cut -f 2 -d " " | sed -e 's/://'`
    # DISKLIST=$(ls -1 /sys/block | grep -E 'hd|sd|cciss|sr|nvme' | tr "\!" "/")
	# for I in ${DISKLIST}; do
    #     SWTIMESTAMP="`date --utc +%Y%m%d.%H%M`"
    #     echo "Collecting Software Manifest for ${I}.  This may take a couple of minutes."
    #     echo >>${SW}
    #     echo "<!-- Software Manifest for ${I} at ${SWTIMESTAMP} -->">>${SW}
    #     echo >>${SW}
    #     virt-inspector2 ${I} >>${SW}
	# done
    
    # PARTLIST=$(fdisk -l 2>/dev/null | grep "^/" | cut -d " " -f 1 | sed -e 's/^/-a /' | tr "\n" " ") 
    # Does this work with, for example, target disk AND scorpion installed?
    # virt-inspector ${PARTLIST}  >>${SW}

	cd ..
	
fi # End of data collection if NOT POSTONLY and NOT REREAD

if [ -z "$REREAD" ]; then
# 2022-06-09 13:15 | Keith Josephson | PCTs have been discontinued by Intel.  Not working after 2020-12-31 and not working on M50CYP.
###############################################################################
# Get last PCT result from Service Partition
###############################################################################
#if [ -s /mnt/sp/pct/result.log ]; then
#	cp /mnt/sp/pct/result.log ${FOLDER}/${PCT}
#fi
#if [ -s /mnt/sp/diags/result.log ]; then
#	cat /mnt/sp/diags/result.log >>${FOLDER}/${PCT}
#fi
#if [ -s /mnt/sp/efipct/result.log ]; then
#	cat /mnt/sp/efipct/result.log >>${FOLDER}/${PCT}
#fi
#if [ -s ~/${SERNUM}.pct.txt ]; then
#	# qa retrieved it from image server and put it here.
#	cat ~/${SERNUM}.pct.txt >>${FOLDER}/${PCT}
#fi
#if [ ! -s "${FOLDER}/${PCT}" ]; then
#    echo "No PCT results found."
#fi

###############################################################################
# Get last MemTest result from EFI Service Partition
###############################################################################
MEMTESTHTML=$(ls -1 /boot/efi/EFI/BOOT/Mem*.html | tail -1)
if [ "${#MEMTESTHTML}" -gt "0" ]; then
    cp -v ${MEMTESTHTML} ${FOLDER}/${MEMTEST}
    cp /boot/efi/EFI/BOOT/mt86.png ${FOLDER}/mt86.png
# else
#     echo "No MemTest86 results found."
fi

###############################################################################
# Get ${SERNUM}.ionstress, ${SERNUM}.memstress, ${SERNUM}.nvmestress result from $CWD
###############################################################################
if ${atION}; then
    # if [ -s "/mnt/img/progress/${SERNUM}.memstress" ]; then
    #     cp -v /mnt/img/progress/${SERNUM}.memstress ${FOLDER}/${FOLDER}.memstress.txt
    # else
    #     echo "No memstress results found."
    # fi
    # 2023-05-09 11:25 | Keith Josephson | If present in $CWD, get ${SERNUM}.ionstress, ${SERNUM}.memstress, ${SERNUM}.nvmestress
    # qa should have already copied from /mnt/img/progress to $CWD
    if [ -s "${SERNUM}.ionstress" ]; then
        /bin/cp ${SERNUM}.ionstress ${FOLDER}/${FOLDER}.ionstress.txt
    fi
    if [ -s "${SERNUM}.memstress" ]; then
        /bin/cp ${SERNUM}.memstress ${FOLDER}/${FOLDER}.memstress.txt
    fi
    if [ -s "${SERNUM}.nvmestress" ]; then
        /bin/cp ${SERNUM}.nvmestress ${FOLDER}/${FOLDER}.nvmestress.txt
    fi
fi

###############################################################################
# Package report
# 2012-11-21 Moved from end of data collection 24 lines above.  Should include PCT.
###############################################################################
cp /var/log/messages ${FOLDER}/$LOG
ls -sh1  >>${FOLDER}/${REPORT}
# 2012-11-21 Added links to other files.

if [ -s ${FOLDER}/${INI} ]; then
	echo "<BR><A HRef=\"/orders/recordreader.cfm?System_UUID=#URL.System_UUID#&amp;RecordType=File&amp;TimeStamp=#URL.TimeStamp#&amp;FileName=${INI}\">${INI}: Intel SysCfg BIOS &amp; FW settings</A>" >>${FOLDER}/${REPORT}
fi
if [ -s ${FOLDER}/${LOG} ]; then
	echo "<BR><A HRef=\"/orders/recordreader.cfm?System_UUID=#URL.System_UUID#&amp;RecordType=File&amp;TimeStamp=#URL.TimeStamp#&amp;FileName=${LOG}\">${LOG}: /var/log/messages at ${TIMESTAMP}</A>" >>${FOLDER}/${REPORT}
fi
echo "End of ${TIMESTAMP} orion report." >>${FOLDER}/${REPORT}
tar cvzf ${FOLDER}.tgz $FOLDER

###############################################################################
# Save reports to Service Partition
###############################################################################
if [ -s /mnt/sp/autoexec.bat ] || [ -s /mnt/sp/startup.nsh ]; then
	mkdir /mnt/sp/orion 2>/dev/null
	cp ${FOLDER}.tgz /mnt/sp/orion/${FOLDER}.tgz
fi

###############################################################################
# Upload reports
###############################################################################

echo "Uploading reports at `date`."
echo "If this takes more than a few seconds, press [Control-C] once to skip ahead."
echo
echo "user anonymous orion@ioncomputer.com" >putreport
echo "bin" >>putreport
echo "cd orion" >>putreport
echo "put ${FOLDER}.tgz" >>putreport
echo "ls ${FOLDER}.tgz" >>putreport
echo "get check ${TIMESTAMP}.check" >>putreport
# Update ORION for next time.
echo "cd .." >>putreport
echo "cd tools" >>putreport
echo "newer orion ${0}.new" >>putreport
echo "bye" >>putreport
ftp -n ftp.ioncomputer.com <putreport >/dev/null 2>&1

if [ -s ${TIMESTAMP}.check ]; then
	echo "Upload of "
	echo "           ${FOLDER}.tgz"
	echo " probably succeeded."
	rm ${TIMESTAMP}.check
	read -t 4
else
	echo "Upload of "
	echo "           ${FOLDER}.tgz"
	echo " probably failed."
	echo
	echo "Please upload "
	echo "           ${FOLDER}.tgz"
	echo "to ftp://ftp.ioncomputer.com/orion/"
	echo "${FOLDER}.tgz"
	echo "    can be found in `pwd`"
	if [ -s /mnt/sp/orion/$REPORT ]; then
		echo
		echo "${FOLDER}.tgz is also available in the"
		echo "    \orion folder of the DOS partition on this device"
		echo "    upload.sh for linux or upload.bat for windows"
		echo "    will transfer the result to the appropriate destination."
	fi
fi
# if [ -s ${0}.new ]; then
# 	mv ${0}.new ${0}; chmod 755 $0; return \
# fi
rm putreport

fi
# End of posting data, if NOT REREAD

if [ -z "$POSTONLY" ]; then

# END of orion; show results

# source colorterm.sh
Start_WhiteOnBlue="echo -en \\033[1;97;44m"
Start_Normal="echo -en \\033[0m"

RES_COL=$((`tput cols` - 6))
Res_Col=$((`tput cols` - 6))
MOVE_TO_STATUS="echo -en \\033[${RES_COL}G"
Move_To_Status="echo -en \\033[${RES_COL}G"
GREEN_OK="echo -e \\033[32m  [OK]\\033[0m"
Green_OK="echo -e \\033[32m  [OK]\\033[0m"
GREEN_DONE="echo -e \\033[32m[DONE]\\033[0m"
Green_Done="echo -e \\033[32m[DONE]\\033[0m"
RED_FIX="echo -e \\033[31m [DO]\\033[0m"
Red_Fix="echo -e \\033[31m [DO]\\033[0m"

# BLUE="\\033[34m"
# NORMAL="\\033[0m"

${Start_WhiteOnBlue}
read -n 1 -p 'Hit [anykey] to view system configuratION.'
${Start_Normal}
# 2020-02-14 11:25 | Keith Josephson | Remove redundant display of FRU data from ${CFG}
# more -p ${FOLDER}/${CFG}
sed -e '/FRU ~SYSSUM~/,/Go pick up your hard copy/d' ${FOLDER}/${CFG} | more -p
${Start_WhiteOnBlue}
echo -n 'Hit [anykey] to view system FirmWare. '
${Start_Normal}
read -n 1
more -p ${FOLDER}/${FW}

${Start_WhiteOnBlue}
echo -n 'Hit [anykey] to view disk and RAID configuratION. '
${Start_Normal}
read -n 1
grep -v "HRef" ${FOLDER}/${RAID} | more -p 
Start_
# ${START_BLUE}
# read -n 1 -p 'Hit [anykey] to view SW Manifest.'
# ${Start_Normal}
# echo
# cat ${FOLDER}/${SW} | sed -n -e '/<operatingsystem>/,/<\/mountpoints>/p' | more

if [ -z "$QA" ]; then
	${Start_WhiteOnBlue}
	echo -n 'Hit [anykey] to view all sensors. '
	${Start_Normal}
	read -n 1
	more -p ${FOLDER}/${SENSOR}
fi

${Start_WhiteOnBlue}
echo -n 'Hit [anykey] to check sensor readings that may not be expected. '
${Start_Normal}
read -n 1
grep ' SDR ' ${FOLDER}/${SENSOR} \
	| egrep -v 'OK|Absent|Present|NotAvailable|OEM|Working|Enabled|Asso|IPMB|FRU|Ready|VRD Temp|0028 SDR|Session Audit|Unknown' \
	| more -p

echo
${Start_WhiteOnBlue}
echo -n 'Hit [anykey] to check sensors for certain other sensor states. '
${Start_Normal}
read -n 1

# Explicitly look for certain other states.
# A fan, for example can be both "Present" AND "Faulty"
grep ' SDR ' ${FOLDER}/${SENSOR} | egrep 'Faulty|Fail|Warn|Crit|RedunDegr' | egrep -v 'VRD Temp|VRD Hot|Unknown' | more -p

echo
${Start_WhiteOnBlue}
echo -n "Hit [anykey] to check for ECC entries in System Event Log (SEL). "
${Start_Normal}
read -n 1
grep 'ECC' ${FOLDER}/${SEL} | more -p
echo
${Start_WhiteOnBlue}
echo -n "Hit [anykey] to check for ERR entries in System Event Log (SEL)."
${Start_Normal}
read -n 1
grep 'ERR' ${FOLDER}/${SEL} | more -p
echo
if [ -e ${FOLDER}/${PCT} ] && [ -s ${FOLDER}/${PCT} ]; then
	echo -e  "\033[36mPCT Results\033[0m."
	ls -lhsG ${FOLDER}/${PCT}
	grep -a Confidence ${FOLDER}/${PCT}
	grep -a "BYO Diagnostics" ${FOLDER}/${PCT}
	grep -a [\*][\*][\*] ${FOLDER}/${PCT} \
		| grep -v [\*][\*][\*][\*][\*] | sort | uniq | more
	grep -a "CPU.cpu_type" ${FOLDER}/${PCT}
	grep -a "CPU.CORES_PER_SOCKET" ${FOLDER}/${PCT}
	grep -a "CACHE.SIZE_02" ${FOLDER}/${PCT}
	grep -a "Number of physical processors" ${FOLDER}/${PCT}
	grep -a ".86B." ${FOLDER}/${PCT}
	grep -a "Hard Drive" ${FOLDER}/${PCT}
fi
grep -a "SCORPION" ${FOLDER}/${REPORT}
fi
echo
# End of data review, if NOT POSTONLY