SATELLITE6

== infra update ==
Comment mettre à jour l infra Linux?
* Nettoyer les SNAP
=== ${SERVER_NAME} ===
* Vérifier le planning et suivre la checklist Z:\system\update-checklist.xls
* Choisir l environnement à mettre à jour :

 ssh.sh ${SERVER_NAME}

<pre>
#Library Dev Rec Pre Prd
envName=Prd

org_name=${COMPANY_NAME}

# init
envArray=(Library Dev Rec Pre Prd)

# Clean and Publish new content if Library
if [ "${envName}" == "Library" ] ;then
  # Clean content views
  for contentView in $(hammer --csv content-view version list |grep -E ',\"\"$' |cut -d"," -f1) ;do
    hammer content-view version delete --id $contentView
  done
  for contentView in $(hammer --csv content-view version list |grep -E ',\"\"$' |cut -d"," -f1) ;do
    hammer content-view version delete --id $contentView
  done
  # End clean
  for contentView in $(hammer --csv content-view list |tail -n +2 |cut -d"," -f1) ;do
    hammer content-view publish \
      --id $contentView \
      --organization $org_name
  done
else
  envNamePrevious=${envArray[0]}
  for t in ${envArray[@]}; do
    [ "$t" == "$envName" ] &&break
    export envNamePrevious=$t
  done
  # Refresh/promote Env
  for contentView in $(hammer --csv content-view version list |tail -n +2 |grep -w "$envNamePrevious" |cut -d"," -f1) ;do
    hammer content-view version promote \
     --id $contentView \
     --organization $org_name \
     --to-lifecycle-environment $envName
  done
fi
</pre>
=== yum check ===
* Déterminer les serveurs à maintenir via le serveur Ansible (réponse lente ! ):
<pre>
ssh.sh intrasys 'timeout 60 yum check-update 2>&1 >/dev/null ;yumTest=$?
if [ $yumTest -eq 100 ] ;then resultMy="update" ;elif [ $yumTest -ne 0 ] ;then
  resultMy="register" ;else needs-restarting -r 2>&1 >/dev/null ;[ $? -eq 1 ] &&resultMy="reboot"
fi
echo -e "$(hostname -s)\t${resultMy}"' |sort -k2r |grep -Ew "update|register|reboot"
</pre>
* Déterminer les configurations à mettre à jour
<pre>
comm.sh intrasys 'find /etc -type f -name "*.rpmnew"' |sed 's#.rpmnew$##' |while read lineMy ;do
  echo "=== ${lineMy}.rpmnew ==="
  comm.sh intrasys 'diff "'${lineMy}.rpmnew'" "'${lineMy}'"'
done
</pre>
==== old ====
# ssh.sh "${hostList}" 'subscription-manager refresh ;yum clean all ;rm -rf /var/cache/yum '  2>&1 >/dev/null
# ssh.sh "${hostList}" 'yum history sync'  2>&1 >/dev/null
# hostList="$(hostList.sh all $(hostList.sh intrasys |grep -v kvm |grep -E '^tsvl.*(p|p-mgt)$' ) |grep -v kvm )"
 #package-cleanup --cleandupes
* Maintenance des packages :
 ssh.sh all 'package-cleanup --cleandupes'
 ssh.sh all 'yum autoremove'
* fix kvm tools
 ssh.sh kvm 'subscription-manager repos --enable=rhel-7-server-rhv-4-tools-rpms'
=== sat6 register ===
 #products list #yum list installed |sed -rn 's#^.*@(.*)$#\1#p' |sort -u
 #repos list
 #subcription needed ? buy
 #enable repos
 #//create repos
 #sync
 #//add autosync
 #create content view
 #  add repos
 #create key
 #  add product
 #content view

<pre>
# kvm
appCode=kvm
# dev rec pre prd
envCode=dev

releaseV=$(cat /etc/redhat-release |sed -rn 's#^.*release ([0-9]+).*$#\1#p' )
activationKey=ak_rhel${releaseV}$([ ! -z "${envCode}" ] &&echo "_${envCode}" )$([ ! -z "${appCode}" ] &&echo "_${appCode}" )
subscription-manager register --org="${COMPANY_NAME}" --activationkey="${activationKey}" --force

# clean SAT6 registration
timeout --signal=9 16 subscription-manager unsubscribe --all
timeout --signal=9 16 subscription-manager unregister
# clean KATELLO service
(systemctl disable goferd 2>/dev/null ||chkconfig goferd off )</dev/null
(systemctl stop goferd 2>/dev/null ||service goferd stop )</dev/null
yum remove katello* gofer* -y
yum autoremove -y
# clean old registration
timeout --signal=9 16 subscription-manager unregister 2>/dev/null
sleep 4
subscription-manager clean
# clean YUM
yum clean all
rm -rf /var/cache/yum
# configure YUM
curl --insecure --output katello-ca-consumer-latest.noarch.rpm https://${SERVER_NAME}.csf.${COMPANY_NAME}.fr/pub/katello-ca-consumer-latest.noarch.rpm
yum localinstall katello-ca-consumer-latest.noarch.rpm -y
rm -f katello-ca-consumer-latest.noarch.rpm
#rpm --force -ivh https://${SERVER_NAME}.csf.${COMPANY_NAME}.fr/pub/katello-ca-consumer-latest.noarch.rpm
subscription-manager register --org="${COMPANY_NAME}" --activationkey="${activationKey}" --force
#sleep 4
# check
subscription-manager repos --enable=*
subscription-manager repos |grep -E "^Enabled:.*1" |wc -l
</pre>
=== yum update ===
<pre>

hostList=""

update_func() {
  # save status
  cat <<EOT >/tmp/script$$.sh
(who -b ;uname -r ;(systemctl list-unit-files 2>/dev/null |sort -u ;chkconfig --list 2>/dev/null |sort -u ) |grep -Ev "^session-|^user-" |grep -Ew "enabled|on" |sed -rn "s#^[[:space:]]*([[:graph:]]+)[[:space:]].*\\\$#\1#p" |sed -r "s#(\.service|\.target|\.socket)##g" ) 2>/dev/null >/tmp/update.service.old.\$(date '+%Y%m%d')
mv -n /tmp/update.service.old.\$(date '+%Y%m%d') /root/update.service.old.\$(date '+%Y%m%d')
wc -l /root/update.service.old.\$(date '+%Y%m%d')
EOT
  script.sh "${hostList}" /tmp/script$$.sh

  # enable all repos
  ssh.sh "${hostList}" 'subscription-manager repos --enable=* 2>/dev/null '

  # enable all repos
  ssh.sh "${hostList}" 'yum clean all'
  
  # snap and standby
  >/tmp/hostList2-$$
  >/tmp/hostList3-$$
  hostList.sh "${hostList}" |grep . |while read hostName ;do
    testCluster=$(ssh.sh "${hostName}" 'pcs status 2>/dev/null' )
    if [ ! -z "${testCluster}" ] ;then
      echo "${testCluster}" |grep -iw standby &&(echo "${testCluster}" |grep -iw online |grep -w "${hostName}" &&echo "${hostName}" >>/tmp/hostList2-$$) ||(ssh.sh "${hostName}" 'sudo pcs node standby ;subscription-manager repos --disable=${COMPANY_NAME}_hp-spp_hp-spp-7Server 2>/dev/null ' ;echo "${hostName}" >>/tmp/hostList3-$$ )
    else
	  echo "null snap"
    fi
  done
  hostList1=$(hostList.sh "${hostList}" |grep -vxf /tmp/hostList2-$$ |shuf )
  hostList3=$(cat /tmp/hostList3-$$ |shuf )

  role.sh "${hostList1}" update_server

  if [ ! -z "${hostList3}" ] ;then
    ssh.sh "${hostList3}" 'subscription-manager repos --enable=* 2>/dev/null '
    role.sh "${hostList3}" update_server
  fi

  cat <<EOT >/tmp/script$$.sh
yum autoremove -y
yum clean all
EOT
  script.sh "${hostList1}" /tmp/script$$.sh
}

>/tmp/hostList2-$$

update_func
</pre>

* Reboot if needed :

<pre>
reboot_func() {
  cat <<EOT >/tmp/script$$.sh
yum install yum-utils -y
yum clean all
rm -rf /var/cache/yum
needs-restarting -r 2>&1 >/dev/null ; [ \$? -eq 1 ] &&(
shutdown -r now )
EOT
  script.sh "${hostList1}" /tmp/script$$.sh
}

status_check() {
  hostList.sh "${hostList1}" |grep . |while read hostName ;do
#    (ping -w1 -c 1 "${hostName}" 2>&1 >/dev/null &&echo "$hostName UP" ||echo "$hostName DOWN" )&
    if [[ ! -z "$(timeout --signal=9 2 ssh.sh "${hostName}" hostname 2>/dev/null )" ]] ;then
      echo "$hostName UP"
    else
      echo "$hostName DOWN"
    fi
  done |sed 's#-.+##'
}

status_check_print() {
  statusNOK="NOK"
  while [ ! -z "${statusNOK}" ] ;do
    statusNOK=$(status_check |grep -w DOWN )
    printf '%s\n' "$(clear; echo "${statusNOK}" )"
    sleep 2
  done
}

echo "reboot"
reboot_func
cat /tmp/hostList2-$$ >/tmp/hostListCluster-$$
status_check_print
</pre>

* Schedule cluster reboots :

<pre>
after_reboot() {
cat <<EOT >/tmp/script$$.sh
# (who -b ;uname -r ;systemctl ;chkconfig 2>/dev/null ) |sed -r "s#^[:space:]+##" >/root/update.service.new.\$(date '+%Y%m%d')
(who -b ;uname -r ;(systemctl list-unit-files 2>/dev/null |sort -u ;chkconfig --list 2>/dev/null |sort -u ) |grep -Ev "^session-|^user-" |grep -Ew "enabled|on" |sed -rn "s#^[[:space:]]*([[:graph:]]+)[[:space:]].*\\\$#\1#p" |sed -r "s#(\.service|\.target|\.socket)##g" ) 2>/dev/null >/root/update.service.new.\$(date '+%Y%m%d')
(cat /root/update.service.new.\$(date '+%Y%m%d') ;echo "=== before ===" ;cat /root/update.service.old.\$(date '+%Y%m%d') ) >/tmp/update.check
awk 'NR == FNR {count[\$0]++; next}; count[\$0] == 1' /tmp/update.check /tmp/update.check
EOT
script.sh "${hostList1}" /tmp/script$$.sh
echo
echo "= Check app flow ="
appCheck.sh "${hostList1}"
}

after_reboot

hostList=$(cat /tmp/hostList2-$$ )
>/tmp/hostList2-$$
echo "${hostList}"
if [ ! -z "${hostList}" ] ;then
  ssh.sh "${hostList}" 'sudo pcs node unstandby --all'
  sleep 5
  update_func
  echo "reboot"
  sleep 3
  reboot_func
  status_check_print
else
  ssh.sh "$(cat /tmp/hostListCluster-$$ )" 'sudo pcs node unstandby --all ;sudo pcs resource relocate run'
fi
</pre>
=== debug port ===
 timeout --kill-after=20 20 curl ${SERVER_NAME}:443
=== test ===
 ansible all -m ping |grep UNREACHABLE
=== old ===
#subscription-manager repos --enable=${COMPANY_NAME}_hp-spp_hp-spp-7Server 2>/dev/null
# yum update kernel -y &&(
# yum update -y --skip-broken &&
# yum update -y)