Backup: Difference between revisions

From Essential
Jump to navigation Jump to search
No edit summary
No edit summary
 
(3 intermediate revisions by the same user not shown)
Line 1: Line 1:
<pre>
<syntaxhighlight lang="bash">
#!/bin/sh  
#!/bin/sh
# backup.sh [host]
# backup.sh [host]


# Init
# Init
binDir="$(dirname $0)"
baseDir="$(realpath "$(dirname $0)"/..)"
backupDir="${binDir}"/../system_restore
dirBackup="${baseDir}"/system_restore
cacheFile=/tmp/$$
oldDir="${baseDir}"/.system_restore.old
appliDir="${baseDir}"/appli_restore
appliOld="${baseDir}"/.appli_restore.old


if [ $# -gt 1 ] ;then
hostList=$(hostList.sh $@)
  hostList=$1
  cat "$2" >${cacheFile}
  fileName="$2"
else
  hostList=$(grep "^t" /etc/ansible/hosts |sort -u)
  cat "$1" >${cacheFile}
  fileName="$1"
fi 


for serverMy in $hostList ;do
cacheFile=/tmp/"$(basename $0)"$$
  # Test acces root
  timeout --kill-after=1 1 ssh -i /etc/ansible/id_intrasys intrasys@${serverMy} "sudo su -c echo 2>&1 >/dev/null" 2>&1 >/dev/null &&(


    sourceDir="${serverMy}:/"
# list sync files
    destinationDir="${backupDir}/${serverMy}/"
rsyncList="${cacheFile}".rsyncList


    oldDir="$(realpath -mL --relative-base . "${destinationDir}").old"
## find uniq cfg files
if [ -z "$(find "${dirBackup}"/rsync.list -mtime -7 2>/dev/null )" ] ;then
  # clean
  rm -f "${dirBackup}"/*.touch "${appliDir}"/*.touch
  echo
  echo "## calculating files"
  # backup old list
  [ -f "${dirBackup}"/rsync.list ] &&cat "${dirBackup}"/rsync.list |gzip >"${oldDir}"/rsync.list.$(date '+%Y%m%d').gz &&rm -f "${dirBackup}"/rsync.list
  [ -f "${appliDir}"/rsync.list ] &&cat "${appliDir}"/rsync.list |gzip >"${appliOld}"/rsync.list.$(date '+%Y%m%d').gz &&rm -f "${appliDir}"/rsync.list


    mkdir -p "${destinationDir}" "${oldDir}"
  echo
  echo "# md5"


    # Backup rpm list
#cat <<EOT >"${cacheFile}".sh
    ${binDir}/clush.sh ${serverMy} "rpm -qa" \
#uptimeDays=\$(((\$(date +%s) - \$(date -d"\$(uptime -s 2>/dev/null ||date )" +%s))/86400+1))
      | gzip >"${backupDir}"/${serverMy}.rpmlist.gz
#(
 
# find /boot -type f -size -9k -mmin +60 -exec grep -Iq . {} \\; -and -print 2>/dev/null
    # Backup cfg files
#  find /etc ! -path '/etc/lvm/backup/*' ! -path '/etc/lvm/archive/*' -type f -atime -\$uptimeDays -size -9k -mmin +60 -exec grep -Iq . {} \\; -and -print 2>/dev/null
    rsync --files-from="${backupDir}/rsync.list" -abrtv --modify-window=1 --delete --force --backup-dir="$oldDir" --suffix=".$(date -u '+%Y-%m-%dT%H_%M_%SZ')" --rsync-path="sudo /usr/bin/rsync" --rsh="ssh -i /etc/ansible/id_intrasys -l intrasys" "intrasys@${sourceDir}" "${destinationDir}"
#  timeout --signal=9 24 find \$(mount |grep ^/dev/ |grep -E "vg_root|rhel_" |cut -d' ' -f3 |grep -vwE '^/(mnt|media)' ) \
t  )
#  -xdev ! -path '/boot/*' ! -path '/etc/*' ! -path '/var/lib/*' -type f -atime -\$uptimeDays -size -9k -mmin +60 -exec grep -Iq . {} \\; -and -print 2>/dev/null \
done
#  |grep -vwEi '(log|logs|deb|dpkg|rpm|yum|cache|tmp|bash_history|old|bck|back)' |head -4096
#!/bin/sh
#  timeout --signal=9 24 find \$(mount |grep ^/dev/ |grep -vE "vg_root|rhel_" |cut -d' ' -f3 |grep -vwE '^/(mnt|media)' ) \
#  -xdev ! -path '/boot/*' ! -path '/etc/*' ! -path '/var/lib/*' -type f -atime -\$uptimeDays -size -9k -mmin +60 -exec grep -Iq . {} \\; -and -print 2>/dev/null \
#  |grep -vwEi '(log|logs|deb|dpkg|rpm|yum|cache|tmp|bash_history|old|bck|back|trace)' |head -4096
#)|grep -vE '(\\.rpmnew|\\.rpmsave|-|~)\$' |awk '!seen[\$0]++' >/tmp/cfgList$$
#EOT


# initialisation
cat <<EOT >"${cacheFile}".sh
binDir="$(dirname $0)"
uptimeDays=\$(((\$(date +%s) - \$(date -d"\$(uptime -s 2>/dev/null ||date )" +%s))/86400+1))
(
  find /boot -type f -size -9k -mmin +60 2>/dev/null
  find /etc ! -path '/etc/lvm/backup/*' ! -path '/etc/lvm/archive/*' -type f -atime -\$uptimeDays -size -9k -mmin +60 2>/dev/null
  timeout --signal=9 16 find /var \
  -xdev ! -path '/boot/*' ! -path '/etc/*' ! -path '/var/lib/*' -type f -atime -\$uptimeDays -size -9k -mmin +60 -exec grep -Iq . {} \\; -and -print 2>/dev/null \
  |grep -vwEi '(log|logs|deb|dpkg|rpm|yum|cache|tmp|bash_history|old|bck|back)' |head -4096
  timeout --signal=9 16 find \$(mount |grep ^/dev/ |grep -E "vg_root|rhel_" |cut -d' ' -f3 |grep -vwE '^/(mnt|media)' ) \
  -xdev ! -path '/var/*' ! -path '/boot/*' ! -path '/etc/*' ! -path '/var/lib/*' -type f -atime -\$uptimeDays -size -9k -mmin +60 -exec grep -Iq . {} \\; -and -print 2>/dev/null \
  |grep -vwEi '(log|logs|deb|dpkg|rpm|yum|cache|tmp|bash_history|old|bck|back)' |head -4096
  timeout --signal=9 32 find \$(mount |grep ^/dev/ |grep -vE "vg_root|rhel_" |cut -d' ' -f3 |grep -vwE '^/(mnt|media)' ) \
  -xdev ! -path '/boot/*' ! -path '/etc/*' ! -path '/var/*' -type f -atime -\$uptimeDays -size -9k -mmin +60 2>/dev/null \
  |grep -vwEi '(log|logs|deb|dpkg|rpm|yum|cache|tmp|bash_history|old|bck|back|trace)' |head -8192
)|grep -vE '(\\.rpmnew|\\.rpmsave|-|~)\$' |awk '!seen[\$0]++' >/tmp/cfgList$$
EOT


#
  script.sh all "${cacheFile}".sh 2>/dev/null >/dev/null
touch ${cacheFile}
chown intrasys: ${cacheFile}
chmod 700 ${cacheFile}


if [ $# -gt 1 ] ;then
# fix 699 duples
   echo "$2" >${cacheFile}
   ssh.sh all "grep -v restore/ /tmp/cfgList$$ >/tmp/cfgList$$2 ;mv -f /tmp/cfgList$$2 /tmp/cfgList$$" 2>/dev/null >/dev/null
  ${binDir}/script.sh "$1" "${cacheFile}"
else
  echo "$1" >${cacheFile}
  ${binDir}/script.sh "${cacheFile}"
fi


rm -f ${cacheFile}
  ssh.sh all 'cat /tmp/cfgList'"$$"' |grep . |while read cfgFile ;do md5sum "${cfgFile}" ;done ;rm -f /tmp/cfgList'"$$" 2>/dev/null \
#!/bin/sh
    >"${cacheFile}".md5
# diff entre serveurs
# "<code bash>"
# APA 20180108


# initialisation des variables
  # ref=(total hosts)/2
binDir="$(dirname $0)"
  triggerValue=$(($(grep "^== " "${cacheFile}".md5 |wc -l) / 4 + 1 ))


# signature du code
  echo
nameBase="$(echo "$1" | tr -c '[:alnum:]' '_' | cut -c 1-68).$(echo "$1" | md5sum | cut -c 1-2).$(date '+%Y%m%d')"
  echo "# uniq"
  awk 'NR == FNR {count[$0]++; next}; count[$0] <= '"${triggerValue}" "${cacheFile}".md5 "${cacheFile}".md5 \
    |awk '!seen[$0]++' >"${cacheFile}".uniq


# si resultat non en cache, lancer le traitement
#debug
[ ! -f "${binDir}/../log/${nameBase}.gz" ] && (
cp -aZ "${cacheFile}".uniq >/tmp/backup-md5-uniq
  # lancement du code
  "${binDir}/"clush.sh "$1" >/dev/null


   # Enleve les lignes en doublons par serveur. Puis ne garde que les lignes uniques entre tous les serveurs.
   echo
   for fileName in "${binDir}/../log/old/"*".${nameBase}.gz" ;do
  echo "# clean result"
     echo "== ${fileName} =="
   cat "${cacheFile}".uniq \
    zcat "${fileName}" \
     |grep -v "^==.*==$" \
       | awk '!seen[$0]++'
       |sed -r 's/^[0-9a-zA-Z]+ +//' >"${cacheFile}".clean
  done \
    | awk '{!seen[$0]++};END{for(i in seen) if(seen[i]==1)print i}' \
      gzip >${cacheFile}2.gz


  # mise en cache
# mv "${cacheFile}".clean "${cacheFile}".clean-1
  mv ${cacheFile}2.gz "${binDir}/../log/${nameBase}.gz"
#  cat "${cacheFile}".clean-1 |grep -E "^/(bin|lib|lib64|opt|sbin|usr)/" |while read lineMy ;do
)
#    rpm -qf "${lineMy}" 2>/dev/null >/dev/null ||echo "${lineMy}"
#  done >"${cacheFile}".clean
#  cat "${cacheFile}".clean-1 |grep -vE "^/(bin|lib|lib64|opt|sbin|usr)/" >>"${cacheFile}".clean


# affiche le resultat
  echo
zcat "${binDir}/../log/${nameBase}.gz"
  echo "# keep system files"
  awk 'NR == FNR {count[$0]++; next}; count[$0] > '"${triggerValue}" "${cacheFile}".clean "${cacheFile}".clean |\
    sort -u \
      >"${cacheFile}".system


# nettoyage traitement
  echo
rm -f ${cacheFile}*
  echo "# keep appli files"
#!/bin/bash
  cat "${cacheFile}".clean \
    |sort -u >"${cacheFile}".appli.2
  comm -23 "${cacheFile}".appli.2 "${cacheFile}".system >"${cacheFile}".appli
 
  # store list
  cat "${cacheFile}".system >"${dirBackup}"/rsync.list
  cat "${cacheFile}".appli |grep -v "^${dirBackup}/" >"${appliDir}"/rsync.list


[ $# -ne 1 ] && echo "
  cat "${baseDir}"/env/files_company |sort -u >"${cacheFile}".rsync-company
# env parameters
  echo
# p : production
  ## remove duplicates from system and appli
# b : preproduction
  cat "${dirBackup}"/rsync.list "${appliDir}"/rsync.list |sort -u >"${cacheFile}".rsync-other
# r : recette
  comm -23 "${cacheFile}".rsync-company "${cacheFile}".rsync-other >"${cacheFile}".rsync
# t : test
# d : dev
" && exit 1


envParam=$1
  # store list
binDir="$(dirname $0)"
  cat "${cacheFile}".rsync >/exploitation/company_restore/rsync.list


grep "^t" ${binDir}/../env/hosts \
# function clean_restore <sourcePath>
   | sort -u \
clean_restore() {
    | grep -E '^.{11}'"${envParam}"
   baseName="$(basename "${sourcePath}")"
#!/bin/sh
  baseDir="$(realpath "$(dirname "${sourcePath}")")"
 
  destinationPath="${baseDir}"/."${baseName}".old
binDir="$(dirname $0)"
  cd "${sourcePath}"
 
  dateMy="$(date -u '+%Y-%m-%dT%H_%M_%SZ')"
filesList="${binDir}"/../system_restore/rsync.list
  find $(find * -maxdepth 0 -type d) ! -type d |sed -r "s#^[^/]+##g" |sort -u >/tmp/rsyncClean$$
 
  comm -23 /tmp/rsyncClean$$ rsync.list >/tmp/rsyncClean$$2
[ ! -f "${filesList}".$(date '+%Y%m%d').gz ] &&(
  mv -f /tmp/rsyncClean$$2 /tmp/rsyncClean$$
 
   cat /tmp/rsyncClean$$ |while read lineMy ;do
   cat "${filesList}" |gzip >"${filesList}".$(date '+%Y%m%d').gz
    ls -1 *"${lineMy}" |while read line2My ;do
 
      sourceFile="${sourcePath}"/"${line2My}"
  ${binDir}/diff.sh 'find /etc -follow -type f -mtime +1 -size -5M -exec md5sum {} \; ;find /boot /.authorized_keys /root /opt -type f -mtime +1 -size -5M -exec md5sum {} \;'
      if [ -f "${sourceFile}" ] ;then
        destinationFile="${destinationPath}"/"${dateMy}"/"${line2My}"
        mkdir -p "$(dirname "${destinationFile}")"
        mv -fv "${sourceFile}" "${destinationFile}"
      fi
    done
  done
  find -type l -delete
  find -empty -type d -delete
  cd - 2>&1 >/dev/null
}


   zcat $(ls -t1 ${binDir}/../log/*find*etc*md5sum*.$(date '+%Y%m%d').gz \
   echo "system
    |head -1) \
appli
      |grep -v "^==.*==$" \
company" |while read item ;do
        |sed -r 's/^[0-9a-zA-Z]+ //' \
    echo "== clean ${item}_restore =="
          |sort \
    sourcePath=/exploitation/${item}_restore
            >"all.${filesList}"
    clean_restore
  cat "all.${filesList}" \
   done
      |uniq >"${filesList}"
)
 
echo "less \"${filesList}\""
#!/bin/bash
 
if [ -z "$@" ] ;then
  MyKey="-ie critical\|error\|warn\|fail"
else
   MyKey="$@"
fi
fi
MyBin="$(basename $0)"


MyTimeOutMin=900
echo
MyTimeOutMax=3600
echo "= packages ="
for serverMy in $hostList ;do
  # Test acces root
  #timeout --kill-after=3 3 ssh -n -oStrictHostKeyChecking=no -i "${baseDir}"/env/id_exploitation intrasys@${serverMy} "sudo su -c echo 2>&1 >/dev/null" 2>&1 >/dev/null ||break


# bash colors
  echo "== ${serverMy} =="
RED='\033[0;31m'
GREEN='\033[0;32m'
NC='\033[0m' # No Color


lsof -X 2>/dev/null |grep " REG " |grep -E " [0-9]+w " |grep -o -P "(\/.+)" |sort -g |uniq |grep -v "/${MyBin}.cfg$" >~/${MyBin}.cfg
  # Backup rpm list
  rpmBackup="${oldDir}"/${serverMy}.rpmlist.gz".$(date -u '+%Y-%m-%dT%H_%M_%SZ')"
  ssh.sh ${serverMy} "rpm -qa" \
    | gzip >/tmp/${serverMy}.rpmlist.gz


cacheFile="/tmp/${MyBin}.$$"
  [ ! -z "$(zdiff "${dirBackup}"/${serverMy}.rpmlist.gz /tmp/${serverMy}.rpmlist.gz)" ] &&mv -f "${dirBackup}"/${serverMy}.rpmlist.gz "${rpmBackup}"
  [ ! -f "${dirBackup}"/${serverMy}.rpmlist.gz ] &&mv /tmp/${serverMy}.rpmlist.gz "${dirBackup}"/${serverMy}.rpmlist.gz
done


function MyStep1 {
# rsync function declaration
>${cacheFile}
rsync2() {
  mkdir -p "${destinationDir}" "${destinationDirOld}"
  chmod 0755 "${destinationDir}" "${destinationDirOld}"
  chown root.root "${destinationDir}" "${destinationDirOld}"
  chcon -h system_u:object_r:root_t:s0 "${destinationDir}" "${destinationDirOld}"


# 3 derniers jours du mois ou 9h à 17h en jour ouvré
  #test sudo
if [ $(date +%d -d "3 day") -lt $(date +%d) ] || { [ $(date +%H) -ge 9 ] && [ $(date +%H) -lt 17 ] && [ $(date +%u) -lt 6 ] ;} ;then
  sudoOpt=sudo
  MyTimeOut=$MyTimeOutMin
  timeout --kill-after=3 3 ssh -n -oStrictHostKeyChecking=no -i "${baseDir}"/env/id_exploitation intrasys@${serverMy} "sudo -n /usr/bin/rsync -h 2>&1 >/dev/null" 2>&1 >/dev/null ||sudoOpt=""
else
  MyTimeOut=$MyTimeOutMax
fi


cat ~/${MyBin}.cfg |while read MyLogPath ;do
   if [ -z "$(find "${destinationDir}" -maxdepth 0 -mtime -7 2>/dev/null )" ] ;then
   if [ -f "$MyLogPath" ] ;then
# APA 20200427 fix multihost
    MyLogElapsed=$(($(date +%s) - $(date +%s -r "$MyLogPath")))
    timeout --signal=9 16 rsync --delete --existing --ignore-existing --ignore-errors -r \
 
    -b --backup-dir="${destinationDirOld}" \
    if [ "$MyLogElapsed" -lt "$MyTimeOut" ] ;then
    --rsync-path="${sudoOpt} /usr/bin/rsync" \
      MyResult="$(tail -666 "$MyLogPath" 2>/dev/null |strings |grep $MyKey |tail -2 |cut -c1-111)"
    --rsh="ssh -i \"${baseDir}\"/env/id_exploitation -l intrasys" \
      if [ -z "$MyResult" ] ;then
    -x "${sourceDir}" "${destinationDir}" 2>/dev/null
        MyResult="$(tail -2 "$MyLogPath" 2>/dev/null |strings |tail -2 |cut -c1-111)"
    touch "${destinationDir}"
        if [ ! -z "$MyResult" ] ;then
          echo -e "${NC}== $MyLogPath ==" >>${cacheFile}
          echo -e "${GREEN}${MyResult}${NC}" >>${cacheFile}
        fi
      else
        echo -e "${NC}== $MyLogPath ==" >>${cacheFile}
        echo -e "${RED}${MyResult}${NC}" >>${cacheFile}
      fi
    fi
   fi
   fi
done
}


MyStep1
  timeout --signal=9 16 rsync --files-from="${destinationDir}"/../rsync.list --modify-window=1 \
cat ${cacheFile} |grep -v -w -f ~/${MyBin}.cfg >~/${MyBin}.ref
  --delete --force --ignore-errors -lptgoDAXb \
  --backup-dir="${destinationDirOld}" \
  --rsync-path="${sudoOpt} /usr/bin/rsync" \
  --rsh="ssh -i \"${baseDir}\"/env/id_exploitation -l intrasys" \
  "${sourceDir}" "${destinationDir}" 2>/dev/null


while true ; do
  #touch "${destinationDir}"
  MyStep1
}
  diff --changed-group-format='%>' --unchanged-group-format='' ~/${MyBin}.ref ${cacheFile} >${cacheFile}.diff
  clear
  cat ${cacheFile}.diff
  sleep 2
done
#!/bin/sh


# Init
echo
sourceDir="$1"
echo "= system ="
destinationDir="$2"
for serverMy in $hostList ;do
  # Test acces root
  #timeout --kill-after=3 3 ssh -n -oStrictHostKeyChecking=no -i "${baseDir}"/env/id_exploitation intrasys@${serverMy} "sudo su -c echo 2>&1 >/dev/null" 2>&1 >/dev/null ||break


backupDir="$(realpath -mL --relative-base . "${destinationDir}").backup"
  echo "== ${serverMy} =="


mkdir -p "${destinationDir}" "${backupDir}"
  sourceDir="${serverMy}:/"
  destinationDir="${dirBackup}/${serverMy}/"
  destinationDirOld="${oldDir}/${serverMy}/$(date -u '+%Y-%m-%dT%H_%M_%SZ')/"
  rsync2
done


# Backup external datas
echo
rsync -abrtv --modify-window=1 --delete --force --backup-dir="$backupDir" --suffix=".$(date -u '+%Y-%m-%dT%H_%M_%SZ')" --rsync-path="sudo /usr/bin/rsync" --rsh="ssh -i /etc/ansible/id_intrasys -l intrasys" -x "intrasys@${sourceDir}" "${destinationDir}"
echo "= appli ="
#!/bin/sh
for serverMy in $hostList ;do
  # Test acces root
  #timeout --kill-after=3 3 ssh -n -oStrictHostKeyChecking=no -i "${baseDir}"/env/id_exploitation intrasys@${serverMy} "sudo su -c echo 2>&1 >/dev/null" 2>&1 >/dev/null ||break


binDir="$(dirname $0)"
  echo "== ${serverMy} =="


touch ${cacheFile}
  sourceDir="${serverMy}:/"
chown intrasys: ${cacheFile}
  destinationDir="${appliDir}/${serverMy}/"
chmod 700 ${cacheFile}
  destinationDirOld="${appliOld}/${serverMy}/$(date -u '+%Y-%m-%dT%H_%M_%SZ')/"
  rsync2
done


if [ $# -gt 1 ] ;then
echo
  hostList=$1
echo "= company ="
  cat "$2" >${cacheFile}
else
  hostList=$(grep "^t" "${binDir}"/../env/hosts |sort -u)
  cat "$1" >${cacheFile}
fi 


destinationBase=/exploitation/company_restore
backupBase=/exploitation/.company_restore.old
filesFrom=/exploitation/company_restore/rsync.list
for serverMy in $hostList ;do
for serverMy in $hostList ;do
   cachePath="${binDir}"/../log/old/${serverMy}."$(cat ${cacheFile} | tr -c '[:alnum:]' '_' | cut -c 1-68)".$(md5sum ${cacheFile} | cut -c 1-2).$(date '+%Y%m%d')
   # Test acces root
  #timeout --kill-after=3 3 ssh -n -oStrictHostKeyChecking=no -i "${baseDir}"/env/id_exploitation intrasys@${serverMy} "sudo su -c echo 2>&1 >/dev/null" 2>&1 >/dev/null ||break
 
  echo "== ${serverMy} =="


   [ ! -f "${cachePath}".gz ] &&\
   sourceDir="${serverMy}:/"
    timeout --kill-after=1 1 ssh -i "${binDir}"/../env/id_intrasys intrasys@${serverMy} "sudo su -c echo 2>&1 >/dev/null" 2>&1 >/dev/null &&(
  destinationDir="${destinationBase}/${serverMy}/"
      timeout 3 scp -i "${binDir}"/../env/id_intrasys -p ${cacheFile} intrasys@${serverMy}:${cacheFile} 2>&1 >/dev/null &&\
  destinationDirOld="${backupBase}/${serverMy}/$(date -u '+%Y-%m-%dT%H_%M_%SZ')/"
      timeout 120 ssh -i "${binDir}"/../env/id_intrasys intrasys@${serverMy} "sudo su -c ${cacheFile}" |gzip >"${cachePath}".gz
   rsync2
      timeout 3 ssh -i "${binDir}"/../env/id_intrasys intrasys@${serverMy} "rm -f ${cacheFile} 2>/dev/null")
   [ -f "${cachePath}".gz ] &&(
    echo "== ${serverMy} =="
    zcat "${cachePath}".gz)
done
done


rm -f ${cacheFile}* 2>/dev/null ;find /tmp -user intrasys -type f -atime +1 -mtime +1 -delete 2>/dev/null
# Clean cache
</pre>
rm -f /tmp/"$(basename $0)"$$* 2>/dev/null
</syntaxhighlight>

Latest revision as of 22:58, 15 December 2023

#!/bin/sh
# backup.sh [host]

# Init
baseDir="$(realpath "$(dirname $0)"/..)"
dirBackup="${baseDir}"/system_restore
oldDir="${baseDir}"/.system_restore.old
appliDir="${baseDir}"/appli_restore
appliOld="${baseDir}"/.appli_restore.old

hostList=$(hostList.sh $@)

cacheFile=/tmp/"$(basename $0)"$$

# list sync files
rsyncList="${cacheFile}".rsyncList

## find uniq cfg files
if [ -z "$(find "${dirBackup}"/rsync.list -mtime -7 2>/dev/null )" ] ;then
  # clean
  rm -f "${dirBackup}"/*.touch "${appliDir}"/*.touch
  echo
  echo "## calculating files"
  # backup old list
  [ -f "${dirBackup}"/rsync.list ] &&cat "${dirBackup}"/rsync.list |gzip >"${oldDir}"/rsync.list.$(date '+%Y%m%d').gz &&rm -f "${dirBackup}"/rsync.list
  [ -f "${appliDir}"/rsync.list ] &&cat "${appliDir}"/rsync.list |gzip >"${appliOld}"/rsync.list.$(date '+%Y%m%d').gz &&rm -f "${appliDir}"/rsync.list

  echo
  echo "# md5"

#cat <<EOT >"${cacheFile}".sh
#uptimeDays=\$(((\$(date +%s) - \$(date -d"\$(uptime -s 2>/dev/null ||date )" +%s))/86400+1))
#(
#  find /boot -type f -size -9k -mmin +60 -exec grep -Iq . {} \\; -and -print 2>/dev/null
#  find /etc ! -path '/etc/lvm/backup/*' ! -path '/etc/lvm/archive/*' -type f -atime -\$uptimeDays -size -9k -mmin +60 -exec grep -Iq . {} \\; -and -print 2>/dev/null
#  timeout --signal=9 24 find \$(mount |grep ^/dev/ |grep -E "vg_root|rhel_" |cut -d' ' -f3 |grep -vwE '^/(mnt|media)' ) \
#   -xdev ! -path '/boot/*' ! -path '/etc/*' ! -path '/var/lib/*' -type f -atime -\$uptimeDays -size -9k -mmin +60 -exec grep -Iq . {} \\; -and -print 2>/dev/null \
#   |grep -vwEi '(log|logs|deb|dpkg|rpm|yum|cache|tmp|bash_history|old|bck|back)' |head -4096
#  timeout --signal=9 24 find \$(mount |grep ^/dev/ |grep -vE "vg_root|rhel_" |cut -d' ' -f3 |grep -vwE '^/(mnt|media)' ) \
#   -xdev ! -path '/boot/*' ! -path '/etc/*' ! -path '/var/lib/*' -type f -atime -\$uptimeDays -size -9k -mmin +60 -exec grep -Iq . {} \\; -and -print 2>/dev/null \
#   |grep -vwEi '(log|logs|deb|dpkg|rpm|yum|cache|tmp|bash_history|old|bck|back|trace)' |head -4096
#)|grep -vE '(\\.rpmnew|\\.rpmsave|-|~)\$' |awk '!seen[\$0]++' >/tmp/cfgList$$
#EOT

cat <<EOT >"${cacheFile}".sh
uptimeDays=\$(((\$(date +%s) - \$(date -d"\$(uptime -s 2>/dev/null ||date )" +%s))/86400+1))
(
  find /boot -type f -size -9k -mmin +60 2>/dev/null
  find /etc ! -path '/etc/lvm/backup/*' ! -path '/etc/lvm/archive/*' -type f -atime -\$uptimeDays -size -9k -mmin +60 2>/dev/null
  timeout --signal=9 16 find /var \
   -xdev ! -path '/boot/*' ! -path '/etc/*' ! -path '/var/lib/*' -type f -atime -\$uptimeDays -size -9k -mmin +60 -exec grep -Iq . {} \\; -and -print 2>/dev/null \
   |grep -vwEi '(log|logs|deb|dpkg|rpm|yum|cache|tmp|bash_history|old|bck|back)' |head -4096
  timeout --signal=9 16 find \$(mount |grep ^/dev/ |grep -E "vg_root|rhel_" |cut -d' ' -f3 |grep -vwE '^/(mnt|media)' ) \
   -xdev ! -path '/var/*' ! -path '/boot/*' ! -path '/etc/*' ! -path '/var/lib/*' -type f -atime -\$uptimeDays -size -9k -mmin +60 -exec grep -Iq . {} \\; -and -print 2>/dev/null \
   |grep -vwEi '(log|logs|deb|dpkg|rpm|yum|cache|tmp|bash_history|old|bck|back)' |head -4096
  timeout --signal=9 32 find \$(mount |grep ^/dev/ |grep -vE "vg_root|rhel_" |cut -d' ' -f3 |grep -vwE '^/(mnt|media)' ) \
   -xdev ! -path '/boot/*' ! -path '/etc/*' ! -path '/var/*' -type f -atime -\$uptimeDays -size -9k -mmin +60 2>/dev/null \
   |grep -vwEi '(log|logs|deb|dpkg|rpm|yum|cache|tmp|bash_history|old|bck|back|trace)' |head -8192
)|grep -vE '(\\.rpmnew|\\.rpmsave|-|~)\$' |awk '!seen[\$0]++' >/tmp/cfgList$$
EOT

  script.sh all "${cacheFile}".sh 2>/dev/null >/dev/null

# fix 699 duples
  ssh.sh all "grep -v restore/ /tmp/cfgList$$ >/tmp/cfgList$$2 ;mv -f /tmp/cfgList$$2 /tmp/cfgList$$" 2>/dev/null >/dev/null

  ssh.sh all 'cat /tmp/cfgList'"$$"' |grep . |while read cfgFile ;do md5sum "${cfgFile}" ;done ;rm -f /tmp/cfgList'"$$" 2>/dev/null \
    >"${cacheFile}".md5

  # ref=(total hosts)/2
  triggerValue=$(($(grep "^== " "${cacheFile}".md5 |wc -l) / 4 + 1 ))

  echo
  echo "# uniq"
  awk 'NR == FNR {count[$0]++; next}; count[$0] <= '"${triggerValue}" "${cacheFile}".md5 "${cacheFile}".md5 \
    |awk '!seen[$0]++' >"${cacheFile}".uniq

#debug
cp -aZ "${cacheFile}".uniq >/tmp/backup-md5-uniq

  echo
  echo "# clean result"
  cat "${cacheFile}".uniq \
    |grep -v "^==.*==$" \
      |sed -r 's/^[0-9a-zA-Z]+ +//' >"${cacheFile}".clean

#  mv "${cacheFile}".clean "${cacheFile}".clean-1
#  cat "${cacheFile}".clean-1 |grep -E "^/(bin|lib|lib64|opt|sbin|usr)/" |while read lineMy ;do
#    rpm -qf "${lineMy}" 2>/dev/null >/dev/null ||echo "${lineMy}"
#  done >"${cacheFile}".clean
#  cat "${cacheFile}".clean-1 |grep -vE "^/(bin|lib|lib64|opt|sbin|usr)/" >>"${cacheFile}".clean

  echo
  echo "# keep system files"
  awk 'NR == FNR {count[$0]++; next}; count[$0] > '"${triggerValue}" "${cacheFile}".clean "${cacheFile}".clean |\
    sort -u \
      >"${cacheFile}".system

  echo
  echo "# keep appli files"
  cat "${cacheFile}".clean \
    |sort -u >"${cacheFile}".appli.2
  comm -23 "${cacheFile}".appli.2 "${cacheFile}".system >"${cacheFile}".appli
  
  # store list
  cat "${cacheFile}".system >"${dirBackup}"/rsync.list
  cat "${cacheFile}".appli |grep -v "^${dirBackup}/" >"${appliDir}"/rsync.list

  cat "${baseDir}"/env/files_company |sort -u >"${cacheFile}".rsync-company
  echo
  ## remove duplicates from system and appli
  cat "${dirBackup}"/rsync.list "${appliDir}"/rsync.list |sort -u >"${cacheFile}".rsync-other
  comm -23 "${cacheFile}".rsync-company "${cacheFile}".rsync-other >"${cacheFile}".rsync

  # store list
  cat "${cacheFile}".rsync >/exploitation/company_restore/rsync.list

# function clean_restore <sourcePath>
clean_restore() {
  baseName="$(basename "${sourcePath}")"
  baseDir="$(realpath "$(dirname "${sourcePath}")")"
  destinationPath="${baseDir}"/."${baseName}".old
  cd "${sourcePath}"
  dateMy="$(date -u '+%Y-%m-%dT%H_%M_%SZ')"
  find $(find * -maxdepth 0 -type d) ! -type d |sed -r "s#^[^/]+##g" |sort -u >/tmp/rsyncClean$$
  comm -23 /tmp/rsyncClean$$ rsync.list >/tmp/rsyncClean$$2
  mv -f /tmp/rsyncClean$$2 /tmp/rsyncClean$$
  cat /tmp/rsyncClean$$ |while read lineMy ;do
    ls -1 *"${lineMy}" |while read line2My ;do
      sourceFile="${sourcePath}"/"${line2My}"
      if [ -f "${sourceFile}" ] ;then
        destinationFile="${destinationPath}"/"${dateMy}"/"${line2My}"
        mkdir -p "$(dirname "${destinationFile}")"
        mv -fv "${sourceFile}" "${destinationFile}"
      fi
    done
  done
  find -type l -delete
  find -empty -type d -delete
  cd - 2>&1 >/dev/null
}

  echo "system
appli
company" |while read item ;do
    echo "== clean ${item}_restore =="
    sourcePath=/exploitation/${item}_restore
    clean_restore
  done
fi

echo
echo "= packages ="
for serverMy in $hostList ;do
  # Test acces root
  #timeout --kill-after=3 3 ssh -n -oStrictHostKeyChecking=no -i "${baseDir}"/env/id_exploitation intrasys@${serverMy} "sudo su -c echo 2>&1 >/dev/null" 2>&1 >/dev/null ||break

  echo "== ${serverMy} =="

  # Backup rpm list
  rpmBackup="${oldDir}"/${serverMy}.rpmlist.gz".$(date -u '+%Y-%m-%dT%H_%M_%SZ')"
  ssh.sh ${serverMy} "rpm -qa" \
    | gzip >/tmp/${serverMy}.rpmlist.gz

  [ ! -z "$(zdiff "${dirBackup}"/${serverMy}.rpmlist.gz /tmp/${serverMy}.rpmlist.gz)" ] &&mv -f "${dirBackup}"/${serverMy}.rpmlist.gz "${rpmBackup}"
  [ ! -f "${dirBackup}"/${serverMy}.rpmlist.gz ] &&mv /tmp/${serverMy}.rpmlist.gz "${dirBackup}"/${serverMy}.rpmlist.gz
done

# rsync function declaration
rsync2() {
  mkdir -p "${destinationDir}" "${destinationDirOld}"
  chmod 0755 "${destinationDir}" "${destinationDirOld}"
  chown root.root "${destinationDir}" "${destinationDirOld}"
  chcon -h system_u:object_r:root_t:s0 "${destinationDir}" "${destinationDirOld}"

  #test sudo
  sudoOpt=sudo
  timeout --kill-after=3 3 ssh -n -oStrictHostKeyChecking=no -i "${baseDir}"/env/id_exploitation intrasys@${serverMy} "sudo -n /usr/bin/rsync -h 2>&1 >/dev/null" 2>&1 >/dev/null ||sudoOpt=""

  if [ -z "$(find "${destinationDir}" -maxdepth 0 -mtime -7 2>/dev/null )" ] ;then
# APA 20200427 fix multihost
    timeout --signal=9 16 rsync --delete --existing --ignore-existing --ignore-errors -r \
     -b --backup-dir="${destinationDirOld}" \
     --rsync-path="${sudoOpt} /usr/bin/rsync" \
     --rsh="ssh -i \"${baseDir}\"/env/id_exploitation -l intrasys" \
     -x "${sourceDir}" "${destinationDir}" 2>/dev/null
    touch "${destinationDir}"
  fi

  timeout --signal=9 16 rsync --files-from="${destinationDir}"/../rsync.list --modify-window=1 \
   --delete --force --ignore-errors -lptgoDAXb \
   --backup-dir="${destinationDirOld}" \
   --rsync-path="${sudoOpt} /usr/bin/rsync" \
   --rsh="ssh -i \"${baseDir}\"/env/id_exploitation -l intrasys" \
   "${sourceDir}" "${destinationDir}" 2>/dev/null

   #touch "${destinationDir}"
}

echo
echo "= system ="
for serverMy in $hostList ;do
  # Test acces root
  #timeout --kill-after=3 3 ssh -n -oStrictHostKeyChecking=no -i "${baseDir}"/env/id_exploitation intrasys@${serverMy} "sudo su -c echo 2>&1 >/dev/null" 2>&1 >/dev/null ||break

  echo "== ${serverMy} =="

  sourceDir="${serverMy}:/"
  destinationDir="${dirBackup}/${serverMy}/"
  destinationDirOld="${oldDir}/${serverMy}/$(date -u '+%Y-%m-%dT%H_%M_%SZ')/"
  rsync2
done

echo
echo "= appli ="
for serverMy in $hostList ;do
  # Test acces root
  #timeout --kill-after=3 3 ssh -n -oStrictHostKeyChecking=no -i "${baseDir}"/env/id_exploitation intrasys@${serverMy} "sudo su -c echo 2>&1 >/dev/null" 2>&1 >/dev/null ||break

  echo "== ${serverMy} =="

  sourceDir="${serverMy}:/"
  destinationDir="${appliDir}/${serverMy}/"
  destinationDirOld="${appliOld}/${serverMy}/$(date -u '+%Y-%m-%dT%H_%M_%SZ')/"
  rsync2
done

echo
echo "= company ="

destinationBase=/exploitation/company_restore
backupBase=/exploitation/.company_restore.old
filesFrom=/exploitation/company_restore/rsync.list
for serverMy in $hostList ;do
  # Test acces root
  #timeout --kill-after=3 3 ssh -n -oStrictHostKeyChecking=no -i "${baseDir}"/env/id_exploitation intrasys@${serverMy} "sudo su -c echo 2>&1 >/dev/null" 2>&1 >/dev/null ||break

  echo "== ${serverMy} =="

  sourceDir="${serverMy}:/"
  destinationDir="${destinationBase}/${serverMy}/"
  destinationDirOld="${backupBase}/${serverMy}/$(date -u '+%Y-%m-%dT%H_%M_%SZ')/"
  rsync2
done

# Clean cache
rm -f /tmp/"$(basename $0)"$$* 2>/dev/null
Retrieved from "https://infocepo.com/wiki/index.php?title=Backup&oldid=1671"