Backup
Jump to navigation
Jump to search
#!/bin/sh
# backup.sh [host]
# Init
baseDir="$(realpath "$(dirname $0)"/..)"
dirBackup="${baseDir}"/system_restore
oldDir="${baseDir}"/.system_restore.old
appliDir="${baseDir}"/appli_restore
appliOld="${baseDir}"/.appli_restore.old
hostList=$(hostList.sh $@)
cacheFile=/tmp/"$(basename $0)"$$
# list sync files
rsyncList="${cacheFile}".rsyncList
## find uniq cfg files
if [ -z "$(find "${dirBackup}"/rsync.list -mtime -7 2>/dev/null )" ] ;then
# clean
rm -f "${dirBackup}"/*.touch "${appliDir}"/*.touch
echo
echo "## calculating files"
# backup old list
[ -f "${dirBackup}"/rsync.list ] &&cat "${dirBackup}"/rsync.list |gzip >"${oldDir}"/rsync.list.$(date '+%Y%m%d').gz &&rm -f "${dirBackup}"/rsync.list
[ -f "${appliDir}"/rsync.list ] &&cat "${appliDir}"/rsync.list |gzip >"${appliOld}"/rsync.list.$(date '+%Y%m%d').gz &&rm -f "${appliDir}"/rsync.list
echo
echo "# md5"
#cat <<EOT >"${cacheFile}".sh
#uptimeDays=\$(((\$(date +%s) - \$(date -d"\$(uptime -s 2>/dev/null ||date )" +%s))/86400+1))
#(
# find /boot -type f -size -9k -mmin +60 -exec grep -Iq . {} \\; -and -print 2>/dev/null
# find /etc ! -path '/etc/lvm/backup/*' ! -path '/etc/lvm/archive/*' -type f -atime -\$uptimeDays -size -9k -mmin +60 -exec grep -Iq . {} \\; -and -print 2>/dev/null
# timeout --signal=9 24 find \$(mount |grep ^/dev/ |grep -E "vg_root|rhel_" |cut -d' ' -f3 |grep -vwE '^/(mnt|media)' ) \
# -xdev ! -path '/boot/*' ! -path '/etc/*' ! -path '/var/lib/*' -type f -atime -\$uptimeDays -size -9k -mmin +60 -exec grep -Iq . {} \\; -and -print 2>/dev/null \
# |grep -vwEi '(log|logs|deb|dpkg|rpm|yum|cache|tmp|bash_history|old|bck|back)' |head -4096
# timeout --signal=9 24 find \$(mount |grep ^/dev/ |grep -vE "vg_root|rhel_" |cut -d' ' -f3 |grep -vwE '^/(mnt|media)' ) \
# -xdev ! -path '/boot/*' ! -path '/etc/*' ! -path '/var/lib/*' -type f -atime -\$uptimeDays -size -9k -mmin +60 -exec grep -Iq . {} \\; -and -print 2>/dev/null \
# |grep -vwEi '(log|logs|deb|dpkg|rpm|yum|cache|tmp|bash_history|old|bck|back|trace)' |head -4096
#)|grep -vE '(\\.rpmnew|\\.rpmsave|-|~)\$' |awk '!seen[\$0]++' >/tmp/cfgList$$
#EOT
cat <<EOT >"${cacheFile}".sh
uptimeDays=\$(((\$(date +%s) - \$(date -d"\$(uptime -s 2>/dev/null ||date )" +%s))/86400+1))
(
find /boot -type f -size -9k -mmin +60 2>/dev/null
find /etc ! -path '/etc/lvm/backup/*' ! -path '/etc/lvm/archive/*' -type f -atime -\$uptimeDays -size -9k -mmin +60 2>/dev/null
timeout --signal=9 16 find /var \
-xdev ! -path '/boot/*' ! -path '/etc/*' ! -path '/var/lib/*' -type f -atime -\$uptimeDays -size -9k -mmin +60 -exec grep -Iq . {} \\; -and -print 2>/dev/null \
|grep -vwEi '(log|logs|deb|dpkg|rpm|yum|cache|tmp|bash_history|old|bck|back)' |head -4096
timeout --signal=9 16 find \$(mount |grep ^/dev/ |grep -E "vg_root|rhel_" |cut -d' ' -f3 |grep -vwE '^/(mnt|media)' ) \
-xdev ! -path '/var/*' ! -path '/boot/*' ! -path '/etc/*' ! -path '/var/lib/*' -type f -atime -\$uptimeDays -size -9k -mmin +60 -exec grep -Iq . {} \\; -and -print 2>/dev/null \
|grep -vwEi '(log|logs|deb|dpkg|rpm|yum|cache|tmp|bash_history|old|bck|back)' |head -4096
timeout --signal=9 32 find \$(mount |grep ^/dev/ |grep -vE "vg_root|rhel_" |cut -d' ' -f3 |grep -vwE '^/(mnt|media)' ) \
-xdev ! -path '/boot/*' ! -path '/etc/*' ! -path '/var/*' -type f -atime -\$uptimeDays -size -9k -mmin +60 2>/dev/null \
|grep -vwEi '(log|logs|deb|dpkg|rpm|yum|cache|tmp|bash_history|old|bck|back|trace)' |head -8192
)|grep -vE '(\\.rpmnew|\\.rpmsave|-|~)\$' |awk '!seen[\$0]++' >/tmp/cfgList$$
EOT
script.sh all "${cacheFile}".sh 2>/dev/null >/dev/null
# fix 699 duples
ssh.sh all "grep -v restore/ /tmp/cfgList$$ >/tmp/cfgList$$2 ;mv -f /tmp/cfgList$$2 /tmp/cfgList$$" 2>/dev/null >/dev/null
ssh.sh all 'cat /tmp/cfgList'"$$"' |grep . |while read cfgFile ;do md5sum "${cfgFile}" ;done ;rm -f /tmp/cfgList'"$$" 2>/dev/null \
>"${cacheFile}".md5
# ref=(total hosts)/2
triggerValue=$(($(grep "^== " "${cacheFile}".md5 |wc -l) / 4 + 1 ))
echo
echo "# uniq"
awk 'NR == FNR {count[$0]++; next}; count[$0] <= '"${triggerValue}" "${cacheFile}".md5 "${cacheFile}".md5 \
|awk '!seen[$0]++' >"${cacheFile}".uniq
#debug
cp -aZ "${cacheFile}".uniq >/tmp/backup-md5-uniq
echo
echo "# clean result"
cat "${cacheFile}".uniq \
|grep -v "^==.*==$" \
|sed -r 's/^[0-9a-zA-Z]+ +//' >"${cacheFile}".clean
# mv "${cacheFile}".clean "${cacheFile}".clean-1
# cat "${cacheFile}".clean-1 |grep -E "^/(bin|lib|lib64|opt|sbin|usr)/" |while read lineMy ;do
# rpm -qf "${lineMy}" 2>/dev/null >/dev/null ||echo "${lineMy}"
# done >"${cacheFile}".clean
# cat "${cacheFile}".clean-1 |grep -vE "^/(bin|lib|lib64|opt|sbin|usr)/" >>"${cacheFile}".clean
echo
echo "# keep system files"
awk 'NR == FNR {count[$0]++; next}; count[$0] > '"${triggerValue}" "${cacheFile}".clean "${cacheFile}".clean |\
sort -u \
>"${cacheFile}".system
echo
echo "# keep appli files"
cat "${cacheFile}".clean \
|sort -u >"${cacheFile}".appli.2
comm -23 "${cacheFile}".appli.2 "${cacheFile}".system >"${cacheFile}".appli
# store list
cat "${cacheFile}".system >"${dirBackup}"/rsync.list
cat "${cacheFile}".appli |grep -v "^${dirBackup}/" >"${appliDir}"/rsync.list
cat "${baseDir}"/env/files_company |sort -u >"${cacheFile}".rsync-company
echo
## remove duplicates from system and appli
cat "${dirBackup}"/rsync.list "${appliDir}"/rsync.list |sort -u >"${cacheFile}".rsync-other
comm -23 "${cacheFile}".rsync-company "${cacheFile}".rsync-other >"${cacheFile}".rsync
# store list
cat "${cacheFile}".rsync >/exploitation/company_restore/rsync.list
# function clean_restore <sourcePath>
clean_restore() {
baseName="$(basename "${sourcePath}")"
baseDir="$(realpath "$(dirname "${sourcePath}")")"
destinationPath="${baseDir}"/."${baseName}".old
cd "${sourcePath}"
dateMy="$(date -u '+%Y-%m-%dT%H_%M_%SZ')"
find $(find * -maxdepth 0 -type d) ! -type d |sed -r "s#^[^/]+##g" |sort -u >/tmp/rsyncClean$$
comm -23 /tmp/rsyncClean$$ rsync.list >/tmp/rsyncClean$$2
mv -f /tmp/rsyncClean$$2 /tmp/rsyncClean$$
cat /tmp/rsyncClean$$ |while read lineMy ;do
ls -1 *"${lineMy}" |while read line2My ;do
sourceFile="${sourcePath}"/"${line2My}"
if [ -f "${sourceFile}" ] ;then
destinationFile="${destinationPath}"/"${dateMy}"/"${line2My}"
mkdir -p "$(dirname "${destinationFile}")"
mv -fv "${sourceFile}" "${destinationFile}"
fi
done
done
find -type l -delete
find -empty -type d -delete
cd - 2>&1 >/dev/null
}
echo "system
appli
company" |while read item ;do
echo "== clean ${item}_restore =="
sourcePath=/exploitation/${item}_restore
clean_restore
done
fi
echo
echo "= packages ="
for serverMy in $hostList ;do
# Test acces root
#timeout --kill-after=3 3 ssh -n -oStrictHostKeyChecking=no -i "${baseDir}"/env/id_exploitation intrasys@${serverMy} "sudo su -c echo 2>&1 >/dev/null" 2>&1 >/dev/null ||break
echo "== ${serverMy} =="
# Backup rpm list
rpmBackup="${oldDir}"/${serverMy}.rpmlist.gz".$(date -u '+%Y-%m-%dT%H_%M_%SZ')"
ssh.sh ${serverMy} "rpm -qa" \
| gzip >/tmp/${serverMy}.rpmlist.gz
[ ! -z "$(zdiff "${dirBackup}"/${serverMy}.rpmlist.gz /tmp/${serverMy}.rpmlist.gz)" ] &&mv -f "${dirBackup}"/${serverMy}.rpmlist.gz "${rpmBackup}"
[ ! -f "${dirBackup}"/${serverMy}.rpmlist.gz ] &&mv /tmp/${serverMy}.rpmlist.gz "${dirBackup}"/${serverMy}.rpmlist.gz
done
# rsync function declaration
rsync2() {
mkdir -p "${destinationDir}" "${destinationDirOld}"
chmod 0755 "${destinationDir}" "${destinationDirOld}"
chown root.root "${destinationDir}" "${destinationDirOld}"
chcon -h system_u:object_r:root_t:s0 "${destinationDir}" "${destinationDirOld}"
#test sudo
sudoOpt=sudo
timeout --kill-after=3 3 ssh -n -oStrictHostKeyChecking=no -i "${baseDir}"/env/id_exploitation intrasys@${serverMy} "sudo -n /usr/bin/rsync -h 2>&1 >/dev/null" 2>&1 >/dev/null ||sudoOpt=""
if [ -z "$(find "${destinationDir}" -maxdepth 0 -mtime -7 2>/dev/null )" ] ;then
# APA 20200427 fix multihost
timeout --signal=9 16 rsync --delete --existing --ignore-existing --ignore-errors -r \
-b --backup-dir="${destinationDirOld}" \
--rsync-path="${sudoOpt} /usr/bin/rsync" \
--rsh="ssh -i \"${baseDir}\"/env/id_exploitation -l intrasys" \
-x "${sourceDir}" "${destinationDir}" 2>/dev/null
touch "${destinationDir}"
fi
timeout --signal=9 16 rsync --files-from="${destinationDir}"/../rsync.list --modify-window=1 \
--delete --force --ignore-errors -lptgoDAXb \
--backup-dir="${destinationDirOld}" \
--rsync-path="${sudoOpt} /usr/bin/rsync" \
--rsh="ssh -i \"${baseDir}\"/env/id_exploitation -l intrasys" \
"${sourceDir}" "${destinationDir}" 2>/dev/null
#touch "${destinationDir}"
}
echo
echo "= system ="
for serverMy in $hostList ;do
# Test acces root
#timeout --kill-after=3 3 ssh -n -oStrictHostKeyChecking=no -i "${baseDir}"/env/id_exploitation intrasys@${serverMy} "sudo su -c echo 2>&1 >/dev/null" 2>&1 >/dev/null ||break
echo "== ${serverMy} =="
sourceDir="${serverMy}:/"
destinationDir="${dirBackup}/${serverMy}/"
destinationDirOld="${oldDir}/${serverMy}/$(date -u '+%Y-%m-%dT%H_%M_%SZ')/"
rsync2
done
echo
echo "= appli ="
for serverMy in $hostList ;do
# Test acces root
#timeout --kill-after=3 3 ssh -n -oStrictHostKeyChecking=no -i "${baseDir}"/env/id_exploitation intrasys@${serverMy} "sudo su -c echo 2>&1 >/dev/null" 2>&1 >/dev/null ||break
echo "== ${serverMy} =="
sourceDir="${serverMy}:/"
destinationDir="${appliDir}/${serverMy}/"
destinationDirOld="${appliOld}/${serverMy}/$(date -u '+%Y-%m-%dT%H_%M_%SZ')/"
rsync2
done
echo
echo "= company ="
destinationBase=/exploitation/company_restore
backupBase=/exploitation/.company_restore.old
filesFrom=/exploitation/company_restore/rsync.list
for serverMy in $hostList ;do
# Test acces root
#timeout --kill-after=3 3 ssh -n -oStrictHostKeyChecking=no -i "${baseDir}"/env/id_exploitation intrasys@${serverMy} "sudo su -c echo 2>&1 >/dev/null" 2>&1 >/dev/null ||break
echo "== ${serverMy} =="
sourceDir="${serverMy}:/"
destinationDir="${destinationBase}/${serverMy}/"
destinationDirOld="${backupBase}/${serverMy}/$(date -u '+%Y-%m-%dT%H_%M_%SZ')/"
rsync2
done
# Clean cache
rm -f /tmp/"$(basename $0)"$$* 2>/dev/null