| Linux hosting5.siteguarding.com 3.10.0-962.3.2.lve1.5.88.el7.x86_64 #1 SMP Fri Sep 26 14:06:42 UTC 2025 x86_64 Path : /usr/local/bin/ |
| Current File : //usr/local/bin/dxm.do_backup_old |
#!/bin/sh
#set -x
# DXM 1.4
# Pavel Piatruk, 2010
# Nick Keefen, 2009-2015
# http://www.nksupport.com/
. /etc/dxm/dxm.conf
#initialize environment and defaults
export PATH="${PATH}:/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin:/root/bin"
export HOME="/root"
export DXM_VERSION="DXM 1.4.2"
#default to unknown on exit
SCRIPT_EXITCODE=3
# initialize logging channels
# &3 and &4: redirected to stdout and stderr for user interaction
exec 3>&1 4>&2
# &1 and &2: redirect to log or null
[ "$LOGLEVEL" -ge 1 ] && exec 1>>"$LOG" 2>&1 || exec 1>/dev/null 2>&1
# &5, &6, &7: debug logging
[ "$LOGLEVEL" -ge 2 ] && exec 5>>"$LOG" || exec 5>/dev/null
[ "$LOGLEVEL" -ge 3 ] && exec 6>>"$LOG" || exec 6>/dev/null
[ "$LOGLEVEL" -ge 4 ] && exec 7>>"$LOG" || exec 7>/dev/null
#cleanup on exit
trap 'exec 2>&4 1>&3 7>&- 6>&- 5>&- 4>&- 3>&-' 0 1 2 3
# logging function
log_message() {
MESSAGE="$1"
shift
: ${MODULE_NAME:="core"}
TIMESTAMP=`date "+%F %T"`
[ "$1" ] || echo "[$TIMESTAMP] [$MODULE_NAME]: $MESSAGE"
while [ "$1" ]; do echo "[$TIMESTAMP] [$MODULE_NAME]: $MESSAGE" >&"$1"; shift; done
}
##################
# BACKUP MODULES #
##################
# mysql dump module
module_mysqldump() {
# start up and check environment
local MODULE_NAME="mysqldump"
log_message "@@ module $MODULE_NAME starting" 1 3
mysql -sNe "SELECT 1" >/dev/null 2>&1 || { log_message "FATAL - MySQL connection failed, aborting" 2 4; MODULE_EXITCODE=2; return $MODULE_EXITCODE; }
mkdir -pv "`dirname $STATUSMYSQL`" >&6
mkdir -pv "$DUMPDIR" >&6
chmod -v 700 "$DUMPDIR" >&6
#autodetect events and routines support
mysqldump --help | grep "\-\-events" >/dev/null 2>&1 && MYSQLDUMPOPT="$MYSQLDUMPOPT --events"
mysqldump --help | grep "\-\-routines" >/dev/null 2>&1 && MYSQLDUMPOPT="$MYSQLDUMPOPT --routines"
local MODULE_EXITCODE=3
local DATABASES_DUMPED=0
local TABLES_DUMPED=0
local ERRORS_SUM=0
local DATABASES_TO_DUMP=`mysql -sNe "SELECT COUNT(SCHEMA_NAME) FROM INFORMATION_SCHEMA.SCHEMATA WHERE SCHEMA_NAME NOT IN ($MYSQL_EXCLUDE_DATABASES) AND SCHEMA_NAME NOT REGEXP '$MYSQL_EXCLUDE_DATABASES_REGEXP'"`
local TABLES_TO_DUMP=`mysql -sNe "SELECT COUNT(TABLE_NAME) FROM INFORMATION_SCHEMA.TABLES WHERE TABLE_SCHEMA NOT IN ($MYSQL_EXCLUDE_DATABASES) AND TABLE_SCHEMA NOT REGEXP '$MYSQL_EXCLUDE_DATABASES_REGEXP' AND CONCAT(TABLE_SCHEMA,'.',TABLE_NAME) NOT IN ($MYSQL_EXCLUDE_TABLES) AND TABLE_NAME NOT REGEXP '$MYSQL_EXCLUDE_TABLES_REGEXP'"`
local TABLES_TO_SKIP=`mysql -sNe "SELECT COUNT(TABLE_NAME) FROM INFORMATION_SCHEMA.TABLES WHERE TABLE_SCHEMA IN ($MYSQL_EXCLUDE_DATABASES) OR TABLE_SCHEMA REGEXP '$MYSQL_EXCLUDE_DATABASES_REGEXP' OR CONCAT(TABLE_SCHEMA,'.',TABLE_NAME) IN ($MYSQL_EXCLUDE_TABLES) OR TABLE_NAME REGEXP '$MYSQL_EXCLUDE_TABLES_REGEXP'"`
# there are some scary regular expressions above, make sure they worked
[ "$DATABASES_TO_DUMP" -gt 0 -a "$TABLES_TO_DUMP" -gt 0 -a "$TABLES_TO_SKIP" -ge 0 ] || { log_message "FATAL - MySQL excludes misconfigured, aborting" 2 4; MODULE_EXITCODE=2; return $MODULE_EXITCODE; }
# dump 'em
log_message "skipping $TABLES_TO_SKIP tables as per configured excludes" 1 3
log_message "excluded tables list follows" 5
mysql -sNe "SELECT CONCAT(TABLE_SCHEMA,'.',TABLE_NAME) FROM INFORMATION_SCHEMA.TABLES WHERE TABLE_SCHEMA IN ($MYSQL_EXCLUDE_DATABASES) OR TABLE_SCHEMA REGEXP '$MYSQL_EXCLUDE_DATABASES_REGEXP' OR CONCAT(TABLE_SCHEMA,'.',TABLE_NAME) IN ($MYSQL_EXCLUDE_TABLES) OR TABLE_NAME REGEXP '$MYSQL_EXCLUDE_TABLES_REGEXP'" >&5
log_message "proceeding to dump $TABLES_TO_DUMP tables in $DATABASES_TO_DUMP databases" 1 3
mysql -sNe "SELECT SCHEMA_NAME FROM INFORMATION_SCHEMA.SCHEMATA WHERE SCHEMA_NAME NOT IN ($MYSQL_EXCLUDE_DATABASES) AND SCHEMA_NAME NOT REGEXP '$MYSQL_EXCLUDE_DATABASES_REGEXP'" > "$DUMPDIR/DB_LIST"
while read DB
do
log_message "($((DATABASES_DUMPED+1))/$DATABASES_TO_DUMP) dumping tables in database $DB" 1 3
mysql -sNe "SELECT TABLE_NAME FROM INFORMATION_SCHEMA.TABLES WHERE TABLE_SCHEMA='$DB' AND CONCAT(TABLE_SCHEMA,'.',TABLE_NAME) NOT IN ($MYSQL_EXCLUDE_TABLES) AND TABLE_NAME NOT REGEXP '$MYSQL_EXCLUDE_TABLES_REGEXP'" > "$DUMPDIR/TABLE_LIST"
while read TABLE
do
log_message " dumping $TABLE" 1 3
mysqldump $MYSQLDUMPOPT "$DB" "$TABLE" >> "$DUMPDIR/$DB.sql"
local DUMP_EXITCODE=$?
log_message " mysqldump exit code: $DUMP_EXITCODE" 5
: $((ERRORS_SUM+=$DUMP_EXITCODE))
[ "$DUMP_EXITCODE" -eq 0 ] && : $((TABLES_DUMPED+=1))
done < "$DUMPDIR/TABLE_LIST"
log_message "purging old dump file $DUMPDIR/$DB.sql.gz" 5
rm -vf "$DUMPDIR/$DB.sql.gz" >&6
log_message "gzipping $DUMPDIR/$DB.sql" 5
gzip -3fv "$DUMPDIR/$DB.sql" >&6 2>&1
chmod -v 600 "$DUMPDIR/$DB.sql.gz" >&6
: $((DATABASES_DUMPED+=1))
done < "$DUMPDIR/DB_LIST"
rm -vf "$DUMPDIR/DB_LIST" "$DUMPDIR/TABLE_LIST" >&6
# report and return
if [ "$ERRORS_SUM" -eq 0 -a "$TABLES_TO_DUMP" -eq "$TABLES_DUMPED" -a "$DATABASES_TO_DUMP" -eq "$DATABASES_DUMPED" ]
then local STATUSMSG="MYSQL OK - dumped $TABLES_DUMPED out of $TABLES_TO_DUMP tables in $DATABASES_DUMPED out of $DATABASES_TO_DUMP databases"
echo "$STATUSMSG" > "$STATUSMYSQL"
MODULE_EXITCODE=0
log_message "$STATUSMSG" 1 3
else local STATUSMSG="MYSQL CRITICAL - $ERRORS_SUM, dumped $TABLES_DUMPED out of $TABLES_TO_DUMP tables in $DATABASES_DUMPED out of $DATABASES_TO_DUMP databases, see $LOG"
echo "$STATUSMSG" > "$STATUSMYSQL"
MODULE_EXITCODE=2
log_message "$STATUSMSG" 2 4
fi
log_message "@@ module $MODULE_NAME complete with exit code $MODULE_EXITCODE, sum of errors is $ERRORS_SUM" 1 3
return $MODULE_EXITCODE
}
################
# MAIN PROGRAM #
################
# startup
log_message "** $DXM_VERSION started as $0 by `readlink /proc/$PPID/exe`" 1 3
# environment checks
[ -r "$EXCLUDE" ] || { log_message "** $DXM_VERSION FATAL - cannot read exclude list $EXCLUDE, aborting" 2 4; exit 2; }
[ -r "$DIRLISTRSYNC" ] || { log_message "** $DXM_VERSION FATAL - cannot read rsync target list $DIRLISTRSYNC, aborting" 2 4; exit 2; }
[ -r "$DIRLISTTAR" ] || { log_message "** $DXM_VERSION FATAL - cannot read tar target list $DIRLISTTAR, aborting" 2 4; exit 2; }
[ -r "$LOCKFILE" ] && [ "$(cat $LOCKFILE | xargs ps -o comm= -p)" = "$(basename $0)" ] && { log_message "** $DXM_VERSION FATAL - lockfile $LOCKFILE exists and process is running, aborting: $(cat /tmp/dxm.lock | xargs ps -o pid= -o comm= -o etime= -p)" 2 4; exit 2; }
# check for deprecated options
for DEPRECATED_OPTION in "EXCLUDED_TABLES_PATTERN" "INCLUDED_TABLES_PATTERN"
do eval "CHECK=\${$DEPRECATED_OPTION}" && [ -n "${CHECK}" ] && log_message "** WARNING - ignoring deprecated option $DEPRECATED_OPTION, please update configuration" 2 4
unset "$DEPRECATED_OPTION"
done
# we didn't die till now, the exit code will be incremented if any module fails
SCRIPT_EXITCODE=0
[ -f "$LOCKFILE" ] && SCRIPT_EXITCODE=1 && log_message "** WARNING - lockfile $LOCKFILE exists but process $(cat $LOCKFILE) is missing; check previous run. $(cat /tmp/dxm.lock | xargs ps -o pid= -o comm= -o etime= -p)" 2 4
echo $$ > "$LOCKFILE" || { log_message "** FATAL - cannot write PID to $LOCKFILE, aborting" 2 4; exit 2; }
log_message "** checks passed, running modules" 1 3
# run mysql dump module
[ "$DOMYSQL" -eq 1 ] && { module_mysqldump; MODULE_EXITCODE=$?; [ "$MODULE_EXITCODE" -gt "$SCRIPT_EXITCODE" ] && SCRIPT_EXITCODE="$MODULE_EXITCODE"; unset MODULE_EXITCODE; }
###############
# PRE-DXM 1.4 #
###############
log_message "@@ now running non-verbose code, refer to $LOG" 1 3
#simple checks
if [ $DOFTP -eq 1 ] && ! which lftp
then
echo "FTP CRITICAL - please install lftp binary" | tee $STATUSFTP
echo "== removing lockfile $LOCKFILE"
rm -vf "$LOCKFILE" >&6
exit
fi
if [ $DOFTP -eq 1 ] && ! lftp $FTPHOST -e "cd $FTPDIR; exit" >&6
then
echo "FTP CRITICAL - cant connect to $FTPHOST or cd to $FTPDIR" | tee $STATUSFTP
fi
if [ $DORSYNC -eq 1 ] && ! /usr/bin/rsync $RSYNCOPT /etc/hosts $REMOTEHOSTRSYNC:$REMOTEDIRRSYNC >&6
then
echo "RSYNC CRITICAL - cant connect to $REMOTEHOSTRSYNC or cd to $REMOTEDIRRSYNC " | tee $STATUSRSYNC
fi
[ $DOFTP -eq 1 ] && ( chmod 600 ~/.netrc
echo 'set net:max-retries 4
set net:reconnect-interval-multiplier 1
set net:reconnect-interval-base 10' > ~/.lftprc )
if [ "$DOGPG" -eq 1 ] && ! gpg -k $GPGPUBKEY >&6
then
echo "GPG CRITICAL - GPGPUBKEY $GPGPUBKEY isnt available" | tee $STATUS
fi
#end of checks
mkdir -pv `dirname $LOG` >&6
TODAY=`date +%Y%m%d`
EXITFTPALL=0
echo -e "\n\n\n@@@@@@@@\n\n===START backup `date` by `readlink /proc/$PPID/exe` \n\n@@@@@@@@\n"
# start mysql backup
# old mysqldump code was here, moved to a clean module
# start postgresql backup
if [ "$DOPOSTGRESQL" -eq 1 ]
then
echo "===START postgresql backup `date` $0 "
mkdir -pv "$PGDUMPDIR" >&6
mkdir -pv `dirname $STATUSPOSTGRESQL` >&6
chmod -v 700 "$PGDUMPDIR" >&6
EXITCODEALL="0"
echo "===we have to backup the following DBs:"
su postgres -c "psql -lAx" | grep '^Name|' | sed -e "s/Name|//" | grep -v template0
for DB in `su postgres -c "psql -lAx" | grep '^Name|' | sed -e "s/Name|//" | grep -v template0`; do
echo ==$DB
su postgres -c "pg_dump $DB" > $PGDUMPDIR/$DB.sql
EXITCODE=$?
gzip -1fv $PGDUMPDIR/$DB.sql >&6 2>&1
chmod -v 600 $PGDUMPDIR/$DB.sql.gz >&6
echo EXITCODE $EXITCODE
EXITCODEALL=$(($EXITCODE+$EXITCODEALL))
done
echo "===END POSTGRESQL `date` $0 "
if [ "$EXITCODEALL" -eq "0" ]
then
echo "POSTGRESQL OK" > $STATUSPOSTGRESQL
else
echo "POSTGRESQL CRITICAL - see $LOG" > $STATUSPOSTGRESQL
fi
fi
#end postgresql backup
#start file backup
if [ "$DOFILES" -eq 1 ] ; then
echo "===START file backup `date` "
mkdir -pv "$DESTDIR" >&6
chmod -v 700 "$DESTDIR" >&6
mkdir -pv `dirname $STATUS` >&6
# if DUMPDIR isnt part of any DIRLISTTAR item, create special TAROPTIONS variable to include DUMPDIR in tar
[ "$DOMYSQL" -eq 1 ] && echo $DUMPDIR | grep -vf $DIRLISTTAR && \
TAROPTIONS="$DUMPDIR" && echo "DUMPDIR wasnt in DIRLISTTAR - fixed"
# ONLY exclude dxm/rsync if DORSYNC=1
[ "$DORSYNC" -eq 1 ] && TAROPTIONS=$TAROPTIONS" -X $DIRLISTRSYNC" && \
echo "$DIRLISTRSYNC has been excluded, since DORSYNC=1"
# 200907829.00.tar, 200907829.00.tar.lst -- full backup
# 200907829.01.tar, 200907829.01.tar.lst -- incr
LASTFULL=`find $DESTDIR -mindepth 1 -maxdepth 1 -name '????????.00.tgz' -printf "%f\n" |sort|tail -n1|cut -d. -f1`
# LASTFULL should be like 20090729.
FORCEFULLBACKUP=0
echo "$FORCEFULLBACKUPDAYS" | grep "`date +%u`" && FORCEFULLBACKUP=1
echo "FORCEFULLBACKUP=$FORCEFULLBACKUP, today is `date +%u`, allowed day list is $FORCEFULLBACKUPDAYS "
if [ -z "$LASTFULL" -o "$FORCEFULLBACKUP" -eq 1 ] || [ "$LASTFULL" -le "`date -d-1week +%Y%m%d`" ] ; then
echo "===last full backup doesnt exist or too old : $LASTFULL. do it as $DESTDIR/$TODAY.00.tgz "
rm -vf $DESTDIR/$TODAY.00.tgz.lst >&6
tar --numeric-owner -czpf "$DESTDIR/$TODAY.00.tgz" -T $DIRLISTTAR -g $DESTDIR/$TODAY.00.tgz.lst \
-X $EXCLUDE --exclude $DESTDIR $TAROPTIONS >&6
EXITTAR=$?
echo "===tar exited with code $EXITTAR"
NEWFILE=$DESTDIR/$TODAY.00.tgz
if [ "$DOGPG" -eq 1 ]
then
echo "== start gpg encryption $NEWFILE"
gpg --batch --encrypt -r $GPGPUBKEY --compress-algo none $NEWFILE >&6
GPGEXITCODE=$?
echo "== GPGEXITCODE was $GPGEXITCODE"
echo "== end gpg encryption"
[ "$GPGEXITCODE" -eq 0 ] || (echo "GPG ERROR" | tee $STATUS ; exit)
echo "== since we have encrypted these files successfully , let's put encrypted files instead of plain" \
| tee -a $LOG
mv -v $NEWFILE.gpg $NEWFILE >&6
fi
if [ "$EXITTAR" -eq "0" ] || [ "$EXITTAR" -eq "1" ] || [ "$EXITTAR" -eq "2" ]
then
echo "TAR OK - `du -h $NEWFILE`"> $STATUS
else
echo "TAR CRITICAL - see $LOG" > $STATUS
fi
if [ "$DOFTP" -eq "1" ]
then
echo "== start FTP upload at `date`"
lftp $FTPHOST -e "cd $FTPDIR; mput $DESTDIR/$TODAY.00.tgz ; exit" >&6
EXITFTP=$?
echo "== $DESTDIR/$TODAY.00.tgz has been uploaded, lftp exited with code $EXITFTP"
EXITFTPALL=$(($EXITFTPALL+$EXITFTP))
lftp $FTPHOST -e "cd $FTPDIR; mput $DESTDIR/$TODAY.00.tgz.lst ; exit" >&6
EXITFTP=$?
echo "== $DESTDIR/$TODAY.00.tgz.lst has been uploaded, lftp exited with code $EXITFTP"
EXITFTPALL=$(($EXITFTPALL+$EXITFTP))
fi
else
# If full backup exists, make increment
echo "===last full backup at $LASTFULL exists. lets make increment to last backup or its last increment"
LASTFULLORINCR=`find $DESTDIR/$LASTFULL*tgz | sort | tail -n1 `
echo "===LASTFULLORINCR is $LASTFULLORINCR"
NEWINDEX=`basename $LASTFULLORINCR | awk -F. '{printf "%.2d\n",$2+1}' `
cp -v $LASTFULLORINCR.lst $DESTDIR/$LASTFULL.$NEWINDEX.tgz.lst >&6
#fi
tar --numeric-owner -czpf "$DESTDIR/$LASTFULL.$NEWINDEX.tgz" -T $DIRLISTTAR -g $DESTDIR/$LASTFULL.$NEWINDEX.tgz.lst \
-X $EXCLUDE --exclude $DESTDIR 2>&1 $TAROPTIONS >&6
EXITTAR=$?
echo "===tar exited with code $EXITTAR"
NEWFILE="$DESTDIR/$LASTFULL.$NEWINDEX.tgz"
if [ "$DOGPG" -eq 1 ]
then
echo "== start gpg encryption $NEWFILE"
rm -vf $NEWFILE.gpg >&6
gpg --batch --encrypt -r $GPGPUBKEY --compress-algo none $NEWFILE >&6
GPGEXITCODE=$?
echo "== GPGEXITCODE was $GPGEXITCODE"
echo "== end gpg encryption"
[ "$GPGEXITCODE" -eq 0 ] || (echo "GPG ERROR" | tee $STATUS; exit )
echo "== since we have encrypted these files successfully , let's put encrypted files instead of plain" \
| tee -a $LOG
mv -v $NEWFILE.gpg $NEWFILE >&6
fi
if [ "$EXITTAR" -eq "0" ] || [ "$EXITTAR" -eq "1" ] || [ "$EXITTAR" -eq "2" ]
then
echo "TAR OK - `du -h $NEWFILE`"> $STATUS
else
echo "TAR CRITICAL - see $LOG" > $STATUS
fi
if [ "$DOFTP" -eq "1" ]
then
echo "== start FTP upload at `date`"
lftp $FTPHOST -e "debug ; cd $FTPDIR; put $DESTDIR/$LASTFULL.$NEWINDEX.tgz ; exit" >&6 2>&1
EXITFTP=$?
echo "== $DESTDIR/$LASTFULL.$NEWINDEX.tgz has been uploaded, lftp exited with code $EXITFTP"
EXITFTPALL=$(($EXITFTPALL+$EXITFTP))
lftp $FTPHOST -e "debug ; cd $FTPDIR; put $DESTDIR/$LASTFULL.$NEWINDEX.tgz.lst ; exit" >&6 2>&1
EXITFTP=$?
echo "== $DESTDIR/$LASTFULL.$NEWINDEX.tgz.lst has been uploaded, lftp exited with code $EXITFTP"
EXITFTPALL=$(($EXITFTPALL+$EXITFTP))
fi
fi
echo "===END file backup `date` " 2>&1
fi
# end file backup
if [ "$DOFTP" -eq 1 ]
then
if [ "$EXITFTPALL" -eq "0" ]
then
echo "FTP OK" > $STATUSFTP
else
echo "FTP CRITICAL - upload error, see $LOG" > $STATUSFTP
fi
fi
# start rsync
if [ "$DORSYNC" -eq 1 ]
then
echo "== START rsync backup at `date`"
STATUS=""
cat "$DIRLISTRSYNC" | while read DIR
do
echo "== rsyncing '$DIR'"
date > "$DIR/.dxm.rsync.date"
/usr/bin/rsync $RSYNCOPT --exclude-from="$EXCLUDE" "$DIR" "$REMOTEHOSTRSYNC":"$REMOTEDIRRSYNC" >&6 2>&1
EXITRSYNC="$?"
echo "rsync exit status was $EXITRSYNC"
if [ "$EXITRSYNC" -eq 0 ] || [ "$EXITRSYNC" -eq 24 ] || [ "$EXITRSYNC" -eq 23 ]
then
:
else
STATUS="$STATUS$DIR=ERR; "
fi
done
if echo "$STATUS" | grep ERR
then
echo "RSYNC CRITICAL - $STATUS" | tee "$STATUSRSYNC"
else
echo "RSYNC OK" | tee "$STATUSRSYNC"
fi
echo "== overall status: `cat $STATUSRSYNC`"
echo "== END rsync backup at `date`"
fi
# end rsync
if [ "$DOFILES" -eq 1 ] ; then
# start cleanup
# del files which DATE (part of filename) is older than KEEP
echo "===START cleanup `date` " 2>&1
for FILE in `find $DESTDIR/ -mindepth 1 -maxdepth 1 -name "????????.*" -printf "%f\n" | sort `;
do DATE=`basename $FILE | cut -d. -f1`
echo -n "===file $FILE was created at $DATE"
if [ "$DATE" -le "`date -d-$KEEP +%Y%m%d`" ]
then
echo removing| tee -a $LOG
rm -vf "$DESTDIR/$FILE" >&6
else
echo keeping| tee -a $LOG
fi
done
echo "===END cleanup `date` "
# end cleanup
fi
echo "== removing lockfile $LOCKFILE"
rm -vf "$LOCKFILE" >&6
####################
# END PRE-DXM 1.4 #
####################
# finish
log_message "** $DXM_VERSION finished with exit code $SCRIPT_EXITCODE" 1 3; exit $SCRIPT_EXITCODE