Geek's Rhapsody

Tag Archives: s3

2016-05-20   System Engineering   No comments

Mysql Backup Script

Features Full Backup (Mysqldump) Incremental Backup (Bin-log) Local Backup & Remote(AWS S3) Upload Sample Environments MySQL bin log prefix : mysql-bin BINLOG_PATH=/home/mysql/data BACKUP_ROOT=/home/backup/mysql S3_PATH=s3://system-backup/{SyatemName}/mysql

backup.mysql.sh

Shell

#!/bin/bash #---------------------------------------- # MySQL Backup Script # Revision 20160531 # Copyright (c) Aiden Kim # http://www.daemon.pe.kr #---------------------------------------- ARCHIVE_PERIOD=60 BINLOG_PATH=/home/mysql/data BASE_PATH=/home/backup/mysql SYSNAME=$(echo $HOSTNAME | tr "[:lower:]" "[:upper:]") S3_PATH=s3://system-backup/${SYSNAME}/mysql STATUS_PATH=${BASE_PATH}/mysql-backup-status ## prepare path if ! [ -d ${BASE_PATH} ]; then /bin/mkdir -p ${BASE_PATH} fi # Full backup if [[ "$1" == "full" ]]; then echo "MySQL Full Backup started `date`..." # prepare directory TSTR=$(date +"%Y%m%d_%H%M%S") BACKUP_PATH=${BASE_PATH}/${TSTR} if ! [[ -d ${BACKUP_PATH} ]]; then /bin/mkdir -p ${BACKUP_PATH} fi /bin/rm -rf ${BACKUP_PATH}/* # get old full backup information if [[ -r ${STATUS_PATH} ]]; then OLD_BACKUP_PATH=`/bin/cat ${STATUS_PATH} | /bin/sed -n '1p'` OLD_START_BINLOG=`/bin/cat ${STATUS_PATH} | /bin/sed -n '2p'` fi # mysqldump /usr/local/mysql/bin/mysqldump --all-databases --routines --events --hex-blob --single-transaction --flush-logs --master-data=2 --include-master-host-port > ${BACKUP_PATH}/full_backup-${TSTR}.sql START_BINLOG=`/bin/sed -rn "1,50 s/^.+ MASTER_LOG_FILE\='(mysql-bin\.[0-9]+)'.+$/\1/p" ${BACKUP_PATH}/full_backup-${TSTR}.sql | /bin/sed -n '1p'` echo "${BACKUP_PATH}" > ${STATUS_PATH} echo "${START_BINLOG}" >> ${STATUS_PATH} /bin/gzip ${BACKUP_PATH}/full_backup-${TSTR}.sql # finalize previous full backup data if [[ -n ${OLD_BACKUP_PATH} ]] && [[ -n ${OLD_START_BINLOG} ]] && [[ -d ${OLD_BACKUP_PATH} ]]; then for FILE in `ls ${BINLOG_PATH}/mysql-bin.?????? | sort -g` do if [[ "${FILE}" < "${BINLOG_PATH}/${START_BINLOG}" ]]; then if [[ "${FILE}" == "${BINLOG_PATH}/${OLD_START_BINLOG}" ]] || [[ "${FILE}" > "${BINLOG_PATH}/${OLD_START_BINLOG}" ]]; then /bin/cp -afpu ${FILE} ${OLD_BACKUP_PATH}/ fi fi done fi # Incremental backup else echo "MySQL Incremental Backup started..." # get previous full backup data if ! [[ -r ${STATUS_PATH} ]]; then echo "Could not find full backup status file." exit 1 fi BACKUP_PATH=`/bin/cat ${STATUS_PATH} | /bin/sed -n '1p'` START_BINLOG=`/bin/cat ${STATUS_PATH} | /bin/sed -n '2p'` TSTR=`/bin/echo ${BACKUP_PATH} | sed -rn "s/^.+\/(20[0-9]{6}_[0-9]{6})$/\1/p"` if ! [[ -d ${BACKUP_PATH} ]]; then echo "Could not find last full backup path." exit 2 fi if ! [[ -d ${BINLOG_PATH} ]]; then echo "Could not find mysql bin-log path." exit 2 fi # flush logs & copy bin-logs /usr/local/mysql/bin/mysqladmin flush-logs BINLOG_CURR=`ls -d ${BINLOG_PATH}/mysql-bin.?????? | sed 's/^.*\.//' | sort -g | tail -n 1` for FILE in `ls ${BINLOG_PATH}/mysql-bin.?????? | sort -g` do if [[ "${BINLOG_PATH}/mysql-bin.${BINLOG_CURR}" != "${FILE}" ]]; then if [[ "${FILE}" == "${BINLOG_PATH}/${START_BINLOG}" ]] || [[ "${FILE}" > "${BINLOG_PATH}/${START_BINLOG}" ]]; then /bin/cp -afpu ${FILE} ${BACKUP_PATH}/ fi fi done fi # sync to s3 bucket /usr/bin/aws s3 sync ${BACKUP_PATH}/ ${S3_PATH}/${TSTR}/ # remove old backups oldDirs=($(/bin/find ${BASE_PATH} -type d -mtime +${ARCHIVE_PERIOD})) oldDirLen=${#oldDirs[@]} for (( i=0; i<${oldDirLen}; i++ )); do /bin/rm -rf ${oldDirs[$i]} done exit 0

1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106

#!/bin/bash #---------------------------------------- # MySQL Backup Script # Revision 20160531 # Copyright (c) Aiden Kim # http://www.daemon.pe.kr #----------------------------------------   ARCHIVE_PERIOD=60 BINLOG_PATH=/home/mysql/data BASE_PATH=/home/backup/mysql SYSNAME=$(echo $HOSTNAME | tr "[:lower:]" "[:upper:]") S3_PATH=s3://system-backup/${SYSNAME}/mysql STATUS_PATH=${BASE_PATH}/mysql-backup-status   ## prepare path if ! [ -d ${BASE_PATH} ]; then   /bin/mkdir -p ${BASE_PATH} fi   # Full backup if [[ "$1" == "full" ]]; then       echo "MySQL Full Backup started `date`..."       # prepare directory     TSTR=$(date +"%Y%m%d_%H%M%S")     BACKUP_PATH=${BASE_PATH}/${TSTR}     if ! [[ -d ${BACKUP_PATH} ]]; then         /bin/mkdir -p ${BACKUP_PATH}     fi     /bin/rm -rf ${BACKUP_PATH}/*       # get old full backup information     if [[ -r ${STATUS_PATH} ]]; then         OLD_BACKUP_PATH=`/bin/cat ${STATUS_PATH} | /bin/sed -n '1p'`         OLD_START_BINLOG=`/bin/cat ${STATUS_PATH} | /bin/sed -n '2p'`     fi       # mysqldump     /usr/local/mysql/bin/mysqldump --all-databases --routines --events --hex-blob --single-transaction --flush-logs --master-data=2 --include-master-host-port > ${BACKUP_PATH}/full_backup-${TSTR}.sql     START_BINLOG=`/bin/sed -rn "1,50 s/^.+ MASTER_LOG_FILE\='(mysql-bin\.[0-9]+)'.+$/\1/p"  ${BACKUP_PATH}/full_backup-${TSTR}.sql | /bin/sed -n '1p'`     echo "${BACKUP_PATH}" > ${STATUS_PATH}     echo "${START_BINLOG}" >> ${STATUS_PATH}     /bin/gzip ${BACKUP_PATH}/full_backup-${TSTR}.sql       # finalize previous full backup data     if [[ -n ${OLD_BACKUP_PATH} ]] && [[ -n ${OLD_START_BINLOG} ]] && [[ -d ${OLD_BACKUP_PATH} ]]; then         for FILE in `ls ${BINLOG_PATH}/mysql-bin.?????? | sort -g`         do             if [[ "${FILE}" < "${BINLOG_PATH}/${START_BINLOG}" ]]; then                 if [[ "${FILE}" == "${BINLOG_PATH}/${OLD_START_BINLOG}" ]] || [[ "${FILE}" > "${BINLOG_PATH}/${OLD_START_BINLOG}" ]]; then                     /bin/cp -afpu ${FILE} ${OLD_BACKUP_PATH}/                 fi             fi         done     fi   # Incremental backup else       echo "MySQL Incremental Backup started..."       # get previous full backup data     if ! [[ -r ${STATUS_PATH} ]]; then         echo "Could not find full backup status file."         exit 1     fi     BACKUP_PATH=`/bin/cat ${STATUS_PATH} | /bin/sed -n '1p'`     START_BINLOG=`/bin/cat ${STATUS_PATH} | /bin/sed -n '2p'`     TSTR=`/bin/echo ${BACKUP_PATH} | sed -rn "s/^.+\/(20[0-9]{6}_[0-9]{6})$/\1/p"`     if ! [[ -d ${BACKUP_PATH} ]]; then         echo "Could not find last full backup path."         exit 2     fi     if ! [[ -d ${BINLOG_PATH} ]]; then         echo "Could not find mysql bin-log path."         exit 2     fi       # flush logs & copy bin-logs     /usr/local/mysql/bin/mysqladmin flush-logs     BINLOG_CURR=`ls -d ${BINLOG_PATH}/mysql-bin.?????? | sed 's/^.*\.//' | sort -g | tail -n 1`     for FILE in `ls ${BINLOG_PATH}/mysql-bin.?????? | sort -g`     do         if [[ "${BINLOG_PATH}/mysql-bin.${BINLOG_CURR}" != "${FILE}" ]]; then             if [[ "${FILE}" == "${BINLOG_PATH}/${START_BINLOG}" ]] || [[ "${FILE}" > "${BINLOG_PATH}/${START_BINLOG}" ]]; then                 /bin/cp -afpu ${FILE} ${BACKUP_PATH}/             fi         fi     done   fi   # sync to s3 bucket /usr/bin/aws s3 sync ${BACKUP_PATH}/ ${S3_PATH}/${TSTR}/   # remove old backups oldDirs=($(/bin/find ${BASE_PATH} -type d -mtime +${ARCHIVE_PERIOD})) oldDirLen=${#oldDirs[@]} for (( i=0; i<${oldDirLen}; i++ )); do     /bin/rm -rf ${oldDirs[$i]} done   exit 0

Executing

Run Examples

Shell

# Full Backup ./backup.mysql.sh full # Incremental Backup ./backup.mysql.sh

1 2 3 4 5

# Full Backup ./backup.mysql.sh full   # Incremental Backup ./backup.mysql.sh

Read More >>

2016-05-19   System Engineering   No comments

Subversion Repogitories Backup Script

Features Full Backup (Dump) Incremental Backup (Dump) Local Backup & Remote(AWS S3) Upload Sample Environments REPOS_ROOT=/home/svn/repos BACKUP_ROOT=/home/backup/svn S3_PATH=s3://svn-backup/svn

backup.svn.sh

Shell

#!/bin/bash #---------------------------------------- # Subversion Repogitories Backup Script # Revision 20160531 # Copyright (c) Aiden Kim # http://www.daemon.pe.kr #---------------------------------------- DSTR=$(date +"%Y%m%d") ARCHIVE_PERIOD=30 REPOS_ROOT=/home/svn/repos BACKUP_ROOT=/home/backup/svn S3_PATH=s3://svn-backup/svn # Each repogitories REPOS_DIRS=`ls -t ${REPOS_ROOT} 2>/dev/null` for DIR in ${REPOS_DIRS}; do # Create directory BACKUP_PATH=${BACKUP_ROOT}/repo-${DIR} if ! [ -d ${BACKUP_PATH} ]; then /bin/mkdir -p ${BACKUP_PATH} fi # Get lastest revision CURR_VER=`/usr/bin/svnlook youngest ${REPOS_ROOT}/${DIR} 2>/dev/null` if ! [[ ${CURR_VER} =~ ^[0-9]+$ ]] ; then echo "Could not get revision of repogitory ${DIR}. Skipped..." continue fi if [[ "$1" == "full" ]]; then # Full dump /usr/bin/svnadmin dump -q -r0:${CURR_VER} ${REPOS_ROOT}/${DIR} > ${BACKUP_PATH}/${DIR}-${DSTR}-full-${CURR_VER}.dump && /usr/bin/gzip -f ${BACKUP_PATH}/${DIR}-${DSTR}-full-${CURR_VER}.dump else # Find last full dump LAST_VER=`/usr/bin/ls -d ${BACKUP_PATH}/*-full-*.dump.gz | sed -r 's/^.*-full-([0-9]+)\.dump\.gz$/\1/' | sort -g | tail -n 1 2>/dev/null` if [[ ${LAST_VER} =~ ^[0-9]+$ ]] ; then if [ ${CURR_VER} -gt ${LAST_VER} ] ; then /usr/bin/svnadmin dump -q --incremental -r${LAST_VER}:${CURR_VER} ${REPOS_ROOT}/${DIR} > ${BACKUP_PATH}/${DIR}-${DSTR}-incremental-${LAST_VER}-${CURR_VER}.dump && /usr/bin/gzip -f ${BACKUP_PATH}/${DIR}-${DSTR}-incremental-${LAST_VER}-${CURR_VER}.dump else echo "Revision of ${DIR} was not changed. Current ${CURR_VER}, Last full-dump ${LAST_VER}. Skipped..." fi else echo "Could not file previous full-dump of ${DIR}. Skipped..." fi fi # Garbage rm -f ${BACKUP_PATH}/*.dump done # Sync to s3 bucket /usr/bin/aws s3 sync ${BACKUP_ROOT}/ ${S3_PATH}/ # Remove old files oldFiles=($(/bin/find ${BACKUP_ROOT} -type f -mtime +${ARCHIVE_PERIOD})) oldFilesLen=${#oldFiles[@]} for (( i=0; i<${oldFilesLen}; i++ )); do /bin/rm -rf ${oldFiles[$i]} done exit 0

1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63

#!/bin/bash #---------------------------------------- # Subversion Repogitories Backup Script # Revision 20160531 # Copyright (c) Aiden Kim # http://www.daemon.pe.kr #----------------------------------------   DSTR=$(date +"%Y%m%d") ARCHIVE_PERIOD=30 REPOS_ROOT=/home/svn/repos BACKUP_ROOT=/home/backup/svn S3_PATH=s3://svn-backup/svn   # Each repogitories REPOS_DIRS=`ls -t ${REPOS_ROOT} 2>/dev/null` for DIR in ${REPOS_DIRS}; do       # Create directory     BACKUP_PATH=${BACKUP_ROOT}/repo-${DIR}     if ! [ -d ${BACKUP_PATH} ]; then         /bin/mkdir -p ${BACKUP_PATH}     fi       # Get lastest revision     CURR_VER=`/usr/bin/svnlook youngest ${REPOS_ROOT}/${DIR} 2>/dev/null`     if ! [[ ${CURR_VER} =~ ^[0-9]+$ ]] ; then         echo "Could not get revision of repogitory ${DIR}. Skipped..."         continue     fi       if [[ "$1" == "full" ]]; then         # Full dump         /usr/bin/svnadmin dump -q -r0:${CURR_VER} ${REPOS_ROOT}/${DIR} > ${BACKUP_PATH}/${DIR}-${DSTR}-full-${CURR_VER}.dump && /usr/bin/gzip -f ${BACKUP_PATH}/${DIR}-${DSTR}-full-${CURR_VER}.dump     else         # Find last full dump         LAST_VER=`/usr/bin/ls -d ${BACKUP_PATH}/*-full-*.dump.gz | sed -r 's/^.*-full-([0-9]+)\.dump\.gz$/\1/' | sort -g | tail -n 1 2>/dev/null`         if [[ ${LAST_VER} =~ ^[0-9]+$ ]] ; then             if [ ${CURR_VER} -gt ${LAST_VER} ] ; then                 /usr/bin/svnadmin dump -q --incremental -r${LAST_VER}:${CURR_VER} ${REPOS_ROOT}/${DIR} > ${BACKUP_PATH}/${DIR}-${DSTR}-incremental-${LAST_VER}-${CURR_VER}.dump && /usr/bin/gzip -f ${BACKUP_PATH}/${DIR}-${DSTR}-incremental-${LAST_VER}-${CURR_VER}.dump             else                 echo "Revision of ${DIR} was not changed. Current ${CURR_VER}, Last full-dump ${LAST_VER}. Skipped..."             fi         else             echo "Could not file previous full-dump of ${DIR}. Skipped..."         fi     fi       # Garbage     rm -f ${BACKUP_PATH}/*.dump done   # Sync to s3 bucket /usr/bin/aws s3 sync ${BACKUP_ROOT}/ ${S3_PATH}/   # Remove old files oldFiles=($(/bin/find ${BACKUP_ROOT} -type f -mtime +${ARCHIVE_PERIOD})) oldFilesLen=${#oldFiles[@]} for (( i=0; i<${oldFilesLen}; i++ )); do     /bin/rm -rf ${oldFiles[$i]} done   exit 0

Executing

Run Examples

Shell

# Full Backup ./backup.svn.sh full # Incremental Backup ./backup.svn.sh

1 2 3 4 5

# Full Backup ./backup.svn.sh full   # Incremental Backup ./backup.svn.sh

 

Read More >>

• Recent Posts

  • OSMC kodi
  • Install SRSd with Postfix on OpenSUSE
  • 자체적으로 rotate되는 로그파일에 logrotate 적용하기
  • Apache 2.4 IP-ACL for ELB (also any other Proxy/LB)
  • Mysql General Procedures for DML using JSON data type

• Archives

  Archives Select Month September 2017  (1) August 2017  (1) October 2016  (1) July 2016  (1) June 2016  (2) May 2016  (2) April 2016  (3) March 2016  (1) January 2016  (1) March 2015  (1) January 2015  (1) December 2014  (3) September 2014  (2) July 2014  (1) June 2014  (1) January 2014  (1) September 2013  (3) August 2013  (1) July 2013  (2) July 2009  (1) February 2009  (2) November 2008  (1) July 2008  (1) February 2008  (1) November 2007  (3) October 2007  (2) August 2007  (4) July 2007  (1) May 2007  (1) March 2007  (1) January 2007  (7) November 2006  (1) September 2006  (4) August 2006  (1) July 2006  (1) May 2006  (1) April 2006  (1) March 2006  (5) January 2006  (1) May 2005  (2) April 2005  (2) March 2005  (11) February 2005  (1) October 2004  (1) August 2004  (1) July 2004  (1)

• Categories

  • ︰Uncategorized (7)
  • Column (20)
  • Developments (27)
  • Others (3)
  • System Engineering (24)
  • Toys (9)

• Tags

  705SC 706SC 707SC ActionScript 3.0 AppLocale AS3 AWS compc Event Flash flex fz50 g300 Gamin HTML json log MAHA 4.0 mail monitoring mysql nagios nms opensuse PDA php postfix procedure reverse proxy s3 sftp smtp source ssh subversion SWC Windows Installer wm5 wmnetmgr wmplayer 오픈웹 접근성 충전기 코딩 표준

• About Aiden Kim

  Web developer
  Software engeneer
  ... in South Korea & Japan

• Meta

  • Entries RSS
  • Contact to Aiden

• Banner