mirror of
https://github.com/deajan/osync
synced 2024-11-11 07:10:40 +00:00
Somehow messy code yet, but working.
This commit is contained in:
deajan
parent
a1420a635b
commit
fab0ae5a03
131
osync.sh
131
osync.sh
@ -1,9 +1,19 @@
|
|||||||
#!/bin/bash
|
#!/bin/bash
|
||||||
|
|
||||||
#### Osyc
|
###### Osync - Rsync based two way sync engine.
|
||||||
## Rsync based two way sync engine. Master/slave setup.
|
###### (L) 2013 by Orsiris "Ozy" de Jong (www.netpower.fr)
|
||||||
|
|
||||||
|
|
||||||
|
## todo:
|
||||||
|
# add dryrun
|
||||||
|
# add logging
|
||||||
|
# add resume on error
|
||||||
|
# add master/slave prevalance
|
||||||
|
# thread sync function
|
||||||
|
# remote functionnality
|
||||||
|
|
||||||
OSYNC_VERSION=0.4
|
OSYNC_VERSION=0.4
|
||||||
OSYNC_BUILD=1507201301
|
OSYNC_BUILD=1607201302
|
||||||
|
|
||||||
DEBUG=no
|
DEBUG=no
|
||||||
SCRIPT_PID=$$
|
SCRIPT_PID=$$
|
||||||
@ -14,10 +24,9 @@ LOCAL_HOST=$(hostname)
|
|||||||
## Default log file until config file is loaded
|
## Default log file until config file is loaded
|
||||||
LOG_FILE=/var/log/osync.log
|
LOG_FILE=/var/log/osync.log
|
||||||
|
|
||||||
## Working directory on master
|
## Working directory. Will keep current file states, backups and soft deleted files.
|
||||||
OSYNC_DIR=".osync_workdir"
|
OSYNC_DIR=".osync_workdir"
|
||||||
|
|
||||||
|
|
||||||
## Log a state message every $KEEP_LOGGING seconds. Should not be equal to soft or hard execution time so your log won't be unnecessary big.
|
## Log a state message every $KEEP_LOGGING seconds. Should not be equal to soft or hard execution time so your log won't be unnecessary big.
|
||||||
KEEP_LOGGING=1801
|
KEEP_LOGGING=1801
|
||||||
|
|
||||||
@ -360,32 +369,44 @@ function CheckMasterSlaveDirs
|
|||||||
|
|
||||||
function LockMaster
|
function LockMaster
|
||||||
{
|
{
|
||||||
echo o
|
echo "Not implemented yet"
|
||||||
}
|
}
|
||||||
|
|
||||||
function LockSlave
|
function LockSlave
|
||||||
{
|
{
|
||||||
echo o
|
echo "Not implemented yet"
|
||||||
|
}
|
||||||
|
|
||||||
|
# Subfunction of Sync
|
||||||
|
function sync_update_master
|
||||||
|
{
|
||||||
|
Log "Updating slave replica."
|
||||||
|
rsync -rlptgodEui --backup --backup-dir "$MASTER_BACKUP_DIR" --exclude "$OSYNC_DIR" --exclude-from "$STATE_DIR/master-deleted-list" --exclude-from "$STATE_DIR/slave-deleted-list" $MASTER_SYNC_DIR/ $SLAVE_SYNC_DIR/
|
||||||
|
}
|
||||||
|
|
||||||
|
# Subfunction of Sync
|
||||||
|
function sync_update_slave
|
||||||
|
{
|
||||||
|
Log "Updating master replica."
|
||||||
|
rsync -rlptgodEui --backup --backup-dir "$SLAVE_BACKUP_DIR" --exclude "$OSYNC_DIR" --exclude-from "$STATE_DIR/slave-deleted-list" --exclude-from "$STATE_DIR/master-deleted-list" $SLAVE_SYNC_DIR/ $MASTER_SYNC_DIR/
|
||||||
}
|
}
|
||||||
|
|
||||||
function Sync
|
function Sync
|
||||||
{
|
{
|
||||||
## decide if local or remote prevalence
|
|
||||||
|
|
||||||
## Lock master dir
|
## Lock master dir
|
||||||
## Lock slave dir
|
## Lock slave dir
|
||||||
|
|
||||||
Log "Starting synchronization task."
|
Log "Starting synchronization task."
|
||||||
|
|
||||||
## Create local file list
|
## Create local file list
|
||||||
Log "Creating current master file list"
|
Log "Creating master replica file list."
|
||||||
rsync -rlptgodE --exclude "$OSYNC_DIR" --list-only $MASTER_SYNC_DIR/ | grep "^-\|^d" | awk '{print $5}' | grep -v "^\.$" > $STATE_DIR/master-tree-current
|
rsync -rlptgodE --exclude "$OSYNC_DIR" --list-only $MASTER_SYNC_DIR/ | grep "^-\|^d" | awk '{print $5}' | grep -v "^\.$" > $STATE_DIR/master-tree-current
|
||||||
Log "Creating current slave file list"
|
Log "Creating slave replica file list."
|
||||||
## Create distant file list
|
## Create distant file list
|
||||||
rsync -rlptgodE --exclude "$OSYNC_DIR" --list-only $SLAVE_SYNC_DIR/ | grep "^-\|^d" | awk '{print $5}' | grep -v "^\.$" > $STATE_DIR/slave-tree-current
|
rsync -rlptgodE --exclude "$OSYNC_DIR" --list-only $SLAVE_SYNC_DIR/ | grep "^-\|^d" | awk '{print $5}' | grep -v "^\.$" > $STATE_DIR/slave-tree-current
|
||||||
|
|
||||||
## diff local file list and before file list except if before file list is empty >> deleted
|
## diff local file list and before file list except if before file list is empty >> deleted
|
||||||
Log "Creating master deleted file list"
|
Log "Creating master replica deleted file list."
|
||||||
if [ -f $STATE_DIR/master-tree-before ]
|
if [ -f $STATE_DIR/master-tree-before ]
|
||||||
then
|
then
|
||||||
comm --nocheck-order -23 $STATE_DIR/master-tree-before $STATE_DIR/master-tree-current > $STATE_DIR/master-deleted-list
|
comm --nocheck-order -23 $STATE_DIR/master-tree-before $STATE_DIR/master-tree-current > $STATE_DIR/master-deleted-list
|
||||||
@ -394,7 +415,7 @@ function Sync
|
|||||||
fi
|
fi
|
||||||
|
|
||||||
## diff local file list and before file list except if before file list is empty >> deleted
|
## diff local file list and before file list except if before file list is empty >> deleted
|
||||||
Log "Creating slave deleted file list"
|
Log "Creating slave replica deleted file list."
|
||||||
if [ -f $STATE_DIR/slave-tree-before ]
|
if [ -f $STATE_DIR/slave-tree-before ]
|
||||||
then
|
then
|
||||||
comm --nocheck-order -23 $STATE_DIR/slave-tree-before $STATE_DIR/slave-tree-current > $STATE_DIR/slave-deleted-list
|
comm --nocheck-order -23 $STATE_DIR/slave-tree-before $STATE_DIR/slave-tree-current > $STATE_DIR/slave-deleted-list
|
||||||
@ -402,23 +423,24 @@ function Sync
|
|||||||
touch $STATE_DIR/slave-deleted-list
|
touch $STATE_DIR/slave-deleted-list
|
||||||
fi
|
fi
|
||||||
|
|
||||||
|
if [ "$CONFLICT_PREVALANCE" != "master" ]
|
||||||
|
then
|
||||||
|
sync_update_slave
|
||||||
|
sync_update_master
|
||||||
|
else
|
||||||
|
sync_update_master
|
||||||
|
sync_update_slave
|
||||||
|
fi
|
||||||
|
|
||||||
## update local -> remote except deleted
|
Log "Propagating deletitions to slave replica."
|
||||||
Log "Updating slave replica"
|
|
||||||
rsync -rlptgodEui --backup --backup-dir "$MASTER_BACKUP_DIR" --exclude "$OSYNC_DIR" --exclude-from "$STATE_DIR/master-deleted-list" --exclude-from "$STATE_DIR/slave-deleted-list" $MASTER_SYNC_DIR/ $SLAVE_SYNC_DIR/
|
|
||||||
## update remote -> local except deleted
|
|
||||||
Log "Updating master replica"
|
|
||||||
rsync -rlptgodEui --backup --backup-dir "$SLAVE_BACKUP_DIR" --exclude "$OSYNC_DIR" --exclude-from "$STATE_DIR/slave-deleted-list" --exclude-from "$STATE_DIR/master-deleted-list" $SLAVE_SYNC_DIR/ $MASTER_SYNC_DIR/
|
|
||||||
|
|
||||||
Log "Propagating deletitions to slave"
|
|
||||||
rsync -rlptgodEui --backup --backup-dir "$MASTER_DELETE_DIR" --delete --exclude "$OSYNC_DIR" --include-from "$STATE_DIR/master-deleted-list" --exclude-from "$STATE_DIR/slave-deleted-list" $MASTER_SYNC_DIR/ $SLAVE_SYNC_DIR/
|
rsync -rlptgodEui --backup --backup-dir "$MASTER_DELETE_DIR" --delete --exclude "$OSYNC_DIR" --include-from "$STATE_DIR/master-deleted-list" --exclude-from "$STATE_DIR/slave-deleted-list" $MASTER_SYNC_DIR/ $SLAVE_SYNC_DIR/
|
||||||
Log "Propagating deletitions to master"
|
Log "Propagating deletitions to master replica."
|
||||||
rsync -rlptgodEui --backup --backup-dir "$SLAVE_DELETE_DIR" --delete --exclude "$OSYNC_DIR" --include-from "$STATE_DIR/slave-deleted-list" --exclude-from "$STATE_DIR/master-deleted-list" $SLAVE_SYNC_DIR/ $MASTER_SYNC_DIR/
|
rsync -rlptgodEui --backup --backup-dir "$SLAVE_DELETE_DIR" --delete --exclude "$OSYNC_DIR" --include-from "$STATE_DIR/slave-deleted-list" --exclude-from "$STATE_DIR/master-deleted-list" $SLAVE_SYNC_DIR/ $MASTER_SYNC_DIR/
|
||||||
|
|
||||||
## Create local file list
|
## Create local file list
|
||||||
Log "Creating new master file list"
|
Log "Creating new master replica file list."
|
||||||
rsync -rlptgodE --exclude "$OSYNC_DIR" --list-only $MASTER_SYNC_DIR/ | grep "^-\|^d" | awk '{print $5}' | grep -v "^\.$" > $STATE_DIR/master-tree-before
|
rsync -rlptgodE --exclude "$OSYNC_DIR" --list-only $MASTER_SYNC_DIR/ | grep "^-\|^d" | awk '{print $5}' | grep -v "^\.$" > $STATE_DIR/master-tree-before
|
||||||
Log "Creating new slave file list"
|
Log "Creating new slave replica file list."
|
||||||
## Create distant file list
|
## Create distant file list
|
||||||
rsync -rlptgodE --exclude "$OSYNC_DIR" --list-only $SLAVE_SYNC_DIR/ | grep "^-\|^d" | awk '{print $5}' | grep -v "^\.$" > $STATE_DIR/slave-tree-before
|
rsync -rlptgodE --exclude "$OSYNC_DIR" --list-only $SLAVE_SYNC_DIR/ | grep "^-\|^d" | awk '{print $5}' | grep -v "^\.$" > $STATE_DIR/slave-tree-before
|
||||||
|
|
||||||
@ -428,7 +450,35 @@ function Sync
|
|||||||
|
|
||||||
function SoftDelete
|
function SoftDelete
|
||||||
{
|
{
|
||||||
echo softd
|
if [ "$CONFLICT_BACKUP" != "no" ]
|
||||||
|
then
|
||||||
|
if [ -d $MASTER_BACKUP_DIR ]
|
||||||
|
then
|
||||||
|
Log "Removing backups older than $CONFLICT_BACKUP_DAYS days on master replica."
|
||||||
|
find $MASTER_BACKUP_DIR/ -ctime +$CONFLICT_BACKUP_DAYS | xargs rm -rf
|
||||||
|
fi
|
||||||
|
|
||||||
|
if [ -d $SLAVE_BACKUP_DIR ]
|
||||||
|
then
|
||||||
|
Log "Removing backups older than $CONFLICT_BACKUP_DAYS days on slave replica."
|
||||||
|
find $SLAVE_BACKUP_DIR/ -ctime +$CONFLICT_BACKUP_DAYS | xargs rm -rf
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
|
||||||
|
if [ "$SOFT_DELETE" != "no" ]
|
||||||
|
then
|
||||||
|
if [ -d $MASTER_DELETE_DIR ]
|
||||||
|
then
|
||||||
|
Log "Removing soft deleted items older than $SOFT_DELETE_DAYS days on master replica."
|
||||||
|
find $MASTER_DELETE_DIR/ -ctime +$SOFT_DELETE_DAYS | xargs rm -rf
|
||||||
|
fi
|
||||||
|
|
||||||
|
if [ -d $SLAVE_DELETE_DIR ]
|
||||||
|
then
|
||||||
|
Log "Removing soft deleted items older than $SOFT_DELETE_DAYS days on slave replica."
|
||||||
|
find $SLAVE_DELETE_DIR/ -ctime +$SOFT_DELETE_DAYS | xargs rm -rf
|
||||||
|
fi
|
||||||
|
fi
|
||||||
}
|
}
|
||||||
|
|
||||||
function Init
|
function Init
|
||||||
@ -443,7 +493,7 @@ function Init
|
|||||||
trap 'TrapError ${LINENO} $?' ERR
|
trap 'TrapError ${LINENO} $?' ERR
|
||||||
fi
|
fi
|
||||||
|
|
||||||
LOG_FILE=/var/log/osync_$osync_VERSION-$BACKUP_ID.log
|
LOG_FILE=/var/log/osync_$OSYNC_VERSION-$SYNC_ID.log
|
||||||
MAIL_ALERT_MSG="Warning: Execution of osync instance $OSYNC_ID (pid $SCRIPT_PID) as $LOCAL_USER@$LOCAL_HOST produced errors."
|
MAIL_ALERT_MSG="Warning: Execution of osync instance $OSYNC_ID (pid $SCRIPT_PID) as $LOCAL_USER@$LOCAL_HOST produced errors."
|
||||||
|
|
||||||
STATE_DIR="$MASTER_SYNC_DIR/$OSYNC_DIR/state"
|
STATE_DIR="$MASTER_SYNC_DIR/$OSYNC_DIR/state"
|
||||||
@ -453,11 +503,14 @@ function Init
|
|||||||
MASTER_DELETE_DIR="$MASTER_SYNC_DIR/$OSYNC_DIR/deleted"
|
MASTER_DELETE_DIR="$MASTER_SYNC_DIR/$OSYNC_DIR/deleted"
|
||||||
SLAVE_BACKUP_DIR="$SLAVE_SYNC_DIR/$OSYNC_DIR/backups"
|
SLAVE_BACKUP_DIR="$SLAVE_SYNC_DIR/$OSYNC_DIR/backups"
|
||||||
SLAVE_DELETE_DIR="$SLAVE_SYNC_DIR/$OSYNC_DIR/deleted"
|
SLAVE_DELETE_DIR="$SLAVE_SYNC_DIR/$OSYNC_DIR/deleted"
|
||||||
}
|
|
||||||
|
## Runner definition
|
||||||
function DryRun
|
if [ "$REMOTE_SYNC" == "yes" ]
|
||||||
{
|
then
|
||||||
echo dry
|
RUNNER="$(which ssh) $SSH_COMP -i $SSH_RSA_PRIVATE_KEY $REMOTE_USER@$REMOTE_HOST -p $REMOTE_PORT"
|
||||||
|
else
|
||||||
|
RUNNER=""
|
||||||
|
fi
|
||||||
}
|
}
|
||||||
|
|
||||||
function Main
|
function Main
|
||||||
@ -515,20 +568,16 @@ then
|
|||||||
then
|
then
|
||||||
Init
|
Init
|
||||||
DATE=$(date)
|
DATE=$(date)
|
||||||
Log "---------------------------------------------------------"
|
Log "-----------------------------------------------------------"
|
||||||
Log "$DATE - Osync v$OSYNC_VERSION script begin."
|
Log "$DATE - Osync v$OSYNC_VERSION script begin."
|
||||||
Log "----------------------------------------------------------"
|
Log "-----------------------------------------------------------"
|
||||||
CheckMasterSlaveDirs
|
CheckMasterSlaveDirs
|
||||||
if [ $? == 0 ]
|
if [ $? == 0 ]
|
||||||
then
|
then
|
||||||
if [ $dryrun -eq 1 ]
|
RunBeforeHook
|
||||||
then
|
Main
|
||||||
DryRun
|
SoftDelete
|
||||||
else
|
RunAfterHook
|
||||||
RunBeforeHook
|
|
||||||
Main
|
|
||||||
RunAfterHook
|
|
||||||
fi
|
|
||||||
CleanUp
|
CleanUp
|
||||||
fi
|
fi
|
||||||
else
|
else
|
||||||
|
|||||||
10
sync.conf
10
sync.conf
@ -1,7 +1,7 @@
|
|||||||
#!/bin/bash
|
#!/bin/bash
|
||||||
|
|
||||||
#### Osync multi-master file synchronizer
|
###### Osync file synchronizer tool based on rsync
|
||||||
#### (L) 2013 by Orsiris "Ozy" de Jong (www.netpower.fr)
|
###### (L) 2013 by Orsiris "Ozy" de Jong (www.netpower.fr)
|
||||||
#### Config file rev 1507201301
|
#### Config file rev 1507201301
|
||||||
|
|
||||||
## Sync job identification, any string you want
|
## Sync job identification, any string you want
|
||||||
@ -38,9 +38,9 @@ RSYNC_COMPRESS=yes
|
|||||||
SOFT_MAX_EXEC_TIME=30000
|
SOFT_MAX_EXEC_TIME=30000
|
||||||
HARD_MAX_EXEC_TIME=36000Ԯ
|
HARD_MAX_EXEC_TIME=36000Ԯ
|
||||||
|
|
||||||
CONFLICT_WINNER=master
|
CONFLICT_PREVALANCE=master
|
||||||
CONFLICT_KEEP_BACKUP=yes
|
CONFLICT_BACKUP=yes
|
||||||
CONFLICT_KEEP_BACKUP_DAYS=30
|
CONFLICT_BACKUP_DAYS=30
|
||||||
|
|
||||||
SOFT_DELETE=yes
|
SOFT_DELETE=yes
|
||||||
SOFT_DELETE_DAYS=30
|
SOFT_DELETE_DAYS=30
|
||||||
|
|||||||
Loading…
Reference in New Issue
Block a user