Second Beta release

CHANGELOG.md

@@ -1,7 +1,11 @@
 # Changelog
 
-## Pre-Release 1.0.1-beta (31 December, 2024)
-* Minor bug fixes, see #5
+## Pre-Release 1.0.1-beta (1 January, 2025)
+* Files are added with the remote modification date, see #2
+* Adding directory and file permission on every synced file and path with 
+option to use custom, see #4
+* Minor general bug fixes like syntax errors
+* Bugfixes: #5
 
 ## Pre-Release 1.0.0-beta (30 December, 2024)
 * Minor bug fixes

install.sh

@@ -106,13 +106,13 @@ fi
 # First create all the directories
 log "Setting up all the directories..."
 log "Creating the directory \"${REPOCONFIGDIR}\""
-mkdir -p "$REPOCONFIGDIR"
+mkdir -m 755 -p "$REPOCONFIGDIR"
 log "Creating the directory \"${INSTALLDIR}\""
-mkdir -p "$INSTALLDIR"
+mkdir -m 555 -p "$INSTALLDIR"
 log "Creating the directory \"${EXCLUDESDIR}\""
-mkdir -p "$EXCLUDESDIR"
+mkdir -m 750 -p "$EXCLUDESDIR"
 log "Creating the directory \"${LOGDIR}\""
-mkdir -p "$LOGDIR"
+mkdir -m 750 -p "$LOGDIR"
 
 # Copy over the files
 log "Copying over all the files to their destinations..."
@@ -125,13 +125,9 @@ cp "${SRC_CONFIGDIR}/mirrorsync.conf" "${CONFIGDIR}/mirrorsync.conf"
 
 # Fix the .version file to readonly
 log "Setting up access rights"
-chmod 555 "${INSTALLDIR}"
 chmod 444 "${INSTALLDIR}/.version"
 chmod 554 "${INSTALLDIR}/mirrorsync.sh"
 chmod 554 "${INSTALLDIR}/httpsync.sh"
-chmod 750 "${EXCLUDESDIR}"
-chmod 750 "${LOGDIR}"
-chmod 755 "${REPOCONFIGDIR}"
 chmod 755 "${CONFIGDIR}"
 chmod 644 "${CONFIGDIR}/mirrorsync.conf"
 

opt/mirrorsync/httpsync.sh

@@ -26,6 +26,9 @@ DST=""
 SYNCLIST=()
 DATE_FORMAT="+%Y-%m-%d %H:%M:%S"
 COMPDATE_FORMAT="+%Y%m%d%H%M%S"
+FILEDATE_FORMAT="+%Y%m%d%H%M.%S"
+DIRPERM=755
+FILEPERM=644
 
 # Log functions
 log() { printf "[%(%F %T)T] %s\n" -1 "$*" >&2; }
@@ -47,6 +50,10 @@ Arguments:
   -h, --help
     Display this usage message and exit.
 
+  --chmod=DXXX,FYYY
+    Set specific file or directory permission on the destination mirror. Only numeric chmod is allowed.
+    Default: D${DIRPERM},F${FILEPERM}
+
   -d, --debug
     Activate Debug Mode, provides a very detailed output to the system console.
 
@@ -83,9 +90,13 @@ Arguments:
 EOF
 }
 
+# Temporary arguments variables
+chmod_arg=""
+
 # Arguments Parser
 while [ "$#" -gt 0 ]; do
     case $1 in
+        --chmod=*) chmod_arg="${1#*=}";;
         -d|--debug) DEBUG_ARG=1;;
         --delete-after) DELETE_AFTER=1;;
         --delete-excluded) DELETE_EXCLUDE=1;;
@@ -114,6 +125,23 @@ DST="${POSITIONAL_ARGS[1]}"
 # If the destination path is empty then do from user position
 if [ -z "$DST" ]; then DST=$(pwd); fi
 
+# If chmod argument is set, change the numbers
+if [ ! -z "$chmod_arg" ]; then
+    dirnum_test=0
+    filenum_test=0
+
+    # Exctract the first 3 numbers after each character match
+    [[ "${TEST}" =~ [dD]([[:digit:]]{3}) ]] && dirnum_test="${BASH_REMATCH[1]}"
+    [[ "${TEST}" =~ [fF]([[:digit:]]{3}) ]] && filenum_test="${BASH_REMATCH[1]}"
+
+    debug "Extracted the following chmod=D${dirnum_test},F${filenum_test}"
+
+    # Test if numbers are correct then add them to global
+    if [ $dirnum_test -gt 0 ]; then DIRPERM=$dirnum_test; fi
+    if [ $filenum_test -gt 0 ]; then FILEPERM=$filenum_test; fi
+
+fi
+
 # Function to validate if value is matched in a array of queries
 # Usage: arraymatch "value_to_test" "array of values to validate"
 arraymatch() {
@@ -234,7 +262,7 @@ parsefilelist() {
                             local modified_dst=$(date -r "$dst")
 
                             local compdate_src=$(date -d "$modified_src" "$COMPDATE_FORMAT")
-                            local compdate_dst=$(date -d "$modified_dst" "$COMPDATE_FORMAT")
+local compdate_dst=$(date -d "$modified_dst" "$COMPDATE_FORMAT")
 
                             # Check if file is changed based on date if date was extracted from source
                             if [ ! -z "$modified_src" ] && [[ $compdate_src > $compdate_dst ]]; then
@@ -264,12 +292,13 @@ parsefilelist() {
                     fi
 
                     local modified_str=$(date -d "$modified_src" "$DATE_FORMAT")
+                    local modified_file=$(date -d "$modified_src" "$FILEDATE_FORMAT")
                     local filesize=$(echo $bytes_src | numfmt --to=iec-i)
 
                     debug "Added a file of size ${filesize}B from \"${url}\" to the list, it was last modifed " \
                           "\"${modified_str}\""
                     # Add to the array
-                    SYNCLIST+=("${url},${dst},${bytes_src},${href}")
+                    SYNCLIST+=("${url},${dst},${bytes_src},${href},${modified_file}")
                 else
                     debug "Not a file \"$url\", ignoring path"
                 fi
@@ -286,7 +315,7 @@ parsefilelist() {
         for localfile in "${localfiles[@]}"
         do
             # Add to the array
-            SYNCLIST+=(",${localfile},,")
+            SYNCLIST+=(",${localfile},,,")
         done
     fi
 }
@@ -365,9 +394,16 @@ do
             if [ -f "$tmpfile" ]; then
                 dstdir="$(dirname "$tmpfile")"
                 # Ensure that a destination directory exists before moving
-                if mkdir -p "$dstdir" 2>&1; then
+                if mkdir -m $DIRPERM -p "$dstdir" 2>&1; then
                     # Then move the file
+                    debug "Moving the file \"${tmpfile}\" to \"${syncinfo[1]}\""
                     mv "$tmpfile" "${syncinfo[1]}" 2>&1
+
+                    debug "Setting permission $FILEPERM on \"${syncinfo[1]}\""
+                    chmod $FILEPERM "${syncinfo[1]}"
+
+                    debug "Changing the files modification date to remotes"
+                    touch -t "${syncinfo[4]}" "${syncinfo[1]}"
                 else
                     error "Could not create the directory \"${dstdir}\" for the transfered file \"${tmpfile}\""
                 fi
@@ -383,10 +419,16 @@ do
 
             # Ensure that a destination directory exists before transfering
             dstdir="$(dirname "${syncinfo[1]}")"
-            if mkdir -p "$dstdir" 2>&1; then
+            if mkdir -m $DIRPERM -p "$dstdir" 2>&1; then
                 # Begin transfer of file
                 actionlog "Transfering \"${syncinfo[0]}\" to \"${syncinfo[1]}\""
                 curl "${opts[@]}" "${syncinfo[0]}" --output "${syncinfo[1]}" 2>&1
+
+                debug "Setting permission $FILEPERM on \"${syncinfo[1]}\""
+                chmod $FILEPERM "${syncinfo[1]}"
+
+                debug "Changing the files modification date to remotes"
+                touch -t "${syncinfo[4]}" "${syncinfo[1]}"
             else
                 error "Could not create the directory \"${dstdir}\" for the remote file \"${syncinfo[0]}\""
             fi

opt/mirrorsync/mirrorsync.sh

@@ -24,6 +24,9 @@ DEBUG_ARG=0
 SHOW_PROGRESS=0
 BARLENGTH=40
 INTEGERCHECK='^[0-9]+$'
+DIRPERM=755
+FILEPERM=644
+
 
 # Log functions for standard output
 log_stdout() { printf "[%(%F %T)T] %s\n" -1 "$*" >&2; }
@@ -86,6 +89,9 @@ progress() {
     fi
 }
 
+# Temporary arguments variables
+chmod_arg=""
+
 # Arguments Help
 usage() {
     cat << EOF
@@ -100,6 +106,10 @@ Arguments:
     Set a path from where this script can read the configurations and settings.
     Default: $CONFIGDIR
 
+  --chmod=DXXX,FYYY
+    Set specific file or directory permission on the destination mirror. Only numeric chmod is allowed.
+    Default: D${DIRPERM},F${FILEPERM}
+
   -d, --debug
     Activate Debug Mode, provides a very detailed output to the system console.
 
@@ -121,6 +131,7 @@ EOF
 # Arguments Parser
 while [ "$#" -gt 0 ]; do
     case $1 in
+        --chmod=*) chmod_arg="${1#*=}";;
         --config-path=*) EXCLUDESDIR="${1#*=}";;
         -d|--debug) DEBUG_ARG=1;;
         --excludes-path=*) EXCLUDESDIR="${1#*=}";;
@@ -144,9 +155,27 @@ LOCKFILE="${EXCLUDESDIR}/mirrorsync.sh.lockfile"
 if [ ! -r "$CONFIGFILE" ]; then
     fatal_stdout "The script configfile \"$CONFIGFILE\" is not availble or readable"
 else
+    debug "Sourcing the main configuration file \"${CONFIGFILE}\""
     source "$CONFIGFILE"
 fi
 
+# If chmod argument is set, change the numbers
+if [ ! -z "$chmod_arg" ]; then
+    dirnum_test=0
+    filenum_test=0
+
+    # Exctract the first 3 numbers after each character match
+    [[ "${TEST}" =~ [dD]([[:digit:]]{3}) ]] && dirnum_test="${BASH_REMATCH[1]}"
+    [[ "${TEST}" =~ [fF]([[:digit:]]{3}) ]] && filenum_test="${BASH_REMATCH[1]}"
+
+    debug "Extracted the following chmod=D${dirnum_test},F${filenum_test}"
+
+    # Test if numbers are correct then add them to global
+    if [ $dirnum_test -gt 0 ]; then DIRPERM=$dirnum_test; fi
+    if [ $filenum_test -gt 0 ]; then FILEPERM=$filenum_test; fi
+
+fi
+
 # Verify repo path exists
 if [ ! -d "$REPOCONFIGDIR" ]; then
     fatal_stdout "The directory \"$REPOCONFIGDIR\" does not exist"
@@ -284,7 +313,7 @@ do
     # Check if directory exists, else create it
     if [ ! -d "$mirrordst" ]; then
         warning "A local path for \"${mirrorname}\" does not exists, will create one"
-        if [ ! mkdir "$mirrordst" 2>&1 ]; then
+        if ! (mkdir -m $DIRPERM "$mirrordst" 2>&1); then
             error "The path \"${mirrordst}\" could not be created. Continuing with the next mirror"
             progresscounter=$((progresscounter+3))
             continue
@@ -464,6 +493,9 @@ do
                 continue
             fi
 
+            # Add the chmod rule to options
+            opts+=(--chmod=D${DIRPERM},F${FILEPERM})
+
             # header for the new log fil
             print_header_updatelog "rsync" "$remotesrc" "$mirrordst" "$transfersize" "$availablesize" "$updatelogfile" \
             "${opts[*]}"
@@ -510,6 +542,9 @@ do
                 continue
             fi
 
+            # Add the chmod rule to options
+            opts+=(--chmod=D${DIRPERM},F${FILEPERM})
+
             # header for the new log fil
             print_header_updatelog "http" "$remotesrc" "$mirrordst" "$transfersize" "$availablesize" "$updatelogfile" \
             "${opts[*]}"