feat(ci): Recovery from backup archive in gitlab-ci

.gitlab-ci.yml

@@ -5,6 +5,7 @@ stages:
   - build
   - deploy
   - backup
+  - recovery
 
 build the website:
   stage: build
@@ -59,3 +60,26 @@ Create the backup:
   script:
     - ssh -p $SSHPORT $SSHCONNECT "mkdir -p ~/web ~/backups/; cd /var/www/sites/r3lab.uni.lu; tar -zcvf ~/backups/backup_$(date +'%Y_%m_%d').tar.gz *"
     - ssh -p $SSHPORT $SSHCONNECT "cd ~/backups/; ls -tr | head -n -3 | xargs --no-run-if-empty rm "
+
+## Recovery from backups
+Recover website from backup:
+  stage: recovery
+  when: manual
+  before_script:
+    - 'which ssh-agent || ( apk add --update openssh )'
+    - eval $(ssh-agent -s)
+    - echo "$SSHPRIVKEY" | tr -d '\r' | ssh-add - > /dev/null
+    - mkdir -p ~/.ssh && chmod 700 ~/.ssh
+    - 'echo -e "Host *\n\tStrictHostKeyChecking no\n\n" > ~/.ssh/config'
+    - echo "$KNOWNHOSTS" > ~/.ssh/known_hosts
+    - chmod 644 ~/.ssh/known_hosts
+  variables:
+    WEB_LOC: /var/www/sites/r3lab.uni.lu
+    BACKUP_LOC: /home/r3pages/backups
+  script:
+    # Upload/update recovery script
+    - scp -P $SSHPORT -r restore.sh $SSHCONNECT:~/restore.sh
+    # First, fail if the backup directory is empty
+    - ssh -p $SSHPORT $SSHCONNECT "find $BACKUP_LOC -mindepth 1 | read"
+    # Remove old files and restore them from archive
+    - ssh -p $SSHPORT $SSHCONNECT "bash restore.sh $WEB_LOC $BACKUP_LOC"

restore.sh

+#!/bin/bash
+
+FILES_LOCATION=$1
+BACKUP_LOCATION=$2
+
+echo "(ok) Executing the script as: $(whoami) in $(pwd)"
+
+if test -z "$FILES_LOCATION"
+then
+      echo "(bad) \$FILES_LOCATION envvar is empty, will exit now."
+      exit 1
+else
+      echo "(ok) Using the following file directory: $FILES_LOCATION"
+fi
+
+if test -z "$BACKUP_LOCATION"
+then
+      echo "(bad) \$BACKUP_LOCATION envvar is empty, will exit now."
+      exit 2
+else
+    echo "(ok) Using the backup location at: $BACKUP_LOCATION"
+fi
+
+if [ "$(ls -A $BACKUP_LOCATION)" ]; then
+    echo "(ok) Backup directory is not empty (good = there's a chance that it contains backup archives)."
+else
+    echo "(bad) Backup directory is empty (no backup archives inside. They should be weekly generated...), will exit now."
+    exit 3
+fi
+
+MOST_RECENT_BACKUP_FILE=`ls -t $BACKUP_LOCATION | grep .*.tar.gz | head -n1`
+if test -z "$MOST_RECENT_BACKUP_FILE"
+then
+    echo "(bad) There are no backups in backup directory, will exit now."
+    exit 4
+else
+    echo "(ok) Using the following backup archive: $MOST_RECENT_BACKUP_FILE"
+fi
+
+cd ~/web;
+DIRECTORY="recovery__old_web_files_$(openssl rand -hex 8)"
+
+echo "(ok) Creating a directory to contain the old files: $DIRECTORY."
+mkdir ~/$DIRECTORY -p
+
+if [ "$(ls -A $DIRECTORY 2>/dev/null)" ]; then
+    echo "(bad) The directory is not empty, will exit now."
+    exit 5    
+fi
+
+echo "(ok) Moving the files from ~/web into ~/$DIRECTORY..."
+mv * ~/$DIRECTORY/. 2>/dev/null
+
+echo "(ok) Started extraction of the backup into ~/web. Consult ~/$DIRECTORY.txt for the log."
+tar -zxvf $BACKUP_LOCATION/$MOST_RECENT_BACKUP_FILE > ~/$DIRECTORY.txt
+
+if [[ $? -eq $zero ]]; then
+  echo "(ok) Finished the recovery. Moved files from ~/web into ~/$DIRECTORY and extracted the backup from $MOST_RECENT_BACKUP_FILE into ~/web."
+  exit 0
+else
+  echo "(bad) Finished the recovery procedure. Moved files from ~/web into ~/$DIRECTORY and tried to extract previous backup from $MOST_RECENT_BACKUP_FILE, but something went wrong ($?)"
+  echo "(bad) Please consult ~/$DIRECTORY.txt for the log."
+  exit 0
+fi
+
+