#!/bin/bash # Script works in the following scenario: # 1. Machine is a debian jessie or more recent, and has systemd # 2. Machine has rkhunter running # 3. Optional if update for gitlab-ce available: Machine has a Gitlab instance which is stopped # by default and needs to be started during upgrade and stopped afterwards # 4. Optional if needrestart available: needrestart will restart all services it can restart, # and we will warn the user if there are too critical services remaining and/or # the running kernel is outdated after upgrading # Remark for 3.: This is only necessary because Gitlab does a database backup before actually # upgrading (which is good), for which it needs to shutdown the Gitlab sub-instances. If # they are already stopped, this fails (which is not good aka. stupid). If this code was # better, we'd only need to start the systemd service. But what can you do ¯\_(ツ)_/¯ # (did I mention already that I'm happy with cgit?) APTBIN="$(which apt||echo "/usr/bin/apt")" RKHBIN="$(which rkhunter||echo "/usr/bin/rkhunter")" # using -e here because we are not to be blamed if the executable isn't - executable [ ! -e "$RKHBIN" ]&&unset RKHBIN #RKHCKP=( "-c" "--sk" "--cronjob" "--rwo" ) RKHCKP=( "-c" "--sk" "--cronjob" "-q" ) RKHUDP=( "--propupd" "--cronjob" "-q" ) STDBIN="$(which systemctl||echo "/bin/systemctl")" GTLBIN="$(which gitlab-ctl||echo "/usr/bin/gitlab-ctl")" GTLUNT="gitlab-runsvdir.service" NDRBIN="$(which needrestart||echo "/usr/sbin/needrestart")" # -e: see RKHBIN [ ! -e "$NDRBIN" ]&&unset NDRBIN NDRCLP=( "-ra" "-blqp" ) NDRCKP=( "-ra" "-bkqp" ) RETSAV="\\033[s" RETCLR="\\033[K" RETRES="\\033[u" RETOGE="$(tput cub 666)[ $(tput setaf 2)OK$(tput sgr0) ]$RETRES$RETCLR""." RETWRN="$(tput cub 666)[$(tput setaf 3)WARN$(tput sgr0)]$RETRES$RETCLR""." RETERR="$(tput cub 666)[$(tput setaf 1)FAIL$(tput sgr0)]$RETRES$RETCLR""." SHUTUP=0 LOGDMP=0;LOGTARG="ln-update" GTLUPD=0 RETVAL=0 retinit() { echo -ne "[....] $1$RETSAV""..." } retinf() { # Delivers the initial log output to be adjusted by return values echo -e "$(tput cub 666)[$(tput setaf 7)INFO$(tput sgr0)]$RETRES$RETCLR"" ($(tput setaf 7)$1$(tput sgr0))." } gitlab-start() { # Starts a stopped gitlab instance: unmasks gitlab-runsvdir, starts that service and executes # gitlab-ctl start. # RC: # 3: Unmasking or starting the systemd service failed # 4: gitlab-ctl start failed [ "$SHUTUP" -ne 1 ]&&retinit "Enabling and launching Gitlab" #"$SUDBIN" "$STDBIN" unmask "$GTLUNT" >/dev/null 2>&1 "$STDBIN" unmask "$GTLUNT" >/dev/null 2>&1 if [ "$?" -ne 0 ];then [ "$SHUTUP" -ne 1 ]&&echo -e "$RETERR" [ "$LOGDMP" -eq 1 ]&&logger -p"error" -t"$LOGTARG" "Unmasking Gitlab systemd unit failed." return 3 fi #"$SUDBIN" "$STDBIN" start "$GTLUNT" >/dev/null 2>&1 "$STDBIN" start "$GTLUNT" >/dev/null 2>&1 if [ "$?" -ne 0 ];then [ "$SHUTUP" -ne 1 ]&&echo -e "$RETERR" [ "$LOGDMP" -eq 1 ]&&logger -p"error" -t"$LOGTARG" "Launching Gitlab systemd unit failed." return 3 fi sleep 5 #"$SUDBIN" "$GTLBIN" start >/dev/null 2>&1 "$GTLBIN" start >/dev/null 2>&1 if [ "$?" -ne 0 ];then [ "$SHUTUP" -ne 1 ]&&echo -e "$RETERR" [ "$LOGDMP" -eq 1 ]&&logger -p"error" -t"$LOGTARG" "Launching Gitlab sub-units failed." return 4 fi [ "$SHUTUP" -ne 1 ]&&echo -e "$RETOGE" [ "$LOGDMP" -eq 1 ]&&logger -p"info" -t"$LOGTARG" "Gitlab enabled and started." } gitlab-stop() { # Stops a started gitlab instance: execution of gitlab-ctl stop succeeded by a stop of # gitlab-runvsdir.service and masking the latter. # RC: # 6: gitlab-ctl stop failed # 7: systemd unit cannot be stopped or cannot be masked. # 8: gitlab sub-units still running, i.e. gitlab-ctl stop did not complete everything; or # the systemd unit is still running [ "$SHUTUP" -ne 1 ]&&retinit "Stopping Gitlab" #"$SUDBIN" "$GTLBIN" stop >/dev/null 2>&1 "$GTLBIN" stop >/dev/null 2>&1 if [ "$?" -ne 0 ];then [ "$SHUTUP" -ne 1 ]&&echo -e "$RETERR" [ "$LOGDMP" -eq 1 ]&&logger -p"error" -t"$LOGTARG" "Stopping Gitlab sub-units failed." return 6 fi #"$SUDBIN" "$STDBIN" stop "$GTLUNT" >/dev/null 2>&1 "$STDBIN" stop "$GTLUNT" >/dev/null 2>&1 if [ "$?" -ne 0 ];then [ "$SHUTUP" -ne 1 ]&&echo -e "$RETERR" [ "$LOGDMP" -eq 1 ]&&logger -p"error" -t"$LOGTARG" "Stopping Gitlab systemd unit failed." return 7 fi #"$SUDBIN" "$STDBIN" mask "$GTLUNT" >/dev/null 2>&1 "$STDBIN" mask "$GTLUNT" >/dev/null 2>&1 case "$?" in 0) [ "$SHUTUP" -ne 1 ]&&echo -e "$RETOGE";; *) [ "$SHUTUP" -ne 1 ]&&echo -e "$RETERR" [ "$LOGDMP" -eq 1 ]&&logger -p"error" -t"$LOGTARG" "Masking Gitlab systemd unit failed." return 7 ;; esac [ "$SHUTUP" -ne 1 ]&&retinit "Checking if Gitlab sub-units are stopped" GLSVCCNT="$("$GTLBIN" status 2>&1|wc -l)" "$GTLBIN" status >/dev/null 2>&1 GLSVCSTS="$?" if [ "$GLSVCSTS" -lt "$GLSVCCNT" ];then [ "$SHUTUP" -ne 1 ]&&echo -e "$RETERR (Stopped Gitlab units: $GLSVCSTS/$GLSVCCNT)" [ "$LOGDMP" -eq 1 ]&&logger -p"error" -t"$LOGTARG" "Not all Gitlab sub-units stopped ($GLSVCSTS/$GLSVCCNT)." return 8 fi "$STDBIN" status "$GTLUNT" >/dev/null 2>&1 TRETVAL="$?" case "$TRETVAL" in 3) [ "$SHUTUP" -ne 1 ]&&echo -e "$RETOGE (Stopped Gitlab units: $GLSVCSTS/$GLSVCCNT, systemd service shut down.)";; *) [ "$SHUTUP" -ne 1 ]&&echo -e "$RETERR (systemd: $TRETVAL)" [ "$LOGDMP" -eq 1 ]&&logger -p"error" -t"$LOGTARG" "Gitlab systemd unit cannot be stopped ($TRETVAL)." return 8 ;; esac [ "$LOGDMP" -eq 1 ]&&logger -p"info" -t"$LOGTARG" "Gitlab stopped and disabled." unset TRETVAL } rkhunter-ck() { # Checks whether RKHunter reports any warnings or errors prior to updating. # This is mainly because after the update we will do a property update to have # the updated binaries and libraries updated in the RKHunter database. To -NOT- # trash any previous errors we will report an error right away and stop any # further execution so the admin can fix the situation and then re-run. # RC: # 1: RKHunter has any current error or warnings [ "$SHUTUP" -ne 1 ]&&retinit "Checking RKHunter state" [ "$LOGDMP" -eq 1 ]&&logger -p"info" -t"$LOGTARG" "Starting RKHunter status check." #"$SUDBIN" "$RKHBIN" "${RKHCKP[@]}" "$RKHBIN" "${RKHCKP[@]}" case "$?" in 0) [ "$SHUTUP" -ne 1 ]&&echo -e "$RETOGE" [ "$LOGDMP" -eq 1 ]&&logger -p"info" -t"$LOGTARG" "RKHunter status check finished." ;; *) [ "$SHUTUP" -ne 1 ]&&echo -e "$RETWRN (check rkhunter logfile)" [ "$LOGDMP" -eq 1 ]&&logger -p"error" -t"$LOGTARG" "RKHunter state inconsistent." return 1 ;; esac } rkhunter-upd() { # Updates the RKHunter property database after the dist-upgrade so we will not confuse # this with our mandatory system changes. # RC: # 1: if this step fails [ "$SHUTUP" -ne 1 ]&&retinit "Updating RKHunter database" "$RKHBIN" "${RKHUDP[@]}" case "$?" in 0) [ "$SHUTUP" -ne 1 ]&&echo -e "$RETOGE" [ "$LOGDMP" -eq 1 ]&&logger -p"info" -t"$LOGTARG" "RKHunter properties update successful." ;; *) [ "$SHUTUP" -ne 1 ]&&echo -e "$RETERR (check rkhunter logfile)" [ "$LOGDMP" -eq 1 ]&&logger -p"error" -t"$LOGTARG" "RKHunter properties update failed." return 1 ;; esac } needrestart-ck() { # Uses needrestart (Debian package) to check whether any outdated kernel or service with outdated # libraries are active. For this, we will use needrestart's autostart features, so be aware that we # -WILL- restart services during the process. # If there is any service that won't be restarted (needrestart is not unhealthily offensive, honest # cheers to the devs from here) or if the running kernel is outdated posterior to our upgrades: # RC: # 255: Outdated services still active or running kernel outdated LOCRETVAL=0 [ "$SHUTUP" -ne 1 ]&&retinit "Checking for services in need of restart" [ "$LOGDMP" -eq 1 ]&&logger -p"info" -t"$LOGTARG" "Checking for services in need of restart..." LLIBRES="$("$NDRBIN" "${NDRCLP[@]}")" NAGRETVAL="$?" case "$NAGRETVAL" in 0) [ "$SHUTUP" -ne 1 ]&&echo -e "$RETOGE" [ "$LOGDMP" -eq 1 ]&&logger -p"info" -t"$LOGTARG" "No remaining services running with outdated libs." ;; *) [ "$SHUTUP" -ne 1 ]&&echo -e "$RETWRN"&&echo "$LLIBRES"|sed 's/^[^-]\+-\ \([^|]\+\)|.*/\1/g' [ "$LOGDMP" -eq 1 ]&&logger -p"warning" -t"$LOGTARG" "Some services could not be restarted: $LLIBRES" LOCRETVAL=255 ;; esac unset LLIBRES [ "$SHUTUP" -ne 1 ]&&retinit "Checking for outdated kernel..." [ "$LOGDMP" -eq 1 ]&&logger -p"info" -t"$LOGTARG" "Checking for outdated kernel..." LKRNRES="$("$NDRBIN" "${NDRCKP[@]}")" NAGRETVAL="$?" case "$NAGRETVAL" in 0) [ "$SHUTUP" -ne 1 ]&&echo -e "$RETOGE" [ "$LOGDMP" -eq 1 ]&&logger -p"info" -t"$LOGTARG" "Kernel is up-to-date." ;; *) [ "$SHUTUP" -ne 1 ]&&echo -e "$RETWRN"&&echo "$LKRNRES"|sed 's/^[^-]\+-\ \([^|]\+\)|.*/\1/g' [ "$LOGDMP" -eq 1 ]&&logger -p"warning" -t"$LOGTARG" "Kernel is outdated: $LKRNRES" LOCRETVAL=255 ;; esac unset LKRNRES return "$LOCRETVAL" } while getopts :lq SHOPT;do case "$SHOPT" in q)SHUTUP=1;; l)LOGDMP=1;; esac done [ "$LOGDMP" -eq 1 ]&&logger -p"info" -t"$LOGTARG" "Starting new run..." [ "$SHUTUP" -ne 1 ]&&retinit "Checking root privileges" #ROOTCK="$(sudo id -u)" ROOTCK="$(id -u)" case "$ROOTCK" in 0) [ "$SHUTUP" -ne 1 ]&&echo -e "$RETOGE";; *) [ "$SHUTUP" -ne 1 ]&&echo -ne "$RETERR"&&echo -e " (We need root rights)" [ "$LOGDMP" -eq 1 ]&&logger -p"warning" -t"$LOGTARG" "Cancelled due to insufficient rights." exit 1 ;; esac [ "$SHUTUP" -ne 1 ]&&retinit "Updating Apt database" #"$SUDBIN" "$APTBIN" -qqq update "$APTBIN" -qqq update 2>/dev/null if [ "$?" -ne 0 ];then [ "$SHUTUP" -ne 1 ]&&echo -e "$RETERR" [ "$LOGDMP" -eq 1 ]&&logger -p"error" -t"$LOGTARG" "Cannot update apt database." exit 2 fi # Make bash not break up the output into array elements on whitespaces but only on encountering line breaks IFS=$'\n' UPDARR=( $("$APTBIN" list --upgradable 2>/dev/null|grep -v "^Listing") ) # ...done. unset IFS UPDCNT="${#UPDARR[@]}" unset UPDARR case "$UPDCNT" in 0) [ "$SHUTUP" -ne 1 ]&&retinf "No updates available :)" exit 0 ;; *) [ "$SHUTUP" -ne 1 ]&&echo -e "$RETOGE ($UPDCNT updates)." [ "$LOGDMP" -eq 1 ]&&logger -p"info" -t"$LOGTARG" "Apt database updated ($UPDCNT updates)." ;; esac "$APTBIN" list --upgradable 2>/dev/null|grep gitlab-ce >/dev/null if [ "$?" -eq 0 ];then [ "$SHUTUP" -ne 1 ]&&echo "Encountered Gitlab update." GTLUPD=1 fi if [ ! -z "$RKHBIN" ];then rkhunter-ck||exit "$?";fi if [ "$GTLUPD" -eq 1 ];then gitlab-start||exit "$?";fi [ "$SHUTUP" -ne 1 ]&&retinit "Upgrading packages" #"$SUDBIN" "$APTBIN" -qqq -y dist-upgrade >/dev/null 2>&1 "$APTBIN" -qqq -y dist-upgrade >/dev/null 2>&1 case "$?" in 0) [ "$SHUTUP" -ne 1 ]&&echo -e "$RETOGE" [ "$LOGDMP" -eq 1 ]&&logger -p"info" -t"$LOGTARG" "Apt dist-upgrade successful." ;; *) [ "$SHUTUP" -ne 1 ]&&echo -e "$RETERR" [ "$LOGDMP" -eq 1 ]&&logger -p"error" -t"$LOGTARG" "Apt dist-upgrade failed." exit 5 ;; esac if [ "$GTLUPD" -eq 1 ];then gitlab-stop||exit "$?";fi if [ ! -z "$RKHBIN" ];then rkhunter-upd||exit "$?";fi if [ ! -z "$NDRBIN" ];then needrestart-ck;NDRRETVAL="$?" # because needrestart-ck only returns warning (RC255), we will only change the script RETVAL # if it has been 0 prior to this: if [ "$RETVAL" -eq 0 ];then RETVAL="$NDRRETVAL";fi unset NDRRETVAL fi RETVAL="$?" exit "$RETVAL"