sync-vm with torrent support

lmn-desktop.yml

@@ -33,6 +33,7 @@
       - krb5-user
       - unattended-upgrades
       - debconf-utils
+      - ctorrent
     extra_pkgs_bpo: [] # [ linux-image-amd64 ]
     ansible_python_interpreter: "/usr/bin/python3"
 

roles/lmn_vm/files/linbo-torrent

@@ -0,0 +1,33 @@
+# default values for linbo-torrenthelper service provided by ctorrent
+# thomas@linuxmuster.net
+# 20220317
+#
+# note: you have to invoke 'linbo-torrent restart' after you have changed any values
+#
+
+# Exit while seed <SEEDHOURS> hours later (default 72 hours)
+SEEDHOURS="100000"
+
+# Max peers count (default 100)
+MAXPEERS="100"
+
+# Min peers count (default 1)
+MINPEERS="1"
+
+# Download slice/block size, unit KB (default 16, max 128)
+SLICESIZE="128"
+
+# Max bandwidth down (unit KB/s, default unlimited)
+MAXDOWN=""
+
+# Max bandwidth up (unit KB/s, default unlimited)
+MAXUP=""
+
+# Supplemental ctorrent options, separated by space (-v: Verbose output for debugging)
+#OPTIONS="-v"
+
+# Timeout in seconds until rsync fallback (client only)
+TIMEOUT="300"
+
+# user to run ctorrent (server only)
+CTUSER="nobody"

roles/lmn_vm/files/linbo-torrenthelper.sh

@@ -0,0 +1,31 @@
+#!/bin/bash
+#
+# thomas@linuxmuster.net
+# GPL v3
+# 20220317
+#
+# linbo ctorrent helper script, started in a screen session by init script
+#
+
+torrent="$1"
+[ -s "$torrent" ] || exit 1
+
+# get ctorrent options from file
+[ -e /etc/default/linbo-torrent ] && source /etc/default/linbo-torrent
+[ -e /home/raphael/git/fvsclient/etc/default/linbo-torrent ] && source /home/raphael/git/fvsclient/etc/default/linbo-torrent
+
+[ -n "$SEEDHOURS" ] &&  OPTIONS="$OPTIONS -e $SEEDHOURS"
+[ -n "$MAXPEERS" ] &&  OPTIONS="$OPTIONS -M $MAXPEERS"
+[ -n "$MINPEERS" ] &&  OPTIONS="$OPTIONS -m $MINPEERS"
+[ -n "$SLICESIZE" ] &&  OPTIONS="$OPTIONS -z $SLICESIZE"
+[ -n "$MAXDOWN" ] &&  OPTIONS="$OPTIONS -D $MAXDOWN"
+[ -n "$MAXUP" ] &&  OPTIONS="$OPTIONS -U $MAXUP"
+OPTIONS="$OPTIONS $torrent"
+
+[ -n "$CTUSER" ] && SUDO="/usr/bin/sudo -u $CTUSER"
+
+while true; do
+ $SUDO /usr/bin/ctorrent $OPTIONS || exit 1
+ # hash check only on initial start, add -f parameter
+ echo "$OPTIONS" | grep -q ^"-f " || OPTIONS="-f $OPTIONS"
+done

roles/lmn_vm/files/run-vm.sh

@@ -10,7 +10,7 @@ Create a new clone, start the vm (if not yet running) and run virt-viewer.
 Squid-Proxy will be started too.
 User Home will be mounted on /media/USERNAME/home
     -n   new clone will be created, even if exists
-    -s   qemu:///session instead of default qemu:///system
+    -s   qemu:///system instead of default qemu:///session
 EOF
 }
 
@@ -21,7 +21,7 @@ exit_script() {
     kill -- -$$ # Sends SIGTERM to child/sub processes
 }
 
-QEMU='qemu:///system'
+QEMU='qemu:///session'
 
 NEWCLONE=0
 
@@ -31,7 +31,7 @@ while getopts ':ns' OPTION; do
             NEWCLONE=1
             ;;
         s)
-            QEMU='qemu:///session'
+            QEMU='qemu:///system'
             ;;
         ?)
         show_help
@@ -49,14 +49,15 @@ if [[  $# -ne 1 ]] ; then
 fi
 
 VM_NAME=$1
+VM_DIR="/tmp/${UID}/vmimages"
 
-if [[ ! -f "/var/lib/libvirt/images/${VM_NAME}.qcow2" ]]; then
+if [[ ! -f "/var/lib/libvirt/images/${VM_NAME}.qcow2" && ! -f "${VM_DIR}/${VM_NAME}.qcow2" ]]; then
     echo "no base VM disk '${VM_NAME}.qcow2' found" >&2
     exit 1
 fi
 
 # check, if we have to start squid
-if [[ ! -f "/tmp/squid.pid" ]]; then
+if ! killall -s 0 squid; then
     echo "starting squid."
     /usr/sbin/squid -f /etc/squid/squid-usermode.conf
 fi
@@ -68,7 +69,6 @@ if ! findmnt "/media/${USER}/home"; then
 fi
 
 export XDG_CONFIG_HOME="/tmp/${UID}/.config"
-VM_DIR="/tmp/${UID}/vmimages"
 
 if  ! virsh --connect="${QEMU}" list | grep "${VM_NAME}-clone"; then
     echo "VM not yet running. Try to clone and start."

roles/lmn_vm/files/sync-vm.sh

@@ -4,23 +4,42 @@ set -eu
 
 show_help() {
     cat << EOF >&2
-Usage: $(basename "$0") [-u vmname] [-d vmname] [-a]"
+Usage: $(basename "$0") [-u vmname] [-d vmname] [-a] [-t]"
 When using option -u (upload), the disk from VM vmname will be synced on server.
 Otherwise the images from images.list and xml-directory will be synced from server.
+Using flag -t all torrents and xml-VM-Definitions will be synced
 EOF
 }
 
+VM_DIR="/tmp/${SUDO_UID}/vmimages"
+
 upload_image() {
     # check if VM-Diskimage exists
-    if [[ ! -f "/var/lib/libvirt/images/${VM_NAME}.qcow2" ]]; then
+    if [[ ! (-f "/var/lib/libvirt/images/${VM_NAME}.qcow2" || -f "${VM_DIR}/${VM_NAME}.qcow2") ]]; then
         echo "File not found ${VM_NAME}.qcow2" >&2
         exit 1
     fi
+    # link private VM-Diskimage to system-Dir
+    if [[ -f "${VM_DIR}/${VM_NAME}.qcow2" \
+          &&   ( -f "/var/lib/libvirt/images/${VM_NAME}.qcow2" && ("${VM_DIR}/${VM_NAME}.qcow2" -nt "/var/lib/libvirt/images/${VM_NAME}.qcow2") \
+          || ! -f "/var/lib/libvirt/images/${VM_NAME}.qcow2") ]]; then
+      echo "copy private VM-Diskimage to system-dir"
+      ln -f "${VM_DIR}/${VM_NAME}.qcow2" "/var/lib/libvirt/images/${VM_NAME}.qcow2"
+    fi
     # check if VM-Machine-Definition XML exists
-    if [[ ! -f "/var/lib/libvirt/images/xml/${VM_NAME}.xml" ]]; then
+    if [[ ! (-f "/var/lib/libvirt/images/xml/${VM_NAME}.xml" || -f "${VM_DIR}/xml/${VM_NAME}") ]]; then
         echo "File not found ${VM_NAME}.xml" >&2
         exit 1
     fi
+    # copy private VM-Maschine-Definition XML to system-Dir
+    if [[ -f "${VM_DIR}/xml/${VM_NAME}.xml" \
+          &&   ( -f "/var/lib/libvirt/images/xml/${VM_NAME}.xml" && $(cmp -s "${VM_DIR}/xml/${VM_NAME}.xml" "/var/lib/libvirt/images/xml/${VM_NAME}.xml") \
+          || ! -f "/var/lib/libvirt/images/xml/${VM_NAME}.xml") ]]; then
+      echo "copy private VM-Maschine-Definition XML to system-dir"
+      cp "${VM_DIR}/${VM_NAME}.xml" "/var/lib/libvirt/images/${VM_NAME}.xml"
+    fi
+    [[ -f "/var/lib/libvirt/images/${VMNAME}.qcow2.torrent" ]] && rsync -av --password-file=/etc/rsync.secret \
+	  "/var/lib/libvirt/images/${VM_NAME}.qcow2" rsync://vmuser@server:/vmimages-upload/
     rsync -av --password-file=/etc/rsync.secret "/var/lib/libvirt/images/${VM_NAME}.qcow2" \
 	  rsync://vmuser@server:/vmimages-upload/
     rsync -av --password-file=/etc/rsync.secret "/var/lib/libvirt/images/xml/${VM_NAME}.xml" \
@@ -43,7 +62,14 @@ sync_all_images() {
 	  /var/lib/libvirt/images/
 }
 
-while getopts ':u:d:a' OPTION; do
+sync_all_torrents() {
+    rsync -av --password-file=/etc/rsync.secret rsync://vmuser@server:/vmimages-download/*.torrent \
+	  /var/lib/libvirt/images/
+    rsync -av --password-file=/etc/rsync.secret rsync://vmuser@server:/vmimages-download/xml \
+	  /var/lib/libvirt/images/
+}
+
+while getopts ':u:d:a:t' OPTION; do
     case "$OPTION" in
         u)
             VM_NAME=$OPTARG
@@ -56,6 +82,9 @@ while getopts ':u:d:a' OPTION; do
         a)
             sync_all_images
             ;;
+        t)
+            sync_all_torrents
+            ;;
         ?)
         show_help
         exit 1

roles/lmn_vm/files/vmimage-torrent

@@ -0,0 +1,213 @@
+#!/bin/bash
+#
+# starts tmux sessions for each valid torrent in LINBODIR
+# thomas@linuxmuster.net
+# 20221103
+#
+
+# read environment
+#. /usr/share/linuxmuster/defaults.sh || exit 1
+#THELPER=$LINBOSHAREDIR/linbo-torrenthelper.sh
+THELPER=linbo-torrenthelper.sh
+#. $LINBOSHAREDIR/helperfunctions.sh || exit 1
+LINBOIMGEXT="qcow2 qdiff"
+LINBOIMGDIR="/var/lib/libvirt/images"
+serverip="10.190.1.1"
+
+# start of functions
+
+# help message
+usage(){
+  echo
+  echo "Info: vmimage-torrent manages the torrent tmux sessions of linbo images."
+  echo
+  echo "Usage:"
+  echo " vmimage-torrent <start|stop|restart|reload|status|create|check> [image_name]"
+  echo " vmimage-torrent attach <image_name|session_name>"
+  echo
+  echo "Note:"
+  echo " * Only qcow2 & qdiff image files located below $LINBOIMGDIR are processed."
+  echo " * The commands \"start\", \"stop\" and \"restart\" may have optionally an image"
+  echo "   filename as parameter. In this case the commands are only applied to the tmux"
+  echo "   session of the certain file. Without an explicit image filename the commands"
+  echo "   were applied to all image file sessions currently running."
+  echo " * An image filename parameter is mandatory with the commands \"check\", \"create\""
+  echo "   and \"attach\"."
+  echo " * \"check\" checks if the image file matches to the correspondig torrent."
+  echo " * \"create\" creates/recreates the torrent of a certain image file."
+  echo " * \"status\" shows a list of currently running torrent tmux sessions."
+  echo " * \"attach\" attaches a torrent tmux session of a certain image. An image or"
+  echo "   session name must be given as parameter."
+  echo "   Press [CTRL+B]+[D] to detach the session again."
+  echo " * \"reload\" is the identical to \"restart\" and is there for backwards compatibility."
+  echo
+  exit 1
+}
+
+# check torrent
+check(){
+  local image="$(basename "$IMGLIST")"
+  local torrent="$image.torrent"
+  local tdir="$(dirname "$IMGLIST")"
+  cd "$tdir"
+  echo "Checking $torrent ..."
+  if ctorrent -c "$torrent"; then
+    echo "Ok!"
+  else
+    echo "Failed!"
+    exit 1
+  fi
+}
+
+# creates torrent files
+create(){
+  local image="$(basename "$IMGLIST")"
+  local tdir="$(dirname "$IMGLIST")"
+  local torrent="${image}.torrent"
+  local session="${torrent//./_}"
+  # stop torrent service
+  vmimage-torrent status | grep -q ^"$session" && vmimage-torrent stop "$IMGLIST"
+  # skip already running torrents
+  echo "Creating $torrent ..."
+  cd "$tdir"
+  rm -f "$torrent"
+  if ctorrent -t -u "http://$serverip:6969/announce" -s "$torrent" "$image" ; then
+    [ "$START" = "no" ] || vmimage-torrent start "$IMGLIST"
+  else
+    echo "Failed!"
+    exit 1
+  fi
+}
+
+# starts torrent tmux sessions
+start(){
+  local item
+  local torrent
+  local image
+  local tdir
+  local session
+  for item in $IMGLIST; do
+    image="$(basename "$item")"
+    torrent="${image}.torrent"
+    tdir="$(dirname "$item")"
+    session="${torrent//./_}"
+    cd "$tdir"
+    if [ ! -s "$image" ]; then
+      echo "Image $image does not exist! Skipping this torrent."
+      continue
+    fi
+    # skip already running torrents
+    if vmimage-torrent status | grep -qw ^"$session"; then
+      echo "tmux session $session is already running."
+      continue
+    fi
+    # create torrent file if there is none
+    if [ ! -e "$torrent" ]; then
+      START="no" vmimage-torrent create "$item" || continue
+    fi
+    echo -n "Starting tmux session $session ... "
+    tmux new -ds "$session" "$THELPER $torrent ; exec $SHELL"
+    sleep 1
+    if vmimage-torrent status | grep -qw ^"$session"; then
+      echo "Ok!"
+    else
+      echo "Failed!"
+    fi
+  done
+}
+
+stop(){
+  if [ -n "$SESSION" ]; then
+    vmimage-torrent status | grep -qw ^"$SESSION" || return
+    tmux kill-session -t "$SESSION"
+  else
+    local item
+    vmimage-torrent status | awk -F\: '{print $1}' | while read item; do
+      tmux kill-session -t "$item"
+    done
+  fi
+}
+
+attach(){
+  if ! tmux list-sessions | grep -qw "$SESSION"; then
+    echo "There is no session $SESSION."
+    exit 1
+  fi
+  echo "Hint: Detach tmux session with [CTRL+B]+[D]."
+  sleep 3
+  tmux attach -t "$SESSION"
+}
+
+status(){
+  tmux list-sessions | grep _torrent
+}
+
+find_images(){
+  local search="$(basename "$1")"
+  if [ -n "$search" ]; then
+    find "$LINBOIMGDIR" -maxdepth 2 -name "$search"
+    return
+  fi
+  local IMGLIST
+  for search in $LINBOIMGEXT; do
+    IMGLIST="$IMGLIST $(find "$LINBOIMGDIR" -maxdepth 2 -name \*.$search)"
+  done
+  # trim leading and trailing spaces
+  echo $IMGLIST | awk '{$1=$1};1'
+}
+
+# end of functions
+
+# check parameters
+if [ -n "$2" ] ; then
+  # trap torrent parameter
+  image="${2/.torrent/}"
+  case "$image" in
+    *.qcow2|*.qdiff)
+      if [ -e "$image" ]; then
+        IMGLIST="$image"
+      else
+        IMGLIST="$(find_images "$image")"
+      fi
+      if [ ! -e "$IMGLIST" ]; then
+        echo "Image file $(basename $image) not found."
+        usage
+      fi
+      filename="$(basename "$IMGLIST")"
+      SESSION="${filename//./_}_torrent"
+      ;;
+    *_torrent)
+      if [ "$1" = "attach" ]; then
+        SESSION="$image"
+      else
+        usage
+      fi
+      ;;
+    *) usage ;;
+  esac
+else
+  case "$1" in
+    stop|status) ;;
+    attach|check|create) usage ;;
+    *)
+      IMGLIST="$(find_images)"
+      if [ -z "$IMGLIST" ]; then
+        echo "No linbo images found."
+        exit 0
+      fi
+      ;;
+  esac
+fi
+
+case "$1" in
+  start) start ;;
+  stop) stop ;;
+  restart|reload) stop ; start ;;
+  status) status ;;
+  create) create ;;
+  check) check ;;
+  attach) attach ;;
+  *) usage ;;
+esac
+
+exit 0

roles/lmn_vm/tasks/main.yml

@@ -63,7 +63,7 @@
     - lmn-link-images
     - lmn-startvirtiofsd
 
-- name: deploy mount home script
+- name: deploy vmimages scripts
   copy:
     src: "{{ item }}"
     dest: /usr/local/bin/
@@ -79,6 +79,16 @@
     - sync-vm.sh
     - link-images.sh
     - start-virtiofsd.sh
+    - linbo-torrenthelper.sh
+    - vmimage-torrent
+
+- name: deploy linbo-torrent defaults
+  copy:
+    src: linbo-torrent
+    dest: /etc/default/
+    owner: root
+    group: root
+    mode: '0755'
 
 - name: Deploy bridge.conf needed for qemu session mode
   lineinfile:
@@ -137,5 +147,5 @@
   register: result
   changed_when: result.stdout | length > 0
   when: >
-    not run_in_installer | default(false) | bool and (ansible_mounts |
+    false and not run_in_installer | default(false) | bool and (ansible_mounts |
     selectattr("mount", "equalto", "/") | list)[0].size_available > 80000000000