From 399c3d0d66df03e5a88ab79a2b1be2fce6f8cd19 Mon Sep 17 00:00:00 2001 From: Raphael Dannecker Date: Wed, 3 May 2023 17:24:27 +0200 Subject: [PATCH] sync-vm with torrent support --- lmn-desktop.yml | 1 + roles/lmn_vm/files/linbo-torrent | 33 ++++ roles/lmn_vm/files/linbo-torrenthelper.sh | 31 ++++ roles/lmn_vm/files/run-vm.sh | 12 +- roles/lmn_vm/files/sync-vm.sh | 37 +++- roles/lmn_vm/files/vmimage-torrent | 213 ++++++++++++++++++++++ roles/lmn_vm/tasks/main.yml | 14 +- 7 files changed, 329 insertions(+), 12 deletions(-) create mode 100644 roles/lmn_vm/files/linbo-torrent create mode 100755 roles/lmn_vm/files/linbo-torrenthelper.sh create mode 100755 roles/lmn_vm/files/vmimage-torrent diff --git a/lmn-desktop.yml b/lmn-desktop.yml index da46224..d312aab 100644 --- a/lmn-desktop.yml +++ b/lmn-desktop.yml @@ -33,6 +33,7 @@ - krb5-user - unattended-upgrades - debconf-utils + - ctorrent extra_pkgs_bpo: [] # [ linux-image-amd64 ] ansible_python_interpreter: "/usr/bin/python3" diff --git a/roles/lmn_vm/files/linbo-torrent b/roles/lmn_vm/files/linbo-torrent new file mode 100644 index 0000000..e22ed0b --- /dev/null +++ b/roles/lmn_vm/files/linbo-torrent @@ -0,0 +1,33 @@ +# default values for linbo-torrenthelper service provided by ctorrent +# thomas@linuxmuster.net +# 20220317 +# +# note: you have to invoke 'linbo-torrent restart' after you have changed any values +# + +# Exit while seed hours later (default 72 hours) +SEEDHOURS="100000" + +# Max peers count (default 100) +MAXPEERS="100" + +# Min peers count (default 1) +MINPEERS="1" + +# Download slice/block size, unit KB (default 16, max 128) +SLICESIZE="128" + +# Max bandwidth down (unit KB/s, default unlimited) +MAXDOWN="" + +# Max bandwidth up (unit KB/s, default unlimited) +MAXUP="" + +# Supplemental ctorrent options, separated by space (-v: Verbose output for debugging) +#OPTIONS="-v" + +# Timeout in seconds until rsync fallback (client only) +TIMEOUT="300" + +# user to run ctorrent (server only) +CTUSER="nobody" diff --git a/roles/lmn_vm/files/linbo-torrenthelper.sh b/roles/lmn_vm/files/linbo-torrenthelper.sh new file mode 100755 index 0000000..2775dcf --- /dev/null +++ b/roles/lmn_vm/files/linbo-torrenthelper.sh @@ -0,0 +1,31 @@ +#!/bin/bash +# +# thomas@linuxmuster.net +# GPL v3 +# 20220317 +# +# linbo ctorrent helper script, started in a screen session by init script +# + +torrent="$1" +[ -s "$torrent" ] || exit 1 + +# get ctorrent options from file +[ -e /etc/default/linbo-torrent ] && source /etc/default/linbo-torrent +[ -e /home/raphael/git/fvsclient/etc/default/linbo-torrent ] && source /home/raphael/git/fvsclient/etc/default/linbo-torrent + +[ -n "$SEEDHOURS" ] && OPTIONS="$OPTIONS -e $SEEDHOURS" +[ -n "$MAXPEERS" ] && OPTIONS="$OPTIONS -M $MAXPEERS" +[ -n "$MINPEERS" ] && OPTIONS="$OPTIONS -m $MINPEERS" +[ -n "$SLICESIZE" ] && OPTIONS="$OPTIONS -z $SLICESIZE" +[ -n "$MAXDOWN" ] && OPTIONS="$OPTIONS -D $MAXDOWN" +[ -n "$MAXUP" ] && OPTIONS="$OPTIONS -U $MAXUP" +OPTIONS="$OPTIONS $torrent" + +[ -n "$CTUSER" ] && SUDO="/usr/bin/sudo -u $CTUSER" + +while true; do + $SUDO /usr/bin/ctorrent $OPTIONS || exit 1 + # hash check only on initial start, add -f parameter + echo "$OPTIONS" | grep -q ^"-f " || OPTIONS="-f $OPTIONS" +done diff --git a/roles/lmn_vm/files/run-vm.sh b/roles/lmn_vm/files/run-vm.sh index 49e7d44..c726a4f 100755 --- a/roles/lmn_vm/files/run-vm.sh +++ b/roles/lmn_vm/files/run-vm.sh @@ -10,7 +10,7 @@ Create a new clone, start the vm (if not yet running) and run virt-viewer. Squid-Proxy will be started too. User Home will be mounted on /media/USERNAME/home -n new clone will be created, even if exists - -s qemu:///session instead of default qemu:///system + -s qemu:///system instead of default qemu:///session EOF } @@ -21,7 +21,7 @@ exit_script() { kill -- -$$ # Sends SIGTERM to child/sub processes } -QEMU='qemu:///system' +QEMU='qemu:///session' NEWCLONE=0 @@ -31,7 +31,7 @@ while getopts ':ns' OPTION; do NEWCLONE=1 ;; s) - QEMU='qemu:///session' + QEMU='qemu:///system' ;; ?) show_help @@ -49,14 +49,15 @@ if [[ $# -ne 1 ]] ; then fi VM_NAME=$1 +VM_DIR="/tmp/${UID}/vmimages" -if [[ ! -f "/var/lib/libvirt/images/${VM_NAME}.qcow2" ]]; then +if [[ ! -f "/var/lib/libvirt/images/${VM_NAME}.qcow2" && ! -f "${VM_DIR}/${VM_NAME}.qcow2" ]]; then echo "no base VM disk '${VM_NAME}.qcow2' found" >&2 exit 1 fi # check, if we have to start squid -if [[ ! -f "/tmp/squid.pid" ]]; then +if ! killall -s 0 squid; then echo "starting squid." /usr/sbin/squid -f /etc/squid/squid-usermode.conf fi @@ -68,7 +69,6 @@ if ! findmnt "/media/${USER}/home"; then fi export XDG_CONFIG_HOME="/tmp/${UID}/.config" -VM_DIR="/tmp/${UID}/vmimages" if ! virsh --connect="${QEMU}" list | grep "${VM_NAME}-clone"; then echo "VM not yet running. Try to clone and start." diff --git a/roles/lmn_vm/files/sync-vm.sh b/roles/lmn_vm/files/sync-vm.sh index 51be74f..471ee69 100755 --- a/roles/lmn_vm/files/sync-vm.sh +++ b/roles/lmn_vm/files/sync-vm.sh @@ -4,23 +4,42 @@ set -eu show_help() { cat << EOF >&2 -Usage: $(basename "$0") [-u vmname] [-d vmname] [-a]" +Usage: $(basename "$0") [-u vmname] [-d vmname] [-a] [-t]" When using option -u (upload), the disk from VM vmname will be synced on server. Otherwise the images from images.list and xml-directory will be synced from server. +Using flag -t all torrents and xml-VM-Definitions will be synced EOF } +VM_DIR="/tmp/${SUDO_UID}/vmimages" + upload_image() { # check if VM-Diskimage exists - if [[ ! -f "/var/lib/libvirt/images/${VM_NAME}.qcow2" ]]; then + if [[ ! (-f "/var/lib/libvirt/images/${VM_NAME}.qcow2" || -f "${VM_DIR}/${VM_NAME}.qcow2") ]]; then echo "File not found ${VM_NAME}.qcow2" >&2 exit 1 fi + # link private VM-Diskimage to system-Dir + if [[ -f "${VM_DIR}/${VM_NAME}.qcow2" \ + && ( -f "/var/lib/libvirt/images/${VM_NAME}.qcow2" && ("${VM_DIR}/${VM_NAME}.qcow2" -nt "/var/lib/libvirt/images/${VM_NAME}.qcow2") \ + || ! -f "/var/lib/libvirt/images/${VM_NAME}.qcow2") ]]; then + echo "copy private VM-Diskimage to system-dir" + ln -f "${VM_DIR}/${VM_NAME}.qcow2" "/var/lib/libvirt/images/${VM_NAME}.qcow2" + fi # check if VM-Machine-Definition XML exists - if [[ ! -f "/var/lib/libvirt/images/xml/${VM_NAME}.xml" ]]; then + if [[ ! (-f "/var/lib/libvirt/images/xml/${VM_NAME}.xml" || -f "${VM_DIR}/xml/${VM_NAME}") ]]; then echo "File not found ${VM_NAME}.xml" >&2 exit 1 fi + # copy private VM-Maschine-Definition XML to system-Dir + if [[ -f "${VM_DIR}/xml/${VM_NAME}.xml" \ + && ( -f "/var/lib/libvirt/images/xml/${VM_NAME}.xml" && $(cmp -s "${VM_DIR}/xml/${VM_NAME}.xml" "/var/lib/libvirt/images/xml/${VM_NAME}.xml") \ + || ! -f "/var/lib/libvirt/images/xml/${VM_NAME}.xml") ]]; then + echo "copy private VM-Maschine-Definition XML to system-dir" + cp "${VM_DIR}/${VM_NAME}.xml" "/var/lib/libvirt/images/${VM_NAME}.xml" + fi + [[ -f "/var/lib/libvirt/images/${VMNAME}.qcow2.torrent" ]] && rsync -av --password-file=/etc/rsync.secret \ + "/var/lib/libvirt/images/${VM_NAME}.qcow2" rsync://vmuser@server:/vmimages-upload/ rsync -av --password-file=/etc/rsync.secret "/var/lib/libvirt/images/${VM_NAME}.qcow2" \ rsync://vmuser@server:/vmimages-upload/ rsync -av --password-file=/etc/rsync.secret "/var/lib/libvirt/images/xml/${VM_NAME}.xml" \ @@ -43,7 +62,14 @@ sync_all_images() { /var/lib/libvirt/images/ } -while getopts ':u:d:a' OPTION; do +sync_all_torrents() { + rsync -av --password-file=/etc/rsync.secret rsync://vmuser@server:/vmimages-download/*.torrent \ + /var/lib/libvirt/images/ + rsync -av --password-file=/etc/rsync.secret rsync://vmuser@server:/vmimages-download/xml \ + /var/lib/libvirt/images/ +} + +while getopts ':u:d:a:t' OPTION; do case "$OPTION" in u) VM_NAME=$OPTARG @@ -56,6 +82,9 @@ while getopts ':u:d:a' OPTION; do a) sync_all_images ;; + t) + sync_all_torrents + ;; ?) show_help exit 1 diff --git a/roles/lmn_vm/files/vmimage-torrent b/roles/lmn_vm/files/vmimage-torrent new file mode 100755 index 0000000..c9a4a20 --- /dev/null +++ b/roles/lmn_vm/files/vmimage-torrent @@ -0,0 +1,213 @@ +#!/bin/bash +# +# starts tmux sessions for each valid torrent in LINBODIR +# thomas@linuxmuster.net +# 20221103 +# + +# read environment +#. /usr/share/linuxmuster/defaults.sh || exit 1 +#THELPER=$LINBOSHAREDIR/linbo-torrenthelper.sh +THELPER=linbo-torrenthelper.sh +#. $LINBOSHAREDIR/helperfunctions.sh || exit 1 +LINBOIMGEXT="qcow2 qdiff" +LINBOIMGDIR="/var/lib/libvirt/images" +serverip="10.190.1.1" + +# start of functions + +# help message +usage(){ + echo + echo "Info: vmimage-torrent manages the torrent tmux sessions of linbo images." + echo + echo "Usage:" + echo " vmimage-torrent [image_name]" + echo " vmimage-torrent attach " + echo + echo "Note:" + echo " * Only qcow2 & qdiff image files located below $LINBOIMGDIR are processed." + echo " * The commands \"start\", \"stop\" and \"restart\" may have optionally an image" + echo " filename as parameter. In this case the commands are only applied to the tmux" + echo " session of the certain file. Without an explicit image filename the commands" + echo " were applied to all image file sessions currently running." + echo " * An image filename parameter is mandatory with the commands \"check\", \"create\"" + echo " and \"attach\"." + echo " * \"check\" checks if the image file matches to the correspondig torrent." + echo " * \"create\" creates/recreates the torrent of a certain image file." + echo " * \"status\" shows a list of currently running torrent tmux sessions." + echo " * \"attach\" attaches a torrent tmux session of a certain image. An image or" + echo " session name must be given as parameter." + echo " Press [CTRL+B]+[D] to detach the session again." + echo " * \"reload\" is the identical to \"restart\" and is there for backwards compatibility." + echo + exit 1 +} + +# check torrent +check(){ + local image="$(basename "$IMGLIST")" + local torrent="$image.torrent" + local tdir="$(dirname "$IMGLIST")" + cd "$tdir" + echo "Checking $torrent ..." + if ctorrent -c "$torrent"; then + echo "Ok!" + else + echo "Failed!" + exit 1 + fi +} + +# creates torrent files +create(){ + local image="$(basename "$IMGLIST")" + local tdir="$(dirname "$IMGLIST")" + local torrent="${image}.torrent" + local session="${torrent//./_}" + # stop torrent service + vmimage-torrent status | grep -q ^"$session" && vmimage-torrent stop "$IMGLIST" + # skip already running torrents + echo "Creating $torrent ..." + cd "$tdir" + rm -f "$torrent" + if ctorrent -t -u "http://$serverip:6969/announce" -s "$torrent" "$image" ; then + [ "$START" = "no" ] || vmimage-torrent start "$IMGLIST" + else + echo "Failed!" + exit 1 + fi +} + +# starts torrent tmux sessions +start(){ + local item + local torrent + local image + local tdir + local session + for item in $IMGLIST; do + image="$(basename "$item")" + torrent="${image}.torrent" + tdir="$(dirname "$item")" + session="${torrent//./_}" + cd "$tdir" + if [ ! -s "$image" ]; then + echo "Image $image does not exist! Skipping this torrent." + continue + fi + # skip already running torrents + if vmimage-torrent status | grep -qw ^"$session"; then + echo "tmux session $session is already running." + continue + fi + # create torrent file if there is none + if [ ! -e "$torrent" ]; then + START="no" vmimage-torrent create "$item" || continue + fi + echo -n "Starting tmux session $session ... " + tmux new -ds "$session" "$THELPER $torrent ; exec $SHELL" + sleep 1 + if vmimage-torrent status | grep -qw ^"$session"; then + echo "Ok!" + else + echo "Failed!" + fi + done +} + +stop(){ + if [ -n "$SESSION" ]; then + vmimage-torrent status | grep -qw ^"$SESSION" || return + tmux kill-session -t "$SESSION" + else + local item + vmimage-torrent status | awk -F\: '{print $1}' | while read item; do + tmux kill-session -t "$item" + done + fi +} + +attach(){ + if ! tmux list-sessions | grep -qw "$SESSION"; then + echo "There is no session $SESSION." + exit 1 + fi + echo "Hint: Detach tmux session with [CTRL+B]+[D]." + sleep 3 + tmux attach -t "$SESSION" +} + +status(){ + tmux list-sessions | grep _torrent +} + +find_images(){ + local search="$(basename "$1")" + if [ -n "$search" ]; then + find "$LINBOIMGDIR" -maxdepth 2 -name "$search" + return + fi + local IMGLIST + for search in $LINBOIMGEXT; do + IMGLIST="$IMGLIST $(find "$LINBOIMGDIR" -maxdepth 2 -name \*.$search)" + done + # trim leading and trailing spaces + echo $IMGLIST | awk '{$1=$1};1' +} + +# end of functions + +# check parameters +if [ -n "$2" ] ; then + # trap torrent parameter + image="${2/.torrent/}" + case "$image" in + *.qcow2|*.qdiff) + if [ -e "$image" ]; then + IMGLIST="$image" + else + IMGLIST="$(find_images "$image")" + fi + if [ ! -e "$IMGLIST" ]; then + echo "Image file $(basename $image) not found." + usage + fi + filename="$(basename "$IMGLIST")" + SESSION="${filename//./_}_torrent" + ;; + *_torrent) + if [ "$1" = "attach" ]; then + SESSION="$image" + else + usage + fi + ;; + *) usage ;; + esac +else + case "$1" in + stop|status) ;; + attach|check|create) usage ;; + *) + IMGLIST="$(find_images)" + if [ -z "$IMGLIST" ]; then + echo "No linbo images found." + exit 0 + fi + ;; + esac +fi + +case "$1" in + start) start ;; + stop) stop ;; + restart|reload) stop ; start ;; + status) status ;; + create) create ;; + check) check ;; + attach) attach ;; + *) usage ;; +esac + +exit 0 diff --git a/roles/lmn_vm/tasks/main.yml b/roles/lmn_vm/tasks/main.yml index b1947d6..1feed81 100644 --- a/roles/lmn_vm/tasks/main.yml +++ b/roles/lmn_vm/tasks/main.yml @@ -63,7 +63,7 @@ - lmn-link-images - lmn-startvirtiofsd -- name: deploy mount home script +- name: deploy vmimages scripts copy: src: "{{ item }}" dest: /usr/local/bin/ @@ -79,6 +79,16 @@ - sync-vm.sh - link-images.sh - start-virtiofsd.sh + - linbo-torrenthelper.sh + - vmimage-torrent + +- name: deploy linbo-torrent defaults + copy: + src: linbo-torrent + dest: /etc/default/ + owner: root + group: root + mode: '0755' - name: Deploy bridge.conf needed for qemu session mode lineinfile: @@ -137,5 +147,5 @@ register: result changed_when: result.stdout | length > 0 when: > - not run_in_installer | default(false) | bool and (ansible_mounts | + false and not run_in_installer | default(false) | bool and (ansible_mounts | selectattr("mount", "equalto", "/") | list)[0].size_available > 80000000000