diff --git a/roles/lmn_vm/files/images.list b/roles/lmn_vm/files/images.list deleted file mode 100644 index 1129e07..0000000 --- a/roles/lmn_vm/files/images.list +++ /dev/null @@ -1,5 +0,0 @@ -win10.qcow2 -win10-SolidWorks.qcow2 -win10-Elektro.qcow2 -deb11.qcow2 -deb11-virtualbox.qcow2 diff --git a/roles/lmn_vm/files/linbo-torrent b/roles/lmn_vm/files/linbo-torrent deleted file mode 100644 index 958b2bb..0000000 --- a/roles/lmn_vm/files/linbo-torrent +++ /dev/null @@ -1,33 +0,0 @@ -# default values for linbo-torrenthelper service provided by ctorrent -# thomas@linuxmuster.net -# 20220317 -# -# note: you have to invoke 'linbo-torrent restart' after you have changed any values -# - -# Exit while seed hours later (default 72 hours) -SEEDHOURS="100000" - -# Max peers count (default 100) -MAXPEERS="100" - -# Min peers count (default 1) -MINPEERS="1" - -# Download slice/block size, unit KB (default 16, max 128) -SLICESIZE="128" - -# Max bandwidth down (unit KB/s, default unlimited) -MAXDOWN="" - -# Max bandwidth up (unit KB/s, default unlimited) -MAXUP="" - -# Supplemental ctorrent options, separated by space (-v: Verbose output for debugging) -#OPTIONS="-v" - -# Timeout in seconds until rsync fallback (client only) -TIMEOUT="300" - -# user to run ctorrent (server only) -CTUSER="lmnsynci" diff --git a/roles/lmn_vm/files/linbo-torrenthelper.sh b/roles/lmn_vm/files/linbo-torrenthelper.sh deleted file mode 100755 index 8c5f2a1..0000000 --- a/roles/lmn_vm/files/linbo-torrenthelper.sh +++ /dev/null @@ -1,31 +0,0 @@ -#!/bin/bash -# -# thomas@linuxmuster.net -# GPL v3 -# 20220317 -# -# linbo ctorrent helper script, started in a screen session by init script -# - -torrent="$1" -[ -s "$torrent" ] || exit 1 - -# get ctorrent options from file -[ -e /etc/default/linbo-torrent ] && source /etc/default/linbo-torrent - -[ -n "$SEEDHOURS" ] && OPTIONS="$OPTIONS -e $SEEDHOURS" -[ -n "$MAXPEERS" ] && OPTIONS="$OPTIONS -M $MAXPEERS" -[ -n "$MINPEERS" ] && OPTIONS="$OPTIONS -m $MINPEERS" -[ -n "$SLICESIZE" ] && OPTIONS="$OPTIONS -z $SLICESIZE" -[ -n "$MAXDOWN" ] && OPTIONS="$OPTIONS -D $MAXDOWN" -[ -n "$MAXUP" ] && OPTIONS="$OPTIONS -U $MAXUP" -OPTIONS="$OPTIONS $torrent" - -[ -n "$CTUSER" ] && SUDO="/usr/bin/sudo -u $CTUSER" - -while true; do - #$SUDO /usr/bin/ctorrent $OPTIONS || exit 1 - nice -n 20 /usr/bin/ctorrent $OPTIONS || exit 1 - # hash check only on initial start, add -f parameter - echo "$OPTIONS" | grep -q ^"-f " || OPTIONS="-f $OPTIONS" -done diff --git a/roles/lmn_vm/files/lmn-link-images b/roles/lmn_vm/files/lmn-link-images deleted file mode 100644 index 15a6a14..0000000 --- a/roles/lmn_vm/files/lmn-link-images +++ /dev/null @@ -1,3 +0,0 @@ -%examusers ALL=(root) NOPASSWD: /usr/local/bin/link-images.sh -%role-student ALL=(root) NOPASSWD: /usr/local/bin/link-images.sh -%role-teacher ALL=(root) NOPASSWD: /usr/local/bin/link-images.sh diff --git a/roles/lmn_vm/files/lmn-mounthome b/roles/lmn_vm/files/lmn-mounthome deleted file mode 100644 index ed958ff..0000000 --- a/roles/lmn_vm/files/lmn-mounthome +++ /dev/null @@ -1,3 +0,0 @@ -%examusers ALL=(root) NOPASSWD: /usr/local/bin/mounthome.sh -%role-student ALL=(root) NOPASSWD: /usr/local/bin/mounthome.sh -%role-teacher ALL=(root) NOPASSWD: /usr/local/bin/mounthome.sh \ No newline at end of file diff --git a/roles/lmn_vm/files/lmn-startvirtiofsd b/roles/lmn_vm/files/lmn-startvirtiofsd deleted file mode 100644 index a56b156..0000000 --- a/roles/lmn_vm/files/lmn-startvirtiofsd +++ /dev/null @@ -1,3 +0,0 @@ -%examusers ALL=(root) NOPASSWD: /usr/local/bin/start-virtiofsd.sh -%role-student ALL=(root) NOPASSWD: /usr/local/bin/start-virtiofsd.sh -%role-teacher ALL=(root) NOPASSWD: /usr/local/bin/start-virtiofsd.sh diff --git a/roles/lmn_vm/files/lmn-sync-vm b/roles/lmn_vm/files/lmn-sync-vm deleted file mode 100644 index 1947e30..0000000 --- a/roles/lmn_vm/files/lmn-sync-vm +++ /dev/null @@ -1,3 +0,0 @@ -%role-teacher ALL=(lmnsynci) NOPASSWD: /usr/local/bin/sync-vm.sh -%role-student ALL=(lmnsynci) NOPASSWD: /usr/local/bin/sync-vm.sh -%examusers ALL=(lmnsynci) NOPASSWD: /usr/local/bin/sync-vm.sh diff --git a/roles/lmn_vm/files/lmn-upload-vm b/roles/lmn_vm/files/lmn-upload-vm deleted file mode 100644 index 599e9d8..0000000 --- a/roles/lmn_vm/files/lmn-upload-vm +++ /dev/null @@ -1 +0,0 @@ -%role-teacher ALL=(root) NOPASSWD: /usr/local/bin/upload-vm.sh diff --git a/roles/lmn_vm/files/lmn-vm b/roles/lmn_vm/files/lmn-vm new file mode 100644 index 0000000..1297be9 --- /dev/null +++ b/roles/lmn_vm/files/lmn-vm @@ -0,0 +1,17 @@ +# vm-sync: Download and synchronize VM-Images and xml-Files +%role-teacher ALL=(lmnsynci) NOPASSWD: /usr/local/bin/vm-sync +%role-student ALL=(lmnsynci) NOPASSWD: /usr/local/bin/vm-sync +%examusers ALL=(lmnsynci) NOPASSWD: /usr/local/bin/vm-sync + +# vm-aria2: Start/Stop aria2 as systemd-service for VM-Images +lmnsynci ALL=(root) NOPASSWD: /usr/local/bin/vm-aria2 + +# vm-link-images: Link VM-Images to User-tmp-directory +%examusers ALL=(root) NOPASSWD: /usr/local/bin/vm-link-images +%role-student ALL=(root) NOPASSWD: /usr/local/bin/vm-link-images +%role-teacher ALL=(root) NOPASSWD: /usr/local/bin/vm-link-images + +# start-virtiofsd: Start Virtiofsd as systemd-service +%examusers ALL=(root) NOPASSWD: /usr/local/bin/start-virtiofsd +%role-student ALL=(root) NOPASSWD: /usr/local/bin/start-virtiofsd +%role-teacher ALL=(root) NOPASSWD: /usr/local/bin/start-virtiofsd diff --git a/roles/lmn_vm/files/upload-vm.sh b/roles/lmn_vm/files/upload-vm.sh deleted file mode 100755 index 7a05074..0000000 --- a/roles/lmn_vm/files/upload-vm.sh +++ /dev/null @@ -1,67 +0,0 @@ -#!/usr/bin/bash -# Push VM-Disk-Image on server -set -eu - -show_help() { - cat << EOF >&2 -Usage: $(basename "$0") vmname" -Create torrent and upload disk, torrent and xml-VM-Definiton on server. -EOF -} - -VM_DIR="/tmp/${SUDO_UID}/vm" - -upload_image() { - # check if VM-Diskimage exists - if [[ ! (-f "/lmn/vm/${VM_NAME}.qcow2" || -f "${VM_DIR}/${VM_NAME}.qcow2") ]]; then - echo "File not found ${VM_NAME}.qcow2" >&2 - exit 1 - fi - # check if VM-Machine-Definition XML exists - if [[ ! (-f "/lmn/vm/${VM_NAME}.xml" || -f "${VM_DIR}/${VM_NAME}.xml") ]]; then - echo "File not found ${VM_NAME}.xml" >&2 - exit 1 - fi - sudo -u lmnsynci /usr/local/bin/vmimage-torrent stop "${VM_NAME}.qcow2" || echo "VMImage-torrent not running" - # link private VM-Diskimage to system-Dir - if [[ -f "${VM_DIR}/${VM_NAME}.qcow2" \ - && ( -f "/lmn/vm/${VM_NAME}.qcow2" && ("${VM_DIR}/${VM_NAME}.qcow2" -nt "/lmn/vm/${VM_NAME}.qcow2") \ - || ! -f "/lmn/vm/${VM_NAME}.qcow2") ]]; then - echo "copy private VM-Diskimage to system-dir" - chown lmnsynci:lmnsynci "${VM_DIR}/${VM_NAME}.qcow2" - ln -f "${VM_DIR}/${VM_NAME}.qcow2" "/lmn/vm/${VM_NAME}.qcow2" - fi - # copy private VM-Maschine-Definition XML to system-Dir - if [[ -f "${VM_DIR}/${VM_NAME}.xml" \ - && ( -f "/lmn/vm/${VM_NAME}.xml" && $(cmp -s "${VM_DIR}/${VM_NAME}.xml" "/lmn/vm/${VM_NAME}.xml") \ - || ! -f "/lmn/vm/${VM_NAME}.xml") ]]; then - echo "copy private VM-Maschine-Definition XML to system-dir" - chown lmnsynci:lmnsynci "${VM_DIR}/${VM_NAME}.xml" - cp -a "${VM_DIR}/${VM_NAME}.xml" "/lmn/vm/" - fi - cd /lmn/vm - # (re-) create torrent file - sudo -u lmnsynci /usr/local/bin/vmimage-torrent create "${VM_NAME}.qcow2" - # create size-information-file - stat -c%s "${VM_NAME}.qcow2" > "${VM_NAME}.qcow2.size" - chown lmnsynci:lmnsynci "${VM_NAME}.qcow2.size" - # Upload Torrent, qcow2 and machine-definition-XML - [[ -f "/lmn/vm/${VM_NAME}.qcow2.torrent" ]] && rsync -av --password-file=/etc/rsync.secret \ - "/lmn/vm/${VM_NAME}.qcow2.torrent" rsync://vmuser@server:/vmimages-upload/ - rsync -av --password-file=/etc/rsync.secret "/lmn/vm/${VM_NAME}.qcow2.size" \ - rsync://vmuser@server:/vmimages-upload/ - rsync -av --password-file=/etc/rsync.secret "/lmn/vm/${VM_NAME}.qcow2" \ - rsync://vmuser@server:/vmimages-upload/ - rsync -av --password-file=/etc/rsync.secret "/lmn/vm/${VM_NAME}.xml" \ - rsync://vmuser@server:/vmimages-upload/ -} - -# if less than one arguments supplied, display usage -if [[ $# -ne 1 ]] ; then - show_help - exit 1 -fi - -VM_NAME=$1 - -upload_image diff --git a/roles/lmn_vm/files/uploadseed b/roles/lmn_vm/files/uploadseed new file mode 100755 index 0000000..eb64945 --- /dev/null +++ b/roles/lmn_vm/files/uploadseed @@ -0,0 +1,83 @@ +#!/usr/bin/python3 + +import os, sys +import subprocess +import xmlrpc.client as xc +import ssl +import argparse + +parser = argparse.ArgumentParser(description='Upload a file to the bittorrent seeder.') +parser.add_argument('--rpc-server', required=True, + help='the RPC server IPaddress:port') +parser.add_argument('--rpc-secret', required=True, + help='the RPC secret') +parser.add_argument('--dht-port', required=True, + help='the DHT port the RPC server is listening on') +parser.add_argument('--no-cert', action='store_true', + help='do not use SSL certificate') +parser.add_argument('--cert', help='the certificate to use for verification') +parser.add_argument('file', help='the file to upload') + +args = parser.parse_args() + +rpcseeder = 'https://' + args.rpc_server + '/rpc' +secret = 'token:' + args.rpc_secret +dhtentry = args.rpc_server.split(':')[0] + ':' + args.dht_port +file2send = args.file +torrent = '/tmp/' + os.path.basename(file2send) + '.torrent' + +ssl_ctx = ssl.create_default_context() +if args.no_cert: + ssl_ctx.check_hostname = False + ssl_ctx.verify_mode = ssl.CERT_NONE + print("Certificate verification disabled.") +elif args.cert is not None: + ssl_ctx.load_verify_locations(args.cert) + +s = xc.ServerProxy(rpcseeder, context = ssl_ctx) + +def make_torrent(): + if os.path.isfile(torrent): + print("Torrent file", torrent, "exists already, please (re)move it.") + sys.exit(1) + + subprocess.run(["/usr/bin/mktorrent", "-l 24", "-v", "-o", torrent, file2send], check=True) + h = subprocess.check_output(["/usr/bin/aria2c", "-S ", torrent]) + for line in h.decode().splitlines(): + if "Info Hash" in line: + return line.split(': ')[1] + +def check_seeds(bthash): + active_seeds = s.aria2.tellActive(secret) + for seed in active_seeds: + f = seed['bittorrent']['info']['name'] + gid = seed['gid'] + ihash = seed['infoHash'] + if f == os.path.basename(file2send): + print(file2send, "is already seeded with GID:", gid) + print("Info Hash is:", ihash) + if bthash == ihash: + print("The torrent file has not changed, exiting.") + return False + else: + print("The torrent file has changed, replacing torrent.") + s.aria2.remove(secret, gid) + return True + print("="*19, " Uploading new torrent with aria2 now. ", "="*19) + return True + +def upload_torrent(): + s.aria2.addTorrent(secret, xc.Binary(open(torrent, mode='rb').read())) + subprocess.run(["/usr/bin/aria2c", + "--dht-entry-point=" + dhtentry, + "--check-integrity", + "--dir=" + os.path.dirname(file2send), + torrent]) + +############################ + +if __name__ == '__main__': + infoHash = make_torrent() + if check_seeds(infoHash): + upload_torrent() + print("Upload finished.") diff --git a/roles/lmn_vm/files/vm-aria2 b/roles/lmn_vm/files/vm-aria2 new file mode 100755 index 0000000..39b3ae6 --- /dev/null +++ b/roles/lmn_vm/files/vm-aria2 @@ -0,0 +1,30 @@ +#!/usr/bin/bash + +set -eu + +# if less than one arguments supplied, display usage +if [[ $# -ne 2 ]]; then + echo "This script takes as input the name of the VM " >&2 + echo "Usage: $0 [start|stop] vm_name" >&2 + exit 1 +fi + +COMMAND="$1" +VM_NAME="$2" + +source /etc/lmn/vm.conf + +if [[ "${COMMAND}" = "start" ]]; then + systemd-run --unit=aria2-"${VM_NAME}" \ + --slice=system-aria2 \ + --uid="$(id -u lmnsynci)" \ + --gid="$(id -g lmnsynci)" \ + --nice=19 \ + --working-directory="${VM_SYSDIR}" \ + --collect \ + --property=Type=exec \ + --property=SuccessExitStatus=1 \ + aria2c --bt-hash-check-seed=true --check-integrity=true --seed-ratio=0.0 --dht-entry-point="${SEEDBOX_HOST}:${SEEDBOX_PORT}" "${VM_SYSDIR}/${VM_NAME}.qcow2.torrent" +elif [[ "${COMMAND}" = "stop" ]]; then + systemctl stop "aria2-${VM_NAME}.service" || echo "Aria2-Service not running" +fi diff --git a/roles/lmn_vm/files/create-vm.sh b/roles/lmn_vm/files/vm-create similarity index 69% rename from roles/lmn_vm/files/create-vm.sh rename to roles/lmn_vm/files/vm-create index bd1669f..5c5548f 100755 --- a/roles/lmn_vm/files/create-vm.sh +++ b/roles/lmn_vm/files/vm-create @@ -2,6 +2,20 @@ # create 1st level-Clones set -eu + +source /etc/lmn/vm.conf + +while getopts ':p' OPTION; do + case "$OPTION" in + p) + PERSISTENT=1 + VM_DIR="${VM_DIR_PERSISTENT}" + ;; + esac +done + +shift "$((OPTIND -1))" + # if less than two arguments supplied, display usage if [[ $# -ne 2 ]]; then echo "This script takes as input the name of the VM to clone" >&2 @@ -11,16 +25,19 @@ fi VM_NAME=$1 VM_CLONE=$2 -VM_DIR="/tmp/${UID}/vm" # Create User-VM-Dir and link system VM-Images [[ -d "${VM_DIR}" ]] || mkdir -p "${VM_DIR}" -sudo /usr/local/bin/link-images.sh +if [[ "${PERSISTENT}" -eq 1 ]]; then + sudo /usr/local/bin/vm-link-images -p +else + sudo /usr/local/bin/vm-link-images +fi # change to image-directory cd "${VM_DIR}" -if { [[ ! -f "${VM_NAME}.xml" ]] && [[ ! -f "/lmn/vm/${VM_NAME}.xml" ]]; } || [[ ! -f "${VM_NAME}.qcow2" ]]; then +if { [[ ! -f "${VM_NAME}.xml" ]] && [[ ! -f "${VM_SYSDIR}/${VM_NAME}.xml" ]]; } || [[ ! -f "${VM_NAME}.qcow2" ]]; then echo "xml or qcow2 File does not exists." >&2 exit 1 fi @@ -32,8 +49,8 @@ chmod a-w "${VM_NAME}-${VM_CLONE}.qcow2" # copy machine-definition-file if [[ -f "${VM_NAME}.xml" ]]; then cp "${VM_NAME}.xml" "${VM_NAME}-${VM_CLONE}.xml" -elif [[ -f "/lmn/vm/${VM_NAME}.xml" ]]; then - cp "/lmn/vm/${VM_NAME}.xml" "${VM_NAME}-${VM_CLONE}.xml" +elif [[ -f "${VM_SYSDIR}/${VM_NAME}.xml" ]]; then + cp "${VM_SYSDIR}/${VM_NAME}.xml" "${VM_NAME}-${VM_CLONE}.xml" else echo "no machine definition file found" >&2 exit 1 diff --git a/roles/lmn_vm/files/link-images.sh b/roles/lmn_vm/files/vm-link-images similarity index 57% rename from roles/lmn_vm/files/link-images.sh rename to roles/lmn_vm/files/vm-link-images index c9dd084..a3a06df 100755 --- a/roles/lmn_vm/files/link-images.sh +++ b/roles/lmn_vm/files/vm-link-images @@ -1,12 +1,22 @@ #!/usr/bin/bash -# link VM in Use-Dir in /tmp +# link VM in User-Dir in /tmp or /var/vm set -eu -# change to image-directory -cd /lmn/vm +source /etc/lmn/vm.conf -VM_DIR="/tmp/${SUDO_UID}/vm" +# change to image-directory +cd "${VM_SYSDIR}" + +while getopts ':p' OPTION; do + case "$OPTION" in + p) + VM_DIR="${VM_DIR_PERSISTENT}" + ;; + esac +done + +shift "$((OPTIND -1))" # link system-VM-Images to User VM Directory for i in *.qcow2; do diff --git a/roles/lmn_vm/files/rebase-vm.sh b/roles/lmn_vm/files/vm-rebase similarity index 89% rename from roles/lmn_vm/files/rebase-vm.sh rename to roles/lmn_vm/files/vm-rebase index d97a778..cc6292e 100755 --- a/roles/lmn_vm/files/rebase-vm.sh +++ b/roles/lmn_vm/files/vm-rebase @@ -10,11 +10,16 @@ This script takes as input the name of the VM to rebase one level down EOF } -while getopts ':n:' OPTION; do +source /etc/lmn/vm.conf + +while getopts ':n:p' OPTION; do case "$OPTION" in n) NEWNAME=$OPTARG ;; + p) + VM_DIR="${VM_DIR_PERSISTENT}" + ;; ?) show_help exit 1 @@ -31,7 +36,6 @@ if [[ $# -ne 1 ]]; then fi # change to Images directory -VM_DIR="/tmp/${UID}/vm" cd "${VM_DIR}" VM_NAME="$1" @@ -70,8 +74,8 @@ if [[ -v NEWNAME ]]; then CURRENTNAME="${CURRENTBASE/.qcow2/}" if [[ -f "${CURRENTNAME}.xml" ]]; then cp "${CURRENTNAME}.xml" "${NEWNAME}.xml" - elif [[ -f "/lmn/vm/${CURRENTNAME}.xml" ]]; then - cp "/lmn/vm/${CURRENTNAME}.xml" "${NEWNAME}.xml" + elif [[ -f "${VM_SYSDIR}/${CURRENTNAME}.xml" ]]; then + cp "${VM_SYSDIR}/${CURRENTNAME}.xml" "${NEWNAME}.xml" else echo "no machine definition file found" >&2 exit 1 diff --git a/roles/lmn_vm/files/run-vm.sh b/roles/lmn_vm/files/vm-run similarity index 71% rename from roles/lmn_vm/files/run-vm.sh rename to roles/lmn_vm/files/vm-run index 1f8a714..c5cbad4 100755 --- a/roles/lmn_vm/files/run-vm.sh +++ b/roles/lmn_vm/files/vm-run @@ -8,8 +8,8 @@ show_help() { Usage: $(basename "$0") [-n] vmname" Create a new clone, start the vm (if not yet running) and run virt-viewer. Squid-Proxy will be started too. -### remove, old ### User Home will be mounted on /media/USERNAME/home -n new clone will be created, even if exists + -p new clone will be created persistent, so available after reboot too -s qemu:///system instead of default qemu:///session EOF } @@ -23,31 +23,34 @@ exit_script() { check_images() { # sync vm-torrents and machine definition file - sudo -u lmnsynci /usr/local/bin/sync-vm.sh -t + sudo -u lmnsynci /usr/local/bin/vm-sync get_file "${VM_NAME}.xml" "${VM_NAME}.qcow2.torrent" + [[ -f "${VM_NAME}" ]] && sudo -u lmnsynci /usr/local/bin/vm-sync delete_outdated_image "${VM_NAME}.qcow2" BACKINGARRAY=() - imgfile="/lmn/vm/${VM_NAME}.qcow2" && [[ -f "${VM_DIR}/${VM_NAME}.qcow2" ]] && imgfile="${VM_DIR}/${VM_NAME}.qcow2" + imgfile="${VM_SYSDIR}/${VM_NAME}.qcow2" && [[ -f "${VM_DIR}/${VM_NAME}.qcow2" ]] && imgfile="${VM_DIR}/${VM_NAME}.qcow2" BACKINGARRAY+=("${imgfile}") echo "Imgfile=$imgfile" if [[ ! -f "${imgfile}" ]] || ! qemu-img info -U "${imgfile}" | grep "file format: qcow2"; then - if [[ ! -f "/lmn/vm/${VM_NAME}.qcow2.torrent" ]]; then + if [[ ! -f "${VM_SYSDIR}/${VM_NAME}.qcow2.torrent" ]]; then echo "no base VM disk '${VM_NAME}.qcow2' found and/or ${VM_NAME} not found on server" >&2 exit 1 fi # sync vm-disk image by torrent echo "Try to sync VM ${VM_NAME} by torrent" - sudo -u lmnsynci /usr/local/bin/sync-vm.sh "${VM_NAME}" + sudo -u lmnsynci /usr/local/bin/vm-sync get_image "${VM_NAME}" fi backingfile=$(qemu-img info -U "${imgfile}" | grep "^backing file:" | cut -d ' ' -f 3) - while [[ ! -z "${backingfile}" ]]; do + while [[ -n "${backingfile}" ]]; do echo "Backingfile required: ${backingfile}" - imgfile="/lmn/vm/${backingfile}" && [[ -f "${VM_DIR}/${backingfile}" ]] && imgfile="${VM_DIR}/${backingfile}" + imgfile="${VM_SYSDIR}/${backingfile}" && [[ -f "${VM_DIR}/${backingfile}" ]] && imgfile="${VM_DIR}/${backingfile}" BACKINGARRAY+=("${imgfile}") if [[ ! -f "${imgfile}" ]] || ! qemu-img info -U "${imgfile}" | grep "file format: qcow2"; then # sync vm-disk image by torrent echo "Try to sync backingfile ${backingfile} by torrent" - sudo -u lmnsynci /usr/local/bin/sync-vm.sh "${backingfile%.qcow2}" + sudo -u lmnsynci /usr/local/bin/vm-sync get_file "${backingfile}.torrent" + [[ -f "${backingfile}" ]] && sudo -u lmnsynci /usr/local/bin/vm-sync delete_outdated_image "${backingfile}" + sudo -u lmnsynci /usr/local/bin/vm-sync get_image "${backingfile%.qcow2}" fi backingfile=$(qemu-img info -U "${imgfile}" | grep "^backing file:" | cut -d ' ' -f 3) done @@ -62,27 +65,29 @@ check_images() { if ! qemu-img check -U "${BACKINGARRAY[$i]}" 2>/dev/null; then echo "check failed!" echo "sync ${BACKINGARRAY[$i]} again" - sudo -u lmnsynci /usr/local/bin/sync-vm.sh $(basename "${BACKINGARRAY[$i]}" .qcow2) + sudo -u lmnsynci /usr/local/bin/vm-sync get_image "$(basename "${BACKINGARRAY[$i]}" .qcow2)" fi done echo "VM-Image and required backingfiles available and checked" } -create-clone() { +create_clone() { local VM_NAME="$1" - local VM_DIR="/tmp/${UID}/vm" local VM_XML="${VM_DIR}/${VM_NAME}-clone.xml" - local VM_SYSDIR="/lmn/vm" - if ! [[ -f "$VM_SYSDIR/${VM_NAME}.xml" && -f "$VM_SYSDIR/${VM_NAME}.qcow2" ]] && ! [[ -f "${VM_DIR}/${VM_NAME}.xml" && -f "${VM_DIR}/${VM_NAME}.qcow2" ]]; then + if ! [[ -f "${VM_SYSDIR}/${VM_NAME}.xml" && -f "${VM_SYSDIR}/${VM_NAME}.qcow2" ]] && ! [[ -f "${VM_DIR}/${VM_NAME}.xml" && -f "${VM_DIR}/${VM_NAME}.qcow2" ]]; then echo "xml or qcow2 File does not exists." >&2 exit 1 fi # Create User-VM-Dir and link system VM-Images [[ -d "${VM_DIR}" ]] || mkdir -p "${VM_DIR}" - sudo /usr/local/bin/link-images.sh + if [[ "${PERSISTENT}" -eq 1 ]]; then + sudo /usr/local/bin/vm-link-images -p + else + sudo /usr/local/bin/vm-link-images + fi # Create backing file cd "${VM_DIR}" @@ -107,9 +112,16 @@ create-clone() { QEMU='qemu:///session' NEWCLONE=0 +PERSISTENT=0 -while getopts ':ns' OPTION; do +source /etc/lmn/vm.conf + +while getopts ':pns' OPTION; do case "$OPTION" in + p) + PERSISTENT=1 + VM_DIR="${VM_DIR_PERSISTENT}" + ;; n) NEWCLONE=1 ;; @@ -132,7 +144,6 @@ if [[ $# -ne 1 ]] ; then fi VM_NAME=$1 -VM_DIR="/tmp/${UID}/vm" # check, if we have to start squid if ! killall -s 0 squid; then @@ -147,14 +158,14 @@ if ! virsh --connect="${QEMU}" list | grep "${VM_NAME}-clone"; then echo "VM not yet running." check_images if [[ "${NEWCLONE}" = 1 ]] || [[ ! -f "${VM_DIR}/${VM_NAME}-clone.qcow2" ]]; then - create-clone "${VM_NAME}" + create_clone "${VM_NAME}" fi # delete the old vm virsh --connect=qemu:///session undefine "${VM_NAME}-clone" || echo "${VM_NAME}-clone did not exist" # finally, create the new vm virsh --connect=qemu:///session define "${VM_DIR}/${VM_NAME}-clone.xml" #trap exit_script SIGHUP SIGINT SIGTERM - [[ "${QEMU}" = 'qemu:///session' ]] && sudo /usr/local/bin/start-virtiofsd.sh "${VM_NAME}" + [[ "${QEMU}" = 'qemu:///session' ]] && sudo /usr/local/bin/vm-virtiofsd "${VM_NAME}" virsh --connect="${QEMU}" start "${VM_NAME}-clone" fi echo "starting viewer" diff --git a/roles/lmn_vm/files/vm-sync b/roles/lmn_vm/files/vm-sync new file mode 100755 index 0000000..5e2c6a8 --- /dev/null +++ b/roles/lmn_vm/files/vm-sync @@ -0,0 +1,131 @@ +#!/usr/bin/bash +# Push/Pull VM-Disk-Image and Infos from server +set -eu + +show_help() { + cat << EOF >&2 +Usage: $(basename "$0") command [args]" +command: + push_file + get_file + get_image + delete_outdated_image +EOF +} + +get_torrent() { + if [[ ! -f "${VM_SYSDIR}/${VM_NAME}.qcow2.torrent" ]]; then + echo "No torrent-File found" + exit 1 + fi + lockfile="/tmp/sync-vm-${VM_NAME}.lock" + if ! flock -n "$lockfile" echo "try to acquire lock"; then + echo torrent seems to be in process. + echo waiting for completion ... + flock -w 3600 "$lockfile" echo "...completed" + sleep 5 + else + ( + if ! flock -n 200; then + echo "failed to acquire lock" + echo "Bitte noch einmal starten." + echo "Beliebige Taste zum Beenden." + read -n 1 + exit 1 + fi + # stop aria2-seeding if running + sudo vm-aria2 stop "${VM_NAME}" + cd "${VM_SYSDIR}" + # get image + aria2c --seed-time=0 --dht-entry-point="${SEEDBOX_HOST}:${SEEDBOX_PORT}" "${VM_SYSDIR}/${VM_NAME}.qcow2.torrent" + # and seed + sudo vm-aria2 start "${VM_NAME}" + if ! flock -u 200; then + echo failed to drop lock + exit 1 + fi + ) 200>"$lockfile" + fi +} + + +get_image_size() { + torrentfile=$1 + length=$(aria2c -S "${torrentfile}" | grep "Total Length" | grep "Total Length" | sed -E 's/.*\(([0-9,]*)\)/\1/' | sed s/,//g) + echo "$length" +} + +delete_outdated_image() { + cd "${VM_SYSDIR}" + qcowsize=$(stat -c%s "${FILENAME}") + if [[ -f "${FILENAME}.torrent" ]] && [[ "${qcowsize}" != $(get_image_size "${FILENAME}.torrent") ]]; then + sudo vm-aria2 stop "${FILENAME%.qcow2}" + rm -f "${FILENAME}" + fi +} + +get_file() { + cd "${VM_SYSDIR}" + wget --no-proxy -O "${FILENAME}" "http://${SEEDBOX_HOST}/aria2/${FILENAME}" || echo "File not found on seedbox" +} + +push_file() { + cd "${VM_SYSDIR}" + uploadseed --rpc-server "${SEEDBOX_HOST}:${SEEDBOX_RPC_PORT}" --dht-port "${SEEDBOX_PORT}" --rpc-secret insecure --no-cert "${FILENAME}" +} + +if [[ "$(id -nu)" != "lmnsynci" ]]; then + echo "$(basename "$0") must be run as lmnsynci user" + show_help + exit 1 +fi + +source /etc/lmn/vm.conf + +while getopts ':' OPTION; do + case "$OPTION" in + ?) + show_help + exit 1 + ;; + esac +done + +shift "$((OPTIND -1))" + +# if less than one arguments supplied, display usage +if [[ $# -lt 1 ]]; then + show_help + exit 1 +fi + +command=$1 +shift + +case "$command" in + push_file) + for FILENAME in "$@"; do + push_file + done + ;; + get_file) + for FILENAME in "$@"; do + get_file + done + ;; + get_image) + for VM_NAME in "$@"; do + get_torrent + done + ;; + delete_outdated_image) + for FILENAME in "$@"; do + delete_outdated_image + done + ;; + *) + show_help + exit 1 + ;; +esac + diff --git a/roles/lmn_vm/files/vm-upload b/roles/lmn_vm/files/vm-upload new file mode 100755 index 0000000..e3d62ee --- /dev/null +++ b/roles/lmn_vm/files/vm-upload @@ -0,0 +1,70 @@ +#!/usr/bin/bash +# Push VM-Disk-Image on server +set -eu + +show_help() { + cat << EOF >&2 +Usage: $(basename "$0") vmname" +Create torrent and upload disk and xml-VM-Definiton on server. +EOF +} + + +upload_image() { + # check if VM-Diskimage exists + if [[ ! (-f "${VM_SYSDIR}/${VM_NAME}.qcow2" || -f "${VM_DIR}/${VM_NAME}.qcow2") ]]; then + echo "File not found ${VM_NAME}.qcow2" >&2 + exit 1 + fi + # check if VM-Machine-Definition XML exists + if [[ ! (-f "${VM_SYSDIR}/${VM_NAME}.xml" || -f "${VM_DIR}/${VM_NAME}.xml") ]]; then + echo "File not found ${VM_NAME}.xml" >&2 + exit 1 + fi + sudo vm-aria2 stop "${VM_NAME}" || echo "VMImage-torrent not running" + # link private VM-Diskimage to system-Dir + if [[ -f "${VM_DIR}/${VM_NAME}.qcow2" \ + && ( -f "${VM_SYSDIR}/${VM_NAME}.qcow2" && ("${VM_DIR}/${VM_NAME}.qcow2" -nt "${VM_SYSDIR}/${VM_NAME}.qcow2") \ + || ! -f "${VM_SYSDIR}/${VM_NAME}.qcow2") ]]; then + echo "copy private VM-Diskimage to system-dir" + chown lmnsynci:lmnsynci "${VM_DIR}/${VM_NAME}.qcow2" + ln -f "${VM_DIR}/${VM_NAME}.qcow2" "${VM_SYSDIR}/${VM_NAME}.qcow2" + fi + # copy private VM-Maschine-Definition XML to system-Dir + if [[ -f "${VM_DIR}/${VM_NAME}.xml" \ + && ( -f "${VM_SYSDIR}/${VM_NAME}.xml" && $(cmp -s "${VM_DIR}/${VM_NAME}.xml" "${VM_SYSDIR}/${VM_NAME}.xml") \ + || ! -f "${VM_SYSDIR}/${VM_NAME}.xml") ]]; then + echo "copy private VM-Maschine-Definition XML to system-dir" + chown lmnsynci:lmnsynci "${VM_DIR}/${VM_NAME}.xml" + cp -a "${VM_DIR}/${VM_NAME}.xml" "${VM_SYSDIR}" + fi + cd "${VM_SYSDIR}" + uploadseed --rpc-server "${SEEDBOX_HOST}:${SEEDBOX_RPC_PORT}" --dht-port "${SEEDBOX_PORT}" --rpc-secret insecure --no-cert "${VM_NAME}.qcow2" + uploadseed --rpc-server "${SEEDBOX_HOST}:${SEEDBOX_RPC_PORT}" --dht-port "${SEEDBOX_PORT}" --rpc-secret insecure --no-cert "${VM_NAME}.xml" +} + +source /etc/lmn/vm.conf + +while getopts ':p' OPTION; do + case "$OPTION" in + p) + VM_DIR="${VM_DIR_PERSISTENT}" + ;; + ?) + show_help + exit 1 + ;; + esac +done + +shift "$((OPTIND -1))" + +# if less than one arguments supplied, display usage +if [[ $# -ne 1 ]] ; then + show_help + exit 1 +fi + +VM_NAME=$1 + +upload_image diff --git a/roles/lmn_vm/files/start-virtiofsd.sh b/roles/lmn_vm/files/vm-virtiofsd similarity index 100% rename from roles/lmn_vm/files/start-virtiofsd.sh rename to roles/lmn_vm/files/vm-virtiofsd diff --git a/roles/lmn_vm/files/vm.conf b/roles/lmn_vm/files/vm.conf new file mode 100644 index 0000000..b10aa9f --- /dev/null +++ b/roles/lmn_vm/files/vm.conf @@ -0,0 +1,14 @@ +# variables for LMN VM submodule + +SEEDBOX_HOST=seedbox.pn.steinbeis.schule +SEEDBOX_PORT=6789 +SEEDBOX_RPC_PORT=6800 + +VM_SYSDIR="/lmn/vm" +if [[ -v SUDO_UID ]]; then + VM_DIR="/tmp/${SUDO_UID}/vm" + VM_DIR_PERSISTENT="/var/vm/${SUDO_UID}" +else + VM_DIR="/tmp/${UID}/vm" + VM_DIR_PERSISTENT="/var/vm/${UID}" +fi diff --git a/roles/lmn_vm/files/vmimage-torrent b/roles/lmn_vm/files/vmimage-torrent deleted file mode 100755 index 4ec8bd6..0000000 --- a/roles/lmn_vm/files/vmimage-torrent +++ /dev/null @@ -1,213 +0,0 @@ -#!/bin/bash -# -# starts tmux sessions for each valid torrent in LINBODIR -# thomas@linuxmuster.net -# 20221103 -# - -# read environment -#. /usr/share/linuxmuster/defaults.sh || exit 1 -#THELPER=$LINBOSHAREDIR/linbo-torrenthelper.sh -THELPER=linbo-torrenthelper.sh -#. $LINBOSHAREDIR/helperfunctions.sh || exit 1 -LINBOIMGEXT="qcow2 qdiff" -LINBOIMGDIR="/lmn/vm" -serverip="10.190.1.1" - -# start of functions - -# help message -usage(){ - echo - echo "Info: vmimage-torrent manages the torrent tmux sessions of linbo images." - echo - echo "Usage:" - echo " vmimage-torrent [image_name]" - echo " vmimage-torrent attach " - echo - echo "Note:" - echo " * Only qcow2 & qdiff image files located below $LINBOIMGDIR are processed." - echo " * The commands \"start\", \"stop\" and \"restart\" may have optionally an image" - echo " filename as parameter. In this case the commands are only applied to the tmux" - echo " session of the certain file. Without an explicit image filename the commands" - echo " were applied to all image file sessions currently running." - echo " * An image filename parameter is mandatory with the commands \"check\", \"create\"" - echo " and \"attach\"." - echo " * \"check\" checks if the image file matches to the correspondig torrent." - echo " * \"create\" creates/recreates the torrent of a certain image file." - echo " * \"status\" shows a list of currently running torrent tmux sessions." - echo " * \"attach\" attaches a torrent tmux session of a certain image. An image or" - echo " session name must be given as parameter." - echo " Press [CTRL+B]+[D] to detach the session again." - echo " * \"reload\" is the identical to \"restart\" and is there for backwards compatibility." - echo - exit 1 -} - -# check torrent -check(){ - local image="$(basename "$IMGLIST")" - local torrent="$image.torrent" - local tdir="$(dirname "$IMGLIST")" - cd "$tdir" - echo "Checking $torrent ..." - if ctorrent -c "$torrent"; then - echo "Ok!" - else - echo "Failed!" - exit 1 - fi -} - -# creates torrent files -create(){ - local image="$(basename "$IMGLIST")" - local tdir="$(dirname "$IMGLIST")" - local torrent="${image}.torrent" - local session="${torrent//./_}" - # stop torrent service - vmimage-torrent status | grep -q ^"$session" && vmimage-torrent stop "$IMGLIST" - # skip already running torrents - echo "Creating $torrent ..." - cd "$tdir" - rm -f "$torrent" - if ctorrent -t -u "http://$serverip:6969/announce" -s "$torrent" "$image" ; then - [ "$START" = "no" ] || vmimage-torrent start "$IMGLIST" - else - echo "Failed!" - exit 1 - fi -} - -# starts torrent tmux sessions -start(){ - local item - local torrent - local image - local tdir - local session - for item in $IMGLIST; do - image="$(basename "$item")" - torrent="${image}.torrent" - tdir="$(dirname "$item")" - session="${torrent//./_}" - cd "$tdir" - if [ ! -s "$image" ]; then - echo "Image $image does not exist! Skipping this torrent." - continue - fi - # skip already running torrents - if vmimage-torrent status | grep -qw ^"$session"; then - echo "tmux session $session is already running." - continue - fi - # create torrent file if there is none - if [ ! -e "$torrent" ]; then - START="no" vmimage-torrent create "$item" || continue - fi - echo -n "Starting tmux session $session ... " - tmux new -ds "$session" "$THELPER $torrent ; exec $SHELL" - sleep 1 - if vmimage-torrent status | grep -qw ^"$session"; then - echo "Ok!" - else - echo "Failed!" - fi - done -} - -stop(){ - if [ -n "$SESSION" ]; then - vmimage-torrent status | grep -qw ^"$SESSION" || return - tmux kill-session -t "$SESSION" - else - local item - vmimage-torrent status | awk -F\: '{print $1}' | while read item; do - tmux kill-session -t "$item" - done - fi -} - -attach(){ - if ! tmux list-sessions | grep -qw "$SESSION"; then - echo "There is no session $SESSION." - exit 1 - fi - echo "Hint: Detach tmux session with [CTRL+B]+[D]." - sleep 3 - tmux attach -t "$SESSION" -} - -status(){ - tmux list-sessions | grep _torrent -} - -find_images(){ - local search="$(basename "$1")" - if [ -n "$search" ]; then - find "$LINBOIMGDIR" -maxdepth 2 -name "$search" - return - fi - local IMGLIST - for search in $LINBOIMGEXT; do - IMGLIST="$IMGLIST $(find "$LINBOIMGDIR" -maxdepth 2 -name \*.$search)" - done - # trim leading and trailing spaces - echo $IMGLIST | awk '{$1=$1};1' -} - -# end of functions - -# check parameters -if [ -n "$2" ] ; then - # trap torrent parameter - image="${2/.torrent/}" - case "$image" in - *.qcow2|*.qdiff) - if [ -e "$image" ]; then - IMGLIST="$image" - else - IMGLIST="$(find_images "$image")" - fi - if [ ! -e "$IMGLIST" ]; then - echo "Image file $(basename $image) not found." - usage - fi - filename="$(basename "$IMGLIST")" - SESSION="${filename//./_}_torrent" - ;; - *_torrent) - if [ "$1" = "attach" ]; then - SESSION="$image" - else - usage - fi - ;; - *) usage ;; - esac -else - case "$1" in - stop|status) ;; - attach|check|create) usage ;; - *) - IMGLIST="$(find_images)" - if [ -z "$IMGLIST" ]; then - echo "No linbo images found." - exit 0 - fi - ;; - esac -fi - -case "$1" in - start) start ;; - stop) stop ;; - restart|reload) stop ; start ;; - status) status ;; - create) create ;; - check) check ;; - attach) attach ;; - *) usage ;; -esac - -exit 0 diff --git a/roles/lmn_vm/files/vmimage-torrent.service b/roles/lmn_vm/files/vmimage-torrent.service deleted file mode 100644 index 576e6e1..0000000 --- a/roles/lmn_vm/files/vmimage-torrent.service +++ /dev/null @@ -1,15 +0,0 @@ -[Unit] -Description=VM-image torrent service -After=network.target - -[Service] -Type=oneshot -RemainAfterExit=yes -User=lmnsynci -Group=lmnsynci -ExecStart=/usr/local/bin/vmimage-torrent start -ExecStop=/usr/local/bin/vmimage-torrent stop -ExecReload=/usr/local/bin/vmimage-torrent reload - -[Install] -WantedBy=multi-user.target diff --git a/roles/lmn_vm/tasks/main.yml b/roles/lmn_vm/tasks/main.yml index 46f0c20..f6c34ff 100644 --- a/roles/lmn_vm/tasks/main.yml +++ b/roles/lmn_vm/tasks/main.yml @@ -13,7 +13,8 @@ - name: install libvirt packages apt: name: - - ctorrent + - aria2 + - mktorrent - libvirt-daemon-system - virt-manager state: latest @@ -80,6 +81,11 @@ system: true create_home: false +- name: Create /etc/lmn directory + file: + path: /etc/lmn + state: directory + - name: Create /lmn directory file: path: /lmn @@ -91,6 +97,12 @@ state: directory mode: '1777' +- name: Create /var/vm directory + file: + path: /var/vm + state: directory + mode: '1777' + - name: Create vm directory file: path: /lmn/vm @@ -127,10 +139,7 @@ mode: '0700' loop: - lmn-mounthome - - lmn-sync-vm - - lmn-upload-vm - - lmn-link-images - - lmn-startvirtiofsd + - lmn-vm - name: Deploy vmimages scripts copy: @@ -141,33 +150,24 @@ mode: '0755' loop: - mounthome.sh - - create-vm.sh - - rebase-vm.sh - - run-vm.sh - - upload-vm.sh - sync-vm.sh - - link-images.sh - - start-virtiofsd.sh - - linbo-torrenthelper.sh - - vmimage-torrent + - vm-create + - vm-rebase + - vm-run + - vm-upload + - vm-sync + - vm-link-images + - vm-virtiofsd - virtiofsd + - vm-aria2 + - uploadseed -- name: Deploy linbo-torrent defaults +- name: Deploy vm configuration file vm.conf copy: - src: linbo-torrent - dest: /etc/default/ + src: vm.conf + dest: /etc/lmn/vm.conf owner: root group: root - mode: '0755' - -- name: Deploy vmimage-torrent.service - copy: - src: vmimage-torrent.service - dest: /etc/systemd/system/ - owner: root - group: root - mode: '0644' - notify: "enable vmimage-torrent.service" - name: Prepare directory for qemu bridge config ansible.builtin.file: