distribute VM-images with aria2 instead of ctorrent

This commit is contained in:
Raphael Dannecker 2024-01-19 20:08:19 +01:00
parent e30a7032a5
commit 40962fd9de
23 changed files with 444 additions and 434 deletions

View file

@ -1,5 +0,0 @@
win10.qcow2
win10-SolidWorks.qcow2
win10-Elektro.qcow2
deb11.qcow2
deb11-virtualbox.qcow2

View file

@ -1,33 +0,0 @@
# default values for linbo-torrenthelper service provided by ctorrent
# thomas@linuxmuster.net
# 20220317
#
# note: you have to invoke 'linbo-torrent restart' after you have changed any values
#
# Exit while seed <SEEDHOURS> hours later (default 72 hours)
SEEDHOURS="100000"
# Max peers count (default 100)
MAXPEERS="100"
# Min peers count (default 1)
MINPEERS="1"
# Download slice/block size, unit KB (default 16, max 128)
SLICESIZE="128"
# Max bandwidth down (unit KB/s, default unlimited)
MAXDOWN=""
# Max bandwidth up (unit KB/s, default unlimited)
MAXUP=""
# Supplemental ctorrent options, separated by space (-v: Verbose output for debugging)
#OPTIONS="-v"
# Timeout in seconds until rsync fallback (client only)
TIMEOUT="300"
# user to run ctorrent (server only)
CTUSER="lmnsynci"

View file

@ -1,31 +0,0 @@
#!/bin/bash
#
# thomas@linuxmuster.net
# GPL v3
# 20220317
#
# linbo ctorrent helper script, started in a screen session by init script
#
torrent="$1"
[ -s "$torrent" ] || exit 1
# get ctorrent options from file
[ -e /etc/default/linbo-torrent ] && source /etc/default/linbo-torrent
[ -n "$SEEDHOURS" ] && OPTIONS="$OPTIONS -e $SEEDHOURS"
[ -n "$MAXPEERS" ] && OPTIONS="$OPTIONS -M $MAXPEERS"
[ -n "$MINPEERS" ] && OPTIONS="$OPTIONS -m $MINPEERS"
[ -n "$SLICESIZE" ] && OPTIONS="$OPTIONS -z $SLICESIZE"
[ -n "$MAXDOWN" ] && OPTIONS="$OPTIONS -D $MAXDOWN"
[ -n "$MAXUP" ] && OPTIONS="$OPTIONS -U $MAXUP"
OPTIONS="$OPTIONS $torrent"
[ -n "$CTUSER" ] && SUDO="/usr/bin/sudo -u $CTUSER"
while true; do
#$SUDO /usr/bin/ctorrent $OPTIONS || exit 1
nice -n 20 /usr/bin/ctorrent $OPTIONS || exit 1
# hash check only on initial start, add -f parameter
echo "$OPTIONS" | grep -q ^"-f " || OPTIONS="-f $OPTIONS"
done

View file

@ -1,3 +0,0 @@
%examusers ALL=(root) NOPASSWD: /usr/local/bin/link-images.sh
%role-student ALL=(root) NOPASSWD: /usr/local/bin/link-images.sh
%role-teacher ALL=(root) NOPASSWD: /usr/local/bin/link-images.sh

View file

@ -1,3 +0,0 @@
%examusers ALL=(root) NOPASSWD: /usr/local/bin/mounthome.sh
%role-student ALL=(root) NOPASSWD: /usr/local/bin/mounthome.sh
%role-teacher ALL=(root) NOPASSWD: /usr/local/bin/mounthome.sh

View file

@ -1,3 +0,0 @@
%examusers ALL=(root) NOPASSWD: /usr/local/bin/start-virtiofsd.sh
%role-student ALL=(root) NOPASSWD: /usr/local/bin/start-virtiofsd.sh
%role-teacher ALL=(root) NOPASSWD: /usr/local/bin/start-virtiofsd.sh

View file

@ -1,3 +0,0 @@
%role-teacher ALL=(lmnsynci) NOPASSWD: /usr/local/bin/sync-vm.sh
%role-student ALL=(lmnsynci) NOPASSWD: /usr/local/bin/sync-vm.sh
%examusers ALL=(lmnsynci) NOPASSWD: /usr/local/bin/sync-vm.sh

View file

@ -1 +0,0 @@
%role-teacher ALL=(root) NOPASSWD: /usr/local/bin/upload-vm.sh

17
roles/lmn_vm/files/lmn-vm Normal file
View file

@ -0,0 +1,17 @@
# vm-sync: Download and synchronize VM-Images and xml-Files
%role-teacher ALL=(lmnsynci) NOPASSWD: /usr/local/bin/vm-sync
%role-student ALL=(lmnsynci) NOPASSWD: /usr/local/bin/vm-sync
%examusers ALL=(lmnsynci) NOPASSWD: /usr/local/bin/vm-sync
# vm-aria2: Start/Stop aria2 as systemd-service for VM-Images
lmnsynci ALL=(root) NOPASSWD: /usr/local/bin/vm-aria2
# vm-link-images: Link VM-Images to User-tmp-directory
%examusers ALL=(root) NOPASSWD: /usr/local/bin/vm-link-images
%role-student ALL=(root) NOPASSWD: /usr/local/bin/vm-link-images
%role-teacher ALL=(root) NOPASSWD: /usr/local/bin/vm-link-images
# start-virtiofsd: Start Virtiofsd as systemd-service
%examusers ALL=(root) NOPASSWD: /usr/local/bin/start-virtiofsd
%role-student ALL=(root) NOPASSWD: /usr/local/bin/start-virtiofsd
%role-teacher ALL=(root) NOPASSWD: /usr/local/bin/start-virtiofsd

View file

@ -1,67 +0,0 @@
#!/usr/bin/bash
# Push VM-Disk-Image on server
set -eu
show_help() {
cat << EOF >&2
Usage: $(basename "$0") vmname"
Create torrent and upload disk, torrent and xml-VM-Definiton on server.
EOF
}
VM_DIR="/tmp/${SUDO_UID}/vm"
upload_image() {
# check if VM-Diskimage exists
if [[ ! (-f "/lmn/vm/${VM_NAME}.qcow2" || -f "${VM_DIR}/${VM_NAME}.qcow2") ]]; then
echo "File not found ${VM_NAME}.qcow2" >&2
exit 1
fi
# check if VM-Machine-Definition XML exists
if [[ ! (-f "/lmn/vm/${VM_NAME}.xml" || -f "${VM_DIR}/${VM_NAME}.xml") ]]; then
echo "File not found ${VM_NAME}.xml" >&2
exit 1
fi
sudo -u lmnsynci /usr/local/bin/vmimage-torrent stop "${VM_NAME}.qcow2" || echo "VMImage-torrent not running"
# link private VM-Diskimage to system-Dir
if [[ -f "${VM_DIR}/${VM_NAME}.qcow2" \
&& ( -f "/lmn/vm/${VM_NAME}.qcow2" && ("${VM_DIR}/${VM_NAME}.qcow2" -nt "/lmn/vm/${VM_NAME}.qcow2") \
|| ! -f "/lmn/vm/${VM_NAME}.qcow2") ]]; then
echo "copy private VM-Diskimage to system-dir"
chown lmnsynci:lmnsynci "${VM_DIR}/${VM_NAME}.qcow2"
ln -f "${VM_DIR}/${VM_NAME}.qcow2" "/lmn/vm/${VM_NAME}.qcow2"
fi
# copy private VM-Maschine-Definition XML to system-Dir
if [[ -f "${VM_DIR}/${VM_NAME}.xml" \
&& ( -f "/lmn/vm/${VM_NAME}.xml" && $(cmp -s "${VM_DIR}/${VM_NAME}.xml" "/lmn/vm/${VM_NAME}.xml") \
|| ! -f "/lmn/vm/${VM_NAME}.xml") ]]; then
echo "copy private VM-Maschine-Definition XML to system-dir"
chown lmnsynci:lmnsynci "${VM_DIR}/${VM_NAME}.xml"
cp -a "${VM_DIR}/${VM_NAME}.xml" "/lmn/vm/"
fi
cd /lmn/vm
# (re-) create torrent file
sudo -u lmnsynci /usr/local/bin/vmimage-torrent create "${VM_NAME}.qcow2"
# create size-information-file
stat -c%s "${VM_NAME}.qcow2" > "${VM_NAME}.qcow2.size"
chown lmnsynci:lmnsynci "${VM_NAME}.qcow2.size"
# Upload Torrent, qcow2 and machine-definition-XML
[[ -f "/lmn/vm/${VM_NAME}.qcow2.torrent" ]] && rsync -av --password-file=/etc/rsync.secret \
"/lmn/vm/${VM_NAME}.qcow2.torrent" rsync://vmuser@server:/vmimages-upload/
rsync -av --password-file=/etc/rsync.secret "/lmn/vm/${VM_NAME}.qcow2.size" \
rsync://vmuser@server:/vmimages-upload/
rsync -av --password-file=/etc/rsync.secret "/lmn/vm/${VM_NAME}.qcow2" \
rsync://vmuser@server:/vmimages-upload/
rsync -av --password-file=/etc/rsync.secret "/lmn/vm/${VM_NAME}.xml" \
rsync://vmuser@server:/vmimages-upload/
}
# if less than one arguments supplied, display usage
if [[ $# -ne 1 ]] ; then
show_help
exit 1
fi
VM_NAME=$1
upload_image

83
roles/lmn_vm/files/uploadseed Executable file
View file

@ -0,0 +1,83 @@
#!/usr/bin/python3
import os, sys
import subprocess
import xmlrpc.client as xc
import ssl
import argparse
parser = argparse.ArgumentParser(description='Upload a file to the bittorrent seeder.')
parser.add_argument('--rpc-server', required=True,
help='the RPC server IPaddress:port')
parser.add_argument('--rpc-secret', required=True,
help='the RPC secret')
parser.add_argument('--dht-port', required=True,
help='the DHT port the RPC server is listening on')
parser.add_argument('--no-cert', action='store_true',
help='do not use SSL certificate')
parser.add_argument('--cert', help='the certificate to use for verification')
parser.add_argument('file', help='the file to upload')
args = parser.parse_args()
rpcseeder = 'https://' + args.rpc_server + '/rpc'
secret = 'token:' + args.rpc_secret
dhtentry = args.rpc_server.split(':')[0] + ':' + args.dht_port
file2send = args.file
torrent = '/tmp/' + os.path.basename(file2send) + '.torrent'
ssl_ctx = ssl.create_default_context()
if args.no_cert:
ssl_ctx.check_hostname = False
ssl_ctx.verify_mode = ssl.CERT_NONE
print("Certificate verification disabled.")
elif args.cert is not None:
ssl_ctx.load_verify_locations(args.cert)
s = xc.ServerProxy(rpcseeder, context = ssl_ctx)
def make_torrent():
if os.path.isfile(torrent):
print("Torrent file", torrent, "exists already, please (re)move it.")
sys.exit(1)
subprocess.run(["/usr/bin/mktorrent", "-l 24", "-v", "-o", torrent, file2send], check=True)
h = subprocess.check_output(["/usr/bin/aria2c", "-S ", torrent])
for line in h.decode().splitlines():
if "Info Hash" in line:
return line.split(': ')[1]
def check_seeds(bthash):
active_seeds = s.aria2.tellActive(secret)
for seed in active_seeds:
f = seed['bittorrent']['info']['name']
gid = seed['gid']
ihash = seed['infoHash']
if f == os.path.basename(file2send):
print(file2send, "is already seeded with GID:", gid)
print("Info Hash is:", ihash)
if bthash == ihash:
print("The torrent file has not changed, exiting.")
return False
else:
print("The torrent file has changed, replacing torrent.")
s.aria2.remove(secret, gid)
return True
print("="*19, " Uploading new torrent with aria2 now. ", "="*19)
return True
def upload_torrent():
s.aria2.addTorrent(secret, xc.Binary(open(torrent, mode='rb').read()))
subprocess.run(["/usr/bin/aria2c",
"--dht-entry-point=" + dhtentry,
"--check-integrity",
"--dir=" + os.path.dirname(file2send),
torrent])
############################
if __name__ == '__main__':
infoHash = make_torrent()
if check_seeds(infoHash):
upload_torrent()
print("Upload finished.")

30
roles/lmn_vm/files/vm-aria2 Executable file
View file

@ -0,0 +1,30 @@
#!/usr/bin/bash
set -eu
# if less than one arguments supplied, display usage
if [[ $# -ne 2 ]]; then
echo "This script takes as input the name of the VM " >&2
echo "Usage: $0 [start|stop] vm_name" >&2
exit 1
fi
COMMAND="$1"
VM_NAME="$2"
source /etc/lmn/vm.conf
if [[ "${COMMAND}" = "start" ]]; then
systemd-run --unit=aria2-"${VM_NAME}" \
--slice=system-aria2 \
--uid="$(id -u lmnsynci)" \
--gid="$(id -g lmnsynci)" \
--nice=19 \
--working-directory="${VM_SYSDIR}" \
--collect \
--property=Type=exec \
--property=SuccessExitStatus=1 \
aria2c --bt-hash-check-seed=true --check-integrity=true --seed-ratio=0.0 --dht-entry-point="${SEEDBOX_HOST}:${SEEDBOX_PORT}" "${VM_SYSDIR}/${VM_NAME}.qcow2.torrent"
elif [[ "${COMMAND}" = "stop" ]]; then
systemctl stop "aria2-${VM_NAME}.service" || echo "Aria2-Service not running"
fi

View file

@ -2,6 +2,20 @@
# create 1st level-Clones
set -eu
source /etc/lmn/vm.conf
while getopts ':p' OPTION; do
case "$OPTION" in
p)
PERSISTENT=1
VM_DIR="${VM_DIR_PERSISTENT}"
;;
esac
done
shift "$((OPTIND -1))"
# if less than two arguments supplied, display usage
if [[ $# -ne 2 ]]; then
echo "This script takes as input the name of the VM to clone" >&2
@ -11,16 +25,19 @@ fi
VM_NAME=$1
VM_CLONE=$2
VM_DIR="/tmp/${UID}/vm"
# Create User-VM-Dir and link system VM-Images
[[ -d "${VM_DIR}" ]] || mkdir -p "${VM_DIR}"
sudo /usr/local/bin/link-images.sh
if [[ "${PERSISTENT}" -eq 1 ]]; then
sudo /usr/local/bin/vm-link-images -p
else
sudo /usr/local/bin/vm-link-images
fi
# change to image-directory
cd "${VM_DIR}"
if { [[ ! -f "${VM_NAME}.xml" ]] && [[ ! -f "/lmn/vm/${VM_NAME}.xml" ]]; } || [[ ! -f "${VM_NAME}.qcow2" ]]; then
if { [[ ! -f "${VM_NAME}.xml" ]] && [[ ! -f "${VM_SYSDIR}/${VM_NAME}.xml" ]]; } || [[ ! -f "${VM_NAME}.qcow2" ]]; then
echo "xml or qcow2 File does not exists." >&2
exit 1
fi
@ -32,8 +49,8 @@ chmod a-w "${VM_NAME}-${VM_CLONE}.qcow2"
# copy machine-definition-file
if [[ -f "${VM_NAME}.xml" ]]; then
cp "${VM_NAME}.xml" "${VM_NAME}-${VM_CLONE}.xml"
elif [[ -f "/lmn/vm/${VM_NAME}.xml" ]]; then
cp "/lmn/vm/${VM_NAME}.xml" "${VM_NAME}-${VM_CLONE}.xml"
elif [[ -f "${VM_SYSDIR}/${VM_NAME}.xml" ]]; then
cp "${VM_SYSDIR}/${VM_NAME}.xml" "${VM_NAME}-${VM_CLONE}.xml"
else
echo "no machine definition file found" >&2
exit 1

View file

@ -1,12 +1,22 @@
#!/usr/bin/bash
# link VM in Use-Dir in /tmp
# link VM in User-Dir in /tmp or /var/vm
set -eu
# change to image-directory
cd /lmn/vm
source /etc/lmn/vm.conf
VM_DIR="/tmp/${SUDO_UID}/vm"
# change to image-directory
cd "${VM_SYSDIR}"
while getopts ':p' OPTION; do
case "$OPTION" in
p)
VM_DIR="${VM_DIR_PERSISTENT}"
;;
esac
done
shift "$((OPTIND -1))"
# link system-VM-Images to User VM Directory
for i in *.qcow2; do

View file

@ -10,11 +10,16 @@ This script takes as input the name of the VM to rebase one level down
EOF
}
while getopts ':n:' OPTION; do
source /etc/lmn/vm.conf
while getopts ':n:p' OPTION; do
case "$OPTION" in
n)
NEWNAME=$OPTARG
;;
p)
VM_DIR="${VM_DIR_PERSISTENT}"
;;
?)
show_help
exit 1
@ -31,7 +36,6 @@ if [[ $# -ne 1 ]]; then
fi
# change to Images directory
VM_DIR="/tmp/${UID}/vm"
cd "${VM_DIR}"
VM_NAME="$1"
@ -70,8 +74,8 @@ if [[ -v NEWNAME ]]; then
CURRENTNAME="${CURRENTBASE/.qcow2/}"
if [[ -f "${CURRENTNAME}.xml" ]]; then
cp "${CURRENTNAME}.xml" "${NEWNAME}.xml"
elif [[ -f "/lmn/vm/${CURRENTNAME}.xml" ]]; then
cp "/lmn/vm/${CURRENTNAME}.xml" "${NEWNAME}.xml"
elif [[ -f "${VM_SYSDIR}/${CURRENTNAME}.xml" ]]; then
cp "${VM_SYSDIR}/${CURRENTNAME}.xml" "${NEWNAME}.xml"
else
echo "no machine definition file found" >&2
exit 1

View file

@ -8,8 +8,8 @@ show_help() {
Usage: $(basename "$0") [-n] vmname"
Create a new clone, start the vm (if not yet running) and run virt-viewer.
Squid-Proxy will be started too.
### remove, old ### User Home will be mounted on /media/USERNAME/home
-n new clone will be created, even if exists
-p new clone will be created persistent, so available after reboot too
-s qemu:///system instead of default qemu:///session
EOF
}
@ -23,31 +23,34 @@ exit_script() {
check_images() {
# sync vm-torrents and machine definition file
sudo -u lmnsynci /usr/local/bin/sync-vm.sh -t
sudo -u lmnsynci /usr/local/bin/vm-sync get_file "${VM_NAME}.xml" "${VM_NAME}.qcow2.torrent"
[[ -f "${VM_NAME}" ]] && sudo -u lmnsynci /usr/local/bin/vm-sync delete_outdated_image "${VM_NAME}.qcow2"
BACKINGARRAY=()
imgfile="/lmn/vm/${VM_NAME}.qcow2" && [[ -f "${VM_DIR}/${VM_NAME}.qcow2" ]] && imgfile="${VM_DIR}/${VM_NAME}.qcow2"
imgfile="${VM_SYSDIR}/${VM_NAME}.qcow2" && [[ -f "${VM_DIR}/${VM_NAME}.qcow2" ]] && imgfile="${VM_DIR}/${VM_NAME}.qcow2"
BACKINGARRAY+=("${imgfile}")
echo "Imgfile=$imgfile"
if [[ ! -f "${imgfile}" ]] || ! qemu-img info -U "${imgfile}" | grep "file format: qcow2"; then
if [[ ! -f "/lmn/vm/${VM_NAME}.qcow2.torrent" ]]; then
if [[ ! -f "${VM_SYSDIR}/${VM_NAME}.qcow2.torrent" ]]; then
echo "no base VM disk '${VM_NAME}.qcow2' found and/or ${VM_NAME} not found on server" >&2
exit 1
fi
# sync vm-disk image by torrent
echo "Try to sync VM ${VM_NAME} by torrent"
sudo -u lmnsynci /usr/local/bin/sync-vm.sh "${VM_NAME}"
sudo -u lmnsynci /usr/local/bin/vm-sync get_image "${VM_NAME}"
fi
backingfile=$(qemu-img info -U "${imgfile}" | grep "^backing file:" | cut -d ' ' -f 3)
while [[ ! -z "${backingfile}" ]]; do
while [[ -n "${backingfile}" ]]; do
echo "Backingfile required: ${backingfile}"
imgfile="/lmn/vm/${backingfile}" && [[ -f "${VM_DIR}/${backingfile}" ]] && imgfile="${VM_DIR}/${backingfile}"
imgfile="${VM_SYSDIR}/${backingfile}" && [[ -f "${VM_DIR}/${backingfile}" ]] && imgfile="${VM_DIR}/${backingfile}"
BACKINGARRAY+=("${imgfile}")
if [[ ! -f "${imgfile}" ]] || ! qemu-img info -U "${imgfile}" | grep "file format: qcow2"; then
# sync vm-disk image by torrent
echo "Try to sync backingfile ${backingfile} by torrent"
sudo -u lmnsynci /usr/local/bin/sync-vm.sh "${backingfile%.qcow2}"
sudo -u lmnsynci /usr/local/bin/vm-sync get_file "${backingfile}.torrent"
[[ -f "${backingfile}" ]] && sudo -u lmnsynci /usr/local/bin/vm-sync delete_outdated_image "${backingfile}"
sudo -u lmnsynci /usr/local/bin/vm-sync get_image "${backingfile%.qcow2}"
fi
backingfile=$(qemu-img info -U "${imgfile}" | grep "^backing file:" | cut -d ' ' -f 3)
done
@ -62,27 +65,29 @@ check_images() {
if ! qemu-img check -U "${BACKINGARRAY[$i]}" 2>/dev/null; then
echo "check failed!"
echo "sync ${BACKINGARRAY[$i]} again"
sudo -u lmnsynci /usr/local/bin/sync-vm.sh $(basename "${BACKINGARRAY[$i]}" .qcow2)
sudo -u lmnsynci /usr/local/bin/vm-sync get_image "$(basename "${BACKINGARRAY[$i]}" .qcow2)"
fi
done
echo "VM-Image and required backingfiles available and checked"
}
create-clone() {
create_clone() {
local VM_NAME="$1"
local VM_DIR="/tmp/${UID}/vm"
local VM_XML="${VM_DIR}/${VM_NAME}-clone.xml"
local VM_SYSDIR="/lmn/vm"
if ! [[ -f "$VM_SYSDIR/${VM_NAME}.xml" && -f "$VM_SYSDIR/${VM_NAME}.qcow2" ]] && ! [[ -f "${VM_DIR}/${VM_NAME}.xml" && -f "${VM_DIR}/${VM_NAME}.qcow2" ]]; then
if ! [[ -f "${VM_SYSDIR}/${VM_NAME}.xml" && -f "${VM_SYSDIR}/${VM_NAME}.qcow2" ]] && ! [[ -f "${VM_DIR}/${VM_NAME}.xml" && -f "${VM_DIR}/${VM_NAME}.qcow2" ]]; then
echo "xml or qcow2 File does not exists." >&2
exit 1
fi
# Create User-VM-Dir and link system VM-Images
[[ -d "${VM_DIR}" ]] || mkdir -p "${VM_DIR}"
sudo /usr/local/bin/link-images.sh
if [[ "${PERSISTENT}" -eq 1 ]]; then
sudo /usr/local/bin/vm-link-images -p
else
sudo /usr/local/bin/vm-link-images
fi
# Create backing file
cd "${VM_DIR}"
@ -107,9 +112,16 @@ create-clone() {
QEMU='qemu:///session'
NEWCLONE=0
PERSISTENT=0
while getopts ':ns' OPTION; do
source /etc/lmn/vm.conf
while getopts ':pns' OPTION; do
case "$OPTION" in
p)
PERSISTENT=1
VM_DIR="${VM_DIR_PERSISTENT}"
;;
n)
NEWCLONE=1
;;
@ -132,7 +144,6 @@ if [[ $# -ne 1 ]] ; then
fi
VM_NAME=$1
VM_DIR="/tmp/${UID}/vm"
# check, if we have to start squid
if ! killall -s 0 squid; then
@ -147,14 +158,14 @@ if ! virsh --connect="${QEMU}" list | grep "${VM_NAME}-clone"; then
echo "VM not yet running."
check_images
if [[ "${NEWCLONE}" = 1 ]] || [[ ! -f "${VM_DIR}/${VM_NAME}-clone.qcow2" ]]; then
create-clone "${VM_NAME}"
create_clone "${VM_NAME}"
fi
# delete the old vm
virsh --connect=qemu:///session undefine "${VM_NAME}-clone" || echo "${VM_NAME}-clone did not exist"
# finally, create the new vm
virsh --connect=qemu:///session define "${VM_DIR}/${VM_NAME}-clone.xml"
#trap exit_script SIGHUP SIGINT SIGTERM
[[ "${QEMU}" = 'qemu:///session' ]] && sudo /usr/local/bin/start-virtiofsd.sh "${VM_NAME}"
[[ "${QEMU}" = 'qemu:///session' ]] && sudo /usr/local/bin/vm-virtiofsd "${VM_NAME}"
virsh --connect="${QEMU}" start "${VM_NAME}-clone"
fi
echo "starting viewer"

131
roles/lmn_vm/files/vm-sync Executable file
View file

@ -0,0 +1,131 @@
#!/usr/bin/bash
# Push/Pull VM-Disk-Image and Infos from server
set -eu
show_help() {
cat << EOF >&2
Usage: $(basename "$0") command [args]"
command:
push_file
get_file
get_image
delete_outdated_image
EOF
}
get_torrent() {
if [[ ! -f "${VM_SYSDIR}/${VM_NAME}.qcow2.torrent" ]]; then
echo "No torrent-File found"
exit 1
fi
lockfile="/tmp/sync-vm-${VM_NAME}.lock"
if ! flock -n "$lockfile" echo "try to acquire lock"; then
echo torrent seems to be in process.
echo waiting for completion ...
flock -w 3600 "$lockfile" echo "...completed"
sleep 5
else
(
if ! flock -n 200; then
echo "failed to acquire lock"
echo "Bitte noch einmal starten."
echo "Beliebige Taste zum Beenden."
read -n 1
exit 1
fi
# stop aria2-seeding if running
sudo vm-aria2 stop "${VM_NAME}"
cd "${VM_SYSDIR}"
# get image
aria2c --seed-time=0 --dht-entry-point="${SEEDBOX_HOST}:${SEEDBOX_PORT}" "${VM_SYSDIR}/${VM_NAME}.qcow2.torrent"
# and seed
sudo vm-aria2 start "${VM_NAME}"
if ! flock -u 200; then
echo failed to drop lock
exit 1
fi
) 200>"$lockfile"
fi
}
get_image_size() {
torrentfile=$1
length=$(aria2c -S "${torrentfile}" | grep "Total Length" | grep "Total Length" | sed -E 's/.*\(([0-9,]*)\)/\1/' | sed s/,//g)
echo "$length"
}
delete_outdated_image() {
cd "${VM_SYSDIR}"
qcowsize=$(stat -c%s "${FILENAME}")
if [[ -f "${FILENAME}.torrent" ]] && [[ "${qcowsize}" != $(get_image_size "${FILENAME}.torrent") ]]; then
sudo vm-aria2 stop "${FILENAME%.qcow2}"
rm -f "${FILENAME}"
fi
}
get_file() {
cd "${VM_SYSDIR}"
wget --no-proxy -O "${FILENAME}" "http://${SEEDBOX_HOST}/aria2/${FILENAME}" || echo "File not found on seedbox"
}
push_file() {
cd "${VM_SYSDIR}"
uploadseed --rpc-server "${SEEDBOX_HOST}:${SEEDBOX_RPC_PORT}" --dht-port "${SEEDBOX_PORT}" --rpc-secret insecure --no-cert "${FILENAME}"
}
if [[ "$(id -nu)" != "lmnsynci" ]]; then
echo "$(basename "$0") must be run as lmnsynci user"
show_help
exit 1
fi
source /etc/lmn/vm.conf
while getopts ':' OPTION; do
case "$OPTION" in
?)
show_help
exit 1
;;
esac
done
shift "$((OPTIND -1))"
# if less than one arguments supplied, display usage
if [[ $# -lt 1 ]]; then
show_help
exit 1
fi
command=$1
shift
case "$command" in
push_file)
for FILENAME in "$@"; do
push_file
done
;;
get_file)
for FILENAME in "$@"; do
get_file
done
;;
get_image)
for VM_NAME in "$@"; do
get_torrent
done
;;
delete_outdated_image)
for FILENAME in "$@"; do
delete_outdated_image
done
;;
*)
show_help
exit 1
;;
esac

70
roles/lmn_vm/files/vm-upload Executable file
View file

@ -0,0 +1,70 @@
#!/usr/bin/bash
# Push VM-Disk-Image on server
set -eu
show_help() {
cat << EOF >&2
Usage: $(basename "$0") vmname"
Create torrent and upload disk and xml-VM-Definiton on server.
EOF
}
upload_image() {
# check if VM-Diskimage exists
if [[ ! (-f "${VM_SYSDIR}/${VM_NAME}.qcow2" || -f "${VM_DIR}/${VM_NAME}.qcow2") ]]; then
echo "File not found ${VM_NAME}.qcow2" >&2
exit 1
fi
# check if VM-Machine-Definition XML exists
if [[ ! (-f "${VM_SYSDIR}/${VM_NAME}.xml" || -f "${VM_DIR}/${VM_NAME}.xml") ]]; then
echo "File not found ${VM_NAME}.xml" >&2
exit 1
fi
sudo vm-aria2 stop "${VM_NAME}" || echo "VMImage-torrent not running"
# link private VM-Diskimage to system-Dir
if [[ -f "${VM_DIR}/${VM_NAME}.qcow2" \
&& ( -f "${VM_SYSDIR}/${VM_NAME}.qcow2" && ("${VM_DIR}/${VM_NAME}.qcow2" -nt "${VM_SYSDIR}/${VM_NAME}.qcow2") \
|| ! -f "${VM_SYSDIR}/${VM_NAME}.qcow2") ]]; then
echo "copy private VM-Diskimage to system-dir"
chown lmnsynci:lmnsynci "${VM_DIR}/${VM_NAME}.qcow2"
ln -f "${VM_DIR}/${VM_NAME}.qcow2" "${VM_SYSDIR}/${VM_NAME}.qcow2"
fi
# copy private VM-Maschine-Definition XML to system-Dir
if [[ -f "${VM_DIR}/${VM_NAME}.xml" \
&& ( -f "${VM_SYSDIR}/${VM_NAME}.xml" && $(cmp -s "${VM_DIR}/${VM_NAME}.xml" "${VM_SYSDIR}/${VM_NAME}.xml") \
|| ! -f "${VM_SYSDIR}/${VM_NAME}.xml") ]]; then
echo "copy private VM-Maschine-Definition XML to system-dir"
chown lmnsynci:lmnsynci "${VM_DIR}/${VM_NAME}.xml"
cp -a "${VM_DIR}/${VM_NAME}.xml" "${VM_SYSDIR}"
fi
cd "${VM_SYSDIR}"
uploadseed --rpc-server "${SEEDBOX_HOST}:${SEEDBOX_RPC_PORT}" --dht-port "${SEEDBOX_PORT}" --rpc-secret insecure --no-cert "${VM_NAME}.qcow2"
uploadseed --rpc-server "${SEEDBOX_HOST}:${SEEDBOX_RPC_PORT}" --dht-port "${SEEDBOX_PORT}" --rpc-secret insecure --no-cert "${VM_NAME}.xml"
}
source /etc/lmn/vm.conf
while getopts ':p' OPTION; do
case "$OPTION" in
p)
VM_DIR="${VM_DIR_PERSISTENT}"
;;
?)
show_help
exit 1
;;
esac
done
shift "$((OPTIND -1))"
# if less than one arguments supplied, display usage
if [[ $# -ne 1 ]] ; then
show_help
exit 1
fi
VM_NAME=$1
upload_image

View file

@ -0,0 +1,14 @@
# variables for LMN VM submodule
SEEDBOX_HOST=seedbox.pn.steinbeis.schule
SEEDBOX_PORT=6789
SEEDBOX_RPC_PORT=6800
VM_SYSDIR="/lmn/vm"
if [[ -v SUDO_UID ]]; then
VM_DIR="/tmp/${SUDO_UID}/vm"
VM_DIR_PERSISTENT="/var/vm/${SUDO_UID}"
else
VM_DIR="/tmp/${UID}/vm"
VM_DIR_PERSISTENT="/var/vm/${UID}"
fi

View file

@ -1,213 +0,0 @@
#!/bin/bash
#
# starts tmux sessions for each valid torrent in LINBODIR
# thomas@linuxmuster.net
# 20221103
#
# read environment
#. /usr/share/linuxmuster/defaults.sh || exit 1
#THELPER=$LINBOSHAREDIR/linbo-torrenthelper.sh
THELPER=linbo-torrenthelper.sh
#. $LINBOSHAREDIR/helperfunctions.sh || exit 1
LINBOIMGEXT="qcow2 qdiff"
LINBOIMGDIR="/lmn/vm"
serverip="10.190.1.1"
# start of functions
# help message
usage(){
echo
echo "Info: vmimage-torrent manages the torrent tmux sessions of linbo images."
echo
echo "Usage:"
echo " vmimage-torrent <start|stop|restart|reload|status|create|check> [image_name]"
echo " vmimage-torrent attach <image_name|session_name>"
echo
echo "Note:"
echo " * Only qcow2 & qdiff image files located below $LINBOIMGDIR are processed."
echo " * The commands \"start\", \"stop\" and \"restart\" may have optionally an image"
echo " filename as parameter. In this case the commands are only applied to the tmux"
echo " session of the certain file. Without an explicit image filename the commands"
echo " were applied to all image file sessions currently running."
echo " * An image filename parameter is mandatory with the commands \"check\", \"create\""
echo " and \"attach\"."
echo " * \"check\" checks if the image file matches to the correspondig torrent."
echo " * \"create\" creates/recreates the torrent of a certain image file."
echo " * \"status\" shows a list of currently running torrent tmux sessions."
echo " * \"attach\" attaches a torrent tmux session of a certain image. An image or"
echo " session name must be given as parameter."
echo " Press [CTRL+B]+[D] to detach the session again."
echo " * \"reload\" is the identical to \"restart\" and is there for backwards compatibility."
echo
exit 1
}
# check torrent
check(){
local image="$(basename "$IMGLIST")"
local torrent="$image.torrent"
local tdir="$(dirname "$IMGLIST")"
cd "$tdir"
echo "Checking $torrent ..."
if ctorrent -c "$torrent"; then
echo "Ok!"
else
echo "Failed!"
exit 1
fi
}
# creates torrent files
create(){
local image="$(basename "$IMGLIST")"
local tdir="$(dirname "$IMGLIST")"
local torrent="${image}.torrent"
local session="${torrent//./_}"
# stop torrent service
vmimage-torrent status | grep -q ^"$session" && vmimage-torrent stop "$IMGLIST"
# skip already running torrents
echo "Creating $torrent ..."
cd "$tdir"
rm -f "$torrent"
if ctorrent -t -u "http://$serverip:6969/announce" -s "$torrent" "$image" ; then
[ "$START" = "no" ] || vmimage-torrent start "$IMGLIST"
else
echo "Failed!"
exit 1
fi
}
# starts torrent tmux sessions
start(){
local item
local torrent
local image
local tdir
local session
for item in $IMGLIST; do
image="$(basename "$item")"
torrent="${image}.torrent"
tdir="$(dirname "$item")"
session="${torrent//./_}"
cd "$tdir"
if [ ! -s "$image" ]; then
echo "Image $image does not exist! Skipping this torrent."
continue
fi
# skip already running torrents
if vmimage-torrent status | grep -qw ^"$session"; then
echo "tmux session $session is already running."
continue
fi
# create torrent file if there is none
if [ ! -e "$torrent" ]; then
START="no" vmimage-torrent create "$item" || continue
fi
echo -n "Starting tmux session $session ... "
tmux new -ds "$session" "$THELPER $torrent ; exec $SHELL"
sleep 1
if vmimage-torrent status | grep -qw ^"$session"; then
echo "Ok!"
else
echo "Failed!"
fi
done
}
stop(){
if [ -n "$SESSION" ]; then
vmimage-torrent status | grep -qw ^"$SESSION" || return
tmux kill-session -t "$SESSION"
else
local item
vmimage-torrent status | awk -F\: '{print $1}' | while read item; do
tmux kill-session -t "$item"
done
fi
}
attach(){
if ! tmux list-sessions | grep -qw "$SESSION"; then
echo "There is no session $SESSION."
exit 1
fi
echo "Hint: Detach tmux session with [CTRL+B]+[D]."
sleep 3
tmux attach -t "$SESSION"
}
status(){
tmux list-sessions | grep _torrent
}
find_images(){
local search="$(basename "$1")"
if [ -n "$search" ]; then
find "$LINBOIMGDIR" -maxdepth 2 -name "$search"
return
fi
local IMGLIST
for search in $LINBOIMGEXT; do
IMGLIST="$IMGLIST $(find "$LINBOIMGDIR" -maxdepth 2 -name \*.$search)"
done
# trim leading and trailing spaces
echo $IMGLIST | awk '{$1=$1};1'
}
# end of functions
# check parameters
if [ -n "$2" ] ; then
# trap torrent parameter
image="${2/.torrent/}"
case "$image" in
*.qcow2|*.qdiff)
if [ -e "$image" ]; then
IMGLIST="$image"
else
IMGLIST="$(find_images "$image")"
fi
if [ ! -e "$IMGLIST" ]; then
echo "Image file $(basename $image) not found."
usage
fi
filename="$(basename "$IMGLIST")"
SESSION="${filename//./_}_torrent"
;;
*_torrent)
if [ "$1" = "attach" ]; then
SESSION="$image"
else
usage
fi
;;
*) usage ;;
esac
else
case "$1" in
stop|status) ;;
attach|check|create) usage ;;
*)
IMGLIST="$(find_images)"
if [ -z "$IMGLIST" ]; then
echo "No linbo images found."
exit 0
fi
;;
esac
fi
case "$1" in
start) start ;;
stop) stop ;;
restart|reload) stop ; start ;;
status) status ;;
create) create ;;
check) check ;;
attach) attach ;;
*) usage ;;
esac
exit 0

View file

@ -1,15 +0,0 @@
[Unit]
Description=VM-image torrent service
After=network.target
[Service]
Type=oneshot
RemainAfterExit=yes
User=lmnsynci
Group=lmnsynci
ExecStart=/usr/local/bin/vmimage-torrent start
ExecStop=/usr/local/bin/vmimage-torrent stop
ExecReload=/usr/local/bin/vmimage-torrent reload
[Install]
WantedBy=multi-user.target

View file

@ -13,7 +13,8 @@
- name: install libvirt packages
apt:
name:
- ctorrent
- aria2
- mktorrent
- libvirt-daemon-system
- virt-manager
state: latest
@ -80,6 +81,11 @@
system: true
create_home: false
- name: Create /etc/lmn directory
file:
path: /etc/lmn
state: directory
- name: Create /lmn directory
file:
path: /lmn
@ -91,6 +97,12 @@
state: directory
mode: '1777'
- name: Create /var/vm directory
file:
path: /var/vm
state: directory
mode: '1777'
- name: Create vm directory
file:
path: /lmn/vm
@ -127,10 +139,7 @@
mode: '0700'
loop:
- lmn-mounthome
- lmn-sync-vm
- lmn-upload-vm
- lmn-link-images
- lmn-startvirtiofsd
- lmn-vm
- name: Deploy vmimages scripts
copy:
@ -141,33 +150,24 @@
mode: '0755'
loop:
- mounthome.sh
- create-vm.sh
- rebase-vm.sh
- run-vm.sh
- upload-vm.sh
- sync-vm.sh
- link-images.sh
- start-virtiofsd.sh
- linbo-torrenthelper.sh
- vmimage-torrent
- vm-create
- vm-rebase
- vm-run
- vm-upload
- vm-sync
- vm-link-images
- vm-virtiofsd
- virtiofsd
- vm-aria2
- uploadseed
- name: Deploy linbo-torrent defaults
- name: Deploy vm configuration file vm.conf
copy:
src: linbo-torrent
dest: /etc/default/
src: vm.conf
dest: /etc/lmn/vm.conf
owner: root
group: root
mode: '0755'
- name: Deploy vmimage-torrent.service
copy:
src: vmimage-torrent.service
dest: /etc/systemd/system/
owner: root
group: root
mode: '0644'
notify: "enable vmimage-torrent.service"
- name: Prepare directory for qemu bridge config
ansible.builtin.file: