sync-vm with torrent support
This commit is contained in:
Raphael Dannecker
parent
50cf61844c
commit
399c3d0d66
7 changed files with 329 additions and 12 deletions
|
|
@ -33,6 +33,7 @@
|
|||
- krb5-user
|
||||
- unattended-upgrades
|
||||
- debconf-utils
|
||||
- ctorrent
|
||||
extra_pkgs_bpo: [] # [ linux-image-amd64 ]
|
||||
ansible_python_interpreter: "/usr/bin/python3"
|
||||
|
||||
|
|
|
|||
33
roles/lmn_vm/files/linbo-torrent
Normal file
33
roles/lmn_vm/files/linbo-torrent
Normal file
|
|
@ -0,0 +1,33 @@
|
|||
# default values for linbo-torrenthelper service provided by ctorrent
|
||||
# thomas@linuxmuster.net
|
||||
# 20220317
|
||||
#
|
||||
# note: you have to invoke 'linbo-torrent restart' after you have changed any values
|
||||
#
|
||||
|
||||
# Exit while seed <SEEDHOURS> hours later (default 72 hours)
|
||||
SEEDHOURS="100000"
|
||||
|
||||
# Max peers count (default 100)
|
||||
MAXPEERS="100"
|
||||
|
||||
# Min peers count (default 1)
|
||||
MINPEERS="1"
|
||||
|
||||
# Download slice/block size, unit KB (default 16, max 128)
|
||||
SLICESIZE="128"
|
||||
|
||||
# Max bandwidth down (unit KB/s, default unlimited)
|
||||
MAXDOWN=""
|
||||
|
||||
# Max bandwidth up (unit KB/s, default unlimited)
|
||||
MAXUP=""
|
||||
|
||||
# Supplemental ctorrent options, separated by space (-v: Verbose output for debugging)
|
||||
#OPTIONS="-v"
|
||||
|
||||
# Timeout in seconds until rsync fallback (client only)
|
||||
TIMEOUT="300"
|
||||
|
||||
# user to run ctorrent (server only)
|
||||
CTUSER="nobody"
|
||||
31
roles/lmn_vm/files/linbo-torrenthelper.sh
Executable file
31
roles/lmn_vm/files/linbo-torrenthelper.sh
Executable file
|
|
@ -0,0 +1,31 @@
|
|||
#!/bin/bash
|
||||
#
|
||||
# thomas@linuxmuster.net
|
||||
# GPL v3
|
||||
# 20220317
|
||||
#
|
||||
# linbo ctorrent helper script, started in a screen session by init script
|
||||
#
|
||||
|
||||
torrent="$1"
|
||||
[ -s "$torrent" ] || exit 1
|
||||
|
||||
# get ctorrent options from file
|
||||
[ -e /etc/default/linbo-torrent ] && source /etc/default/linbo-torrent
|
||||
[ -e /home/raphael/git/fvsclient/etc/default/linbo-torrent ] && source /home/raphael/git/fvsclient/etc/default/linbo-torrent
|
||||
|
||||
[ -n "$SEEDHOURS" ] && OPTIONS="$OPTIONS -e $SEEDHOURS"
|
||||
[ -n "$MAXPEERS" ] && OPTIONS="$OPTIONS -M $MAXPEERS"
|
||||
[ -n "$MINPEERS" ] && OPTIONS="$OPTIONS -m $MINPEERS"
|
||||
[ -n "$SLICESIZE" ] && OPTIONS="$OPTIONS -z $SLICESIZE"
|
||||
[ -n "$MAXDOWN" ] && OPTIONS="$OPTIONS -D $MAXDOWN"
|
||||
[ -n "$MAXUP" ] && OPTIONS="$OPTIONS -U $MAXUP"
|
||||
OPTIONS="$OPTIONS $torrent"
|
||||
|
||||
[ -n "$CTUSER" ] && SUDO="/usr/bin/sudo -u $CTUSER"
|
||||
|
||||
while true; do
|
||||
$SUDO /usr/bin/ctorrent $OPTIONS || exit 1
|
||||
# hash check only on initial start, add -f parameter
|
||||
echo "$OPTIONS" | grep -q ^"-f " || OPTIONS="-f $OPTIONS"
|
||||
done
|
||||
|
|
@ -10,7 +10,7 @@ Create a new clone, start the vm (if not yet running) and run virt-viewer.
|
|||
Squid-Proxy will be started too.
|
||||
User Home will be mounted on /media/USERNAME/home
|
||||
-n new clone will be created, even if exists
|
||||
-s qemu:///session instead of default qemu:///system
|
||||
-s qemu:///system instead of default qemu:///session
|
||||
EOF
|
||||
}
|
||||
|
||||
|
|
@ -21,7 +21,7 @@ exit_script() {
|
|||
kill -- -$$ # Sends SIGTERM to child/sub processes
|
||||
}
|
||||
|
||||
QEMU='qemu:///system'
|
||||
QEMU='qemu:///session'
|
||||
|
||||
NEWCLONE=0
|
||||
|
||||
|
|
@ -31,7 +31,7 @@ while getopts ':ns' OPTION; do
|
|||
NEWCLONE=1
|
||||
;;
|
||||
s)
|
||||
QEMU='qemu:///session'
|
||||
QEMU='qemu:///system'
|
||||
;;
|
||||
?)
|
||||
show_help
|
||||
|
|
@ -49,14 +49,15 @@ if [[ $# -ne 1 ]] ; then
|
|||
fi
|
||||
|
||||
VM_NAME=$1
|
||||
VM_DIR="/tmp/${UID}/vmimages"
|
||||
|
||||
if [[ ! -f "/var/lib/libvirt/images/${VM_NAME}.qcow2" ]]; then
|
||||
if [[ ! -f "/var/lib/libvirt/images/${VM_NAME}.qcow2" && ! -f "${VM_DIR}/${VM_NAME}.qcow2" ]]; then
|
||||
echo "no base VM disk '${VM_NAME}.qcow2' found" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# check, if we have to start squid
|
||||
if [[ ! -f "/tmp/squid.pid" ]]; then
|
||||
if ! killall -s 0 squid; then
|
||||
echo "starting squid."
|
||||
/usr/sbin/squid -f /etc/squid/squid-usermode.conf
|
||||
fi
|
||||
|
|
@ -68,7 +69,6 @@ if ! findmnt "/media/${USER}/home"; then
|
|||
fi
|
||||
|
||||
export XDG_CONFIG_HOME="/tmp/${UID}/.config"
|
||||
VM_DIR="/tmp/${UID}/vmimages"
|
||||
|
||||
if ! virsh --connect="${QEMU}" list | grep "${VM_NAME}-clone"; then
|
||||
echo "VM not yet running. Try to clone and start."
|
||||
|
|
|
|||
|
|
@ -4,23 +4,42 @@ set -eu
|
|||
|
||||
show_help() {
|
||||
cat << EOF >&2
|
||||
Usage: $(basename "$0") [-u vmname] [-d vmname] [-a]"
|
||||
Usage: $(basename "$0") [-u vmname] [-d vmname] [-a] [-t]"
|
||||
When using option -u (upload), the disk from VM vmname will be synced on server.
|
||||
Otherwise the images from images.list and xml-directory will be synced from server.
|
||||
Using flag -t all torrents and xml-VM-Definitions will be synced
|
||||
EOF
|
||||
}
|
||||
|
||||
VM_DIR="/tmp/${SUDO_UID}/vmimages"
|
||||
|
||||
upload_image() {
|
||||
# check if VM-Diskimage exists
|
||||
if [[ ! -f "/var/lib/libvirt/images/${VM_NAME}.qcow2" ]]; then
|
||||
if [[ ! (-f "/var/lib/libvirt/images/${VM_NAME}.qcow2" || -f "${VM_DIR}/${VM_NAME}.qcow2") ]]; then
|
||||
echo "File not found ${VM_NAME}.qcow2" >&2
|
||||
exit 1
|
||||
fi
|
||||
# link private VM-Diskimage to system-Dir
|
||||
if [[ -f "${VM_DIR}/${VM_NAME}.qcow2" \
|
||||
&& ( -f "/var/lib/libvirt/images/${VM_NAME}.qcow2" && ("${VM_DIR}/${VM_NAME}.qcow2" -nt "/var/lib/libvirt/images/${VM_NAME}.qcow2") \
|
||||
|| ! -f "/var/lib/libvirt/images/${VM_NAME}.qcow2") ]]; then
|
||||
echo "copy private VM-Diskimage to system-dir"
|
||||
ln -f "${VM_DIR}/${VM_NAME}.qcow2" "/var/lib/libvirt/images/${VM_NAME}.qcow2"
|
||||
fi
|
||||
# check if VM-Machine-Definition XML exists
|
||||
if [[ ! -f "/var/lib/libvirt/images/xml/${VM_NAME}.xml" ]]; then
|
||||
if [[ ! (-f "/var/lib/libvirt/images/xml/${VM_NAME}.xml" || -f "${VM_DIR}/xml/${VM_NAME}") ]]; then
|
||||
echo "File not found ${VM_NAME}.xml" >&2
|
||||
exit 1
|
||||
fi
|
||||
# copy private VM-Maschine-Definition XML to system-Dir
|
||||
if [[ -f "${VM_DIR}/xml/${VM_NAME}.xml" \
|
||||
&& ( -f "/var/lib/libvirt/images/xml/${VM_NAME}.xml" && $(cmp -s "${VM_DIR}/xml/${VM_NAME}.xml" "/var/lib/libvirt/images/xml/${VM_NAME}.xml") \
|
||||
|| ! -f "/var/lib/libvirt/images/xml/${VM_NAME}.xml") ]]; then
|
||||
echo "copy private VM-Maschine-Definition XML to system-dir"
|
||||
cp "${VM_DIR}/${VM_NAME}.xml" "/var/lib/libvirt/images/${VM_NAME}.xml"
|
||||
fi
|
||||
[[ -f "/var/lib/libvirt/images/${VMNAME}.qcow2.torrent" ]] && rsync -av --password-file=/etc/rsync.secret \
|
||||
"/var/lib/libvirt/images/${VM_NAME}.qcow2" rsync://vmuser@server:/vmimages-upload/
|
||||
rsync -av --password-file=/etc/rsync.secret "/var/lib/libvirt/images/${VM_NAME}.qcow2" \
|
||||
rsync://vmuser@server:/vmimages-upload/
|
||||
rsync -av --password-file=/etc/rsync.secret "/var/lib/libvirt/images/xml/${VM_NAME}.xml" \
|
||||
|
|
@ -43,7 +62,14 @@ sync_all_images() {
|
|||
/var/lib/libvirt/images/
|
||||
}
|
||||
|
||||
while getopts ':u:d:a' OPTION; do
|
||||
sync_all_torrents() {
|
||||
rsync -av --password-file=/etc/rsync.secret rsync://vmuser@server:/vmimages-download/*.torrent \
|
||||
/var/lib/libvirt/images/
|
||||
rsync -av --password-file=/etc/rsync.secret rsync://vmuser@server:/vmimages-download/xml \
|
||||
/var/lib/libvirt/images/
|
||||
}
|
||||
|
||||
while getopts ':u:d:a:t' OPTION; do
|
||||
case "$OPTION" in
|
||||
u)
|
||||
VM_NAME=$OPTARG
|
||||
|
|
@ -56,6 +82,9 @@ while getopts ':u:d:a' OPTION; do
|
|||
a)
|
||||
sync_all_images
|
||||
;;
|
||||
t)
|
||||
sync_all_torrents
|
||||
;;
|
||||
?)
|
||||
show_help
|
||||
exit 1
|
||||
|
|
|
|||
213
roles/lmn_vm/files/vmimage-torrent
Executable file
213
roles/lmn_vm/files/vmimage-torrent
Executable file
|
|
@ -0,0 +1,213 @@
|
|||
#!/bin/bash
|
||||
#
|
||||
# starts tmux sessions for each valid torrent in LINBODIR
|
||||
# thomas@linuxmuster.net
|
||||
# 20221103
|
||||
#
|
||||
|
||||
# read environment
|
||||
#. /usr/share/linuxmuster/defaults.sh || exit 1
|
||||
#THELPER=$LINBOSHAREDIR/linbo-torrenthelper.sh
|
||||
THELPER=linbo-torrenthelper.sh
|
||||
#. $LINBOSHAREDIR/helperfunctions.sh || exit 1
|
||||
LINBOIMGEXT="qcow2 qdiff"
|
||||
LINBOIMGDIR="/var/lib/libvirt/images"
|
||||
serverip="10.190.1.1"
|
||||
|
||||
# start of functions
|
||||
|
||||
# help message
|
||||
usage(){
|
||||
echo
|
||||
echo "Info: vmimage-torrent manages the torrent tmux sessions of linbo images."
|
||||
echo
|
||||
echo "Usage:"
|
||||
echo " vmimage-torrent <start|stop|restart|reload|status|create|check> [image_name]"
|
||||
echo " vmimage-torrent attach <image_name|session_name>"
|
||||
echo
|
||||
echo "Note:"
|
||||
echo " * Only qcow2 & qdiff image files located below $LINBOIMGDIR are processed."
|
||||
echo " * The commands \"start\", \"stop\" and \"restart\" may have optionally an image"
|
||||
echo " filename as parameter. In this case the commands are only applied to the tmux"
|
||||
echo " session of the certain file. Without an explicit image filename the commands"
|
||||
echo " were applied to all image file sessions currently running."
|
||||
echo " * An image filename parameter is mandatory with the commands \"check\", \"create\""
|
||||
echo " and \"attach\"."
|
||||
echo " * \"check\" checks if the image file matches to the correspondig torrent."
|
||||
echo " * \"create\" creates/recreates the torrent of a certain image file."
|
||||
echo " * \"status\" shows a list of currently running torrent tmux sessions."
|
||||
echo " * \"attach\" attaches a torrent tmux session of a certain image. An image or"
|
||||
echo " session name must be given as parameter."
|
||||
echo " Press [CTRL+B]+[D] to detach the session again."
|
||||
echo " * \"reload\" is the identical to \"restart\" and is there for backwards compatibility."
|
||||
echo
|
||||
exit 1
|
||||
}
|
||||
|
||||
# check torrent
|
||||
check(){
|
||||
local image="$(basename "$IMGLIST")"
|
||||
local torrent="$image.torrent"
|
||||
local tdir="$(dirname "$IMGLIST")"
|
||||
cd "$tdir"
|
||||
echo "Checking $torrent ..."
|
||||
if ctorrent -c "$torrent"; then
|
||||
echo "Ok!"
|
||||
else
|
||||
echo "Failed!"
|
||||
exit 1
|
||||
fi
|
||||
}
|
||||
|
||||
# creates torrent files
|
||||
create(){
|
||||
local image="$(basename "$IMGLIST")"
|
||||
local tdir="$(dirname "$IMGLIST")"
|
||||
local torrent="${image}.torrent"
|
||||
local session="${torrent//./_}"
|
||||
# stop torrent service
|
||||
vmimage-torrent status | grep -q ^"$session" && vmimage-torrent stop "$IMGLIST"
|
||||
# skip already running torrents
|
||||
echo "Creating $torrent ..."
|
||||
cd "$tdir"
|
||||
rm -f "$torrent"
|
||||
if ctorrent -t -u "http://$serverip:6969/announce" -s "$torrent" "$image" ; then
|
||||
[ "$START" = "no" ] || vmimage-torrent start "$IMGLIST"
|
||||
else
|
||||
echo "Failed!"
|
||||
exit 1
|
||||
fi
|
||||
}
|
||||
|
||||
# starts torrent tmux sessions
|
||||
start(){
|
||||
local item
|
||||
local torrent
|
||||
local image
|
||||
local tdir
|
||||
local session
|
||||
for item in $IMGLIST; do
|
||||
image="$(basename "$item")"
|
||||
torrent="${image}.torrent"
|
||||
tdir="$(dirname "$item")"
|
||||
session="${torrent//./_}"
|
||||
cd "$tdir"
|
||||
if [ ! -s "$image" ]; then
|
||||
echo "Image $image does not exist! Skipping this torrent."
|
||||
continue
|
||||
fi
|
||||
# skip already running torrents
|
||||
if vmimage-torrent status | grep -qw ^"$session"; then
|
||||
echo "tmux session $session is already running."
|
||||
continue
|
||||
fi
|
||||
# create torrent file if there is none
|
||||
if [ ! -e "$torrent" ]; then
|
||||
START="no" vmimage-torrent create "$item" || continue
|
||||
fi
|
||||
echo -n "Starting tmux session $session ... "
|
||||
tmux new -ds "$session" "$THELPER $torrent ; exec $SHELL"
|
||||
sleep 1
|
||||
if vmimage-torrent status | grep -qw ^"$session"; then
|
||||
echo "Ok!"
|
||||
else
|
||||
echo "Failed!"
|
||||
fi
|
||||
done
|
||||
}
|
||||
|
||||
stop(){
|
||||
if [ -n "$SESSION" ]; then
|
||||
vmimage-torrent status | grep -qw ^"$SESSION" || return
|
||||
tmux kill-session -t "$SESSION"
|
||||
else
|
||||
local item
|
||||
vmimage-torrent status | awk -F\: '{print $1}' | while read item; do
|
||||
tmux kill-session -t "$item"
|
||||
done
|
||||
fi
|
||||
}
|
||||
|
||||
attach(){
|
||||
if ! tmux list-sessions | grep -qw "$SESSION"; then
|
||||
echo "There is no session $SESSION."
|
||||
exit 1
|
||||
fi
|
||||
echo "Hint: Detach tmux session with [CTRL+B]+[D]."
|
||||
sleep 3
|
||||
tmux attach -t "$SESSION"
|
||||
}
|
||||
|
||||
status(){
|
||||
tmux list-sessions | grep _torrent
|
||||
}
|
||||
|
||||
find_images(){
|
||||
local search="$(basename "$1")"
|
||||
if [ -n "$search" ]; then
|
||||
find "$LINBOIMGDIR" -maxdepth 2 -name "$search"
|
||||
return
|
||||
fi
|
||||
local IMGLIST
|
||||
for search in $LINBOIMGEXT; do
|
||||
IMGLIST="$IMGLIST $(find "$LINBOIMGDIR" -maxdepth 2 -name \*.$search)"
|
||||
done
|
||||
# trim leading and trailing spaces
|
||||
echo $IMGLIST | awk '{$1=$1};1'
|
||||
}
|
||||
|
||||
# end of functions
|
||||
|
||||
# check parameters
|
||||
if [ -n "$2" ] ; then
|
||||
# trap torrent parameter
|
||||
image="${2/.torrent/}"
|
||||
case "$image" in
|
||||
*.qcow2|*.qdiff)
|
||||
if [ -e "$image" ]; then
|
||||
IMGLIST="$image"
|
||||
else
|
||||
IMGLIST="$(find_images "$image")"
|
||||
fi
|
||||
if [ ! -e "$IMGLIST" ]; then
|
||||
echo "Image file $(basename $image) not found."
|
||||
usage
|
||||
fi
|
||||
filename="$(basename "$IMGLIST")"
|
||||
SESSION="${filename//./_}_torrent"
|
||||
;;
|
||||
*_torrent)
|
||||
if [ "$1" = "attach" ]; then
|
||||
SESSION="$image"
|
||||
else
|
||||
usage
|
||||
fi
|
||||
;;
|
||||
*) usage ;;
|
||||
esac
|
||||
else
|
||||
case "$1" in
|
||||
stop|status) ;;
|
||||
attach|check|create) usage ;;
|
||||
*)
|
||||
IMGLIST="$(find_images)"
|
||||
if [ -z "$IMGLIST" ]; then
|
||||
echo "No linbo images found."
|
||||
exit 0
|
||||
fi
|
||||
;;
|
||||
esac
|
||||
fi
|
||||
|
||||
case "$1" in
|
||||
start) start ;;
|
||||
stop) stop ;;
|
||||
restart|reload) stop ; start ;;
|
||||
status) status ;;
|
||||
create) create ;;
|
||||
check) check ;;
|
||||
attach) attach ;;
|
||||
*) usage ;;
|
||||
esac
|
||||
|
||||
exit 0
|
||||
|
|
@ -63,7 +63,7 @@
|
|||
- lmn-link-images
|
||||
- lmn-startvirtiofsd
|
||||
|
||||
- name: deploy mount home script
|
||||
- name: deploy vmimages scripts
|
||||
copy:
|
||||
src: "{{ item }}"
|
||||
dest: /usr/local/bin/
|
||||
|
|
@ -79,6 +79,16 @@
|
|||
- sync-vm.sh
|
||||
- link-images.sh
|
||||
- start-virtiofsd.sh
|
||||
- linbo-torrenthelper.sh
|
||||
- vmimage-torrent
|
||||
|
||||
- name: deploy linbo-torrent defaults
|
||||
copy:
|
||||
src: linbo-torrent
|
||||
dest: /etc/default/
|
||||
owner: root
|
||||
group: root
|
||||
mode: '0755'
|
||||
|
||||
- name: Deploy bridge.conf needed for qemu session mode
|
||||
lineinfile:
|
||||
|
|
@ -137,5 +147,5 @@
|
|||
register: result
|
||||
changed_when: result.stdout | length > 0
|
||||
when: >
|
||||
not run_in_installer | default(false) | bool and (ansible_mounts |
|
||||
false and not run_in_installer | default(false) | bool and (ansible_mounts |
|
||||
selectattr("mount", "equalto", "/") | list)[0].size_available > 80000000000
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue