Merge pull request 'do not use submodules because they do not always get loaded' (#20) from earl-warren/setup-forgejo:wip-lxc-helpers into main

Reviewed-on: https://code.forgejo.org/actions/setup-forgejo/pulls/20
Reviewed-by: dachary <dachary@noreply.code.forgejo.org>

.forgejo/workflows/forgejo-sh.yml

@@ -11,8 +11,6 @@ jobs:
     runs-on: self-hosted
     steps:
       - uses: actions/checkout@v3
-        with:
-          submodules: 'true'
       - run: |
           set -x
           ./forgejo-dependencies.sh install_docker

.forgejo/workflows/integration.yml

@@ -4,8 +4,6 @@ jobs:
     runs-on: self-hosted
     steps:
       - uses: actions/checkout@v3
-        with:
-          submodules: 'true'
       - run: |
           set -x
           LXC_IP_PREFIX=10.0.9 ./forgejo-dependencies.sh

.gitmodules

@@ -1,3 +0,0 @@
-[submodule "lxc-helpers"]
-	path = lxc-helpers
-	url = https://code.forgejo.org/forgejo/lxc-helpers

action.yml

@@ -75,6 +75,9 @@ runs:
   steps:
     - run: echo "${{ github.action_path }}" >> $GITHUB_PATH
       shell: bash
+    - uses: actions/checkout@v3
+      with:
+        submodules: 'true'
     - id: forgejo
       run: |
         cd $(mktemp -d)

forgejo-dependencies.sh

@@ -3,7 +3,7 @@
 
 set -x
 
-source $(dirname $0)/lxc-helpers/lxc-helpers-lib.sh
+source $(dirname $0)/lxc-helpers-lib.sh
 
 : ${LXC_IP_PREFIX:=10.0.8}
 

forgejo-test-helper.sh

@@ -138,10 +138,10 @@ function push_self_action() {
     local tag="$4"
 
     local dir="$DIR/self"
-    git clone --recurse-submodules . $dir
+    git clone . $dir
     (
 	cd $dir
-	rm -fr .forgejo .git lxc-helpers/.git
+	rm -fr .forgejo .git
 	git init
 	git checkout -b main
 	git remote add origin "$url/$owner/$self_action"

lxc-helpers

@@ -1 +0,0 @@
-Subproject commit b0766388a2624bc56258053576c7c09dba2e7cf5

lxc-helpers-lib.sh

@@ -0,0 +1,299 @@
+#!/bin/bash
+# SPDX-License-Identifier: MIT
+
+LXC_SELF_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+LXC_BIN=/usr/local/bin
+
+: ${LXC_SUDO:=}
+: ${LXC_CONTAINER_RELEASE:=bookworm}
+: ${LXC_HOME:=/home}
+: ${LXC_VERBOSE:=false}
+
+source /etc/os-release
+
+function lxc_release() {
+    echo $VERSION_CODENAME
+}
+
+function lxc_template_release() {
+    echo lxc-helpers-$LXC_CONTAINER_RELEASE
+}
+
+function lxc_root() {
+    local name="$1"
+
+    echo /var/lib/lxc/$name/rootfs
+}
+
+function lxc_config() {
+    local name="$1"
+
+    echo /var/lib/lxc/$name/config
+}
+
+function lxc_container_run() {
+    local name="$1"
+    shift
+
+    $LXC_SUDO lxc-attach --clear-env --name $name -- "$@"
+}
+
+function lxc_container_run_script_as() {
+    local name="$1"
+    local user="$2"
+    local script="$3"
+
+    $LXC_SUDO chmod +x $(lxc_root $name)$script
+    $LXC_SUDO lxc-attach --name $name -- sudo --user $user $script
+}
+
+function lxc_container_run_script() {
+    local name="$1"
+    local script="$2"
+
+    $LXC_SUDO chmod +x $(lxc_root $name)$script
+    lxc_container_run $name $script
+}
+
+function lxc_container_inside() {
+    local name="$1"
+    shift
+
+    lxc_container_run $name $LXC_BIN/lxc-helpers.sh "$@"
+}
+
+function lxc_container_user_install() {
+    local name="$1"
+    local user_id="$2"
+    local user="$3"
+
+    if test "$user" = root ; then
+	return
+    fi
+
+    local root=$(lxc_root $name)
+
+    if ! $LXC_SUDO grep --quiet "^$user " $root/etc/sudoers ; then
+	$LXC_SUDO tee $root/usr/local/bin/lxc-helpers-create-user.sh > /dev/null <<EOF
+#!/bin/bash
+set -ex
+
+mkdir -p $LXC_HOME
+useradd --base-dir $LXC_HOME --create-home --shell /bin/bash --uid $user_id $user
+for group in docker kvm libvirt ; do
+    if grep --quiet \$group /etc/group ; then adduser $user \$group ; fi
+done
+echo "$user ALL=(ALL) NOPASSWD:ALL" >> /etc/sudoers
+sudo --user $user ssh-keygen -b 2048 -N '' -f $LXC_HOME/$user/.ssh/id_rsa
+EOF
+	lxc_container_run_script $name /usr/local/bin/lxc-helpers-create-user.sh
+    fi
+}
+
+function lxc_maybe_sudo() {
+    if test $(id -u) != 0 ; then
+        LXC_SUDO=sudo
+    fi
+}
+
+function lxc_prepare_environment() {
+    lxc_maybe_sudo
+    if ! $(which lxc-create > /dev/null) ; then
+	$LXC_SUDO apt-get install -y -qq make git libvirt0 libpam-cgfs bridge-utils uidmap dnsmasq-base dnsmasq dnsmasq-utils qemu-user-static
+    fi
+}
+
+function lxc_container_configure() {
+    local name="$1"
+    
+    $LXC_SUDO tee -a $(lxc_config $name) > /dev/null <<'EOF'
+security.nesting = true
+lxc.cap.drop =
+lxc.apparmor.profile = unconfined
+#
+# /dev/net (docker won't work without /dev/net/tun)
+#
+lxc.cgroup2.devices.allow = c 10:200 rwm
+lxc.mount.entry = /dev/net dev/net none bind,create=dir 0 0
+#
+# /dev/kvm (libvirt / kvm won't work without /dev/kvm)
+#
+lxc.cgroup2.devices.allow = c 10:232 rwm
+lxc.mount.entry = /dev/kvm dev/kvm none bind,create=file 0 0
+#
+# /dev/loop
+#
+lxc.cgroup2.devices.allow = c 10:237 rwm
+lxc.cgroup2.devices.allow = b 7:* rwm
+lxc.mount.entry = /dev/loop-control dev/loop-control none bind,create=file 0 0
+#
+# /dev/mapper
+#
+lxc.cgroup2.devices.allow = c 10:236 rwm
+lxc.mount.entry = /dev/mapper dev/mapper none bind,create=dir 0 0
+#
+# /dev/fuse
+#
+lxc.cgroup2.devices.allow = b 10:229 rwm
+lxc.mount.entry = /dev/fuse dev/fuse none bind,create=file 0 0
+EOF
+
+
+    #
+    # Wait for the network to come up
+    #
+    local wait_networking=$(lxc_root $name)/usr/local/bin/lxc-helpers-wait-networking.sh
+    $LXC_SUDO tee $wait_networking > /dev/null <<'EOF'
+#!/bin/sh -e
+for d in $(seq 60); do
+  getent hosts wikipedia.org > /dev/null && break
+  sleep 1
+done
+getent hosts wikipedia.org > /dev/null || getent hosts wikipedia.org
+EOF
+    $LXC_SUDO chmod +x $wait_networking
+}
+
+function lxc_container_mount() {
+    local name="$1"
+    local dir="$2"
+
+    local config=$(lxc_config $name)
+
+    if ! $LXC_SUDO grep --quiet "lxc.mount.entry = $dir" $config ; then
+	local relative_dir=${dir##/}
+	$LXC_SUDO tee -a $config > /dev/null <<< "lxc.mount.entry = $dir $relative_dir none bind,create=dir 0 0"
+    fi
+}
+
+
+function lxc_container_start() {
+    local name="$1"
+
+    if lxc_running $name ; then
+	return
+    fi
+
+    local logs
+    if $LXC_VERBOSE; then
+	logs="--logfile=/dev/tty"
+    fi
+
+    $LXC_SUDO lxc-start $logs $name
+    $LXC_SUDO lxc-wait --name $name --state RUNNING
+    lxc_container_run $name /usr/local/bin/lxc-helpers-wait-networking.sh
+}
+
+function lxc_container_stop() {
+    local name="$1"
+
+    $LXC_SUDO lxc-ls -1 --running --filter="^$name" | while read container ; do
+	$LXC_SUDO lxc-stop --kill --name="$container"
+    done
+}
+
+function lxc_container_destroy() {
+    local name="$1"
+    local root="$2"
+
+    if lxc_exists "$name" ; then
+	lxc_container_stop $name $root
+	$LXC_SUDO lxc-destroy --force --name="$name"
+    fi
+}
+
+function lxc_exists() {
+    local name="$1"
+
+    test "$($LXC_SUDO lxc-ls --filter=^$name)"
+}
+    
+function lxc_running() {
+    local name="$1"
+
+    test "$($LXC_SUDO lxc-ls --running --filter=^$name)"
+}
+
+function lxc_build_template_release() {
+    local name="$(lxc_template_release)"
+    
+    if lxc_exists $name ; then
+	return
+    fi
+
+    local root=$(lxc_root $name)
+    local packages="sudo,git,python3"
+    $LXC_SUDO lxc-create --name $name --template debian -- --release=$LXC_CONTAINER_RELEASE --packages="$packages"
+    $LXC_SUDO cp -a $LXC_SELF_DIR/lxc-helpers*.sh $root/$LXC_BIN
+    lxc_container_configure $name
+}
+
+function lxc_build_template() {
+    local name="$1"
+    local newname="$2"
+
+    if lxc_exists $newname ; then
+	return
+    fi
+
+    if test "$name" = "$(lxc_template_release)" ; then
+	lxc_build_template_release
+    fi
+    
+    $LXC_SUDO lxc-copy --name=$name --newname=$newname
+}
+
+function lxc_apt_install() {
+    local name="$1"
+    shift
+
+    lxc_container_inside $name lxc_apt_install_inside "$@"
+}
+
+function lxc_apt_install_inside() {
+    DEBIAN_FRONTEND=noninteractive apt-get install -y -qq "$@"
+}
+
+function lxc_install_lxc() {
+    local name="$1"
+    local prefix="$2"
+
+    lxc_container_inside $name lxc_install_lxc_inside $prefix
+}
+
+function lxc_install_lxc_inside() {
+    local prefix="$1"
+    
+    local packages="make git libvirt0 libpam-cgfs bridge-utils uidmap dnsmasq-base dnsmasq dnsmasq-utils qemu-user-static"
+    if test "$(lxc_release)" = bookworm ; then
+	packages="$packages lxc-templates debootstrap distro-info"
+    fi
+
+    lxc_apt_install_inside $packages
+
+    if ! systemctl is-active --quiet lxc-net; then
+	systemctl disable --now dnsmasq
+	apt-get install -y -qq lxc
+	systemctl stop lxc-net
+	sed -i -e '/ConditionVirtualization/d' $root/usr/lib/systemd/system/lxc-net.service
+	systemctl daemon-reload
+	cat >> /etc/default/lxc-net <<EOF
+LXC_ADDR="$prefix.1"
+LXC_NETMASK="255.255.255.0"
+LXC_NETWORK="$prefix.0/24"
+LXC_DHCP_RANGE="$prefix.2,$prefix.254"
+LXC_DHCP_MAX="253"
+EOF
+	systemctl start lxc-net
+    fi
+}
+
+function lxc_install_docker() {
+    local name="$1"
+
+    lxc_container_inside $name lxc_install_docker_inside
+}
+
+function lxc_install_docker_inside() {
+    lxc_apt_install_inside docker.io docker-compose
+}

lxc-helpers.sh

@@ -0,0 +1,129 @@
+#!/usr/bin/env bash
+# SPDX-License-Identifier: MIT
+
+set -e
+
+source $(dirname $0)/lxc-helpers-lib.sh
+
+function verbose() {
+    set -x
+    PS4='${BASH_SOURCE[0]}:$LINENO: ${FUNCNAME[0]}:  '
+    LXC_VERBOSE=true
+}
+
+function help() {
+    cat <<'EOF'
+lxc-helpers - LXC container management helpers
+
+SYNOPSIS
+
+   lxc-helpers [-v|--verbose] [-h|--help]
+	       [-o|--os {bookworm|bullseye} (default bookworm)]
+	       command [arguments]
+
+DESCRIPTION
+
+   A thin shell based layer on top of LXC to create, populate, run and
+   destroy LXC containers. A container is created from a copy of an
+   existing container.
+
+CREATE AND DESTROY
+
+   lxc_prepare_environment
+
+       Install LXC dependencies.
+
+   lxc_template_release
+
+       Echo the name of the container for the Operating System
+       specified with `--os`.
+
+   lxc_build_template `existing_container` `new_container`
+
+       Copy `existing_container` into `new_container`. If
+       `existing_container` is equal to $(lxc_template_release) it
+       will be created on demand.
+
+   lxc_container_mount `name` `path`
+
+       Configure `name` container to bind mount `path` so that it is
+       also accessible at `path` from within the container.
+
+   lxc_container_start `name`
+
+       Start the `name` container.
+
+   lxc_container_stop `name`
+
+       Unmount all bind mounted directories and stop the `name`
+       container.
+
+   lxc_container_destroy `name`
+
+       Call lxc_container_stop `name` and destroy the container.
+
+ACTIONS IN THE CONTAINER
+
+   For some command lxc_something `name` that can be called from outside the container
+   there is an equivalent function lxc_something_inside that can be called from inside
+   the container.
+
+   lxc_install_lxc `name` `prefix`
+   lxc_install_lxc_inside `prefix`
+
+      Install LXC in the `name` container to allow the creation of
+      named containers. `prefix` is a class C IP prefix from which
+      containers will obtain their IP (for instance 10.40.50).
+
+   lxc_container_run_script `name` `path`
+   lxc_container_run_script_as `name` `user` `path`
+
+      Run the script found at `path` within the `name` container. The
+      environment is cleared before running the script. The first form
+      will run as root, the second form will impersonate `user`.
+
+   lxc_container_user_install `name` `user_id` `user` [`homedir` default `/home`]
+
+      Create the `user` with `user_id` in the `name` container with a
+      HOME at `/homedir/user`. Passwordless sudo permissions are
+      granted to `user`. It is made a member of the groups docker, kvm
+      and libvirt if they exist already. A SSH key is created.
+
+      Example: lxc_container_user_install mycontainer $(id -u) $(USER)
+
+EOF
+}
+
+function main() {
+    local options=$(getopt -o hvo --long help,verbose,os: -- "$@")
+    [ $? -eq 0 ] || {
+	echo "Incorrect options provided"
+	exit 1
+    }
+    eval set -- "$options"
+    while true; do
+	case "$1" in
+	    -v | --verbose)
+		verbose
+		;;
+	    -h | --help)
+		help
+		;;
+	    -o | --os)
+		LXC_CONTAINER_RELEASE=$2
+		shift
+		;;
+	    --)
+		shift
+		break
+		;;
+	esac
+	shift
+    done
+
+    lxc_maybe_sudo
+
+    "$@"
+}
+
+main "$@"

testdata/sanity-checks/.forgejo/workflows/test.yml

@@ -10,8 +10,6 @@ jobs:
           echo +++++++++++++++++++++++++++++++++++++++++++
           echo about to actions/checkout@v3 for SELF@vTest
       - uses: actions/checkout@v3
-        with:
-          submodules: 'true'
       - id: forgejo
         uses: SELF@vTest
         with: