how to setup rebuilderd on Debian (temporary URL)
(version 0.9, feedback much welcome.)
USER=rebuilderd
# sudo adduser --system --home /srv/rebuilderd --shell /bin/bash --group $USER ; sudo passwd -d $USER
REBUILDERD_HOSTNAME=rebuilderd.example.org
sudo su - rebuilderd
#than as rebuilderd user:
echo '$unshare_tmpdir_template = "/srv/rebuilderd/tmp/tmp.sbuild.XXXXXXXXXX";' > ~/.sbuildrc
# to get a working sbuild with unshare backend
sudo apt install sbuild uidmap
# also needed
# to get a working rebuilderd-worker, debrebuild, debootsnap and debsnap (all from src:devscripts)
sudo apt install rebuilderd-worker sbuild uidmap diffoscope devscripts mmdebstrap devscripts equivs apt-utils mmdebstrap python3-debian python3-requests libdpkg-perl libjson-perl sbuild uidmap libstring-shellquote-perl dpkg-dev debian-keyring
#
# the filesystem layout used was
#
# sudo vcreate -L 1TiB rebuilderd
# sudo lvcreate -L 1T lvm -n tmp
# sudo mkfs.ext4 /dev/mapper/lvm-tmp
# sudo mkfs.ext4 /dev/mapper/lvm-rebuilderd
# echo "/dev/mapper/lvm-tmp /tmp ext4 defaults,noatime,mode=1777 0 0" | sudo tee -a /etc/fstab
# echo "/dev/mapper/lvm-rebuilderd /srv/rebuilderd ext4 defaults 0 1" | sudo tee -a /etc/fstab
# sudo mount /srv/rebuilderd
sudo mkdir -p /srv/rebuilderd/cache
sudo mkdir -p /srv/rebuilderd/results # do we use that?
sudo mkdir -p /srv/rebuilderd/tmp
sudo chown -R $USER:$USER /srv/rebuilderd
# needed for sbuild with unshare backend:
# make sure $USER is mentioned in /etc/sub(uid|gid)
# (this is only the case for installations done with bookworm or later)
for i in uid gid ; do
grep $USER /etc/sub$i || echo $USER must be added to /etc/sub$i manually
done
# to build and run rebuilderd-worker
sudo apt install cargo pkg-config liblzma-dev scdoc libzstd-dev libssl-dev libsqlite3-dev
# to also run rebuilderd
sudo apt install jq nginx sqlite3
# build and install rebuilderd
cd
git clone https://github.com/kpcyrd/rebuilderd && cd rebuilderd
git checkout 782f371ba14d32258d3579a10a66a9d6aef4aca3
make
sudo make DESTDIR=/opt install
sudo vi /opt/etc/rebuilderd-worker.conf
# use upstream provided default and make sure it has /opt in backend path
[backend."debian"]
path = "/opt/usr/libexec/rebuilderd/rebuilder-debian.sh"
sudo vi /opt/usr/libexec/rebuilderd/rebuilder-debian.sh
# so it has this content:
#!/bin/sh
set -xe
cd "$(dirname "$1")"
mkdir -p etc/apt
mkdir -p var/lib/apt/lists/
echo 'deb-src [signed-by=/usr/share/keyrings/debian-archive-keyring.gpg] https://deb.debian.org/debian unstable main' > etc/apt/sources.list
apt-get -o Dir=. update
apt-get -o Dir=. source --download-only "$(basename "$1" | cut -d_ -f1)"
/usr/bin/debrebuild --buildresult="${REBUILDERD_OUTDIR}" --builder=sbuild+unshare --cache=/srv/rebuilderd/cache -- "${1}"
# Note: currently the distro, eg trixie, has to be defined in /opt/usr/libexec/rebuilderd/rebuilder-debian.sh
# *and* /opt/etc/rebuilderd-sync.conf
# the former shall be addressed by https://github.com/kpcyrd/rebuilderd/issues/154
#
# FIXME: this should be done using services
#
# then start rebuilderd as $USER in screen:
/opt/usr/bin/rebuilderd -c /opt/etc/rebuilderd.conf
#
# run this in screen as $USER to update the list of packages frequently
while true ; do env REBUILDERD_COOKIE_PATH=~/.local/share/rebuilderd-auth-cookie /opt/usr/bin/rebuildctl -H http://127.0.0.1:8484 pkgs sync-profile --sync-config /opt/etc/rebuilderd-sync.conf debian-main ; echo sleeping 3h now ; sleep 3h ; done
# for rebuilding arch:any only:
#
# while true ; do /opt/usr/bin/rebuildctl pkgs sync-profile --sync-config /opt/etc/rebuilderd-sync.conf debian-main --print-json | jq '.[] | select(.architecture != "all")' | jq -s | /opt/usr/bin/rebuildctl pkgs sync-stdin debian main ; echo sleeping 3h now ; sleep 3h ; done
#
# until https://github.com/kpcyrd/rebuilderd/issues/152 has been fixed,
# it's also useful to randomize the queue initially once, run as $USER:
echo "update queue set priority = ABS(RANDOM()) % 10000;" |sqlite3 ~/rebuilderd.db
#
# finally start some workers as $USER in screen:
cd
mkdir 1 2
cd 1
/opt/usr/bin/rebuilderd-worker -c /opt/etc/rebuilderd-worker.conf connect
cd 2
/opt/usr/bin/rebuilderd-worker -c /opt/etc/rebuilderd-worker.conf connect
#
# FIXME: this should be done, aehm, differently ;)
#
# setup nginx
cd /var/www/html/
wget https://salsa.debian.org/qa/jenkins.debian.net/-/raw/master/hosts/osuosl5-amd64/var/www/html/index.html
sed -i "s#reproduce.debian.net#$REBUILDERD_HOSTNAME#g" index.html
# add to /etc/nginx/sites-enabled/default
location /api/ {
proxy_pass http://127.0.0.1:8484;
}
# it gets messy, how to run more archs on one machine:
cd ; mkdir -p all ; cd all ; /opt/usr/bin/rebuilderd -c /opt/etc/rebuilderd-all.conf
while true ; do env REBUILDERD_COOKIE_PATH=~/.local/share/rebuilderd-auth-cookie /opt/usr/bin/rebuildctl -H http://127.0.0.1:8489 pkgs sync-profile --sync-config /opt/etc/rebuilderd-sync-all.conf debian-main --print-json | jq '.[] | select(.architecture = "all")' | jq -s | env REBUILDERD_COOKIE_PATH=~/.local/share/rebuilderd-auth-cookie /opt/usr/bin/rebuildctl -H http://127.0.0.1:8489 pkgs sync-stdin debian main ; echo sleeping 3h now ; sleep 3h ; done
# it gets messy, how to run more archs on one machine:
cd ; mkdir -p amd64 ; cd amd64 ; /opt/usr/bin/rebuilderd -c /opt/etc/rebuilderd-amd64.conf
while true ; do env REBUILDERD_COOKIE_PATH=~/.local/share/rebuilderd-auth-cookie /opt/usr/bin/rebuildctl -H http://127.0.0.1:8484 pkgs sync-profile --sync-config /opt/etc/rebuilderd-sync-amd64.conf debian-main --print-json | jq '.[] | select(.architecture != "all")' | jq -s | env REBUILDERD_COOKIE_PATH=~/.local/share/rebuilderd-auth-cookie /opt/usr/bin/rebuildctl -H http://127.0.0.1:8484 pkgs sync-stdin debian main ; echo sleeping 3h now ; sleep 3h ; done
cd ; mkdir -p arm64 ; cd arm64 ; /opt/usr/bin/rebuilderd -c /opt/etc/rebuilderd-arm64.conf
while true ; do env REBUILDERD_COOKIE_PATH=~/.local/share/rebuilderd-auth-cookie /opt/usr/bin/rebuildctl -H http://127.0.0.1:8486 pkgs sync-profile --sync-config /opt/etc/rebuilderd-sync-arm64.conf debian-main --print-json | jq '.[] | select(.architecture != "all")' | jq -s | env REBUILDERD_COOKIE_PATH=~/.local/share/rebuilderd-auth-cookie /opt/usr/bin/rebuildctl -H http://127.0.0.1:8486 pkgs sync-stdin debian main ; echo sleeping 3h now ; sleep 3h ; done
cd ; mkdir -p armhf ; cd armhf ; /opt/usr/bin/rebuilderd -c /opt/etc/rebuilderd-armhf.conf
while true ; do env REBUILDERD_COOKIE_PATH=~/.local/share/rebuilderd-auth-cookie /opt/usr/bin/rebuildctl -H http://127.0.0.1:8487 pkgs sync-profile --sync-config /opt/etc/rebuilderd-sync-armhf.conf debian-main --print-json | jq '.[] | select(.architecture != "all")' | jq -s | env REBUILDERD_COOKIE_PATH=~/.local/share/rebuilderd-auth-cookie /opt/usr/bin/rebuildctl -H http://127.0.0.1:8487 pkgs sync-stdin debian main ; echo sleeping 3h now ; sleep 3h ; done
cd ; mkdir -p i386 ; cd i386 ; /opt/usr/bin/rebuilderd -c /opt/etc/rebuilderd-i386.conf
while true ; do env REBUILDERD_COOKIE_PATH=~/.local/share/rebuilderd-auth-cookie /opt/usr/bin/rebuildctl -H http://127.0.0.1:8485 pkgs sync-profile --sync-config /opt/etc/rebuilderd-sync-i386.conf debian-main --print-json | jq '.[] | select(.architecture != "all")' | jq -s | env REBUILDERD_COOKIE_PATH=~/.local/share/rebuilderd-auth-cookie /opt/usr/bin/rebuildctl -H http://127.0.0.1:8485 pkgs sync-stdin debian main ; echo sleeping 3h now ; sleep 3h ; done
cd ; mkdir -p riscv64 ; cd riscv64 ; /opt/usr/bin/rebuilderd -c /opt/etc/rebuilderd-riscv64.conf
while true ; do env REBUILDERD_COOKIE_PATH=~/.local/share/rebuilderd-auth-cookie /opt/usr/bin/rebuildctl -H http://127.0.0.1:8488 pkgs sync-profile --sync-config /opt/etc/rebuilderd-sync-riscv64.conf debian-main --print-json | jq '.[] | select(.architecture != "all")' | jq -s | env REBUILDERD_COOKIE_PATH=~/.local/share/rebuilderd-auth-cookie /opt/usr/bin/rebuildctl -H http://127.0.0.1:8488 pkgs sync-stdin debian main ; echo sleeping 3h now ; sleep 3h ; done
# with rebuilderd-worker from trixie:
rebuilderd-worker -c /opt/etc/rebuilderd-worker.conf connect
# backup and vacuum
echo "VACUUM;" | sqlite3 rebuilderd.db
date ; echo "VACUUM;" | sqlite3 rebuilderd.db ; date ; ls -lart rebuilderd.db