Building a XEN dom0 Linux kernel

The domU instructions below have also turned into a scripts:

Introduction

You need at least Linux v4.2 for the XEN dom0/U code to be available. Slackware 14.2 has v4.4 so that is fine – you could just recompile the sources provided from the K set. But let us get things staight with the latest versions from

Preparing

Grab the latest stable and verify it

ver=5.2.14

cd /usr/src/
wget https://cdn.kernel.org/pub/linux/kernel/v5.x/linux-$ver.tar.xz
wget https://cdn.kernel.org/pub/linux/kernel/v5.x/linux-$ver.tar.sign
unxz linux-$ver.tar.xz
#gpg2 --recv-keys 0x6092693E --keyserver wwwkeys.pgp.net
gpg2 --verify linux-$ver.tar.sign
tar xf linux-$ver.tar

or xen testing,

#git clone git://git.kernel.org/pub/scm/linux/kernel/git/xen/tip.git
#cd tip/
#git reset --hard
#git checkout -b xen-next origin/linux-next
#git pull

eventually grab the Reiser4 patch

wget https://sourceforge.net/projects/reiser4/files/reiser4-for-linux-5.x/reiser4-for-5.2.0.patch.gz

differenciate the code bases

#rm -rf dom0/
mv linux-$ver/ dom0/
cd dom0/
rm -f linux
ln -s dom0 linux

#rm -rf domU/
mv linux-$ver/ domU/
cd domU/

and apply some patches,

zcat ../reiser4-for-5.2.0.patch.gz | patch -p1

in case of previous undergoing, start clean,

ls -alhF .config*
rm -f .config*
make -j16 distclean
#make -j16 mrproper

and proceed,

mv -i Makefile Makefile.dist
sed 's/^EXTRAVERSION =/EXTRAVERSION = .slackxenreiser4/' Makefile.dist > Makefile
sed 's/^EXTRAVERSION =/EXTRAVERSION = .domureiser4/' Makefile.dist > Makefile

dom0 specific,

wget -O - http://mirror.yandex.ru/slackware/slackware64-current/kernels/huge.s/config > .config
make olddefconfig

domU specific,

#make help | grep defconfig
#make ARCH=x86_64 tinyconfig
#make ARCH=x86_64 defconfig
make x86_64_defconfig # same as defconfig

and proceed,

cat >> .config <<-EOF9
CONFIG_BTRFS_FS=y
CONFIG_GFS2_FS=y
CONFIG_JFS_FS=y
CONFIG_REISERFS_FS=y
CONFIG_REISER4_FS=y
CONFIG_XFS_FS=y
CONFIG_BRIDGE=y
CONFIG_VLAN_8021Q=y
CONFIG_IKCONFIG=y
CONFIG_IKCONFIG_PROC=y
CONFIG_NFS_V4_1=y
CONFIG_NFS_V4_2=y
CONFIG_NFS_V4_1_MIGRATION=y
CONFIG_NFSD=y
CONFIG_NFSD_V3=y
CONFIG_NFSD_V4=y
CONFIG_CIFS=y
EOF9
make olddefconfig
grep REISER4_FS .config

The Docker Situation

cd /usr/src/
#wget https://raw.githubusercontent.com/moby/moby/master/contrib/check-config.sh
#same as wget https://raw.githubusercontent.com/coreos/docker/master/contrib/check-config.sh
wget https://raw.githubusercontent.com/docker/docker/master/contrib/check-config.sh
chmod +x check-config.sh

cd dom0/
cd domU/
../check-config.sh .config | grep missing | sed -r 's/^[[:space:]]*- .*(CONFIG_[^:]+).*: .*missing.*/\1=y/'
vi .config
G

COPY/PASTE into .config

make -j16 olddefconfig && echo DONE
grep VXLAN .config
grep OVERLAY_FS .config

#k8s
cat >> .config <<-EOF9
CONFIG_NETFILTER_ADVANCED=y
CONFIG_IP_NF_TARGET_REDIRECT=y
CONFIG_NETFILTER_XT_MATCH_COMMENT=y
EOF9
make -j16 olddefconfig && echo DONE
grep NETFILTER_ADVANCED .config
grep IP_NF_TARGET_REDIRECT .config
grep NETFILTER_XT_MATCH_COMMENT .config

note. Linux v5 – AUFS_FS is totally gone

XEN

make it generally XEN capable,

make xenconfig
make olddefconfig && echo DONE
grep CONFIG_XEN_PVH= .config

#cat >> .config <<-EOF9
#CONFIG_PARAVIRT_SPINLOCKS=y
#EOF9
#grep CONFIG_PARAVIRT_SPINLOCKS .config

DOM0 SPECIFIC

cat >> .config <<-EOF9
CONFIG_UFS_FS=y
CONFIG_UFS_FS_WRITE=y
EOF9
make olddefconfig

grep CEPH .config
grep BLK_DEV_RBD .config
grep INFINIBAND .config
grep INFINIBAND_ISER .config
grep SCSI_LOWLEVEL .config
grep ISCSI_TCP .config
grep BRIDGE_NF_EBTABLES .config
grep BRIDGE_EBT_ .config

TMEM does not seem to work on dom0

xen:tmem: frontswap enabled, RAM provided by Xen Transcendent Memory
xen:tmem: cleancache enabled, RAM provided by Xen Transcendent Memory
xen_selfballoon: Xen selfballooning driver disabled for domain0

DOMU SPECIFIC

domU only & TMEM

grep XEN_DOM0 .config
cat >> .config <<-EOF9
CONFIG_XEN_DOM0=n
CONFIG_XEN_TMEM=y
CONFIG_CLEANCACHE=y
CONFIG_FRONTSWAP=y
CONFIG_SWAP=y
CONFIG_XEN_SELFBALLOONING=y
EOF9
make -j16 olddefconfig

disable as much modules as possible

grep =m$ .config
mv -f .config .config.m
#sed 's/=m$/=n/' .config.m > .config
sed 's/=m$/=y/' .config.m > .config
make -j16 olddefconfig
grep =m$ .config

you will notice those remain

CONFIG_XEN_TMEM=m
CONFIG_PNFS_FLEXFILE_LAYOUT=m

Building & Delivering Modules

j=$((`grep ^processor /proc/cpuinfo | tail -1 | awk '{print $3}'` + 1))
lscpu | grep ^Thread

time nice make -j$j > ../make.dom0.log && echo DONE
time nice make -j$j > ../make.domU.log && echo DONE

version=`file arch/x86/boot/bzImage | awk '{print $9}'`
echo $version
ls -lhF /lib/modules/
[[ -d /lib/modules/$version/ ]] && rm -rf /lib/modules/$version/ && echo cleaned-up
make -j$j modules_install >/dev/null && echo DONE
du -sh /lib/modules/$version/ # dom0 207M 210M 209M 205M
du -sh /lib/modules/$version/ # domU 84K / 1.1M
find /lib/modules/$version/
mkdir -p /data/kernels/
tar czf /data/kernels/lib.modules.$version.tar.gz -C /lib/modules/ $version/

DOM0 DELIVERY SPECIFIC

depmod -a $version

ls -lF arch/x86_64/boot/bzImage
ls -lF arch/x86/boot/bzImage
file arch/x86/boot/bzImage

mv -f /vmlinuz.config /vmlinuz.config.old
mv -f /vmlinuz.map /vmlinuz.map.old
mv -f /vmlinuz /vmlinuz.old
cp -i .config /vmlinuz.config
cp -i System.map /vmlinuz.map
cp -i arch/x86/boot/bzImage /vmlinuz
file /vmlinuz.old /vmlinuz

#mv -f /boot/efi/EFI/Slackware/vmlinuz /boot/efi/EFI/Slackware/vmlinuz.old
#cp -i arch/x86/boot/bzImage /boot/efi/EFI/Slackware/vmlinuz
#file /boot/efi/EFI/Slackware/vmlinuz.old /boot/efi/EFI/Slackware/vmlinuz*

some may need kernel headers

tar czf /data/kernels/dom0.$version.tar.gz -C /usr/src/ dom0/

check that it works after reboot,

xl li # no guest
shutdown -r now

and eventually deliver the new dom0 kernel onto the XEN farm,

ls -lhF /vmlinuz
ls -lhF /lib/modules/

#cp -f .config /data/kernels/vmlinuz.dom0.config
#cp -f System.map /data/kernels/vmlinuz.dom0.map
#ls -lkF /data/kernels/vmlinuz
#file /data/kernels/vmlinuz

version=`uname -r`

ssh slack2 mv -f /vmlinuz /vmlinuz.old
scp /vmlinuz slack2:/
ssh slack2 ls -lhF /lib/modules/
ssh slack2 "tar xzf /data/kernels/lib.modules.$version.tar.gz -C /lib/modules/ && echo done"
ssh slack2 depmod -a $version

DOMU DELIVERY SPECIFIC

ls -lF arch/x86_64/boot/bzImage
ls -lF arch/x86/boot/bzImage # / 11115792

SHOULD NOT BE A SYMLINK OTHERWISE YOU WILL OVER-WRITE THE TARGET,

ls -lhF /data/kernels/vmlinuz
file /data/kernels/vmlinuz

mv -f /data/kernels/vmlinuz.config /data/kernels/vmlinuz.config.old
mv -f /data/kernels/vmlinuz.map /data/kernels/vmlinuz.map.old
mv -f /data/kernels/vmlinuz /data/kernels/vmlinuz.old
cp -f .config /data/kernels/vmlinuz.config
cp -f System.map /data/kernels/vmlinuz.map
cp -f arch/x86/boot/bzImage /data/kernels/vmlinuz

clean-up from build system

echo /lib/modules/$version/
ls -ldF /lib/modules/$version/
rm -rf /lib/modules/$version/

as XEM_TMEM module keeps being a module, we need to deliver it to the guests to use TMEM,

guest=GUEST

cd /data/guests/$guest/
fsck.ext4 $guest.ext4
mkdir lala/
mount -o loop,rw $guest.ext4 lala/

mkdir -p lala/lib/modules/
ls -alkF lala/lib/modules/
rm -rf lala/lib/modules/*
#for ball in `ls -1 /data/kernels/lib.modules.*.tar.gz`; do
#        echo -n $ball...
#        tar xzf $ball -C lala/lib/modules/ && echo done
#done; unset ball
tar xzf /data/kernels/lib.modules.$version.tar.gz -C lala/lib/modules/
ls -alkF lala/lib/modules/

cat lala/etc/modules
echo tmem >> lala/etc/modules

umount lala/
rmdir lala/
xl create $guest -c

and verify that everything works as expected onto a new guest, including TMEM and Docker,

zcat /proc/config.gz | grep TMEM
zcat /proc/config.gz | grep CLEANCACHE
lsmod | grep tmem
watch free -m

#zcat /proc/config.gz | grep OVERLAY
#docker ps -a

Resources

CONFIG_IKCONFIG: Kernel .config support https://cateee.net/lkddb/web-lkddb/IKCONFIG.html

Kernel/IKCONFIG Support https://wiki.gentoo.org/wiki/Kernel/IKCONFIG_Support

Trash

domU new school,

ls -lhF /lib/modules/
echo $version
rm -rf /lib/modules/$version/
cp -f .config /data/kernels/vmlinuz.$version.config
cp -f System.map /data/kernels/vmlinuz.$version.map
cp -f arch/x86/boot/bzImage /data/kernels/vmlinuz.$version
ln -sf vmlinuz.$version /data/kernels/vmlinuz

dom0 docker to be added,

CONFIG_NF_NAT_IPV4=y
CONFIG_IP_NF_TARGET_MASQUERADE=y
CONFIG_IP_NF_NAT=y
CONFIG_MEMCG_SWAP_ENABLED=y
#CONFIG_CGROUP_HUGETLB=y
CONFIG_IP_NF_TARGET_REDIRECT=y
CONFIG_EXT3_FS_XATTR=y
CONFIG_AUFS_FS=y

domU docker to be added,

CONFIG_CGROUP_DEVICE=y
CONFIG_MEMCG=y
CONFIG_VETH=y
CONFIG_BRIDGE_NETFILTER=y
CONFIG_NETFILTER_XT_MATCH_IPVS=y
CONFIG_USER_NS=y
CONFIG_CGROUP_PIDS=y
CONFIG_MEMCG_SWAP=y
CONFIG_MEMCG_SWAP_ENABLED=y
CONFIG_BLK_CGROUP=y
CONFIG_BLK_DEV_THROTTLING=y
CONFIG_CFQ_GROUP_IOSCHED=y
CONFIG_CGROUP_PERF=y
#CONFIG_CGROUP_HUGETLB=y
CONFIG_NET_CLS_CGROUP=y
CONFIG_CGROUP_NET_PRIO=y
CONFIG_CFS_BANDWIDTH=y
CONFIG_RT_GROUP_SCHED=y
CONFIG_IP_NF_TARGET_REDIRECT=y
CONFIG_IP_VS=y
CONFIG_IP_VS_NFCT=y
#CONFIG_IP_VS_PROTO_TCP=y
#CONFIG_IP_VS_PROTO_UDP=y
CONFIG_IP_VS_RR=y
CONFIG_VXLAN=y
CONFIG_INET_ESP=y
CONFIG_INET_XFRM_MODE_TRANSPORT=y
CONFIG_IPVLAN=y
CONFIG_MACVLAN=y
CONFIG_DUMMY=y
CONFIG_NF_NAT_TFTP=y
CONFIG_NF_CONNTRACK_TFTP=y
CONFIG_AUFS_FS=y
CONFIG_BTRFS_FS_POSIX_ACL=y
CONFIG_DM_THIN_PROVISIONING=y
CONFIG_OVERLAY_FS=y

e.g. for dom0,

cat >> .config <<-EOF
CONFIG_NF_NAT_IPV4=y
CONFIG_IP_NF_TARGET_MASQUERADE=y
CONFIG_IP_NF_NAT=y
CONFIG_MEMCG_SWAP_ENABLED=y
CONFIG_CGROUP_HUGETLB=y
CONFIG_EXT3_FS_XATTR=y
CONFIG_AUFS_FS=y
EOF

e.g. for domU,

CONFIG_CGROUP_DEVICE=y
CONFIG_MEMCG=y
CONFIG_VETH=y
CONFIG_BRIDGE_NETFILTER=y
CONFIG_NETFILTER_XT_MATCH_IPVS=y
CONFIG_USER_NS=y
CONFIG_CGROUP_PIDS=y
CONFIG_MEMCG_SWAP=y
CONFIG_MEMCG_SWAP_ENABLED=y
CONFIG_BLK_CGROUP=y
CONFIG_BLK_DEV_THROTTLING=y
CONFIG_CFQ_GROUP_IOSCHED=y
CONFIG_CGROUP_PERF=y
CONFIG_NET_CLS_CGROUP=y
CONFIG_CGROUP_NET_PRIO=y
CONFIG_CFS_BANDWIDTH=y
CONFIG_RT_GROUP_SCHED=y
CONFIG_IP_VS=y
CONFIG_IP_VS_NFCT=y
CONFIG_IP_VS_RR=y
CONFIG_VXLAN=y
CONFIG_INET_ESP=y
CONFIG_INET_XFRM_MODE_TRANSPORT=y
CONFIG_IPVLAN=y
CONFIG_MACVLAN=y
CONFIG_DUMMY=y
CONFIG_NF_NAT_TFTP=y
CONFIG_NF_CONNTRACK_TFTP=y
CONFIG_AUFS_FS=y
CONFIG_BTRFS_FS_POSIX_ACL=y
CONFIG_DM_THIN_PROVISIONING=y

Nethence | Pub | Lab | Pbraun | SNE Russia | xhtml