FreeBSD jail 模板基于 nullfs 和 unionfs 手工创建<<推荐>>

Helo

贡献者：iceage
原文链接

介绍

此模板提供了完整的系统环境处理脚本，包括网路的检测，内核模块的加载，NAT的处理等，完全满足系统自治的要求，基本不需要人为干预，部署新 jail 只需要创建两个空目录即可。调整 jail 模板只需要在宿主机上 chroot 到 basejail 中修改环境或安装软件即可。

jai模板制作原理

每个jail系统都需要完整的base 文件树，而base有1G+的大小，单个jail无所谓，如果系统启动多个jail，空间就是不小的开支。所以把base中系统文件部分单独提取出来然后以只读目录的形式重新挂载到每个jail中，这样只用一份母本文件就可以实现每个jail公用，达到节省空间的目的。

系统需求：

内核支持 IPFW 编译支持或者模块支持
内核支持 VIMAGE 编译支持，12.1默认支持
内核支持 epair 编译支持或者模块支持
内核支持 bridge 编译支持或者模块支持

目标：

basejail 公共模板目录 jail_name_write 用户增量可写目录
base来源：光盘

制作：

1、挂载光驱准备 jail 所需的系统 base

#mount -t cd9660 /dev/cd0 /mnt

2、准备 basejail 公共模板目录

#cd /usr/local
#mkdir basejail

3、解压 base.tgz 以及 ports.txz(可选) 到 basejail 模板目录。

#tar xvzfp /mnt/usr/freebsd-dist/base.txz -C /usr/local/basejail
#tar xvzfp /mnt/usr/freebsd-dist/ports.txz -C /usr/local/basejail
#tar xvzfp /mnt/usr/freebsd-dist/lib32.txz -C /usr/local/basejail

4、chroot 到 basejail 公共模板目录，并设置公共 jail 模板环境

#chroot /usr/local/basejail
jail# echo "nameserver 8.8.8.8" >> /etc/resolv.conf
jail# mount -t devfs none dev
jail# pkg update -f 
jail# pkg install -y vim-console 
jail# pkg lock -y vim-console
jail# cd /usr/ports
jail# portsnap auto
jail# make fetchindex
jail# sysrc sendmail_enable=none
jail# sysrc syslogd_flags="-ss"
jail# exit

测试

/usr/local/basejail 为模板公用目录
/var/jails 为 jail 运行目录

5、建立 jail 运行目录，并部署两个 jail 目录

#mkdir -p /var/jails/jail1
#mkdir -p /var/jails/jail1_write
#
#mkdir -p /var/jails/jail2
#mkdir -p /var/jails/jail2_write

6、编辑 jial 配置文件 /etc/jail.conf

####################################################
#
#      所有 jails 的公共配置
#
####################################################
vnet;
vnet.interface  =  "${iface}b";
host.hostname   =  "${jail_name}.org";
path            =  "/var/jails/${jail_name}";
persist         =  1;
$log            = "/var/log/jails/${jail_name}_console.log";
$bridge         = "bridge0";

#公共目录 basejail 以 nullfs 只读方式挂载
mount           =  "/usr/local/basejail /var/jails/${jail_name} nullfs ro 0 0";

#可写目录以 unionfs 方式挂载，所有文件修改以增量形式保存在 jail_name_write 目录中
mount          +=  "/var/jails/${jail_name}_write /var/jails/${jail_name} unionfs rw 0 0";

exec.clean;
exec.system_user = "root";
exec.jail_user   = "root";
exec.start       =  "/bin/sh /etc/rc";
exec.start      += "ifconfig ${iface}b inet ${ipaddr}";
exec.start      += "route add default 172.16.1.254";
exec.start      += "ifconfig ${iface}b name em0";
exec.start      += "ipfw add 65534 allow ip from any to any";
exec.stop        =  "/bin/sh /etc/rc.shutdown";

#使用脚本处理 jail 启动前环境
exec.prestart    =  "/var/jails/prestart_script $iface $bridge $log $jail_name $ipaddr";

exec.prestop    =  "ifconfig em0 -vnet ${jail_name}";

#使用脚本处理 jail 停止后环境
exec.poststop   =  "/var/jails/poststop_script $iface $bridge";
exec.consolelog = "$log";
devfs_ruleset   =  "4";
mount.devfs     =  1;
#mount.procfs;
mount.fdescfs;  #/dev下标准输入输出设备,没有它,ps明令等不能正常

##########################################################################
#                 生产中增减 jail 只需要修改以下示例配置，重置jail也只需要
#                 在宿主机中删除 /var/jails/jailname_write 目录下的文件即可
##########################################################################

#定义jail1
jail1{
        $iface          = "epair1";
        $jail_name      = "jail1";
        $ipaddr         = 172.16.1.2;
}

#定义jail2
jail2{
        $iface          = "epair2";
        $jail_name      = "jail2";
        $ipaddr         = 172.16.1.3;
        
        # 如果jial 中运行 postgresql 需要支持以下sysvipc选项
        #sysvmsg         = new;
        #sysvsem         = new;
        #sysvshm         = new;
}

7、jail 前置脚本文件 /var/jails/prestart_script

#!/bin/sh

bridge=$2
epair=$1
log=$3
jail_name=$4
ipaddr=$5


checkkernel()
{
        if [ -z "`kldstat | grep 'if_tap'`" ];then
                `kldload if_tap`
                echo "if_tap_load=YES" >> /boot/loader.conf
        fi

        if [ -z "`kldstat | grep 'if_bridge'`" ];then
                `kldload if_bridge`
                echo "if_bridge_load=YES" >> /boot/loader.conf
        fi

        if [ -z "`kldstat | grep 'if_epair'`" ];then
                `kldload if_epair`
                echo "if_epair_load=YES" >> /boot/loader.conf
        fi

        if [ -z "`kldstat | grep 'ipfw'`" ];then
                `kldload ipfw`
                echo "ipfw_load=YES" >> /boot/loader.conf
                echo "net.inet.ip.fw.default_to_accept=1" >> /boot/loader.conf
        fi

        if [ -z "`kldstat | grep 'ipfw_nat'`" ];then
                `kldload ipfw_nat`
                echo "ipfw_nat_load=YES" >> /boot/loader.conf
        fi

        if [ -z "`kldstat | grep 'libalias'`" ];then
                `kldload libalias`
                echo "libalias_load=YES" >> /boot/loader.conf
        fi
}


checklog()
{
        if [ ! -f "$log" ];then
                dirname=`dirname $log`
                if [ ! -d "$dirname" ];then
                        `mkdir -p $dirname`
                fi

                `touch $log`
        fi
}

interface()
{
        descr="${jail_name} ${ipaddr}"

        if [ -z "`ifconfig | grep $bridge`" ];then
                `ifconfig $bridge create up`
        fi


        if [ -z "`ifconfig | grep $epair`" ];then
                `ifconfig $epair create up`
                `ifconfig ${epair}a description "_______[ ${descr} ]_______"`
        fi

        if [ -z "`ifconfig ${bridge}| grep ${epair}a`" ];then
                `ifconfig ${bridge} addm ${epair}a`
        fi
}



addnat()
{

        num=`ipfw list | wc -l`
        num=`echo $num | sed -e 's/^[ ]*//g' | sed -e 's/[ ]*$//g'`

        if [ -z "`sysctl net.inet.ip.forwarding | grep 1`" ];then
                `sysctl net.inet.ip.forwarding=1`
                echo "net.inet.ip.forwarding=1" >> /etc/sysctl.conf
        fi


        if [ -z "`ifconfig ${bridge} | grep 172.16.1.254`" ];then
                `ifconfig ${bridge} delete`
                `ifconfig ${bridge} inet 172.16.1.254/24`
        fi

        if [ "${num}" -le "1" ];then
                `/usr/local/etc/ipfw`
        fi

}

checkkernel
checklog
interface 
addnat

8、jail 后置脚本文件 /var/jails/poststop_script

#!/bin/sh

bridge=$2
epair=$1

interface()
{
        num=`jls | wc -l`
        num=`echo $num | sed -e 's/^[ ]*//g' | sed -e 's/[ ]*$//g'`

        if [ ! -z "`ifconfig ${bridge} | grep ${epair}a`" ];then
                `ifconfig ${bridge} deletem ${epair}a`
        fi

        if [ ! -z "`ifconfig | grep ${epair}a`" ];then
                `ifconfig ${epair}a destroy`
        fi

        if [ "${num}" -eq "1" ];then
                `ifconfig $bridge destroy`
        fi
}

interface

9、ipfw 防火墙脚本 /usr/local/etc/ipfw ，不需要通过 rc 调用

#!/bin/sh

if="em0"     #修改这里适配你的物理网卡
net="172.16.1.0/24"
#proxy="172.16.1.2"
cmd="ipfw -fq add"

ipfw  -fq nat 1 delete
ipfw  -fq delete 500
ipfw  -fq delete 501

ipfw  -fq nat 1 config if $if #\
##       如果有端口映射需求请打开以下映射选项
#        redirect_port tcp ${proxy}:22 2202 \
#        redirect_port tcp ${proxy}:80 80

$cmd 500 nat 1 ip from $net to not $net out via $if
$cmd 501 nat 1 ip from not $net to any  in  via $if
$cmd 65534 allow ip from any to any

处理脚本权限与内核模块

#chmod +x /var/jails/poststop_script
#chmod +x /var/jails/prestart_script
#chmod +x /usr/local/etc/ipfw
#sysrc -f /boot/loader.conf ipfw_load=YES
#sysrc -f /boot/loader.conf ipfw_nat_load=YES    
#sysrc -f /boot/loader.conf libalias_load=YES 
#sysrc -f /boot/loader.conf if_bridge_load=YES
#sysrc -f /boot/loader.conf if_tap_load=YES 
#echo  "net.inet.ip.fw.default_to_accept=1" >> /boot/loader.conf    
#echo  "net.inet.ip.fw.verbose=1" >> /etc/sysctl.conf 
#echo  "net.inet.ip.fw.verbose_limit=3 " >> /etc/sysctl.conf

10、启动/停止所有 jail

#sysrc jail_enable=yes
#service jail start
#service jail stop

单独启动或者重启某一个jail

#service jail start jail1
#service jail restart jail2

11、进入 jail1

#jexec jail1

Helo

删除basejail因为是系统文件无法删除用chflgs修改文件标识符

chflags -R 0 *

Helo

FreeBSD挂载ISO镜像

root@nas:~ # mdconfig -f /nas/data/iso/FreeBSD-14.1-RELEASE-amd64-dvd1.iso
md0
root@nas:~ # mount -t cd9660 /dev/md0 /mnt/

Helo

1.VM Bhyve 和 jail脚本有干扰 bridge 和 switch
2.ipfw 系统禁用因为jail脚本检测会干扰

Helo

最终脚本（HOST：FreeBSD13.3 JAIL：FreeBSD13.2）：
评估HOST 14版本也可以但是和vm冲突暂缓验证

vim /etc/jail.conf

####################################################
#
#      所有 jails 的公共配置
#
####################################################
vnet;
vnet.interface  =  "${iface}b";
host.hostname   =  "${jail_name}.nas.mozii.org";
path            =  "/var/jail/jails/${jail_name}";
persist         =  1;
$log            = "/var/log/jails/${jail_name}_console.log";
$bridge         = "bridge0";

#公共目录 basejail 以 nullfs 只读方式挂载
mount           =  "/var/jail/basejail /var/jail/jails/${jail_name} nullfs ro 0 0";
mount          +=  "/var/jail/jails/${jail_name}_write /var/jail/jails/${jail_name} unionfs rw 0 0";

exec.clean;
exec.system_user = "root";
exec.jail_user   = "root";
exec.start       =  "/bin/sh /etc/rc";
exec.start      += "ifconfig ${iface}b inet ${ipaddr}";
exec.start      += "route add default 172.16.1.254";
exec.start      += "ifconfig ${iface}b name em0";
exec.start      += "ipfw add 65534 allow ip from any to any";
exec.stop        =  "/bin/sh /etc/rc.shutdown";

#使用脚本处理 jail 启动前环境准备
exec.prestart    =  "/var/jail/jails/prestart_script.sh $iface $bridge $log $jail_name";

#注意：
#下面一条步骤不可省略，否则 vnet 模式下 kernel 会 panic,产生 
#nd6_dad_timer: cancel DAD on epair0 because of ND6_IFF_IFDISABLED. 导致系统宕机

exec.prestop    =  "ifconfig em0 -vnet ${jail_name}";

#使用脚本处理 jail 停止后环境
exec.poststop   =  "/var/jail/jails/poststop_script.sh $iface $bridge";
exec.consolelog = "$log";
devfs_ruleset   =  "4";
mount.devfs     =  1;
#mount.procfs;
mount.fdescfs;  #/dev下标准输入输出设备,没有它,ps明令等不能正常

###########################################################################
#           以下为 jail   定义区
###########################################################################


## 所有网站代理
nginx {
        $iface          = "epair250";
        $jail_name      = "nginx";
	$ipaddr         = 172.16.1.250/24;
}

## Postgresql 数据库
postgres {
        $iface          = "epair2";
        $jail_name      = "postgres";
        $ipaddr         = 172.16.1.2/24;
        sysvmsg         = new;
        sysvsem         = new;
        sysvshm         = new;
}

## MySQL 数据库
mysql {
        $iface          = "epair3";
        $jail_name      = "mysql";
        $ipaddr         = 172.16.1.3/24;
}


## Redis 数据库
redis {
        $iface          = "epair5";
        $jail_name      = "redis";
        $ipaddr         = 172.16.1.5/24;
}

vim /var/jail/jails/prestart_script.sh

#!/bin/sh

bridge=$2
epair=$1
log=$3
jail_name=$4
ipaddr=$5


checkkernel()
{
#        if [ -z "`kldstat | grep 'if_tap'`" ];then
#                `kldload if_tap`
#                echo "if_tap_load=YES" >> /boot/loader.conf
#        fi

        if [ -z "`kldstat | grep 'if_bridge'`" ];then
                `kldload if_bridge`
                echo "if_bridge_load=YES" >> /boot/loader.conf
        fi

        if [ -z "`kldstat | grep 'if_epair'`" ];then
                `kldload if_epair`
                echo "if_epair_load=YES" >> /boot/loader.conf
        fi

        if [ -z "`kldstat | grep 'ipfw'`" ];then
                `kldload ipfw`
                echo "ipfw_load=YES" >> /boot/loader.conf
                echo "net.inet.ip.fw.default_to_accept=1" >> /boot/loader.conf
        fi

        if [ -z "`kldstat | grep 'ipfw_nat'`" ];then
                `kldload ipfw_nat`
                echo "ipfw_nat_load=YES" >> /boot/loader.conf
        fi

        if [ -z "`kldstat | grep 'libalias'`" ];then
                `kldload libalias`
                echo "libalias_load=YES" >> /boot/loader.conf
        fi
}


checklog()
{
        if [ ! -f "$log" ];then
                dirname=`dirname $log`
                if [ ! -d "$dirname" ];then
                        `mkdir -p $dirname`
                fi

                `touch $log`
        fi
}

interface()
{
        descr="${jail_name} ${ipaddr}"

        if [ -z "`ifconfig | grep $bridge`" ];then
                `ifconfig $bridge create up`
        fi


        if [ -z "`ifconfig | grep $epair`" ];then
                `ifconfig $epair create up`
                `ifconfig ${epair}a description "_______[ ${descr} ]_______"`
        fi

        if [ -z "`ifconfig ${bridge}| grep ${epair}a`" ];then
                `ifconfig ${bridge} addm ${epair}a`
        fi
}



addnat()
{

        num=`ipfw list | wc -l`
        num=`echo $num | sed -e 's/^[ ]*//g' | sed -e 's/[ ]*$//g'`

        if [ -z "`sysctl net.inet.ip.forwarding | grep 1`" ];then
                `sysctl net.inet.ip.forwarding=1`
                echo "net.inet.ip.forwarding=1" >> /etc/sysctl.conf
        fi


        if [ -z "`ifconfig ${bridge} | grep 172.16.1.254`" ];then
                `ifconfig ${bridge} delete`
                `ifconfig ${bridge} inet 172.16.1.254/24`
        fi

        if [ "${num}" -le "1" ];then
                `/usr/local/etc/ipfw`
        fi

}

heckkernel
checklog
interface 
addnat

##关闭系统ipfw开关 停止服务
vim /usr/local/etc/ipfw

#!/bin/sh

if="vtnet0"     #修改这里适配你的物理网卡
net="172.16.1.0/24"
nginx="172.16.1.250"
cmd="ipfw -fq add"

ipfw  -fq nat 1 delete
ipfw  -fq flush
ipfw  -fq delete 500
ipfw  -fq delete 501

ipfw  -fq nat 1 config if $if \
        redirect_port tcp ${nginx}:80 80 \
        redirect_port tcp ${nginx}:443 443

$cmd 500 nat 1 ip from $net to not $net out via $if
$cmd 501 nat 1 ip from not $net to any  in  via $if
$cmd 65534 allow ip from any to any