Script A: Switchdev Mode with Linux Bridge Configuration
The following commands are used to create and bring up two VFs in switchdev mode, and to configure TC-Flower PF filters. Namespaces on the host allow for easy testing of the switchdev feature without VM creation, but a similar exercise could be done with VMs instead of namespaces.
This script can be used as a reference to run at boot time so PF eSwitch will boot in switchdev mode after every reboot.
===========================================================================================
#!/bin/bash
set -x
#set -e
DEVLINK=devlink
TC=tc
$BR=br0
PF1=ens4f0 # (PF whose eSwitch will be configured in switchdev mode. Change accordingly.)
PF1_PCI="pci/0000:af:00.0"
PF1_IP=192.168.66.16
VF1=ens4f0v0
VF2=ens4f0v1
VF1_PCI=0000:af:01.0
VF2_PCI=0000:af:01.1
VF1_MAC=52:54:00:00:16:01
VF2_MAC=52:54:00:00:16:02
VF1_IP=192.168.66.161
VF2_IP=192.168.66.162
VF1_PR=eth0
VF2_PR=eth1
PEER_IP=192.168.66.10
MASK=24
PEER_MAC=68:05:ca:a3:7b:10
rmmod ice
modprobe ice
sleep 2
#1. Make sure that there are no VFs
echo 0 > /sys/class/net/$PF1/device/sriov_numvfs
#2. Create a bridge
ip link add $BR type bridge 2> /dev/null
# To allow PF to be added to bridge as uplink
# PF needs to be added to bridge prior to entering switchdev and creating VFs
#3. Add PF as UpLink port to the bridge
ip link set $PF1 master $BR
#4. Change eSwitch mode to switchdev
$DEVLINK dev eswitch set $PF1_PCI mode switchdev
# Check the current eSwitch mode
$DEVLINK dev eswitch show $PF1_PCI
#5. Create 2 SR-IOV VFs
echo 2 > /sys/class/net/$PF1/device/sriov_numvfs
#6. Configure VF MAC Addresses
ip link set $PF1 vf 0 mac $VF1_MAC
ip link set $PF1 vf 1 mac $VF2_MAC
#7. Add VF Port Representors to the bridge and bring all of them up
ip link set $VF1_PR master $BR
ip link set $VF2_PR master $BR
ip link set $VF1_PR up
ip link set $VF2_PR up
ip link set $PF1 up
ip link set $BR up
#8. Delete IP address on PF and assign IP address to bridge
ip addr del $PF1_IP/24 dev $PF1
ip addr add $PF1_IP/24 dev $BR
#9. Create 2 network namespaces: ns1, ns2
ip netns add ns1 2> /dev/null
ip netns add ns2 2> /dev/null
sleep 2
#10. Move VF1 and VF2 to ns
ip link set $VF1 netns ns1
ip link set $VF2 netns ns2
#11. Add IP Addresses and bring up VF interfaces moved to namespaces
ip netns exec ns1 ip link set $VF1 up
ip netns exec ns2 ip link set $VF2 up
ip netns exec ns1 ip addr add $VF1_IP/$MASK dev $VF1
ip netns exec ns2 ip addr add $VF2_IP/$MASK dev $VF2
# Enable hw-tc-offload on PF (Uplink port) and VF Port Representors
#12. To offload tc filters to the hardware hw-tc-offload must be enabled on the VFs Port Representor (VF_PR)
ethtool -K $PF1 hw-tc-offload on
ethtool -K $VF1_PR hw-tc-offload on
ethtool -K $VF2_PR hw-tc-offload on
# Verify settings:
ethtool -k $PF1 | grep "hw-tc"
ethtool -k $VF1_PR | grep "hw-tc"
ethtool -k $VF2_PR | grep "hw-tc"
#13. Enable ingress qdisc on PF (Uplink port) and VF Port Representors
$TC qdisc add dev $PF1 ingress
$TC qdisc add dev $VF1_PR ingress
$TC qdisc add dev $VF2_PR ingress
#14. Add filter with skip_sw to offload to hardware
#Add tc filter for VF1 -> PEER (unicast ip)
$TC filter add dev $VF1_PR ingress protocol ip prio 1 flower src_mac $VF1_MAC dst_mac
$PEER_MAC skip_sw action mirred egress redirect dev $PF1
#Add tc filter for VF1 -> VF2 (unicast ip)
$TC filter add dev $VF1_PR ingress protocol ip prio 1 flower src_mac $VF1_MAC dst_mac
$VF2_MAC skip_sw action mirred egress redirect dev $VF2_PR
#Add tc filter for VF2 -> PEER (unicast ip)
$TC filter add dev $VF2_PR ingress protocol ip prio 1 flower src_mac $VF2_MAC dst_mac
$PEER_MAC skip_sw action mirred egress redirect dev $PF1
#Add tc filter for VF2 -> VF1 (unicast ip)
$TC filter add dev $VF2_PR ingress protocol ip prio 1 flower src_mac $VF2_MAC dst_mac
$VF1_MAC skip_sw action mirred egress redirect dev $VF1_PR
#Add tc filter for PEER -> VF1 (unicast ip)
$TC filter add dev $PF1 ingress protocol ip prio 1 flower src_mac $PEER_MAC dst_mac
$VF1_MAC skip_sw action mirred egress redirect dev $VF1_PR
#Add tc filter for PEER -> VF2 (unicast ip)
$TC filter add dev $PF1 ingress protocol ip prio 1 flower src_mac $PEER_MAC dst_mac
$VF2_MAC skip_sw action mirred egress redirect dev $VF2_PR
sleep 2
#15. Do a ping from VF1 to PEER
ip netns exec ns1 ping -c3 $PEER_IP
#16. Do a ping from VF2 to PEER
ip netns exec ns2 ping -c3 $PEER_IP
#17. Do a ping from VF1 to VF2
ip netns exec ns1 ping -c3 $VF2_IP
===========================================================================================