通常情况下，Linux的网卡中断是由一个CPU核心来处理的，当承担高流量的场景下，会出现一些诡异的情况（网卡尚未达到瓶颈，但是却出现丢包的情况）
成都创新互联公司是一家专业的成都网站建设公司，我们专注成都做网站、成都网站制作、成都外贸网站建设、网络营销、企业网站建设，友情链接，广告投放为企业客户提供一站式建站解决方案，能带给客户新的互联网理念。从网站结构的规划UI设计到用户体验提高，创新互联力求做到尽善尽美。
这种时候，我们最好看下网卡中断是不是缺少调优。
优化3要点：网卡多队列+irq affinity亲缘性设置+关闭irqbalance
目前阿里云官方提供的centos和ubuntu镜像里面，已经自带了优化脚本，内容如下:
centos7的脚本路径在 /usr/sbin/ecs_mq_rps_rfs 具体内容如下：
#!/bin/bash
# This is the default setting of networking multiqueue and irq affinity
# 1. enable multiqueue if available
# 2. irq affinity optimization
# 3. stop irqbalance service
# set and check multiqueue
function set_check_multiqueue()
{
eth=$1
log_file=$2
queue_num=$(ethtool -l $eth | grep -ia5 'pre-set' | grep -i combined | awk {'print $2'})
if [ $queue_num -gt 1 ]; then
# set multiqueue
ethtool -L $eth combined $queue_num
# check multiqueue setting
cur_q_num=$(ethtool -l $eth | grep -iA5 current | grep -i combined | awk {'print $2'})
if [ "X$queue_num" != "X$cur_q_num" ]; then
echo "Failed to set $eth queue size to $queue_num" >> $log_file
echo "after setting, pre-set queue num: $queue_num , current: $cur_q_num" >> $log_file
return 1
else
echo "OK. set $eth queue size to $queue_num" >> $log_file
fi
else
echo "only support $queue_num queue; no need to enable multiqueue on $eth" >> $log_file
fi
}
#set irq affinity
function set_irq_smpaffinity()
{
log_file=$1
node_dir=/sys/devices/system/node
for i in $(ls -d $node_dir/node*); do
i=${i/*node/}
done
echo "max node :$i" >> $log_file
node_cpumax=$(cat /sys/devices/system/node/node${i}/cpulist |awk -F- '{print $NF}')
irqs=($(cat /proc/interrupts |grep virtio |grep put | awk -F: '{print $1}'))
core=0
for irq in ${irqs[@]};do
VEC=$core
if [ $VEC -ge 32 ];then
let "IDX = $VEC / 32"
MASK_FILL=""
MASK_ZERO="00000000"
for ((i=1; i<=$IDX;i++))
do
MASK_FILL="${MASK_FILL},${MASK_ZERO}"
done
let "VEC -= 32 * $IDX"
MASK_TMP=$((1<<$VEC))
MASK=$(printf "%X%s" $MASK_TMP $MASK_FILL)
else
MASK_TMP=$((1<<$VEC))
MASK=$(printf "%X" $MASK_TMP)
fi
echo $MASK > /proc/irq/$irq/smp_affinity
echo "mask:$MASK, irq:$irq" >> $log_file
core=$(((core+1)%(node_cpumax+1)))
done
}
# stop irqbalance service
function stop_irqblance()
{
log_file=$1
ret=0
if [ "X" != "X$(ps -ef | grep irqbalance | grep -v grep)" ]; then
if which systemctl;then
systemctl stop irqbalance
else
service irqbalance stop
fi
if [ $? -ne 0 ]; then
echo "Failed to stop irqbalance" >> $log_file
ret=1
fi
else
echo "OK. irqbalance stoped." >> $log_file
fi
return $ret
}
# main logic
function main()
{
ecs_network_log=/var/log/ecs_network_optimization.log
ret_value=0
echo "running $0" > $ecs_network_log
echo "======== ECS network setting starts $(date +'%Y-%m-%d %H:%M:%S') ========" >> $ecs_network_log
# we assume your NIC interface(s) is/are like eth*
eth_dirs=$(ls -d /sys/class/net/eth*)
if [ "X$eth_dirs" = "X" ]; then
echo "ERROR! can not find any ethX in /sys/class/net/ dir." >> $ecs_network_log
ret_value=1
fi
for i in $eth_dirs
do
cur_eth=$(basename $i)
echo "optimize network performance: current device $cur_eth" >> $ecs_network_log
# only optimize virtio_net device
driver=$(basename $(readlink $i/device/driver))
if ! echo $driver | grep -q virtio; then
echo "ignore device $cur_eth with driver $driver" >> $ecs_network_log
continue
fi
echo "set and check multiqueue on $cur_eth" >> $ecs_network_log
set_check_multiqueue $cur_eth $ecs_network_log
if [ $? -ne 0 ]; then
echo "Failed to set multiqueue on $cur_eth" >> $ecs_network_log
ret_value=1
fi
done
stop_irqblance $ecs_network_log
set_irq_smpaffinity $ecs_network_log
echo "======== ECS network setting END $(date +'%Y-%m-%d %H:%M:%S') ========" >> $ecs_network_log
return $ret_value
}
# program starts here
main
exit $?查询的rps绑定情况的脚本 get_rps.sh
#!/bin/bash # 获取当前rps情况 for i in $(ls /sys/class/net/eth0/queues/rx-*/rps_cpus); do echo $i cat $i done
参考文档:
https://help.aliyun.com/knowledge_detail/52559.html 【推荐阅读】
https://www.jianshu.com/p/09bb5d5a72ba
https://help.aliyun.com/document_detail/25378.html#g6
https://tech.meituan.com/2018/03/16/redis-high-concurrency-optimization.html 【推荐阅读】
