centos 7.x 安装 yugabytedb 的优化设置

太过爱你忘了你带给我的痛 2022-10-30 13:23 230阅读 0赞

os: centos 7.4.1708

# yum #

centos-release-scl  
epel

# yum install -y centos-release-scl;
    wget -P /etc/yum.repos.d/ http://mirrors.aliyun.com/repo/epel-7.repo;
    yum clean all ; 
    yum makecache ; 
    
    yum install -y zip unzip dos2unix pigz iputils iftop git tree sysstat ntp \
    make cmake cmake3 gcc gcc-c++ lrzsz mpstat dstat wget bison unixODBC \
    e2fsprogs e2fsprogs-devel e4fsprogs e4fsprogs-devel xfsprogs \
    smartmontools coreutils glib2 perl perl-devel perl-Pod-Markdown perl-ExtUtils* perl-Text-CSV* \
    zlib zlib-devel openldap openldap-devel flex readline readline-devel \
    systemtap systemtap-sdt-devel python python-devel tcl tcl-devel sgml-common docbook* stylesheets \
    openjade sgml-tools libxslt libxslt-devel libxml2 libxml2-devel openssl openssl-devel \
    pam pam-devel libicu libicu-devel uuid uuid-devel libuuid-devel jadetex bzip2 \
    automake autoconf libtool hdparm numactl libtool-ltdl libtool-ltdl-devel \
    lz4 lz4-devel lz4-static  wget curl \
    devtoolset-7-binutils devtoolset-7-gcc devtoolset-7-gcc-c++ devtoolset-7-libstdc++-devel \
    devtoolset-7-runtime libedit-devel llvm-toolset-7-clang  llvm-toolset-7-clang-libs  \
    llvm-toolset-7-compiler-rt  llvm-toolset-7-libomp  llvm-toolset-7-llvm-libs  \
    llvm-toolset-7-runtime llvm5.0 llvm5.0-devel llvm5.0-libs

python2 生命已经终结，后面考虑安装python3

# yum install -y python3 python3-libs python3-devel python3-pip python3-setuptools
    
    # alternatives --set python /usr/bin/python3

另外参考<<centos 使用 scl 软件集>>

# user #

# groupadd -g 5000 yugabyte;
    useradd -g yugabyte -u 5000 yugabyte -d /home/yugabyte;
    usermod -g yugabyte yugabyte;
    mkdir -p /var/lib/yugabytedb ; 
    chown -R yugabyte:yugabyte /var/lib/yugabytedb;

yugabyte 用户的环境变量，先占位，后面修改

# vi /home/yugabyte/.bash_profile
    
    export PGHOME=/usr/yugabytedb-
    export PGDATA=/var/lib/yugabytedb 
    
    export LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:${PGHOME}/lib
    export LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:/lib:/usr/lib:/usr/local/lib
    export PATH=${PGHOME}/bin:$PATH
    export MANPATH=${PGHOME}/share/man:$MANPATH
    
    export TMP=/tmp
    export TEMP=/tmp
    export TMPDIR=/tmp

# rc.local #

# vi /etc/rc.local
    
    ############################################
    #禁用THP
    ############################################
    if test -f /sys/kernel/mm/transparent_hugepage/enabled; then
       echo never > /sys/kernel/mm/transparent_hugepage/enabled
    fi
    if test -f /sys/kernel/mm/transparent_hugepage/defrag; then
       echo never > /sys/kernel/mm/transparent_hugepage/defrag
    fi
    
    
    ############################################
    #预读块大小
    ############################################
    /usr/sbin/blockdev --setra 16384 /dev/sda
    
    
    ############################################
    #设置 io scheduler
    ############################################
    # noop 
    # deadline
    # cfq
    ############################################
    echo deadline > /sys/block/sda/queue/scheduler
    
    
    ############################################
    #设置 write through
    ############################################
    # yum install hdparm
    #
    #set write type 
    # 0:write through
    # 1:write back
    # hdparm -I /dev/sda |grep "Write cache"
    # hdparm -W 0 /dev/sda
    # hdparm -W 1 /dev/sda
    ############################################
    hdparm -W 0 /dev/sda 
    
    
    ############################################
    #清空 iptables
    ############################################
    iptables -F

记得授予可执行权限

# chmod u+x /etc/rc.d/rc.local

# sysctl.conf #

# vi /etc/sysctl.conf
    
    #共享内存设置,默认已经足够大
    #kernel.shmall = 18446744073692774399    
    #kernel.shmmax = 18446744073692774399    
    kernel.shmmni = 819200 
    
    kernel.sched_migration_cost_ns=50000
    kernel.sched_autogroup_enabled=1
    
    #信号量设置
    #kernel.sem = 250 32000 100 128 #这个设置在很多场合都报错了,有公式计算
    #SEMMSL SEMMNS SEMOPM SEMMNI
    #SEMMSL = 50100
    #SEMMNS = SEMMSL × SEMMNI = 50100 * 1280 = 64128000
    #SEMOPM = SEMMSL = 50100
    #SEMMNI = 1280
    kernel.sem = 50100 64128000 50100 1280
    
    #接收套接字缓冲区大小
    net.core.rmem_default = 262144    
    net.core.rmem_max = 4194304 
    #发送套接字缓冲区大小
    net.core.wmem_default = 262144    
    net.core.wmem_max = 4194304 
    
    fs.aio-max-nr = 1048576
    fs.nr_open=20480000
    fs.file-max = 7672460
    
    net.ipv6.conf.all.disable_ipv6 = 1
    
    net.ipv4.ip_local_port_range = 9000 65000
    net.ipv4.tcp_keepalive_time = 60
    net.ipv4.tcp_keepalive_probes = 9
    net.ipv4.tcp_keepalive_intvl =60
    net.ipv4.tcp_retries2 = 5
    net.ipv4.tcp_mem = 8388608 12582912 16777216
    net.ipv4.tcp_rmem = 8192 87380 16777216    
    net.ipv4.tcp_wmem = 8192 65536 16777216  
    
    #操作系统脏页，当大内存时需要调低
    vm.dirty_ratio = 5 
    vm.dirty_bytes=536870912
    vm.dirty_background_ratio=1
    vm.dirty_background_bytes = 134217728
    vm.dirty_expire_centisecs = 3000 
    vm.dirty_writeback_centisecs = 100
    
    #内存分配方式,允许超分配
    vm.overcommit_memory = 2
    vm.overcommit_ratio = 90
    vm.swappiness = 1
    
    #大页设置,大内存建议启动
    #vm.nr_hugepages = xxxx
    #grep ^Hugepagesize /proc/meminfo

# sysctl -p

# nptdate #

yugabytedb 对时间的要求比较严格

# crontab -e
    
    # 每1分钟
    */1 * * * * /usr/sbin/ntpdate ntp.sjtu.edu.cn >> /var/log/ntpdate.log 2>&1 &

# limits.conf #

https://docs.yugabyte.com/latest/deploy/manual-deployment/system-config/\#ulimits

# vi /etc/security/limits.conf
    
    *                -       core            unlimited
    *                -       data            unlimited
    *                -       fsize           unlimited
    *                -       sigpending      119934
    *                -       memlock         unlimited
    *                -       rss             unlimited
    *                -       nofile          1048576
    *                -       msgqueue        819200
    *                -       stack           unlimited
    *                -       cpu             unlimited
    *                -       nproc           unlimited 
    *                -       locks           unlimited

有个坑需要注意下，里面默认设置了非root用户的最大进程数为4096

# cat /etc/security/limits.d/20-nproc.conf
    
    # Default limit for number of user's processes to prevent
    # accidental fork bombs.
    # See rhbz #432903 for reasoning.
    
    *          soft    nproc     4096
    root       soft    nproc     unlimited

删除该文件，或者修改 4096 为更大值

# rm /etc/security/limits.d/20-nproc.conf

# systemd ulimit #

systemd 的资源限制稍有不同，主要是 centos 对 资源限制做了调整。

# man systemd-system.conf

# vi /etc/systemd/system.conf 
    
    DefaultLimitNOFILE=1024000
    DefaultLimitNPROC=1024000
    DefaultLimitMEMLOCK=infinity
    DefaultLimitCORE=infinity
    DefaultLimitSTACK=infinity
    DefaultLimitAS=infinity
    DefaultLimitFSIZE=infinity
    DefaultLimitSIGPENDING=119934

# systemctl daemon-reload
    # systemctl restart sshd.service
    
    # cat /proc/YOUR-PID/limits

# 文件系统优化 #

xfs

# vi /etc/fstab
    
    /dev/sdd1 /data nodev,discard,noatime,inode64,allocsize=16m,errors=remount-ro 0 0

ext4

# vi /etc/fstab
    
    /dev/sdd1 /data defaults,noatime,nodiratime,nodelalloc,barrier=0 0 0

# selinux disable #

# vi /etc/selinux/config
    SELINUX=DISABLED

或者

# vi /etc/sysconfig/selinux
    SELINUX=DISABLED

# sed -i 's/SELINUX=enforcing/SELINUX=disabled/' /etc/selinux/config

# firewalld disable #