使用haproxy实现反向代理负载均衡实战

环境准备：两台虚拟机

# yum install -y gcc glibc gcc-c++ make screen tree lrzsz

node1源码编译安装haproxy

[root@node1 ~]# cd /usr/local/src

[root@node1 src]# wget http://www.haproxy.org/download/1.7/src/haproxy-1.7.5.tar.gz

[root@node1 src]# tar zxf haproxy-1.7.5.tar.gz

[root@node1 src]# cd haproxy-1.7.5

[root@node1 haproxy-1.7.5]# make TARGET=linux2628 PREFIX=/usr/local/haproxy-1.7.5

[root@node1 haproxy-1.7.5]# make install

[root@node1 haproxy-1.7.5]# cp /usr/local/sbin/haproxy /usr/sbin/

[root@node1 haproxy-1.7.5]# haproxy -v

HA-Proxy version 1.7.5 2017/04/03

Copyright 2000-2017 Willy Tarreau <willy@haproxy.org>

编辑Haproxy启动脚本

[root@node1 haproxy-1.7.5]# cp examples/haproxy.init /etc/init.d/haproxy

[root@node1 haproxy-1.7.5]# chmod 755 /etc/init.d/haproxy

针对配置文件的路径创建以下文件

[root@node1 haproxy-1.7.5]# useradd -r haproxy

[root@node1 haproxy-1.7.5]# mkdir /etc/haproxy

[root@node1 haproxy-1.7.5]# mkdir /var/lib/haproxy

[root@node1 haproxy-1.7.5]# mkdir /var/run/haproxy

编辑haproxy配置文件，配置log，并启动

[root@linux-node1 haproxy]# vim /etc/haproxy/haproxy.cfg

globallog 127.0.0.1 local3 infochroot /var/lib/haproxyuser haproxygroup haproxydaemondefaultslog globalmode httpoption httplogoption dontlognulltimeout connect 5000timeout client 50000timeout server 50000frontend www_chinasoft_commode httpbind *:80stats uri /haproxy?statsdefault_backend www_chinasoft_backendbackend www_chinasoft_backendoption httpchk GET /index.htmlbalance roundrobinserver node1 192.168.3.140:8080 check inter 2000 rise 3 fall 3 weight 5server node2 192.168.3.200:8080 check inter 2000 rise 3 fall 3 weight 5server mini1 192.168.3.12:8080 check inter 2000 rise 3 fall 3 weight 5

 

*******************************************************

global #全局配置，在所有配置段中都生效

log 127.0.0.1 local3 info #记录日志

chroot /var/lib/haproxy

user haproxy

group haproxy

daemon

defaults #默认配置，可以被前端和后端继承

log global #使用global的log设置

mode http #使用http模式，也可以使用tcp模式

option httplog #启动http请求的log

option dontlognull #在日志中不记录空连接（空连接：健康检查的链接）

timeout connect 5000 #长连接超时时间

timeout client 50000 #客户端连接超时

timeout server 50000 #RS连接超时

frontend www_chinasoft_com #前端配置 + 一个配置段的名字（最好不要乱写，和项目直接相关最佳）

mode http #使用http模式，也可以使用tcp模式

bind *:80 #监听80端口

stats uri /haproxy?stats #状态页面dashboard

default_backend www_chinasoft_com_backend #对应的backend名称

backend www_chinasoft_com_backend #对应的frontend的default_backend

#source cookie SERVERID

option httpchk GET /index.html #检测url

balance roundrobin #使用rr负载均衡方式

server node1 192.168.3.140:8080 check inter 2000 rise 3 fall 3 weight 5

server node2 192.168.3.200:8080 check inter 2000 rise 3 fall 3 weight 5

server mini1 192.168.3.12:8080 check inter 2000 rise 3 fall 3 weight 1 #RS健康检测时间间隔2秒，重试三次，失败三次不可用，权重1

*******************************************************

打开haproxy的日志

# vim /etc/rsyslog.conf15 $ModLoad imudp #打开注释16 $UDPServerRun 514 #打开注释74 local3.* /var/log/haproxy.log #local3的路径

[root@node1 haproxy-1.7.5]# /etc/init.d/haproxy start

[root@node1 haproxy-1.7.5]# systemctl restart rsyslog.service

[root@node1 haproxy-1.7.5]# touch /var/log/haproxy.log

[root@node1 haproxy-1.7.5]# chown -R haproxy.haproxy /var/log/haproxy.log

[root@node1 haproxy-1.7.5]# /etc/init.d/haproxy restart

Restarting haproxy (via systemctl): [ OK ]

[root@node1 haproxy-1.7.5]# tail -f /var/log/haproxy.log

May 4 03:23:50 localhost haproxy[5793]: Stopping frontend www_chinasoft_com in 0 ms.

May 4 03:23:50 localhost haproxy[5793]: Stopping backend www_chinasoft_backend in 0 ms.

May 4 03:23:50 localhost haproxy[5793]: Proxy www_chinasoft_com stopped (FE: 0 conns, BE: 0 conns).

May 4 03:23:50 localhost haproxy[5793]: Proxy www_chinasoft_backend stopped (FE: 0 conns, BE: 0 conns).

May 4 03:23:50 localhost haproxy[5848]: Proxy www_chinasoft_com started.

[root@node1 ~]# sed -i 's/index.html/chinasoft.html/g' /etc/haproxy/haproxy.cfg

[root@node1 ~]# /etc/init.d/haproxy restart

Restarting haproxy (via systemctl): [ OK ]

[root@node1 ~]#

Message from syslogd@localhost at May 4 12:20:49 ...

haproxy[6076]: backend www_chinasoft_backend has no server available!

下面是检测url和uri的几种方式

option httpchk

option httpchk <uri>

option httpchk <method> <uri>

option httpchk <method> <uri> <version>

更改配置文件获取客户端的真实ip

在banckend配置段加入一个option

option forwardfor header X-REAL-IP #X-REAL-IP是自定义的一个名称

通过acl设置虚拟主机，一个前端可以对应多个后端，而实际生产环境建议一个frontend对应一个backend，并重载（生产不建议restart，restart会断开现有链接）

[root@node1 ~]# cat /etc/haproxy/haproxy.cfg globallog 127.0.0.1 local3 infochroot /var/lib/haproxyuser haproxygroup haproxydaemondefaultslog globalmode httpoption httplogoption dontlognulltimeout connect 5000timeout client 50000timeout server 50000frontend www_chinasoft_commode httpbind *:80stats uri /haproxy?statsdefault_backend www_chinasoft_backend    # 默认的backendacl other_chinasoft_com hdr_end(host) other.chinasoft.com    # other_chinasoft_com：给此acl起一个名字；hdr(host)：固定格式，用来识别host，如果没有匹配到acl，即访问default的bankcend    use_backend other_chinasoft_com_backend if other_chinasoft_combackend www_chinasoft_backendoption forwardfor header X-REAL-IPoption httpchk GET /index.htmlbalance roundrobinserver node1 192.168.3.140:8080 check inter 2000 rise 3 fall 3 weight 5backend other_chinasoft_com_backendoption forwardfor header X-REAL-IPoption httpchk GET /index.htmlbalance roundrobinserver node2 192.168.3.200:8080 check inter 2000 rise 3 fall 3 weight 1server mini1 192.168.3.12:8080 check inter 2000 rise 3 fall 3 weight 1

 

在本地电脑使用host解析

192.168.3.140 www.chinasoft.com 192.168.3.140 other.chinasoft.com 192.168.3.12 other.chinasoft.com

通过浏览器访问不同的域名

在fortend添加acl，根据静态文件，设置不同的backend（类似于location），注释的两行和前两行意义相同，分别是通过url正则匹配和url的后缀匹配

frontend www_chinasoft_commode httpbind *:80stats uri /haproxy?statsdefault_backend www_chinasoft_backendacl other_chinasoft_com hdr_end(host) other.chinasoft.comuse_backend other_chinasoft_com_backend if other_chinasoft_com#acl is_static_reg url_reg /*.(css|jpg|jpeg|png|js|gif)$#use_backend other_chinasoft_com_backend if is_static_regacl is_static_path path_end .gif .png .css .jpg .jpeguse_backend other_chinasoft_com_backend if is_static_path

[root@mini1 ~]# echo 'this is static test page <br> 192.168.3.12' > /var/www/html/hello.js

[root@mini3 ~]# echo 'this is static test page <br> 192.168.3.200' > /var/www/html/hello.js

[root@node1 html]# /etc/init.d/haproxy restart

Restarting haproxy (via systemctl): [ OK ]

其他形式的acl，正则或者UA（可以理解为nginx的location）,更多形式的acl，请参考：http://cbonte.github.io/haproxy-dconv/configuration-1.6.html#acl

acl is_do_path url_reg /chuck.douse_backend other_chuck-blog_com_backend if is_do_pathacl is_UA_path hdr_reg(User-Agent) -i andrioduse_backend other_chuck-blog_com_backend if is_UA_path

四、haproxy的动态维护

在配置文件添加socket

[root@node1 html]# head -8 /etc/haproxy/haproxy.cfg

global

log 127.0.0.1 local3 info

chroot /var/lib/haproxy

user haproxy

group haproxy

daemon

stats socket /var/lib/haproxy/haproxy.sock mode 600 level admin # 指定socket文件路径，权限，管理级别

stats timeout 2m # 指定超时时间

[root@node1 html]# /etc/init.d/haproxy restart

Restarting haproxy (via systemctl): [ OK ]

[root@node1 html]# ll /var/lib/haproxy/

total 0

srw------- 1 root root 0 May 4 14:38 haproxy.sock

安装socat

[root@node1 ~]# yum install -y socat

使用help查看socat的事情

[root@node1 ~]# echo 'help' | socat stdio /var/lib/haproxy/haproxy.sock Unknown command. Please enter one of the following commands only :help : this messageprompt : toggle interactive mode with promptquit : disconnectdisable agent : disable agent checks (use 'set server' instead)disable health : disable health checks (use 'set server' instead)disable server : disable a server for maintenance (use 'set server' instead)enable agent : enable agent checks (use 'set server' instead)enable health : enable health checks (use 'set server' instead)enable server : enable a disabled server (use 'set server' instead)set maxconn server : change a server's maxconn settingset server : change a server's state, weight or addressget weight : report a server's current weightset weight : change a server's weight (deprecated)disable frontend : temporarily disable specific frontendenable frontend : re-enable specific frontendset maxconn frontend : change a frontend's maxconn settingshow servers state [id]: dump volatile server information (for backend 
     
      )show backend : list backends in the current running configshutdown frontend : stop a specific frontendclear table : remove an entry from a tableset table [id] : update or create a table entry's datashow table [id]: report table usage stats or dump this table's contentsshow errors : report last request and response errors for each proxyclear counters : clear max statistics counters (add 'all' for all counters)show info : report information about the running processshow stat : report counters for each proxy and servershow sess [id] : report the list of current sessions or dump this sessionshutdown session : kill a specific sessionshutdown sessions server : kill sessions on a servershow pools : report information about the memory pools usageadd acl : add acl entryclear acl 
      
        : clear the content of this acldel acl : delete acl entryget acl : report the patterns matching a sample for an ACLshow acl [id] : report available acls or dump an acl's contentsadd map : add map entryclear map 
       
         : clear the content of this mapdel map : delete map entryget map : report the keys and values matching a sample for a mapset map : modify map entryshow map [id] : report available maps or dump a map's contentsshow stat resolvers [id]: dumps counters from all resolvers section andassociated name serversset maxconn global : change the per-process maxconn settingset rate-limit : change a rate limiting valueset timeout : change a timeout settingshow env [var] : dump environment variables known to the process
       
      
     

 

查看info信息，内容值可以利用来监控

[root@node1 ~]# echo "show info" |socat stdio /var/lib/haproxy/haproxy.sock Name: HAProxyVersion: 1.7.5Release_date: 2017/04/03Nbproc: 1Process_num: 1Pid: 6902Uptime: 0d 0h03m26sUptime_sec: 206Memmax_MB: 0PoolAlloc_MB: 0PoolUsed_MB: 0PoolFailed: 0Ulimit-n: 4034Maxsock: 4034Maxconn: 2000Hard_maxconn: 2000CurrConns: 0CumConns: 3CumReq: 3Maxpipes: 0PipesUsed: 0PipesFree: 0ConnRate: 0ConnRateLimit: 0MaxConnRate: 0SessRate: 0SessRateLimit: 0MaxSessRate: 0CompressBpsIn: 0CompressBpsOut: 0CompressBpsRateLim: 0Tasks: 9Run_queue: 1Idle_pct: 100node: node1

关闭linux-node2主机

[root@node1 ~]# echo "disable server other_chinasoft_com_backend/node2" |socat stdio /var/lib/haproxy/haproxy.sock

可以看到node2进入了维护（maintain）状态

 

打开node2主机(只对现有已经写到配置文件中的server生效，不能用来新增节点)

[root@node1 ~]# echo "enable server other_chinasoft_com_backend/node2" |socat stdio /var/lib/haproxy/haproxy.sock

 

五、生产环境遇到的问题

haproxy的本地端口可能用尽，解决方案如下4条

1）更改local的端口范围,调整内核参数

[root@node1 ~]# cat /proc/sys/net/ipv4/ip_local_port_range

32768 60999

2）调整timewait的端口复用，设置为1

[root@node1 ~]# cat /proc/sys/net/ipv4/tcp_tw_reuse

0

3)缩短tcp_wait的时间，不建议修改

[root@node1 ~]# cat /proc/sys/net/ipv4/tcp_fin_timeout

60

4)终极方案：增加为多个ip，自然端口数就够了