HaProxy

Keepalived + Haproxy with SELinux

昨天和開發同仁討論到有一些內部的服務需要開放給外部人員或供應商使用，要考慮未來內部服務的接入或改版，這個架構其實用到快爛了，用反向代理(Reverse Proxy) 可輕易解決，又考量冗餘性，所以 keepalived 也是必要的，至少在單一主機故障或服務異常時能主動切換，爭取些時間讓系統人員能進一步查找問題及修復，過去這樣的架構也用的很熟悉，不過過去大多將　　SELinux 關閉以省的麻煩，但總覺的缺了一點什麼，因此這次開啟 SELinux 後果然遇到了幾個 Permission Denied的 問題, 查了一下文檔，其實加上 SELinux Tags 就可以解決。下面記錄一下便於自己下次可重覆使用。

準備環境：  VM: haproxy1 (192.168.122.211)
          VM: haproxy2 (192.168.122.212)
          IP: VIP(192.168.122.210)

１. 安裝架設 haproxy+keepalived
haproxy1 / haproxy2
安裝套件

# yum install haproxy keepalived -y
# Adding to kernel parameters
echo 1024 60999 > /proc/sys/net/ipv4/ip_local_port_range
echo 30 > /proc/sys/net/ipv4/tcp_fin_timeout
echo 4096 > /proc/sys/net/ipv4/tcp_max_syn_backlog
echo 262144 > /proc/sys/net/ipv4/tcp_max_tw_buckets
echo 262144 > /proc/sys/net/ipv4/tcp_max_orphans
echo 300 > /proc/sys/net/ipv4/tcp_keepalive_time
echo 1 > /proc/sys/net/ipv4/tcp_tw_recycle
echo 0 > /proc/sys/net/ipv4/tcp_timestamps
echo 0 > /proc/sys/net/ipv4/tcp_ecn
echo 1 > /proc/sys/net/ipv4/tcp_sack
echo 0 > /proc/sys/net/ipv4/tcp_dsack

設置配置檔
Master(haproxy1)

haproxy

#---------------------------------------------------------------------
# Example configuration for a possible web application. See the
# full configuration options online.
#
# http://haproxy.1wt.eu/download/1.4/doc/configuration.txt
#
#---------------------------------------------------------------------

#---------------------------------------------------------------------
# Global settings
#---------------------------------------------------------------------
global
# to have these messages end up in /var/log/haproxy.log you will
# need to:
#
# 1) configure syslog to accept network log events. This is done
# by adding the '-r' option to the SYSLOGD_OPTIONS in
# /etc/sysconfig/syslog
#
# 2) configure local2 events to go to the /var/log/haproxy.log
# file. A line like the following can be added to
# /etc/sysconfig/syslog
#
# local2.* /var/log/haproxy.log
#
log 127.0.0.1 local2

chroot /var/lib/haproxy
nbproc 1
#cpu-map 1 0
#cpu-map 2 1
pidfile /var/run/haproxy.pid
maxconn 10000
user haproxy
group haproxy
daemon
ssl-server-verify none
tune.ssl.default-dh-param 2048
# turn on stats unix socket
stats socket /var/lib/haproxy/stats

#---------------------------------------------------------------------
# common defaults that all the 'listen' and 'backend' sections will
# use if not designated in their block
#---------------------------------------------------------------------
defaults
mode http
log global
option httplog
#option dontlognull
option http-keep-alive
#option http-server-close
#option forwardfor except 127.0.0.0/8
compression algo gzip
option redispatch
retries 3
timeout http-request 10s
timeout queue 1m
timeout connect 10s
timeout client 1m
timeout server 1m
timeout http-keep-alive 10s
timeout check 10s
maxconn 10000
#capture request header X-Forwarded-For len 500
#capture request header Host len 500
#capture request header X-Request-UID len 500


#---------------------------------------------------------------------
# main frontend which proxys to the backends
#---------------------------------------------------------------------
frontend http-in
mode http
bind *:80
# Important Change: Production or Staging
# bind *:443 ssl crt /etc/pki/tls/certs/xxx-SelfSigned.pem no-sslv3 # 2048 # Production


capture request header X-Forwarded-For len 500
capture request header Referer len 2083
capture request header User-Agent len 200
http-response replace-header X-Forwarded-For (.*),\+(.*) \1


# ACL Section
# EC 1.0 ACL
# HTTPS
acl fugo_svc1 url_beg -i /WebService1/ 

# ACL for virtual hosting 
acl fugo_web hdr(host) -i xxx.ccc.com.tw
acl www_web hdr(host) -i www.ccc.com.tw


use_backend backend_fugo_svc1 if fugo_svc1
default_backend backend_default # Default Backend

# ---------------------------------------------------------------
# Backend Start
# ---------------------------------------------------------------

backend backend_fugo_svc1 # Fugo Service 1 設定
mode http
balance roundrobin
#option forwardfor
http-request set-header X-Forward-Port %[dst_port]
http-request set-header X-Client-IP %[src]
http-response replace-header X-Forwarded-For (.*),\+(.*) \1
http-response replace-header X-Client-IP (.*),\+(.*) \1
server http-webservice1-fugo xx.xx.xx.xx:80 check

backend backend_default # Fugo Service 1 設定
mode http
balance roundrobin
#option forwardfor
http-request set-header X-Forward-Port %[dst_port]
http-request set-header X-Client-IP %[src]
http-response replace-header X-Forwarded-For (.*),\+(.*) \1
http-response replace-header X-Client-IP (.*),\+(.*) \1
server http-webservice1-fugo xx.xx.xx.xx:80 check


listen stats
mode http
bind *:8080
stats refresh 30s
stats enable
stats uri /haproxy?stats
stats hide-version
stats auth admin:[noP@ssword]

keepalived

/etc/keepalived/keepalived.conf

! Configuration File for keepalived

global_defs {
   #notification_email {
   #  jerry.w@xxx.com.tw
   #}
   #notification_email_from jerry.w@xxx.com.tw
   #smtp_server 192.168.200.1
   #smtp_connect_timeout 30
   router_id LVS_DEVEL
   vrrp_skip_check_adv_addr
   vrrp_strict
   vrrp_garp_interval 0
   vrrp_gna_interval 0
}

vrrp_script chk_haproxy {

  script "/etc/keepalived/check_haproxy.sh"
  interval 5
  weight 2
}

vrrp_instance VI_1 {
    state MASTER             # [MASTER|BACKUP]
    interface eth0
    virtual_router_id 51     # 同一個 Segment 不能 ID 衝突 
    priority 150
    nopreempt
    advert_int 5
    authentication {
        auth_type PASS
        auth_pass xxxx
    }
    track_script {
        chk_haproxy
    }
    virtual_ipaddress {
        192.168.122.210
    }
    notify_master "/etc/keepalived/clean_arp.sh 192.168.122.211"
}

/etc/keepalived/clean_arp.sh

#!/bin/sh
VIP=$1
GATEWAY=192.168.122.1 
/sbin/arping -I eth0 -c 5 -s $VIP $GATEWAY &>/dev/null

/etc/keepalived/check_haproxy.sh

#!/bin/bash
if [ $(ps -C haproxy --no-header | wc -l) -eq 0 ]; then
  /usr/bin/systemctl start haproxy
  sleep 3

  if [ $(ps -C haproxy --no-header | wc -l) -eq 0 ]; then
    /usr/bin/systemctl stop keepalived
  fi
fi

Backup(haproxy2)

haproxy

同 haproxy1

keepalived

  參照 haproxy1修改 state 欄位

2.開啟相關防火牆規則 (haproxy service port & vrrp)

# CentOS firewalld, IPTables 則自行對應 
# firewall-cmd --add-rich-rule='rule protocol value="vrrp" accept' --permanent
# firewall-cmd --add-port=8080/tcp --add-port=80/tcp --permanent;firewall-cmd --reload
4. 啟動 keepalived 服務
# systemctl enable keepalived
# systemctl start keepalived
  1. 除錯 avc denied 問題
# chcon -t keepalived_unconfined_script_exec_t /etc/keepalived/check_haproxy.sh
# chcon -t keepalived_unconfined_script_exec_t /etc/keepalived/clean_arp.sh

Author: jerryw1974

learning and focus on computer science, cloud infrastructure, virtualization and information security, technical, networking,platform system and cyber-security related topic.