一，tengine 安装和配置1.基础库安装：pcre，openssl，geoip，luajit#mkdir -p /var/log/nginx#mkdir -p /var/nginx_temp/{nginx_client,nginx_proxy,nginx_fastcgi,nginx_temp}#cd /data/soft#wget #tar zxvf pcre-8.36.tar.gz#cd pcre-8.36#./configure --prefix=/usr/local#make && make install#cd /data/soft#wget #cd openssl-1.0.1m./config shared --prefix=/usr/local#make && make install#cd /data/soft#wget #tar zxvf GeoIP-1.6.5.tar.gz#cd GeoIP-1.6.5./configure --prefix=/usr/local#make && make install#cd /data/soft#wget #tar zxvf LuaJIT-2.0.4.tar.gz#cd LuaJIT-2.0.4#make PREFIX=/usr/localmake install #cd /data/soft#tar xvzf tengine-1.5.2.tar.gz#cd tengine-1.5.2#./configure --prefix=/opt/nginx \--lock-path=/var/lock/nginx.lock \--pid-path=/var/run/nginx.pid \--error-log-path=/var/log/nginx/error.log \--http-log-path=/var/log/nginx/access.log \--user=nobody \--group=nobody \--with-pcre=../pcre-8.36 \--with-pcre-opt=-fPIC \--with-openssl=../openssl-1.0.1m \--with-openssl-opt=-fPIC \--with-backtrace_module \--with-http_stub_status_module \--with-http_gzip_static_module \--with-http_realip_module \--with-http_concat_module=shared \--with-http_sysguard_module=shared \--with-http_limit_conn_module=shared \--with-http_limit_req_module=shared \--with-http_split_clients_module=shared \--with-http_footer_filter_module=shared \--with-http_geoip_module=shared \--with-http_sub_module=shared \--with-http_access_module=shared \--with-http_upstream_ip_hash_module=shared \--with-http_upstream_least_conn_module=shared \--with-http_referer_module=shared \--with-http_rewrite_module=shared \--with-http_memcached_module=shared \--with-http_upstream_session_sticky_module=shared \--with-http_addition_module=shared \--with-http_xslt_module=shared \--with-http_p_w_picpath_filter_module=shared \--with-http_user_agent_module=shared \--with-http_empty_gif_module=shared \--with-http_browser_module=shared \--with-http_map_module=shared \--with-http_userid_filter_module=shared \--with-http_charset_filter_module=shared \--with-http_trim_filter_module=shared \--with-http_lua_module=shared \--without-http_fastcgi_module \--without-http_uwsgi_module \--without-http_scgi_module \--without-select_module \--without-poll_module \-with-http_fastcgi_module=shared \--with-http_uwsgi_module=shared \--http-client-body-temp-path=/var/nginx_temp/nginx_client \--http-proxy-temp-path=/var/nginx_temp/nginx_proxy \--http-fastcgi-temp-path=/var/nginx_temp/nginx_fastcgi \ --with-cc-opt='-g -O2 -fstack-protector --param=ssp-buffer-size=4 -Wformat -Werror=format-security' --with-ld-opt='-Wl,-Bsymbolic-functions -Wl,-z,relro’ # make && make install#mkdir -p /opt/nginx/conf/{upstream,vhost.d}#cd /opt#mv nginx tengine-1.5.2 && ln -s tengine-1.5.2 nginx#rm -f /opt/nginx/conf/*.default2.tengine 主配置#vi /opt/nginx/conf/nginx.confuser nobody nobody; worker_processes auto;worker_cpu_affinity auto; error_log /var/log/nginx/error.log crit;pid /var/run/nginx.pid; worker_rlimit_nofile 65535; dso { load ngx_http_rewrite_module.so; load ngx_http_access_module.so; load ngx_http_limit_conn_module.so; load ngx_http_limit_req_module.so; load ngx_http_sysguard_module.so; load ngx_http_lua_module.so;} events { use epoll; worker_connections 10240;} http { server_tokens off; server_tag off; autoindex off; access_log off; include mime.types; default_type application/octet-stream; server_names_hash_bucket_size 128; client_header_buffer_size 32k; large_client_header_buffers 4 32k; client_max_body_size 30m; client_body_buffer_size 256k; sendfile on; tcp_nopush on; keepalive_timeout 60; tcp_nodelay on; gzip on; gzip_min_length 1k; gzip_buffers 4 16k; gzip_http_version 1.1; gzip_comp_level 6; gzip_types text/plain application/x-javascript text/css application/xml application/javascript text/javascript p_w_picpath/jpeg p_w_picpath/gif p_w_picpath/png; gzip_vary on; proxy_connect_timeout 120; proxy_read_timeout 120; proxy_send_timeout 120; proxy_buffer_size 128k; proxy_buffers 4 128k; proxy_busy_buffers_size 256k; proxy_temp_file_write_size 256k; proxy_headers_hash_max_size 1024; proxy_headers_hash_bucket_size 128; proxy_redirect off; proxy_set_header Host $host; proxy_set_header X-Real-IP $remote_addr; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; proxy_temp_path /var/nginx_temp/nginx_temp; log_format main '$remote_addr - $remote_user [$time_local] "$request" ' '$status $body_bytes_sent "$http_referer" ' '"$http_user_agent" "$http_x_forwarded_for" ' '$request_time';log_format ha '$remote_addr\t-\t$remote_user\t[$time_local]\t"$request"\t' '$status $body_bytes_sent\t"$http_referer"\t' '"$http_user_agent"\t$request_time\t' '$cookie_user_id__baidu_com\t$cookie_place_id__baidu_com\t' '$cookie_elatitude\t$cookie_elongitude\t' '$cookie_fixed_cid\t$cookie_temp_cid\t$http_x_requested_with\t$cookie_track_id\t$sent_http_content_type\t' '$http_x_baidu_requestid\t$host'; server { sysguard on; sysguard_load load=50 action=/loadlimit; sysguard_mem swapratio=24% action=/swaplimit; sysguard_mem free=100M action=/freelimit; location /loadlimit { return 503; } location /swaplimit { return 503; } location /freelimit { return 503; } } include upstream/*.conf; include vhost.d/*.conf;}3. 负载均衡池配置#vim /opt/nginx/conf/upstream/baidi.comupstream b_baidi_SERVER { consistent_hash $request_uri; server 100.10.111.210:8080 id=101 weight=8; server 100.10.111.180:8080 id=102 weight=10; server 100.10.210.139:8080 id=103 weight=10; server 100.10.112.163:8080 id=104 weight=5; check interval=3000 rise=2 fall=3 timeout=1000 type=http port=8080; check_http_send "OPTIONS http://baidi.com/api/check; check_http_expect_alive http_2xx http_3xx;}4vhost 设置#vi /opt/nginx/conf/vhost.d/baidi.com.confserver { listen 80; listen 443 ssl; server_name baidi.com; ssl_certificate /opt/nginx/conf/ssl/baidi.crt; ssl_certificate_key /opt/nginx/conf/ssl/baidi.key; ssl_protocols TLSv1 TLSv1.1 TLSv1.2; set $rewrite_status 0; if ($https_status = off) { set $rewrite_status "${rewrite_status}1"; } if ($scheme = http) { set $rewrite_status "${rewrite_status}2"; } if ($https = on) { set $https_status $https; } if ($rewrite_status = 012) { rewrite / https://$http_host$request_uri permanent; break; } access_log /var/log/nginx/baidi.com.access.log ha; error_log /var/log/nginx/baidi.com.error.log; add_header Access-Control-Allow-Origin *; add_header Access-Control-Allow-Credentials true; add_header Access-Control-Allow-Headers Content-Type; add_header Access-Control-Allow-Methods POST; location / { proxy_next_upstream http_502 http_504 error timeout invalid_header; proxy_redirect off; proxy_set_header Host $host; proxy_set_header X-Real-IP $remote_addr; proxy_set_header REMOTE-HOST $remote_addr; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; proxy_set_header X-Forwarded-For $remote_addr; proxy_pass http://b_baidi_SERVER; proxy_set_header Host $host; } location ~ health_status { check_status; allow 127.0.0.1; allow 10.0.0.0/8; allow 172.0.0.0/8; allow 192.168.0.0/16; deny all; }}4 启动脚本#vim /etc/init.d/nginx#!/bin/sh## nginx - this script start and stop the nginx daemon## chkconfig: 2345# description: Startup script for nginx# processname: nginx# config: /opt/nginx/conf/nginx.conf# pidfile: /var/run/nginx.pid## code by rocketzhang#PATH=/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin DAEMON="/opt/nginx/sbin/nginx"CNFFILE="/opt/nginx/conf/nginx.conf"PIDFILE="/var/run/nginx.pid"LOCKFILE="/var/lock/nginx.lock" RETVAL=0 set -e[[ -x "$DAEMON" ]] || exit 0 start() { [[ -x $DAEMON ]] || exit 5 [[ -f $CNFFILE ]] || exit 6 echo -n "Startting Nginx......" $DAEMON -c $CNFFILE || echo -n "Nginx already running!" RETVAL=$? echo [[ $RETVAL -eq 0 ]] && touch $LOCKFILE} stop() { echo -n "Stopping Nginx......" if [[ -n `ps aux | grep nginx | awk '/master/{print $2}'` ]]; then kill -QUIT `ps aux | grep nginx | awk '/master/{print $2}'` RETVAL=$? echo [[ $RETVAL -eq 0 ]] && rm -f $LOCKFILE $PIDFILE fi} reload() { echo -n "Reloading Nginx......" if [[ -n `ps aux | grep nginx | awk '/master/{print $2}'` ]]; then kill -HUP `ps aux | grep nginx | awk '/master/{print $2}'` RETVAL=$? echo fi} case "$1" instart) start ;; stop) stop ;; reload) reload ;; restart) stop sleep 1 start ;; *) echo "Usage: service nginx {start|stop|reload|restart}" RETVAL=1 ;;esac exit $RETVAL5.# 服务启动 #chmod +x /etc/init.d/nginx#chkconfig nginx on#service nginx start主节点配置二，keepaived 安装和配置1.基础安装#yum -y install keepalived#vim /etc/keepalived/keepalived.confglobal_defs { notification_email { rickyhui@yahoo.com } notification_email_from yanming.zhang@ele.me smtp_server 127.0.0.1 smtp_connect_timeout 30 router_id NGX_DEVEL1} vrrp_script check_nginx_health { script "/opt/scripts/check_nginx_for_keepalived.sh" interval 1 weight -2} vrrp_instance VI_1 { state MASTER interface bond0 virtual_router_id 51 priority 100 advert_int 1 smtp_alert authentication { auth_type PASS auth_pass 123456 } virtual_ipaddress { 10.0.20.200 } track_script { check_nginx_health } notify_master "/opt/scripts/notify_for_keepalived.sh master" notify_bakcup "/opt/scripts/notify_for_keepalived.sh backup" notify_fault "/opt/scripts/notify_for_keepalived.sh fault"}备节点配置#vim /etc/keepalived/keepalived.confglobal_defs { notification_email { rickyhui@yahoo.com } notification_email_from rickyhui@yahoo.com smtp_server 127.0.0.1 smtp_connect_timeout 30 router_id NGX_DEVEL2} vrrp_script check_nginx_health { script "/opt/scripts/check_nginx_for_keepalived.sh" interval 1 weight -2} vrrp_instance VI_1 { state BACKUP interface bond0 virtual_router_id 51 priority 90 advert_int 1 smtp_alert authentication { auth_type PASS auth_pass 123456 } virtual_ipaddress { 10.0.20.200 } track_script { check_nginx_health } notify_master "/opt/scripts/notify_for_keepalived.sh master" notify_bakcup "/opt/scripts/notify_for_keepalived.sh backup" notify_fault "/opt/scripts/notify_for_keepalived.sh fault"}#脚本 #vim /opt/scripts/check_nginx_for_keepalived.sh#!/bin/shPATH=/sbin:/bin:/usr/sbin:/usr/bin:/usr/local/bin:/usr/local/sbin if [[ -z `ps aux | grep 'nginx:' | grep -v grep` ]]; then /etc/init.d/nginx start && sleep 5 if [[ -z `ps aux | grep 'nginx:' | grep -v grep` ]]; then /sbin/service keepalived stop fifiVIP 漂移短信通知脚本# vim /opt/scripts/notify_for_keepalived.sh#!/bin/shPATH=/sbin:/bin:/usr/sbin:/usr/bin:/usr/local/bin:/usr/local/sbin KEEPALIVE_CONF="/etc/keepalived/keepalived.conf"VIP=`grep -A 1 virtual_ipaddress ${KEEPALIVE_CONF} | tail -1 | sed 's/\t//g; s/ //g'` ## 短信发送API_KEY="xxxxxxx"URL=" http://baidi.com/api/send/"PHONE_NUM_FILE="/opt/scripts/phone_num_list.txt" function SEND_SMS_INFO() { MESSAGE="$1" [[ ! -e $PHONE_NUM_FILE || -z `sed '/^$/d' $PHONE_NUM_FILE` ]] && exit 1 [[ -n "$MESSAGE" ]] || exit 1 while read PHONE_INFO do PHONE_NUM=`echo $PHONE_INFO | awk -F '|' '{print $1}'` curl -H "X-Authentication-Token:${TOKEN}" -d "sender_key=${API_KEY}&receiver=${PHONE_NUM}&message=${MESSAGE}" "${URL}" done < $PHONE_NUM_FILE} case "$1" inmaster|backup|fault) SEND_SMS_INFO "(`hostname`) The search proxy server's vip has changed to master $1" exit 0 ;; *) echo "Usage: `basename $0` {master|backup|fault}" exit 1esac# service keepalived start注意：最终的实现要求是，VIP地址必须随着nginx服务的正常启动而漂移，否则就没有意义。比如：主节点的nginx服务挂了，但没有被正常拉起来，而keepalived服务是正常的，此时，由于主节点keepalived的优先级高，VIP地址就会漂移到主节点，就会导致无法正常提供服务，所以必须考虑这种特殊情况。