optimize offline detection

Author: janusec2

README.md

@@ -1,4 +1,4 @@
-# [Janusec Application Gateway / JANUSEC应用网关](https://www.janusec.com/)   [![Tweet](https://img.shields.io/twitter/url/http/shields.io.svg?style=social)](https://twitter.com/intent/tweet?text=Protect%20web%20applications%20from%20network%20attacks%20with%20open%20source%20Janusec%20Application%20Gateway&url=https://github.com/Janusec/janusec&via=janusec&hashtags=waf,web,application,firewall,gateway)
+# Janusec Application Gateway / JANUSEC应用网关  
 
 [![Build Status](https://travis-ci.org/Janusec/janusec.svg?branch=master)](https://travis-ci.org/Janusec/janusec)
 
@@ -52,21 +52,20 @@
 
 ## 产品网站   
 
-https://doc.janusec.com/cn/   
+https://janusec.github.io/cn/   
+
 
 ## 需求   
 
-* PostgreSQL 10/11/12+ (开发环境，及生产环境主节点需要)  
-* Debian 9/10+, CentOS/RHEL 7/8+, 首选Debian 10+    
+* PostgreSQL 10/11/12/13/14+ (开发环境，及生产环境主节点需要)  
+* Debian 9/10/11+, CentOS/RHEL 7/8+, 首选Debian 10+    
 * systemd  
 * nftables  
 * Golang 1.15+ (仅开发环境需要)  
 
 ## 部署快速指引    
 
-详细文档可在这里获取： [Janusec应用网关快速入门](https://doc.janusec.com/cn/quick-start/)  
-
-如希望快速体验，可尝试使用 [Docker镜像](https://www.janusec.com/articles/opensource/1615470598.html)    
+详细文档可在这里获取： [Janusec应用网关快速入门](https://janusec.github.io/cn/quick-start/)  
 
 ## 开发快速指引   
 
@@ -103,7 +102,7 @@ Janusec将自动加密数据库口令
 只使用主节点时，任意应用域名均可用于访问管理入口。   
 如果使用了副本节点，应为主节点申请一个单独的域名。   
 
-[Janusec应用网关配置](https://doc.janusec.com/cn/quick-start/)   
+[Janusec应用网关配置](https://janusec.github.io/cn/quick-start/)   
 
 ## 发布 
 
@@ -119,15 +118,15 @@ Janusec将自动加密数据库口令
 
 Web化管理所需的文件在 `./static/janusec-admin/` 目录, 源码在 [Janusec-Admin Github](https://github.com/Janusec/janusec-admin) ，前端源码使用Angular 9.  
 
-## 许可证  
+## 多许可证  
 
-Janusec应用网关源文件使用GNU [AGPLv3](http://www.gnu.org/licenses/agpl-3.0.html)授权.    
+JANUSEC应用网关开源版本的源文件使用GNU [AGPLv3](http://www.gnu.org/licenses/agpl-3.0.html)授权.     
+专业增强特性版本闭源发布，增强特性包括：GSLB、Cookie合规（应用无需修改）等。   
 
 ## 支持  
 
-* 产品网站 [https://doc.janusec.com/cn/](https://doc.janusec.com/cn/)   
-* 官方网站: [https://www.janusec.com/](https://www.janusec.com/)  
-* Email: `support#janusec.com`  
+* 产品网站 [https://janusec.github.io/cn/](https://janusec.github.io/cn/)   
+* Email: `support`**@**`janusec.com`  
 * QQ群: 776900157  
 
 
@@ -185,17 +184,15 @@ https://janusec.github.io/
 
 ## Requirements    
 
-* PostgreSQL 10/11/12+ (Required by Development and Primary Node of Deployment)  
-* Debian 9/10+, CentOS/RHEL 7/8+, Debian 10+ is preferred       
+* PostgreSQL 10/11/12/13/14+ (Required by Development and Primary Node of Deployment)  
+* Debian 9/10/11+, CentOS/RHEL 7/8+, Debian 10+ is preferred       
 * systemd  
 * nftables  
 * Golang 1.15+ (Required by Development Only)  
 
 ## Quick Start for Deployment     
 
-Detailed documentation is available at： [Janusec Application Gateway Quick Start](https://janusec.github.io/documentation/quick-start/).  
-
-You can also try it with [Docker Image](https://www.janusec.com/articles/opensource/1615470598.html)    
+Detailed documentation is available at： [Janusec Application Gateway Quick Start](https://janusec.github.io/documentation/quick-start/).   
 
 ## Quick Start for Developer   
 
@@ -247,13 +244,14 @@ The release package is under `./dist` .
 
 Release directory is `./static/janusec-admin/` , and source code is available at [Janusec-Admin Github](https://github.com/Janusec/janusec-admin) with Angular 9.  
 
-## LICENSE   
+## Multiple LICENSES   
+
+The open source files are made available under the terms of the GNU Affero General Public License ([GNU AGPLv3](http://www.gnu.org/licenses/agpl-3.0.html)).   
 
-Janusec Application Gateway source files are made available under the terms of the GNU Affero General Public License ([GNU AGPLv3](http://www.gnu.org/licenses/agpl-3.0.html)).   
+The professional enhanced version is released in closed source, and the enhanced features including GSLB, Cookie compliance (No need to modify applications), etc.   
 
 ## Support   
 
-* Product: [https://janusec.github.io/](https://janusec.github.io/)   
-* Official site : [https://www.janusec.com/](https://www.janusec.com/)  
-* Email: `support#janusec.com`  
+* Product: [https://janusec.github.io/](https://janusec.github.io/)    
+* Email: `support`**@**`janusec.com`  
 * QQ Group: 776900157 

backend/application.go

@@ -74,6 +74,8 @@ func SelectBackendRoute(app *models.Application, r *http.Request, srcIP string)
 	// get online destinations
 	var onlineDests = []*models.Destination{}
 	for _, dest := range dests {
+		dest.Mutex.Lock()
+		defer dest.Mutex.Unlock()
 		if dest.Online {
 			onlineDests = append(onlineDests, dest)
 		}

backend/destination.go

@@ -9,11 +9,18 @@ package backend
 
 import (
 	"encoding/json"
+	"janusec/data"
 	"janusec/models"
 	"janusec/utils"
 	"net"
 	"net/http"
 	"time"
+
+	"github.com/patrickmn/go-cache"
+)
+
+var (
+	offlineCache = cache.New(30*time.Second, 30*time.Second)
 )
 
 // ContainsDestinationID ...
@@ -31,13 +38,17 @@ func ContainsDestinationID(destinations []*models.Destination, destID int64) boo
 func CheckOfflineDestinations(nowTimeStamp int64) {
 	for _, app := range Apps {
 		for _, dest := range app.Destinations {
+			dest.Mutex.Lock()
+			defer dest.Mutex.Unlock()
 			if dest.RouteType == models.ReverseProxyRoute && !dest.Online {
-				go func(dest *models.Destination) {
-					conn, err := net.DialTimeout("tcp", dest.Destination, time.Second)
+				go func(dest2 *models.Destination) {
+					conn, err := net.DialTimeout("tcp", dest2.Destination, time.Second)
 					if err == nil {
 						defer conn.Close()
-						dest.Online = true
-						dest.CheckTime = nowTimeStamp
+						dest2.Mutex.Lock()
+						defer dest2.Mutex.Unlock()
+						dest2.Online = true
+						dest2.CheckTime = nowTimeStamp
 					}
 				}(dest)
 			} else if dest.RouteType == models.K8S_Ingress && !dest.Online {
@@ -54,8 +65,6 @@ func CheckOfflineDestinations(nowTimeStamp int64) {
 				if err != nil {
 					utils.DebugPrintln("Unmarshal K8S API", err)
 				}
-				dest.Mutex.Lock()
-				defer dest.Mutex.Unlock()
 				dest.Pods = ""
 				for _, podItem := range pods.Items {
 					if podItem.Status.Phase == "Running" {
@@ -71,3 +80,45 @@ func CheckOfflineDestinations(nowTimeStamp int64) {
 		}
 	}
 }
+
+// SetDestinationOffline added on Mar 23, 2024, v1.5.0
+func SetDestinationOffline(dest *models.Destination) {
+	targetDest := dest.Destination
+	if dest.RouteType == models.K8S_Ingress {
+		targetDest = dest.PodsAPI
+	}
+	if count, ok := offlineCache.Get(targetDest); !ok {
+		offlineCache.Set(targetDest, int64(1), cache.DefaultExpiration)
+	} else {
+		nowCount := count.(int64) + int64(1)
+		if nowCount > 5 {
+			// more than 5 requests timeout
+			dest.Online = false
+			app, err := GetApplicationByID(dest.AppID)
+			if err == nil {
+				sendOfflineNotification(app, targetDest)
+			}
+		}
+		offlineCache.Set(targetDest, nowCount, cache.DefaultExpiration)
+	}
+}
+
+// sendOfflineNotification ...
+func sendOfflineNotification(app *models.Application, dest string) {
+	var emails string
+	if data.IsPrimary {
+		emails = data.DAL.GetAppAdminAndOwnerEmails(app.Owner)
+	} else {
+		emails = data.NodeSetting.SMTP.AdminEmails
+	}
+	mailBody := "Backend server: " + dest + " (" + app.Name + ") was offline."
+	if len(mailBody) > 0 && len(emails) > 0 {
+		go utils.SendEmail(data.NodeSetting.SMTP.SMTPServer,
+			data.NodeSetting.SMTP.SMTPPort,
+			data.NodeSetting.SMTP.SMTPAccount,
+			data.NodeSetting.SMTP.SMTPPassword,
+			emails,
+			"[JANUSEC] Backend server offline notification",
+			mailBody)
+	}
+}

backend/k8s.go

@@ -27,7 +27,7 @@ func UpdatePods(dest *models.Destination, nowTimeStamp int64) {
 	if err != nil {
 		utils.DebugPrintln("Check K8S API GetResponse", err)
 		dest.CheckTime = nowTimeStamp
-		dest.Online = false
+		SetDestinationOffline(dest)
 	}
 	pods := models.PODS{}
 	err = json.Unmarshal(resp, &pods)
@@ -96,7 +96,7 @@ func SelectPodFromVIPTarget(dest *models.VipTarget, srcIP string) string {
 		if err != nil {
 			utils.DebugPrintln("Check K8S API GetResponse", err)
 			dest.CheckTime = nowTimeStamp
-			dest.Online = false
+			SetVipTargetOffline(dest)
 		}
 		pods := models.PODS{}
 		err = json.Unmarshal(resp, &pods)

backend/vip_app.go

@@ -104,22 +104,14 @@ func UDPForwarding(vipApp *models.VipApp, udpListenConn *net.UDPConn) {
 			//fmt.Println("UDPForwarding ReadMsgUDP", err)
 			break
 		}
-
 		vipTarget := SelectVipTarget(vipApp, clientAddr.String())
-		if err != nil {
-			//fmt.Println("UDPForwarding ResolveUDPAddr", err)
-			break
-		}
 		if vipTarget != nil {
 			vipTarget.CheckTime = time.Now().Unix()
 			targetAddr, _ := net.ResolveUDPAddr("udp", vipTarget.Destination)
 			udpTargetConn, err := net.DialUDP("udp", nil, targetAddr)
 			if err != nil {
 				utils.DebugPrintln("UDPForwarding DialUDP could not connect to target", vipTarget.Destination, err)
-				vipTarget.Online = false
-				if data.NodeSetting.SMTP.SMTPEnabled {
-					sendVIPOfflineNotification(vipApp, vipTarget.Destination)
-				}
+				SetVipTargetOffline(vipTarget)
 				break
 			}
 			if udpTargetConn == nil {
@@ -138,10 +130,7 @@ func UDPForwarding(vipApp *models.VipApp, udpListenConn *net.UDPConn) {
 				for {
 					n, _, err := udpTargetConn.ReadFromUDP(dataBuf)
 					if err != nil {
-						vipTarget.Online = false
-						if data.NodeSetting.SMTP.SMTPEnabled {
-							sendVIPOfflineNotification(vipApp, vipTarget.Destination)
-						}
+						SetVipTargetOffline(vipTarget)
 						break
 					}
 					// Response to client
@@ -190,10 +179,7 @@ func TCPForwarding(vipApp *models.VipApp, vipListener net.Listener) {
 			vipTarget.CheckTime = time.Now().Unix()
 			if err != nil {
 				utils.DebugPrintln("TCPForwarding could not connect to target", targetDest, err)
-				vipTarget.Online = false
-				if data.NodeSetting.SMTP.SMTPEnabled {
-					sendVIPOfflineNotification(vipApp, targetDest)
-				}
+				SetVipTargetOffline(vipTarget)
 				continue
 			}
 			vipTarget.Online = true
@@ -385,23 +371,3 @@ func GetVipAppIndex(vipAppID int64) int {
 	}
 	return -1
 }
-
-// sendVIPOfflineNotification ...
-func sendVIPOfflineNotification(app *models.VipApp, dest string) {
-	var emails string
-	if data.IsPrimary {
-		emails = data.DAL.GetAppAdminAndOwnerEmails(app.Owner)
-	} else {
-		emails = data.NodeSetting.SMTP.AdminEmails
-	}
-	mailBody := "Backend virtual IP server: " + dest + " (" + app.Name + ") was offline."
-	if len(mailBody) > 0 && len(emails) > 0 {
-		go utils.SendEmail(data.NodeSetting.SMTP.SMTPServer,
-			data.NodeSetting.SMTP.SMTPPort,
-			data.NodeSetting.SMTP.SMTPAccount,
-			data.NodeSetting.SMTP.SMTPPassword,
-			emails,
-			"[JANUSEC] Backend server offline notification",
-			mailBody)
-	}
-}

backend/vip_target.go

@@ -13,6 +13,8 @@ import (
 	"janusec/utils"
 	"net"
 	"time"
+
+	"github.com/patrickmn/go-cache"
 )
 
 // DeleteVipTargetsByAppID delete backend targets for port forwarding
@@ -42,7 +44,7 @@ func CheckOfflineVipTargets(nowTimeStamp int64) {
 						targetAddr, _ := net.ResolveUDPAddr("udp", vTarget.Destination)
 						udpTargetConn, err := net.DialUDP("udp", nil, targetAddr)
 						if err != nil {
-							vTarget.Online = false
+							SetVipTargetOffline(vTarget)
 							return
 						}
 						// udpTargetConn will be closed in go thread
@@ -51,7 +53,7 @@ func CheckOfflineVipTargets(nowTimeStamp int64) {
 							data := make([]byte, 2048)
 							_, _, err := udpConn.ReadFromUDP(data)
 							if err != nil {
-								vipTarget.Online = false
+								SetVipTargetOffline(vipTarget)
 							} else {
 								vipTarget.Online = true
 							}
@@ -61,7 +63,7 @@ func CheckOfflineVipTargets(nowTimeStamp int64) {
 						// send test data to target
 						_, err = udpTargetConn.Write([]byte("Hi"))
 						if err != nil {
-							vTarget.Online = false
+							SetVipTargetOffline(vTarget)
 							return
 						}
 					}
@@ -79,3 +81,44 @@ func ContainsTargetID(targets []*models.VipTarget, targetID int64) bool {
 	}
 	return false
 }
+
+func SetVipTargetOffline(dest *models.VipTarget) {
+	target := dest.Destination
+	if dest.RouteType == models.K8S_Ingress {
+		target = dest.PodsAPI
+	}
+	if count, ok := offlineCache.Get(target); !ok {
+		offlineCache.Set(target, int64(1), cache.DefaultExpiration)
+	} else {
+		nowCount := count.(int64) + int64(1)
+		if nowCount > 5 {
+			// more than 5 requests timeout
+			dest.Online = false
+			app, err := GetVipAppByID(dest.VipAppID)
+			if err == nil {
+				sendVIPOfflineNotification(app, target)
+			}
+		}
+		offlineCache.Set(target, nowCount, cache.DefaultExpiration)
+	}
+}
+
+// sendVIPOfflineNotification ...
+func sendVIPOfflineNotification(app *models.VipApp, dest string) {
+	var emails string
+	if data.IsPrimary {
+		emails = data.DAL.GetAppAdminAndOwnerEmails(app.Owner)
+	} else {
+		emails = data.NodeSetting.SMTP.AdminEmails
+	}
+	mailBody := "Backend virtual IP: " + dest + " (" + app.Name + ") was offline."
+	if len(mailBody) > 0 && len(emails) > 0 {
+		go utils.SendEmail(data.NodeSetting.SMTP.SMTPServer,
+			data.NodeSetting.SMTP.SMTPPort,
+			data.NodeSetting.SMTP.SMTPAccount,
+			data.NodeSetting.SMTP.SMTPPassword,
+			emails,
+			"[JANUSEC] Backend server offline notification",
+			mailBody)
+	}
+}

data/data.go

@@ -35,7 +35,7 @@ var (
 	// IsPrimary i.e. Is Primary Node
 	IsPrimary bool
 	// Version of JANUSEC
-	Version = "1.4.2"
+	Version = "1.5.0"
 )
 
 // InitConfig init Data Access Layer