Include IPCat support in Beta™

Author: dsamarin

README.md

@@ -27,6 +27,7 @@ The blacklist syntax is similar to that of [gitignore][1]. An optional prefix `!
 4. Match Hostname [e.g. `crawl-66-249-66-1.googlebot.com`]
 5. Match Hostname RegExp [e.g.: `~ .*\.cox\.net`]
 6. Match Geofence [e.g.: `@ 39.377297 -74.451082 (7km)`]
+7. Match [ipcat][2] [e.g. `ipcat Cloudflare Inc`]
 
 ### Whitelist
 
@@ -63,5 +64,13 @@ shady.com
 ~ (.*)\.shady\.com # Block subdomains of shady
 ```
 
+## IPCat
+
+A rule that begins with "`ipcat `" will be matched with the IPCat database by name.
+This rule is currently in development so this syntax and usage may change. Use `*`
+to match a multitude of characters like a glob, e.g. `!ipcat Akamai` to whitelist
+Akamai requests.
+
 
 [1]: https://git-scm.com/docs/gitignore
+[2]: https://github.com/oftn-oswg/ipcat

admin.go

@@ -56,6 +56,7 @@ func NewAdminHandler(db *ZerodropDB, config *ZerodropConfig) *AdminHandler {
 	return handler
 }
 
+// AdminPageData represents the data served to the admin templates.
 type AdminPageData struct {
 	Error    string
 	Title    string
@@ -64,6 +65,7 @@ type AdminPageData struct {
 	Entries  []ZerodropEntry
 }
 
+// ServeLogin renders the login page.
 func (a *AdminHandler) ServeLogin(w http.ResponseWriter, r *http.Request) {
 	page := a.Config.Base + "admin/login"
 	if r.URL.Path != "/login" {
@@ -79,6 +81,7 @@ func (a *AdminHandler) ServeLogin(w http.ResponseWriter, r *http.Request) {
 	}
 }
 
+// ServeNew renders the new entry page.
 func (a *AdminHandler) ServeNew(w http.ResponseWriter, r *http.Request) {
 	if r.Method == "POST" {
 		err := r.ParseMultipartForm(int64(a.Config.UploadMaxSize))
@@ -181,6 +184,7 @@ func (a *AdminHandler) ServeNew(w http.ResponseWriter, r *http.Request) {
 	}
 }
 
+// ServeMain serves the entry list.
 func (a *AdminHandler) ServeMain(w http.ResponseWriter, r *http.Request) {
 	data := &AdminPageData{Title: "Zerodrop Admin", LoggedIn: true, Config: a.Config}
 

auth.go

@@ -14,16 +14,19 @@ import (
 	jwt "github.com/dgrijalva/jwt-go"
 )
 
+// AuthCredentials represents the secrets required to maintain authentication.
 type AuthCredentials struct {
 	Digest string
 	Secret []byte
 }
 
+// AuthClaims represents the claims of the JWT (JSON Web Token)
 type AuthClaims struct {
 	Auth bool `json:"admin"`
 	jwt.StandardClaims
 }
 
+// AuthHandler wraps another handler with authentication methods and requirements.
 type AuthHandler struct {
 	Success         http.Handler
 	Failure         http.Handler

blacklist.go

@@ -8,21 +8,33 @@ import (
 	"strconv"
 	"strings"
 
+	"github.com/client9/ipcat"
 	"github.com/oschwald/geoip2-golang"
 )
 
-type ZerodropBlacklistItem struct {
+// BlacklistContext is a structure used to contain the external data
+// used to categorize IP addresses needed for specific rules, like
+// the geolocation database used for geofencing or the ipcat database.
+type BlacklistContext struct {
+	GeoDB *geoip2.Reader
+	IPSet *ipcat.IntervalSet
+}
+
+// BlacklistRule is a structure that represents a rule or comment as part
+// of a blacklist.
+type BlacklistRule struct {
 	Comment  string
 	Negation bool
 	All      bool
 	Network  *net.IPNet
 	IP       net.IP
 	Hostname string
 	Regexp   *regexp.Regexp
-	Geofence *ZerodropGeofence
+	Geofence *Geofence
+	IPCat    string
 }
 
-func (i ZerodropBlacklistItem) String() (value string) {
+func (i BlacklistRule) String() (value string) {
 	if i.Negation {
 		value += "!"
 	}
@@ -61,18 +73,23 @@ func (i ZerodropBlacklistItem) String() (value string) {
 		return
 	}
 
+	if i.IPCat != "" {
+		value += "ipcat " + i.IPCat
+	}
+
 	if i.Comment != "" {
 		value += "# " + i.Comment
 	}
 
 	return
 }
 
-type ZerodropBlacklist struct {
-	List []*ZerodropBlacklistItem
+// Blacklist is a list of BlacklistRules
+type Blacklist struct {
+	List []*BlacklistRule
 }
 
-func (b ZerodropBlacklist) String() string {
+func (b Blacklist) String() string {
 	itemCount := 0
 
 	// Stringify items
@@ -106,9 +123,10 @@ var geofenceUnits = map[string]float64{
 	"ft": 1609.0 / 5280.0,
 }
 
-func ParseBlacklist(text string) ZerodropBlacklist {
+// ParseBlacklist parses a text blacklist and returns a Blacklist object.
+func ParseBlacklist(text string) Blacklist {
 	lines := strings.Split(text, "\n")
-	blacklist := ZerodropBlacklist{List: []*ZerodropBlacklistItem{}}
+	blacklist := Blacklist{List: []*BlacklistRule{}}
 
 	for _, line := range lines {
 		// A line with # serves as a comment.
@@ -123,7 +141,7 @@ func ParseBlacklist(text string) ZerodropBlacklist {
 			continue
 		}
 
-		item := &ZerodropBlacklistItem{}
+		item := &BlacklistRule{}
 
 		// An optional prefix "!" which negates the pattern;
 		// any matching address/host excluded by a previous pattern
@@ -141,6 +159,14 @@ func ParseBlacklist(text string) ZerodropBlacklist {
 			continue
 		}
 
+		// IPCat database query match
+		if line[:6] == "ipcat " {
+			ipcat := strings.TrimSpace(line[6:])
+			item.IPCat = ipcat
+			blacklist.Add(item)
+			continue
+		}
+
 		switch line[0] {
 		case '@':
 			// An optional prefix "@" indicates a geofencing target.
@@ -203,7 +229,7 @@ func ParseBlacklist(text string) ZerodropBlacklist {
 				continue
 			}
 
-			item.Geofence = &ZerodropGeofence{
+			item.Geofence = &Geofence{
 				Latitude:  lat,
 				Longitude: lng,
 				Radius:    radius,
@@ -251,14 +277,17 @@ func ParseBlacklist(text string) ZerodropBlacklist {
 	return blacklist
 }
 
-func (b *ZerodropBlacklist) Add(item *ZerodropBlacklistItem) {
+// Add appends a BlacklistRule to the Blacklist.
+func (b *Blacklist) Add(item *BlacklistRule) {
 	b.List = append(b.List, item)
 }
 
-func (b *ZerodropBlacklist) Allow(ip net.IP, geodb *geoip2.Reader) bool {
+// Allow decides whether the Blacklist permits the selected IP address.
+func (b *Blacklist) Allow(ctx *BlacklistContext, ip net.IP) bool {
 	allow := true
 
-	user := (*ZerodropGeofence)(nil)
+	user := (*Geofence)(nil)
+	category := (*ipcat.Interval)(nil)
 
 	for _, item := range b.List {
 		match := false
@@ -311,18 +340,18 @@ func (b *ZerodropBlacklist) Allow(ip net.IP, geodb *geoip2.Reader) bool {
 				}
 			}
 		} else if item.Geofence != nil {
-			if geodb == nil {
-				log.Println("Denying access to geofenced blacklist: No GeoDB provided.")
+			if ctx.GeoDB == nil {
+				log.Println("Denying access by geofence rule error: no database provided")
 				return false
 			}
 
 			if user == nil {
-				record, err := geodb.City(ip)
+				record, err := ctx.GeoDB.City(ip)
 				if err != nil {
-					log.Println("Denying access to geofenced blacklist: Error loading IP.")
+					log.Printf("Denying access by geofence rule error: %s", err.Error())
 					return false
 				}
-				user = &ZerodropGeofence{
+				user = &Geofence{
 					Latitude:  record.Location.Latitude,
 					Longitude: record.Location.Longitude,
 					Radius:    float64(record.Location.AccuracyRadius) * 1000.0, // Convert km to m
@@ -338,8 +367,41 @@ func (b *ZerodropBlacklist) Allow(ip net.IP, geodb *geoip2.Reader) bool {
 				// Blacklist if user intersects at all with bounds
 				match = !(boundsIntersect&IsDisjoint != 0)
 			}
+		} else if item.IPCat != "" {
+			if ctx.IPSet == nil {
+				log.Println("Denying access by ipcat rule error: no database provided")
+				return false
+			}
+
+			if category == nil {
+				ipv4 := ip.To4()
+				if ipv4 != nil {
+					dots := ipv4.String()
+					interval, err := ctx.IPSet.Contains(dots)
+					if err != nil {
+						log.Printf("Denying access by ipcat rule error: %s", err.Error())
+						return false
+					}
+					category = interval
+				}
+			}
+
+			if category != nil {
+				var err error
+
+				name := strings.ToLower(category.Name)
+				search := strings.Replace(regexp.QuoteMeta(strings.ToLower(item.IPCat)), `\*`, `.*`, -1)
+				match, err = regexp.MatchString(search, name)
+				if err != nil {
+					log.Printf("Denying access by ipcat rule error: %s", err.Error())
+					return false
+				}
+			}
+
+			return false
 		}
 
+		// TODO: Allow early termination based on negation flags
 		if match {
 			allow = item.Negation
 		}

db.go

@@ -7,20 +7,21 @@ import (
 	"time"
 )
 
+// ZerodropEntry is a page entry.
 type ZerodropEntry struct {
 	db                   *ZerodropDB
-	Name                 string            // The request URI used to access this entry
-	URL                  string            // The URL that this entry references
-	Filename             string            // The location of the file in the uploads directory
-	ContentType          string            // The MIME type to serve as Content-Type header
-	Redirect             bool              // Indicates whether to redirect instead of proxy
-	Creation             time.Time         // The time this entry was created
-	AccessBlacklist      ZerodropBlacklist // Blacklist
-	AccessBlacklistCount int               // Number of requests that have been caught by the blacklist
-	AccessExpire         bool              // Indicates whether to expire after finite access
-	AccessExpireCount    int               // The number of requests on this entry before expiry
-	AccessCount          int               // The number of times this has been accessed
-	AccessTrain          bool              // Whether training is active
+	Name                 string    // The request URI used to access this entry
+	URL                  string    // The URL that this entry references
+	Filename             string    // The location of the file in the uploads directory
+	ContentType          string    // The MIME type to serve as Content-Type header
+	Redirect             bool      // Indicates whether to redirect instead of proxy
+	Creation             time.Time // The time this entry was created
+	AccessBlacklist      Blacklist // Blacklist
+	AccessBlacklistCount int       // Number of requests that have been caught by the blacklist
+	AccessExpire         bool      // Indicates whether to expire after finite access
+	AccessExpireCount    int       // The number of requests on this entry before expiry
+	AccessCount          int       // The number of times this has been accessed
+	AccessTrain          bool      // Whether training is active
 }
 
 // ZerodropDB represents a database connection.
@@ -29,11 +30,13 @@ type ZerodropDB struct {
 	mapping map[string]ZerodropEntry
 }
 
+// Connect opens a connection to the backend.
 func (d *ZerodropDB) Connect() error {
 	d.mapping = map[string]ZerodropEntry{}
 	return nil
 }
 
+// Get returns the entry with the specified name.
 func (d *ZerodropDB) Get(name string) (entry ZerodropEntry, ok bool) {
 	entry, ok = d.mapping[name]
 	return
@@ -57,16 +60,19 @@ func (d *ZerodropDB) List() []ZerodropEntry {
 	return list
 }
 
-func (db *ZerodropDB) Create(entry *ZerodropEntry) error {
-	entry.db = db
-	db.mapping[entry.Name] = *entry
+// Create adds an entry to the database.
+func (d *ZerodropDB) Create(entry *ZerodropEntry) error {
+	entry.db = d
+	d.mapping[entry.Name] = *entry
 	return nil
 }
 
+// Remove removes an entry from the database.
 func (d *ZerodropDB) Remove(name string) {
 	delete(d.mapping, name)
 }
 
+// Clear resets the database by removing all entries.
 func (d *ZerodropDB) Clear() {
 	d.Connect()
 }
@@ -76,6 +82,7 @@ func (e *ZerodropEntry) IsExpired() bool {
 	return e.AccessExpire && (e.AccessCount >= e.AccessExpireCount)
 }
 
+// Update saves changes to the entry to the database it belongs to.
 func (e *ZerodropEntry) Update() error {
 	if e.db != nil {
 		return e.db.Create(e)

geofence.go

@@ -4,20 +4,27 @@ import (
 	"github.com/kellydunn/golang-geo"
 )
 
-type ZerodropGeofence struct {
+// Geofence represents a point on the Earth with an accuracy radius.
+type Geofence struct {
 	Latitude, Longitude, Radius float64
 }
 
-type ZerodropIntersection uint
+// SetIntersection is a description of the relationship between two sets.
+type SetIntersection uint
 
 const (
-	IsDisjoint ZerodropIntersection = 1 << iota
+	// IsDisjoint means that the two sets have no common elements.
+	IsDisjoint SetIntersection = 1 << iota
+
+	// IsSubset means the first set is a subset of the second.
 	IsSubset
+
+	// IsSuperset means the second set is a subset of the first.
 	IsSuperset
 )
 
-// Intersection gets the intersection data for two points with accuracy radius
-func (mi *ZerodropGeofence) Intersection(tu *ZerodropGeofence) (i ZerodropIntersection) {
+// Intersection describes the relationship between two geofences
+func (mi *Geofence) Intersection(tu *Geofence) (i SetIntersection) {
 	miPoint := geo.NewPoint(mi.Latitude, mi.Longitude)
 	tuPoint := geo.NewPoint(tu.Latitude, tu.Longitude)
 	distance := miPoint.GreatCircleDistance(tuPoint) * 1000

main.go

@@ -22,6 +22,7 @@ type ZerodropConfig struct {
 	AuthDigest string `default:"11a55ac5de2beb9146e01386dd978a13bb9b99388f5eb52e37f69a32e3d5f11e"`
 
 	GeoDB string
+	IPCat string
 
 	UploadDirectory   string `default:"."`
 	UploadPermissions uint32 `default:"0600"`