added support for custom regex on field scope (-fs "[regex pattern]")

README.md

@@ -156,7 +156,7 @@ HEADLESS:
 SCOPE:
    -cs, -crawl-scope string[]       in scope url regex to be followed by crawler
    -cos, -crawl-out-scope string[]  out of scope url regex to be excluded by crawler
-   -fs, -field-scope string         pre-defined scope field (dn,rdn,fqdn) (default "rdn")
+   -fs, -field-scope string  pre-defined scope field (dn,rdn,fqdn) or custom regex (e.g., '(company-staging.io|company.com)') (default "rdn")
    -ns, -no-scope                   disables host based default scope
    -do, -display-out-scope          display external endpoint from scoped crawling
 

cmd/katana/main.go

@@ -109,7 +109,7 @@ pipelines offering both headless and non-headless crawling.`)
 	flagSet.CreateGroup("scope", "Scope",
 		flagSet.StringSliceVarP(&options.Scope, "crawl-scope", "cs", nil, "in scope url regex to be followed by crawler", goflags.FileCommaSeparatedStringSliceOptions),
 		flagSet.StringSliceVarP(&options.OutOfScope, "crawl-out-scope", "cos", nil, "out of scope url regex to be excluded by crawler", goflags.FileCommaSeparatedStringSliceOptions),
-		flagSet.StringVarP(&options.FieldScope, "field-scope", "fs", "rdn", "pre-defined scope field (dn,rdn,fqdn)"),
+		flagSet.StringVarP(&options.FieldScope, "field-scope", "fs", "rdn", "pre-defined scope field (dn,rdn,fqdn) or custom regex (e.g., '(company-staging.io|company.com)')"),
 		flagSet.BoolVarP(&options.NoScope, "no-scope", "ns", false, "disables host based default scope"),
 		flagSet.BoolVarP(&options.DisplayOutScope, "display-out-scope", "do", false, "display external endpoint from scoped crawling"),
 	)

pkg/utils/scope/scope.go

@@ -12,10 +12,11 @@ import (
 
 // Manager manages scope for crawling process
 type Manager struct {
-	inScope    []*regexp.Regexp
-	outOfScope []*regexp.Regexp
-	noScope    bool
-	fieldScope dnsScopeField
+	inScope           []*regexp.Regexp
+	outOfScope        []*regexp.Regexp
+	noScope           bool
+	fieldScope        dnsScopeField
+	fieldScopePattern *regexp.Regexp
 }
 
 type dnsScopeField int
@@ -24,6 +25,7 @@ const (
 	dnDnsScopeField dnsScopeField = iota + 1
 	rdnDnsScopeField
 	fqdnDNSScopeField
+	customDNSScopeField
 )
 
 var stringToDNSScopeField = map[string]dnsScopeField{
@@ -39,7 +41,12 @@ func NewManager(inScope, outOfScope []string, fieldScope string, noScope bool) (
 	}
 
 	if scopeValue, ok := stringToDNSScopeField[fieldScope]; !ok {
-		return nil, fmt.Errorf("invalid dns scope field specified: %s", fieldScope)
+		manager.fieldScope = customDNSScopeField
+		if compiled, err := regexp.Compile(fieldScope); err != nil {
+			return nil, fmt.Errorf("could not compile regex %s: %s", fieldScope, err)
+		} else {
+			manager.fieldScopePattern = compiled
+		}
 	} else {
 		manager.fieldScope = scopeValue
 	}
@@ -108,7 +115,12 @@ func (m *Manager) validateURL(URL string) (bool, error) {
 
 func (m *Manager) validateDNS(hostname, rootHostname string) (bool, error) {
 	parsed := net.ParseIP(hostname)
-
+	if m.fieldScope == customDNSScopeField {
+		// If we have a custom regex, we need to match it against the full hostname
+		if m.fieldScopePattern.MatchString(hostname) {
+			return true, nil
+		}
+	}
 	if m.fieldScope == fqdnDNSScopeField || parsed != nil {
 		matched := strings.EqualFold(hostname, rootHostname)
 		return matched, nil