Make failing sources non-default and exclude from tests (#1475)

* make `dnsdumpster` non-default * minor * make `subdomaincenter` non-default
projectdiscovery · Dec 12, 2024 · 2b6c5f6 · 2b6c5f6
1 parent 31f5b69
commit 2b6c5f6
Show file tree

Hide file tree

Showing 4 changed files with 13 additions and 11 deletions.
diff --git a/v2/pkg/passive/sources_test.go b/v2/pkg/passive/sources_test.go
@@ -68,7 +68,7 @@ var (
 		"chinaz",
 		"crtsh",
 		"digitorus",
-		"dnsdumpster",
+		// "dnsdumpster", //failing with "unexpected status code 403 received"
 		"dnsrepo",
 		"fofa",
 		"fullhunt",
@@ -89,7 +89,7 @@ var (
 		// "threatminer",
 		// "reconcloud",
 		"builtwith",
-		"subdomaincenter",
+		// "subdomaincenter", //failing with empty result
 	}
 
 	expectedDefaultRecursiveSources = []string{

diff --git a/v2/pkg/passive/sources_wo_auth_test.go b/v2/pkg/passive/sources_wo_auth_test.go
@@ -24,13 +24,15 @@ func TestSourcesWithoutKeys(t *testing.T) {
 	}
 
 	ignoredSources := []string{
-		"commoncrawl",    // commoncrawl is under resourced and will likely time-out so step over it for this test https://groups.google.com/u/2/g/common-crawl/c/3QmQjFA_3y4/m/vTbhGqIBBQAJ
-		"riddler",        // failing due to cloudfront protection
-		"crtsh",          // Fails in GH Action (possibly IP-based ban) causing a timeout.
-		"hackertarget",   // Fails in GH Action (possibly IP-based ban) but works locally
-		"waybackarchive", // Fails randomly
-		"alienvault",     // 503 Service Temporarily Unavailable
-		"digitorus",      // failing with "Failed to retrieve certificate"
+		"commoncrawl",     // commoncrawl is under resourced and will likely time-out so step over it for this test https://groups.google.com/u/2/g/common-crawl/c/3QmQjFA_3y4/m/vTbhGqIBBQAJ
+		"riddler",         // failing due to cloudfront protection
+		"crtsh",           // Fails in GH Action (possibly IP-based ban) causing a timeout.
+		"hackertarget",    // Fails in GH Action (possibly IP-based ban) but works locally
+		"waybackarchive",  // Fails randomly
+		"alienvault",      // 503 Service Temporarily Unavailable
+		"digitorus",       // failing with "Failed to retrieve certificate"
+		"dnsdumpster",     // failing with "unexpected status code 403 received"
+		"subdomaincenter", //failing with empty result or random reason /shrug
 	}
 
 	domain := "hackerone.com"

diff --git a/v2/pkg/subscraping/sources/dnsdumpster/dnsdumpster.go b/v2/pkg/subscraping/sources/dnsdumpster/dnsdumpster.go
@@ -118,7 +118,7 @@ func (s *Source) Name() string {
 }
 
 func (s *Source) IsDefault() bool {
-	return true
+	return false
 }
 
 func (s *Source) HasRecursiveSupport() bool {

diff --git a/v2/pkg/subscraping/sources/subdomaincenter/subdomaincenter.go b/v2/pkg/subscraping/sources/subdomaincenter/subdomaincenter.go
@@ -63,7 +63,7 @@ func (s *Source) Name() string {
 }
 
 func (s *Source) IsDefault() bool {
-	return true
+	return false
 }
 
 func (s *Source) HasRecursiveSupport() bool {