diff --git a/v2/examples/main.go b/v2/examples/main.go
index d6baf9d91..8edf7ea72 100644
--- a/v2/examples/main.go
+++ b/v2/examples/main.go
@@ -30,8 +30,9 @@ func main() {
}
output := &bytes.Buffer{}
+ var sourceMap map[string]map[string]struct{}
// To run subdomain enumeration on a single domain
- if err = subfinder.EnumerateSingleDomainWithCtx(context.Background(), "hackerone.com", []io.Writer{output}); err != nil {
+ if sourceMap, err = subfinder.EnumerateSingleDomainWithCtx(context.Background(), "hackerone.com", []io.Writer{output}); err != nil {
log.Fatalf("failed to enumerate single domain: %v", err)
}
@@ -47,4 +48,13 @@ func main() {
// print the output
log.Println(output.String())
+
+ // Or use sourceMap to access the results in your application
+ for subdomain, sources := range sourceMap {
+ sourcesList := make([]string, 0, len(sources))
+ for source := range sources {
+ sourcesList = append(sourcesList, source)
+ }
+ log.Printf("%s %s (%d)\n", subdomain, sourcesList, len(sources))
+ }
}
diff --git a/v2/go.mod b/v2/go.mod
index 9bf75920b..992aef1ce 100644
--- a/v2/go.mod
+++ b/v2/go.mod
@@ -10,10 +10,10 @@ require (
github.com/projectdiscovery/chaos-client v0.5.2
github.com/projectdiscovery/dnsx v1.2.1
github.com/projectdiscovery/fdmax v0.0.4
- github.com/projectdiscovery/gologger v1.1.27
- github.com/projectdiscovery/ratelimit v0.0.55
- github.com/projectdiscovery/retryablehttp-go v1.0.82
- github.com/projectdiscovery/utils v0.2.15
+ github.com/projectdiscovery/gologger v1.1.40
+ github.com/projectdiscovery/ratelimit v0.0.68
+ github.com/projectdiscovery/retryablehttp-go v1.0.95
+ github.com/projectdiscovery/utils v0.4.6
github.com/rs/xid v1.5.0
github.com/stretchr/testify v1.9.0
github.com/tomnomnom/linkheader v0.0.0-20180905144013-02ca5825eb80
@@ -71,8 +71,8 @@ require (
github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c // indirect
github.com/projectdiscovery/blackrock v0.0.1 // indirect
github.com/projectdiscovery/cdncheck v1.1.0 // indirect
- github.com/projectdiscovery/fastdialer v0.2.9 // indirect
- github.com/projectdiscovery/hmap v0.0.62 // indirect
+ github.com/projectdiscovery/fastdialer v0.2.14 // indirect
+ github.com/projectdiscovery/hmap v0.0.74 // indirect
github.com/projectdiscovery/machineid v0.0.0-20240226150047-2e2c51e35983 // indirect
github.com/projectdiscovery/networkpolicy v0.0.9 // indirect
github.com/refraction-networking/utls v1.6.7 // indirect
@@ -102,12 +102,12 @@ require (
github.com/zmap/zcrypto v0.0.0-20230422215203-9a665e1e9968 // indirect
go.etcd.io/bbolt v1.3.7 // indirect
go.uber.org/multierr v1.11.0 // indirect
- golang.org/x/crypto v0.27.0 // indirect
+ golang.org/x/crypto v0.31.0 // indirect
golang.org/x/mod v0.17.0 // indirect
golang.org/x/oauth2 v0.11.0 // indirect
- golang.org/x/sync v0.8.0 // indirect
- golang.org/x/term v0.24.0 // indirect
- golang.org/x/text v0.18.0 // indirect
+ golang.org/x/sync v0.10.0 // indirect
+ golang.org/x/term v0.27.0 // indirect
+ golang.org/x/text v0.21.0 // indirect
golang.org/x/time v0.5.0 // indirect
golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d // indirect
google.golang.org/appengine v1.6.7 // indirect
@@ -125,7 +125,7 @@ require (
github.com/pkg/errors v0.9.1 // indirect
github.com/pmezard/go-difflib v1.0.0 // indirect
github.com/projectdiscovery/goflags v0.1.64
- github.com/projectdiscovery/retryabledns v1.0.80 // indirect
- golang.org/x/net v0.29.0 // indirect
- golang.org/x/sys v0.25.0 // indirect
+ github.com/projectdiscovery/retryabledns v1.0.93 // indirect
+ golang.org/x/net v0.33.0 // indirect
+ golang.org/x/sys v0.28.0 // indirect
)
diff --git a/v2/go.sum b/v2/go.sum
index 818b0505c..1ec44c74b 100644
--- a/v2/go.sum
+++ b/v2/go.sum
@@ -196,28 +196,28 @@ github.com/projectdiscovery/chaos-client v0.5.2 h1:dN+7GXEypsJAbCD//dBcUxzAEAEH1
github.com/projectdiscovery/chaos-client v0.5.2/go.mod h1:KnoJ/NJPhll42uaqlDga6oafFfNw5l2XI2ajRijtDuU=
github.com/projectdiscovery/dnsx v1.2.1 h1:TxslYvp1Z/YZ4CP/J0gx5RYpvXREnVmyoacmTcGu5yg=
github.com/projectdiscovery/dnsx v1.2.1/go.mod h1:6dAsMCEDu7FArZy2qjyTeUQrqpZ4ITLU11fcmUvFqt0=
-github.com/projectdiscovery/fastdialer v0.2.9 h1:vDCqxVMCyUu3oVEizEK1K8K+CCcLkVDW3X2HfiWaVFA=
-github.com/projectdiscovery/fastdialer v0.2.9/go.mod h1:mYv5QaNBDDSHlZO9DI0niRMw+G5hUzwIhs8QixSElUI=
+github.com/projectdiscovery/fastdialer v0.2.14 h1:/cndy+5celjoYzbk4LksHYOCTpFGIJY8RF/EK31Opjs=
+github.com/projectdiscovery/fastdialer v0.2.14/go.mod h1:z5yKQ/YWaVrBMfdL6f5J7VytUx9wxc5vs/Lf51QelCw=
github.com/projectdiscovery/fdmax v0.0.4 h1:K9tIl5MUZrEMzjvwn/G4drsHms2aufTn1xUdeVcmhmc=
github.com/projectdiscovery/fdmax v0.0.4/go.mod h1:oZLqbhMuJ5FmcoaalOm31B1P4Vka/CqP50nWjgtSz+I=
github.com/projectdiscovery/goflags v0.1.64 h1:FDfwdt9N97Hi8OuhbkDlKtVttpc/CRMIWQVa08VsHsI=
github.com/projectdiscovery/goflags v0.1.64/go.mod h1:3FyHIVQtnycNOc1LE3O1jj/XR5XuMdF9QfHd0ujhnX4=
-github.com/projectdiscovery/gologger v1.1.27 h1:et/adsKS0jAkPZNUuZTJ+J4U/Ofadxu6Bj3NlUYs1e8=
-github.com/projectdiscovery/gologger v1.1.27/go.mod h1:TXLvCbofuDyQlweDkSBanN083w51QKT4EmyKAVMQ+Ts=
-github.com/projectdiscovery/hmap v0.0.62 h1:Pb3omgGQkRHB2EBDhgXniUXVC8Jz2CBzpSMKZYRUKFs=
-github.com/projectdiscovery/hmap v0.0.62/go.mod h1:pNheW3ukrLOcRUDFWrq8OAZiizhosDonIKRVg/4PGnM=
+github.com/projectdiscovery/gologger v1.1.40 h1:FSIhKnYKzuIEIz3RTg6JX9JtDKgkEzIEf2v5RYckoQ4=
+github.com/projectdiscovery/gologger v1.1.40/go.mod h1:8AUxYXmClqOWJgZ5wknNn5rRK3UlrXQ/r9JjX+gp5Gg=
+github.com/projectdiscovery/hmap v0.0.74 h1:j0TpS9fJxisfdGcIRjaZ3qgyjP3pBkRJf12ZWv64rOI=
+github.com/projectdiscovery/hmap v0.0.74/go.mod h1:qEPAdq/gWQU/IEI+QMzSyL+HYdqayR64V9vGTI/W38c=
github.com/projectdiscovery/machineid v0.0.0-20240226150047-2e2c51e35983 h1:ZScLodGSezQVwsQDtBSMFp72WDq0nNN+KE/5DHKY5QE=
github.com/projectdiscovery/machineid v0.0.0-20240226150047-2e2c51e35983/go.mod h1:3G3BRKui7nMuDFAZKR/M2hiOLtaOmyukT20g88qRQjI=
github.com/projectdiscovery/networkpolicy v0.0.9 h1:IrlDoYZagNNO8y+7iZeHT8k5izE+nek7TdtvEBwCxqk=
github.com/projectdiscovery/networkpolicy v0.0.9/go.mod h1:XFJ2Lnv8BE/ziQCFjBHMsH1w6VmkPiQtk+NlBpdMU7M=
-github.com/projectdiscovery/ratelimit v0.0.55 h1:K72IbJX/Lm4vbCtTcZ6Z8C5lWKL4vEhPYeiopFOWdqg=
-github.com/projectdiscovery/ratelimit v0.0.55/go.mod h1:IpuZAnf3OIoUkXuO8CTAC/l0Fv50/ZfRrbRi6gufTwE=
-github.com/projectdiscovery/retryabledns v1.0.80 h1:P1oqWHZdF/IMtxXV4mrtENTNsghYojVlrNcv8dZAMu8=
-github.com/projectdiscovery/retryabledns v1.0.80/go.mod h1:BdCE4+Lph6hkNx+f+EjOnG+K8Z6shFWs9ZiTj6YcNvU=
-github.com/projectdiscovery/retryablehttp-go v1.0.82 h1:XS+ZEKO291KGZVhHMO+RhvNvVE+IK8VHeM5i6CQ4Ixc=
-github.com/projectdiscovery/retryablehttp-go v1.0.82/go.mod h1:JvlWId6aXT8en1ly22///wHoRQwSsMNAttdJ/YkhHHY=
-github.com/projectdiscovery/utils v0.2.15 h1:FO3n7uhLazHtat3qBxWIq0JCfLoR/IvP5JXAOSKaQ5w=
-github.com/projectdiscovery/utils v0.2.15/go.mod h1:2NAFRu8j/82bkVqx2TcsZFdgtUOnHEUi7u6s3lv79Lo=
+github.com/projectdiscovery/ratelimit v0.0.68 h1:gMLD1aB4R8w7BIpKvtQf6TNb6+5zsJO9WSRWZ9pxwe4=
+github.com/projectdiscovery/ratelimit v0.0.68/go.mod h1:ieU9nNu9Ie8nVMKdj3bsX3JA3kfNI8qn4pkNXsyRxsw=
+github.com/projectdiscovery/retryabledns v1.0.93 h1:iKcEEEH77WwUf5EGimhHxCDdqBF2kOl7WhQi3VQXB8Q=
+github.com/projectdiscovery/retryabledns v1.0.93/go.mod h1:f5HmPdVr3CUm4tHHiB0UyiZVQTYYAKTqfoj8M2gCvqo=
+github.com/projectdiscovery/retryablehttp-go v1.0.95 h1:5CHhWLMovX1MD9W3HzlsMBY3xA+dyeqta2gSWo3j92E=
+github.com/projectdiscovery/retryablehttp-go v1.0.95/go.mod h1:/7CHaD7vqnqBD++AI0JsJdcYyq1Wbf4vMhddjy7sZjI=
+github.com/projectdiscovery/utils v0.4.6 h1:lwbS5d/f70wyDwuwF6lAVkn390hEI/0LOtqyqJEI+qE=
+github.com/projectdiscovery/utils v0.4.6/go.mod h1:eevtW7+x7ydrBdmOenmHdqqJKRv3VqY2QUR7vs4qRfU=
github.com/refraction-networking/utls v1.6.7 h1:zVJ7sP1dJx/WtVuITug3qYUq034cDq9B2MR1K67ULZM=
github.com/refraction-networking/utls v1.6.7/go.mod h1:BC3O4vQzye5hqpmDTWUqi4P5DDhzJfkV1tdqtawQIH0=
github.com/rivo/uniseg v0.1.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
@@ -333,8 +333,8 @@ golang.org/x/crypto v0.0.0-20211209193657-4570a0811e8b/go.mod h1:IxCIyHEi3zRg3s0
golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58=
golang.org/x/crypto v0.7.0/go.mod h1:pYwdfH91IfpZVANVyUOhSIPZaFoJGxTFbZhFTx+dXZU=
golang.org/x/crypto v0.11.0/go.mod h1:xgJhtzW8F9jGdVFWZESrid1U1bjeNy4zgy5cRr/CIio=
-golang.org/x/crypto v0.27.0 h1:GXm2NjJrPaiv/h1tb2UH8QfgC/hOf/+z0p6PT8o1w7A=
-golang.org/x/crypto v0.27.0/go.mod h1:1Xngt8kV6Dvbssa53Ziq6Eqn0HqbZi5Z6R0ZpwQzt70=
+golang.org/x/crypto v0.31.0 h1:ihbySMvVjLAeSH1IbfcRTkD/iNscyz8rGzjF/E5hV6U=
+golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk=
golang.org/x/exp v0.0.0-20230420155640-133eef4313cb h1:rhjz/8Mbfa8xROFiH+MQphmAmgqRM0bOMnytznhWEXk=
golang.org/x/exp v0.0.0-20230420155640-133eef4313cb/go.mod h1:V1LtkGg67GoY2N1AnLN78QLrzxkLyJw7RJb1gzOOz9w=
golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
@@ -355,8 +355,8 @@ golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc=
golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg=
golang.org/x/net v0.12.0/go.mod h1:zEVYFnQC7m/vmpQFELhcD1EWkZlX69l4oqgmer6hfKA=
-golang.org/x/net v0.29.0 h1:5ORfpBpCs4HzDYoodCDBbwHzdR5UrLBZ3sOnUJmFoHo=
-golang.org/x/net v0.29.0/go.mod h1:gLkgy8jTGERgjzMic6DS9+SP0ajcu6Xu3Orq/SpETg0=
+golang.org/x/net v0.33.0 h1:74SYHlV8BIgHIFC/LrYkOGIwL19eTYXQ5wc6TBuO36I=
+golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4=
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
golang.org/x/oauth2 v0.5.0/go.mod h1:9/XBHVqLaWO3/BRHs5jbpYCnOZVjj5V0ndyaAM7KB4I=
golang.org/x/oauth2 v0.6.0/go.mod h1:ycmewcwgD4Rpr3eZJLSB4Kyyljb3qDh40vJ8STE5HKw=
@@ -366,8 +366,8 @@ golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJ
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.8.0 h1:3NFvSEYkUoMifnESzZl15y791HH1qU2xm6eCJU5ZPXQ=
-golang.org/x/sync v0.8.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
+golang.org/x/sync v0.10.0 h1:3NQrjDixjgGwUOCaF8w2+VYHv0Ve/vGYSbdkTa98gmQ=
+golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
@@ -392,8 +392,8 @@ golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.25.0 h1:r+8e+loiHxRqhXVl6ML1nO3l1+oFoWbnlu2Ehimmi34=
-golang.org/x/sys v0.25.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/sys v0.28.0 h1:Fksou7UEQUWlKvIdsqzJmUmCX3cZuD2+P3XyyzwMhlA=
+golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
@@ -401,8 +401,8 @@ golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
golang.org/x/term v0.6.0/go.mod h1:m6U89DPEgQRMq3DNkDClhWw02AUbt2daBVO4cn4Hv9U=
golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo=
golang.org/x/term v0.10.0/go.mod h1:lpqdcUyK/oCiQxvxVrppt5ggO2KCZ5QblwqPnfZ6d5o=
-golang.org/x/term v0.24.0 h1:Mh5cbb+Zk2hqqXNO7S1iTjEphVL+jb8ZWaqh/g+JWkM=
-golang.org/x/term v0.24.0/go.mod h1:lOBK/LVxemqiMij05LGJ0tzNr8xlmwBRJ81PX6wVLH8=
+golang.org/x/term v0.27.0 h1:WP60Sv1nlK1T6SupCHbXzSaN0b9wUmsPoRS9b61A23Q=
+golang.org/x/term v0.27.0/go.mod h1:iMsnZpn0cago0GOrHO2+Y7u7JPn5AylBrcoWkElMTSM=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
@@ -413,8 +413,8 @@ golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
golang.org/x/text v0.11.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
-golang.org/x/text v0.18.0 h1:XvMDiNzPAl0jr17s6W9lcaIhGUfUORdGCNsuLmPG224=
-golang.org/x/text v0.18.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY=
+golang.org/x/text v0.21.0 h1:zyQAAkrwaneQ066sspRyJaG9VNi/YJ1NfzcGB3hZ/qo=
+golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ=
golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk=
golang.org/x/time v0.5.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM=
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
diff --git a/v2/pkg/passive/sources.go b/v2/pkg/passive/sources.go
index 03ff7bdce..b8ce15472 100644
--- a/v2/pkg/passive/sources.go
+++ b/v2/pkg/passive/sources.go
@@ -18,7 +18,6 @@ import (
"github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/certspotter"
"github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/chaos"
"github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/chinaz"
- "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/columbus"
"github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/commoncrawl"
"github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/crtsh"
"github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/digitorus"
@@ -30,6 +29,7 @@ import (
"github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/fullhunt"
"github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/github"
"github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/hackertarget"
+ "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/hudsonrock"
"github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/hunter"
"github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/intelx"
"github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/leakix"
@@ -42,7 +42,6 @@ import (
"github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/securitytrails"
"github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/shodan"
"github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/sitedossier"
- "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/subdomaincenter"
"github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/threatbook"
"github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/virustotal"
"github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/waybackarchive"
@@ -62,7 +61,6 @@ var AllSources = [...]subscraping.Source{
&certspotter.Source{},
&chaos.Source{},
&chinaz.Source{},
- &columbus.Source{},
&commoncrawl.Source{},
&crtsh.Source{},
&digitorus.Source{},
@@ -95,7 +93,7 @@ var AllSources = [...]subscraping.Source{
// &threatminer.Source{}, // failing api
// &reconcloud.Source{}, // failing due to cloudflare bot protection
&builtwith.Source{},
- &subdomaincenter.Source{},
+ &hudsonrock.Source{},
}
var sourceWarnings = mapsutil.NewSyncLockMap[string, string](
@@ -128,7 +126,7 @@ func New(sourceNames, excludedSourceNames []string, useAllSources, useSourcesSup
if len(sourceNames) > 0 {
for _, source := range sourceNames {
if NameSourceMap[source] == nil {
- gologger.Fatal().Msgf("There is no source with the name: %s", source)
+ gologger.Warning().Msgf("There is no source with the name: %s", source)
} else {
sources[source] = NameSourceMap[source]
}
diff --git a/v2/pkg/passive/sources_test.go b/v2/pkg/passive/sources_test.go
index 3866c064e..3223117e3 100644
--- a/v2/pkg/passive/sources_test.go
+++ b/v2/pkg/passive/sources_test.go
@@ -21,7 +21,6 @@ var (
"certspotter",
"chaos",
"chinaz",
- "columbus",
"commoncrawl",
"crtsh",
"digitorus",
@@ -54,7 +53,7 @@ var (
// "threatminer",
// "reconcloud",
"builtwith",
- "subdomaincenter",
+ "hudsonrock",
}
expectedDefaultSources = []string{
@@ -67,7 +66,6 @@ var (
"censys",
"chaos",
"chinaz",
- "columbus",
"crtsh",
"digitorus",
"dnsdumpster",
@@ -91,7 +89,6 @@ var (
// "threatminer",
// "reconcloud",
"builtwith",
- "subdomaincenter",
}
expectedDefaultRecursiveSources = []string{
@@ -100,7 +97,6 @@ var (
"bufferover",
"certspotter",
"crtsh",
- "dnsdumpster",
"dnsdb",
"digitorus",
"hackertarget",
diff --git a/v2/pkg/passive/sources_wo_auth_test.go b/v2/pkg/passive/sources_wo_auth_test.go
index db30aba8a..d5aef6530 100644
--- a/v2/pkg/passive/sources_wo_auth_test.go
+++ b/v2/pkg/passive/sources_wo_auth_test.go
@@ -24,13 +24,14 @@ func TestSourcesWithoutKeys(t *testing.T) {
}
ignoredSources := []string{
- "commoncrawl", // commoncrawl is under resourced and will likely time-out so step over it for this test https://groups.google.com/u/2/g/common-crawl/c/3QmQjFA_3y4/m/vTbhGqIBBQAJ
- "riddler", // failing due to cloudfront protection
- "crtsh", // Fails in GH Action (possibly IP-based ban) causing a timeout.
- "hackertarget", // Fails in GH Action (possibly IP-based ban) but works locally
- "waybackarchive", // Fails randomly
- "alienvault", // 503 Service Temporarily Unavailable
- "digitorus", // failing with "Failed to retrieve certificate"
+ "commoncrawl", // commoncrawl is under resourced and will likely time-out so step over it for this test https://groups.google.com/u/2/g/common-crawl/c/3QmQjFA_3y4/m/vTbhGqIBBQAJ
+ "riddler", // failing due to cloudfront protection
+ "crtsh", // Fails in GH Action (possibly IP-based ban) causing a timeout.
+ "hackertarget", // Fails in GH Action (possibly IP-based ban) but works locally
+ "waybackarchive", // Fails randomly
+ "alienvault", // 503 Service Temporarily Unavailable
+ "digitorus", // failing with "Failed to retrieve certificate"
+ "dnsdumpster", // failing with "unexpected status code 403 received"
}
domain := "hackerone.com"
diff --git a/v2/pkg/runner/banners.go b/v2/pkg/runner/banners.go
index 5d6af1227..1e9af43bc 100644
--- a/v2/pkg/runner/banners.go
+++ b/v2/pkg/runner/banners.go
@@ -17,7 +17,7 @@ const banner = `
const ToolName = `subfinder`
// Version is the current version of subfinder
-const version = `v2.6.7`
+const version = `v2.6.8`
// showBanner is used to show the banner to the user
func showBanner() {
diff --git a/v2/pkg/runner/enumerate.go b/v2/pkg/runner/enumerate.go
index 59c5d5041..f8849d960 100644
--- a/v2/pkg/runner/enumerate.go
+++ b/v2/pkg/runner/enumerate.go
@@ -19,18 +19,20 @@ import (
const maxNumCount = 2
var replacer = strings.NewReplacer(
+ "•.", "",
+ "•", "",
"*.", "",
"http://", "",
"https://", "",
)
// EnumerateSingleDomain wraps EnumerateSingleDomainWithCtx with an empty context
-func (r *Runner) EnumerateSingleDomain(domain string, writers []io.Writer) error {
+func (r *Runner) EnumerateSingleDomain(domain string, writers []io.Writer) (map[string]map[string]struct{}, error) {
return r.EnumerateSingleDomainWithCtx(context.Background(), domain, writers)
}
// EnumerateSingleDomainWithCtx performs subdomain enumeration against a single domain
-func (r *Runner) EnumerateSingleDomainWithCtx(ctx context.Context, domain string, writers []io.Writer) error {
+func (r *Runner) EnumerateSingleDomainWithCtx(ctx context.Context, domain string, writers []io.Writer) (map[string]map[string]struct{}, error) {
gologger.Info().Msgf("Enumerating subdomains for %s\n", domain)
// Check if the user has asked to remove wildcards explicitly.
@@ -61,14 +63,15 @@ func (r *Runner) EnumerateSingleDomainWithCtx(ctx context.Context, domain string
for result := range passiveResults {
switch result.Type {
case subscraping.Error:
- gologger.Warning().Msgf("Could not run source %s: %s\n", result.Source, result.Error)
+ gologger.Warning().Msgf("Encountered an error with source %s: %s\n", result.Source, result.Error)
case subscraping.Subdomain:
+ subdomain := replacer.Replace(result.Value)
+
// Validate the subdomain found and remove wildcards from
- if !strings.HasSuffix(result.Value, "."+domain) {
+ if !strings.HasSuffix(subdomain, "."+domain) {
skippedCounts[result.Source]++
continue
}
- subdomain := replacer.Replace(result.Value)
if matchSubdomain := r.filterAndMatchSubdomain(subdomain); matchSubdomain {
if _, ok := uniqueMap[subdomain]; !ok {
@@ -145,7 +148,7 @@ func (r *Runner) EnumerateSingleDomainWithCtx(ctx context.Context, domain string
}
if err != nil {
gologger.Error().Msgf("Could not write results for %s: %s\n", domain, err)
- return err
+ return nil, err
}
}
@@ -186,7 +189,7 @@ func (r *Runner) EnumerateSingleDomainWithCtx(ctx context.Context, domain string
printStatistics(statistics)
}
- return nil
+ return sourceMap, nil
}
func (r *Runner) filterAndMatchSubdomain(subdomain string) bool {
diff --git a/v2/pkg/runner/options.go b/v2/pkg/runner/options.go
index 32562cf17..5e3d4b172 100644
--- a/v2/pkg/runner/options.go
+++ b/v2/pkg/runner/options.go
@@ -257,5 +257,5 @@ var defaultRateLimits = []string{
"netlas=1/s",
// "gitlab=2/s",
"github=83/m",
- "subdomaincenter=2/m",
+ "hudsonrock=5/s",
}
diff --git a/v2/pkg/runner/runner.go b/v2/pkg/runner/runner.go
index e5cb3740b..3a79f637f 100644
--- a/v2/pkg/runner/runner.go
+++ b/v2/pkg/runner/runner.go
@@ -136,7 +136,7 @@ func (r *Runner) EnumerateMultipleDomainsWithCtx(ctx context.Context, reader io.
return err
}
- err = r.EnumerateSingleDomainWithCtx(ctx, domain, append(writers, file))
+ _, err = r.EnumerateSingleDomainWithCtx(ctx, domain, append(writers, file))
file.Close()
} else if r.options.OutputDirectory != "" {
@@ -154,11 +154,11 @@ func (r *Runner) EnumerateMultipleDomainsWithCtx(ctx context.Context, reader io.
return err
}
- err = r.EnumerateSingleDomainWithCtx(ctx, domain, append(writers, file))
+ _, err = r.EnumerateSingleDomainWithCtx(ctx, domain, append(writers, file))
file.Close()
} else {
- err = r.EnumerateSingleDomainWithCtx(ctx, domain, writers)
+ _, err = r.EnumerateSingleDomainWithCtx(ctx, domain, writers)
}
if err != nil {
return err
diff --git a/v2/pkg/subscraping/sources/dnsdumpster/dnsdumpster.go b/v2/pkg/subscraping/sources/dnsdumpster/dnsdumpster.go
index 8e6afbd4a..2155e31cf 100644
--- a/v2/pkg/subscraping/sources/dnsdumpster/dnsdumpster.go
+++ b/v2/pkg/subscraping/sources/dnsdumpster/dnsdumpster.go
@@ -3,67 +3,29 @@ package dnsdumpster
import (
"context"
+ "encoding/json"
"fmt"
- "io"
- "net/url"
- "regexp"
- "strings"
"time"
"github.com/projectdiscovery/subfinder/v2/pkg/subscraping"
)
-// CSRFSubMatchLength CSRF regex submatch length
-const CSRFSubMatchLength = 2
-
-var re = regexp.MustCompile("")
-
-// getCSRFToken gets the CSRF Token from the page
-func getCSRFToken(page string) string {
- if subs := re.FindStringSubmatch(page); len(subs) == CSRFSubMatchLength {
- return strings.TrimSpace(subs[1])
- }
- return ""
-}
-
-// postForm posts a form for a domain and returns the response
-func postForm(ctx context.Context, session *subscraping.Session, token, domain string) (string, error) {
- params := url.Values{
- "csrfmiddlewaretoken": {token},
- "targetip": {domain},
- "user": {"free"},
- }
-
- resp, err := session.HTTPRequest(
- ctx,
- "POST",
- "https://dnsdumpster.com/",
- fmt.Sprintf("csrftoken=%s; Domain=dnsdumpster.com", token),
- map[string]string{
- "Content-Type": "application/x-www-form-urlencoded",
- "Referer": "https://dnsdumpster.com",
- "X-CSRF-Token": token,
- },
- strings.NewReader(params.Encode()),
- subscraping.BasicAuth{},
- )
-
- if err != nil {
- session.DiscardHTTPResponse(resp)
- return "", err
- }
-
- // Now, grab the entire page
- in, err := io.ReadAll(resp.Body)
- resp.Body.Close()
- return string(in), err
+type response struct {
+ A []struct {
+ Host string `json:"host"`
+ } `json:"a"`
+ Ns []struct {
+ Host string `json:"host"`
+ } `json:"ns"`
}
// Source is the passive scraping agent
type Source struct {
+ apiKeys []string
timeTaken time.Duration
errors int
results int
+ skipped bool
}
// Run function returns all subdomains found with the service
@@ -78,35 +40,35 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se
close(results)
}(time.Now())
- resp, err := session.SimpleGet(ctx, "https://dnsdumpster.com/")
- if err != nil {
- results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
- s.errors++
- session.DiscardHTTPResponse(resp)
+ randomApiKey := subscraping.PickRandom(s.apiKeys, s.Name())
+ if randomApiKey == "" {
+ s.skipped = true
return
}
- body, err := io.ReadAll(resp.Body)
+ resp, err := session.Get(ctx, fmt.Sprintf("https://api.dnsdumpster.com/domain/%s", domain), "", map[string]string{"X-API-Key": randomApiKey})
if err != nil {
results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
s.errors++
- resp.Body.Close()
+ session.DiscardHTTPResponse(resp)
return
}
- resp.Body.Close()
+ defer resp.Body.Close()
- csrfToken := getCSRFToken(string(body))
- data, err := postForm(ctx, session, csrfToken, domain)
+ var response response
+ err = json.NewDecoder(resp.Body).Decode(&response)
if err != nil {
results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
s.errors++
+ resp.Body.Close()
return
}
- for _, subdomain := range session.Extractor.Extract(data) {
- results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain}
+ for _, record := range append(response.A, response.Ns...) {
+ results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: record.Host}
s.results++
}
+
}()
return results
@@ -122,15 +84,15 @@ func (s *Source) IsDefault() bool {
}
func (s *Source) HasRecursiveSupport() bool {
- return true
+ return false
}
func (s *Source) NeedsKey() bool {
- return false
+ return true
}
-func (s *Source) AddApiKeys(_ []string) {
- // no key needed
+func (s *Source) AddApiKeys(keys []string) {
+ s.apiKeys = keys
}
func (s *Source) Statistics() subscraping.Statistics {
@@ -138,5 +100,6 @@ func (s *Source) Statistics() subscraping.Statistics {
Errors: s.errors,
Results: s.results,
TimeTaken: s.timeTaken,
+ Skipped: s.skipped,
}
}
diff --git a/v2/pkg/subscraping/sources/github/github.go b/v2/pkg/subscraping/sources/github/github.go
index e1a77d727..6034b2dbc 100644
--- a/v2/pkg/subscraping/sources/github/github.go
+++ b/v2/pkg/subscraping/sources/github/github.go
@@ -11,6 +11,7 @@ import (
"regexp"
"strconv"
"strings"
+ "sync"
"time"
jsoniter "github.com/json-iterator/go"
@@ -142,40 +143,58 @@ func (s *Source) enumerate(ctx context.Context, searchURL string, domainRegexp *
// proccesItems process github response items
func (s *Source) proccesItems(ctx context.Context, items []item, domainRegexp *regexp.Regexp, name string, session *subscraping.Session, results chan subscraping.Result) error {
- for _, item := range items {
- // find subdomains in code
- resp, err := session.SimpleGet(ctx, rawURL(item.HTMLURL))
- if err != nil {
- if resp != nil && resp.StatusCode != http.StatusNotFound {
- session.DiscardHTTPResponse(resp)
+ var wg sync.WaitGroup
+ errChan := make(chan error, len(items))
+
+ for _, responseItem := range items {
+ wg.Add(1)
+ go func(responseItem item) {
+ defer wg.Done()
+
+ // find subdomains in code
+ resp, err := session.SimpleGet(ctx, rawURL(responseItem.HTMLURL))
+ if err != nil {
+ if resp != nil && resp.StatusCode != http.StatusNotFound {
+ session.DiscardHTTPResponse(resp)
+ }
+ errChan <- err
+ return
}
- return err
- }
- if resp.StatusCode == http.StatusOK {
- scanner := bufio.NewScanner(resp.Body)
- for scanner.Scan() {
- line := scanner.Text()
- if line == "" {
- continue
+ if resp.StatusCode == http.StatusOK {
+ scanner := bufio.NewScanner(resp.Body)
+ for scanner.Scan() {
+ line := scanner.Text()
+ if line == "" {
+ continue
+ }
+ for _, subdomain := range domainRegexp.FindAllString(normalizeContent(line), -1) {
+ results <- subscraping.Result{Source: name, Type: subscraping.Subdomain, Value: subdomain}
+ s.results++
+ }
}
- for _, subdomain := range domainRegexp.FindAllString(normalizeContent(line), -1) {
+ resp.Body.Close()
+ }
+
+ // find subdomains in text matches
+ for _, textMatch := range responseItem.TextMatches {
+ for _, subdomain := range domainRegexp.FindAllString(normalizeContent(textMatch.Fragment), -1) {
results <- subscraping.Result{Source: name, Type: subscraping.Subdomain, Value: subdomain}
s.results++
-
}
}
- resp.Body.Close()
- }
+ }(responseItem)
+ }
- // find subdomains in text matches
- for _, textMatch := range item.TextMatches {
- for _, subdomain := range domainRegexp.FindAllString(normalizeContent(textMatch.Fragment), -1) {
- results <- subscraping.Result{Source: name, Type: subscraping.Subdomain, Value: subdomain}
- s.results++
- }
+ wg.Wait()
+ close(errChan)
+
+ for err := range errChan {
+ if err != nil {
+ return err
}
}
+
return nil
}
diff --git a/v2/pkg/subscraping/sources/columbus/columbus.go b/v2/pkg/subscraping/sources/hudsonrock/hudsonrock.go
similarity index 62%
rename from v2/pkg/subscraping/sources/columbus/columbus.go
rename to v2/pkg/subscraping/sources/hudsonrock/hudsonrock.go
index 83432ab92..b109a3192 100644
--- a/v2/pkg/subscraping/sources/columbus/columbus.go
+++ b/v2/pkg/subscraping/sources/hudsonrock/hudsonrock.go
@@ -1,16 +1,26 @@
-// Package columbus logic
-package columbus
+// Package hudsonrock logic
+package hudsonrock
import (
"context"
+ "encoding/json"
"fmt"
"time"
- jsoniter "github.com/json-iterator/go"
-
"github.com/projectdiscovery/subfinder/v2/pkg/subscraping"
)
+type hudsonrockResponse struct {
+ Data struct {
+ EmployeesUrls []struct {
+ URL string `json:"url"`
+ } `json:"employees_urls"`
+ ClientsUrls []struct {
+ URL string `json:"url"`
+ } `json:"clients_urls"`
+ } `json:"data"`
+}
+
// Source is the passive scraping agent
type Source struct {
timeTaken time.Duration
@@ -30,16 +40,17 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se
close(results)
}(time.Now())
- resp, err := session.SimpleGet(ctx, fmt.Sprintf("https://columbus.elmasy.com/api/lookup/%s", domain))
+ resp, err := session.SimpleGet(ctx, fmt.Sprintf("https://cavalier.hudsonrock.com/api/json/v2/osint-tools/urls-by-domain?domain=%s", domain))
if err != nil {
results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
s.errors++
session.DiscardHTTPResponse(resp)
return
}
+ defer resp.Body.Close()
- var subdomains []string
- err = jsoniter.NewDecoder(resp.Body).Decode(&subdomains)
+ var response hudsonrockResponse
+ err = json.NewDecoder(resp.Body).Decode(&response)
if err != nil {
results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
s.errors++
@@ -47,14 +58,11 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se
return
}
- resp.Body.Close()
-
- for _, record := range subdomains {
- if record == "" {
- continue
+ for _, record := range append(response.Data.EmployeesUrls, response.Data.ClientsUrls...) {
+ for _, subdomain := range session.Extractor.Extract(record.URL) {
+ results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain}
+ s.results++
}
- results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: record + "." + domain}
- s.results++
}
}()
@@ -64,11 +72,11 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se
// Name returns the name of the source
func (s *Source) Name() string {
- return "columbus"
+ return "hudsonrock"
}
func (s *Source) IsDefault() bool {
- return true
+ return false
}
func (s *Source) HasRecursiveSupport() bool {
diff --git a/v2/pkg/subscraping/sources/sitedossier/sitedossier.go b/v2/pkg/subscraping/sources/sitedossier/sitedossier.go
index 91c86298e..dacc3c66b 100644
--- a/v2/pkg/subscraping/sources/sitedossier/sitedossier.go
+++ b/v2/pkg/subscraping/sources/sitedossier/sitedossier.go
@@ -18,12 +18,6 @@ const SleepRandIntn = 5
var reNext = regexp.MustCompile(``)
-type agent struct {
- results chan subscraping.Result
- errors int
- session *subscraping.Session
-}
-
// Source is the passive scraping agent
type Source struct {
timeTaken time.Duration
@@ -37,58 +31,52 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se
s.errors = 0
s.results = 0
- a := agent{
- session: session,
- results: results,
- }
-
go func() {
defer func(startTime time.Time) {
s.timeTaken = time.Since(startTime)
- close(a.results)
+ close(results)
}(time.Now())
- a.enumerate(ctx, fmt.Sprintf("http://www.sitedossier.com/parentdomain/%s", domain))
- s.errors = a.errors
- s.results = len(a.results)
+ s.enumerate(ctx, session, fmt.Sprintf("http://www.sitedossier.com/parentdomain/%s", domain), results)
}()
- return a.results
+ return results
}
-func (a *agent) enumerate(ctx context.Context, baseURL string) {
+func (s *Source) enumerate(ctx context.Context, session *subscraping.Session, baseURL string, results chan subscraping.Result) {
select {
case <-ctx.Done():
return
default:
}
- resp, err := a.session.SimpleGet(ctx, baseURL)
+ resp, err := session.SimpleGet(ctx, baseURL)
isnotfound := resp != nil && resp.StatusCode == http.StatusNotFound
if err != nil && !isnotfound {
- a.results <- subscraping.Result{Source: "sitedossier", Type: subscraping.Error, Error: err}
- a.errors++
- a.session.DiscardHTTPResponse(resp)
+ results <- subscraping.Result{Source: "sitedossier", Type: subscraping.Error, Error: err}
+ s.errors++
+ session.DiscardHTTPResponse(resp)
return
}
body, err := io.ReadAll(resp.Body)
if err != nil {
- a.results <- subscraping.Result{Source: "sitedossier", Type: subscraping.Error, Error: err}
- a.errors++
+ results <- subscraping.Result{Source: "sitedossier", Type: subscraping.Error, Error: err}
+ s.errors++
resp.Body.Close()
return
}
resp.Body.Close()
src := string(body)
- for _, subdomain := range a.session.Extractor.Extract(src) {
- a.results <- subscraping.Result{Source: "sitedossier", Type: subscraping.Subdomain, Value: subdomain}
+ for _, subdomain := range session.Extractor.Extract(src) {
+ results <- subscraping.Result{Source: "sitedossier", Type: subscraping.Subdomain, Value: subdomain}
+ s.results++
}
match := reNext.FindStringSubmatch(src)
if len(match) > 0 {
- a.enumerate(ctx, fmt.Sprintf("http://www.sitedossier.com%s", match[1]))
+ s.enumerate(ctx, session, fmt.Sprintf("http://www.sitedossier.com%s", match[1]), results)
}
}
diff --git a/v2/pkg/subscraping/sources/subdomaincenter/subdomaincenter.go b/v2/pkg/subscraping/sources/subdomaincenter/subdomaincenter.go
deleted file mode 100644
index 40873f32a..000000000
--- a/v2/pkg/subscraping/sources/subdomaincenter/subdomaincenter.go
+++ /dev/null
@@ -1,87 +0,0 @@
-// Package subdomaincenter logic
-package subdomaincenter
-
-import (
- "context"
- "fmt"
- "time"
-
- jsoniter "github.com/json-iterator/go"
-
- "github.com/projectdiscovery/subfinder/v2/pkg/subscraping"
-)
-
-// Source is the passive scraping agent
-type Source struct {
- timeTaken time.Duration
- errors int
- results int
-}
-
-// Run function returns all subdomains found with the service
-func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result {
- results := make(chan subscraping.Result)
- s.errors = 0
- s.results = 0
-
- go func() {
- defer func(startTime time.Time) {
- s.timeTaken = time.Since(startTime)
- close(results)
- }(time.Now())
-
- resp, err := session.SimpleGet(ctx, fmt.Sprintf("https://api.subdomain.center/?domain=%s", domain))
- if err != nil {
- results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
- s.errors++
- session.DiscardHTTPResponse(resp)
- return
- }
-
- var subdomains []string
- err = jsoniter.NewDecoder(resp.Body).Decode(&subdomains)
- if err != nil {
- results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
- s.errors++
- resp.Body.Close()
- return
- }
- resp.Body.Close()
-
- for _, subdomain := range subdomains {
- results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain}
- s.results++
- }
- }()
-
- return results
-}
-
-// Name returns the name of the source
-func (s *Source) Name() string {
- return "subdomaincenter"
-}
-
-func (s *Source) IsDefault() bool {
- return true
-}
-
-func (s *Source) HasRecursiveSupport() bool {
- return false
-}
-
-func (s *Source) NeedsKey() bool {
- return false
-}
-
-func (s *Source) AddApiKeys(_ []string) {
- // no key needed
-}
-
-func (s *Source) Statistics() subscraping.Statistics {
- return subscraping.Statistics{
- Errors: s.errors,
- Results: s.results,
- TimeTaken: s.timeTaken,
- }
-}