Browse Source

Bug Fixes relating to Closing File Handles

OnionScan was having issues with larger batch runs do to
failure to close network connection once they had been used.

This commit fixes this issue, as well as adds a quick exit from
the main OnionScan loop due to timeout - once the timeout is crossed
no further protocol scans will take place.

Fixes #51
master
Sarah Jamie Lewis 4 years ago
parent
commit
872555c1df
22 changed files with 311 additions and 225 deletions
  1. +14
    -1
      config/onionscan_config.go
  2. +13
    -11
      main.go
  3. +57
    -39
      onionscan.go
  4. +7
    -7
      protocol/bitcoin_scanner.go
  5. +7
    -7
      protocol/ftp_scanner.go
  6. +36
    -33
      protocol/http_scanner.go
  7. +12
    -10
      protocol/irc_scanner.go
  8. +7
    -6
      protocol/mongodb_scanner.go
  9. +7
    -7
      protocol/ricochet_scanner.go
  10. +7
    -7
      protocol/smtp_scanner.go
  11. +18
    -8
      protocol/ssh_scanner.go
  12. +7
    -7
      protocol/vnc_scanner.go
  13. +12
    -11
      protocol/xmpp_scanner.go
  14. +25
    -13
      report/onionscanreport.go
  15. +13
    -8
      report/report_generator.go
  16. +12
    -11
      scans/apache_mod_status.go
  17. +13
    -12
      scans/check_directory_listing.go
  18. +4
    -3
      scans/check_exif.go
  19. +4
    -5
      scans/pgp_content_scan.go
  20. +15
    -0
      scans/private_key.go
  21. +2
    -1
      scans/scanner.go
  22. +19
    -18
      scans/standard-page-scan.go

+ 14
- 1
config/onionscan_config.go View File

@@ -1,6 +1,7 @@
package config

import (
"log"
"time"
)

@@ -9,13 +10,25 @@ type OnionscanConfig struct {
DirectoryDepth int
Fingerprint bool
Timeout time.Duration
Verbose bool
}

func Configure(torProxyAddress string, directoryDepth int, fingerprint bool, timeout int) *OnionscanConfig {
func Configure(torProxyAddress string, directoryDepth int, fingerprint bool, timeout int, verbose bool) *OnionscanConfig {
onionScan := new(OnionscanConfig)
onionScan.TorProxyAddress = torProxyAddress
onionScan.DirectoryDepth = directoryDepth
onionScan.Fingerprint = fingerprint
onionScan.Timeout = time.Duration(time.Second * time.Duration(timeout))
onionScan.Verbose = verbose
return onionScan
}

func (os *OnionscanConfig) LogInfo(message string) {
if os.Verbose {
log.Printf("INFO: %v", message)
}
}

func (os *OnionscanConfig) LogError(err error) {
log.Printf("ERROR: %v", err)
}

+ 13
- 11
main.go View File

@@ -1,6 +1,7 @@
package main

import (
"errors"
"flag"
"fmt"
"github.com/s-rah/onionscan/config"
@@ -28,6 +29,7 @@ func main() {
fingerprint := flag.Bool("fingerprint", true, "true disables some deeper scans e.g. directory probing with the aim of just getting a fingerprint of the service.")
list := flag.String("list", "", "If provided OnionScan will attempt to read from the given list, rather than the provided hidden service")
timeout := flag.Int("timeout", 120, "read timeout for connecting to onion services")
batch := flag.Int("batch", 10, "number of onions to scan concurrently")

flag.Parse()

@@ -58,22 +60,17 @@ func main() {
log.Printf("This might take a few minutes..\n\n")

onionScan := new(OnionScan)
onionScan.Config = config.Configure(*torProxyAddress, *directoryDepth, *fingerprint, *timeout)
onionScan.Config = config.Configure(*torProxyAddress, *directoryDepth, *fingerprint, *timeout, *verbose)

reports := make(chan *report.OnionScanReport)

if !*verbose {
log.SetOutput(ioutil.Discard)
}

count := 0
max := 100
if max > len(onionsToScan) {
max = len(onionsToScan)
if *batch > len(onionsToScan) {
*batch = len(onionsToScan)
}

// Run an initial batch of 100 requests (or less...)
for count < max {
for count < *batch {
go onionScan.Scan(onionsToScan[count], reports)
count++
}
@@ -89,11 +86,16 @@ func main() {
}

received++
if scanReport.TimedOut {
onionScan.Config.LogError(errors.New(scanReport.HiddenService + " timed out"))
}

file := scanReport.HiddenService + "." + *reportFile

if *jsonReport {
report.GenerateJsonReport(*reportFile, scanReport)
report.GenerateJsonReport(file, scanReport)
} else if *simpleReport {
report.GenerateSimpleReport(*reportFile, scanReport)
report.GenerateSimpleReport(file, scanReport)
}
}
}

+ 57
- 39
onionscan.go View File

@@ -6,12 +6,62 @@ import (
"github.com/s-rah/onionscan/report"
"github.com/s-rah/onionscan/utils"
"strings"
"time"
)

type OnionScan struct {
Config *config.OnionscanConfig
}

func (os *OnionScan) PerformNextAction(report *report.OnionScanReport) {
switch report.NextAction {
case "web":
wps := new(protocol.HTTPProtocolScanner)
wps.ScanProtocol(report.HiddenService, os.Config, report)
report.NextAction = "ssh"
case "ssh":
sps := new(protocol.SSHProtocolScanner)
sps.ScanProtocol(report.HiddenService, os.Config, report)
report.NextAction = "irc"
case "irc":
ips := new(protocol.IRCProtocolScanner)
ips.ScanProtocol(report.HiddenService, os.Config, report)
report.NextAction = "ricochet"
case "ricochet":
rps := new(protocol.RicochetProtocolScanner)
rps.ScanProtocol(report.HiddenService, os.Config, report)
report.NextAction = "ftp"
case "ftp":
fps := new(protocol.FTPProtocolScanner)
fps.ScanProtocol(report.HiddenService, os.Config, report)
report.NextAction = "smtp"
case "smtp":
smps := new(protocol.SMTPProtocolScanner)
smps.ScanProtocol(report.HiddenService, os.Config, report)
report.NextAction = "mongodb"
case "mongodb":
mdbps := new(protocol.MongoDBProtocolScanner)
mdbps.ScanProtocol(report.HiddenService, os.Config, report)
report.NextAction = "vnc"
case "vnc":
vncps := new(protocol.VNCProtocolScanner)
vncps.ScanProtocol(report.HiddenService, os.Config, report)
report.NextAction = "xmpp"
case "xmpp":
xmppps := new(protocol.XMPPProtocolScanner)
xmppps.ScanProtocol(report.HiddenService, os.Config, report)
report.NextAction = "bitcoin"
case "bitcoin":
bps := new(protocol.BitcoinProtocolScanner)
bps.ScanProtocol(report.HiddenService, os.Config, report)
report.NextAction = "none"
case "none":
return
default:
report.NextAction = "web"
}
}

func (os *OnionScan) Scan(hiddenService string, out chan *report.OnionScanReport) {

// Remove Extra Prefix
@@ -23,45 +73,13 @@ func (os *OnionScan) Scan(hiddenService string, out chan *report.OnionScanReport

report := report.NewOnionScanReport(hiddenService)

// HTTP
hps := new(protocol.HTTPProtocolScanner)
hps.ScanProtocol(hiddenService, os.Config, report)

// SSH
sps := new(protocol.SSHProtocolScanner)
sps.ScanProtocol(hiddenService, os.Config, report)

// Ricochet
rps := new(protocol.RicochetProtocolScanner)
rps.ScanProtocol(hiddenService, os.Config, report)

// Bitcoin
bps := new(protocol.BitcoinProtocolScanner)
bps.ScanProtocol(hiddenService, os.Config, report)

//IRC
ips := new(protocol.IRCProtocolScanner)
ips.ScanProtocol(hiddenService, os.Config, report)

//FTP
fps := new(protocol.FTPProtocolScanner)
fps.ScanProtocol(hiddenService, os.Config, report)

//SMTP
smps := new(protocol.SMTPProtocolScanner)
smps.ScanProtocol(hiddenService, os.Config, report)

//MongoDb
mdbps := new(protocol.MongoDBProtocolScanner)
mdbps.ScanProtocol(hiddenService, os.Config, report)

//VNC
vncps := new(protocol.VNCProtocolScanner)
vncps.ScanProtocol(hiddenService, os.Config, report)

//XMPP
xmppps := new(protocol.XMPPProtocolScanner)
xmppps.ScanProtocol(hiddenService, os.Config, report)
for report.NextAction != "none" {
os.PerformNextAction(report)
if time.Now().Sub(report.DateScanned).Seconds() > os.Config.Timeout.Seconds() {
report.TimedOut = true
report.NextAction = "none"
}
}

out <- report
}

+ 7
- 7
protocol/bitcoin_scanner.go View File

@@ -1,26 +1,26 @@
package protocol

import (
"fmt"
"github.com/s-rah/onionscan/config"
"github.com/s-rah/onionscan/report"
"github.com/s-rah/onionscan/utils"
"log"
)

type BitcoinProtocolScanner struct {
}

func (rps *BitcoinProtocolScanner) ScanProtocol(hiddenService string, onionscanConfig *config.OnionscanConfig, report *report.OnionScanReport) {
func (rps *BitcoinProtocolScanner) ScanProtocol(hiddenService string, osc *config.OnionscanConfig, report *report.OnionScanReport) {
// Bitcoin
log.Printf("Checking %s Bitcoin(8333)\n", hiddenService)
_, err := utils.GetNetworkConnection(hiddenService, 8333, onionscanConfig.TorProxyAddress, onionscanConfig.Timeout)
osc.LogInfo(fmt.Sprintf("Checking %s Bitcoin(8333)\n", hiddenService))
conn, err := utils.GetNetworkConnection(hiddenService, 8333, osc.TorProxyAddress, osc.Timeout)
if err != nil {
log.Printf("Failed to connect to service on port 8333\n")
osc.LogInfo("Failed to connect to service on port 8333\n")
report.BitcoinDetected = false
} else {
log.Printf("Detected possible Bitcoin instance\n")
osc.LogInfo("Detected possible Bitcoin instance\n")
// TODO: Actual Analysis
report.BitcoinDetected = true
}
conn.Close()
}

+ 7
- 7
protocol/ftp_scanner.go View File

@@ -4,21 +4,21 @@ import (
"bufio"
"crypto/sha1"
"encoding/hex"
"fmt"
"github.com/s-rah/onionscan/config"
"github.com/s-rah/onionscan/report"
"github.com/s-rah/onionscan/utils"
"log"
)

type FTPProtocolScanner struct {
}

func (sps *FTPProtocolScanner) ScanProtocol(hiddenService string, onionscanConfig *config.OnionscanConfig, report *report.OnionScanReport) {
func (sps *FTPProtocolScanner) ScanProtocol(hiddenService string, osc *config.OnionscanConfig, report *report.OnionScanReport) {
// FTP
log.Printf("Checking %s FTP(21)\n", hiddenService)
conn, err := utils.GetNetworkConnection(hiddenService, 21, onionscanConfig.TorProxyAddress, onionscanConfig.Timeout)
osc.LogInfo(fmt.Sprintf("Checking %s FTP(21)\n", hiddenService))
conn, err := utils.GetNetworkConnection(hiddenService, 21, osc.TorProxyAddress, osc.Timeout)
if err != nil {
log.Printf("Failed to connect to service on port 21\n")
osc.LogInfo("Failed to connect to service on port 21\n")
report.FTPDetected = false
} else {
// TODO FTP Checking
@@ -29,8 +29,8 @@ func (sps *FTPProtocolScanner) ScanProtocol(hiddenService string, onionscanConfi
report.FTPBanner = banner
hash := sha1.Sum([]byte(banner))
report.FTPFingerprint = hex.EncodeToString(hash[:])
log.Printf("Found FTP Banner: %s (%s)", banner, report.FTPFingerprint)
osc.LogInfo(fmt.Sprintf("Found FTP Banner: %s (%s)", banner, report.FTPFingerprint))
}
}
conn.Close()
}

+ 36
- 33
protocol/http_scanner.go View File

@@ -1,13 +1,13 @@
package protocol

import (
"fmt"
"github.com/s-rah/onionscan/config"
"github.com/s-rah/onionscan/report"
"github.com/s-rah/onionscan/scans"
"github.com/s-rah/onionscan/utils"
"h12.me/socks"
"io/ioutil"
"log"
"net/http"
"strings"
)
@@ -24,63 +24,67 @@ var (
"/products", "/products/cat"}
)

func (hps *HTTPProtocolScanner) ScanProtocol(hiddenService string, onionscanConfig *config.OnionscanConfig, report *report.OnionScanReport) {
func (hps *HTTPProtocolScanner) ScanProtocol(hiddenService string, osc *config.OnionscanConfig, report *report.OnionScanReport) {

// HTTP
log.Printf("Checking %s http(80)\n", hiddenService)
_, err := utils.GetNetworkConnection(hiddenService, 80, onionscanConfig.TorProxyAddress, onionscanConfig.Timeout)
osc.LogInfo(fmt.Sprintf("Checking %s http(80)\n", hiddenService))
conn, err := utils.GetNetworkConnection(hiddenService, 80, osc.TorProxyAddress, osc.Timeout)
if err != nil {
log.Printf("Failed to connect to service on port 80\n")
osc.LogInfo("Failed to connect to service on port 80\n")
report.WebDetected = false
return
conn.Close()
} else {
log.Printf("Found potential service on http(80)\n")
osc.LogInfo("Found potential service on http(80)\n")
report.WebDetected = true
dialSocksProxy := socks.DialSocksProxy(socks.SOCKS5, onionscanConfig.TorProxyAddress)
conn.Close()
dialSocksProxy := socks.DialSocksProxy(socks.SOCKS5, osc.TorProxyAddress)
transportConfig := &http.Transport{
Dial: dialSocksProxy,
}
hps.Client = &http.Client{Transport: transportConfig}
// FIXME This should probably be moved to it's own file now.
response, err := hps.Client.Get("http://" + hiddenService)
if err == nil {
// Reading all http headers
osc.LogInfo(fmt.Sprintf("HTTP response headers: %s\n", report.ServerVersion))
responseHeaders := response.Header
for key := range responseHeaders {
value := responseHeaders.Get(key)
// normalize by strings.ToUpper(key) to avoid case sensitive checking
report.AddResponseHeader(strings.ToUpper(key), value)
}

if err != nil {
log.Printf("Failed to connect to service on port 80\n")
return
}

// Reading all http headers
log.Printf("HTTP response headers: %s\n", report.ServerVersion)
responseHeaders := response.Header
for key := range responseHeaders {
value := responseHeaders.Get(key)
// normalize by strings.ToUpper(key) to avoid case sensitive checking
report.AddResponseHeader(strings.ToUpper(key), value)
log.Printf("\t%s : %s\n", strings.ToUpper(key), value)
report.ServerVersion = responseHeaders.Get("Server")
response.Body.Close()
} else {
osc.LogError(err)
}

report.ServerVersion = responseHeaders.Get("Server")

// Apache mod-status Check
hps.ScanPage(hiddenService, "/server-status", report, scans.ApacheModStatus)
hps.ScanPage(hiddenService, "/", report, scans.StandardPageScan)
hps.ScanPage(hiddenService, "/", report, osc, scans.StandardPageScan)
hps.ScanPage(hiddenService, "/server-status", report, osc, scans.ApacheModStatus)
hps.ScanPage(hiddenService, "/private_key", report, osc, scans.PrivateKeyScan)

if onionscanConfig.Fingerprint == false {
log.Printf("\tScanning Common and Referenced Directories\n")
if osc.Fingerprint == false {
osc.LogInfo("\tScanning Common and Referenced Directories\n")
directories := append(CommonDirectories, report.PageReferencedDirectories...)
utils.RemoveDuplicates(&directories)

for _, directory := range directories {
hps.ScanPage(hiddenService, directory, report, scans.CheckDirectoryListing(onionscanConfig.DirectoryDepth))
hps.ScanPage(hiddenService, directory, report, osc, scans.CheckDirectoryListing(osc.DirectoryDepth))
}
}
}
log.Printf("\n")
}

func (hps *HTTPProtocolScanner) ScanPage(hiddenService string, page string, report *report.OnionScanReport, f func(scans.Scanner, string, int, string, *report.OnionScanReport)) {
_, contents, responseCode := hps.ScrapePage(hiddenService, page)
f(hps, page, responseCode, string(contents), report)
func (hps *HTTPProtocolScanner) ScanPage(hiddenService string, page string, report *report.OnionScanReport, osc *config.OnionscanConfig, f func(scans.Scanner, string, int, string, *report.OnionScanReport, *config.OnionscanConfig)) {
err, contents, responseCode := hps.ScrapePage(hiddenService, page)
if err == nil {
f(hps, page, responseCode, string(contents), report, osc)
return
} else {
osc.LogError(err)
}
}

func (hps *HTTPProtocolScanner) ScrapePage(hiddenService string, page string) (error, []byte, int) {
@@ -92,7 +96,6 @@ func (hps *HTTPProtocolScanner) ScrapePage(hiddenService string, page string) (e
}
response, err := hps.Client.Get("http://" + page)
if err != nil {
log.Printf("Error connecting to http://%s %s\n", page, err)
return err, nil, -1
}
defer response.Body.Close()


+ 12
- 10
protocol/irc_scanner.go View File

@@ -1,36 +1,38 @@
package protocol

import (
"fmt"
"github.com/s-rah/onionscan/config"
"github.com/s-rah/onionscan/report"
"github.com/s-rah/onionscan/utils"
"log"
)

type IRCProtocolScanner struct {
}

func (rps *IRCProtocolScanner) ScanProtocol(hiddenService string, onionscanConfig *config.OnionscanConfig, report *report.OnionScanReport) {
func (rps *IRCProtocolScanner) ScanProtocol(hiddenService string, osc *config.OnionscanConfig, report *report.OnionScanReport) {
// IRC
log.Printf("Checking %s IRC(6667)\n", hiddenService)
_, err := utils.GetNetworkConnection(hiddenService, 6667, onionscanConfig.TorProxyAddress, onionscanConfig.Timeout)
osc.LogInfo(fmt.Sprintf("Checking %s IRC(6667)\n", hiddenService))
conn, err := utils.GetNetworkConnection(hiddenService, 6667, osc.TorProxyAddress, osc.Timeout)
if err != nil {
log.Printf("Failed to connect to service on port 6667\n")
osc.LogInfo("Failed to connect to service on port 6667\n")
report.IRCDetected = false
} else {
log.Printf("Detected possible IRC instance\n")
osc.LogInfo("Detected possible IRC instance\n")
// TODO: Actual Analysis
report.IRCDetected = true
}
conn.Close()

// IRC
log.Printf("Checking %s IRC(6697)\n", hiddenService)
_, err = utils.GetNetworkConnection(hiddenService, 6697, onionscanConfig.TorProxyAddress, onionscanConfig.Timeout)
osc.LogInfo(fmt.Sprintf("Checking %s IRC(6697)\n", hiddenService))
conn, err = utils.GetNetworkConnection(hiddenService, 6697, osc.TorProxyAddress, osc.Timeout)
if err != nil {
log.Printf("Failed to connect to service on port 6697\n")
osc.LogInfo("Failed to connect to service on port 6697\n")
} else {
log.Printf("Detected possible IRC (secure) instance\n")
osc.LogInfo("Detected possible IRC (secure) instance\n")
// TODO: Actual Analysis
report.IRCDetected = true
}
conn.Close()
}

+ 7
- 6
protocol/mongodb_scanner.go View File

@@ -1,26 +1,27 @@
package protocol

import (
"fmt"
"github.com/s-rah/onionscan/config"
"github.com/s-rah/onionscan/report"
"github.com/s-rah/onionscan/utils"
"log"
)

type MongoDBProtocolScanner struct {
}

func (rps *MongoDBProtocolScanner) ScanProtocol(hiddenService string, onionscanConfig *config.OnionscanConfig, report *report.OnionScanReport) {
func (rps *MongoDBProtocolScanner) ScanProtocol(hiddenService string, osc *config.OnionscanConfig, report *report.OnionScanReport) {
// MongoDB
log.Printf("Checking %s MongoDB(27017)\n", hiddenService)
_, err := utils.GetNetworkConnection(hiddenService, 27017, onionscanConfig.TorProxyAddress, onionscanConfig.Timeout)
osc.LogInfo(fmt.Sprintf("Checking %s MongoDB(27017)\n", hiddenService))
conn, err := utils.GetNetworkConnection(hiddenService, 27017, osc.TorProxyAddress, osc.Timeout)
if err != nil {
log.Printf("Failed to connect to service on port 27017\n")
osc.LogInfo("Failed to connect to service on port 27017\n")
report.MongoDBDetected = false
} else {
log.Printf("Detected possible MongoDB instance\n")
osc.LogInfo("Detected possible MongoDB instance\n")
// TODO: Actual Analysis
report.MongoDBDetected = true
}
conn.Close()

}

+ 7
- 7
protocol/ricochet_scanner.go View File

@@ -1,26 +1,26 @@
package protocol

import (
"fmt"
"github.com/s-rah/onionscan/config"
"github.com/s-rah/onionscan/report"
"github.com/s-rah/onionscan/utils"
"log"
)

type RicochetProtocolScanner struct {
}

func (rps *RicochetProtocolScanner) ScanProtocol(hiddenService string, onionscanConfig *config.OnionscanConfig, report *report.OnionScanReport) {
func (rps *RicochetProtocolScanner) ScanProtocol(hiddenService string, osc *config.OnionscanConfig, report *report.OnionScanReport) {
// Ricochet
log.Printf("Checking %s ricochet(9878)\n", hiddenService)
_, err := utils.GetNetworkConnection(hiddenService, 9878, onionscanConfig.TorProxyAddress, onionscanConfig.Timeout)
osc.LogInfo(fmt.Sprintf("Checking %s ricochet(9878)\n", hiddenService))
conn, err := utils.GetNetworkConnection(hiddenService, 9878, osc.TorProxyAddress, osc.Timeout)
if err != nil {
log.Printf("Failed to connect to service on port 9878\n")
osc.LogInfo("Failed to connect to service on port 9878\n")
report.RicochetDetected = false
} else {
log.Printf("Detected possible ricochet instance\n")
osc.LogInfo("Detected possible ricochet instance\n")
// TODO: Actual Analysis
report.RicochetDetected = true
}
conn.Close()
}

+ 7
- 7
protocol/smtp_scanner.go View File

@@ -4,21 +4,21 @@ import (
"bufio"
"crypto/sha1"
"encoding/hex"
"fmt"
"github.com/s-rah/onionscan/config"
"github.com/s-rah/onionscan/report"
"github.com/s-rah/onionscan/utils"
"log"
)

type SMTPProtocolScanner struct {
}

func (sps *SMTPProtocolScanner) ScanProtocol(hiddenService string, onionscanConfig *config.OnionscanConfig, report *report.OnionScanReport) {
func (sps *SMTPProtocolScanner) ScanProtocol(hiddenService string, osc *config.OnionscanConfig, report *report.OnionScanReport) {
// SMTP
log.Printf("Checking %s SMTP(25)\n", hiddenService)
conn, err := utils.GetNetworkConnection(hiddenService, 25, onionscanConfig.TorProxyAddress, onionscanConfig.Timeout)
osc.LogInfo(fmt.Sprintf("Checking %s SMTP(25)\n", hiddenService))
conn, err := utils.GetNetworkConnection(hiddenService, 25, osc.TorProxyAddress, osc.Timeout)
if err != nil {
log.Printf("Failed to connect to service on port 25\n")
osc.LogInfo("Failed to connect to service on port 25\n")
report.SMTPDetected = false
} else {
// TODO SMTP Checking
@@ -29,8 +29,8 @@ func (sps *SMTPProtocolScanner) ScanProtocol(hiddenService string, onionscanConf
report.SMTPBanner = banner
hash := sha1.Sum([]byte(banner))
report.SMTPFingerprint = hex.EncodeToString(hash[:])
log.Printf("Found SMTP Banner: %s (%s)", banner, report.SMTPFingerprint)
osc.LogInfo(fmt.Sprintf("Found SMTP Banner: %s (%s)", banner, report.SMTPFingerprint))
}
}
conn.Close()
}

+ 18
- 8
protocol/ssh_scanner.go View File

@@ -1,6 +1,7 @@
package protocol

import (
"bufio"
"crypto/md5"
"errors"
"fmt"
@@ -8,20 +9,20 @@ import (
"github.com/s-rah/onionscan/report"
"github.com/s-rah/onionscan/utils"
"golang.org/x/crypto/ssh"
"log"
"net"
)

type SSHProtocolScanner struct {
}

func (sps *SSHProtocolScanner) ScanProtocol(hiddenService string, onionscanConfig *config.OnionscanConfig, report *report.OnionScanReport) {
func (sps *SSHProtocolScanner) ScanProtocol(hiddenService string, osc *config.OnionscanConfig, report *report.OnionScanReport) {
// SSH
log.Printf("Checking %s ssh(22)\n", hiddenService)
conn, err := utils.GetNetworkConnection(hiddenService, 22, onionscanConfig.TorProxyAddress, onionscanConfig.Timeout)
osc.LogInfo(fmt.Sprintf("Checking %s ssh(22)\n", hiddenService))
conn, err := utils.GetNetworkConnection(hiddenService, 22, osc.TorProxyAddress, osc.Timeout)
if err != nil {
log.Printf("Failed to connect to service on port 22\n")
osc.LogInfo("Failed to connect to service on port 22\n")
report.SSHDetected = false
conn.Close()
} else {
// TODO SSH Checking
report.SSHDetected = true
@@ -41,13 +42,22 @@ func (sps *SSHProtocolScanner) ScanProtocol(hiddenService string, onionscanConfi
}
}
report.SSHKey = fingerprint
log.Printf("Found SSH Key %s\n", fingerprint)
osc.LogInfo(fmt.Sprintf("Found SSH Key %s\n", fingerprint))
// We don't want to continue
return errors.New("error")
},
}
ssh.NewClientConn(conn, hiddenService+":22", config)

conn.Close()
conn, err = utils.GetNetworkConnection(hiddenService, 22, osc.TorProxyAddress, osc.Timeout)
if err == nil {
reader := bufio.NewReader(conn)
banner, err := reader.ReadString('\n')
if err == nil {
report.SSHBanner = banner
osc.LogInfo(fmt.Sprintf("Found SSH Banner: %s (%s)", banner))
}
}
conn.Close()
}

}

+ 7
- 7
protocol/vnc_scanner.go View File

@@ -1,26 +1,26 @@
package protocol

import (
"fmt"
"github.com/s-rah/onionscan/config"
"github.com/s-rah/onionscan/report"
"github.com/s-rah/onionscan/utils"
"log"
)

type VNCProtocolScanner struct {
}

func (vncps *VNCProtocolScanner) ScanProtocol(hiddenService string, onionscanConfig *config.OnionscanConfig, report *report.OnionScanReport) {
func (vncps *VNCProtocolScanner) ScanProtocol(hiddenService string, osc *config.OnionscanConfig, report *report.OnionScanReport) {
// MongoDB
log.Printf("Checking %s VNC(5900)\n", hiddenService)
_, err := utils.GetNetworkConnection(hiddenService, 5900, onionscanConfig.TorProxyAddress, onionscanConfig.Timeout)
osc.LogInfo(fmt.Sprintf("Checking %s VNC(5900)\n", hiddenService))
conn, err := utils.GetNetworkConnection(hiddenService, 5900, osc.TorProxyAddress, osc.Timeout)
if err != nil {
log.Printf("Failed to connect to service on port 5900\n")
osc.LogInfo("Failed to connect to service on port 5900\n")
report.VNCDetected = false
} else {
log.Printf("Detected possible VNC instance\n")
osc.LogInfo("Detected possible VNC instance\n")
// TODO: Actual Analysis
report.VNCDetected = true
}
conn.Close()
}

+ 12
- 11
protocol/xmpp_scanner.go View File

@@ -1,36 +1,37 @@
package protocol

import (
"fmt"
"github.com/s-rah/onionscan/config"
"github.com/s-rah/onionscan/report"
"github.com/s-rah/onionscan/utils"
"log"
)

type XMPPProtocolScanner struct {
}

func (rps *XMPPProtocolScanner) ScanProtocol(hiddenService string, onionscanConfig *config.OnionscanConfig, report *report.OnionScanReport) {
func (rps *XMPPProtocolScanner) ScanProtocol(hiddenService string, osc *config.OnionscanConfig, report *report.OnionScanReport) {
// XMPP
log.Printf("Checking %s XMPP(5222)\n", hiddenService)
_, err := utils.GetNetworkConnection(hiddenService, 5222, onionscanConfig.TorProxyAddress, onionscanConfig.Timeout)
osc.LogInfo(fmt.Sprintf("Checking %s XMPP(5222)\n", hiddenService))
conn, err := utils.GetNetworkConnection(hiddenService, 5222, osc.TorProxyAddress, osc.Timeout)
if err != nil {
log.Printf("Failed to connect to service on port 5222\n")
osc.LogInfo("Failed to connect to service on port 5222\n")
report.XMPPDetected = false
} else {
log.Printf("Detected possible XMPP instance\n")
osc.LogInfo("Detected possible XMPP instance\n")
// TODO: Actual Analysis
report.XMPPDetected = true
}
conn.Close()
// XMPP
log.Printf("Checking %s XMPP(5223)\n", hiddenService)
_, err = utils.GetNetworkConnection(hiddenService, 5223, onionscanConfig.TorProxyAddress, onionscanConfig.Timeout)
osc.LogInfo(fmt.Sprintf("Checking %s XMPP(5223)\n", hiddenService))
conn, err = utils.GetNetworkConnection(hiddenService, 5223, osc.TorProxyAddress, osc.Timeout)
if err != nil {
log.Printf("Failed to connect to service on port 5223\n")
osc.LogInfo("Failed to connect to service on port 5223\n")
} else {
log.Printf("Detected possible XMPP (secure) instance\n")
osc.LogInfo("Detected possible XMPP (secure) instance\n")
// TODO: Actual Analysis
report.XMPPDetected = true
}
conn.Close()
}

+ 25
- 13
report/onionscanreport.go View File

@@ -4,6 +4,7 @@ import (
"encoding/json"
"github.com/s-rah/onionscan/utils"
"io/ioutil"
"time"
)

type ExifTag struct {
@@ -23,19 +24,22 @@ type PGPKey struct {
}

type OnionScanReport struct {
HiddenService string `json:"hiddenService"`
HiddenService string `json:"hiddenService"`
DateScanned time.Time `json:"dateScanned"`

// Summary
WebDetected bool `json:"webDetected"`
SSHDetected bool `json:"sshDetected"`
RicochetDetected bool `json:"ricochetDetected"`
IRCDetected bool `json:"ircDetected"`
FTPDetected bool `json:"ftpDetected"`
SMTPDetected bool `json:"smtpDetected"`
BitcoinDetected bool `json:"bitcoinDetected"`
MongoDBDetected bool `json:"mongodbDetected"`
VNCDetected bool `json:"vncDetected"`
XMPPDetected bool `json:"xmppDetected"`
WebDetected bool `json:"webDetected"`
SSHDetected bool `json:"sshDetected"`
RicochetDetected bool `json:"ricochetDetected"`
IRCDetected bool `json:"ircDetected"`
FTPDetected bool `json:"ftpDetected"`
SMTPDetected bool `json:"smtpDetected"`
BitcoinDetected bool `json:"bitcoinDetected"`
MongoDBDetected bool `json:"mongodbDetected"`
VNCDetected bool `json:"vncDetected"`
XMPPDetected bool `json:"xmppDetected"`
SkynetDetected bool `json:"skynetDetected"`
PrivateKeyDetected bool `json:"privateKeyDetected"`

// Web Specific
ServerPoweredBy string `json:"serverPoweredBy"`
@@ -60,7 +64,8 @@ type OnionScanReport struct {
BitcoinAddresses []string `json:"bitcoinAddresses"`

// SSH
SSHKey string `json:"sshKey"`
SSHKey string `json:"sshKey"`
SSHBanner string `json:"sshBanner"`

// FTP
FTPFingerprint string `json:"ftpFingerprint"`
@@ -69,6 +74,9 @@ type OnionScanReport struct {
// SMTP
SMTPFingerprint string `json:"smtpFingerprint"`
SMTPBanner string `json:"smtpBanner"`

NextAction string `json:"lastAction"`
TimedOut bool
}

func LoadReportFromFile(filename string) (OnionScanReport, error) {
@@ -82,7 +90,11 @@ func LoadReportFromFile(filename string) (OnionScanReport, error) {
}

func NewOnionScanReport(hiddenService string) *OnionScanReport {
return &OnionScanReport{HiddenService: hiddenService, ResponseHeaders: make(map[string]string)}
report := new(OnionScanReport)
report.HiddenService = hiddenService
report.ResponseHeaders = make(map[string]string)
report.DateScanned = time.Now()
return report
}

func (osr *OnionScanReport) AddOpenDirectory(dir string) {


+ 13
- 8
report/report_generator.go View File

@@ -5,6 +5,7 @@ import (
"fmt"
"log"
"os"
"time"
)

func GenerateJsonReport(reportFile string, report *OnionScanReport) {
@@ -14,10 +15,12 @@ func GenerateJsonReport(reportFile string, report *OnionScanReport) {
buffer.WriteString(fmt.Sprintf("%s\n", jsonOut))

if len(reportFile) > 0 {
f, err := os.Create(report.HiddenService + "." + reportFile)
if err != nil {
log.Fatalf("Cannot create report file: %s", err)
panic(err)
f, err := os.Create(reportFile)

for err != nil {
log.Printf("Cannot create report file: %s...trying again in 5 seconds...", err)
time.Sleep(time.Second * 5)
f, err = os.Create(reportFile)
}

defer f.Close()
@@ -172,10 +175,12 @@ func GenerateSimpleReport(reportFile string, report *OnionScanReport) {
}

if len(reportFile) > 0 {
f, err := os.Create(report.HiddenService + "." + reportFile)
if err != nil {
log.Fatalf("Cannot create report file: %s", err)
panic(err)
f, err := os.Create(reportFile)

for err != nil {
log.Printf("Cannot create report file: %s...trying again in 5 seconds...", err)
time.Sleep(time.Second * 5)
f, err = os.Create(reportFile)
}

defer f.Close()


+ 12
- 11
scans/apache_mod_status.go View File

@@ -1,58 +1,59 @@
package scans

import (
"fmt"
"github.com/s-rah/onionscan/config"
"github.com/s-rah/onionscan/report"
"github.com/s-rah/onionscan/utils"
"log"
"regexp"
"strings"
)

func ApacheModStatus(scan Scanner, page string, status int, contents string, report *report.OnionScanReport) {
func ApacheModStatus(scan Scanner, page string, status int, contents string, report *report.OnionScanReport, osc *config.OnionscanConfig) {
if status == 200 {
r := regexp.MustCompile(`Server Version: (.*)</dt>`)
serverVersion := r.FindStringSubmatch(string(contents))

// Check if this looks like a mod_status page. Sometimes sites simply load their index.
if len(serverVersion) > 1 {
log.Printf("Detected Apache mod_status Exposed...\033[091mAlert!\033[0m\n")
osc.LogInfo("Detected Apache mod_status Exposed...\033[091mAlert!\033[0m\n")
report.FoundApacheModStatus = true

log.Printf("\t Using mod_status Server Version: %s\n", serverVersion[1])
osc.LogInfo(fmt.Sprintf("\t Using mod_status Server Version: %s\n", serverVersion[1]))
report.ServerVersion = serverVersion[1]

// Check for co-hosted onion services.
log.Printf("Scanning for Co-Hosted Onions\n")
osc.LogInfo("Scanning for Co-Hosted Onions\n")
r = regexp.MustCompile(`[a-z0-9]+.onion(:[0-9]{0-5})?`)
foundServices := r.FindAllString(string(contents), -1)
utils.RemoveDuplicates(&foundServices)
for _, onion := range foundServices {
if onion != report.HiddenService {
log.Printf("\t \033[091mAlert!\033[0m Found Co-Hosted Onions: %s\n", onion)
osc.LogInfo(fmt.Sprintf("\t \033[091mAlert!\033[0m Found Co-Hosted Onions: %s\n", onion))
report.AddRelatedOnionService(onion)
}
}

// Check for co-hosted onion services.
log.Printf("Scanning for Co-Hosted Clearnet Domains\n")
osc.LogInfo("Scanning for Co-Hosted Clearnet Domains\n")
r = regexp.MustCompile(`>(([a-zA-Z]{1})|([a-zA-Z]{1}[a-zA-Z]{1})|([a-zA-Z]{1}[0-9]{1})|([0-9]{1}[a-zA-Z]{1})|([a-zA-Z0-9][a-zA-Z0-9-_]{1,61}[a-zA-Z0-9]))\.([a-zA-Z]{2,6}|[a-zA-Z0-9-]{2,30}\.[a-zA-Z]{2,3})`)
foundServices = r.FindAllString(string(contents), -1)
utils.RemoveDuplicates(&foundServices)
for _, domain := range foundServices {
if strings.Contains(domain, ".onion") == false {
log.Printf("\t \033[091mAlert!\033[0m Found Co-Hosted Service: %s\n", domain[1:])
osc.LogInfo(fmt.Sprintf("\t \033[091mAlert!\033[0m Found Co-Hosted Service: %s\n", domain[1:]))
report.AddRelatedClearnetDomain(domain[4:])
}
}

// Check for IP Addresses
log.Printf("Scanning for IP Addresses (clearweb clients, and servers)\n")
osc.LogInfo("Scanning for IP Addresses (clearweb clients, and servers)\n")
r = regexp.MustCompile(`(?:(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.){3}(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)`)
foundIPs := r.FindAllString(string(contents), -1)
utils.RemoveDuplicates(&foundIPs)
for _, ip := range foundIPs {
if ip != "127.0.0.1" {
log.Printf("\t \033[091mAlert!\033[0m Found IP Address : %s\n", ip)
osc.LogInfo(fmt.Sprintf("\t \033[091mAlert!\033[0m Found IP Address : %s\n", ip))
report.AddIPAddress(ip)
}
}
@@ -60,7 +61,7 @@ func ApacheModStatus(scan Scanner, page string, status int, contents string, rep
}
}
if !report.FoundApacheModStatus {
log.Printf("\tApache mod_status Not Exposed...\033[92mGood!\033[0m\n")
osc.LogInfo("\tApache mod_status Not Exposed...\033[92mGood!\033[0m\n")
report.FoundApacheModStatus = false
}
}

+ 13
- 12
scans/check_directory_listing.go View File

@@ -1,35 +1,36 @@
package scans

import (
"fmt"
"github.com/s-rah/onionscan/config"
"github.com/s-rah/onionscan/report"
"log"
"regexp"
"strings"
)

func CheckDirectoryListing(depth int) func(Scanner, string, int, string, *report.OnionScanReport) {
return func(scan Scanner, dir string, status int, contents string, report *report.OnionScanReport) {
CheckDirectoryListingDepth(scan, dir, status, depth, contents, report)
func CheckDirectoryListing(depth int) func(Scanner, string, int, string, *report.OnionScanReport, *config.OnionscanConfig) {
return func(scan Scanner, dir string, status int, contents string, report *report.OnionScanReport, osc *config.OnionscanConfig) {
CheckDirectoryListingDepth(scan, dir, status, depth, contents, report, osc)
}
}

func CheckDirectoryListingDepth(scan Scanner, dir string, status int, depth int, contents string, report *report.OnionScanReport) {
func CheckDirectoryListingDepth(scan Scanner, dir string, status int, depth int, contents string, report *report.OnionScanReport, osc *config.OnionscanConfig) {
if status == 200 && strings.Contains(string(contents), "Index of "+dir) {
log.Printf("Detected Open Directory %s...\033[091mAlert!\033[0m\n", dir)
osc.LogInfo(fmt.Sprintf("Detected Open Directory %s...\033[091mAlert!\033[0m\n", dir))

report.AddOpenDirectory(dir)

r := regexp.MustCompile(`href="((.*?\.jpg)|(.*?\.png)|(.*?\.jpeg)|(.*?\.gif))"`)
foundImages := r.FindAllStringSubmatch(string(contents), -1)
for _, image := range foundImages {
log.Printf("\t Found image %s/%s\n", dir, image[1])
scan.ScanPage(report.HiddenService, dir+"/"+image[1], report, CheckExif)
osc.LogInfo(fmt.Sprintf("\t Found image %s/%s\n", dir, image[1]))
scan.ScanPage(report.HiddenService, dir+"/"+image[1], report, osc, CheckExif)
}

r = regexp.MustCompile(`href="((.*\.zip)|(.*\.tar)|(.*\.gz)|(.*\.pst)|(.*\.txt))"`)
interestingFiles := r.FindAllStringSubmatch(string(contents), -1)
for _, file := range interestingFiles {
log.Printf("\t Found interesting file %s/%s\n", dir, file[1])
osc.LogInfo(fmt.Sprintf("\t Found interesting file %s/%s\n", dir, file[1]))
//TODO: We can do further analysis here, for now, just report them.
report.AddInterestingFile(dir + "/" + file[1])
}
@@ -37,14 +38,14 @@ func CheckDirectoryListingDepth(scan Scanner, dir string, status int, depth int,
r = regexp.MustCompile(`href="([^/](.*?))/"`)
subDir := r.FindAllStringSubmatch(string(contents), -1)
for _, file := range subDir {
log.Printf("\t Found subdir %s/%s\n", dir, file[1])
osc.LogInfo(fmt.Sprintf("\t Found subdir %s/%s\n", dir, file[1]))
//TODO: We can do further analysis here, for now, just report them.
if depth > 0 {
scan.ScanPage(report.HiddenService, dir+"/"+file[1], report, CheckDirectoryListing(depth-1))
scan.ScanPage(report.HiddenService, dir+"/"+file[1], report, osc, CheckDirectoryListing(depth-1))
}
}

} else {
log.Printf("Directory %s either doesn't exist or is not readable\n", dir)
osc.LogInfo(fmt.Sprintf("Directory %s either doesn't exist or is not readable\n", dir))
}
}

+ 4
- 3
scans/check_exif.go View File

@@ -1,14 +1,15 @@
package scans

import (
"fmt"
"github.com/s-rah/onionscan/config"
"github.com/s-rah/onionscan/report"
"github.com/xiam/exif"
"io"
"log"
"strings"
)

func CheckExif(scan Scanner, page string, status int, contents string, report *report.OnionScanReport) {
func CheckExif(scan Scanner, page string, status int, contents string, report *report.OnionScanReport, osc *config.OnionscanConfig) {
if status == 200 {
reader := exif.New()
_, err := io.Copy(reader, strings.NewReader(contents))
@@ -31,7 +32,7 @@ func CheckExif(scan Scanner, page string, status int, contents string, report *r
report.AddExifImage(page)

for name, val := range reader.Tags {
log.Printf("\t \033[091mAlert!\033[0m Found Exif Tag%s: %s\n", name, val)
osc.LogInfo(fmt.Sprintf("\t \033[091mAlert!\033[0m Found Exif Tag%s: %s\n", name, val))
report.AddExifTag(name, val)
}
}


+ 4
- 5
scans/pgp_content_scan.go View File

@@ -3,7 +3,6 @@ package scans
import (
"github.com/s-rah/onionscan/report"
"golang.org/x/crypto/openpgp"
"log"
"regexp"
"strings"
)
@@ -12,17 +11,17 @@ type PGPContentScan struct {
}

func (cs *PGPContentScan) ScanContent(content string, report *report.OnionScanReport) {
log.Printf("Scanning for PGP Key\n")
//log.Printf("Scanning for PGP Key\n")
pgpRegexp := regexp.MustCompile("-----BEGIN PGP PUBLIC KEY BLOCK-----((?s).*)-----END PGP PUBLIC KEY BLOCK-----")
foundPGP := pgpRegexp.FindAllString(content, -1)
for _, keyString := range foundPGP {
keys, err := openpgp.ReadArmoredKeyRing(strings.NewReader(keyString))
if err != nil {
log.Printf("ERROR: %s\n", err)
// log.Printf("ERROR: %s\n", err)
continue
}
if len(keys) < 1 || len(keys[0].Subkeys) < 1 || len(keys[0].Identities) < 1 {
log.Printf("ERROR: failed to accept key\n")
// log.Printf("ERROR: failed to accept key\n")
continue
}

@@ -30,7 +29,7 @@ func (cs *PGPContentScan) ScanContent(content string, report *report.OnionScanRe
for identity = range keys[0].Identities {
break
}
log.Printf("\tFound PGP Key fingerprint: %s belonging to %s", keys[0].Subkeys[0].PublicKey.KeyIdShortString(), identity)
// log.Printf("\tFound PGP Key fingerprint: %s belonging to %s", keys[0].Subkeys[0].PublicKey.KeyIdShortString(), identity)

report.AddPGPKey(keyString, identity, keys[0].Subkeys[0].PublicKey.KeyIdShortString())
}


+ 15
- 0
scans/private_key.go View File

@@ -0,0 +1,15 @@
package scans

import (
"fmt"
"github.com/s-rah/onionscan/config"
"github.com/s-rah/onionscan/report"
)

func PrivateKeyScan(scan Scanner, page string, status int, contents string, report *report.OnionScanReport, osc *config.OnionscanConfig) {
osc.LogInfo(fmt.Sprintf("Scanning %s\n", page))
if status == 200 {
osc.LogInfo(fmt.Sprintf("\tPrivate Key %s is Accessible!!\n", page))
report.PrivateKeyDetected = true
}
}

+ 2
- 1
scans/scanner.go View File

@@ -1,10 +1,11 @@
package scans

import (
"github.com/s-rah/onionscan/config"
"github.com/s-rah/onionscan/report"
)

type Scanner interface {
ScanPage(string, string, *report.OnionScanReport, func(Scanner, string, int, string, *report.OnionScanReport))
ScanPage(string, string, *report.OnionScanReport, *config.OnionscanConfig, func(Scanner, string, int, string, *report.OnionScanReport, *config.OnionscanConfig))
ScrapePage(string, string) (error, []byte, int)
}

+ 19
- 18
scans/standard-page-scan.go View File

@@ -3,19 +3,20 @@ package scans
import (
"crypto/sha1"
"encoding/hex"
"fmt"
"github.com/s-rah/onionscan/config"
"github.com/s-rah/onionscan/report"
"github.com/s-rah/onionscan/utils"
"golang.org/x/net/html"
"log"
"net/url"
"regexp"
"strings"
)

func StandardPageScan(scan Scanner, page string, status int, contents string, report *report.OnionScanReport) {
log.Printf("Scanning %s\n", page)
func StandardPageScan(scan Scanner, page string, status int, contents string, report *report.OnionScanReport, osc *config.OnionscanConfig) {
osc.LogInfo(fmt.Sprintf("Scanning %s\n", page))
if status == 200 {
log.Printf("\tPage %s is Accessible\n", page)
osc.LogInfo(fmt.Sprintf("\tPage %s is Accessible\n", page))

hash := sha1.Sum([]byte(contents))
report.Hashes = append(report.Hashes, hex.EncodeToString(hash[:]))
@@ -27,14 +28,14 @@ func StandardPageScan(scan Scanner, page string, status int, contents string, re
var startIndex = strings.Index(contents, "<title>")
var endIndex = strings.Index(contents, "</title>")
var pageTitle = contents[startIndex+len("<title>") : endIndex]
log.Printf("\tPage Title: %s\n", pageTitle)
osc.LogInfo(fmt.Sprintf("\tPage Title: %s\n", pageTitle))
report.PageTitle = pageTitle
}

new(PGPContentScan).ScanContent(contents, report)
//new(BitcoinContentScan).ScanContent(contents, report)

log.Printf("\tScanning for Images\n")
osc.LogInfo("\tScanning for Images\n")
var domains []string
var cssLinks []string

@@ -72,31 +73,31 @@ func StandardPageScan(scan Scanner, page string, status int, contents string, re
baseUrl, err := url.Parse(imageUrl)
if err == nil {
if utils.WithoutSubdomains(baseUrl.Host) == utils.WithoutSubdomains(report.HiddenService) {
scan.ScanPage(report.HiddenService, utils.WithoutProtocol(imageUrl), report, CheckExif)
log.Printf("\t Found internal image %s\n", imageUrl)
scan.ScanPage(report.HiddenService, utils.WithoutProtocol(imageUrl), report, osc, CheckExif)
osc.LogInfo(fmt.Sprintf("\t Found internal image %s\n", imageUrl))
} else {
log.Printf("\t Not scanning remote image %s\n", imageUrl)
osc.LogInfo(fmt.Sprintf("\t Not scanning remote image %s\n", imageUrl))
}
}
}
}

log.Printf("\tScanning for CSS Fonts and Background Images\n")
osc.LogInfo("\tScanning for CSS Fonts and Background Images\n")
utils.RemoveDuplicates(&cssLinks)
for _, cssUrl := range cssLinks {
log.Printf("\tScanning CSS file: %s\n", cssUrl)
osc.LogInfo(fmt.Sprintf("\tScanning CSS file: %s\n", cssUrl))
_, cssContents, _ := scan.ScrapePage(report.HiddenService, utils.WithoutProtocol(cssUrl))
domains = append(domains, utils.ExtractDomains(string(cssContents))[:]...)
}

log.Printf("\tScanning for Links\n")
osc.LogInfo("\tScanning for Links\n")
domains = append(domains, utils.ExtractDomains(contents)...)
utils.RemoveDuplicates(&domains)
for _, domain := range domains {
baseUrl, err := url.Parse(domain)
if err == nil {
if baseUrl.Host != "" && utils.WithoutSubdomains(baseUrl.Host) != utils.WithoutSubdomains(report.HiddenService) {
log.Printf("Found Related URL %s\n", domain)
osc.LogInfo(fmt.Sprintf("Found Related URL %s\n", domain))
// TODO: Lots of information here which needs to be processed.
// * Links to standard sites - google / bitpay etc.
// * Links to other onion sites
@@ -104,13 +105,13 @@ func StandardPageScan(scan Scanner, page string, status int, contents string, re
report.AddLinkedSite(baseUrl.Host)
} else {
// * Process FQDN internal links
log.Printf("Found Internal URL %s\n", domain)
osc.LogInfo(fmt.Sprintf("Found Internal URL %s\n", domain))
report.AddInternalPage(baseUrl.Host)
}
}
}

log.Printf("\tScanning for Referenced Directories\n")
osc.LogInfo("\tScanning for Referenced Directories\n")
r := regexp.MustCompile("(src|href)=\"([^\"]*)\"")
foundPaths := r.FindAllStringSubmatch(string(contents), -1)
for _, regexpResults := range foundPaths {
@@ -121,13 +122,13 @@ func StandardPageScan(scan Scanner, page string, status int, contents string, re

term := strings.LastIndex(path, "/")
if term > 0 {
log.Printf("\t Found Referenced Directory %s\n", path[:term])
osc.LogInfo(fmt.Sprintf("\t Found Referenced Directory %s\n", path[:term]))
report.AddPageReferencedDirectory(utils.WithoutProtocol(path[:term]))
}
}
} else if status == 403 {
log.Printf("\tPage %s%s is Forbidden\n", report.HiddenService, page)
osc.LogInfo(fmt.Sprintf("\tPage %s%s is Forbidden\n", report.HiddenService, page))
} else if status == 404 {
log.Printf("\tPage %s%s is Does Not Exist\n", report.HiddenService, page)
osc.LogInfo(fmt.Sprintf("\tPage %s%s is Does Not Exist\n", report.HiddenService, page))
}
}

Loading…
Cancel
Save