From 872555c1df2d9988b356a9d64b420941c157983a Mon Sep 17 00:00:00 2001 From: Sarah Jamie Lewis Date: Sat, 30 Jul 2016 17:00:42 -0700 Subject: [PATCH] Bug Fixes relating to Closing File Handles OnionScan was having issues with larger batch runs do to failure to close network connection once they had been used. This commit fixes this issue, as well as adds a quick exit from the main OnionScan loop due to timeout - once the timeout is crossed no further protocol scans will take place. Fixes #51 --- config/onionscan_config.go | 15 ++++- main.go | 24 ++++---- onionscan.go | 96 +++++++++++++++++++------------- protocol/bitcoin_scanner.go | 14 ++--- protocol/ftp_scanner.go | 14 ++--- protocol/http_scanner.go | 69 ++++++++++++----------- protocol/irc_scanner.go | 22 ++++---- protocol/mongodb_scanner.go | 13 +++-- protocol/ricochet_scanner.go | 14 ++--- protocol/smtp_scanner.go | 14 ++--- protocol/ssh_scanner.go | 26 ++++++--- protocol/vnc_scanner.go | 14 ++--- protocol/xmpp_scanner.go | 23 ++++---- report/onionscanreport.go | 38 ++++++++----- report/report_generator.go | 21 ++++--- scans/apache_mod_status.go | 23 ++++---- scans/check_directory_listing.go | 25 +++++---- scans/check_exif.go | 7 ++- scans/pgp_content_scan.go | 9 ++- scans/private_key.go | 15 +++++ scans/scanner.go | 3 +- scans/standard-page-scan.go | 37 ++++++------ 22 files changed, 311 insertions(+), 225 deletions(-) create mode 100644 scans/private_key.go diff --git a/config/onionscan_config.go b/config/onionscan_config.go index b789781..32cf166 100644 --- a/config/onionscan_config.go +++ b/config/onionscan_config.go @@ -1,6 +1,7 @@ package config import ( + "log" "time" ) @@ -9,13 +10,25 @@ type OnionscanConfig struct { DirectoryDepth int Fingerprint bool Timeout time.Duration + Verbose bool } -func Configure(torProxyAddress string, directoryDepth int, fingerprint bool, timeout int) *OnionscanConfig { +func Configure(torProxyAddress string, directoryDepth int, fingerprint bool, timeout int, verbose bool) *OnionscanConfig { onionScan := new(OnionscanConfig) onionScan.TorProxyAddress = torProxyAddress onionScan.DirectoryDepth = directoryDepth onionScan.Fingerprint = fingerprint onionScan.Timeout = time.Duration(time.Second * time.Duration(timeout)) + onionScan.Verbose = verbose return onionScan } + +func (os *OnionscanConfig) LogInfo(message string) { + if os.Verbose { + log.Printf("INFO: %v", message) + } +} + +func (os *OnionscanConfig) LogError(err error) { + log.Printf("ERROR: %v", err) +} diff --git a/main.go b/main.go index 2201923..1d39d1e 100644 --- a/main.go +++ b/main.go @@ -1,6 +1,7 @@ package main import ( + "errors" "flag" "fmt" "github.com/s-rah/onionscan/config" @@ -28,6 +29,7 @@ func main() { fingerprint := flag.Bool("fingerprint", true, "true disables some deeper scans e.g. directory probing with the aim of just getting a fingerprint of the service.") list := flag.String("list", "", "If provided OnionScan will attempt to read from the given list, rather than the provided hidden service") timeout := flag.Int("timeout", 120, "read timeout for connecting to onion services") + batch := flag.Int("batch", 10, "number of onions to scan concurrently") flag.Parse() @@ -58,22 +60,17 @@ func main() { log.Printf("This might take a few minutes..\n\n") onionScan := new(OnionScan) - onionScan.Config = config.Configure(*torProxyAddress, *directoryDepth, *fingerprint, *timeout) + onionScan.Config = config.Configure(*torProxyAddress, *directoryDepth, *fingerprint, *timeout, *verbose) reports := make(chan *report.OnionScanReport) - if !*verbose { - log.SetOutput(ioutil.Discard) - } - count := 0 - max := 100 - if max > len(onionsToScan) { - max = len(onionsToScan) + if *batch > len(onionsToScan) { + *batch = len(onionsToScan) } // Run an initial batch of 100 requests (or less...) - for count < max { + for count < *batch { go onionScan.Scan(onionsToScan[count], reports) count++ } @@ -89,11 +86,16 @@ func main() { } received++ + if scanReport.TimedOut { + onionScan.Config.LogError(errors.New(scanReport.HiddenService + " timed out")) + } + + file := scanReport.HiddenService + "." + *reportFile if *jsonReport { - report.GenerateJsonReport(*reportFile, scanReport) + report.GenerateJsonReport(file, scanReport) } else if *simpleReport { - report.GenerateSimpleReport(*reportFile, scanReport) + report.GenerateSimpleReport(file, scanReport) } } } diff --git a/onionscan.go b/onionscan.go index 4ca8665..8cdf3d5 100644 --- a/onionscan.go +++ b/onionscan.go @@ -6,12 +6,62 @@ import ( "github.com/s-rah/onionscan/report" "github.com/s-rah/onionscan/utils" "strings" + "time" ) type OnionScan struct { Config *config.OnionscanConfig } +func (os *OnionScan) PerformNextAction(report *report.OnionScanReport) { + switch report.NextAction { + case "web": + wps := new(protocol.HTTPProtocolScanner) + wps.ScanProtocol(report.HiddenService, os.Config, report) + report.NextAction = "ssh" + case "ssh": + sps := new(protocol.SSHProtocolScanner) + sps.ScanProtocol(report.HiddenService, os.Config, report) + report.NextAction = "irc" + case "irc": + ips := new(protocol.IRCProtocolScanner) + ips.ScanProtocol(report.HiddenService, os.Config, report) + report.NextAction = "ricochet" + case "ricochet": + rps := new(protocol.RicochetProtocolScanner) + rps.ScanProtocol(report.HiddenService, os.Config, report) + report.NextAction = "ftp" + case "ftp": + fps := new(protocol.FTPProtocolScanner) + fps.ScanProtocol(report.HiddenService, os.Config, report) + report.NextAction = "smtp" + case "smtp": + smps := new(protocol.SMTPProtocolScanner) + smps.ScanProtocol(report.HiddenService, os.Config, report) + report.NextAction = "mongodb" + case "mongodb": + mdbps := new(protocol.MongoDBProtocolScanner) + mdbps.ScanProtocol(report.HiddenService, os.Config, report) + report.NextAction = "vnc" + case "vnc": + vncps := new(protocol.VNCProtocolScanner) + vncps.ScanProtocol(report.HiddenService, os.Config, report) + report.NextAction = "xmpp" + case "xmpp": + xmppps := new(protocol.XMPPProtocolScanner) + xmppps.ScanProtocol(report.HiddenService, os.Config, report) + report.NextAction = "bitcoin" + case "bitcoin": + bps := new(protocol.BitcoinProtocolScanner) + bps.ScanProtocol(report.HiddenService, os.Config, report) + report.NextAction = "none" + case "none": + return + default: + report.NextAction = "web" + } +} + func (os *OnionScan) Scan(hiddenService string, out chan *report.OnionScanReport) { // Remove Extra Prefix @@ -23,45 +73,13 @@ func (os *OnionScan) Scan(hiddenService string, out chan *report.OnionScanReport report := report.NewOnionScanReport(hiddenService) - // HTTP - hps := new(protocol.HTTPProtocolScanner) - hps.ScanProtocol(hiddenService, os.Config, report) - - // SSH - sps := new(protocol.SSHProtocolScanner) - sps.ScanProtocol(hiddenService, os.Config, report) - - // Ricochet - rps := new(protocol.RicochetProtocolScanner) - rps.ScanProtocol(hiddenService, os.Config, report) - - // Bitcoin - bps := new(protocol.BitcoinProtocolScanner) - bps.ScanProtocol(hiddenService, os.Config, report) - - //IRC - ips := new(protocol.IRCProtocolScanner) - ips.ScanProtocol(hiddenService, os.Config, report) - - //FTP - fps := new(protocol.FTPProtocolScanner) - fps.ScanProtocol(hiddenService, os.Config, report) - - //SMTP - smps := new(protocol.SMTPProtocolScanner) - smps.ScanProtocol(hiddenService, os.Config, report) - - //MongoDb - mdbps := new(protocol.MongoDBProtocolScanner) - mdbps.ScanProtocol(hiddenService, os.Config, report) - - //VNC - vncps := new(protocol.VNCProtocolScanner) - vncps.ScanProtocol(hiddenService, os.Config, report) - - //XMPP - xmppps := new(protocol.XMPPProtocolScanner) - xmppps.ScanProtocol(hiddenService, os.Config, report) + for report.NextAction != "none" { + os.PerformNextAction(report) + if time.Now().Sub(report.DateScanned).Seconds() > os.Config.Timeout.Seconds() { + report.TimedOut = true + report.NextAction = "none" + } + } out <- report } diff --git a/protocol/bitcoin_scanner.go b/protocol/bitcoin_scanner.go index 321a0a1..cbfa53e 100644 --- a/protocol/bitcoin_scanner.go +++ b/protocol/bitcoin_scanner.go @@ -1,26 +1,26 @@ package protocol import ( + "fmt" "github.com/s-rah/onionscan/config" "github.com/s-rah/onionscan/report" "github.com/s-rah/onionscan/utils" - "log" ) type BitcoinProtocolScanner struct { } -func (rps *BitcoinProtocolScanner) ScanProtocol(hiddenService string, onionscanConfig *config.OnionscanConfig, report *report.OnionScanReport) { +func (rps *BitcoinProtocolScanner) ScanProtocol(hiddenService string, osc *config.OnionscanConfig, report *report.OnionScanReport) { // Bitcoin - log.Printf("Checking %s Bitcoin(8333)\n", hiddenService) - _, err := utils.GetNetworkConnection(hiddenService, 8333, onionscanConfig.TorProxyAddress, onionscanConfig.Timeout) + osc.LogInfo(fmt.Sprintf("Checking %s Bitcoin(8333)\n", hiddenService)) + conn, err := utils.GetNetworkConnection(hiddenService, 8333, osc.TorProxyAddress, osc.Timeout) if err != nil { - log.Printf("Failed to connect to service on port 8333\n") + osc.LogInfo("Failed to connect to service on port 8333\n") report.BitcoinDetected = false } else { - log.Printf("Detected possible Bitcoin instance\n") + osc.LogInfo("Detected possible Bitcoin instance\n") // TODO: Actual Analysis report.BitcoinDetected = true } - + conn.Close() } diff --git a/protocol/ftp_scanner.go b/protocol/ftp_scanner.go index c454041..813294b 100644 --- a/protocol/ftp_scanner.go +++ b/protocol/ftp_scanner.go @@ -4,21 +4,21 @@ import ( "bufio" "crypto/sha1" "encoding/hex" + "fmt" "github.com/s-rah/onionscan/config" "github.com/s-rah/onionscan/report" "github.com/s-rah/onionscan/utils" - "log" ) type FTPProtocolScanner struct { } -func (sps *FTPProtocolScanner) ScanProtocol(hiddenService string, onionscanConfig *config.OnionscanConfig, report *report.OnionScanReport) { +func (sps *FTPProtocolScanner) ScanProtocol(hiddenService string, osc *config.OnionscanConfig, report *report.OnionScanReport) { // FTP - log.Printf("Checking %s FTP(21)\n", hiddenService) - conn, err := utils.GetNetworkConnection(hiddenService, 21, onionscanConfig.TorProxyAddress, onionscanConfig.Timeout) + osc.LogInfo(fmt.Sprintf("Checking %s FTP(21)\n", hiddenService)) + conn, err := utils.GetNetworkConnection(hiddenService, 21, osc.TorProxyAddress, osc.Timeout) if err != nil { - log.Printf("Failed to connect to service on port 21\n") + osc.LogInfo("Failed to connect to service on port 21\n") report.FTPDetected = false } else { // TODO FTP Checking @@ -29,8 +29,8 @@ func (sps *FTPProtocolScanner) ScanProtocol(hiddenService string, onionscanConfi report.FTPBanner = banner hash := sha1.Sum([]byte(banner)) report.FTPFingerprint = hex.EncodeToString(hash[:]) - log.Printf("Found FTP Banner: %s (%s)", banner, report.FTPFingerprint) + osc.LogInfo(fmt.Sprintf("Found FTP Banner: %s (%s)", banner, report.FTPFingerprint)) } } - + conn.Close() } diff --git a/protocol/http_scanner.go b/protocol/http_scanner.go index 654e57d..03858d9 100644 --- a/protocol/http_scanner.go +++ b/protocol/http_scanner.go @@ -1,13 +1,13 @@ package protocol import ( + "fmt" "github.com/s-rah/onionscan/config" "github.com/s-rah/onionscan/report" "github.com/s-rah/onionscan/scans" "github.com/s-rah/onionscan/utils" "h12.me/socks" "io/ioutil" - "log" "net/http" "strings" ) @@ -24,63 +24,67 @@ var ( "/products", "/products/cat"} ) -func (hps *HTTPProtocolScanner) ScanProtocol(hiddenService string, onionscanConfig *config.OnionscanConfig, report *report.OnionScanReport) { +func (hps *HTTPProtocolScanner) ScanProtocol(hiddenService string, osc *config.OnionscanConfig, report *report.OnionScanReport) { // HTTP - log.Printf("Checking %s http(80)\n", hiddenService) - _, err := utils.GetNetworkConnection(hiddenService, 80, onionscanConfig.TorProxyAddress, onionscanConfig.Timeout) + osc.LogInfo(fmt.Sprintf("Checking %s http(80)\n", hiddenService)) + conn, err := utils.GetNetworkConnection(hiddenService, 80, osc.TorProxyAddress, osc.Timeout) if err != nil { - log.Printf("Failed to connect to service on port 80\n") + osc.LogInfo("Failed to connect to service on port 80\n") report.WebDetected = false - return + conn.Close() } else { - log.Printf("Found potential service on http(80)\n") + osc.LogInfo("Found potential service on http(80)\n") report.WebDetected = true - dialSocksProxy := socks.DialSocksProxy(socks.SOCKS5, onionscanConfig.TorProxyAddress) + conn.Close() + dialSocksProxy := socks.DialSocksProxy(socks.SOCKS5, osc.TorProxyAddress) transportConfig := &http.Transport{ Dial: dialSocksProxy, } hps.Client = &http.Client{Transport: transportConfig} // FIXME This should probably be moved to it's own file now. response, err := hps.Client.Get("http://" + hiddenService) + if err == nil { + // Reading all http headers + osc.LogInfo(fmt.Sprintf("HTTP response headers: %s\n", report.ServerVersion)) + responseHeaders := response.Header + for key := range responseHeaders { + value := responseHeaders.Get(key) + // normalize by strings.ToUpper(key) to avoid case sensitive checking + report.AddResponseHeader(strings.ToUpper(key), value) + } - if err != nil { - log.Printf("Failed to connect to service on port 80\n") - return + report.ServerVersion = responseHeaders.Get("Server") + response.Body.Close() + } else { + osc.LogError(err) } - // Reading all http headers - log.Printf("HTTP response headers: %s\n", report.ServerVersion) - responseHeaders := response.Header - for key := range responseHeaders { - value := responseHeaders.Get(key) - // normalize by strings.ToUpper(key) to avoid case sensitive checking - report.AddResponseHeader(strings.ToUpper(key), value) - log.Printf("\t%s : %s\n", strings.ToUpper(key), value) - } - - report.ServerVersion = responseHeaders.Get("Server") - // Apache mod-status Check - hps.ScanPage(hiddenService, "/server-status", report, scans.ApacheModStatus) - hps.ScanPage(hiddenService, "/", report, scans.StandardPageScan) + hps.ScanPage(hiddenService, "/", report, osc, scans.StandardPageScan) + hps.ScanPage(hiddenService, "/server-status", report, osc, scans.ApacheModStatus) + hps.ScanPage(hiddenService, "/private_key", report, osc, scans.PrivateKeyScan) - if onionscanConfig.Fingerprint == false { - log.Printf("\tScanning Common and Referenced Directories\n") + if osc.Fingerprint == false { + osc.LogInfo("\tScanning Common and Referenced Directories\n") directories := append(CommonDirectories, report.PageReferencedDirectories...) utils.RemoveDuplicates(&directories) for _, directory := range directories { - hps.ScanPage(hiddenService, directory, report, scans.CheckDirectoryListing(onionscanConfig.DirectoryDepth)) + hps.ScanPage(hiddenService, directory, report, osc, scans.CheckDirectoryListing(osc.DirectoryDepth)) } } } - log.Printf("\n") } -func (hps *HTTPProtocolScanner) ScanPage(hiddenService string, page string, report *report.OnionScanReport, f func(scans.Scanner, string, int, string, *report.OnionScanReport)) { - _, contents, responseCode := hps.ScrapePage(hiddenService, page) - f(hps, page, responseCode, string(contents), report) +func (hps *HTTPProtocolScanner) ScanPage(hiddenService string, page string, report *report.OnionScanReport, osc *config.OnionscanConfig, f func(scans.Scanner, string, int, string, *report.OnionScanReport, *config.OnionscanConfig)) { + err, contents, responseCode := hps.ScrapePage(hiddenService, page) + if err == nil { + f(hps, page, responseCode, string(contents), report, osc) + return + } else { + osc.LogError(err) + } } func (hps *HTTPProtocolScanner) ScrapePage(hiddenService string, page string) (error, []byte, int) { @@ -92,7 +96,6 @@ func (hps *HTTPProtocolScanner) ScrapePage(hiddenService string, page string) (e } response, err := hps.Client.Get("http://" + page) if err != nil { - log.Printf("Error connecting to http://%s %s\n", page, err) return err, nil, -1 } defer response.Body.Close() diff --git a/protocol/irc_scanner.go b/protocol/irc_scanner.go index 01369db..1a1dee3 100644 --- a/protocol/irc_scanner.go +++ b/protocol/irc_scanner.go @@ -1,36 +1,38 @@ package protocol import ( + "fmt" "github.com/s-rah/onionscan/config" "github.com/s-rah/onionscan/report" "github.com/s-rah/onionscan/utils" - "log" ) type IRCProtocolScanner struct { } -func (rps *IRCProtocolScanner) ScanProtocol(hiddenService string, onionscanConfig *config.OnionscanConfig, report *report.OnionScanReport) { +func (rps *IRCProtocolScanner) ScanProtocol(hiddenService string, osc *config.OnionscanConfig, report *report.OnionScanReport) { // IRC - log.Printf("Checking %s IRC(6667)\n", hiddenService) - _, err := utils.GetNetworkConnection(hiddenService, 6667, onionscanConfig.TorProxyAddress, onionscanConfig.Timeout) + osc.LogInfo(fmt.Sprintf("Checking %s IRC(6667)\n", hiddenService)) + conn, err := utils.GetNetworkConnection(hiddenService, 6667, osc.TorProxyAddress, osc.Timeout) if err != nil { - log.Printf("Failed to connect to service on port 6667\n") + osc.LogInfo("Failed to connect to service on port 6667\n") report.IRCDetected = false } else { - log.Printf("Detected possible IRC instance\n") + osc.LogInfo("Detected possible IRC instance\n") // TODO: Actual Analysis report.IRCDetected = true } + conn.Close() // IRC - log.Printf("Checking %s IRC(6697)\n", hiddenService) - _, err = utils.GetNetworkConnection(hiddenService, 6697, onionscanConfig.TorProxyAddress, onionscanConfig.Timeout) + osc.LogInfo(fmt.Sprintf("Checking %s IRC(6697)\n", hiddenService)) + conn, err = utils.GetNetworkConnection(hiddenService, 6697, osc.TorProxyAddress, osc.Timeout) if err != nil { - log.Printf("Failed to connect to service on port 6697\n") + osc.LogInfo("Failed to connect to service on port 6697\n") } else { - log.Printf("Detected possible IRC (secure) instance\n") + osc.LogInfo("Detected possible IRC (secure) instance\n") // TODO: Actual Analysis report.IRCDetected = true } + conn.Close() } diff --git a/protocol/mongodb_scanner.go b/protocol/mongodb_scanner.go index 429aa53..f04b857 100644 --- a/protocol/mongodb_scanner.go +++ b/protocol/mongodb_scanner.go @@ -1,26 +1,27 @@ package protocol import ( + "fmt" "github.com/s-rah/onionscan/config" "github.com/s-rah/onionscan/report" "github.com/s-rah/onionscan/utils" - "log" ) type MongoDBProtocolScanner struct { } -func (rps *MongoDBProtocolScanner) ScanProtocol(hiddenService string, onionscanConfig *config.OnionscanConfig, report *report.OnionScanReport) { +func (rps *MongoDBProtocolScanner) ScanProtocol(hiddenService string, osc *config.OnionscanConfig, report *report.OnionScanReport) { // MongoDB - log.Printf("Checking %s MongoDB(27017)\n", hiddenService) - _, err := utils.GetNetworkConnection(hiddenService, 27017, onionscanConfig.TorProxyAddress, onionscanConfig.Timeout) + osc.LogInfo(fmt.Sprintf("Checking %s MongoDB(27017)\n", hiddenService)) + conn, err := utils.GetNetworkConnection(hiddenService, 27017, osc.TorProxyAddress, osc.Timeout) if err != nil { - log.Printf("Failed to connect to service on port 27017\n") + osc.LogInfo("Failed to connect to service on port 27017\n") report.MongoDBDetected = false } else { - log.Printf("Detected possible MongoDB instance\n") + osc.LogInfo("Detected possible MongoDB instance\n") // TODO: Actual Analysis report.MongoDBDetected = true } + conn.Close() } diff --git a/protocol/ricochet_scanner.go b/protocol/ricochet_scanner.go index 06820de..0274d85 100644 --- a/protocol/ricochet_scanner.go +++ b/protocol/ricochet_scanner.go @@ -1,26 +1,26 @@ package protocol import ( + "fmt" "github.com/s-rah/onionscan/config" "github.com/s-rah/onionscan/report" "github.com/s-rah/onionscan/utils" - "log" ) type RicochetProtocolScanner struct { } -func (rps *RicochetProtocolScanner) ScanProtocol(hiddenService string, onionscanConfig *config.OnionscanConfig, report *report.OnionScanReport) { +func (rps *RicochetProtocolScanner) ScanProtocol(hiddenService string, osc *config.OnionscanConfig, report *report.OnionScanReport) { // Ricochet - log.Printf("Checking %s ricochet(9878)\n", hiddenService) - _, err := utils.GetNetworkConnection(hiddenService, 9878, onionscanConfig.TorProxyAddress, onionscanConfig.Timeout) + osc.LogInfo(fmt.Sprintf("Checking %s ricochet(9878)\n", hiddenService)) + conn, err := utils.GetNetworkConnection(hiddenService, 9878, osc.TorProxyAddress, osc.Timeout) if err != nil { - log.Printf("Failed to connect to service on port 9878\n") + osc.LogInfo("Failed to connect to service on port 9878\n") report.RicochetDetected = false } else { - log.Printf("Detected possible ricochet instance\n") + osc.LogInfo("Detected possible ricochet instance\n") // TODO: Actual Analysis report.RicochetDetected = true } - + conn.Close() } diff --git a/protocol/smtp_scanner.go b/protocol/smtp_scanner.go index d6b44fc..45d1e5d 100644 --- a/protocol/smtp_scanner.go +++ b/protocol/smtp_scanner.go @@ -4,21 +4,21 @@ import ( "bufio" "crypto/sha1" "encoding/hex" + "fmt" "github.com/s-rah/onionscan/config" "github.com/s-rah/onionscan/report" "github.com/s-rah/onionscan/utils" - "log" ) type SMTPProtocolScanner struct { } -func (sps *SMTPProtocolScanner) ScanProtocol(hiddenService string, onionscanConfig *config.OnionscanConfig, report *report.OnionScanReport) { +func (sps *SMTPProtocolScanner) ScanProtocol(hiddenService string, osc *config.OnionscanConfig, report *report.OnionScanReport) { // SMTP - log.Printf("Checking %s SMTP(25)\n", hiddenService) - conn, err := utils.GetNetworkConnection(hiddenService, 25, onionscanConfig.TorProxyAddress, onionscanConfig.Timeout) + osc.LogInfo(fmt.Sprintf("Checking %s SMTP(25)\n", hiddenService)) + conn, err := utils.GetNetworkConnection(hiddenService, 25, osc.TorProxyAddress, osc.Timeout) if err != nil { - log.Printf("Failed to connect to service on port 25\n") + osc.LogInfo("Failed to connect to service on port 25\n") report.SMTPDetected = false } else { // TODO SMTP Checking @@ -29,8 +29,8 @@ func (sps *SMTPProtocolScanner) ScanProtocol(hiddenService string, onionscanConf report.SMTPBanner = banner hash := sha1.Sum([]byte(banner)) report.SMTPFingerprint = hex.EncodeToString(hash[:]) - log.Printf("Found SMTP Banner: %s (%s)", banner, report.SMTPFingerprint) + osc.LogInfo(fmt.Sprintf("Found SMTP Banner: %s (%s)", banner, report.SMTPFingerprint)) } } - + conn.Close() } diff --git a/protocol/ssh_scanner.go b/protocol/ssh_scanner.go index d788193..3e0ae79 100644 --- a/protocol/ssh_scanner.go +++ b/protocol/ssh_scanner.go @@ -1,6 +1,7 @@ package protocol import ( + "bufio" "crypto/md5" "errors" "fmt" @@ -8,20 +9,20 @@ import ( "github.com/s-rah/onionscan/report" "github.com/s-rah/onionscan/utils" "golang.org/x/crypto/ssh" - "log" "net" ) type SSHProtocolScanner struct { } -func (sps *SSHProtocolScanner) ScanProtocol(hiddenService string, onionscanConfig *config.OnionscanConfig, report *report.OnionScanReport) { +func (sps *SSHProtocolScanner) ScanProtocol(hiddenService string, osc *config.OnionscanConfig, report *report.OnionScanReport) { // SSH - log.Printf("Checking %s ssh(22)\n", hiddenService) - conn, err := utils.GetNetworkConnection(hiddenService, 22, onionscanConfig.TorProxyAddress, onionscanConfig.Timeout) + osc.LogInfo(fmt.Sprintf("Checking %s ssh(22)\n", hiddenService)) + conn, err := utils.GetNetworkConnection(hiddenService, 22, osc.TorProxyAddress, osc.Timeout) if err != nil { - log.Printf("Failed to connect to service on port 22\n") + osc.LogInfo("Failed to connect to service on port 22\n") report.SSHDetected = false + conn.Close() } else { // TODO SSH Checking report.SSHDetected = true @@ -41,13 +42,22 @@ func (sps *SSHProtocolScanner) ScanProtocol(hiddenService string, onionscanConfi } } report.SSHKey = fingerprint - log.Printf("Found SSH Key %s\n", fingerprint) + osc.LogInfo(fmt.Sprintf("Found SSH Key %s\n", fingerprint)) // We don't want to continue return errors.New("error") }, } ssh.NewClientConn(conn, hiddenService+":22", config) - + conn.Close() + conn, err = utils.GetNetworkConnection(hiddenService, 22, osc.TorProxyAddress, osc.Timeout) + if err == nil { + reader := bufio.NewReader(conn) + banner, err := reader.ReadString('\n') + if err == nil { + report.SSHBanner = banner + osc.LogInfo(fmt.Sprintf("Found SSH Banner: %s (%s)", banner)) + } + } + conn.Close() } - } diff --git a/protocol/vnc_scanner.go b/protocol/vnc_scanner.go index 71f72e0..68f8603 100644 --- a/protocol/vnc_scanner.go +++ b/protocol/vnc_scanner.go @@ -1,26 +1,26 @@ package protocol import ( + "fmt" "github.com/s-rah/onionscan/config" "github.com/s-rah/onionscan/report" "github.com/s-rah/onionscan/utils" - "log" ) type VNCProtocolScanner struct { } -func (vncps *VNCProtocolScanner) ScanProtocol(hiddenService string, onionscanConfig *config.OnionscanConfig, report *report.OnionScanReport) { +func (vncps *VNCProtocolScanner) ScanProtocol(hiddenService string, osc *config.OnionscanConfig, report *report.OnionScanReport) { // MongoDB - log.Printf("Checking %s VNC(5900)\n", hiddenService) - _, err := utils.GetNetworkConnection(hiddenService, 5900, onionscanConfig.TorProxyAddress, onionscanConfig.Timeout) + osc.LogInfo(fmt.Sprintf("Checking %s VNC(5900)\n", hiddenService)) + conn, err := utils.GetNetworkConnection(hiddenService, 5900, osc.TorProxyAddress, osc.Timeout) if err != nil { - log.Printf("Failed to connect to service on port 5900\n") + osc.LogInfo("Failed to connect to service on port 5900\n") report.VNCDetected = false } else { - log.Printf("Detected possible VNC instance\n") + osc.LogInfo("Detected possible VNC instance\n") // TODO: Actual Analysis report.VNCDetected = true } - + conn.Close() } diff --git a/protocol/xmpp_scanner.go b/protocol/xmpp_scanner.go index 5c1d2c4..14d7323 100644 --- a/protocol/xmpp_scanner.go +++ b/protocol/xmpp_scanner.go @@ -1,36 +1,37 @@ package protocol import ( + "fmt" "github.com/s-rah/onionscan/config" "github.com/s-rah/onionscan/report" "github.com/s-rah/onionscan/utils" - "log" ) type XMPPProtocolScanner struct { } -func (rps *XMPPProtocolScanner) ScanProtocol(hiddenService string, onionscanConfig *config.OnionscanConfig, report *report.OnionScanReport) { +func (rps *XMPPProtocolScanner) ScanProtocol(hiddenService string, osc *config.OnionscanConfig, report *report.OnionScanReport) { // XMPP - log.Printf("Checking %s XMPP(5222)\n", hiddenService) - _, err := utils.GetNetworkConnection(hiddenService, 5222, onionscanConfig.TorProxyAddress, onionscanConfig.Timeout) + osc.LogInfo(fmt.Sprintf("Checking %s XMPP(5222)\n", hiddenService)) + conn, err := utils.GetNetworkConnection(hiddenService, 5222, osc.TorProxyAddress, osc.Timeout) if err != nil { - log.Printf("Failed to connect to service on port 5222\n") + osc.LogInfo("Failed to connect to service on port 5222\n") report.XMPPDetected = false } else { - log.Printf("Detected possible XMPP instance\n") + osc.LogInfo("Detected possible XMPP instance\n") // TODO: Actual Analysis report.XMPPDetected = true } - + conn.Close() // XMPP - log.Printf("Checking %s XMPP(5223)\n", hiddenService) - _, err = utils.GetNetworkConnection(hiddenService, 5223, onionscanConfig.TorProxyAddress, onionscanConfig.Timeout) + osc.LogInfo(fmt.Sprintf("Checking %s XMPP(5223)\n", hiddenService)) + conn, err = utils.GetNetworkConnection(hiddenService, 5223, osc.TorProxyAddress, osc.Timeout) if err != nil { - log.Printf("Failed to connect to service on port 5223\n") + osc.LogInfo("Failed to connect to service on port 5223\n") } else { - log.Printf("Detected possible XMPP (secure) instance\n") + osc.LogInfo("Detected possible XMPP (secure) instance\n") // TODO: Actual Analysis report.XMPPDetected = true } + conn.Close() } diff --git a/report/onionscanreport.go b/report/onionscanreport.go index ad09771..20f1452 100644 --- a/report/onionscanreport.go +++ b/report/onionscanreport.go @@ -4,6 +4,7 @@ import ( "encoding/json" "github.com/s-rah/onionscan/utils" "io/ioutil" + "time" ) type ExifTag struct { @@ -23,19 +24,22 @@ type PGPKey struct { } type OnionScanReport struct { - HiddenService string `json:"hiddenService"` + HiddenService string `json:"hiddenService"` + DateScanned time.Time `json:"dateScanned"` // Summary - WebDetected bool `json:"webDetected"` - SSHDetected bool `json:"sshDetected"` - RicochetDetected bool `json:"ricochetDetected"` - IRCDetected bool `json:"ircDetected"` - FTPDetected bool `json:"ftpDetected"` - SMTPDetected bool `json:"smtpDetected"` - BitcoinDetected bool `json:"bitcoinDetected"` - MongoDBDetected bool `json:"mongodbDetected"` - VNCDetected bool `json:"vncDetected"` - XMPPDetected bool `json:"xmppDetected"` + WebDetected bool `json:"webDetected"` + SSHDetected bool `json:"sshDetected"` + RicochetDetected bool `json:"ricochetDetected"` + IRCDetected bool `json:"ircDetected"` + FTPDetected bool `json:"ftpDetected"` + SMTPDetected bool `json:"smtpDetected"` + BitcoinDetected bool `json:"bitcoinDetected"` + MongoDBDetected bool `json:"mongodbDetected"` + VNCDetected bool `json:"vncDetected"` + XMPPDetected bool `json:"xmppDetected"` + SkynetDetected bool `json:"skynetDetected"` + PrivateKeyDetected bool `json:"privateKeyDetected"` // Web Specific ServerPoweredBy string `json:"serverPoweredBy"` @@ -60,7 +64,8 @@ type OnionScanReport struct { BitcoinAddresses []string `json:"bitcoinAddresses"` // SSH - SSHKey string `json:"sshKey"` + SSHKey string `json:"sshKey"` + SSHBanner string `json:"sshBanner"` // FTP FTPFingerprint string `json:"ftpFingerprint"` @@ -69,6 +74,9 @@ type OnionScanReport struct { // SMTP SMTPFingerprint string `json:"smtpFingerprint"` SMTPBanner string `json:"smtpBanner"` + + NextAction string `json:"lastAction"` + TimedOut bool } func LoadReportFromFile(filename string) (OnionScanReport, error) { @@ -82,7 +90,11 @@ func LoadReportFromFile(filename string) (OnionScanReport, error) { } func NewOnionScanReport(hiddenService string) *OnionScanReport { - return &OnionScanReport{HiddenService: hiddenService, ResponseHeaders: make(map[string]string)} + report := new(OnionScanReport) + report.HiddenService = hiddenService + report.ResponseHeaders = make(map[string]string) + report.DateScanned = time.Now() + return report } func (osr *OnionScanReport) AddOpenDirectory(dir string) { diff --git a/report/report_generator.go b/report/report_generator.go index 91cabe9..14d1da0 100644 --- a/report/report_generator.go +++ b/report/report_generator.go @@ -5,6 +5,7 @@ import ( "fmt" "log" "os" + "time" ) func GenerateJsonReport(reportFile string, report *OnionScanReport) { @@ -14,10 +15,12 @@ func GenerateJsonReport(reportFile string, report *OnionScanReport) { buffer.WriteString(fmt.Sprintf("%s\n", jsonOut)) if len(reportFile) > 0 { - f, err := os.Create(report.HiddenService + "." + reportFile) - if err != nil { - log.Fatalf("Cannot create report file: %s", err) - panic(err) + f, err := os.Create(reportFile) + + for err != nil { + log.Printf("Cannot create report file: %s...trying again in 5 seconds...", err) + time.Sleep(time.Second * 5) + f, err = os.Create(reportFile) } defer f.Close() @@ -172,10 +175,12 @@ func GenerateSimpleReport(reportFile string, report *OnionScanReport) { } if len(reportFile) > 0 { - f, err := os.Create(report.HiddenService + "." + reportFile) - if err != nil { - log.Fatalf("Cannot create report file: %s", err) - panic(err) + f, err := os.Create(reportFile) + + for err != nil { + log.Printf("Cannot create report file: %s...trying again in 5 seconds...", err) + time.Sleep(time.Second * 5) + f, err = os.Create(reportFile) } defer f.Close() diff --git a/scans/apache_mod_status.go b/scans/apache_mod_status.go index 8ab3c97..d4c1767 100644 --- a/scans/apache_mod_status.go +++ b/scans/apache_mod_status.go @@ -1,58 +1,59 @@ package scans import ( + "fmt" + "github.com/s-rah/onionscan/config" "github.com/s-rah/onionscan/report" "github.com/s-rah/onionscan/utils" - "log" "regexp" "strings" ) -func ApacheModStatus(scan Scanner, page string, status int, contents string, report *report.OnionScanReport) { +func ApacheModStatus(scan Scanner, page string, status int, contents string, report *report.OnionScanReport, osc *config.OnionscanConfig) { if status == 200 { r := regexp.MustCompile(`Server Version: (.*)`) serverVersion := r.FindStringSubmatch(string(contents)) // Check if this looks like a mod_status page. Sometimes sites simply load their index. if len(serverVersion) > 1 { - log.Printf("Detected Apache mod_status Exposed...\033[091mAlert!\033[0m\n") + osc.LogInfo("Detected Apache mod_status Exposed...\033[091mAlert!\033[0m\n") report.FoundApacheModStatus = true - log.Printf("\t Using mod_status Server Version: %s\n", serverVersion[1]) + osc.LogInfo(fmt.Sprintf("\t Using mod_status Server Version: %s\n", serverVersion[1])) report.ServerVersion = serverVersion[1] // Check for co-hosted onion services. - log.Printf("Scanning for Co-Hosted Onions\n") + osc.LogInfo("Scanning for Co-Hosted Onions\n") r = regexp.MustCompile(`[a-z0-9]+.onion(:[0-9]{0-5})?`) foundServices := r.FindAllString(string(contents), -1) utils.RemoveDuplicates(&foundServices) for _, onion := range foundServices { if onion != report.HiddenService { - log.Printf("\t \033[091mAlert!\033[0m Found Co-Hosted Onions: %s\n", onion) + osc.LogInfo(fmt.Sprintf("\t \033[091mAlert!\033[0m Found Co-Hosted Onions: %s\n", onion)) report.AddRelatedOnionService(onion) } } // Check for co-hosted onion services. - log.Printf("Scanning for Co-Hosted Clearnet Domains\n") + osc.LogInfo("Scanning for Co-Hosted Clearnet Domains\n") r = regexp.MustCompile(`>(([a-zA-Z]{1})|([a-zA-Z]{1}[a-zA-Z]{1})|([a-zA-Z]{1}[0-9]{1})|([0-9]{1}[a-zA-Z]{1})|([a-zA-Z0-9][a-zA-Z0-9-_]{1,61}[a-zA-Z0-9]))\.([a-zA-Z]{2,6}|[a-zA-Z0-9-]{2,30}\.[a-zA-Z]{2,3})`) foundServices = r.FindAllString(string(contents), -1) utils.RemoveDuplicates(&foundServices) for _, domain := range foundServices { if strings.Contains(domain, ".onion") == false { - log.Printf("\t \033[091mAlert!\033[0m Found Co-Hosted Service: %s\n", domain[1:]) + osc.LogInfo(fmt.Sprintf("\t \033[091mAlert!\033[0m Found Co-Hosted Service: %s\n", domain[1:])) report.AddRelatedClearnetDomain(domain[4:]) } } // Check for IP Addresses - log.Printf("Scanning for IP Addresses (clearweb clients, and servers)\n") + osc.LogInfo("Scanning for IP Addresses (clearweb clients, and servers)\n") r = regexp.MustCompile(`(?:(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.){3}(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)`) foundIPs := r.FindAllString(string(contents), -1) utils.RemoveDuplicates(&foundIPs) for _, ip := range foundIPs { if ip != "127.0.0.1" { - log.Printf("\t \033[091mAlert!\033[0m Found IP Address : %s\n", ip) + osc.LogInfo(fmt.Sprintf("\t \033[091mAlert!\033[0m Found IP Address : %s\n", ip)) report.AddIPAddress(ip) } } @@ -60,7 +61,7 @@ func ApacheModStatus(scan Scanner, page string, status int, contents string, rep } } if !report.FoundApacheModStatus { - log.Printf("\tApache mod_status Not Exposed...\033[92mGood!\033[0m\n") + osc.LogInfo("\tApache mod_status Not Exposed...\033[92mGood!\033[0m\n") report.FoundApacheModStatus = false } } diff --git a/scans/check_directory_listing.go b/scans/check_directory_listing.go index 98a3d7a..2934c9a 100644 --- a/scans/check_directory_listing.go +++ b/scans/check_directory_listing.go @@ -1,35 +1,36 @@ package scans import ( + "fmt" + "github.com/s-rah/onionscan/config" "github.com/s-rah/onionscan/report" - "log" "regexp" "strings" ) -func CheckDirectoryListing(depth int) func(Scanner, string, int, string, *report.OnionScanReport) { - return func(scan Scanner, dir string, status int, contents string, report *report.OnionScanReport) { - CheckDirectoryListingDepth(scan, dir, status, depth, contents, report) +func CheckDirectoryListing(depth int) func(Scanner, string, int, string, *report.OnionScanReport, *config.OnionscanConfig) { + return func(scan Scanner, dir string, status int, contents string, report *report.OnionScanReport, osc *config.OnionscanConfig) { + CheckDirectoryListingDepth(scan, dir, status, depth, contents, report, osc) } } -func CheckDirectoryListingDepth(scan Scanner, dir string, status int, depth int, contents string, report *report.OnionScanReport) { +func CheckDirectoryListingDepth(scan Scanner, dir string, status int, depth int, contents string, report *report.OnionScanReport, osc *config.OnionscanConfig) { if status == 200 && strings.Contains(string(contents), "Index of "+dir) { - log.Printf("Detected Open Directory %s...\033[091mAlert!\033[0m\n", dir) + osc.LogInfo(fmt.Sprintf("Detected Open Directory %s...\033[091mAlert!\033[0m\n", dir)) report.AddOpenDirectory(dir) r := regexp.MustCompile(`href="((.*?\.jpg)|(.*?\.png)|(.*?\.jpeg)|(.*?\.gif))"`) foundImages := r.FindAllStringSubmatch(string(contents), -1) for _, image := range foundImages { - log.Printf("\t Found image %s/%s\n", dir, image[1]) - scan.ScanPage(report.HiddenService, dir+"/"+image[1], report, CheckExif) + osc.LogInfo(fmt.Sprintf("\t Found image %s/%s\n", dir, image[1])) + scan.ScanPage(report.HiddenService, dir+"/"+image[1], report, osc, CheckExif) } r = regexp.MustCompile(`href="((.*\.zip)|(.*\.tar)|(.*\.gz)|(.*\.pst)|(.*\.txt))"`) interestingFiles := r.FindAllStringSubmatch(string(contents), -1) for _, file := range interestingFiles { - log.Printf("\t Found interesting file %s/%s\n", dir, file[1]) + osc.LogInfo(fmt.Sprintf("\t Found interesting file %s/%s\n", dir, file[1])) //TODO: We can do further analysis here, for now, just report them. report.AddInterestingFile(dir + "/" + file[1]) } @@ -37,14 +38,14 @@ func CheckDirectoryListingDepth(scan Scanner, dir string, status int, depth int, r = regexp.MustCompile(`href="([^/](.*?))/"`) subDir := r.FindAllStringSubmatch(string(contents), -1) for _, file := range subDir { - log.Printf("\t Found subdir %s/%s\n", dir, file[1]) + osc.LogInfo(fmt.Sprintf("\t Found subdir %s/%s\n", dir, file[1])) //TODO: We can do further analysis here, for now, just report them. if depth > 0 { - scan.ScanPage(report.HiddenService, dir+"/"+file[1], report, CheckDirectoryListing(depth-1)) + scan.ScanPage(report.HiddenService, dir+"/"+file[1], report, osc, CheckDirectoryListing(depth-1)) } } } else { - log.Printf("Directory %s either doesn't exist or is not readable\n", dir) + osc.LogInfo(fmt.Sprintf("Directory %s either doesn't exist or is not readable\n", dir)) } } diff --git a/scans/check_exif.go b/scans/check_exif.go index 8a35812..a3af5cf 100644 --- a/scans/check_exif.go +++ b/scans/check_exif.go @@ -1,14 +1,15 @@ package scans import ( + "fmt" + "github.com/s-rah/onionscan/config" "github.com/s-rah/onionscan/report" "github.com/xiam/exif" "io" - "log" "strings" ) -func CheckExif(scan Scanner, page string, status int, contents string, report *report.OnionScanReport) { +func CheckExif(scan Scanner, page string, status int, contents string, report *report.OnionScanReport, osc *config.OnionscanConfig) { if status == 200 { reader := exif.New() _, err := io.Copy(reader, strings.NewReader(contents)) @@ -31,7 +32,7 @@ func CheckExif(scan Scanner, page string, status int, contents string, report *r report.AddExifImage(page) for name, val := range reader.Tags { - log.Printf("\t \033[091mAlert!\033[0m Found Exif Tag%s: %s\n", name, val) + osc.LogInfo(fmt.Sprintf("\t \033[091mAlert!\033[0m Found Exif Tag%s: %s\n", name, val)) report.AddExifTag(name, val) } } diff --git a/scans/pgp_content_scan.go b/scans/pgp_content_scan.go index 7d4ff1f..bbb1b81 100644 --- a/scans/pgp_content_scan.go +++ b/scans/pgp_content_scan.go @@ -3,7 +3,6 @@ package scans import ( "github.com/s-rah/onionscan/report" "golang.org/x/crypto/openpgp" - "log" "regexp" "strings" ) @@ -12,17 +11,17 @@ type PGPContentScan struct { } func (cs *PGPContentScan) ScanContent(content string, report *report.OnionScanReport) { - log.Printf("Scanning for PGP Key\n") + //log.Printf("Scanning for PGP Key\n") pgpRegexp := regexp.MustCompile("-----BEGIN PGP PUBLIC KEY BLOCK-----((?s).*)-----END PGP PUBLIC KEY BLOCK-----") foundPGP := pgpRegexp.FindAllString(content, -1) for _, keyString := range foundPGP { keys, err := openpgp.ReadArmoredKeyRing(strings.NewReader(keyString)) if err != nil { - log.Printf("ERROR: %s\n", err) + // log.Printf("ERROR: %s\n", err) continue } if len(keys) < 1 || len(keys[0].Subkeys) < 1 || len(keys[0].Identities) < 1 { - log.Printf("ERROR: failed to accept key\n") + // log.Printf("ERROR: failed to accept key\n") continue } @@ -30,7 +29,7 @@ func (cs *PGPContentScan) ScanContent(content string, report *report.OnionScanRe for identity = range keys[0].Identities { break } - log.Printf("\tFound PGP Key fingerprint: %s belonging to %s", keys[0].Subkeys[0].PublicKey.KeyIdShortString(), identity) + // log.Printf("\tFound PGP Key fingerprint: %s belonging to %s", keys[0].Subkeys[0].PublicKey.KeyIdShortString(), identity) report.AddPGPKey(keyString, identity, keys[0].Subkeys[0].PublicKey.KeyIdShortString()) } diff --git a/scans/private_key.go b/scans/private_key.go new file mode 100644 index 0000000..14c638b --- /dev/null +++ b/scans/private_key.go @@ -0,0 +1,15 @@ +package scans + +import ( + "fmt" + "github.com/s-rah/onionscan/config" + "github.com/s-rah/onionscan/report" +) + +func PrivateKeyScan(scan Scanner, page string, status int, contents string, report *report.OnionScanReport, osc *config.OnionscanConfig) { + osc.LogInfo(fmt.Sprintf("Scanning %s\n", page)) + if status == 200 { + osc.LogInfo(fmt.Sprintf("\tPrivate Key %s is Accessible!!\n", page)) + report.PrivateKeyDetected = true + } +} diff --git a/scans/scanner.go b/scans/scanner.go index 3c38a13..a812a08 100644 --- a/scans/scanner.go +++ b/scans/scanner.go @@ -1,10 +1,11 @@ package scans import ( + "github.com/s-rah/onionscan/config" "github.com/s-rah/onionscan/report" ) type Scanner interface { - ScanPage(string, string, *report.OnionScanReport, func(Scanner, string, int, string, *report.OnionScanReport)) + ScanPage(string, string, *report.OnionScanReport, *config.OnionscanConfig, func(Scanner, string, int, string, *report.OnionScanReport, *config.OnionscanConfig)) ScrapePage(string, string) (error, []byte, int) } diff --git a/scans/standard-page-scan.go b/scans/standard-page-scan.go index 99ee343..cfb4b98 100644 --- a/scans/standard-page-scan.go +++ b/scans/standard-page-scan.go @@ -3,19 +3,20 @@ package scans import ( "crypto/sha1" "encoding/hex" + "fmt" + "github.com/s-rah/onionscan/config" "github.com/s-rah/onionscan/report" "github.com/s-rah/onionscan/utils" "golang.org/x/net/html" - "log" "net/url" "regexp" "strings" ) -func StandardPageScan(scan Scanner, page string, status int, contents string, report *report.OnionScanReport) { - log.Printf("Scanning %s\n", page) +func StandardPageScan(scan Scanner, page string, status int, contents string, report *report.OnionScanReport, osc *config.OnionscanConfig) { + osc.LogInfo(fmt.Sprintf("Scanning %s\n", page)) if status == 200 { - log.Printf("\tPage %s is Accessible\n", page) + osc.LogInfo(fmt.Sprintf("\tPage %s is Accessible\n", page)) hash := sha1.Sum([]byte(contents)) report.Hashes = append(report.Hashes, hex.EncodeToString(hash[:])) @@ -27,14 +28,14 @@ func StandardPageScan(scan Scanner, page string, status int, contents string, re var startIndex = strings.Index(contents, "") var endIndex = strings.Index(contents, "") var pageTitle = contents[startIndex+len("") : endIndex] - log.Printf("\tPage Title: %s\n", pageTitle) + osc.LogInfo(fmt.Sprintf("\tPage Title: %s\n", pageTitle)) report.PageTitle = pageTitle } new(PGPContentScan).ScanContent(contents, report) //new(BitcoinContentScan).ScanContent(contents, report) - log.Printf("\tScanning for Images\n") + osc.LogInfo("\tScanning for Images\n") var domains []string var cssLinks []string @@ -72,31 +73,31 @@ func StandardPageScan(scan Scanner, page string, status int, contents string, re baseUrl, err := url.Parse(imageUrl) if err == nil { if utils.WithoutSubdomains(baseUrl.Host) == utils.WithoutSubdomains(report.HiddenService) { - scan.ScanPage(report.HiddenService, utils.WithoutProtocol(imageUrl), report, CheckExif) - log.Printf("\t Found internal image %s\n", imageUrl) + scan.ScanPage(report.HiddenService, utils.WithoutProtocol(imageUrl), report, osc, CheckExif) + osc.LogInfo(fmt.Sprintf("\t Found internal image %s\n", imageUrl)) } else { - log.Printf("\t Not scanning remote image %s\n", imageUrl) + osc.LogInfo(fmt.Sprintf("\t Not scanning remote image %s\n", imageUrl)) } } } } - log.Printf("\tScanning for CSS Fonts and Background Images\n") + osc.LogInfo("\tScanning for CSS Fonts and Background Images\n") utils.RemoveDuplicates(&cssLinks) for _, cssUrl := range cssLinks { - log.Printf("\tScanning CSS file: %s\n", cssUrl) + osc.LogInfo(fmt.Sprintf("\tScanning CSS file: %s\n", cssUrl)) _, cssContents, _ := scan.ScrapePage(report.HiddenService, utils.WithoutProtocol(cssUrl)) domains = append(domains, utils.ExtractDomains(string(cssContents))[:]...) } - log.Printf("\tScanning for Links\n") + osc.LogInfo("\tScanning for Links\n") domains = append(domains, utils.ExtractDomains(contents)...) utils.RemoveDuplicates(&domains) for _, domain := range domains { baseUrl, err := url.Parse(domain) if err == nil { if baseUrl.Host != "" && utils.WithoutSubdomains(baseUrl.Host) != utils.WithoutSubdomains(report.HiddenService) { - log.Printf("Found Related URL %s\n", domain) + osc.LogInfo(fmt.Sprintf("Found Related URL %s\n", domain)) // TODO: Lots of information here which needs to be processed. // * Links to standard sites - google / bitpay etc. // * Links to other onion sites @@ -104,13 +105,13 @@ func StandardPageScan(scan Scanner, page string, status int, contents string, re report.AddLinkedSite(baseUrl.Host) } else { // * Process FQDN internal links - log.Printf("Found Internal URL %s\n", domain) + osc.LogInfo(fmt.Sprintf("Found Internal URL %s\n", domain)) report.AddInternalPage(baseUrl.Host) } } } - log.Printf("\tScanning for Referenced Directories\n") + osc.LogInfo("\tScanning for Referenced Directories\n") r := regexp.MustCompile("(src|href)=\"([^\"]*)\"") foundPaths := r.FindAllStringSubmatch(string(contents), -1) for _, regexpResults := range foundPaths { @@ -121,13 +122,13 @@ func StandardPageScan(scan Scanner, page string, status int, contents string, re term := strings.LastIndex(path, "/") if term > 0 { - log.Printf("\t Found Referenced Directory %s\n", path[:term]) + osc.LogInfo(fmt.Sprintf("\t Found Referenced Directory %s\n", path[:term])) report.AddPageReferencedDirectory(utils.WithoutProtocol(path[:term])) } } } else if status == 403 { - log.Printf("\tPage %s%s is Forbidden\n", report.HiddenService, page) + osc.LogInfo(fmt.Sprintf("\tPage %s%s is Forbidden\n", report.HiddenService, page)) } else if status == 404 { - log.Printf("\tPage %s%s is Does Not Exist\n", report.HiddenService, page) + osc.LogInfo(fmt.Sprintf("\tPage %s%s is Does Not Exist\n", report.HiddenService, page)) } }