Don't Crash When Tor Proxy is Not Up

This commit adds support for scanning tls endpoints (:443)
and extracts all the certificate data to report for further analysis

README.md

@@ -58,7 +58,7 @@ This [should not be news](http://arstechnica.com/security/2016/02/default-settin
 * Determine client IP addresses if you are co-hosting a clearnet site.
 * Determine your IP address if your setup allows.
 * Determine other sites you are co-hosting.
-* Determine how active your site it.
+* Determine how active your site is.
 * Find secret or hidden areas of your site
 * and much, much more.
 

config/onionscan_config.go

@@ -1,15 +1,34 @@
 package config
 
-import ()
+import (
+	"log"
+	"time"
+)
 
 type OnionscanConfig struct {
 	TorProxyAddress string
 	DirectoryDepth  int
+	Fingerprint     bool
+	Timeout         time.Duration
+	Verbose         bool
 }
 
-func Configure(torProxyAddress string, directoryDepth int) *OnionscanConfig {
+func Configure(torProxyAddress string, directoryDepth int, fingerprint bool, timeout int, verbose bool) *OnionscanConfig {
 	onionScan := new(OnionscanConfig)
 	onionScan.TorProxyAddress = torProxyAddress
 	onionScan.DirectoryDepth = directoryDepth
+	onionScan.Fingerprint = fingerprint
+	onionScan.Timeout = time.Duration(time.Second * time.Duration(timeout))
+	onionScan.Verbose = verbose
 	return onionScan
 }
+
+func (os *OnionscanConfig) LogInfo(message string) {
+	if os.Verbose {
+		log.Printf("INFO: %v", message)
+	}
+}
+
+func (os *OnionscanConfig) LogError(err error) {
+	log.Printf("ERROR: %v", err)
+}

main.go

@@ -1,6 +1,7 @@
 package main
 
 import (
+	"errors"
 	"flag"
 	"fmt"
 	"github.com/s-rah/onionscan/config"
@@ -8,52 +9,96 @@ import (
 	"io/ioutil"
 	"log"
 	"os"
+	"strings"
 )
 
 func main() {
 
 	flag.Usage = func() {
 		fmt.Printf("Usage of %s:\n", os.Args[0])
-		fmt.Printf("    onionscan [flags] hiddenservice\n")
+		fmt.Printf("    onionscan [flags] hiddenservice | onionscan [flags] --list list\n")
 		flag.PrintDefaults()
 	}
 
 	torProxyAddress := flag.String("torProxyAddress", "127.0.0.1:9050", "the address of the tor proxy to use")
 	simpleReport := flag.Bool("simpleReport", true, "print out a simple report detailing what is wrong and how to fix it, true by default")
-	reportFile := flag.String("reportFile", "", "the file destination path for report file")
+	reportFile := flag.String("reportFile", "", "the file destination path for report file - if given, the prefix of the file will be the scanned onion service. If not given, the report will be written to stdout")
 	jsonReport := flag.Bool("jsonReport", false, "print out a json report providing a detailed report of the scan.")
 	verbose := flag.Bool("verbose", false, "print out a verbose log output of the scan")
 	directoryDepth := flag.Int("depth", 100, "depth of directory scan recursion (default: 100)")
+	fingerprint := flag.Bool("fingerprint", true, "true disables some deeper scans e.g. directory probing with the aim of just getting a fingerprint of the service.")
+	list := flag.String("list", "", "If provided OnionScan will attempt to read from the given list, rather than the provided hidden service")
+	timeout := flag.Int("timeout", 120, "read timeout for connecting to onion services")
+	batch := flag.Int("batch", 10, "number of onions to scan concurrently")
 
 	flag.Parse()
 
-	if len(flag.Args()) != 1 {
+	if len(flag.Args()) != 1 && *list == "" {
 		flag.Usage()
 		os.Exit(1)
 	}
 
-	hiddenService := flag.Args()[0]
+	if !*simpleReport && !*jsonReport {
+		log.Fatalf("You must set one of --simpleReport or --jsonReport")
+	}
 
-	log.Printf("Starting Scan of %s\n", hiddenService)
+	onionsToScan := []string{}
+	if *list == "" {
+		onionsToScan = append(onionsToScan, flag.Args()[0])
+		log.Printf("Starting Scan of %s\n", flag.Args()[0])
+	} else {
+		content, err := ioutil.ReadFile(*list)
+		if err != nil {
+			log.Fatalf("Could not read onion file %s\n", *list)
+		}
+		onions := strings.Split(string(content), "\n")
+		for _, onion := range onions[0 : len(onions)-1] {
+			onionsToScan = append(onionsToScan, onion)
+		}
+		log.Printf("Starting Scan of %d onion services\n", len(onionsToScan))
+	}
 	log.Printf("This might take a few minutes..\n\n")
 
-	if !*verbose {
-		log.SetOutput(ioutil.Discard)
-	}
-
 	onionScan := new(OnionScan)
-	onionScan.Config = config.Configure(*torProxyAddress, *directoryDepth)
-	scanReport, err := onionScan.Scan(hiddenService)
+	onionScan.Config = config.Configure(*torProxyAddress, *directoryDepth, *fingerprint, *timeout, *verbose)
 
-	if err != nil {
-		log.Fatalf("Error running scanner: %s", err)
+	reports := make(chan *report.OnionScanReport)
+
+	count := 0
+	if *batch > len(onionsToScan) {
+		*batch = len(onionsToScan)
 	}
 
-	if *jsonReport {
-		report.GenerateJsonReport(*reportFile, scanReport)
+	// Run an initial batch of 100 requests (or less...)
+	for count < *batch {
+		go onionScan.Scan(onionsToScan[count], reports)
+		count++
 	}
 
-	if *simpleReport {
-		report.GenerateSimpleReport(*reportFile, scanReport)
+	received := 0
+	for received < len(onionsToScan) {
+		scanReport := <-reports
+
+		// After the initial batch, it's one in one out to prevent proxy overload.
+		if count < len(onionsToScan) {
+			go onionScan.Scan(onionsToScan[count], reports)
+			count++
+		}
+
+		received++
+		if scanReport.TimedOut {
+			onionScan.Config.LogError(errors.New(scanReport.HiddenService + " timed out"))
+		}
+
+		file := *reportFile
+		if file != "" {
+			file = scanReport.HiddenService + "." + *reportFile
+		}
+
+		if *jsonReport {
+			report.GenerateJsonReport(file, scanReport)
+		} else if *simpleReport {
+			report.GenerateSimpleReport(file, scanReport)
+		}
 	}
 }

onionscan.go

@@ -1,20 +1,72 @@
 package main
 
 import (
-	"errors"
-	"fmt"
 	"github.com/s-rah/onionscan/config"
 	"github.com/s-rah/onionscan/protocol"
 	"github.com/s-rah/onionscan/report"
 	"github.com/s-rah/onionscan/utils"
 	"strings"
+	"time"
 )
 
 type OnionScan struct {
 	Config *config.OnionscanConfig
 }
 
-func (os *OnionScan) Scan(hiddenService string) (*report.OnionScanReport, error) {
+func (os *OnionScan) PerformNextAction(report *report.OnionScanReport) {
+	switch report.NextAction {
+	case "web":
+		wps := new(protocol.HTTPProtocolScanner)
+		wps.ScanProtocol(report.HiddenService, os.Config, report)
+		report.NextAction = "tls"
+	case "tls":
+		tps := new(protocol.TLSProtocolScanner)
+		tps.ScanProtocol(report.HiddenService, os.Config, report)
+		report.NextAction = "ssh"
+	case "ssh":
+		sps := new(protocol.SSHProtocolScanner)
+		sps.ScanProtocol(report.HiddenService, os.Config, report)
+		report.NextAction = "irc"
+	case "irc":
+		ips := new(protocol.IRCProtocolScanner)
+		ips.ScanProtocol(report.HiddenService, os.Config, report)
+		report.NextAction = "ricochet"
+	case "ricochet":
+		rps := new(protocol.RicochetProtocolScanner)
+		rps.ScanProtocol(report.HiddenService, os.Config, report)
+		report.NextAction = "ftp"
+	case "ftp":
+		fps := new(protocol.FTPProtocolScanner)
+		fps.ScanProtocol(report.HiddenService, os.Config, report)
+		report.NextAction = "smtp"
+	case "smtp":
+		smps := new(protocol.SMTPProtocolScanner)
+		smps.ScanProtocol(report.HiddenService, os.Config, report)
+		report.NextAction = "mongodb"
+	case "mongodb":
+		mdbps := new(protocol.MongoDBProtocolScanner)
+		mdbps.ScanProtocol(report.HiddenService, os.Config, report)
+		report.NextAction = "vnc"
+	case "vnc":
+		vncps := new(protocol.VNCProtocolScanner)
+		vncps.ScanProtocol(report.HiddenService, os.Config, report)
+		report.NextAction = "xmpp"
+	case "xmpp":
+		xmppps := new(protocol.XMPPProtocolScanner)
+		xmppps.ScanProtocol(report.HiddenService, os.Config, report)
+		report.NextAction = "bitcoin"
+	case "bitcoin":
+		bps := new(protocol.BitcoinProtocolScanner)
+		bps.ScanProtocol(report.HiddenService, os.Config, report)
+		report.NextAction = "none"
+	case "none":
+		return
+	default:
+		report.NextAction = "web"
+	}
+}
+
+func (os *OnionScan) Scan(hiddenService string, out chan *report.OnionScanReport) {
 
 	// Remove Extra Prefix
 	hiddenService = utils.WithoutProtocol(hiddenService)
@@ -25,42 +77,13 @@ func (os *OnionScan) Scan(hiddenService string) (*report.OnionScanReport, error)
 
 	report := report.NewOnionScanReport(hiddenService)
 
-	// HTTP
-	hps := new(protocol.HTTPProtocolScanner)
-	hps.ScanProtocol(hiddenService, os.Config, report)
-
-	// SSH
-	sps := new(protocol.SSHProtocolScanner)
-	sps.ScanProtocol(hiddenService, os.Config, report)
-
-	// Ricochet
-	rps := new(protocol.RicochetProtocolScanner)
-	rps.ScanProtocol(hiddenService, os.Config, report)
-
-	// Bitcoin
-	bps := new(protocol.BitcoinProtocolScanner)
-	bps.ScanProtocol(hiddenService, os.Config, report)
-
-	//IRC
-	ips := new(protocol.IRCProtocolScanner)
-	ips.ScanProtocol(hiddenService, os.Config, report)
-
-	//FTP
-	fps := new(protocol.FTPProtocolScanner)
-	fps.ScanProtocol(hiddenService, os.Config, report)
-
-	//SMTP
-	smps := new(protocol.SMTPProtocolScanner)
-	smps.ScanProtocol(hiddenService, os.Config, report)
-
-	//MongoDb
-	mdbps := new(protocol.MongoDBProtocolScanner)
-	mdbps.ScanProtocol(hiddenService, os.Config, report)
-
-	if !report.WebDetected && !report.SSHDetected && !report.RicochetDetected && !report.BitcoinDetected && !report.IRCDetected && !report.FTPDetected && !report.SMTPDetected && !report.MongoDBDetected {
-		fmt.Printf("Unable to connect to this Tor Hidden Service on any known protocol.\n")
-		return nil, errors.New("Unable to connect to this Tor Hidden Service on any known protocol.")
+	for report.NextAction != "none" {
+		os.PerformNextAction(report)
+		if time.Now().Sub(report.DateScanned).Seconds() > os.Config.Timeout.Seconds() {
+			report.TimedOut = true
+			report.NextAction = "none"
+		}
 	}
 
-	return report, nil
+	out <- report
 }

protocol/bitcoin_scanner.go

@@ -1,26 +1,28 @@
 package protocol
 
 import (
+	"fmt"
 	"github.com/s-rah/onionscan/config"
 	"github.com/s-rah/onionscan/report"
-	"h12.me/socks"
-	"log"
+	"github.com/s-rah/onionscan/utils"
 )
 
 type BitcoinProtocolScanner struct {
 }
 
-func (rps *BitcoinProtocolScanner) ScanProtocol(hiddenService string, onionscanConfig *config.OnionscanConfig, report *report.OnionScanReport) {
+func (rps *BitcoinProtocolScanner) ScanProtocol(hiddenService string, osc *config.OnionscanConfig, report *report.OnionScanReport) {
 	// Bitcoin
-	log.Printf("Checking %s Bitcoin(8333)\n", hiddenService)
-	_, err := socks.DialSocksProxy(socks.SOCKS5, onionscanConfig.TorProxyAddress)("", hiddenService+":8333")
+	osc.LogInfo(fmt.Sprintf("Checking %s Bitcoin(8333)\n", hiddenService))
+	conn, err := utils.GetNetworkConnection(hiddenService, 8333, osc.TorProxyAddress, osc.Timeout)
 	if err != nil {
-		log.Printf("Failed to connect to service on port 8333\n")
+		osc.LogInfo("Failed to connect to service on port 8333\n")
 		report.BitcoinDetected = false
 	} else {
-		log.Printf("Detected possible Bitcoin instance\n")
+		osc.LogInfo("Detected possible Bitcoin instance\n")
 		// TODO: Actual Analysis
 		report.BitcoinDetected = true
 	}
-
+	if conn != nil {
+		conn.Close()
+	}
 }

protocol/ftp_scanner.go

@@ -1,25 +1,37 @@
 package protocol
 
 import (
+	"bufio"
+	"crypto/sha1"
+	"encoding/hex"
+	"fmt"
 	"github.com/s-rah/onionscan/config"
 	"github.com/s-rah/onionscan/report"
-	"h12.me/socks"
-	"log"
+	"github.com/s-rah/onionscan/utils"
 )
 
 type FTPProtocolScanner struct {
 }
 
-func (sps *FTPProtocolScanner) ScanProtocol(hiddenService string, onionscanConfig *config.OnionscanConfig, report *report.OnionScanReport) {
+func (sps *FTPProtocolScanner) ScanProtocol(hiddenService string, osc *config.OnionscanConfig, report *report.OnionScanReport) {
 	// FTP
-	log.Printf("Checking %s FTP(22)\n", hiddenService)
-	_, err := socks.DialSocksProxy(socks.SOCKS5, onionscanConfig.TorProxyAddress)("", hiddenService+":21")
+	osc.LogInfo(fmt.Sprintf("Checking %s FTP(21)\n", hiddenService))
+	conn, err := utils.GetNetworkConnection(hiddenService, 21, osc.TorProxyAddress, osc.Timeout)
 	if err != nil {
-		log.Printf("Failed to connect to service on port 21\n")
+		osc.LogInfo("Failed to connect to service on port 21\n")
 		report.FTPDetected = false
 	} else {
-		// TODO FTP Checking
 		report.FTPDetected = true
+		reader := bufio.NewReader(conn)
+		banner, err := reader.ReadString('\n')
+		if err == nil {
+			report.FTPBanner = banner
+			hash := sha1.Sum([]byte(banner))
+			report.FTPFingerprint = hex.EncodeToString(hash[:])
+			osc.LogInfo(fmt.Sprintf("Found FTP Banner: %s (%s)", banner, report.FTPFingerprint))
+		}
+	}
+	if conn != nil {
+		conn.Close()
 	}
-
 }

protocol/http_scanner.go

@@ -1,13 +1,14 @@
 package protocol
 
 import (
+	"crypto/tls"
+	"fmt"
 	"github.com/s-rah/onionscan/config"
 	"github.com/s-rah/onionscan/report"
 	"github.com/s-rah/onionscan/scans"
 	"github.com/s-rah/onionscan/utils"
 	"h12.me/socks"
 	"io/ioutil"
-	"log"
 	"net/http"
 	"strings"
 )
@@ -24,67 +25,86 @@ var (
 		"/products", "/products/cat"}
 )
 
-func (hps *HTTPProtocolScanner) ScanProtocol(hiddenService string, onionscanConfig *config.OnionscanConfig, report *report.OnionScanReport) {
+func (hps *HTTPProtocolScanner) ScanProtocol(hiddenService string, osc *config.OnionscanConfig, report *report.OnionScanReport) {
 
 	// HTTP
-	log.Printf("Checking %s http(80)\n", hiddenService)
-	_, err := socks.DialSocksProxy(socks.SOCKS5, onionscanConfig.TorProxyAddress)("", hiddenService+":80")
+	osc.LogInfo(fmt.Sprintf("Checking %s http(80)\n", hiddenService))
+	conn, err := utils.GetNetworkConnection(hiddenService, 80, osc.TorProxyAddress, osc.Timeout)
 	if err != nil {
-		log.Printf("Failed to connect to service on port 80\n")
+		osc.LogInfo("Failed to connect to service on port 80\n")
 		report.WebDetected = false
-		return
-	} else {
-		log.Printf("Found potential service on http(80)\n")
-		report.WebDetected = true
-		dialSocksProxy := socks.DialSocksProxy(socks.SOCKS5, onionscanConfig.TorProxyAddress)
-		transportConfig := &http.Transport{
-			Dial: dialSocksProxy,
+		if conn != nil {
+			conn.Close()
+		}
+	} else {
+		osc.LogInfo("Found potential service on http(80)\n")
+		report.WebDetected = true
+		conn.Close()
+		dialSocksProxy := socks.DialSocksProxy(socks.SOCKS5, osc.TorProxyAddress)
+		transportConfig := &http.Transport{
+			Dial:            dialSocksProxy,
+			TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
+		}
+		hps.Client = &http.Client{
+			Transport: transportConfig,
 		}
-		hps.Client = &http.Client{Transport: transportConfig}
 		// FIXME This should probably be moved to it's own file now.
 		response, err := hps.Client.Get("http://" + hiddenService)
+		if err == nil {
+			// Reading all http headers
+			osc.LogInfo(fmt.Sprintf("HTTP response headers: %s\n", report.ServerVersion))
+			responseHeaders := response.Header
+			for key := range responseHeaders {
+				value := responseHeaders.Get(key)
+				// normalize by strings.ToUpper(key) to avoid case sensitive checking
+				report.AddResponseHeader(strings.ToUpper(key), value)
+			}
 
-		if err != nil {
-			log.Printf("Failed to connect to service on port 80\n")
-			return
+			report.ServerVersion = responseHeaders.Get("Server")
+			response.Body.Close()
+		} else {
+			osc.LogError(err)
 		}
 
-		// Reading all http headers
-		log.Printf("HTTP response headers: %s\n", report.ServerVersion)
-		responseHeaders := response.Header
-		for key := range responseHeaders {
-			value := responseHeaders.Get(key)
-			// normalize by strings.ToUpper(key) to avoid case sensitive checking
-			report.AddResponseHeader(strings.ToUpper(key), value)
-			log.Printf("\t%s : %s\n", strings.ToUpper(key), value)
-		}
-
-		report.ServerVersion = responseHeaders.Get("Server")
-
 		// Apache mod-status Check
-		hps.ScanPage(hiddenService, "/server-status", report, scans.ApacheModStatus)
-		hps.ScanPage(hiddenService, "/", report, scans.StandardPageScan)
+		hps.ScanPage(hiddenService, "/", report, osc, scans.StandardPageScan)
+		hps.ScanPage(hiddenService, "/server-status", report, osc, scans.ApacheModStatus)
+		hps.ScanPage(hiddenService, "/private_key", report, osc, scans.PrivateKeyScan)
 
-		log.Printf("\tScanning Common and Referenced Directories\n")
-		directories := append(CommonDirectories, report.PageReferencedDirectories...)
-		utils.RemoveDuplicates(&directories)
-		for _, directory := range directories {
-			hps.ScanPage(hiddenService, directory, report, scans.CheckDirectoryListing(onionscanConfig.DirectoryDepth))
+		if osc.Fingerprint == false {
+			osc.LogInfo("\tScanning Common and Referenced Directories\n")
+			directories := append(CommonDirectories, report.PageReferencedDirectories...)
+			utils.RemoveDuplicates(&directories)
+
+			for _, directory := range directories {
+				hps.ScanPage(hiddenService, directory, report, osc, scans.CheckDirectoryListing(osc.DirectoryDepth))
+			}
 		}
 	}
-	log.Printf("\n")
 }
 
-func (hps *HTTPProtocolScanner) ScanPage(hiddenService string, page string, report *report.OnionScanReport, f func(scans.Scanner, string, int, string, *report.OnionScanReport)) {
+func (hps *HTTPProtocolScanner) ScanPage(hiddenService string, page string, report *report.OnionScanReport, osc *config.OnionscanConfig, f func(scans.Scanner, string, int, string, *report.OnionScanReport, *config.OnionscanConfig)) {
+	err, contents, responseCode := hps.ScrapePage(hiddenService, page)
+	if err == nil {
+		f(hps, page, responseCode, string(contents), report, osc)
+		return
+	} else {
+		osc.LogError(err)
+	}
+}
+
+func (hps *HTTPProtocolScanner) ScrapePage(hiddenService string, page string) (error, []byte, int) {
 	if !strings.Contains(page, utils.WithoutSubdomains(hiddenService)) {
+		if !strings.HasPrefix(page, "/") {
+			page = "/" + page
+		}
 		page = hiddenService + page
 	}
 	response, err := hps.Client.Get("http://" + page)
 	if err != nil {
-		log.Printf("Error connecting to http://%s %s\n", page, err)
-		return
+		return err, nil, -1
 	}
 	defer response.Body.Close()
 	contents, _ := ioutil.ReadAll(response.Body)
-	f(hps, page, response.StatusCode, string(contents), report)
+	return nil, contents, response.StatusCode
 }

protocol/irc_scanner.go

@@ -1,26 +1,42 @@
 package protocol
 
 import (
+	"fmt"
 	"github.com/s-rah/onionscan/config"
 	"github.com/s-rah/onionscan/report"
-	"h12.me/socks"
-	"log"
+	"github.com/s-rah/onionscan/utils"
 )
 
 type IRCProtocolScanner struct {
 }
 
-func (rps *IRCProtocolScanner) ScanProtocol(hiddenService string, onionscanConfig *config.OnionscanConfig, report *report.OnionScanReport) {
+func (rps *IRCProtocolScanner) ScanProtocol(hiddenService string, osc *config.OnionscanConfig, report *report.OnionScanReport) {
 	// IRC
-	log.Printf("Checking %s IRC(6667)\n", hiddenService)
-	_, err := socks.DialSocksProxy(socks.SOCKS5, onionscanConfig.TorProxyAddress)("", hiddenService+":6667")
+	osc.LogInfo(fmt.Sprintf("Checking %s IRC(6667)\n", hiddenService))
+	conn, err := utils.GetNetworkConnection(hiddenService, 6667, osc.TorProxyAddress, osc.Timeout)
 	if err != nil {
-		log.Printf("Failed to connect to service on port 6667\n")
+		osc.LogInfo("Failed to connect to service on port 6667\n")
 		report.IRCDetected = false
 	} else {
-		log.Printf("Detected possible IRC instance\n")
+		osc.LogInfo("Detected possible IRC instance\n")
 		// TODO: Actual Analysis
 		report.IRCDetected = true
 	}
+	if conn != nil {
+		conn.Close()
+	}
 
+	// IRC
+	osc.LogInfo(fmt.Sprintf("Checking %s IRC(6697)\n", hiddenService))
+	conn, err = utils.GetNetworkConnection(hiddenService, 6697, osc.TorProxyAddress, osc.Timeout)
+	if err != nil {
+		osc.LogInfo("Failed to connect to service on port 6697\n")
+	} else {
+		osc.LogInfo("Detected possible IRC (secure) instance\n")
+		// TODO: Actual Analysis
+		report.IRCDetected = true
+	}
+	if conn != nil {
+		conn.Close()
+	}
 }

protocol/mongodb_scanner.go

@@ -1,26 +1,29 @@
 package protocol
 
 import (
+	"fmt"
 	"github.com/s-rah/onionscan/config"
 	"github.com/s-rah/onionscan/report"
-	"h12.me/socks"
-	"log"
+	"github.com/s-rah/onionscan/utils"
 )
 
 type MongoDBProtocolScanner struct {
 }
 
-func (rps *MongoDBProtocolScanner) ScanProtocol(hiddenService string, onionscanConfig *config.OnionscanConfig, report *report.OnionScanReport) {
+func (rps *MongoDBProtocolScanner) ScanProtocol(hiddenService string, osc *config.OnionscanConfig, report *report.OnionScanReport) {
 	// MongoDB
-	log.Printf("Checking %s MongoDB(27017)\n", hiddenService)
-	_, err := socks.DialSocksProxy(socks.SOCKS5, onionscanConfig.TorProxyAddress)("", hiddenService+":27017")
+	osc.LogInfo(fmt.Sprintf("Checking %s MongoDB(27017)\n", hiddenService))
+	conn, err := utils.GetNetworkConnection(hiddenService, 27017, osc.TorProxyAddress, osc.Timeout)
 	if err != nil {
-		log.Printf("Failed to connect to service on port 27017\n")
+		osc.LogInfo("Failed to connect to service on port 27017\n")
 		report.MongoDBDetected = false
 	} else {
-		log.Printf("Detected possible MongoDB instance\n")
+		osc.LogInfo("Detected possible MongoDB instance\n")
 		// TODO: Actual Analysis
 		report.MongoDBDetected = true
 	}
+	if conn != nil {
+		conn.Close()
+	}
 
 }

protocol/ricochet_scanner.go

@@ -1,26 +1,28 @@
 package protocol
 
 import (
+	"fmt"
 	"github.com/s-rah/onionscan/config"
 	"github.com/s-rah/onionscan/report"
-	"h12.me/socks"
-	"log"
+	"github.com/s-rah/onionscan/utils"
 )
 
 type RicochetProtocolScanner struct {
 }
 
-func (rps *RicochetProtocolScanner) ScanProtocol(hiddenService string, onionscanConfig *config.OnionscanConfig, report *report.OnionScanReport) {
+func (rps *RicochetProtocolScanner) ScanProtocol(hiddenService string, osc *config.OnionscanConfig, report *report.OnionScanReport) {
 	// Ricochet
-	log.Printf("Checking %s ricochet(9878)\n", hiddenService)
-	_, err := socks.DialSocksProxy(socks.SOCKS5, onionscanConfig.TorProxyAddress)("", hiddenService+":9878")
+	osc.LogInfo(fmt.Sprintf("Checking %s ricochet(9878)\n", hiddenService))
+	conn, err := utils.GetNetworkConnection(hiddenService, 9878, osc.TorProxyAddress, osc.Timeout)
 	if err != nil {
-		log.Printf("Failed to connect to service on port 9878\n")
+		osc.LogInfo("Failed to connect to service on port 9878\n")
 		report.RicochetDetected = false
 	} else {
-		log.Printf("Detected possible ricochet instance\n")
+		osc.LogInfo("Detected possible ricochet instance\n")
 		// TODO: Actual Analysis
 		report.RicochetDetected = true
 	}
-
+	if conn != nil {
+		conn.Close()
+	}
 }

protocol/smtp_scanner.go

@@ -1,25 +1,38 @@
 package protocol
 
 import (
+	"bufio"
+	"crypto/sha1"
+	"encoding/hex"
+	"fmt"
 	"github.com/s-rah/onionscan/config"
 	"github.com/s-rah/onionscan/report"
-	"h12.me/socks"
-	"log"
+	"github.com/s-rah/onionscan/utils"
 )
 
 type SMTPProtocolScanner struct {
 }
 
-func (sps *SMTPProtocolScanner) ScanProtocol(hiddenService string, onionscanConfig *config.OnionscanConfig, report *report.OnionScanReport) {
+func (sps *SMTPProtocolScanner) ScanProtocol(hiddenService string, osc *config.OnionscanConfig, report *report.OnionScanReport) {
 	// SMTP
-	log.Printf("Checking %s SMTP(25)\n", hiddenService)
-	_, err := socks.DialSocksProxy(socks.SOCKS5, onionscanConfig.TorProxyAddress)("", hiddenService+":25")
+	osc.LogInfo(fmt.Sprintf("Checking %s SMTP(25)\n", hiddenService))
+	conn, err := utils.GetNetworkConnection(hiddenService, 25, osc.TorProxyAddress, osc.Timeout)
 	if err != nil {
-		log.Printf("Failed to connect to service on port 25\n")
+		osc.LogInfo("Failed to connect to service on port 25\n")
 		report.SMTPDetected = false
 	} else {
 		// TODO SMTP Checking
 		report.SMTPDetected = true
+		reader := bufio.NewReader(conn)
+		banner, err := reader.ReadString('\n')
+		if err == nil {
+			report.SMTPBanner = banner
+			hash := sha1.Sum([]byte(banner))
+			report.SMTPFingerprint = hex.EncodeToString(hash[:])
+			osc.LogInfo(fmt.Sprintf("Found SMTP Banner: %s (%s)", banner, report.SMTPFingerprint))
+		}
+	}
+	if conn != nil {
+		conn.Close()
 	}
-
 }

protocol/ssh_scanner.go

@@ -1,27 +1,30 @@
 package protocol
 
 import (
+	"bufio"
 	"crypto/md5"
 	"errors"
 	"fmt"
 	"github.com/s-rah/onionscan/config"
 	"github.com/s-rah/onionscan/report"
+	"github.com/s-rah/onionscan/utils"
 	"golang.org/x/crypto/ssh"
-	"h12.me/socks"
-	"log"
 	"net"
 )
 
 type SSHProtocolScanner struct {
 }
 
-func (sps *SSHProtocolScanner) ScanProtocol(hiddenService string, onionscanConfig *config.OnionscanConfig, report *report.OnionScanReport) {
+func (sps *SSHProtocolScanner) ScanProtocol(hiddenService string, osc *config.OnionscanConfig, report *report.OnionScanReport) {
 	// SSH
-	log.Printf("Checking %s ssh(22)\n", hiddenService)
-	conn, err := socks.DialSocksProxy(socks.SOCKS5, onionscanConfig.TorProxyAddress)("", hiddenService+":22")
+	osc.LogInfo(fmt.Sprintf("Checking %s ssh(22)\n", hiddenService))
+	conn, err := utils.GetNetworkConnection(hiddenService, 22, osc.TorProxyAddress, osc.Timeout)
 	if err != nil {
-		log.Printf("Failed to connect to service on port 22\n")
+		osc.LogInfo("Failed to connect to service on port 22\n")
 		report.SSHDetected = false
+		if conn != nil {
+			conn.Close()
+		}
 	} else {
 		// TODO SSH Checking
 		report.SSHDetected = true
@@ -41,13 +44,26 @@ func (sps *SSHProtocolScanner) ScanProtocol(hiddenService string, onionscanConfi
 					}
 				}
 				report.SSHKey = fingerprint
-				log.Printf("Found SSH Key %s\n", fingerprint)
+				osc.LogInfo(fmt.Sprintf("Found SSH Key %s\n", fingerprint))
 				// We don't want to continue
 				return errors.New("error")
 			},
 		}
 		ssh.NewClientConn(conn, hiddenService+":22", config)
-
+		if conn != nil {
+			conn.Close()
+		}
+		conn, err = utils.GetNetworkConnection(hiddenService, 22, osc.TorProxyAddress, osc.Timeout)
+		if err == nil {
+			reader := bufio.NewReader(conn)
+			banner, err := reader.ReadString('\n')
+			if err == nil {
+				report.SSHBanner = banner
+				osc.LogInfo(fmt.Sprintf("Found SSH Banner: %s (%s)", banner))
+			}
+		}
+		if conn != nil {
+			conn.Close()
+		}
 	}
-
 }

protocol/tls_scanner.go

@@ -0,0 +1,37 @@
+package protocol
+
+import (
+	"crypto/tls"
+	"fmt"
+	"github.com/s-rah/onionscan/config"
+	"github.com/s-rah/onionscan/report"
+	"github.com/s-rah/onionscan/utils"
+)
+
+type TLSProtocolScanner struct {
+}
+
+func (sps *TLSProtocolScanner) ScanProtocol(hiddenService string, osc *config.OnionscanConfig, report *report.OnionScanReport) {
+	osc.LogInfo(fmt.Sprintf("Checking %s TLS(443)\n", hiddenService))
+	conn, err := utils.GetNetworkConnection(hiddenService, 443, osc.TorProxyAddress, osc.Timeout)
+	if err != nil {
+		osc.LogInfo("Failed to connect to service on port 443\n")
+		report.TLSDetected = false
+	} else {
+		osc.LogInfo("Found TLS Endpoint\n")
+		report.TLSDetected = true
+		config := &tls.Config{
+			InsecureSkipVerify: true,
+		}
+		tlsConn := tls.Client(conn, config)
+		tlsConn.Write([]byte("GET / HTTP/1.1\r\n\r\n"))
+		for _, certificate := range tlsConn.ConnectionState().PeerCertificates {
+			osc.LogInfo(fmt.Sprintf("Found Certificate %v \n", certificate))
+			report.Certificates = append(report.Certificates, *certificate)
+		}
+		tlsConn.Close()
+	}
+	if conn != nil {
+		conn.Close()
+	}
+}

protocol/vnc_scanner.go

@@ -0,0 +1,28 @@
+package protocol
+
+import (
+	"fmt"
+	"github.com/s-rah/onionscan/config"
+	"github.com/s-rah/onionscan/report"
+	"github.com/s-rah/onionscan/utils"
+)
+
+type VNCProtocolScanner struct {
+}
+
+func (vncps *VNCProtocolScanner) ScanProtocol(hiddenService string, osc *config.OnionscanConfig, report *report.OnionScanReport) {
+	// MongoDB
+	osc.LogInfo(fmt.Sprintf("Checking %s VNC(5900)\n", hiddenService))
+	conn, err := utils.GetNetworkConnection(hiddenService, 5900, osc.TorProxyAddress, osc.Timeout)
+	if err != nil {
+		osc.LogInfo("Failed to connect to service on port 5900\n")
+		report.VNCDetected = false
+	} else {
+		osc.LogInfo("Detected possible VNC instance\n")
+		// TODO: Actual Analysis
+		report.VNCDetected = true
+	}
+	if conn != nil {
+		conn.Close()
+	}
+}

protocol/xmpp_scanner.go

@@ -0,0 +1,41 @@
+package protocol
+
+import (
+	"fmt"
+	"github.com/s-rah/onionscan/config"
+	"github.com/s-rah/onionscan/report"
+	"github.com/s-rah/onionscan/utils"
+)
+
+type XMPPProtocolScanner struct {
+}
+
+func (rps *XMPPProtocolScanner) ScanProtocol(hiddenService string, osc *config.OnionscanConfig, report *report.OnionScanReport) {
+	// XMPP
+	osc.LogInfo(fmt.Sprintf("Checking %s XMPP(5222)\n", hiddenService))
+	conn, err := utils.GetNetworkConnection(hiddenService, 5222, osc.TorProxyAddress, osc.Timeout)
+	if err != nil {
+		osc.LogInfo("Failed to connect to service on port 5222\n")
+		report.XMPPDetected = false
+	} else {
+		osc.LogInfo("Detected possible XMPP instance\n")
+		// TODO: Actual Analysis
+		report.XMPPDetected = true
+	}
+	if conn != nil {
+		conn.Close()
+	}
+	// XMPP
+	osc.LogInfo(fmt.Sprintf("Checking %s XMPP(5223)\n", hiddenService))
+	conn, err = utils.GetNetworkConnection(hiddenService, 5223, osc.TorProxyAddress, osc.Timeout)
+	if err != nil {
+		osc.LogInfo("Failed to connect to service on port 5223\n")
+	} else {
+		osc.LogInfo("Detected possible XMPP (secure) instance\n")
+		// TODO: Actual Analysis
+		report.XMPPDetected = true
+	}
+	if conn != nil {
+		conn.Close()
+	}
+}

report/onionscanreport.go

@@ -1,9 +1,11 @@
 package report
 
 import (
+	"crypto/x509"
 	"encoding/json"
 	"github.com/s-rah/onionscan/utils"
 	"io/ioutil"
+	"time"
 )
 
 type ExifTag struct {
@@ -16,37 +18,70 @@ type ExifImage struct {
 	ExifTags []ExifTag `json:"exifTags"`
 }
 
+type PGPKey struct {
+	ArmoredKey  string `json:"armoredKey"`
+	Identity    string `json:"identity"`
+	FingerPrint string `json:"fingerprint"`
+}
+
 type OnionScanReport struct {
-	WebDetected      bool `json:"webDetected"`
-	SSHDetected      bool `json:"sshDetected"`
-	RicochetDetected bool `json:"ricochetDetected"`
-	IRCDetected      bool `json:"ircDetected"`
-	FTPDetected      bool `json:"ftpDetected"`
-	SMTPDetected     bool `json:"smtpDetected"`
+	HiddenService string    `json:"hiddenService"`
+	DateScanned   time.Time `json:"dateScanned"`
 
-	BitcoinDetected bool `json:"bitcoinDetected"`
-	MongoDBDetected bool `json:"mongodbDetected"`
+	// Summary
+	WebDetected        bool `json:"webDetected"`
+	TLSDetected        bool `json:"tlsDetected"`
+	SSHDetected        bool `json:"sshDetected"`
+	RicochetDetected   bool `json:"ricochetDetected"`
+	IRCDetected        bool `json:"ircDetected"`
+	FTPDetected        bool `json:"ftpDetected"`
+	SMTPDetected       bool `json:"smtpDetected"`
+	BitcoinDetected    bool `json:"bitcoinDetected"`
+	MongoDBDetected    bool `json:"mongodbDetected"`
+	VNCDetected        bool `json:"vncDetected"`
+	XMPPDetected       bool `json:"xmppDetected"`
+	SkynetDetected     bool `json:"skynetDetected"`
+	PrivateKeyDetected bool `json:"privateKeyDetected"`
 
-	HiddenService             string      `json:"hiddenService"`
-	ServerPoweredBy           string      `json:"serverPoweredBy"`
-	ServerVersion             string      `json:"serverVersion"`
-	FoundApacheModStatus      bool        `json:"foundApacheModStatus"`
-	RelatedOnionServices      []string    `json:"relatedOnionServices"`
-	RelatedClearnetDomains    []string    `json:"relatedOnionDomains"`
-	LinkedSites               []string    `json:"linkedSites"`
-	InternalPages             []string    `json:"InternalPages"`
-	IP                        []string    `json:"ipAddresses"`
-	OpenDirectories           []string    `json:"openDirectories"`
-	ExifImages                []ExifImage `json:"exifImages"`
-	InterestingFiles          []string    `json:"interestingFiles"`
-	PageReferencedDirectories []string    `json:"pageReferencedDirectories"`
-	PGPKeys                   []string    `json:"pgpKeys"`
+	// Web Specific
+	ServerPoweredBy           string            `json:"serverPoweredBy"`
+	ServerVersion             string            `json:"serverVersion"`
+	FoundApacheModStatus      bool              `json:"foundApacheModStatus"`
+	RelatedOnionServices      []string          `json:"relatedOnionServices"`
+	RelatedClearnetDomains    []string          `json:"relatedOnionDomains"`
+	LinkedSites               []string          `json:"linkedSites"`
+	InternalPages             []string          `json:"internalPages"`
+	IP                        []string          `json:"ipAddresses"`
+	OpenDirectories           []string          `json:"openDirectories"`
+	ExifImages                []ExifImage       `json:"exifImages"`
+	InterestingFiles          []string          `json:"interestingFiles"`
+	PageReferencedDirectories []string          `json:"pageReferencedDirectories"`
+	PGPKeys                   []PGPKey          `json:"pgpKeys"`
+	Hashes                    []string          `json:"hashes"`
+	Snapshot                  string            `json:"snapshot"`
+	PageTitle                 string            `json:"pageTitle"`
+	ResponseHeaders           map[string]string `json:"responseHeaders"`
 
-	Hashes          []string          `json:"hashes"`
-	SSHKey          string            `json:"sshKey"`
-	Snapshot        string            `json:"snapshot"`
-	PageTitle       string            `json:"pageTitle"`
-	ResponseHeaders map[string]string `json:"responseHeaders"`
+	// TLS
+	Certificates []x509.Certificate `json:"certificates"`
+
+	//Bitcoin
+	BitcoinAddresses []string `json:"bitcoinAddresses"`
+
+	// SSH
+	SSHKey    string `json:"sshKey"`
+	SSHBanner string `json:"sshBanner"`
+
+	// FTP
+	FTPFingerprint string `json:"ftpFingerprint"`
+	FTPBanner      string `json:"ftpBanner"`
+
+	// SMTP
+	SMTPFingerprint string `json:"smtpFingerprint"`
+	SMTPBanner      string `json:"smtpBanner"`
+
+	NextAction string `json:"lastAction"`
+	TimedOut   bool
 }
 
 func LoadReportFromFile(filename string) (OnionScanReport, error) {
@@ -60,7 +95,11 @@ func LoadReportFromFile(filename string) (OnionScanReport, error) {
 }
 
 func NewOnionScanReport(hiddenService string) *OnionScanReport {
-	return &OnionScanReport{HiddenService: hiddenService, ResponseHeaders: make(map[string]string)}
+	report := new(OnionScanReport)
+	report.HiddenService = hiddenService
+	report.ResponseHeaders = make(map[string]string)
+	report.DateScanned = time.Now()
+	return report
 }
 
 func (osr *OnionScanReport) AddOpenDirectory(dir string) {
@@ -93,9 +132,9 @@ func (osr *OnionScanReport) AddInternalPage(site string) {
 	utils.RemoveDuplicates(&osr.InternalPages)
 }
 
-func (osr *OnionScanReport) AddPGPKey(key string) {
-	osr.PGPKeys = append(osr.PGPKeys, key)
-	utils.RemoveDuplicates(&osr.PGPKeys)
+func (osr *OnionScanReport) AddPGPKey(armoredKey, identity, fingerprint string) {
+	osr.PGPKeys = append(osr.PGPKeys, PGPKey{armoredKey, identity, fingerprint})
+	//TODO map of fingerprint:PGPKeys? and  utils.RemoveDuplicates(&osr.PGPKeys)
 }
 
 func (osr *OnionScanReport) AddResponseHeader(name string, value string) {

report/report_generator.go

@@ -5,6 +5,7 @@ import (
 	"fmt"
 	"log"
 	"os"
+	"time"
 )
 
 func GenerateJsonReport(reportFile string, report *OnionScanReport) {
@@ -15,9 +16,11 @@ func GenerateJsonReport(reportFile string, report *OnionScanReport) {
 
 	if len(reportFile) > 0 {
 		f, err := os.Create(reportFile)
-		if err != nil {
-			log.Fatalf("Cannot create report file: %s", err)
-			panic(err)
+
+		for err != nil {
+			log.Printf("Cannot create report file: %s...trying again in 5 seconds...", err)
+			time.Sleep(time.Second * 5)
+			f, err = os.Create(reportFile)
 		}
 
 		defer f.Close()
@@ -173,9 +176,11 @@ func GenerateSimpleReport(reportFile string, report *OnionScanReport) {
 
 	if len(reportFile) > 0 {
 		f, err := os.Create(reportFile)
-		if err != nil {
-			log.Fatalf("Cannot create report file: %s", err)
-			panic(err)
+
+		for err != nil {
+			log.Printf("Cannot create report file: %s...trying again in 5 seconds...", err)
+			time.Sleep(time.Second * 5)
+			f, err = os.Create(reportFile)
 		}
 
 		defer f.Close()

scans/apache_mod_status.go

@@ -1,58 +1,59 @@
 package scans
 
 import (
+	"fmt"
+	"github.com/s-rah/onionscan/config"
 	"github.com/s-rah/onionscan/report"
 	"github.com/s-rah/onionscan/utils"
-	"log"
 	"regexp"
 	"strings"
 )
 
-func ApacheModStatus(scan Scanner, page string, status int, contents string, report *report.OnionScanReport) {
+func ApacheModStatus(scan Scanner, page string, status int, contents string, report *report.OnionScanReport, osc *config.OnionscanConfig) {
 	if status == 200 {
 		r := regexp.MustCompile(`Server Version: (.*)</dt>`)
 		serverVersion := r.FindStringSubmatch(string(contents))
 
 		// Check if this looks like a mod_status page. Sometimes sites simply load their index.
 		if len(serverVersion) > 1 {
-			log.Printf("Detected Apache mod_status Exposed...\033[091mAlert!\033[0m\n")
+			osc.LogInfo("Detected Apache mod_status Exposed...\033[091mAlert!\033[0m\n")
 			report.FoundApacheModStatus = true
 
-			log.Printf("\t Using mod_status Server Version: %s\n", serverVersion[1])
+			osc.LogInfo(fmt.Sprintf("\t Using mod_status Server Version: %s\n", serverVersion[1]))
 			report.ServerVersion = serverVersion[1]
 
 			// Check for co-hosted onion services.
-			log.Printf("Scanning for Co-Hosted Onions\n")
+			osc.LogInfo("Scanning for Co-Hosted Onions\n")
 			r = regexp.MustCompile(`[a-z0-9]+.onion(:[0-9]{0-5})?`)
 			foundServices := r.FindAllString(string(contents), -1)
 			utils.RemoveDuplicates(&foundServices)
 			for _, onion := range foundServices {
 				if onion != report.HiddenService {
-					log.Printf("\t \033[091mAlert!\033[0m Found Co-Hosted Onions: %s\n", onion)
+					osc.LogInfo(fmt.Sprintf("\t \033[091mAlert!\033[0m Found Co-Hosted Onions: %s\n", onion))
 					report.AddRelatedOnionService(onion)
 				}
 			}
 
 			// Check for co-hosted onion services.
-			log.Printf("Scanning for Co-Hosted Clearnet Domains\n")
+			osc.LogInfo("Scanning for Co-Hosted Clearnet Domains\n")
 			r = regexp.MustCompile(`>(([a-zA-Z]{1})|([a-zA-Z]{1}[a-zA-Z]{1})|([a-zA-Z]{1}[0-9]{1})|([0-9]{1}[a-zA-Z]{1})|([a-zA-Z0-9][a-zA-Z0-9-_]{1,61}[a-zA-Z0-9]))\.([a-zA-Z]{2,6}|[a-zA-Z0-9-]{2,30}\.[a-zA-Z]{2,3})`)
 			foundServices = r.FindAllString(string(contents), -1)
 			utils.RemoveDuplicates(&foundServices)
 			for _, domain := range foundServices {
 				if strings.Contains(domain, ".onion") == false {
-					log.Printf("\t \033[091mAlert!\033[0m Found Co-Hosted Service: %s\n", domain[1:])
+					osc.LogInfo(fmt.Sprintf("\t \033[091mAlert!\033[0m Found Co-Hosted Service: %s\n", domain[1:]))
 					report.AddRelatedClearnetDomain(domain[4:])
 				}
 			}
 
 			// Check for IP Addresses
-			log.Printf("Scanning for IP Addresses (clearweb clients, and servers)\n")
+			osc.LogInfo("Scanning for IP Addresses (clearweb clients, and servers)\n")
 			r = regexp.MustCompile(`(?:(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.){3}(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)`)
 			foundIPs := r.FindAllString(string(contents), -1)
 			utils.RemoveDuplicates(&foundIPs)
 			for _, ip := range foundIPs {
 				if ip != "127.0.0.1" {
-					log.Printf("\t \033[091mAlert!\033[0m Found IP Address : %s\n", ip)
+					osc.LogInfo(fmt.Sprintf("\t \033[091mAlert!\033[0m Found IP Address : %s\n", ip))
 					report.AddIPAddress(ip)
 				}
 			}
@@ -60,7 +61,7 @@ func ApacheModStatus(scan Scanner, page string, status int, contents string, rep
 		}
 	}
 	if !report.FoundApacheModStatus {
-		log.Printf("\tApache mod_status Not Exposed...\033[92mGood!\033[0m\n")
+		osc.LogInfo("\tApache mod_status Not Exposed...\033[92mGood!\033[0m\n")
 		report.FoundApacheModStatus = false
 	}
 }

scans/bitcoin_content_scan.go

@@ -0,0 +1,20 @@
+package scans
+
+import (
+	"github.com/s-rah/onionscan/report"
+	"log"
+	"regexp"
+)
+
+type BitcoinContentScan struct {
+}
+
+func (cs *BitcoinContentScan) ScanContent(content string, report *report.OnionScanReport) {
+	log.Printf("Scanning for Bitcoin Address\n")
+	bitcoinAddressRegexp := regexp.MustCompile("[1|3][A-Za-z0-9]{25,34}")
+	foundBitcoinAddress := bitcoinAddressRegexp.FindAllString(content, -1)
+	for _, ba := range foundBitcoinAddress {
+		log.Printf("Found Bitcoin Address: %s", ba)
+		report.BitcoinAddresses = append(report.BitcoinAddresses, ba)
+	}
+}

scans/check_directory_listing.go

@@ -1,35 +1,36 @@
 package scans
 
 import (
+	"fmt"
+	"github.com/s-rah/onionscan/config"
 	"github.com/s-rah/onionscan/report"
-	"log"
 	"regexp"
 	"strings"
 )
 
-func CheckDirectoryListing(depth int) func(Scanner, string, int, string, *report.OnionScanReport) {
-	return func(scan Scanner, dir string, status int, contents string, report *report.OnionScanReport) {
-		CheckDirectoryListingDepth(scan, dir, status, depth, contents, report)
+func CheckDirectoryListing(depth int) func(Scanner, string, int, string, *report.OnionScanReport, *config.OnionscanConfig) {
+	return func(scan Scanner, dir string, status int, contents string, report *report.OnionScanReport, osc *config.OnionscanConfig) {
+		CheckDirectoryListingDepth(scan, dir, status, depth, contents, report, osc)
 	}
 }
 
-func CheckDirectoryListingDepth(scan Scanner, dir string, status int, depth int, contents string, report *report.OnionScanReport) {
+func CheckDirectoryListingDepth(scan Scanner, dir string, status int, depth int, contents string, report *report.OnionScanReport, osc *config.OnionscanConfig) {
 	if status == 200 && strings.Contains(string(contents), "Index of "+dir) {
-		log.Printf("Detected Open Directory %s...\033[091mAlert!\033[0m\n", dir)
+		osc.LogInfo(fmt.Sprintf("Detected Open Directory %s...\033[091mAlert!\033[0m\n", dir))
 
 		report.AddOpenDirectory(dir)
 
 		r := regexp.MustCompile(`href="((.*?\.jpg)|(.*?\.png)|(.*?\.jpeg)|(.*?\.gif))"`)
 		foundImages := r.FindAllStringSubmatch(string(contents), -1)
 		for _, image := range foundImages {
-			log.Printf("\t Found image %s/%s\n", dir, image[1])
-			scan.ScanPage(report.HiddenService, dir+"/"+image[1], report, CheckExif)
+			osc.LogInfo(fmt.Sprintf("\t Found image %s/%s\n", dir, image[1]))
+			scan.ScanPage(report.HiddenService, dir+"/"+image[1], report, osc, CheckExif)
 		}
 
 		r = regexp.MustCompile(`href="((.*\.zip)|(.*\.tar)|(.*\.gz)|(.*\.pst)|(.*\.txt))"`)
 		interestingFiles := r.FindAllStringSubmatch(string(contents), -1)
 		for _, file := range interestingFiles {
-			log.Printf("\t Found interesting file %s/%s\n", dir, file[1])
+			osc.LogInfo(fmt.Sprintf("\t Found interesting file %s/%s\n", dir, file[1]))
 			//TODO: We can do further analysis here, for now, just report them.
 			report.AddInterestingFile(dir + "/" + file[1])
 		}
@@ -37,14 +38,14 @@ func CheckDirectoryListingDepth(scan Scanner, dir string, status int, depth int,
 		r = regexp.MustCompile(`href="([^/](.*?))/"`)
 		subDir := r.FindAllStringSubmatch(string(contents), -1)
 		for _, file := range subDir {
-			log.Printf("\t Found subdir %s/%s\n", dir, file[1])
+			osc.LogInfo(fmt.Sprintf("\t Found subdir %s/%s\n", dir, file[1]))
 			//TODO: We can do further analysis here, for now, just report them.
 			if depth > 0 {
-				scan.ScanPage(report.HiddenService, dir+"/"+file[1], report, CheckDirectoryListing(depth-1))
+				scan.ScanPage(report.HiddenService, dir+"/"+file[1], report, osc, CheckDirectoryListing(depth-1))
 			}
 		}
 
 	} else {
-		log.Printf("Directory %s either doesn't exist or is not readable\n", dir)
+		osc.LogInfo(fmt.Sprintf("Directory %s either doesn't exist or is not readable\n", dir))
 	}
 }

scans/check_exif.go

@@ -1,14 +1,15 @@
 package scans
 
 import (
+	"fmt"
+	"github.com/s-rah/onionscan/config"
 	"github.com/s-rah/onionscan/report"
 	"github.com/xiam/exif"
 	"io"
-	"log"
 	"strings"
 )
 
-func CheckExif(scan Scanner, page string, status int, contents string, report *report.OnionScanReport) {
+func CheckExif(scan Scanner, page string, status int, contents string, report *report.OnionScanReport, osc *config.OnionscanConfig) {
 	if status == 200 {
 		reader := exif.New()
 		_, err := io.Copy(reader, strings.NewReader(contents))
@@ -31,7 +32,7 @@ func CheckExif(scan Scanner, page string, status int, contents string, report *r
 			report.AddExifImage(page)
 
 			for name, val := range reader.Tags {
-				log.Printf("\t \033[091mAlert!\033[0m Found Exif Tag%s: %s\n", name, val)
+				osc.LogInfo(fmt.Sprintf("\t \033[091mAlert!\033[0m Found Exif Tag%s: %s\n", name, val))
 				report.AddExifTag(name, val)
 			}
 		}

scans/pgp_content_scan.go

@@ -2,18 +2,35 @@ package scans
 
 import (
 	"github.com/s-rah/onionscan/report"
-	"log"
+	"golang.org/x/crypto/openpgp"
 	"regexp"
+	"strings"
 )
 
 type PGPContentScan struct {
 }
 
 func (cs *PGPContentScan) ScanContent(content string, report *report.OnionScanReport) {
-	log.Printf("\tScanning for PGP Key\n")
+	//log.Printf("Scanning for PGP Key\n")
 	pgpRegexp := regexp.MustCompile("-----BEGIN PGP PUBLIC KEY BLOCK-----((?s).*)-----END PGP PUBLIC KEY BLOCK-----")
 	foundPGP := pgpRegexp.FindAllString(content, -1)
-	for _, key := range foundPGP {
-		report.AddPGPKey(key)
+	for _, keyString := range foundPGP {
+		keys, err := openpgp.ReadArmoredKeyRing(strings.NewReader(keyString))
+		if err != nil {
+			//		log.Printf("ERROR: %s\n", err)
+			continue
+		}
+		if len(keys) < 1 || len(keys[0].Subkeys) < 1 || len(keys[0].Identities) < 1 {
+			//		log.Printf("ERROR: failed to accept key\n")
+			continue
+		}
+
+		var identity string
+		for identity = range keys[0].Identities {
+			break
+		}
+		//	log.Printf("\tFound PGP Key fingerprint: %s belonging to %s", keys[0].Subkeys[0].PublicKey.KeyIdShortString(), identity)
+
+		report.AddPGPKey(keyString, identity, keys[0].Subkeys[0].PublicKey.KeyIdShortString())
 	}
 }

scans/private_key.go

@@ -0,0 +1,15 @@
+package scans
+
+import (
+	"fmt"
+	"github.com/s-rah/onionscan/config"
+	"github.com/s-rah/onionscan/report"
+)
+
+func PrivateKeyScan(scan Scanner, page string, status int, contents string, report *report.OnionScanReport, osc *config.OnionscanConfig) {
+	osc.LogInfo(fmt.Sprintf("Scanning %s\n", page))
+	if status == 200 {
+		osc.LogInfo(fmt.Sprintf("\tPrivate Key %s is Accessible!!\n", page))
+		report.PrivateKeyDetected = true
+	}
+}

scans/scanner.go

@@ -1,9 +1,11 @@
 package scans
 
 import (
+	"github.com/s-rah/onionscan/config"
 	"github.com/s-rah/onionscan/report"
 )
 
 type Scanner interface {
-	ScanPage(string, string, *report.OnionScanReport, func(Scanner, string, int, string, *report.OnionScanReport))
+	ScanPage(string, string, *report.OnionScanReport, *config.OnionscanConfig, func(Scanner, string, int, string, *report.OnionScanReport, *config.OnionscanConfig))
+	ScrapePage(string, string) (error, []byte, int)
 }

scans/standard-page-scan.go

@@ -3,19 +3,20 @@ package scans
 import (
 	"crypto/sha1"
 	"encoding/hex"
+	"fmt"
+	"github.com/s-rah/onionscan/config"
 	"github.com/s-rah/onionscan/report"
 	"github.com/s-rah/onionscan/utils"
 	"golang.org/x/net/html"
-	"log"
 	"net/url"
 	"regexp"
 	"strings"
 )
 
-func StandardPageScan(scan Scanner, page string, status int, contents string, report *report.OnionScanReport) {
-	log.Printf("Scanning %s\n", page)
+func StandardPageScan(scan Scanner, page string, status int, contents string, report *report.OnionScanReport, osc *config.OnionscanConfig) {
+	osc.LogInfo(fmt.Sprintf("Scanning %s\n", page))
 	if status == 200 {
-		log.Printf("\tPage %s is Accessible\n", page)
+		osc.LogInfo(fmt.Sprintf("\tPage %s is Accessible\n", page))
 
 		hash := sha1.Sum([]byte(contents))
 		report.Hashes = append(report.Hashes, hex.EncodeToString(hash[:]))
@@ -27,14 +28,16 @@ func StandardPageScan(scan Scanner, page string, status int, contents string, re
 			var startIndex = strings.Index(contents, "<title>")
 			var endIndex = strings.Index(contents, "</title>")
 			var pageTitle = contents[startIndex+len("<title>") : endIndex]
-			log.Printf("\tPage Title: %s\n", pageTitle)
+			osc.LogInfo(fmt.Sprintf("\tPage Title: %s\n", pageTitle))
 			report.PageTitle = pageTitle
 		}
 
 		new(PGPContentScan).ScanContent(contents, report)
+		//new(BitcoinContentScan).ScanContent(contents, report)
 
-		log.Printf("\tScanning for Images\n")
-		domains := utils.ExtractDomains(contents)
+		osc.LogInfo("\tScanning for Images\n")
+		var domains []string
+		var cssLinks []string
 
 		// parser based on http://schier.co/blog/2015/04/26/a-simple-web-scraper-in-go.html
 		z := html.NewTokenizer(strings.NewReader(contents))
@@ -49,8 +52,8 @@ func StandardPageScan(scan Scanner, page string, status int, contents string, re
 			// TODO: don't crawl links with nofollow
 
 			if tt == html.StartTagToken {
-				isLink := t.Data == "a"
-				if isLink {
+				// links
+				if t.Data == "a" {
 					linkUrl := utils.GetAttribute(t, "href")
 					if len(linkUrl) > 1 {
 						domains = append(domains, linkUrl)
@@ -58,56 +61,74 @@ func StandardPageScan(scan Scanner, page string, status int, contents string, re
 				}
 			}
 
-			isImage := t.Data == "img"
-			if isImage {
+			// css <link>
+			if t.Data == "link" && utils.GetAttribute(t, "rel") == "stylesheet" {
+				cssLinks = append(cssLinks, utils.GetAttribute(t, "href"))
+			}
+
+			// images
+			if t.Data == "img" {
 				imageUrl := utils.GetAttribute(t, "src")
 
-				baseUrl, _ := url.Parse(imageUrl)
-				if utils.WithoutSubdomains(baseUrl.Host) == utils.WithoutSubdomains(report.HiddenService) {
-					scan.ScanPage(report.HiddenService, utils.WithoutProtocol(imageUrl), report, CheckExif)
-					log.Printf("\t Found internal image %s\n", imageUrl)
-				} else {
-					log.Printf("\t Not scanning remote image %s\n", imageUrl)
+				baseUrl, err := url.Parse(imageUrl)
+				if err == nil {
+					if utils.WithoutSubdomains(baseUrl.Host) == utils.WithoutSubdomains(report.HiddenService) {
+						scan.ScanPage(report.HiddenService, utils.WithoutProtocol(imageUrl), report, osc, CheckExif)
+						osc.LogInfo(fmt.Sprintf("\t Found internal image %s\n", imageUrl))
+					} else {
+						osc.LogInfo(fmt.Sprintf("\t Not scanning remote image %s\n", imageUrl))
+					}
 				}
 			}
 		}
 
-		log.Printf("\tScanning for Links\n")
+		osc.LogInfo("\tScanning for CSS Fonts and Background Images\n")
+		utils.RemoveDuplicates(&cssLinks)
+		for _, cssUrl := range cssLinks {
+			osc.LogInfo(fmt.Sprintf("\tScanning CSS file: %s\n", cssUrl))
+			_, cssContents, _ := scan.ScrapePage(report.HiddenService, utils.WithoutProtocol(cssUrl))
+			domains = append(domains, utils.ExtractDomains(string(cssContents))[:]...)
+		}
 
+		osc.LogInfo("\tScanning for Links\n")
+		domains = append(domains, utils.ExtractDomains(contents)...)
+		utils.RemoveDuplicates(&domains)
 		for _, domain := range domains {
-			baseUrl, _ := url.Parse(domain)
-			if baseUrl.Host != "" && utils.WithoutSubdomains(baseUrl.Host) != utils.WithoutSubdomains(report.HiddenService) {
-				log.Printf("Found Related URL %s\n", domain)
-				// TODO: Lots of information here which needs to be processed.
-				// * Links to standard sites - google / bitpay etc.
-				// * Links to other onion sites
-				// * Links to obscure clearnet sites.
-				report.AddLinkedSite(baseUrl.Host)
-			} else {
-				// * Process FQDN internal links
-				log.Printf("Found Internal URL %s\n", domain)
-				report.AddInternalPage(baseUrl.Host)
+			baseUrl, err := url.Parse(domain)
+			if err == nil {
+				if baseUrl.Host != "" && utils.WithoutSubdomains(baseUrl.Host) != utils.WithoutSubdomains(report.HiddenService) {
+					osc.LogInfo(fmt.Sprintf("Found Related URL %s\n", domain))
+					// TODO: Lots of information here which needs to be processed.
+					// * Links to standard sites - google / bitpay etc.
+					// * Links to other onion sites
+					// * Links to obscure clearnet sites.
+					report.AddLinkedSite(baseUrl.Host)
+				} else {
+					// * Process FQDN internal links
+					osc.LogInfo(fmt.Sprintf("Found Internal URL %s\n", domain))
+					report.AddInternalPage(baseUrl.Host)
+				}
 			}
 		}
 
-		log.Printf("\tScanning for Referenced Directories\n")
+		osc.LogInfo("\tScanning for Referenced Directories\n")
 		r := regexp.MustCompile("(src|href)=\"([^\"]*)\"")
 		foundPaths := r.FindAllStringSubmatch(string(contents), -1)
 		for _, regexpResults := range foundPaths {
 			path := regexpResults[2]
-			if strings.HasPrefix(path, "http") && !strings.Contains(path, utils.WithoutSubdomains(report.HiddenService)) {
+			if (strings.HasPrefix(path, "http") || strings.HasPrefix(path, "//")) && !strings.Contains(path, utils.WithoutSubdomains(report.HiddenService)) {
 				continue
 			}
 
 			term := strings.LastIndex(path, "/")
 			if term > 0 {
-				log.Printf("\t Found Referenced Directory %s\n", path[:term])
+				osc.LogInfo(fmt.Sprintf("\t Found Referenced Directory %s\n", path[:term]))
 				report.AddPageReferencedDirectory(utils.WithoutProtocol(path[:term]))
 			}
 		}
 	} else if status == 403 {
-		log.Printf("\tPage %s%s is Forbidden\n", report.HiddenService, page)
+		osc.LogInfo(fmt.Sprintf("\tPage %s%s is Forbidden\n", report.HiddenService, page))
 	} else if status == 404 {
-		log.Printf("\tPage %s%s is Does Not Exist\n", report.HiddenService, page)
+		osc.LogInfo(fmt.Sprintf("\tPage %s%s is Does Not Exist\n", report.HiddenService, page))
 	}
 }

utils/networking.go

@@ -0,0 +1,17 @@
+package utils
+
+import (
+	"h12.me/socks"
+	"net"
+	"strconv"
+	"time"
+)
+
+func GetNetworkConnection(onionService string, port int, proxy string, timeout time.Duration) (net.Conn, error) {
+	portNumber := strconv.Itoa(port)
+	conn, err := socks.DialSocksProxy(socks.SOCKS5, proxy)("", onionService+":"+portNumber)
+	if err == nil {
+		conn.SetDeadline(time.Now().Add(timeout))
+	}
+	return conn, err
+}

utils/url_parsing.go

@@ -2,11 +2,23 @@ package utils
 
 import (
 	"github.com/mvdan/xurls"
+	"regexp"
 	"strings"
 )
 
 func ExtractDomains(content string) []string {
-	return xurls.Strict.FindAllString(content, -1)
+	domains := xurls.Strict.FindAllString(content, -1)
+	cssurlregex := regexp.MustCompile(`(?i)url\((.*?)\)`)
+	cssDomains := cssurlregex.FindAllString(content, -1)
+	for _, cssDomain := range cssDomains {
+		if strings.HasPrefix(strings.ToLower(cssDomain), "url(") {
+			cssDomain = cssDomain[4 : len(cssDomain)-1]
+		}
+		if !strings.HasSuffix(cssDomain, ":before") && !strings.HasSuffix(cssDomain, ":after") {
+			domains = append(domains, cssDomain)
+		}
+	}
+	return domains
 }
 
 func WithoutSubdomains(urlhost string) string {
@@ -25,5 +37,8 @@ func WithoutProtocol(url string) string {
 	if strings.HasPrefix(url, "https://") {
 		return url[8:]
 	}
+	if strings.HasPrefix(url, "//") {
+		return url[2:]
+	}
 	return url
 }