cleaning up, reporting

README.md

@@ -58,7 +58,7 @@ This [should not be news](http://arstechnica.com/security/2016/02/default-settin
 * Determine client IP addresses if you are co-hosting a clearnet site.
 * Determine your IP address if your setup allows.
 * Determine other sites you are co-hosting.
-* Determine how active your site is.
+* Determine how active your site it.
 * Find secret or hidden areas of your site
 * and much, much more.
 

config/onionscan_config.go

@@ -1,34 +1,15 @@
 package config
 
-import (
+import ()
-	"log"
-	"time"
-)
 
 type OnionscanConfig struct {
 	TorProxyAddress string
 	DirectoryDepth  int
-	Fingerprint     bool
-	Timeout         time.Duration
-	Verbose         bool
 }
 
-func Configure(torProxyAddress string, directoryDepth int, fingerprint bool, timeout int, verbose bool) *OnionscanConfig {
+func Configure(torProxyAddress string, directoryDepth int) *OnionscanConfig {
 	onionScan := new(OnionscanConfig)
 	onionScan.TorProxyAddress = torProxyAddress
 	onionScan.DirectoryDepth = directoryDepth
-	onionScan.Fingerprint = fingerprint
-	onionScan.Timeout = time.Duration(time.Second * time.Duration(timeout))
-	onionScan.Verbose = verbose
 	return onionScan
 }
-
-func (os *OnionscanConfig) LogInfo(message string) {
-	if os.Verbose {
-		log.Printf("INFO: %v", message)
-	}
-}
-
-func (os *OnionscanConfig) LogError(err error) {
-	log.Printf("ERROR: %v", err)
-}

main.go

@@ -1,7 +1,6 @@
 package main
 
 import (
-	"errors"
 	"flag"
 	"fmt"
 	"github.com/s-rah/onionscan/config"
@@ -9,96 +8,52 @@ import (
 	"io/ioutil"
 	"log"
 	"os"
-	"strings"
 )
 
 func main() {
 
 	flag.Usage = func() {
 		fmt.Printf("Usage of %s:\n", os.Args[0])
-		fmt.Printf("    onionscan [flags] hiddenservice | onionscan [flags] --list list\n")
+		fmt.Printf("    onionscan [flags] hiddenservice\n")
 		flag.PrintDefaults()
 	}
 
 	torProxyAddress := flag.String("torProxyAddress", "127.0.0.1:9050", "the address of the tor proxy to use")
 	simpleReport := flag.Bool("simpleReport", true, "print out a simple report detailing what is wrong and how to fix it, true by default")
-	reportFile := flag.String("reportFile", "", "the file destination path for report file - if given, the prefix of the file will be the scanned onion service. If not given, the report will be written to stdout")
+	reportFile := flag.String("reportFile", "", "the file destination path for report file")
 	jsonReport := flag.Bool("jsonReport", false, "print out a json report providing a detailed report of the scan.")
 	verbose := flag.Bool("verbose", false, "print out a verbose log output of the scan")
 	directoryDepth := flag.Int("depth", 100, "depth of directory scan recursion (default: 100)")
-	fingerprint := flag.Bool("fingerprint", true, "true disables some deeper scans e.g. directory probing with the aim of just getting a fingerprint of the service.")
-	list := flag.String("list", "", "If provided OnionScan will attempt to read from the given list, rather than the provided hidden service")
-	timeout := flag.Int("timeout", 120, "read timeout for connecting to onion services")
-	batch := flag.Int("batch", 10, "number of onions to scan concurrently")
 
 	flag.Parse()
 
-	if len(flag.Args()) != 1 && *list == "" {
+	if len(flag.Args()) != 1 {
 		flag.Usage()
 		os.Exit(1)
 	}
 
-	if !*simpleReport && !*jsonReport {
+	hiddenService := flag.Args()[0]
-		log.Fatalf("You must set one of --simpleReport or --jsonReport")
-	}
 
-	onionsToScan := []string{}
+	log.Printf("Starting Scan of %s\n", hiddenService)
-	if *list == "" {
-		onionsToScan = append(onionsToScan, flag.Args()[0])
-		log.Printf("Starting Scan of %s\n", flag.Args()[0])
-	} else {
-		content, err := ioutil.ReadFile(*list)
-		if err != nil {
-			log.Fatalf("Could not read onion file %s\n", *list)
-		}
-		onions := strings.Split(string(content), "\n")
-		for _, onion := range onions[0 : len(onions)-1] {
-			onionsToScan = append(onionsToScan, onion)
-		}
-		log.Printf("Starting Scan of %d onion services\n", len(onionsToScan))
-	}
 	log.Printf("This might take a few minutes..\n\n")
 
+	if !*verbose {
+		log.SetOutput(ioutil.Discard)
+	}
+
 	onionScan := new(OnionScan)
-	onionScan.Config = config.Configure(*torProxyAddress, *directoryDepth, *fingerprint, *timeout, *verbose)
+	onionScan.Config = config.Configure(*torProxyAddress, *directoryDepth)
+	scanReport, err := onionScan.Scan(hiddenService)
 
-	reports := make(chan *report.OnionScanReport)
+	if err != nil {
-
+		log.Fatalf("Error running scanner: %s", err)
-	count := 0
-	if *batch > len(onionsToScan) {
-		*batch = len(onionsToScan)
 	}
 
-	// Run an initial batch of 100 requests (or less...)
+	if *jsonReport {
-	for count < *batch {
+		report.GenerateJsonReport(*reportFile, scanReport)
-		go onionScan.Scan(onionsToScan[count], reports)
-		count++
 	}
 
-	received := 0
+	if *simpleReport {
-	for received < len(onionsToScan) {
+		report.GenerateSimpleReport(*reportFile, scanReport)
-		scanReport := <-reports
-
-		// After the initial batch, it's one in one out to prevent proxy overload.
-		if count < len(onionsToScan) {
-			go onionScan.Scan(onionsToScan[count], reports)
-			count++
-		}
-
-		received++
-		if scanReport.TimedOut {
-			onionScan.Config.LogError(errors.New(scanReport.HiddenService + " timed out"))
-		}
-
-		file := *reportFile
-		if file != "" {
-			file = scanReport.HiddenService + "." + *reportFile
-		}
-
-		if *jsonReport {
-			report.GenerateJsonReport(file, scanReport)
-		} else if *simpleReport {
-			report.GenerateSimpleReport(file, scanReport)
-		}
 	}
 }

onionscan.go

@@ -1,72 +1,20 @@
 package main
 
 import (
+	"errors"
+	"fmt"
 	"github.com/s-rah/onionscan/config"
 	"github.com/s-rah/onionscan/protocol"
 	"github.com/s-rah/onionscan/report"
 	"github.com/s-rah/onionscan/utils"
 	"strings"
-	"time"
 )
 
 type OnionScan struct {
 	Config *config.OnionscanConfig
 }
 
-func (os *OnionScan) PerformNextAction(report *report.OnionScanReport) {
+func (os *OnionScan) Scan(hiddenService string) (*report.OnionScanReport, error) {
-	switch report.NextAction {
-	case "web":
-		wps := new(protocol.HTTPProtocolScanner)
-		wps.ScanProtocol(report.HiddenService, os.Config, report)
-		report.NextAction = "tls"
-	case "tls":
-		tps := new(protocol.TLSProtocolScanner)
-		tps.ScanProtocol(report.HiddenService, os.Config, report)
-		report.NextAction = "ssh"
-	case "ssh":
-		sps := new(protocol.SSHProtocolScanner)
-		sps.ScanProtocol(report.HiddenService, os.Config, report)
-		report.NextAction = "irc"
-	case "irc":
-		ips := new(protocol.IRCProtocolScanner)
-		ips.ScanProtocol(report.HiddenService, os.Config, report)
-		report.NextAction = "ricochet"
-	case "ricochet":
-		rps := new(protocol.RicochetProtocolScanner)
-		rps.ScanProtocol(report.HiddenService, os.Config, report)
-		report.NextAction = "ftp"
-	case "ftp":
-		fps := new(protocol.FTPProtocolScanner)
-		fps.ScanProtocol(report.HiddenService, os.Config, report)
-		report.NextAction = "smtp"
-	case "smtp":
-		smps := new(protocol.SMTPProtocolScanner)
-		smps.ScanProtocol(report.HiddenService, os.Config, report)
-		report.NextAction = "mongodb"
-	case "mongodb":
-		mdbps := new(protocol.MongoDBProtocolScanner)
-		mdbps.ScanProtocol(report.HiddenService, os.Config, report)
-		report.NextAction = "vnc"
-	case "vnc":
-		vncps := new(protocol.VNCProtocolScanner)
-		vncps.ScanProtocol(report.HiddenService, os.Config, report)
-		report.NextAction = "xmpp"
-	case "xmpp":
-		xmppps := new(protocol.XMPPProtocolScanner)
-		xmppps.ScanProtocol(report.HiddenService, os.Config, report)
-		report.NextAction = "bitcoin"
-	case "bitcoin":
-		bps := new(protocol.BitcoinProtocolScanner)
-		bps.ScanProtocol(report.HiddenService, os.Config, report)
-		report.NextAction = "none"
-	case "none":
-		return
-	default:
-		report.NextAction = "web"
-	}
-}
-
-func (os *OnionScan) Scan(hiddenService string, out chan *report.OnionScanReport) {
 
 	// Remove Extra Prefix
 	hiddenService = utils.WithoutProtocol(hiddenService)
@@ -77,13 +25,42 @@ func (os *OnionScan) Scan(hiddenService string, out chan *report.OnionScanReport
 
 	report := report.NewOnionScanReport(hiddenService)
 
-	for report.NextAction != "none" {
+	// HTTP
-		os.PerformNextAction(report)
+	hps := new(protocol.HTTPProtocolScanner)
-		if time.Now().Sub(report.DateScanned).Seconds() > os.Config.Timeout.Seconds() {
+	hps.ScanProtocol(hiddenService, os.Config, report)
-			report.TimedOut = true
+
-			report.NextAction = "none"
+	// SSH
-		}
+	sps := new(protocol.SSHProtocolScanner)
+	sps.ScanProtocol(hiddenService, os.Config, report)
+
+	// Ricochet
+	rps := new(protocol.RicochetProtocolScanner)
+	rps.ScanProtocol(hiddenService, os.Config, report)
+
+	// Bitcoin
+	bps := new(protocol.BitcoinProtocolScanner)
+	bps.ScanProtocol(hiddenService, os.Config, report)
+
+	//IRC
+	ips := new(protocol.IRCProtocolScanner)
+	ips.ScanProtocol(hiddenService, os.Config, report)
+
+	//FTP
+	fps := new(protocol.FTPProtocolScanner)
+	fps.ScanProtocol(hiddenService, os.Config, report)
+
+	//SMTP
+	smps := new(protocol.SMTPProtocolScanner)
+	smps.ScanProtocol(hiddenService, os.Config, report)
+
+	//MongoDb
+	mdbps := new(protocol.MongoDBProtocolScanner)
+	mdbps.ScanProtocol(hiddenService, os.Config, report)
+
+	if !report.WebDetected && !report.SSHDetected && !report.RicochetDetected && !report.BitcoinDetected && !report.IRCDetected && !report.FTPDetected && !report.SMTPDetected && !report.MongoDBDetected {
+		fmt.Printf("Unable to connect to this Tor Hidden Service on any known protocol.\n")
+		return nil, errors.New("Unable to connect to this Tor Hidden Service on any known protocol.")
 	}
 
-	out <- report
+	return report, nil
 }

protocol/bitcoin_scanner.go

@@ -1,28 +1,26 @@
 package protocol
 
 import (
-	"fmt"
 	"github.com/s-rah/onionscan/config"
 	"github.com/s-rah/onionscan/report"
-	"github.com/s-rah/onionscan/utils"
+	"h12.me/socks"
+	"log"
 )
 
 type BitcoinProtocolScanner struct {
 }
 
-func (rps *BitcoinProtocolScanner) ScanProtocol(hiddenService string, osc *config.OnionscanConfig, report *report.OnionScanReport) {
+func (rps *BitcoinProtocolScanner) ScanProtocol(hiddenService string, onionscanConfig *config.OnionscanConfig, report *report.OnionScanReport) {
 	// Bitcoin
-	osc.LogInfo(fmt.Sprintf("Checking %s Bitcoin(8333)\n", hiddenService))
+	log.Printf("Checking %s Bitcoin(8333)\n", hiddenService)
-	conn, err := utils.GetNetworkConnection(hiddenService, 8333, osc.TorProxyAddress, osc.Timeout)
+	_, err := socks.DialSocksProxy(socks.SOCKS5, onionscanConfig.TorProxyAddress)("", hiddenService+":8333")
 	if err != nil {
-		osc.LogInfo("Failed to connect to service on port 8333\n")
+		log.Printf("Failed to connect to service on port 8333\n")
 		report.BitcoinDetected = false
 	} else {
-		osc.LogInfo("Detected possible Bitcoin instance\n")
+		log.Printf("Detected possible Bitcoin instance\n")
 		// TODO: Actual Analysis
 		report.BitcoinDetected = true
 	}
-	if conn != nil {
+
-		conn.Close()
-	}
 }

protocol/ftp_scanner.go

@@ -1,37 +1,25 @@
 package protocol
 
 import (
-	"bufio"
-	"crypto/sha1"
-	"encoding/hex"
-	"fmt"
 	"github.com/s-rah/onionscan/config"
 	"github.com/s-rah/onionscan/report"
-	"github.com/s-rah/onionscan/utils"
+	"h12.me/socks"
+	"log"
 )
 
 type FTPProtocolScanner struct {
 }
 
-func (sps *FTPProtocolScanner) ScanProtocol(hiddenService string, osc *config.OnionscanConfig, report *report.OnionScanReport) {
+func (sps *FTPProtocolScanner) ScanProtocol(hiddenService string, onionscanConfig *config.OnionscanConfig, report *report.OnionScanReport) {
 	// FTP
-	osc.LogInfo(fmt.Sprintf("Checking %s FTP(21)\n", hiddenService))
+	log.Printf("Checking %s FTP(22)\n", hiddenService)
-	conn, err := utils.GetNetworkConnection(hiddenService, 21, osc.TorProxyAddress, osc.Timeout)
+	_, err := socks.DialSocksProxy(socks.SOCKS5, onionscanConfig.TorProxyAddress)("", hiddenService+":21")
 	if err != nil {
-		osc.LogInfo("Failed to connect to service on port 21\n")
+		log.Printf("Failed to connect to service on port 21\n")
 		report.FTPDetected = false
 	} else {
+		// TODO FTP Checking
 		report.FTPDetected = true
-		reader := bufio.NewReader(conn)
-		banner, err := reader.ReadString('\n')
-		if err == nil {
-			report.FTPBanner = banner
-			hash := sha1.Sum([]byte(banner))
-			report.FTPFingerprint = hex.EncodeToString(hash[:])
-			osc.LogInfo(fmt.Sprintf("Found FTP Banner: %s (%s)", banner, report.FTPFingerprint))
-		}
-	}
-	if conn != nil {
-		conn.Close()
 	}
+
 }

protocol/http_scanner.go

@@ -1,14 +1,13 @@
 package protocol
 
 import (
-	"crypto/tls"
-	"fmt"
 	"github.com/s-rah/onionscan/config"
 	"github.com/s-rah/onionscan/report"
 	"github.com/s-rah/onionscan/scans"
 	"github.com/s-rah/onionscan/utils"
 	"h12.me/socks"
 	"io/ioutil"
+	"log"
 	"net/http"
 	"strings"
 )
@@ -25,86 +24,67 @@ var (
 		"/products", "/products/cat"}
 )
 
-func (hps *HTTPProtocolScanner) ScanProtocol(hiddenService string, osc *config.OnionscanConfig, report *report.OnionScanReport) {
+func (hps *HTTPProtocolScanner) ScanProtocol(hiddenService string, onionscanConfig *config.OnionscanConfig, report *report.OnionScanReport) {
 
 	// HTTP
-	osc.LogInfo(fmt.Sprintf("Checking %s http(80)\n", hiddenService))
+	log.Printf("Checking %s http(80)\n", hiddenService)
-	conn, err := utils.GetNetworkConnection(hiddenService, 80, osc.TorProxyAddress, osc.Timeout)
+	_, err := socks.DialSocksProxy(socks.SOCKS5, onionscanConfig.TorProxyAddress)("", hiddenService+":80")
 	if err != nil {
-		osc.LogInfo("Failed to connect to service on port 80\n")
+		log.Printf("Failed to connect to service on port 80\n")
 		report.WebDetected = false
-		if conn != nil {
-			conn.Close()
-		}
-	} else {
-		osc.LogInfo("Found potential service on http(80)\n")
-		report.WebDetected = true
-		conn.Close()
-		dialSocksProxy := socks.DialSocksProxy(socks.SOCKS5, osc.TorProxyAddress)
-		transportConfig := &http.Transport{
-			Dial:            dialSocksProxy,
-			TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
-		}
-		hps.Client = &http.Client{
-			Transport: transportConfig,
-		}
-		// FIXME This should probably be moved to it's own file now.
-		response, err := hps.Client.Get("http://" + hiddenService)
-		if err == nil {
-			// Reading all http headers
-			osc.LogInfo(fmt.Sprintf("HTTP response headers: %s\n", report.ServerVersion))
-			responseHeaders := response.Header
-			for key := range responseHeaders {
-				value := responseHeaders.Get(key)
-				// normalize by strings.ToUpper(key) to avoid case sensitive checking
-				report.AddResponseHeader(strings.ToUpper(key), value)
-			}
-
-			report.ServerVersion = responseHeaders.Get("Server")
-			response.Body.Close()
-		} else {
-			osc.LogError(err)
-		}
-
-		// Apache mod-status Check
-		hps.ScanPage(hiddenService, "/", report, osc, scans.StandardPageScan)
-		hps.ScanPage(hiddenService, "/server-status", report, osc, scans.ApacheModStatus)
-		hps.ScanPage(hiddenService, "/private_key", report, osc, scans.PrivateKeyScan)
-
-		if osc.Fingerprint == false {
-			osc.LogInfo("\tScanning Common and Referenced Directories\n")
-			directories := append(CommonDirectories, report.PageReferencedDirectories...)
-			utils.RemoveDuplicates(&directories)
-
-			for _, directory := range directories {
-				hps.ScanPage(hiddenService, directory, report, osc, scans.CheckDirectoryListing(osc.DirectoryDepth))
-			}
-		}
-	}
-}
-
-func (hps *HTTPProtocolScanner) ScanPage(hiddenService string, page string, report *report.OnionScanReport, osc *config.OnionscanConfig, f func(scans.Scanner, string, int, string, *report.OnionScanReport, *config.OnionscanConfig)) {
-	err, contents, responseCode := hps.ScrapePage(hiddenService, page)
-	if err == nil {
-		f(hps, page, responseCode, string(contents), report, osc)
 		return
 	} else {
-		osc.LogError(err)
+		log.Printf("Found potential service on http(80)\n")
+		report.WebDetected = true
+		dialSocksProxy := socks.DialSocksProxy(socks.SOCKS5, onionscanConfig.TorProxyAddress)
+		transportConfig := &http.Transport{
+			Dial: dialSocksProxy,
+		}
+		hps.Client = &http.Client{Transport: transportConfig}
+		// FIXME This should probably be moved to it's own file now.
+		response, err := hps.Client.Get("http://" + hiddenService)
+
+		if err != nil {
+			log.Printf("Failed to connect to service on port 80\n")
+			return
+		}
+
+		// Reading all http headers
+		log.Printf("HTTP response headers: %s\n", report.ServerVersion)
+		responseHeaders := response.Header
+		for key := range responseHeaders {
+			value := responseHeaders.Get(key)
+			// normalize by strings.ToUpper(key) to avoid case sensitive checking
+			report.AddResponseHeader(strings.ToUpper(key), value)
+			log.Printf("\t%s : %s\n", strings.ToUpper(key), value)
+		}
+
+		report.ServerVersion = responseHeaders.Get("Server")
+
+		// Apache mod-status Check
+		hps.ScanPage(hiddenService, "/server-status", report, scans.ApacheModStatus)
+		hps.ScanPage(hiddenService, "/", report, scans.StandardPageScan)
+
+		log.Printf("\tScanning Common and Referenced Directories\n")
+		directories := append(CommonDirectories, report.PageReferencedDirectories...)
+		utils.RemoveDuplicates(&directories)
+		for _, directory := range directories {
+			hps.ScanPage(hiddenService, directory, report, scans.CheckDirectoryListing(onionscanConfig.DirectoryDepth))
+		}
 	}
+	log.Printf("\n")
 }
 
-func (hps *HTTPProtocolScanner) ScrapePage(hiddenService string, page string) (error, []byte, int) {
+func (hps *HTTPProtocolScanner) ScanPage(hiddenService string, page string, report *report.OnionScanReport, f func(scans.Scanner, string, int, string, *report.OnionScanReport)) {
 	if !strings.Contains(page, utils.WithoutSubdomains(hiddenService)) {
-		if !strings.HasPrefix(page, "/") {
-			page = "/" + page
-		}
 		page = hiddenService + page
 	}
 	response, err := hps.Client.Get("http://" + page)
 	if err != nil {
-		return err, nil, -1
+		log.Printf("Error connecting to http://%s %s\n", page, err)
+		return
 	}
 	defer response.Body.Close()
 	contents, _ := ioutil.ReadAll(response.Body)
-	return nil, contents, response.StatusCode
+	f(hps, page, response.StatusCode, string(contents), report)
 }

protocol/irc_scanner.go

@@ -1,42 +1,26 @@
 package protocol
 
 import (
-	"fmt"
 	"github.com/s-rah/onionscan/config"
 	"github.com/s-rah/onionscan/report"
-	"github.com/s-rah/onionscan/utils"
+	"h12.me/socks"
+	"log"
 )
 
 type IRCProtocolScanner struct {
 }
 
-func (rps *IRCProtocolScanner) ScanProtocol(hiddenService string, osc *config.OnionscanConfig, report *report.OnionScanReport) {
+func (rps *IRCProtocolScanner) ScanProtocol(hiddenService string, onionscanConfig *config.OnionscanConfig, report *report.OnionScanReport) {
 	// IRC
-	osc.LogInfo(fmt.Sprintf("Checking %s IRC(6667)\n", hiddenService))
+	log.Printf("Checking %s IRC(6667)\n", hiddenService)
-	conn, err := utils.GetNetworkConnection(hiddenService, 6667, osc.TorProxyAddress, osc.Timeout)
+	_, err := socks.DialSocksProxy(socks.SOCKS5, onionscanConfig.TorProxyAddress)("", hiddenService+":6667")
 	if err != nil {
-		osc.LogInfo("Failed to connect to service on port 6667\n")
+		log.Printf("Failed to connect to service on port 6667\n")
 		report.IRCDetected = false
 	} else {
-		osc.LogInfo("Detected possible IRC instance\n")
+		log.Printf("Detected possible IRC instance\n")
 		// TODO: Actual Analysis
 		report.IRCDetected = true
 	}
-	if conn != nil {
-		conn.Close()
-	}
 
-	// IRC
-	osc.LogInfo(fmt.Sprintf("Checking %s IRC(6697)\n", hiddenService))
-	conn, err = utils.GetNetworkConnection(hiddenService, 6697, osc.TorProxyAddress, osc.Timeout)
-	if err != nil {
-		osc.LogInfo("Failed to connect to service on port 6697\n")
-	} else {
-		osc.LogInfo("Detected possible IRC (secure) instance\n")
-		// TODO: Actual Analysis
-		report.IRCDetected = true
-	}
-	if conn != nil {
-		conn.Close()
-	}
 }

protocol/mongodb_scanner.go

@@ -1,29 +1,26 @@
 package protocol
 
 import (
-	"fmt"
 	"github.com/s-rah/onionscan/config"
 	"github.com/s-rah/onionscan/report"
-	"github.com/s-rah/onionscan/utils"
+	"h12.me/socks"
+	"log"
 )
 
 type MongoDBProtocolScanner struct {
 }
 
-func (rps *MongoDBProtocolScanner) ScanProtocol(hiddenService string, osc *config.OnionscanConfig, report *report.OnionScanReport) {
+func (rps *MongoDBProtocolScanner) ScanProtocol(hiddenService string, onionscanConfig *config.OnionscanConfig, report *report.OnionScanReport) {
 	// MongoDB
-	osc.LogInfo(fmt.Sprintf("Checking %s MongoDB(27017)\n", hiddenService))
+	log.Printf("Checking %s MongoDB(27017)\n", hiddenService)
-	conn, err := utils.GetNetworkConnection(hiddenService, 27017, osc.TorProxyAddress, osc.Timeout)
+	_, err := socks.DialSocksProxy(socks.SOCKS5, onionscanConfig.TorProxyAddress)("", hiddenService+":27017")
 	if err != nil {
-		osc.LogInfo("Failed to connect to service on port 27017\n")
+		log.Printf("Failed to connect to service on port 27017\n")
 		report.MongoDBDetected = false
 	} else {
-		osc.LogInfo("Detected possible MongoDB instance\n")
+		log.Printf("Detected possible MongoDB instance\n")
 		// TODO: Actual Analysis
 		report.MongoDBDetected = true
 	}
-	if conn != nil {
-		conn.Close()
-	}
 
 }

protocol/ricochet_scanner.go

@@ -1,28 +1,26 @@
 package protocol
 
 import (
-	"fmt"
 	"github.com/s-rah/onionscan/config"
 	"github.com/s-rah/onionscan/report"
-	"github.com/s-rah/onionscan/utils"
+	"h12.me/socks"
+	"log"
 )
 
 type RicochetProtocolScanner struct {
 }
 
-func (rps *RicochetProtocolScanner) ScanProtocol(hiddenService string, osc *config.OnionscanConfig, report *report.OnionScanReport) {
+func (rps *RicochetProtocolScanner) ScanProtocol(hiddenService string, onionscanConfig *config.OnionscanConfig, report *report.OnionScanReport) {
 	// Ricochet
-	osc.LogInfo(fmt.Sprintf("Checking %s ricochet(9878)\n", hiddenService))
+	log.Printf("Checking %s ricochet(9878)\n", hiddenService)
-	conn, err := utils.GetNetworkConnection(hiddenService, 9878, osc.TorProxyAddress, osc.Timeout)
+	_, err := socks.DialSocksProxy(socks.SOCKS5, onionscanConfig.TorProxyAddress)("", hiddenService+":9878")
 	if err != nil {
-		osc.LogInfo("Failed to connect to service on port 9878\n")
+		log.Printf("Failed to connect to service on port 9878\n")
 		report.RicochetDetected = false
 	} else {
-		osc.LogInfo("Detected possible ricochet instance\n")
+		log.Printf("Detected possible ricochet instance\n")
 		// TODO: Actual Analysis
 		report.RicochetDetected = true
 	}
-	if conn != nil {
+
-		conn.Close()
-	}
 }

protocol/smtp_scanner.go

@@ -1,38 +1,25 @@
 package protocol
 
 import (
-	"bufio"
-	"crypto/sha1"
-	"encoding/hex"
-	"fmt"
 	"github.com/s-rah/onionscan/config"
 	"github.com/s-rah/onionscan/report"
-	"github.com/s-rah/onionscan/utils"
+	"h12.me/socks"
+	"log"
 )
 
 type SMTPProtocolScanner struct {
 }
 
-func (sps *SMTPProtocolScanner) ScanProtocol(hiddenService string, osc *config.OnionscanConfig, report *report.OnionScanReport) {
+func (sps *SMTPProtocolScanner) ScanProtocol(hiddenService string, onionscanConfig *config.OnionscanConfig, report *report.OnionScanReport) {
 	// SMTP
-	osc.LogInfo(fmt.Sprintf("Checking %s SMTP(25)\n", hiddenService))
+	log.Printf("Checking %s SMTP(25)\n", hiddenService)
-	conn, err := utils.GetNetworkConnection(hiddenService, 25, osc.TorProxyAddress, osc.Timeout)
+	_, err := socks.DialSocksProxy(socks.SOCKS5, onionscanConfig.TorProxyAddress)("", hiddenService+":25")
 	if err != nil {
-		osc.LogInfo("Failed to connect to service on port 25\n")
+		log.Printf("Failed to connect to service on port 25\n")
 		report.SMTPDetected = false
 	} else {
 		// TODO SMTP Checking
 		report.SMTPDetected = true
-		reader := bufio.NewReader(conn)
-		banner, err := reader.ReadString('\n')
-		if err == nil {
-			report.SMTPBanner = banner
-			hash := sha1.Sum([]byte(banner))
-			report.SMTPFingerprint = hex.EncodeToString(hash[:])
-			osc.LogInfo(fmt.Sprintf("Found SMTP Banner: %s (%s)", banner, report.SMTPFingerprint))
-		}
-	}
-	if conn != nil {
-		conn.Close()
 	}
+
 }

protocol/ssh_scanner.go

@@ -1,30 +1,27 @@
 package protocol
 
 import (
-	"bufio"
 	"crypto/md5"
 	"errors"
 	"fmt"
 	"github.com/s-rah/onionscan/config"
 	"github.com/s-rah/onionscan/report"
-	"github.com/s-rah/onionscan/utils"
 	"golang.org/x/crypto/ssh"
+	"h12.me/socks"
+	"log"
 	"net"
 )
 
 type SSHProtocolScanner struct {
 }
 
-func (sps *SSHProtocolScanner) ScanProtocol(hiddenService string, osc *config.OnionscanConfig, report *report.OnionScanReport) {
+func (sps *SSHProtocolScanner) ScanProtocol(hiddenService string, onionscanConfig *config.OnionscanConfig, report *report.OnionScanReport) {
 	// SSH
-	osc.LogInfo(fmt.Sprintf("Checking %s ssh(22)\n", hiddenService))
+	log.Printf("Checking %s ssh(22)\n", hiddenService)
-	conn, err := utils.GetNetworkConnection(hiddenService, 22, osc.TorProxyAddress, osc.Timeout)
+	conn, err := socks.DialSocksProxy(socks.SOCKS5, onionscanConfig.TorProxyAddress)("", hiddenService+":22")
 	if err != nil {
-		osc.LogInfo("Failed to connect to service on port 22\n")
+		log.Printf("Failed to connect to service on port 22\n")
 		report.SSHDetected = false
-		if conn != nil {
-			conn.Close()
-		}
 	} else {
 		// TODO SSH Checking
 		report.SSHDetected = true
@@ -44,26 +41,13 @@ func (sps *SSHProtocolScanner) ScanProtocol(hiddenService string, osc *config.On
 					}
 				}
 				report.SSHKey = fingerprint
-				osc.LogInfo(fmt.Sprintf("Found SSH Key %s\n", fingerprint))
+				log.Printf("Found SSH Key %s\n", fingerprint)
 				// We don't want to continue
 				return errors.New("error")
 			},
 		}
 		ssh.NewClientConn(conn, hiddenService+":22", config)
-		if conn != nil {
+
-			conn.Close()
-		}
-		conn, err = utils.GetNetworkConnection(hiddenService, 22, osc.TorProxyAddress, osc.Timeout)
-		if err == nil {
-			reader := bufio.NewReader(conn)
-			banner, err := reader.ReadString('\n')
-			if err == nil {
-				report.SSHBanner = banner
-				osc.LogInfo(fmt.Sprintf("Found SSH Banner: %s (%s)", banner))
-			}
-		}
-		if conn != nil {
-			conn.Close()
-		}
 	}
+
 }

protocol/tls_scanner.go

@@ -1,37 +0,0 @@
-package protocol
-
-import (
-	"crypto/tls"
-	"fmt"
-	"github.com/s-rah/onionscan/config"
-	"github.com/s-rah/onionscan/report"
-	"github.com/s-rah/onionscan/utils"
-)
-
-type TLSProtocolScanner struct {
-}
-
-func (sps *TLSProtocolScanner) ScanProtocol(hiddenService string, osc *config.OnionscanConfig, report *report.OnionScanReport) {
-	osc.LogInfo(fmt.Sprintf("Checking %s TLS(443)\n", hiddenService))
-	conn, err := utils.GetNetworkConnection(hiddenService, 443, osc.TorProxyAddress, osc.Timeout)
-	if err != nil {
-		osc.LogInfo("Failed to connect to service on port 443\n")
-		report.TLSDetected = false
-	} else {
-		osc.LogInfo("Found TLS Endpoint\n")
-		report.TLSDetected = true
-		config := &tls.Config{
-			InsecureSkipVerify: true,
-		}
-		tlsConn := tls.Client(conn, config)
-		tlsConn.Write([]byte("GET / HTTP/1.1\r\n\r\n"))
-		for _, certificate := range tlsConn.ConnectionState().PeerCertificates {
-			osc.LogInfo(fmt.Sprintf("Found Certificate %v \n", certificate))
-			report.Certificates = append(report.Certificates, *certificate)
-		}
-		tlsConn.Close()
-	}
-	if conn != nil {
-		conn.Close()
-	}
-}

protocol/vnc_scanner.go

@@ -1,28 +0,0 @@
-package protocol
-
-import (
-	"fmt"
-	"github.com/s-rah/onionscan/config"
-	"github.com/s-rah/onionscan/report"
-	"github.com/s-rah/onionscan/utils"
-)
-
-type VNCProtocolScanner struct {
-}
-
-func (vncps *VNCProtocolScanner) ScanProtocol(hiddenService string, osc *config.OnionscanConfig, report *report.OnionScanReport) {
-	// MongoDB
-	osc.LogInfo(fmt.Sprintf("Checking %s VNC(5900)\n", hiddenService))
-	conn, err := utils.GetNetworkConnection(hiddenService, 5900, osc.TorProxyAddress, osc.Timeout)
-	if err != nil {
-		osc.LogInfo("Failed to connect to service on port 5900\n")
-		report.VNCDetected = false
-	} else {
-		osc.LogInfo("Detected possible VNC instance\n")
-		// TODO: Actual Analysis
-		report.VNCDetected = true
-	}
-	if conn != nil {
-		conn.Close()
-	}
-}

protocol/xmpp_scanner.go

@@ -1,41 +0,0 @@
-package protocol
-
-import (
-	"fmt"
-	"github.com/s-rah/onionscan/config"
-	"github.com/s-rah/onionscan/report"
-	"github.com/s-rah/onionscan/utils"
-)
-
-type XMPPProtocolScanner struct {
-}
-
-func (rps *XMPPProtocolScanner) ScanProtocol(hiddenService string, osc *config.OnionscanConfig, report *report.OnionScanReport) {
-	// XMPP
-	osc.LogInfo(fmt.Sprintf("Checking %s XMPP(5222)\n", hiddenService))
-	conn, err := utils.GetNetworkConnection(hiddenService, 5222, osc.TorProxyAddress, osc.Timeout)
-	if err != nil {
-		osc.LogInfo("Failed to connect to service on port 5222\n")
-		report.XMPPDetected = false
-	} else {
-		osc.LogInfo("Detected possible XMPP instance\n")
-		// TODO: Actual Analysis
-		report.XMPPDetected = true
-	}
-	if conn != nil {
-		conn.Close()
-	}
-	// XMPP
-	osc.LogInfo(fmt.Sprintf("Checking %s XMPP(5223)\n", hiddenService))
-	conn, err = utils.GetNetworkConnection(hiddenService, 5223, osc.TorProxyAddress, osc.Timeout)
-	if err != nil {
-		osc.LogInfo("Failed to connect to service on port 5223\n")
-	} else {
-		osc.LogInfo("Detected possible XMPP (secure) instance\n")
-		// TODO: Actual Analysis
-		report.XMPPDetected = true
-	}
-	if conn != nil {
-		conn.Close()
-	}
-}

report/onionscanreport.go

@@ -1,11 +1,9 @@
 package report
 
 import (
-	"crypto/x509"
 	"encoding/json"
 	"github.com/s-rah/onionscan/utils"
 	"io/ioutil"
-	"time"
 )
 
 type ExifTag struct {
@@ -19,69 +17,42 @@ type ExifImage struct {
 }
 
 type PGPKey struct {
-	ArmoredKey  string `json:"armoredKey"`
+	ArmoredKey	string	`json:"armoredKey"`
-	Identity    string `json:"identity"`
+	Identity	string	`json:"identity"`
-	FingerPrint string `json:"fingerprint"`
+	FingerPrint	string	`json:"fingerprint"`
 }
 
 type OnionScanReport struct {
-	HiddenService string    `json:"hiddenService"`
+	WebDetected      bool `json:"webDetected"`
-	DateScanned   time.Time `json:"dateScanned"`
+	SSHDetected      bool `json:"sshDetected"`
+	RicochetDetected bool `json:"ricochetDetected"`
+	IRCDetected      bool `json:"ircDetected"`
+	FTPDetected      bool `json:"ftpDetected"`
+	SMTPDetected     bool `json:"smtpDetected"`
 
-	// Summary
+	BitcoinDetected bool `json:"bitcoinDetected"`
-	WebDetected        bool `json:"webDetected"`
+	MongoDBDetected bool `json:"mongodbDetected"`
-	TLSDetected        bool `json:"tlsDetected"`
-	SSHDetected        bool `json:"sshDetected"`
-	RicochetDetected   bool `json:"ricochetDetected"`
-	IRCDetected        bool `json:"ircDetected"`
-	FTPDetected        bool `json:"ftpDetected"`
-	SMTPDetected       bool `json:"smtpDetected"`
-	BitcoinDetected    bool `json:"bitcoinDetected"`
-	MongoDBDetected    bool `json:"mongodbDetected"`
-	VNCDetected        bool `json:"vncDetected"`
-	XMPPDetected       bool `json:"xmppDetected"`
-	SkynetDetected     bool `json:"skynetDetected"`
-	PrivateKeyDetected bool `json:"privateKeyDetected"`
 
-	// Web Specific
+	HiddenService             string      `json:"hiddenService"`
-	ServerPoweredBy           string            `json:"serverPoweredBy"`
+	ServerPoweredBy           string      `json:"serverPoweredBy"`
-	ServerVersion             string            `json:"serverVersion"`
+	ServerVersion             string      `json:"serverVersion"`
-	FoundApacheModStatus      bool              `json:"foundApacheModStatus"`
+	FoundApacheModStatus      bool        `json:"foundApacheModStatus"`
-	RelatedOnionServices      []string          `json:"relatedOnionServices"`
+	RelatedOnionServices      []string    `json:"relatedOnionServices"`
-	RelatedClearnetDomains    []string          `json:"relatedOnionDomains"`
+	RelatedClearnetDomains    []string    `json:"relatedOnionDomains"`
-	LinkedSites               []string          `json:"linkedSites"`
+	LinkedSites               []string    `json:"linkedSites"`
-	InternalPages             []string          `json:"internalPages"`
+	InternalPages             []string    `json:"InternalPages"`
-	IP                        []string          `json:"ipAddresses"`
+	IP                        []string    `json:"ipAddresses"`
-	OpenDirectories           []string          `json:"openDirectories"`
+	OpenDirectories           []string    `json:"openDirectories"`
-	ExifImages                []ExifImage       `json:"exifImages"`
+	ExifImages                []ExifImage `json:"exifImages"`
-	InterestingFiles          []string          `json:"interestingFiles"`
+	InterestingFiles          []string    `json:"interestingFiles"`
-	PageReferencedDirectories []string          `json:"pageReferencedDirectories"`
+	PageReferencedDirectories []string    `json:"pageReferencedDirectories"`
-	PGPKeys                   []PGPKey          `json:"pgpKeys"`
+	PGPKeys                   []PGPKey    `json:"pgpKeys"`
-	Hashes                    []string          `json:"hashes"`
-	Snapshot                  string            `json:"snapshot"`
-	PageTitle                 string            `json:"pageTitle"`
-	ResponseHeaders           map[string]string `json:"responseHeaders"`
 
-	// TLS
+	Hashes          []string          `json:"hashes"`
-	Certificates []x509.Certificate `json:"certificates"`
+	SSHKey          string            `json:"sshKey"`
-
+	Snapshot        string            `json:"snapshot"`
-	//Bitcoin
+	PageTitle       string            `json:"pageTitle"`
-	BitcoinAddresses []string `json:"bitcoinAddresses"`
+	ResponseHeaders map[string]string `json:"responseHeaders"`
-
-	// SSH
-	SSHKey    string `json:"sshKey"`
-	SSHBanner string `json:"sshBanner"`
-
-	// FTP
-	FTPFingerprint string `json:"ftpFingerprint"`
-	FTPBanner      string `json:"ftpBanner"`
-
-	// SMTP
-	SMTPFingerprint string `json:"smtpFingerprint"`
-	SMTPBanner      string `json:"smtpBanner"`
-
-	NextAction string `json:"lastAction"`
-	TimedOut   bool
 }
 
 func LoadReportFromFile(filename string) (OnionScanReport, error) {
@@ -95,11 +66,7 @@ func LoadReportFromFile(filename string) (OnionScanReport, error) {
 }
 
 func NewOnionScanReport(hiddenService string) *OnionScanReport {
-	report := new(OnionScanReport)
+	return &OnionScanReport{HiddenService: hiddenService, ResponseHeaders: make(map[string]string)}
-	report.HiddenService = hiddenService
-	report.ResponseHeaders = make(map[string]string)
-	report.DateScanned = time.Now()
-	return report
 }
 
 func (osr *OnionScanReport) AddOpenDirectory(dir string) {

report/report_generator.go

@@ -5,7 +5,6 @@ import (
 	"fmt"
 	"log"
 	"os"
-	"time"
 )
 
 func GenerateJsonReport(reportFile string, report *OnionScanReport) {
@@ -16,11 +15,9 @@ func GenerateJsonReport(reportFile string, report *OnionScanReport) {
 
 	if len(reportFile) > 0 {
 		f, err := os.Create(reportFile)
-
+		if err != nil {
-		for err != nil {
+			log.Fatalf("Cannot create report file: %s", err)
-			log.Printf("Cannot create report file: %s...trying again in 5 seconds...", err)
+			panic(err)
-			time.Sleep(time.Second * 5)
-			f, err = os.Create(reportFile)
 		}
 
 		defer f.Close()
@@ -176,11 +173,9 @@ func GenerateSimpleReport(reportFile string, report *OnionScanReport) {
 
 	if len(reportFile) > 0 {
 		f, err := os.Create(reportFile)
-
+		if err != nil {
-		for err != nil {
+			log.Fatalf("Cannot create report file: %s", err)
-			log.Printf("Cannot create report file: %s...trying again in 5 seconds...", err)
+			panic(err)
-			time.Sleep(time.Second * 5)
-			f, err = os.Create(reportFile)
 		}
 
 		defer f.Close()

scans/apache_mod_status.go

@@ -1,59 +1,58 @@
 package scans
 
 import (
-	"fmt"
-	"github.com/s-rah/onionscan/config"
 	"github.com/s-rah/onionscan/report"
 	"github.com/s-rah/onionscan/utils"
+	"log"
 	"regexp"
 	"strings"
 )
 
-func ApacheModStatus(scan Scanner, page string, status int, contents string, report *report.OnionScanReport, osc *config.OnionscanConfig) {
+func ApacheModStatus(scan Scanner, page string, status int, contents string, report *report.OnionScanReport) {
 	if status == 200 {
 		r := regexp.MustCompile(`Server Version: (.*)</dt>`)
 		serverVersion := r.FindStringSubmatch(string(contents))
 
 		// Check if this looks like a mod_status page. Sometimes sites simply load their index.
 		if len(serverVersion) > 1 {
-			osc.LogInfo("Detected Apache mod_status Exposed...\033[091mAlert!\033[0m\n")
+			log.Printf("Detected Apache mod_status Exposed...\033[091mAlert!\033[0m\n")
 			report.FoundApacheModStatus = true
 
-			osc.LogInfo(fmt.Sprintf("\t Using mod_status Server Version: %s\n", serverVersion[1]))
+			log.Printf("\t Using mod_status Server Version: %s\n", serverVersion[1])
 			report.ServerVersion = serverVersion[1]
 
 			// Check for co-hosted onion services.
-			osc.LogInfo("Scanning for Co-Hosted Onions\n")
+			log.Printf("Scanning for Co-Hosted Onions\n")
 			r = regexp.MustCompile(`[a-z0-9]+.onion(:[0-9]{0-5})?`)
 			foundServices := r.FindAllString(string(contents), -1)
 			utils.RemoveDuplicates(&foundServices)
 			for _, onion := range foundServices {
 				if onion != report.HiddenService {
-					osc.LogInfo(fmt.Sprintf("\t \033[091mAlert!\033[0m Found Co-Hosted Onions: %s\n", onion))
+					log.Printf("\t \033[091mAlert!\033[0m Found Co-Hosted Onions: %s\n", onion)
 					report.AddRelatedOnionService(onion)
 				}
 			}
 
 			// Check for co-hosted onion services.
-			osc.LogInfo("Scanning for Co-Hosted Clearnet Domains\n")
+			log.Printf("Scanning for Co-Hosted Clearnet Domains\n")
 			r = regexp.MustCompile(`>(([a-zA-Z]{1})|([a-zA-Z]{1}[a-zA-Z]{1})|([a-zA-Z]{1}[0-9]{1})|([0-9]{1}[a-zA-Z]{1})|([a-zA-Z0-9][a-zA-Z0-9-_]{1,61}[a-zA-Z0-9]))\.([a-zA-Z]{2,6}|[a-zA-Z0-9-]{2,30}\.[a-zA-Z]{2,3})`)
 			foundServices = r.FindAllString(string(contents), -1)
 			utils.RemoveDuplicates(&foundServices)
 			for _, domain := range foundServices {
 				if strings.Contains(domain, ".onion") == false {
-					osc.LogInfo(fmt.Sprintf("\t \033[091mAlert!\033[0m Found Co-Hosted Service: %s\n", domain[1:]))
+					log.Printf("\t \033[091mAlert!\033[0m Found Co-Hosted Service: %s\n", domain[1:])
 					report.AddRelatedClearnetDomain(domain[4:])
 				}
 			}
 
 			// Check for IP Addresses
-			osc.LogInfo("Scanning for IP Addresses (clearweb clients, and servers)\n")
+			log.Printf("Scanning for IP Addresses (clearweb clients, and servers)\n")
 			r = regexp.MustCompile(`(?:(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.){3}(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)`)
 			foundIPs := r.FindAllString(string(contents), -1)
 			utils.RemoveDuplicates(&foundIPs)
 			for _, ip := range foundIPs {
 				if ip != "127.0.0.1" {
-					osc.LogInfo(fmt.Sprintf("\t \033[091mAlert!\033[0m Found IP Address : %s\n", ip))
+					log.Printf("\t \033[091mAlert!\033[0m Found IP Address : %s\n", ip)
 					report.AddIPAddress(ip)
 				}
 			}
@@ -61,7 +60,7 @@ func ApacheModStatus(scan Scanner, page string, status int, contents string, rep
 		}
 	}
 	if !report.FoundApacheModStatus {
-		osc.LogInfo("\tApache mod_status Not Exposed...\033[92mGood!\033[0m\n")
+		log.Printf("\tApache mod_status Not Exposed...\033[92mGood!\033[0m\n")
 		report.FoundApacheModStatus = false
 	}
 }

scans/bitcoin_content_scan.go

@@ -1,20 +0,0 @@
-package scans
-
-import (
-	"github.com/s-rah/onionscan/report"
-	"log"
-	"regexp"
-)
-
-type BitcoinContentScan struct {
-}
-
-func (cs *BitcoinContentScan) ScanContent(content string, report *report.OnionScanReport) {
-	log.Printf("Scanning for Bitcoin Address\n")
-	bitcoinAddressRegexp := regexp.MustCompile("[1|3][A-Za-z0-9]{25,34}")
-	foundBitcoinAddress := bitcoinAddressRegexp.FindAllString(content, -1)
-	for _, ba := range foundBitcoinAddress {
-		log.Printf("Found Bitcoin Address: %s", ba)
-		report.BitcoinAddresses = append(report.BitcoinAddresses, ba)
-	}
-}

scans/check_directory_listing.go

@@ -1,36 +1,35 @@
 package scans
 
 import (
-	"fmt"
-	"github.com/s-rah/onionscan/config"
 	"github.com/s-rah/onionscan/report"
+	"log"
 	"regexp"
 	"strings"
 )
 
-func CheckDirectoryListing(depth int) func(Scanner, string, int, string, *report.OnionScanReport, *config.OnionscanConfig) {
+func CheckDirectoryListing(depth int) func(Scanner, string, int, string, *report.OnionScanReport) {
-	return func(scan Scanner, dir string, status int, contents string, report *report.OnionScanReport, osc *config.OnionscanConfig) {
+	return func(scan Scanner, dir string, status int, contents string, report *report.OnionScanReport) {
-		CheckDirectoryListingDepth(scan, dir, status, depth, contents, report, osc)
+		CheckDirectoryListingDepth(scan, dir, status, depth, contents, report)
 	}
 }
 
-func CheckDirectoryListingDepth(scan Scanner, dir string, status int, depth int, contents string, report *report.OnionScanReport, osc *config.OnionscanConfig) {
+func CheckDirectoryListingDepth(scan Scanner, dir string, status int, depth int, contents string, report *report.OnionScanReport) {
 	if status == 200 && strings.Contains(string(contents), "Index of "+dir) {
-		osc.LogInfo(fmt.Sprintf("Detected Open Directory %s...\033[091mAlert!\033[0m\n", dir))
+		log.Printf("Detected Open Directory %s...\033[091mAlert!\033[0m\n", dir)
 
 		report.AddOpenDirectory(dir)
 
 		r := regexp.MustCompile(`href="((.*?\.jpg)|(.*?\.png)|(.*?\.jpeg)|(.*?\.gif))"`)
 		foundImages := r.FindAllStringSubmatch(string(contents), -1)
 		for _, image := range foundImages {
-			osc.LogInfo(fmt.Sprintf("\t Found image %s/%s\n", dir, image[1]))
+			log.Printf("\t Found image %s/%s\n", dir, image[1])
-			scan.ScanPage(report.HiddenService, dir+"/"+image[1], report, osc, CheckExif)
+			scan.ScanPage(report.HiddenService, dir+"/"+image[1], report, CheckExif)
 		}
 
 		r = regexp.MustCompile(`href="((.*\.zip)|(.*\.tar)|(.*\.gz)|(.*\.pst)|(.*\.txt))"`)
 		interestingFiles := r.FindAllStringSubmatch(string(contents), -1)
 		for _, file := range interestingFiles {
-			osc.LogInfo(fmt.Sprintf("\t Found interesting file %s/%s\n", dir, file[1]))
+			log.Printf("\t Found interesting file %s/%s\n", dir, file[1])
 			//TODO: We can do further analysis here, for now, just report them.
 			report.AddInterestingFile(dir + "/" + file[1])
 		}
@@ -38,14 +37,14 @@ func CheckDirectoryListingDepth(scan Scanner, dir string, status int, depth int,
 		r = regexp.MustCompile(`href="([^/](.*?))/"`)
 		subDir := r.FindAllStringSubmatch(string(contents), -1)
 		for _, file := range subDir {
-			osc.LogInfo(fmt.Sprintf("\t Found subdir %s/%s\n", dir, file[1]))
+			log.Printf("\t Found subdir %s/%s\n", dir, file[1])
 			//TODO: We can do further analysis here, for now, just report them.
 			if depth > 0 {
-				scan.ScanPage(report.HiddenService, dir+"/"+file[1], report, osc, CheckDirectoryListing(depth-1))
+				scan.ScanPage(report.HiddenService, dir+"/"+file[1], report, CheckDirectoryListing(depth-1))
 			}
 		}
 
 	} else {
-		osc.LogInfo(fmt.Sprintf("Directory %s either doesn't exist or is not readable\n", dir))
+		log.Printf("Directory %s either doesn't exist or is not readable\n", dir)
 	}
 }

scans/check_exif.go

@@ -1,15 +1,14 @@
 package scans
 
 import (
-	"fmt"
-	"github.com/s-rah/onionscan/config"
 	"github.com/s-rah/onionscan/report"
 	"github.com/xiam/exif"
 	"io"
+	"log"
 	"strings"
 )
 
-func CheckExif(scan Scanner, page string, status int, contents string, report *report.OnionScanReport, osc *config.OnionscanConfig) {
+func CheckExif(scan Scanner, page string, status int, contents string, report *report.OnionScanReport) {
 	if status == 200 {
 		reader := exif.New()
 		_, err := io.Copy(reader, strings.NewReader(contents))
@@ -32,7 +31,7 @@ func CheckExif(scan Scanner, page string, status int, contents string, report *r
 			report.AddExifImage(page)
 
 			for name, val := range reader.Tags {
-				osc.LogInfo(fmt.Sprintf("\t \033[091mAlert!\033[0m Found Exif Tag%s: %s\n", name, val))
+				log.Printf("\t \033[091mAlert!\033[0m Found Exif Tag%s: %s\n", name, val)
 				report.AddExifTag(name, val)
 			}
 		}

scans/pgp_content_scan.go

@@ -3,34 +3,38 @@ package scans
 import (
 	"github.com/s-rah/onionscan/report"
 	"golang.org/x/crypto/openpgp"
+	"log"
 	"regexp"
 	"strings"
+	"fmt"
 )
 
 type PGPContentScan struct {
 }
 
 func (cs *PGPContentScan) ScanContent(content string, report *report.OnionScanReport) {
-	//log.Printf("Scanning for PGP Key\n")
+	log.Printf("Scanning for PGP Key\n")
 	pgpRegexp := regexp.MustCompile("-----BEGIN PGP PUBLIC KEY BLOCK-----((?s).*)-----END PGP PUBLIC KEY BLOCK-----")
 	foundPGP := pgpRegexp.FindAllString(content, -1)
 	for _, keyString := range foundPGP {
-		keys, err := openpgp.ReadArmoredKeyRing(strings.NewReader(keyString))
+		keys, err := openpgp.ReadArmoredKeyRing(strings.NewReader(keyString));
 		if err != nil {
-			//		log.Printf("ERROR: %s\n", err)
+			log.Printf("ERROR: %s\n", err)
 			continue
 		}
-		if len(keys) < 1 || len(keys[0].Subkeys) < 1 || len(keys[0].Identities) < 1 {
+		if len(keys) < 1 || len(keys[0].Subkeys) < 1 || len(keys[0].Identities) < 1{
-			//		log.Printf("ERROR: failed to accept key\n")
+			log.Printf("ERROR: failed to accept key\n")
 			continue
 		}
 
 		var identity string
-		for identity = range keys[0].Identities {
+		for identity, _ = range keys[0].Identities {
 			break
 		}
-		//	log.Printf("\tFound PGP Key fingerprint: %s belonging to %s", keys[0].Subkeys[0].PublicKey.KeyIdShortString(), identity)
+		var fingerprint string
+		fingerprint = fmt.Sprintf("%X", keys[0].Subkeys[0].PublicKey.Fingerprint)
+		log.Printf("\tFound PGP Key fingerprint: %s belonging to %s", fingerprint, identity)
 
-		report.AddPGPKey(keyString, identity, keys[0].Subkeys[0].PublicKey.KeyIdShortString())
+		report.AddPGPKey(keyString, identity, fingerprint)
 	}
 }

scans/private_key.go

@@ -1,15 +0,0 @@
-package scans
-
-import (
-	"fmt"
-	"github.com/s-rah/onionscan/config"
-	"github.com/s-rah/onionscan/report"
-)
-
-func PrivateKeyScan(scan Scanner, page string, status int, contents string, report *report.OnionScanReport, osc *config.OnionscanConfig) {
-	osc.LogInfo(fmt.Sprintf("Scanning %s\n", page))
-	if status == 200 {
-		osc.LogInfo(fmt.Sprintf("\tPrivate Key %s is Accessible!!\n", page))
-		report.PrivateKeyDetected = true
-	}
-}

scans/scanner.go

@@ -1,11 +1,9 @@
 package scans
 
 import (
-	"github.com/s-rah/onionscan/config"
 	"github.com/s-rah/onionscan/report"
 )
 
 type Scanner interface {
-	ScanPage(string, string, *report.OnionScanReport, *config.OnionscanConfig, func(Scanner, string, int, string, *report.OnionScanReport, *config.OnionscanConfig))
+	ScanPage(string, string, *report.OnionScanReport, func(Scanner, string, int, string, *report.OnionScanReport))
-	ScrapePage(string, string) (error, []byte, int)
 }

scans/standard-page-scan.go

@@ -3,20 +3,19 @@ package scans
 import (
 	"crypto/sha1"
 	"encoding/hex"
-	"fmt"
-	"github.com/s-rah/onionscan/config"
 	"github.com/s-rah/onionscan/report"
 	"github.com/s-rah/onionscan/utils"
 	"golang.org/x/net/html"
+	"log"
 	"net/url"
 	"regexp"
 	"strings"
 )
 
-func StandardPageScan(scan Scanner, page string, status int, contents string, report *report.OnionScanReport, osc *config.OnionscanConfig) {
+func StandardPageScan(scan Scanner, page string, status int, contents string, report *report.OnionScanReport) {
-	osc.LogInfo(fmt.Sprintf("Scanning %s\n", page))
+	log.Printf("Scanning %s\n", page)
 	if status == 200 {
-		osc.LogInfo(fmt.Sprintf("\tPage %s is Accessible\n", page))
+		log.Printf("\tPage %s is Accessible\n", page)
 
 		hash := sha1.Sum([]byte(contents))
 		report.Hashes = append(report.Hashes, hex.EncodeToString(hash[:]))
@@ -28,16 +27,14 @@ func StandardPageScan(scan Scanner, page string, status int, contents string, re
 			var startIndex = strings.Index(contents, "<title>")
 			var endIndex = strings.Index(contents, "</title>")
 			var pageTitle = contents[startIndex+len("<title>") : endIndex]
-			osc.LogInfo(fmt.Sprintf("\tPage Title: %s\n", pageTitle))
+			log.Printf("\tPage Title: %s\n", pageTitle)
 			report.PageTitle = pageTitle
 		}
 
 		new(PGPContentScan).ScanContent(contents, report)
-		//new(BitcoinContentScan).ScanContent(contents, report)
 
-		osc.LogInfo("\tScanning for Images\n")
+		log.Printf("\tScanning for Images\n")
-		var domains []string
+		domains := utils.ExtractDomains(contents)
-		var cssLinks []string
 
 		// parser based on http://schier.co/blog/2015/04/26/a-simple-web-scraper-in-go.html
 		z := html.NewTokenizer(strings.NewReader(contents))
@@ -52,8 +49,8 @@ func StandardPageScan(scan Scanner, page string, status int, contents string, re
 			// TODO: don't crawl links with nofollow
 
 			if tt == html.StartTagToken {
-				// links
+				isLink := t.Data == "a"
-				if t.Data == "a" {
+				if isLink {
 					linkUrl := utils.GetAttribute(t, "href")
 					if len(linkUrl) > 1 {
 						domains = append(domains, linkUrl)
@@ -61,74 +58,56 @@ func StandardPageScan(scan Scanner, page string, status int, contents string, re
 				}
 			}
 
-			// css <link>
+			isImage := t.Data == "img"
-			if t.Data == "link" && utils.GetAttribute(t, "rel") == "stylesheet" {
+			if isImage {
-				cssLinks = append(cssLinks, utils.GetAttribute(t, "href"))
-			}
-
-			// images
-			if t.Data == "img" {
 				imageUrl := utils.GetAttribute(t, "src")
 
-				baseUrl, err := url.Parse(imageUrl)
+				baseUrl, _ := url.Parse(imageUrl)
-				if err == nil {
+				if utils.WithoutSubdomains(baseUrl.Host) == utils.WithoutSubdomains(report.HiddenService) {
-					if utils.WithoutSubdomains(baseUrl.Host) == utils.WithoutSubdomains(report.HiddenService) {
+					scan.ScanPage(report.HiddenService, utils.WithoutProtocol(imageUrl), report, CheckExif)
-						scan.ScanPage(report.HiddenService, utils.WithoutProtocol(imageUrl), report, osc, CheckExif)
+					log.Printf("\t Found internal image %s\n", imageUrl)
-						osc.LogInfo(fmt.Sprintf("\t Found internal image %s\n", imageUrl))
-					} else {
-						osc.LogInfo(fmt.Sprintf("\t Not scanning remote image %s\n", imageUrl))
-					}
-				}
-			}
-		}
-
-		osc.LogInfo("\tScanning for CSS Fonts and Background Images\n")
-		utils.RemoveDuplicates(&cssLinks)
-		for _, cssUrl := range cssLinks {
-			osc.LogInfo(fmt.Sprintf("\tScanning CSS file: %s\n", cssUrl))
-			_, cssContents, _ := scan.ScrapePage(report.HiddenService, utils.WithoutProtocol(cssUrl))
-			domains = append(domains, utils.ExtractDomains(string(cssContents))[:]...)
-		}
-
-		osc.LogInfo("\tScanning for Links\n")
-		domains = append(domains, utils.ExtractDomains(contents)...)
-		utils.RemoveDuplicates(&domains)
-		for _, domain := range domains {
-			baseUrl, err := url.Parse(domain)
-			if err == nil {
-				if baseUrl.Host != "" && utils.WithoutSubdomains(baseUrl.Host) != utils.WithoutSubdomains(report.HiddenService) {
-					osc.LogInfo(fmt.Sprintf("Found Related URL %s\n", domain))
-					// TODO: Lots of information here which needs to be processed.
-					// * Links to standard sites - google / bitpay etc.
-					// * Links to other onion sites
-					// * Links to obscure clearnet sites.
-					report.AddLinkedSite(baseUrl.Host)
 				} else {
-					// * Process FQDN internal links
+					log.Printf("\t Not scanning remote image %s\n", imageUrl)
-					osc.LogInfo(fmt.Sprintf("Found Internal URL %s\n", domain))
-					report.AddInternalPage(baseUrl.Host)
 				}
 			}
 		}
 
-		osc.LogInfo("\tScanning for Referenced Directories\n")
+		log.Printf("\tScanning for Links\n")
+
+		for _, domain := range domains {
+			baseUrl, _ := url.Parse(domain)
+			if baseUrl.Host != "" && utils.WithoutSubdomains(baseUrl.Host) != utils.WithoutSubdomains(report.HiddenService) {
+				log.Printf("Found Related URL %s\n", domain)
+				// TODO: Lots of information here which needs to be processed.
+				// * Links to standard sites - google / bitpay etc.
+				// * Links to other onion sites
+				// * Links to obscure clearnet sites.
+				report.AddLinkedSite(baseUrl.Host)
+			} else {
+				// * Process FQDN internal links
+				log.Printf("Found Internal URL %s\n", domain)
+				report.AddInternalPage(baseUrl.Host)
+			}
+		}
+
+		log.Printf("\tScanning for Referenced Directories\n")
 		r := regexp.MustCompile("(src|href)=\"([^\"]*)\"")
 		foundPaths := r.FindAllStringSubmatch(string(contents), -1)
 		for _, regexpResults := range foundPaths {
 			path := regexpResults[2]
-			if (strings.HasPrefix(path, "http") || strings.HasPrefix(path, "//")) && !strings.Contains(path, utils.WithoutSubdomains(report.HiddenService)) {
+			if strings.HasPrefix(path, "http") && !strings.Contains(path, utils.WithoutSubdomains(report.HiddenService)) {
 				continue
 			}
 
 			term := strings.LastIndex(path, "/")
 			if term > 0 {
-				osc.LogInfo(fmt.Sprintf("\t Found Referenced Directory %s\n", path[:term]))
+				log.Printf("\t Found Referenced Directory %s\n", path[:term])
 				report.AddPageReferencedDirectory(utils.WithoutProtocol(path[:term]))
 			}
 		}
 	} else if status == 403 {
-		osc.LogInfo(fmt.Sprintf("\tPage %s%s is Forbidden\n", report.HiddenService, page))
+		log.Printf("\tPage %s%s is Forbidden\n", report.HiddenService, page)
 	} else if status == 404 {
-		osc.LogInfo(fmt.Sprintf("\tPage %s%s is Does Not Exist\n", report.HiddenService, page))
+		log.Printf("\tPage %s%s is Does Not Exist\n", report.HiddenService, page)
 	}
 }

utils/networking.go

@@ -1,17 +0,0 @@
-package utils
-
-import (
-	"h12.me/socks"
-	"net"
-	"strconv"
-	"time"
-)
-
-func GetNetworkConnection(onionService string, port int, proxy string, timeout time.Duration) (net.Conn, error) {
-	portNumber := strconv.Itoa(port)
-	conn, err := socks.DialSocksProxy(socks.SOCKS5, proxy)("", onionService+":"+portNumber)
-	if err == nil {
-		conn.SetDeadline(time.Now().Add(timeout))
-	}
-	return conn, err
-}

utils/url_parsing.go

@@ -2,23 +2,11 @@ package utils
 
 import (
 	"github.com/mvdan/xurls"
-	"regexp"
 	"strings"
 )
 
 func ExtractDomains(content string) []string {
-	domains := xurls.Strict.FindAllString(content, -1)
+	return xurls.Strict.FindAllString(content, -1)
-	cssurlregex := regexp.MustCompile(`(?i)url\((.*?)\)`)
-	cssDomains := cssurlregex.FindAllString(content, -1)
-	for _, cssDomain := range cssDomains {
-		if strings.HasPrefix(strings.ToLower(cssDomain), "url(") {
-			cssDomain = cssDomain[4 : len(cssDomain)-1]
-		}
-		if !strings.HasSuffix(cssDomain, ":before") && !strings.HasSuffix(cssDomain, ":after") {
-			domains = append(domains, cssDomain)
-		}
-	}
-	return domains
 }
 
 func WithoutSubdomains(urlhost string) string {
@@ -37,8 +25,5 @@ func WithoutProtocol(url string) string {
 	if strings.HasPrefix(url, "https://") {
 		return url[8:]
 	}
-	if strings.HasPrefix(url, "//") {
-		return url[2:]
-	}
 	return url
 }