Load Onions from a File

Fixes #21 - Also enforces timeouts and cleans up
some code.

config/onionscan_config.go

@@ -1,15 +1,21 @@
 package config
 
+import (
+	"time"
+)
+
 type OnionscanConfig struct {
 	TorProxyAddress string
 	DirectoryDepth  int
 	Fingerprint     bool
+	Timeout         time.Duration
 }
 
-func Configure(torProxyAddress string, directoryDepth int, fingerprint bool) *OnionscanConfig {
+func Configure(torProxyAddress string, directoryDepth int, fingerprint bool, timeout int) *OnionscanConfig {
 	onionScan := new(OnionscanConfig)
 	onionScan.TorProxyAddress = torProxyAddress
 	onionScan.DirectoryDepth = directoryDepth
 	onionScan.Fingerprint = fingerprint
+	onionScan.Timeout = time.Duration(time.Second * time.Duration(timeout))
 	return onionScan
 }

main.go

@@ -8,54 +8,88 @@ import (
 	"io/ioutil"
 	"log"
 	"os"
+	"strings"
 )
 
 func main() {
 
 	flag.Usage = func() {
 		fmt.Printf("Usage of %s:\n", os.Args[0])
-		fmt.Printf("    onionscan [flags] hiddenservice\n")
+		fmt.Printf("    onionscan [flags] hiddenservice | onionscan [flags] --list list\n")
 		flag.PrintDefaults()
 	}
 
 	torProxyAddress := flag.String("torProxyAddress", "127.0.0.1:9050", "the address of the tor proxy to use")
 	simpleReport := flag.Bool("simpleReport", true, "print out a simple report detailing what is wrong and how to fix it, true by default")
-	reportFile := flag.String("reportFile", "", "the file destination path for report file")
+	reportFile := flag.String("reportFile", "", "the file destination path for report file - if given, the prefix of the file will be the scanned onion service. If not given, the report will be written to stdout")
 	jsonReport := flag.Bool("jsonReport", false, "print out a json report providing a detailed report of the scan.")
 	verbose := flag.Bool("verbose", false, "print out a verbose log output of the scan")
 	directoryDepth := flag.Int("depth", 100, "depth of directory scan recursion (default: 100)")
-	fingerprint := flag.Bool("fingerprint", true, "whether to conduct a full scan, or just fingerprint possible ports")
+	fingerprint := flag.Bool("fingerprint", true, "true disables some deeper scans e.g. directory probing with the aim of just getting a fingerprint of the service.")
+	list := flag.String("list", "", "If provided OnionScan will attempt to read from the given list, rather than the provided hidden service")
+	timeout := flag.Int("timeout", 120, "read timeout for connecting to onion services")
 
 	flag.Parse()
 
-	if len(flag.Args()) != 1 {
+	if len(flag.Args()) != 1 && *list == "" {
 		flag.Usage()
 		os.Exit(1)
 	}
 
-	hiddenService := flag.Args()[0]
+	if !*simpleReport && !*jsonReport {
+		log.Fatalf("You must set one of --simpleReport or --jsonReport")
+	}
 
-	log.Printf("Starting Scan of %s\n", hiddenService)
+	onionsToScan := []string{}
+	if *list == "" {
+		onionsToScan = append(onionsToScan, flag.Args()[0])
+		log.Printf("Starting Scan of %s\n", flag.Args()[0])
+	} else {
+		content, err := ioutil.ReadFile(*list)
+		if err != nil {
+			log.Fatalf("Could not read onion file %s\n", *list)
+		}
+		onions := strings.Split(string(content), "\n")
+		onionsToScan = append(onionsToScan, onions...)
+		log.Printf("Starting Scan of %d onion services\n", len(onionsToScan)-1)
+	}
 	log.Printf("This might take a few minutes..\n\n")
 
+	onionScan := new(OnionScan)
+	onionScan.Config = config.Configure(*torProxyAddress, *directoryDepth, *fingerprint, *timeout)
+
+	reports := make(chan *report.OnionScanReport)
+
 	if !*verbose {
 		log.SetOutput(ioutil.Discard)
 	}
 
-	onionScan := new(OnionScan)
-	onionScan.Config = config.Configure(*torProxyAddress, *directoryDepth, *fingerprint)
-	scanReport, err := onionScan.Scan(hiddenService)
-
-	if *jsonReport {
-		report.GenerateJsonReport(*reportFile, scanReport)
+	count := 0
+	max := 100
+	if max > len(onionsToScan)-1 {
+		max = len(onionsToScan) - 1
 	}
 
-	if *simpleReport {
-		report.GenerateSimpleReport(*reportFile, scanReport)
+	// Run an initial batch of 100 requests (or less...)
+	for count < max {
+		go onionScan.Scan(onionsToScan[count], reports)
+		count++
 	}
 
-	if !*jsonReport && err != nil {
-		log.Fatalf("Error running scanner: %s", err)
-	}
+	received := 0
+	for received < len(onionsToScan)-1 {
+		scanReport := <-reports
+		// After the initial batch, it's one in one out to prevent proxy overload.
+		if count < len(onionsToScan)-1 {
+			go onionScan.Scan(onionsToScan[count], reports)
+			count++
+		}
+		received++
 
+		if *jsonReport {
+			report.GenerateJsonReport(*reportFile, scanReport)
+		} else if *simpleReport {
+			report.GenerateSimpleReport(*reportFile, scanReport)
+		}
+	}
 }

onionscan.go

@@ -1,12 +1,10 @@
 package main
 
 import (
-	"errors"
 	"github.com/s-rah/onionscan/config"
 	"github.com/s-rah/onionscan/protocol"
 	"github.com/s-rah/onionscan/report"
 	"github.com/s-rah/onionscan/utils"
-	"log"
 	"strings"
 )
 
@@ -14,7 +12,7 @@ type OnionScan struct {
 	Config *config.OnionscanConfig
 }
 
-func (os *OnionScan) Scan(hiddenService string) (*report.OnionScanReport, error) {
+func (os *OnionScan) Scan(hiddenService string, out chan *report.OnionScanReport) {
 
 	// Remove Extra Prefix
 	hiddenService = utils.WithoutProtocol(hiddenService)
@@ -61,10 +59,5 @@ func (os *OnionScan) Scan(hiddenService string) (*report.OnionScanReport, error)
 	vncps := new(protocol.VNCProtocolScanner)
 	vncps.ScanProtocol(hiddenService, os.Config, report)
 
-	if !report.WebDetected && !report.SSHDetected && !report.RicochetDetected && !report.BitcoinDetected && !report.IRCDetected && !report.FTPDetected && !report.SMTPDetected && !report.MongoDBDetected {
-		log.Printf("Unable to connect to this Tor Hidden Service on any known protocol.\n")
-		return nil, errors.New("Unable to connect to this Tor Hidden Service on any known protocol.")
-	}
-
-	return report, nil
+	out <- report
 }

protocol/bitcoin_scanner.go

@@ -3,7 +3,7 @@ package protocol
 import (
 	"github.com/s-rah/onionscan/config"
 	"github.com/s-rah/onionscan/report"
-	"h12.me/socks"
+	"github.com/s-rah/onionscan/utils"
 	"log"
 )
 
@@ -13,7 +13,7 @@ type BitcoinProtocolScanner struct {
 func (rps *BitcoinProtocolScanner) ScanProtocol(hiddenService string, onionscanConfig *config.OnionscanConfig, report *report.OnionScanReport) {
 	// Bitcoin
 	log.Printf("Checking %s Bitcoin(8333)\n", hiddenService)
-	_, err := socks.DialSocksProxy(socks.SOCKS5, onionscanConfig.TorProxyAddress)("", hiddenService+":8333")
+	_, err := utils.GetNetworkConnection(hiddenService, 8333, onionscanConfig.TorProxyAddress, onionscanConfig.Timeout)
 	if err != nil {
 		log.Printf("Failed to connect to service on port 8333\n")
 		report.BitcoinDetected = false

protocol/ftp_scanner.go

@@ -6,7 +6,7 @@ import (
 	"encoding/hex"
 	"github.com/s-rah/onionscan/config"
 	"github.com/s-rah/onionscan/report"
-	"h12.me/socks"
+	"github.com/s-rah/onionscan/utils"
 	"log"
 )
 
@@ -16,7 +16,7 @@ type FTPProtocolScanner struct {
 func (sps *FTPProtocolScanner) ScanProtocol(hiddenService string, onionscanConfig *config.OnionscanConfig, report *report.OnionScanReport) {
 	// FTP
 	log.Printf("Checking %s FTP(21)\n", hiddenService)
-	conn, err := socks.DialSocksProxy(socks.SOCKS5, onionscanConfig.TorProxyAddress)("", hiddenService+":21")
+	conn, err := utils.GetNetworkConnection(hiddenService, 21, onionscanConfig.TorProxyAddress, onionscanConfig.Timeout)
 	if err != nil {
 		log.Printf("Failed to connect to service on port 21\n")
 		report.FTPDetected = false

protocol/http_scanner.go

@@ -28,7 +28,7 @@ func (hps *HTTPProtocolScanner) ScanProtocol(hiddenService string, onionscanConf
 
 	// HTTP
 	log.Printf("Checking %s http(80)\n", hiddenService)
-	_, err := socks.DialSocksProxy(socks.SOCKS5, onionscanConfig.TorProxyAddress)("", hiddenService+":80")
+	_, err := utils.GetNetworkConnection(hiddenService, 80, onionscanConfig.TorProxyAddress, onionscanConfig.Timeout)
 	if err != nil {
 		log.Printf("Failed to connect to service on port 80\n")
 		report.WebDetected = false
@@ -40,7 +40,7 @@ func (hps *HTTPProtocolScanner) ScanProtocol(hiddenService string, onionscanConf
 		transportConfig := &http.Transport{
 			Dial: dialSocksProxy,
 		}
-		hps.Client = &http.Client{Transport: transportConfig}
+		hps.Client = &http.Client{Transport: transportConfig, Timeout: onionscanConfig.Timeout}
 		// FIXME This should probably be moved to it's own file now.
 		response, err := hps.Client.Get("http://" + hiddenService)
 

protocol/irc_scanner.go

@@ -3,7 +3,7 @@ package protocol
 import (
 	"github.com/s-rah/onionscan/config"
 	"github.com/s-rah/onionscan/report"
-	"h12.me/socks"
+	"github.com/s-rah/onionscan/utils"
 	"log"
 )
 
@@ -13,7 +13,7 @@ type IRCProtocolScanner struct {
 func (rps *IRCProtocolScanner) ScanProtocol(hiddenService string, onionscanConfig *config.OnionscanConfig, report *report.OnionScanReport) {
 	// IRC
 	log.Printf("Checking %s IRC(6667)\n", hiddenService)
-	_, err := socks.DialSocksProxy(socks.SOCKS5, onionscanConfig.TorProxyAddress)("", hiddenService+":6667")
+	_, err := utils.GetNetworkConnection(hiddenService, 6667, onionscanConfig.TorProxyAddress, onionscanConfig.Timeout)
 	if err != nil {
 		log.Printf("Failed to connect to service on port 6667\n")
 		report.IRCDetected = false
@@ -23,4 +23,14 @@ func (rps *IRCProtocolScanner) ScanProtocol(hiddenService string, onionscanConfi
 		report.IRCDetected = true
 	}
 
+	// IRC
+	log.Printf("Checking %s IRC(6697)\n", hiddenService)
+	_, err = utils.GetNetworkConnection(hiddenService, 6697, onionscanConfig.TorProxyAddress, onionscanConfig.Timeout)
+	if err != nil {
+		log.Printf("Failed to connect to service on port 6697\n")
+	} else {
+		log.Printf("Detected possible IRC (secure) instance\n")
+		// TODO: Actual Analysis
+		report.IRCDetected = true
+	}
 }

protocol/mongodb_scanner.go

@@ -3,7 +3,7 @@ package protocol
 import (
 	"github.com/s-rah/onionscan/config"
 	"github.com/s-rah/onionscan/report"
-	"h12.me/socks"
+	"github.com/s-rah/onionscan/utils"
 	"log"
 )
 
@@ -13,7 +13,7 @@ type MongoDBProtocolScanner struct {
 func (rps *MongoDBProtocolScanner) ScanProtocol(hiddenService string, onionscanConfig *config.OnionscanConfig, report *report.OnionScanReport) {
 	// MongoDB
 	log.Printf("Checking %s MongoDB(27017)\n", hiddenService)
-	_, err := socks.DialSocksProxy(socks.SOCKS5, onionscanConfig.TorProxyAddress)("", hiddenService+":27017")
+	_, err := utils.GetNetworkConnection(hiddenService, 27017, onionscanConfig.TorProxyAddress, onionscanConfig.Timeout)
 	if err != nil {
 		log.Printf("Failed to connect to service on port 27017\n")
 		report.MongoDBDetected = false

protocol/ricochet_scanner.go

@@ -3,7 +3,7 @@ package protocol
 import (
 	"github.com/s-rah/onionscan/config"
 	"github.com/s-rah/onionscan/report"
-	"h12.me/socks"
+	"github.com/s-rah/onionscan/utils"
 	"log"
 )
 
@@ -13,7 +13,7 @@ type RicochetProtocolScanner struct {
 func (rps *RicochetProtocolScanner) ScanProtocol(hiddenService string, onionscanConfig *config.OnionscanConfig, report *report.OnionScanReport) {
 	// Ricochet
 	log.Printf("Checking %s ricochet(9878)\n", hiddenService)
-	_, err := socks.DialSocksProxy(socks.SOCKS5, onionscanConfig.TorProxyAddress)("", hiddenService+":9878")
+	_, err := utils.GetNetworkConnection(hiddenService, 9878, onionscanConfig.TorProxyAddress, onionscanConfig.Timeout)
 	if err != nil {
 		log.Printf("Failed to connect to service on port 9878\n")
 		report.RicochetDetected = false

protocol/smtp_scanner.go

@@ -6,7 +6,7 @@ import (
 	"encoding/hex"
 	"github.com/s-rah/onionscan/config"
 	"github.com/s-rah/onionscan/report"
-	"h12.me/socks"
+	"github.com/s-rah/onionscan/utils"
 	"log"
 )
 
@@ -16,7 +16,7 @@ type SMTPProtocolScanner struct {
 func (sps *SMTPProtocolScanner) ScanProtocol(hiddenService string, onionscanConfig *config.OnionscanConfig, report *report.OnionScanReport) {
 	// SMTP
 	log.Printf("Checking %s SMTP(25)\n", hiddenService)
-	conn, err := socks.DialSocksProxy(socks.SOCKS5, onionscanConfig.TorProxyAddress)("", hiddenService+":25")
+	conn, err := utils.GetNetworkConnection(hiddenService, 25, onionscanConfig.TorProxyAddress, onionscanConfig.Timeout)
 	if err != nil {
 		log.Printf("Failed to connect to service on port 25\n")
 		report.SMTPDetected = false

protocol/ssh_scanner.go

@@ -6,8 +6,8 @@ import (
 	"fmt"
 	"github.com/s-rah/onionscan/config"
 	"github.com/s-rah/onionscan/report"
+	"github.com/s-rah/onionscan/utils"
 	"golang.org/x/crypto/ssh"
-	"h12.me/socks"
 	"log"
 	"net"
 )
@@ -18,7 +18,7 @@ type SSHProtocolScanner struct {
 func (sps *SSHProtocolScanner) ScanProtocol(hiddenService string, onionscanConfig *config.OnionscanConfig, report *report.OnionScanReport) {
 	// SSH
 	log.Printf("Checking %s ssh(22)\n", hiddenService)
-	conn, err := socks.DialSocksProxy(socks.SOCKS5, onionscanConfig.TorProxyAddress)("", hiddenService+":22")
+	conn, err := utils.GetNetworkConnection(hiddenService, 22, onionscanConfig.TorProxyAddress, onionscanConfig.Timeout)
 	if err != nil {
 		log.Printf("Failed to connect to service on port 22\n")
 		report.SSHDetected = false

protocol/vnc_scanner.go

@@ -3,7 +3,7 @@ package protocol
 import (
 	"github.com/s-rah/onionscan/config"
 	"github.com/s-rah/onionscan/report"
-	"h12.me/socks"
+	"github.com/s-rah/onionscan/utils"
 	"log"
 )
 
@@ -13,7 +13,7 @@ type VNCProtocolScanner struct {
 func (vncps *VNCProtocolScanner) ScanProtocol(hiddenService string, onionscanConfig *config.OnionscanConfig, report *report.OnionScanReport) {
 	// MongoDB
 	log.Printf("Checking %s VNC(5900)\n", hiddenService)
-	_, err := socks.DialSocksProxy(socks.SOCKS5, onionscanConfig.TorProxyAddress)("", hiddenService+":5900")
+	_, err := utils.GetNetworkConnection(hiddenService, 5900, onionscanConfig.TorProxyAddress, onionscanConfig.Timeout)
 	if err != nil {
 		log.Printf("Failed to connect to service on port 5900\n")
 		report.VNCDetected = false

report/report_generator.go

@@ -14,7 +14,7 @@ func GenerateJsonReport(reportFile string, report *OnionScanReport) {
 	buffer.WriteString(fmt.Sprintf("%s\n", jsonOut))
 
 	if len(reportFile) > 0 {
-		f, err := os.Create(reportFile)
+		f, err := os.Create(report.HiddenService + "." + reportFile)
 		if err != nil {
 			log.Fatalf("Cannot create report file: %s", err)
 			panic(err)
@@ -172,7 +172,7 @@ func GenerateSimpleReport(reportFile string, report *OnionScanReport) {
 	}
 
 	if len(reportFile) > 0 {
-		f, err := os.Create(reportFile)
+		f, err := os.Create(report.HiddenService + "." + reportFile)
 		if err != nil {
 			log.Fatalf("Cannot create report file: %s", err)
 			panic(err)

utils/networking.go

@@ -0,0 +1,15 @@
+package utils
+
+import (
+	"h12.me/socks"
+	"net"
+	"strconv"
+	"time"
+)
+
+func GetNetworkConnection(onionService string, port int, proxy string, timeout time.Duration) (net.Conn, error) {
+	portNumber := strconv.Itoa(port)
+	conn, err := socks.DialSocksProxy(socks.SOCKS5, proxy)("", onionService+":"+portNumber)
+	conn.SetDeadline(time.Now().Add(timeout))
+	return conn, err
+}