Pull referenced directories from page scan and scan them along with

common directories in http scanner

protocol/http_scanner.go

@@ -1,19 +1,28 @@
 package protocol
 
 import (
-	"github.com/s-rah/onionscan/scans"
 	"github.com/s-rah/onionscan/report"
-	"net/http"
+	"github.com/s-rah/onionscan/scans"
-	"io/ioutil"
+	"github.com/s-rah/onionscan/utils"
 	"h12.me/socks"
+	"io/ioutil"
 	"log"
+	"net/http"
 )
 
 type HTTPProtocolScanner struct {
-	Client          *http.Client
+	Client *http.Client
 }
 
-func (hps * HTTPProtocolScanner) ScanProtocol(hiddenService string, proxyAddress string, report *report.OnionScanReport) {
+var (
+	CommonDirectories = []string{"/style", "/styles", "/css", "/uploads", "/images", "/img", "/static",
+		// Lots of Wordpress installs which don't lock down directory listings
+		"/wp-content/uploads",
+		// Common with torshops created onions
+		"/products", "/products/cat"}
+)
+
+func (hps *HTTPProtocolScanner) ScanProtocol(hiddenService string, proxyAddress string, report *report.OnionScanReport) {
 
 	// HTTP
 	log.Printf("Checking %s http(80)\n", hiddenService)
@@ -45,25 +54,17 @@ func (hps * HTTPProtocolScanner) ScanProtocol(hiddenService string, proxyAddress
 		hps.ScanPage(hiddenService, "/server-status", report, scans.ApacheModStatus)
 		hps.ScanPage(hiddenService, "/", report, scans.StandardPageScan)
 
-		hps.ScanPage(hiddenService, "/style", report, scans.CheckDirectoryListing)
+		log.Printf("\tScanning Common and Referenced Directories\n")
-		hps.ScanPage(hiddenService, "/styles", report, scans.CheckDirectoryListing)
+		directories := append(CommonDirectories, report.PageReferencedDirectories...)
-		hps.ScanPage(hiddenService, "/css", report, scans.CheckDirectoryListing)
+		utils.RemoveDuplicates(&directories)
-		hps.ScanPage(hiddenService, "/uploads", report, scans.CheckDirectoryListing)
+		for _, directory := range directories {
-		hps.ScanPage(hiddenService, "/images", report, scans.CheckDirectoryListing)
+			hps.ScanPage(hiddenService, directory, report, scans.CheckDirectoryListing)
-		hps.ScanPage(hiddenService, "/img", report, scans.CheckDirectoryListing)
+		}
-		hps.ScanPage(hiddenService, "/static", report, scans.CheckDirectoryListing)
-
-		// Lots of Wordpress installs which don't lock down directory listings
-		hps.ScanPage(hiddenService, "/wp-content/uploads", report, scans.CheckDirectoryListing)
-
-		// Common with torshops created onions
-		hps.ScanPage(hiddenService, "/products", report, scans.CheckDirectoryListing)
-		hps.ScanPage(hiddenService, "/products/cat", report, scans.CheckDirectoryListing)
 	}
 	log.Printf("\n")
 }
 
-func (hps * HTTPProtocolScanner) ScanPage(hiddenService string, page string, report *report.OnionScanReport, f func(scans.Scanner, string, int, string, *report.OnionScanReport)) {
+func (hps *HTTPProtocolScanner) ScanPage(hiddenService string, page string, report *report.OnionScanReport, f func(scans.Scanner, string, int, string, *report.OnionScanReport)) {
 	response, err := hps.Client.Get("http://" + hiddenService + page)
 	if err != nil {
 		log.Printf("Error connecting to %s%s %s\n", hiddenService, page, err)

report/onionscanreport.go

@@ -2,58 +2,58 @@ package report
 
 import (
 	"encoding/json"
-	"io/ioutil"
 	"github.com/s-rah/onionscan/utils"
+	"io/ioutil"
 )
 
 type ExifTag struct {
-	Name string `json:"name"`
+	Name  string `json:"name"`
-	Value string`json:"value"`
+	Value string `json:"value"`
 }
 
 type ExifImage struct {
-	Location string		`json:"location"`
+	Location string    `json:"location"`
-	ExifTags	     []ExifTag `json:"exifTags"`	
+	ExifTags []ExifTag `json:"exifTags"`
 }
 
 type OnionScanReport struct {
+	WebDetected      bool `json:"webDetected"`
+	SSHDetected      bool `json:"sshDetected"`
+	RicochetDetected bool `json:"ricochetDetected"`
+	IRCDetected      bool `json:"ircDetected"`
+	FTPDetected      bool `json:"ftpDetected"`
+	SMTPDetected     bool `json:"smtpDetected"`
 
-	WebDetected	    bool `json:"webDetected"`
+	BitcoinDetected bool `json:"bitcoinDetected"`
-	SSHDetected	    bool `json:"sshDetected"`
-	RicochetDetected	    bool `json:"ricochetDetected"`
-	IRCDetected		bool `json:"ircDetected"`
-	FTPDetected		bool `json:"ftpDetected"`
-	SMTPDetected		bool `json:"smtpDetected"`
 
-	BitcoinDetected		bool `json:"bitcoinDetected"`
+	HiddenService             string      `json:"hiddenService"`
+	ServerPoweredBy           string      `json:"serverPoweredBy"`
+	ServerVersion             string      `json:"serverVersion"`
+	FoundApacheModStatus      bool        `json:"foundApacheModStatus"`
+	RelatedOnionServices      []string    `json:"relatedOnionServices"`
+	RelatedClearnetDomains    []string    `json:"relatedOnionDomains"`
+	LinkedSites               []string    `json:"linkedSites"`
+	IP                        []string    `json:"ipAddresses"`
+	OpenDirectories           []string    `json:"openDirectories"`
+	ExifImages                []ExifImage `json:"exifImages"`
+	InterestingFiles          []string    `json:"interestingFiles"`
+	PageReferencedDirectories []string    `json:"pageReferencedDirectories"`
 
-	HiddenService        string   `json:"hiddenService"`
+	Hashes   []string `json:"hashes"`
-	ServerPoweredBy	     string   `json:"serverPoweredBy"`
+	SSHKey   string   `json:"sshKey"`
-	ServerVersion        string   `json:"serverVersion"`
+	Snapshot string   `json:"snapshot"`
-	FoundApacheModStatus bool     `json:"foundApacheModStatus"`
-	RelatedOnionServices []string `json:"relatedOnionServices"`
-	RelatedClearnetDomains []string `json:"relatedOnionDomains"`
-	LinkedSites	[]string `json:"linkedSites"`
-	IP		     []string `json:"ipAddresses"`
-	OpenDirectories      []string `json:"openDirectories"`	
-	ExifImages	     []ExifImage `json:"exifImages"`
-	InterestingFiles     []string `json:"interestingFiles"`
-	Hashes		     []string `json:"hashes"`
-	SSHKey 		string `json:"sshKey"`
-	Snapshot	string `json:"snapshot"`
 }
 
 func LoadReportFromFile(filename string) (OnionScanReport, error) {
-    dat, err := ioutil.ReadFile(filename)
+	dat, err := ioutil.ReadFile(filename)
-    if err != nil {
+	if err != nil {
-        return OnionScanReport{}, err
+		return OnionScanReport{}, err
-    }
+	}
-    res := OnionScanReport{}
+	res := OnionScanReport{}
-    err = json.Unmarshal(dat, &res)
+	err = json.Unmarshal(dat, &res)
-    return res, err
+	return res, err
 }
 
-
 func NewOnionScanReport(hiddenService string) *OnionScanReport {
 	return &OnionScanReport{HiddenService: hiddenService}
 }
@@ -63,15 +63,15 @@ func (osr *OnionScanReport) AddOpenDirectory(dir string) {
 }
 
 func (osr *OnionScanReport) AddRelatedOnionService(service string) {
-	osr.RelatedOnionServices  = append(osr.RelatedOnionServices, service)
+	osr.RelatedOnionServices = append(osr.RelatedOnionServices, service)
 }
 
 func (osr *OnionScanReport) AddRelatedClearnetDomain(domain string) {
-	osr.RelatedClearnetDomains  = append(osr.RelatedClearnetDomains, domain)
+	osr.RelatedClearnetDomains = append(osr.RelatedClearnetDomains, domain)
 }
 
 func (osr *OnionScanReport) AddInterestingFile(file string) {
-	osr.InterestingFiles  = append(osr.InterestingFiles, file)
+	osr.InterestingFiles = append(osr.InterestingFiles, file)
 }
 
 func (osr *OnionScanReport) AddIPAddress(ip string) {
@@ -84,17 +84,21 @@ func (osr *OnionScanReport) AddLinkedSite(site string) {
 }
 
 func (osr *OnionScanReport) Serialize() (string, error) {
-	report,err := json.Marshal(osr)
+	report, err := json.Marshal(osr)
 	if err != nil {
 		return "", err
-	}	
+	}
 	return string(report), nil
 }
 
 func (osr *OnionScanReport) AddExifImage(location string) {
-	osr.ExifImages = append(osr.ExifImages, ExifImage{location, []ExifTag{}})	
+	osr.ExifImages = append(osr.ExifImages, ExifImage{location, []ExifTag{}})
 }
 
 func (osr *OnionScanReport) AddExifTag(name string, value string) {
-	osr.ExifImages[len(osr.ExifImages)-1].ExifTags = append(osr.ExifImages[len(osr.ExifImages)-1].ExifTags , ExifTag{name, value})	
+	osr.ExifImages[len(osr.ExifImages)-1].ExifTags = append(osr.ExifImages[len(osr.ExifImages)-1].ExifTags, ExifTag{name, value})
+}
+
+func (osr *OnionScanReport) AddPageReferencedDirectory(directory string) {
+	osr.PageReferencedDirectories = append(osr.PageReferencedDirectories, directory)
 }

scans/standard-page-scan.go

@@ -1,41 +1,41 @@
 package scans
 
 import (
-	"github.com/s-rah/onionscan/report"
-	"github.com/s-rah/onionscan/utils"
-	"net/url"
-	"log"
-	"regexp"
-	"strings"
 	"crypto/sha1"
 	"encoding/hex"
+	"github.com/s-rah/onionscan/report"
+	"github.com/s-rah/onionscan/utils"
+	"log"
+	"net/url"
+	"regexp"
+	"strings"
 )
 
 func StandardPageScan(scan Scanner, page string, status int, contents string, report *report.OnionScanReport) {
 	log.Printf("Scanning %s%s\n", report.HiddenService, page)
 	if status == 200 {
 		log.Printf("\tPage %s%s is Accessible\n", report.HiddenService, page)
-		
+
 		hash := sha1.Sum([]byte(contents))
 		report.Hashes = append(report.Hashes, hex.EncodeToString(hash[:]))
 		report.Snapshot = contents
 
 		domains := utils.ExtractDomains(contents)
-		
+
-		for _,domain := range domains {
+		for _, domain := range domains {
 			if !strings.HasPrefix(domain, "http://"+report.HiddenService) {
 				log.Printf("Found Related URL %s\n", domain)
 				// TODO: Lots of information here which needs to be processed.
 				// * Links to standard sites - google / bitpay etc.
 				// * Links to other onion sites
 				// * Links to obscure clearnet sites.
-				baseUrl,_ := url.Parse(domain)
+				baseUrl, _ := url.Parse(domain)
 				report.AddLinkedSite(baseUrl.Host)
 			} else {
-				// * Process Internal links
+				// * Process FQDN internal links (unlikly)
 				log.Printf("Found Internal URL %s\n", domain)
 			}
-		} 
+		}
 
 		log.Printf("\tScanning for Images\n")
 		r := regexp.MustCompile("src=\"(" + "http://" + report.HiddenService + "/)?((.*?\\.jpg)|(.*?\\.png)|(.*?\\.jpeg)|(.*?\\.gif))\"")
@@ -44,11 +44,25 @@ func StandardPageScan(scan Scanner, page string, status int, contents string, re
 			log.Printf("\t Found image %s\n", image[2])
 			scan.ScanPage(report.HiddenService, "/"+image[2], report, CheckExif)
 		}
+
+		log.Printf("\tScanning for Referenced Directories\n")
+		r = regexp.MustCompile("(src|href)=\"([^\"]*)\"")
+		foundPaths := r.FindAllStringSubmatch(string(contents), -1)
+		for _, regexpResults := range foundPaths {
+			path := regexpResults[2]
+			if strings.HasPrefix(path, "http") {
+				continue
+			}
+
+			term := strings.LastIndex(path, "/")
+			if term > 0 {
+				log.Printf("\t Found Referenced Directory %s\n", path[:term])
+				report.AddPageReferencedDirectory(path[:term])
+			}
+		}
 	} else if status == 403 {
 		log.Printf("\tPage %s%s is Forbidden\n", report.HiddenService, page)
 	} else if status == 404 {
 		log.Printf("\tPage %s%s is Does Not Exist\n", report.HiddenService, page)
 	}
 }
-
-