Improving Standard Page Scan

.gitignore

@@ -1 +1,36 @@
+# common files
 *~
+*.log
+*.bak
+*.tmp
+*.swp
+*.lock
+
+
+# Eclipse
+.classpath
+.project
+.settings
+.metadata
+.factorypath
+
+# IDEA IntelliJ
+*.ipr
+*.iws
+*.iml
+.idea
+.idea_modules
+out
+atlassian-ide-plugin.xml
+
+# Windows
+Thumbs.db
+ehthumbs.db
+Desktop.ini
+
+# KDE directory preferences
+.directory
+
+# OS X
+.DS_Store
+.Trashes

protocol/http_scanner.go

@@ -36,10 +36,16 @@ func (hps * HTTPProtocolScanner) ScanProtocol(hiddenService string, proxyAddress
 			return
 		}
 
-		// Initial Attempt at Resolving Server Type
+		// Reading all http headers
-		log.Printf("Attempting to Derive Server Type from Headers..\n")
+		log.Printf("HTTP response headers: %s\n", report.ServerVersion)
-		report.ServerVersion = response.Header.Get("Server")
+		responseHeaders := response.Header
-		log.Printf("\tServer Version: %s\n", report.ServerVersion)
+		for key := range responseHeaders {
+			value := responseHeaders.Get(key)
+			report.AddResponseHeader(key, value)
+			log.Printf("\t%s : %s\n", key, value)
+		}
+
+		report.ServerVersion = responseHeaders.Get("Server")
 
 		// Apache mod-status Check
 		hps.ScanPage(hiddenService, "/server-status", report, scans.ApacheModStatus)

report/onionscanreport.go

@@ -4,6 +4,7 @@ import (
 	"encoding/json"
 	"io/ioutil"
 	"github.com/s-rah/onionscan/utils"
+	"fmt"
 )
 
 type ExifTag struct {
@@ -18,29 +19,31 @@ type ExifImage struct {
 
 type OnionScanReport struct {
 
-	WebDetected	    bool `json:"webDetected"`
+	WebDetected		bool `json:"webDetected"`
-	SSHDetected	    bool `json:"sshDetected"`
+	SSHDetected	    	bool `json:"sshDetected"`
-	RicochetDetected	    bool `json:"ricochetDetected"`
+	RicochetDetected	bool `json:"ricochetDetected"`
 	IRCDetected		bool `json:"ircDetected"`
 	FTPDetected		bool `json:"ftpDetected"`
 	SMTPDetected		bool `json:"smtpDetected"`
 
 	BitcoinDetected		bool `json:"bitcoinDetected"`
 
-	HiddenService        string   `json:"hiddenService"`
+	HiddenService        	string   `json:"hiddenService"`
-	ServerPoweredBy	     string   `json:"serverPoweredBy"`
+	PageTitle            	string   `json:"pageTitle"`
-	ServerVersion        string   `json:"serverVersion"`
+	ResponseHeaders	     	[]string `json:"responseHeaders"`
-	FoundApacheModStatus bool     `json:"foundApacheModStatus"`
+	ServerPoweredBy	     	string   `json:"serverPoweredBy"`
-	RelatedOnionServices []string `json:"relatedOnionServices"`
+	ServerVersion        	string   `json:"serverVersion"`
-	RelatedClearnetDomains []string `json:"relatedOnionDomains"`
+	FoundApacheModStatus 	bool     `json:"foundApacheModStatus"`
-	LinkedSites	[]string `json:"linkedSites"`
+	RelatedOnionServices 	[]string `json:"relatedOnionServices"`
-	IP		     []string `json:"ipAddresses"`
+	RelatedClearnetDomains 	[]string `json:"relatedOnionDomains"`
-	OpenDirectories      []string `json:"openDirectories"`	
+	LinkedSites		[]string `json:"linkedSites"`
-	ExifImages	     []ExifImage `json:"exifImages"`
+	IP		     	[]string `json:"ipAddresses"`
-	InterestingFiles     []string `json:"interestingFiles"`
+	OpenDirectories      	[]string `json:"openDirectories"`
-	Hashes		     []string `json:"hashes"`
+	ExifImages	     	[]ExifImage `json:"exifImages"`
-	SSHKey 		string `json:"sshKey"`
+	InterestingFiles     	[]string `json:"interestingFiles"`
-	Snapshot	string `json:"snapshot"`
+	Hashes		     	[]string `json:"hashes"`
+	SSHKey 			string `json:"sshKey"`
+	Snapshot		string `json:"snapshot"`
 }
 
 func LoadReportFromFile(filename string) (OnionScanReport, error) {
@@ -83,6 +86,11 @@ func (osr *OnionScanReport) AddLinkedSite(site string) {
 	utils.RemoveDuplicates(&osr.LinkedSites)
 }
 
+func (osr *OnionScanReport) AddResponseHeader(name string, value string) {
+	header := fmt.Sprintf("%s : %s ", name, value)
+	osr.ResponseHeaders = append(osr.ResponseHeaders, header)
+}
+
 func (osr *OnionScanReport) Serialize() (string, error) {
 	report,err := json.Marshal(osr)
 	if err != nil {

scans/standard-page-scan.go

@@ -20,6 +20,16 @@ func StandardPageScan(scan Scanner, page string, status int, contents string, re
 		report.Hashes = append(report.Hashes, hex.EncodeToString(hash[:]))
 		report.Snapshot = contents
 
+		// Try resolve page title if present
+		isTitlePresent := strings.Contains(contents, "<title>")
+		if isTitlePresent {
+			var startIndex = strings.Index(contents, "<title>")
+			var endIndex = strings.Index(contents, "</title>")
+			var pageTitle = contents[startIndex+len("<title>"):endIndex]
+			log.Printf("\tPage Title: %s\n", pageTitle)
+			report.PageTitle = pageTitle
+		}
+
 		domains := utils.ExtractDomains(contents)
 		
 		for _,domain := range domains {