Improving Standard Page Scan

.gitignore

@@ -1 +1,36 @@
+# common files
 *~
+*.log
+*.bak
+*.tmp
+*.swp
+*.lock
+
+
+# Eclipse
+.classpath
+.project
+.settings
+.metadata
+.factorypath
+
+# IDEA IntelliJ
+*.ipr
+*.iws
+*.iml
+.idea
+.idea_modules
+out
+atlassian-ide-plugin.xml
+
+# Windows
+Thumbs.db
+ehthumbs.db
+Desktop.ini
+
+# KDE directory preferences
+.directory
+
+# OS X
+.DS_Store
+.Trashes

protocol/http_scanner.go

@@ -36,10 +36,16 @@ func (hps * HTTPProtocolScanner) ScanProtocol(hiddenService string, proxyAddress
 			return
 		}
 
-		// Initial Attempt at Resolving Server Type
-		log.Printf("Attempting to Derive Server Type from Headers..\n")
-		report.ServerVersion = response.Header.Get("Server")
-		log.Printf("\tServer Version: %s\n", report.ServerVersion)
+		// Reading all http headers
+		log.Printf("HTTP response headers: %s\n", report.ServerVersion)
+		responseHeaders := response.Header
+		for key := range responseHeaders {
+			value := responseHeaders.Get(key)
+			report.AddResponseHeader(key, value)
+			log.Printf("\t%s : %s\n", key, value)
+		}
+
+		report.ServerVersion = responseHeaders.Get("Server")
 
 		// Apache mod-status Check
 		hps.ScanPage(hiddenService, "/server-status", report, scans.ApacheModStatus)

report/onionscanreport.go

@@ -4,6 +4,7 @@ import (
 	"encoding/json"
 	"io/ioutil"
 	"github.com/s-rah/onionscan/utils"
+	"fmt"
 )
 
 type ExifTag struct {
@@ -28,6 +29,8 @@ type OnionScanReport struct {
 	BitcoinDetected		bool `json:"bitcoinDetected"`
 
 	HiddenService        	string   `json:"hiddenService"`
+	PageTitle            	string   `json:"pageTitle"`
+	ResponseHeaders	     	[]string `json:"responseHeaders"`
 	ServerPoweredBy	     	string   `json:"serverPoweredBy"`
 	ServerVersion        	string   `json:"serverVersion"`
 	FoundApacheModStatus 	bool     `json:"foundApacheModStatus"`
@@ -83,6 +86,11 @@ func (osr *OnionScanReport) AddLinkedSite(site string) {
 	utils.RemoveDuplicates(&osr.LinkedSites)
 }
 
+func (osr *OnionScanReport) AddResponseHeader(name string, value string) {
+	header := fmt.Sprintf("%s : %s ", name, value)
+	osr.ResponseHeaders = append(osr.ResponseHeaders, header)
+}
+
 func (osr *OnionScanReport) Serialize() (string, error) {
 	report,err := json.Marshal(osr)
 	if err != nil {

scans/standard-page-scan.go

@@ -20,6 +20,16 @@ func StandardPageScan(scan Scanner, page string, status int, contents string, re
 		report.Hashes = append(report.Hashes, hex.EncodeToString(hash[:]))
 		report.Snapshot = contents
 
+		// Try resolve page title if present
+		isTitlePresent := strings.Contains(contents, "<title>")
+		if isTitlePresent {
+			var startIndex = strings.Index(contents, "<title>")
+			var endIndex = strings.Index(contents, "</title>")
+			var pageTitle = contents[startIndex+len("<title>"):endIndex]
+			log.Printf("\tPage Title: %s\n", pageTitle)
+			report.PageTitle = pageTitle
+		}
+
 		domains := utils.ExtractDomains(contents)
 		
 		for _,domain := range domains {