Merge pull request #41 from mapmeld/master

Scan CSS style tags and stylesheet links

protocol/http_scanner.go

@@ -76,15 +76,23 @@ func (hps *HTTPProtocolScanner) ScanProtocol(hiddenService string, onionscanConf
 }
 
 func (hps *HTTPProtocolScanner) ScanPage(hiddenService string, page string, report *report.OnionScanReport, f func(scans.Scanner, string, int, string, *report.OnionScanReport)) {
+	_, contents, responseCode := hps.ScrapePage(hiddenService, page)
+	f(hps, page, responseCode, string(contents), report)
+}
+
+func (hps *HTTPProtocolScanner) ScrapePage(hiddenService string, page string) (error, []byte, int) {
 	if !strings.Contains(page, utils.WithoutSubdomains(hiddenService)) {
+		if !strings.HasPrefix(page, "/") {
+			page = "/" + page
+		}
 		page = hiddenService + page
 	}
 	response, err := hps.Client.Get("http://" + page)
 	if err != nil {
 		log.Printf("Error connecting to http://%s %s\n", page, err)
-		return
+		return err, nil, -1
 	}
 	defer response.Body.Close()
 	contents, _ := ioutil.ReadAll(response.Body)
-	f(hps, page, response.StatusCode, string(contents), report)
+	return nil, contents, response.StatusCode
 }

scans/scanner.go

@@ -6,4 +6,5 @@ import (
 
 type Scanner interface {
 	ScanPage(string, string, *report.OnionScanReport, func(Scanner, string, int, string, *report.OnionScanReport))
+	ScrapePage(string, string) (error, []byte, int)
 }

scans/standard-page-scan.go

@@ -34,7 +34,8 @@ func StandardPageScan(scan Scanner, page string, status int, contents string, re
 		new(PGPContentScan).ScanContent(contents, report)
 
 		log.Printf("\tScanning for Images\n")
-		domains := utils.ExtractDomains(contents)
+		var domains []string
+		var cssLinks []string
 
 		// parser based on http://schier.co/blog/2015/04/26/a-simple-web-scraper-in-go.html
 		z := html.NewTokenizer(strings.NewReader(contents))
@@ -49,8 +50,8 @@ func StandardPageScan(scan Scanner, page string, status int, contents string, re
 			// TODO: don't crawl links with nofollow
 
 			if tt == html.StartTagToken {
-				isLink := t.Data == "a"
-				if isLink {
+				// links
+				if t.Data == "a" {
 					linkUrl := utils.GetAttribute(t, "href")
 					if len(linkUrl) > 1 {
 						domains = append(domains, linkUrl)
@@ -58,8 +59,13 @@ func StandardPageScan(scan Scanner, page string, status int, contents string, re
 				}
 			}
 
-			isImage := t.Data == "img"
-			if isImage {
+			// css <link>
+			if t.Data == "link" && utils.GetAttribute(t, "rel") == "stylesheet" {
+				cssLinks = append(cssLinks, utils.GetAttribute(t, "href"))
+			}
+
+			// images
+			if t.Data == "img" {
 				imageUrl := utils.GetAttribute(t, "src")
 
 				baseUrl, _ := url.Parse(imageUrl)
@@ -72,8 +78,15 @@ func StandardPageScan(scan Scanner, page string, status int, contents string, re
 			}
 		}
 
-		log.Printf("\tScanning for Links\n")
+		log.Printf("\tScanning for CSS Fonts and Background Images\n")
+		for _, cssUrl := range cssLinks {
+			log.Printf("\tScanning CSS file: %s\n", cssUrl)
+			_, cssContents, _ := scan.ScrapePage(report.HiddenService, utils.WithoutProtocol(cssUrl))
+			domains = append(domains, utils.ExtractDomains(string(cssContents))[0:]...)
+		}
 
+		log.Printf("\tScanning for Links\n")
+		domains = append(domains, utils.ExtractDomains(contents)...)
 		for _, domain := range domains {
 			baseUrl, _ := url.Parse(domain)
 			if baseUrl.Host != "" && utils.WithoutSubdomains(baseUrl.Host) != utils.WithoutSubdomains(report.HiddenService) {
@@ -95,7 +108,7 @@ func StandardPageScan(scan Scanner, page string, status int, contents string, re
 		foundPaths := r.FindAllStringSubmatch(string(contents), -1)
 		for _, regexpResults := range foundPaths {
 			path := regexpResults[2]
-			if strings.HasPrefix(path, "http") && !strings.Contains(path, utils.WithoutSubdomains(report.HiddenService)) {
+			if (strings.HasPrefix(path, "http") || strings.HasPrefix(path, "//")) && !strings.Contains(path, utils.WithoutSubdomains(report.HiddenService)) {
 				continue
 			}
 

utils/url_parsing.go

@@ -2,11 +2,23 @@ package utils
 
 import (
 	"github.com/mvdan/xurls"
+	"regexp"
 	"strings"
 )
 
 func ExtractDomains(content string) []string {
-	return xurls.Strict.FindAllString(content, -1)
+	domains := xurls.Strict.FindAllString(content, -1)
+	cssurlregex := regexp.MustCompile(`(?i)url\((.*?)\)`)
+	cssDomains := cssurlregex.FindAllString(content, -1)
+	for _, cssDomain := range cssDomains {
+		if strings.HasPrefix(strings.ToLower(cssDomain), "url(") {
+			cssDomain = cssDomain[4 : len(cssDomain)-1]
+		}
+		if !strings.HasSuffix(cssDomain, ":before") && !strings.HasSuffix(cssDomain, ":after") {
+			domains = append(domains, cssDomain)
+		}
+	}
+	return domains
 }
 
 func WithoutSubdomains(urlhost string) string {
@@ -25,5 +37,8 @@ func WithoutProtocol(url string) string {
 	if strings.HasPrefix(url, "https://") {
 		return url[8:]
 	}
+	if strings.HasPrefix(url, "//") {
+		return url[2:]
+	}
 	return url
 }