dan
/
onionscan
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Merge pull request #28 from dballard/directory-scan 

Pull referenced directories from page scan and scan them along with common directories in http scanner
This commit is contained in:
Sarah Jamie Lewis 2016-04-26 17:34:00 -07:00
parent 44e6d5c955 72bf9d6a3a
commit 93c56a44ed
3 changed files with 94 additions and 75 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

41
protocol/http_scanner.go
View File

@ -1,19 +1,28 @@
package protocol
import (
"github.com/s-rah/onionscan/scans"
"github.com/s-rah/onionscan/report"
"net/http"
"io/ioutil"
"github.com/s-rah/onionscan/scans"
"github.com/s-rah/onionscan/utils"
"h12.me/socks"
"io/ioutil"
"log"
"net/http"
)
type HTTPProtocolScanner struct {
Client *http.Client
Client *http.Client
}
func (hps * HTTPProtocolScanner) ScanProtocol(hiddenService string, proxyAddress string, report *report.OnionScanReport) {
var (
CommonDirectories = []string{"/style", "/styles", "/css", "/uploads", "/images", "/img", "/static",
// Lots of Wordpress installs which don't lock down directory listings
"/wp-content/uploads",
// Common with torshops created onions
"/products", "/products/cat"}
)
func (hps *HTTPProtocolScanner) ScanProtocol(hiddenService string, proxyAddress string, report *report.OnionScanReport) {
// HTTP
log.Printf("Checking %s http(80)\n", hiddenService)
@ -45,25 +54,17 @@ func (hps * HTTPProtocolScanner) ScanProtocol(hiddenService string, proxyAddress
hps.ScanPage(hiddenService, "/server-status", report, scans.ApacheModStatus)
hps.ScanPage(hiddenService, "/", report, scans.StandardPageScan)
hps.ScanPage(hiddenService, "/style", report, scans.CheckDirectoryListing)
hps.ScanPage(hiddenService, "/styles", report, scans.CheckDirectoryListing)
hps.ScanPage(hiddenService, "/css", report, scans.CheckDirectoryListing)
hps.ScanPage(hiddenService, "/uploads", report, scans.CheckDirectoryListing)
hps.ScanPage(hiddenService, "/images", report, scans.CheckDirectoryListing)
hps.ScanPage(hiddenService, "/img", report, scans.CheckDirectoryListing)
hps.ScanPage(hiddenService, "/static", report, scans.CheckDirectoryListing)
// Lots of Wordpress installs which don't lock down directory listings
hps.ScanPage(hiddenService, "/wp-content/uploads", report, scans.CheckDirectoryListing)
// Common with torshops created onions
hps.ScanPage(hiddenService, "/products", report, scans.CheckDirectoryListing)
hps.ScanPage(hiddenService, "/products/cat", report, scans.CheckDirectoryListing)
log.Printf("\tScanning Common and Referenced Directories\n")
directories := append(CommonDirectories, report.PageReferencedDirectories...)
utils.RemoveDuplicates(&directories)
for _, directory := range directories {
hps.ScanPage(hiddenService, directory, report, scans.CheckDirectoryListing)
}
}
log.Printf("\n")
}
func (hps * HTTPProtocolScanner) ScanPage(hiddenService string, page string, report *report.OnionScanReport, f func(scans.Scanner, string, int, string, *report.OnionScanReport)) {
func (hps *HTTPProtocolScanner) ScanPage(hiddenService string, page string, report *report.OnionScanReport, f func(scans.Scanner, string, int, string, *report.OnionScanReport)) {
response, err := hps.Client.Get("http://" + hiddenService + page)
if err != nil {
log.Printf("Error connecting to %s%s %s\n", hiddenService, page, err)

82
report/onionscanreport.go
View File

@ -2,58 +2,58 @@ package report
import (
"encoding/json"
"io/ioutil"
"github.com/s-rah/onionscan/utils"
"io/ioutil"
)
type ExifTag struct {
Name string `json:"name"`
Value string`json:"value"`
Name string `json:"name"`
Value string `json:"value"`
}
type ExifImage struct {
Location string `json:"location"`
ExifTags []ExifTag `json:"exifTags"`
Location string `json:"location"`
ExifTags []ExifTag `json:"exifTags"`
}
type OnionScanReport struct {
WebDetected bool `json:"webDetected"`
SSHDetected bool `json:"sshDetected"`
RicochetDetected bool `json:"ricochetDetected"`
IRCDetected bool `json:"ircDetected"`
FTPDetected bool `json:"ftpDetected"`
SMTPDetected bool `json:"smtpDetected"`
WebDetected bool `json:"webDetected"`
SSHDetected bool `json:"sshDetected"`
RicochetDetected bool `json:"ricochetDetected"`
IRCDetected bool `json:"ircDetected"`
FTPDetected bool `json:"ftpDetected"`
SMTPDetected bool `json:"smtpDetected"`
BitcoinDetected bool `json:"bitcoinDetected"`
BitcoinDetected bool `json:"bitcoinDetected"`
HiddenService string `json:"hiddenService"`
ServerPoweredBy string `json:"serverPoweredBy"`
ServerVersion string `json:"serverVersion"`
FoundApacheModStatus bool `json:"foundApacheModStatus"`
RelatedOnionServices []string `json:"relatedOnionServices"`
RelatedClearnetDomains []string `json:"relatedOnionDomains"`
LinkedSites []string `json:"linkedSites"`
IP []string `json:"ipAddresses"`
OpenDirectories []string `json:"openDirectories"`
ExifImages []ExifImage `json:"exifImages"`
InterestingFiles []string `json:"interestingFiles"`
PageReferencedDirectories []string `json:"pageReferencedDirectories"`
HiddenService string `json:"hiddenService"`
ServerPoweredBy string `json:"serverPoweredBy"`
ServerVersion string `json:"serverVersion"`
FoundApacheModStatus bool `json:"foundApacheModStatus"`
RelatedOnionServices []string `json:"relatedOnionServices"`
RelatedClearnetDomains []string `json:"relatedOnionDomains"`
LinkedSites []string `json:"linkedSites"`
IP []string `json:"ipAddresses"`
OpenDirectories []string `json:"openDirectories"`
ExifImages []ExifImage `json:"exifImages"`
InterestingFiles []string `json:"interestingFiles"`
Hashes []string `json:"hashes"`
SSHKey string `json:"sshKey"`
Snapshot string `json:"snapshot"`
Hashes []string `json:"hashes"`
SSHKey string `json:"sshKey"`
Snapshot string `json:"snapshot"`
}
func LoadReportFromFile(filename string) (OnionScanReport, error) {
dat, err := ioutil.ReadFile(filename)
if err != nil {
return OnionScanReport{}, err
}
res := OnionScanReport{}
err = json.Unmarshal(dat, &res)
return res, err
dat, err := ioutil.ReadFile(filename)
if err != nil {
return OnionScanReport{}, err
}
res := OnionScanReport{}
err = json.Unmarshal(dat, &res)
return res, err
}
func NewOnionScanReport(hiddenService string) *OnionScanReport {
return &OnionScanReport{HiddenService: hiddenService}
}
@ -63,15 +63,15 @@ func (osr *OnionScanReport) AddOpenDirectory(dir string) {
}
func (osr *OnionScanReport) AddRelatedOnionService(service string) {
osr.RelatedOnionServices = append(osr.RelatedOnionServices, service)
osr.RelatedOnionServices = append(osr.RelatedOnionServices, service)
}
func (osr *OnionScanReport) AddRelatedClearnetDomain(domain string) {
osr.RelatedClearnetDomains = append(osr.RelatedClearnetDomains, domain)
osr.RelatedClearnetDomains = append(osr.RelatedClearnetDomains, domain)
}
func (osr *OnionScanReport) AddInterestingFile(file string) {
osr.InterestingFiles = append(osr.InterestingFiles, file)
osr.InterestingFiles = append(osr.InterestingFiles, file)
}
func (osr *OnionScanReport) AddIPAddress(ip string) {
@ -84,7 +84,7 @@ func (osr *OnionScanReport) AddLinkedSite(site string) {
}
func (osr *OnionScanReport) Serialize() (string, error) {
report,err := json.Marshal(osr)
report, err := json.Marshal(osr)
if err != nil {
return "", err
}
@ -96,5 +96,9 @@ func (osr *OnionScanReport) AddExifImage(location string) {
}
func (osr *OnionScanReport) AddExifTag(name string, value string) {
osr.ExifImages[len(osr.ExifImages)-1].ExifTags = append(osr.ExifImages[len(osr.ExifImages)-1].ExifTags , ExifTag{name, value})
osr.ExifImages[len(osr.ExifImages)-1].ExifTags = append(osr.ExifImages[len(osr.ExifImages)-1].ExifTags, ExifTag{name, value})
}
func (osr *OnionScanReport) AddPageReferencedDirectory(directory string) {
osr.PageReferencedDirectories = append(osr.PageReferencedDirectories, directory)
}

36
scans/standard-page-scan.go
View File

@ -1,14 +1,14 @@
package scans
import (
"github.com/s-rah/onionscan/report"
"github.com/s-rah/onionscan/utils"
"net/url"
"log"
"regexp"
"strings"
"crypto/sha1"
"encoding/hex"
"github.com/s-rah/onionscan/report"
"github.com/s-rah/onionscan/utils"
"log"
"net/url"
"regexp"
"strings"
)
func StandardPageScan(scan Scanner, page string, status int, contents string, report *report.OnionScanReport) {
@ -22,17 +22,17 @@ func StandardPageScan(scan Scanner, page string, status int, contents string, re
domains := utils.ExtractDomains(contents)
for _,domain := range domains {
for _, domain := range domains {
if !strings.HasPrefix(domain, "http://"+report.HiddenService) {
log.Printf("Found Related URL %s\n", domain)
// TODO: Lots of information here which needs to be processed.
// * Links to standard sites - google / bitpay etc.
// * Links to other onion sites
// * Links to obscure clearnet sites.
baseUrl,_ := url.Parse(domain)
baseUrl, _ := url.Parse(domain)
report.AddLinkedSite(baseUrl.Host)
} else {
// * Process Internal links
// * Process FQDN internal links (unlikly)
log.Printf("Found Internal URL %s\n", domain)
}
}
@ -44,11 +44,25 @@ func StandardPageScan(scan Scanner, page string, status int, contents string, re
log.Printf("\t Found image %s\n", image[2])
scan.ScanPage(report.HiddenService, "/"+image[2], report, CheckExif)
}
log.Printf("\tScanning for Referenced Directories\n")
r = regexp.MustCompile("(src|href)=\"([^\"]*)\"")
foundPaths := r.FindAllStringSubmatch(string(contents), -1)
for _, regexpResults := range foundPaths {
path := regexpResults[2]
if strings.HasPrefix(path, "http") {
continue
}
term := strings.LastIndex(path, "/")
if term > 0 {
log.Printf("\t Found Referenced Directory %s\n", path[:term])
report.AddPageReferencedDirectory(path[:term])
}
}
} else if status == 403 {
log.Printf("\tPage %s%s is Forbidden\n", report.HiddenService, page)
} else if status == 404 {
log.Printf("\tPage %s%s is Does Not Exist\n", report.HiddenService, page)
}
}