description = [[
Gets favicons and makes a database with their frequencies.

Quick start:
Save this script to the scripts subdirectory of your nmap working directory and
cd into it.
	mkdir ~/favicon
	mkdir ~/favicon/hash ~/favicon/icon
Then run a command like this:
	./nmap --datadir . -n -PN -d --max-retries 2 --script=favicon --script-args hashdir=/home/user/favicon/hash,icondir=/home/user/favicon/icon -p 80 -iR 20000 -oN favicon-%Y%m%d-%H%M%S.nmap

The script downloads each favicon it finds and stores it in a file in ICON_DIR
named after the MD5 hash of the favicon.

<code>
$ cd ~/favicon
$ ls icon
17F03417CBF92B80992B7CA7A566FB0C.ico  C89ECD7675567625E5755A7A9C31632D.ico
379A65BEB4D412765FCF9FBBDEECD416.ico  C8BFCB5728998AC6C3DA90EA5CD2340A.ico
7131EF7073ED685BF2987B9061C65D36.ico  CB5AA723DDDB0734CEC459F2B9C3B1C4.ico
88733EE53676A47FC354A61C32516E82.ico  D16A0DA12074DAE41980A6918D33F031.ico
A3C7BE1BCF382EA413C30453A4ACF638.ico  D41D8CD98F00B204E9800998ECF8427E.ico
A8FE5B8AE2C445A33AC41B33CCC9A120.ico  D4DA62A788942AAB81D033C9E49D57CB.ico
B6141EFEE8D8E64DBC23539F99F7238E.ico  ECF508711C226CCDA02D58853B31D7A7.ico
C3FB27F0BF8AC3171C8105726D61380A.ico
</code>

It also stores the host name and port of each host that had a favicon in a file
in HASH_DIR, also named with the MD5 sum.

<code>
$ cd ~/favicon
$ ls hash
17F03417CBF92B80992B7CA7A566FB0C  C89ECD7675567625E5755A7A9C31632D
379A65BEB4D412765FCF9FBBDEECD416  C8BFCB5728998AC6C3DA90EA5CD2340A
7131EF7073ED685BF2987B9061C65D36  CB5AA723DDDB0734CEC459F2B9C3B1C4
88733EE53676A47FC354A61C32516E82  D16A0DA12074DAE41980A6918D33F031
A3C7BE1BCF382EA413C30453A4ACF638  D41D8CD98F00B204E9800998ECF8427E
A8FE5B8AE2C445A33AC41B33CCC9A120  D4DA62A788942AAB81D033C9E49D57CB
B6141EFEE8D8E64DBC23539F99F7238E  ECF508711C226CCDA02D58853B31D7A7
C3FB27F0BF8AC3171C8105726D61380A
</code>

Each of these files contains a list of each host and port that had that
particular favicon, one per line.

<code>
$ cd ~/favicon
$ cat hash/D16A0DA12074DAE41980A6918D33F031
190.166.207.187:80
125.25.91.250:80
</code>

You can see which are the most common icons with <code>wc</code>.

<code>
$ cd ~/favicon
$ wc -l hash/* | sort -n
  1 hash/17F03417CBF92B80992B7CA7A566FB0C
  1 hash/379A65BEB4D412765FCF9FBBDEECD416
  1 hash/7131EF7073ED685BF2987B9061C65D36
  1 hash/88733EE53676A47FC354A61C32516E82
  1 hash/A3C7BE1BCF382EA413C30453A4ACF638
  1 hash/B6141EFEE8D8E64DBC23539F99F7238E
  1 hash/C3FB27F0BF8AC3171C8105726D61380A
  1 hash/C89ECD7675567625E5755A7A9C31632D
  1 hash/C8BFCB5728998AC6C3DA90EA5CD2340A
  1 hash/CB5AA723DDDB0734CEC459F2B9C3B1C4
  1 hash/D4DA62A788942AAB81D033C9E49D57CB
  1 hash/ECF508711C226CCDA02D58853B31D7A7
  2 hash/D16A0DA12074DAE41980A6918D33F031
  4 hash/D41D8CD98F00B204E9800998ECF8427E
 19 hash/A8FE5B8AE2C445A33AC41B33CCC9A120
 37 total
</code>
]]

author = "David Fifield <david@bamsoftware com>"

license = "Same as Nmap--See http://nmap.org/book/man-legal.html"

categories = {"discovery", "safe"}

require("bin")
require("http")
require("nmap")
require("openssl")
require("shortport")
require("stdnse")
require("url")

portrule = shortport.port_or_service({80, 443, 8080, 8443},
	{"http", "https", "http-alt", "https-alt"})

-- You must create hash and icon directories and provide hashdir and icondir
-- script arguments. Run
-- 	mkdir ~/favicon
-- 	mkdir ~/favicon/hash ~/favicon/icon
-- Then use --script-args hashdir=/home/user/favicon/hash,icondir=/home/user/favicon/icon.
-- The icondir should be common to all scans. The hashdir may be different to
-- segregate the results of different target sets.
local HASH_DIR = nmap.registry.args.hashdir
local ICON_DIR = nmap.registry.args.icondir

if not ICON_DIR or not HASH_DIR then
	error("\nUse --script-args hashdir=.../favicon/hash,icondir=.../favicon/icon.")
end

local REDIRECT_LIMIT = 5
local MAX_SIZE = 10000000

local parse_url_relative
local http_get_redirected, root_doc, favicon_link_url

action = function(host, port)
	local root_host, root_port, root_path, body
	local favicon_url, favicon_body
	local target, hash
	local _

	local ip, name = host.ip, host.targetname or "<name>"
	host = host.targetname or host.ip
	port = port.number
	root_host, root_port, root_path, body = http_get_redirected(host, port,
		"/", name, ip, REDIRECT_LIMIT)
	if body then
		stdnse.print_debug("favicon: %s %s redirects led " ..
			"to %s:%d%s", name, ip, root_host, root_port, root_path)
		favicon_url = favicon_link_url(body)
	end
	if favicon_url then
		stdnse.print_debug("favicon: %s %s <link> " ..
			"favicon from %s", name, ip, favicon_url)
		local scheme, favicon_host, favicon_port, favicon_path =
			parse_url_relative(favicon_url, root_host, root_port, root_path)
		if scheme == "http" or scheme == "https" then
			_, _, _, favicon_body = http_get_redirected(favicon_host, favicon_port, favicon_path, name, ip, REDIRECT_LIMIT)
		end
		target = root_host .. ":" ..  tostring(root_port)
	else
		stdnse.print_debug("favicon: %s %s no <link>", name, ip)
	end
	if not favicon_body then
		stdnse.print_debug("favicon: %s %s default " ..
			"favicon from %s:%d/favicon.ico", name, ip, host, port)
		_, _, _, favicon_body = http_get_redirected(host, port, "/favicon.ico", name, ip, REDIRECT_LIMIT)
		target = host .. ":" ..  tostring(port)
	end
	if not favicon_body then
		return
	elseif #favicon_body > MAX_SIZE then
		stdnse.print_debug("favicon: %s %s oversized (%d)",
			name, ip, #favicon_body, favicon_url)
		return
	else
		_, hash = bin.unpack("H16", openssl.md5(favicon_body))
	end

	local f, line
	local f = io.open(HASH_DIR .. "/" .. hash, "r")
	if f then
		-- We've seen this hash already.
		for line in f:lines() do
			if line == target then
				-- Already seen this host too.
				return hash .. ": " .. target .. " already present."
			end
		end
		f:close()

		-- Append the target to the list.
		f = assert(io.open(HASH_DIR .. "/" .. hash, "a"))
		f:write(target .. "\n")
		f:close()

		return hash .. ": added " .. target .. "."
	else
		-- New hash. Save the favicon.
		f = assert(io.open(ICON_DIR .. "/" .. hash .. ".ico", "wb"))
		f:write(favicon_body)
		f:close()

		-- Append the target to the list.
		f = assert(io.open(HASH_DIR .. "/" .. hash, "a"))
		f:write(target .. "\n")
		f:close()

		return hash .. ": new with " .. target .. "."
	end
end

function dirname(path)
	local dir
	dir = string.match(path, "^(.*)/")
	return dir or ""
end

-- Return a URL's scheme, host, port, and path, filling in the results with the
-- given host, port, and path if the URL is relative. Return nil if the scheme
-- is not "http" or "https".
function parse_url_relative(u, host, port, path)
	local defaultport, scheme, abspath
	u = url.parse(u)
	scheme = u.scheme or "http"
	if scheme == "http" then
		defaultport = 80
	elseif scheme == "https" then
		defaultport = 443
	else
		return nil
	end
	abspath = u.path or ""
	if not string.find(abspath, "^/") then
		abspath = dirname(path) .. "/" .. abspath
	end
	return scheme, u.host or host, u.port or defaultport, abspath
end

-- Do an HTTP GET, following at most limit redirections. Return the final host,
-- port, path, and body; or nil if any of the redirections failed.
function http_get_redirected(host, port, path, name, ip, limit)
	if not limit then
		limit = 0
	end
	local count = 0

	while count <= limit do
		stdnse.print_debug("favicon: %s %s link %d/%d get %s:%d%s",
			name, ip, count, limit, host, port, path)
		local response = http.get(host, port, path, {bypass_cache = true, no_cache = true, no_cache_body = true, max_content_length = MAX_SIZE})
		if not response.status then
			stdnse.print_debug("favicon: %s %s link %d/%d failed %s:%d%s",
				name, ip, count, limit, host, port, path)
			break
		end
		stdnse.print_debug("favicon: %s %s link %d/%d received %s:%d%s",
			name, ip, count, limit, host, port, path)
		local status = tonumber(response.status)
		if status == 200 then
			return host, port, path, response.body
		elseif status >= 300 and status < 400 and response.header.location then
			local scheme
			scheme, host, port, path = parse_url_relative(response.header.location, host, port, path)
			stdnse.print_debug("favicon: %s %s link %d/%d redirects to %s:%d%s",
				name, ip, count, limit, host, port, path)
			if not (scheme == "http" or scheme == "https") then
				break
			end
		else
			break
		end
		count = count + 1
	end

	return nil
end

-- grep an HTML body for an element of the form
--   <link rel="icon" href="...">
--   <link rel="shortcut icon" href="...">
-- and return the href.
function favicon_link_url(body)
	local _, i, j
	local rel, href, word

	-- Loop through link elements.
	i = 0
	while i do
		_, i = string.find(body, "<%s*[Ll][Ii][Nn][Kk]%s", i + 1)
		if not i then
			return nil
		end
		-- Loop through attributes.
		j = i
		while true do
			local name, quote, value
			_, j, name, quote, value = string.find(body, "^%s*(%w+)%s*=%s*([\"'])(.-)%2", j + 1)
			if not j then
				break
			end
			if string.lower(name) == "rel" then
				rel = value
			elseif string.lower(name) == "href" then
				href = value
			end
		end
		for word in string.gmatch(rel or "", "%S+") do
			if string.lower(word) == "icon" then
				return href
			end
		end
	end
end