diff --git a/local.bib b/local.bib index 9258610..7f8c4b5 100644 --- a/local.bib +++ b/local.bib @@ -75,19 +75,36 @@ url = {http://citeseerx.ist.psu.edu/viewdoc/download?doi=10.1.1.133.5778&rep=rep1&type=pdf}, } +@misc{meek-wiki, + title = {meek}, + key = {meek}, + howpublished = {Tor Bug Tracker \& Wiki}, + url = {https://trac.torproject.org/projects/tor/wiki/doc/meek}, +} + @misc{snowflake-wiki, title = {Snowflake}, key = {Snowflake}, + howpublished = {Tor Bug Tracker \& Wiki}, url = {https://trac.torproject.org/projects/tor/wiki/doc/Snowflake}, } -@article{FifieldGilEpnerWebRTC, +@misc{goagent-wiki, + author = {David Fifield}, + title = {{GoAgent}: Further notes on {App Engine} and speculation about a pluggable transport}, + month = oct, + year = 2013, + howpublished = {Tor Bug Tracker \& Wiki}, + url = {https://trac.torproject.org/projects/tor/wiki/doc/GoAgent?action=diff&version=2&old_version=1}, +} + +@techreport{FifieldGilEpnerWebRTC, title = {Fingerprintability of {WebRTC}}, author = {David Fifield and Mia Gil Epner}, - journal = {CoRR}, - volume = {abs/1605.08805}, - year = {2016}, - url = {https://arxiv.org/abs/1605.08805}, + institution = {}, + month = may, + year = 2016, + url = {https://arxiv.org/abs/1605.08805v1}, } @inproceedings{Khattak2016doyousee, @@ -133,7 +150,7 @@ year = 2013, month = jan, publisher = {Citizen Lab}, - url = {https://citizenlab.org/2013/01/planet-blue-coat-mapping-global-censorship-and-surveillance-tools/}, + url = {https://citizenlab.ca/2013/01/planet-blue-coat-mapping-global-censorship-and-surveillance-tools/}, } @misc{CitizenLab2013opakistan, @@ -142,7 +159,7 @@ year = 2013, month = jun, publisher = {Citizen Lab}, - url = {https://citizenlab.org/2013/06/o-pakistan/}, + url = {https://citizenlab.ca/2013/06/o-pakistan/}, } @inproceedings{Mathrani2010a, @@ -394,6 +411,7 @@ @techreport{LovecruftDeValence2017a, author = {Lovecruft, Isis Agora and de Valence, Henry}, title = {{HYPHAE}: Social Secret Sharing}, + institution = {}, number = {2017-04-21}, month = apr, year = {2017}, @@ -403,8 +421,207 @@ @techreport{Sovran2008b, author = {Yair Sovran and Jinyang Li and Lakshminarayanan Subramanian}, title = {Unblocking the {Internet}: Social Networks Stymie Censors}, - organization = {New York University}, + institution = {New York University}, number = {TR2008-918}, - year = {2008}, + year = 2008, url = {http://kscope.news.cs.nyu.edu/pub/TR-2008-918.pdf}, } + +@misc{tor-dev-meek-announcement, + author = {David Fifield}, + title = {A simple {HTTP} transport and big ideas}, + month = jan, + year = 2014, + howpublished = {tor-dev mailing list}, + url = {https://lists.torproject.org/pipermail/tor-dev/2014-January/006159.html}, +} + +@misc{tor-dev-howto-use-lantern-pt, + author = {David Fifield}, + title = {{HOWTO} use {Lantern} as a pluggable transport}, + month = mar, + year = 2014, + howpublished = {tor-dev mailing list}, + url = {https://lists.torproject.org/pipermail/tor-dev/2014-March/006356.html}, +} + +@misc{tor-trac-8860, + author = {Arlo Breault and David Fifield and George Kadianakis}, + title = {Registration over {App Engine}}, + month = may, + year = 2013, + url = {https://bugs.torproject.org/8860}, +} + +@misc{tor-trac-10935, + author = {David Fifield and George Kadianakis and Georg Koppen and Mark Smith}, + title = {Make bundles featuring meek}, + month = feb, + year = 2014, + url = {https://bugs.torproject.org/10935}, +} + +@misc{tor-trac-17473, + author = {David Fifield and Georg Koppen and Klaus Layer}, + title = {Update the meek-amazon fingerprint to B9E7141C594AF25699E0079C1F0146F409495296}, + month = oct, + year = 2015, + url = {https://bugs.torproject.org/17473}, +} + +@techreport{tor-tr-2012-10-001, + author = {Karsten Loesing}, + title = {Counting daily bridge users}, + institution = {The Tor Project}, + number = {2012-10-001}, + month = oct, + year = 2012, + url = {https://research.torproject.org/techreports/counting-daily-bridge-users-2012-10-24.pdf}, +} + +@misc{tor-blog-tor-browser-364-and-40-alpha-1-are-released, + author = {Erinn Clark}, + title = {{Tor Browser} 3.6.4 and 4.0-alpha-1 are released}, + month = aug, + year = 2014, + howpublished = {The Tor Blog}, + url = {https://blog.torproject.org/tor-browser-364-and-40-alpha-1-are-released}, +} + +@misc{tor-blog-tor-browser-40-released, + author = {Mike Perry}, + title = {{Tor Browser} 4.0 is released}, + month = oct, + year = 2014, + howpublished = {The Tor Blog}, + url = {https://blog.torproject.org/tor-browser-40-released}, +} + +@misc{tor-blog-how-use-meek-pluggable-transport, + author = {David Fifield}, + title = {How to use the ``meek'' pluggable transport}, + month = aug, + year = 2015, + howpublished = {The Tor Blog}, + url = {https://blog.torproject.org/how-use-meek-pluggable-transport}, +} + +@misc{tor-dev-user-counting-confusion, + author = {David Fifield}, + title = {Why the seeming correlation between flash proxy and meek on metrics graphs?}, + month = sep, + year = 2014, + howpublished = {tor-dev mailing list}, + url = {https://lists.torproject.org/pipermail/tor-dev/2014-September/007484.html}, +} + +@misc{openitp-usability-hackathon, + author = {Willow Brugh}, + title = {{San Francisco} Hackathon/{DiscoTech} (+ {RightsCon} + {Responsible Data Forum})}, + month = mar, + year = 2014, + url = {http://codesign.mit.edu/2014/03/sfdiscotech/}, +} + +@misc{google-transparency-cn-201405, + title = {{Google Transparency Report}: {China}, All Products, {May} 31, 2014--Present}, + author = {Google}, + month = jul, + year = 2014, + url = {https://www.google.com/transparencyreport/traffic/disruptions/124/}, +} + +@misc{greatfire-google-block-cn, + title = {{Google} disrupted prior to {Tiananmen} Anniversary; Mirror sites enable uncensored access to information}, + author = {Percy Alpha}, + month = jun, + year = 2014, + url = {https://en.greatfire.org/blog/2014/jun/google-disrupted-prior-tiananmen-anniversary-mirror-sites-enable-uncensored-access}, +} + +@misc{DouBarrWallStreetJournal, + author = {Eva Dou and Alistair Barr}, + title = {{U.S.} Cloud Providers Face Backlash From {China's} Censors}, + howpublished = {Wall Street Journal}, + month = mar, + year = 2015, + url = {https://www.wsj.com/articles/u-s-cloud-providers-face-backlash-from-chinas-censors-1426541126}, +} + +@misc{PrinceCloudflareHackerNews, + author = {Matthew Prince}, + month = mar, + year = 2015, + howpublished = {Hacker News comment}, + url = {https://news.ycombinator.com/item?id=9234367}, +} + +@techreport{Nasr2017a, + title = {Enemy At the Gateways: A Game Theoretic Approach to Proxy Distribution}, + author = {Milad Nasr and Sadegh Farhang and Amir Houmansadr and Jens Grossklags}, + institution = {}, + month = sep, + year = 2017, + url = {https://arxiv.org/abs/1709.04030v1}, +} + +@book{OpenNet2008AccessDenied, + editor = {Deibert, Ronald and Palfrey, John G. and Rohozinski, Rafal and Zittrain, Jonathan}, + title = {Access denied: the practice and policy of global {Internet} filtering}, + publisher = {MIT Press}, + year = 2008, + address = {Cambridge, Mass}, + isbn = {978-0-262-54196-1}, + url = {http://access.opennet.net/?page_id=61}, +} + +@misc{tor-dev-meek-azure-persistent, + title = {Big performance improvement for meek-azure}, + month = apr, + year = 2015, + howpublished = {tor-dev mailing list}, + url = {https://lists.torproject.org/pipermail/tor-dev/2015-April/008637.html}, +} + +@techreport{citizenlab-great-cannon, + author = {Bill Marczak and Nicholas Weaver and Jakub Dalek and Roya Ensafi and David Fifield and Sarah McKune and Arn Rey and John Scott-Railton and Ron Deibert and Vern Paxson}, + title = {China's Great Cannon}, + month = apr, + year = 2015, + institution = {The Citizen Lab}, + url = {https://citizenlab.ca/2015/04/chinas-great-cannon/}, +} + +@misc{greatfire-we-are-under-attack, + author = {Charlie Smith}, + month = mar, + year = 2015, + howpublished = {GreatFire}, + url = {https://en.greatfire.org/blog/2015/mar/we-are-under-attack}, +} + +@misc{tor-dev-meek-azure-outage-201508, + author = {David Fifield}, + title = {Outage of meek-azure}, + month = aug, + year = 2015, + howpublished = {tor-dev mailing list}, + url = {https://lists.torproject.org/pipermail/tor-talk/2015-August/038780.html}, +} + +@misc{google-cloud-service-terms-20150326000133, + author = {{Google Cloud Platform}}, + title = {Service Specific Terms}, + month = mar, + year = 2015, + url = {https://web.archive.org/web/20150326000133/https://cloud.google.com/terms/service-terms}, +} + +@misc{fireeye-apt29_domain_frontin, + author = {Matthew Dunwoody}, + title = {{APT29} Domain Fronting With {TOR}}, + month = mar, + year = 2017, + howpublished = {FireEye Threat Research Blog}, + url = {https://www.fireeye.com/blog/threat-research/2017/03/apt29_domain_frontin.html}, +} diff --git a/thesis.tex b/thesis.tex index 7f9b04b..35b6d91 100644 --- a/thesis.tex +++ b/thesis.tex @@ -20,6 +20,9 @@ % biblatex manual: % "When using the hyperref package, it is preferable to load it after biblatex." \usepackage[backend=biber,maxbibnames=99,backref=true]{biblatex} +% Remove "URL: " prefix from bibliography URLs. Original declaration is from +% texmf-dist/tex/latex/biblatex/biblatex.def. +\DeclareFieldFormat{url}{\url{#1}} \bibliography{local,censor-local,censor} \usepackage[hidelinks]{hyperref} \urlstyle{same} @@ -411,6 +414,7 @@ obfuscated protocol then prevents blocking by content. \item rBridge~\cite{Wang2013a} \item Salmon~\cite{Douglas2016a} \item Hyphae~\cite{LovecruftDeValence2017a} +\item Enemy at the Gateways~\cite{Nasr2017a} \end{itemize} @@ -680,6 +684,8 @@ They demonstrate that the two-level nature of the blocking system unintentionally makes it an oracle that can reveal the IP addresses of sites in the secret blocking list. +\cite{OpenNet2008AccessDenied} + For a decade, the OpenNet Initiative produced reports on Internet filtering and surveillance in dozens of countries, until it ceased operation in 2014. @@ -1255,6 +1261,472 @@ which I coauthored with Chang Lan, Rod Hynes, Percy Wegmann, and Vern Paxson. \section{An unvarnished history of meek deployment} +\begin{figure} +\centering +\includegraphics{figures/metrics-clients-meek} +\caption{ +Estimated mean number of concurrent users +of the meek pluggable transport. +} +\label{fig:metrics-clients-meek} +\end{figure} + +Fielding a circumvention and keeping it running is full of unexpected challenges. +At the time of the publication of the domain fronting paper~\cite{Fifield2015a-local} in 2015, +meek had been deployed only a year and a half. +Here I will recount the entire history of the deployment project, +from inception to the present, a period of over three years. +I have been the main developer and project leader +of meek over its entire existence. +I hope to share the benefit of my experience +by commentating the history with surprises +and lessons learned. +Figure~\ref{fig:metrics-clients-meek} +shows the estimated concurrent number of users +of meek over its entire existence. +The counts come from Tor Metrics~\cite{tor-tr-2012-10-001}. + +The prehistory of meek begins in 2013 with flash proxy. +Flash proxy clients need a secure way to register their address +to a central facilitator, in order that flash proxies can connect back to them. +Initially we had only two means of registration: +flashproxy-reg-http, sending client registrations directly over HTTP; +and flashproxy-reg-email, sending client registrations to a special email address. +We knew that flashproxy-reg-http was easily blockable; +flashproxy-reg-email had good blocking resistance +but was somewhat slow and complicated, +requiring a server to poll for new messages. +At some point, Jacob Appelbaum showed me an example of using +domain fronting---though we didn't have a name for it then---to +access a simple HTTP-rewriting proxy on App Engine. +I eventually realized that the same trick would work for flash proxy rendezvous. +I proposed a design~\cite{tor-trac-8860} in May 2013 +and within a month Arlo Breault had written flashproxy-reg-appspot, +which worked just like flashproxy-reg-http, +but fronted through \nolinkurl{www.google.com} rather than +contacting the registration server directly. +The fronting-based registration became flash proxy's preferred method, +being faster and simpler than the email-based one. + +The development into a full-fledged bidirectional transport seems slow, in retrospect. +All the pieces were there; it was only a matter of putting them together. +I did not appreciate the potential of domain fronting when I saw it for the first time. +Even after the introduction of flashproxy-reg-appspot, +months passed before the beginning of meek. +The whole idea behind flash proxy registration was that the registration channel +could be of low quality---unidirectional, low-bandwidth, and high-latency---because +it was only used to bootstrap into a more capable channel (WebSocket). +Email fits well into this model: +not good for a general-purpose channel, +just good enough for rendezvous. +The fronting-based HTTP channel, however, +was much more capable, +bidirectional with reasonably high performance. +Rather than handing off the client to a flash proxy, +it should be possible to carry all the client's traffic through the same +domain-fronted channel. +It was during this time that I first became aware of GoAgent +through the ``Collateral Freedom'' report of Robinson et~al.~\cite{Robinson2013a}. +According to the report, GoAgent, +which used a less secure form of domain fronting than what meek would have, +was the most used circumvention tool among +a group of users in China. +I read the source code of GoAgent in October 2013 +and wrote ideas about writing a similar pluggable transport~\cite{goagent-wiki} +which would become meek. + +I lost time in premature optimization of meek's network performance. +I was thinking about the request--response nature of HTTP, +and how requests and responses could conceivably arrive out of order +(even if reordering was unlikely to occur in practice, because of the +keepalive connections and HTTP pipelining). +I made several attempts at a TCP-like reliability and sequencing layer, +none of which were satisfactory. +I wrote a simplified experimental prototype called ``meeker,'' +which simply prepended an HTTP header before the client and server streams, +but meeker only worked for direct connections, +not through an HTTP-aware intermediary like App Engine. +When I explained these difficulties to George Kadianakis in December 2013, +he advised me to forget the complexity and implement the simplest +thing that could work, which was good advice. +I started working on a version that strictly serialized request--response pairs, +which architecture meek still uses today. + + +\subsection{2014} + +According to the Git revision history, +I started working on the source code of meek proper on January 26, 2014. +I made the first public announcement on January 31, 2014, +in a post to the \mbox{tor-dev} mailing list\index{tor-dev mailing list} +titled ``A simple HTTP transport and big ideas''~\cite{tor-dev-meek-announcement}. +(If the development time seems short, +it's only because months of prototypes and false starts.) +In the post, I linked to the source code, +described the protocol, +and explained how to try it, +using an App Engine instance I had set up shortly before. +At this time there was no web browser TLS camouflage, +and only App Engine was supported. +I was not yet using the term ``domain fronting.'' +The ``big ideas'' of the title were as follows: +we could run one big public bridge rather than relying +on multiple smaller bridges as other transports did; +a web server with a PHP ``reflector'' script could do the same forwarding as a CDN, +providing a diversity of access points even without domain fronting; +we could combine meek with authentication and serve a 404 +to unauthenticated users; +and Cloudflare and other CDNs are alternatives to App Engine. +We did end up running a public bridge for public benefit +(and worrying over how to pay for it), +and deploying on platforms other than App Engine +(with Tor we never used Cloudflare specifically, but did others). +Arlo Breault would write a PHP reflector, +though there was never a repository of public meek reflectors +as there were for other types of Tor bridge. +Combining meek with authentication never happened; +it was never needed for our public domain-fronted instances +because active probing doesn't help the censor in those cases anyway. + +During the spring 2014 semester (January--May) +I was enrolled in Vern Paxson's Internet/Network Security course +along with fellow student Chang Lan. +We made the development and security evaluation of meek +our course project. +During this time we built browser TLS camouflage extensions, +tested and polished the code, +and ran performance tests. +Our final report, +``Blocking-resistant communication through high-value web services,'' +was the kernel of our later paper on domain fronting. + +% I began the process of getting +% meek integrated into Tor Browser in February 2014~\cite{tor-trac-10935}. +% A lot happened in the next few months, +% before the integration was finished in August 2014. +% I am grateful to the Tor Browser developers +% Kathleen Brade, Georg Koppen, and Mark Smith; +% and the volunteers on the \mbox{tor-qa} mailing list\index{tor-qa mailing list} +% for their assistance during this time especially. +% Along the way, I extended the base meek client with +% a browser-based TLS camouflage module +% using the same Firefox core on which Tor Browser is based. + +In March 2014, I met some developers of Lantern +at a one-day hackathon sponsored by OpenITP~\cite{openitp-usability-hackathon}. +Lantern developer Percy Wegmann and I realized that the meek code I had been working on +could act as a glue layer between Tor and the HTTP proxy exposed by Lantern, +in effect allowing you to use Lantern as a pluggable transport for Tor. +We worked out a prototype and wrote a summary of the process~\cite{tor-dev-howto-use-lantern-pt}. +Even though our specific application that day did not use domain fronting, +the early contact with other circumvention developers was valuable. + +June 2014 brought a surprise: +the Great Firewall of China blocked all Google services~\cite{google-transparency-cn-201405,greatfire-google-block-cn}. +It would be hubris to think that it was in response +to the nascent deployment of meek on App Engine; +a more likely cause was Google's decision to start using HTTPS +for web searches, +which would foil URL keyword filtering. +Nevertheless, the blocking cast doubt on the feasibility of domain fronting: +I had believed that blocking all of Google would be too costly +in terms of collateral damage to be sustained for long +by any censor, even the Great Firewall, +and that belief was wrong. +At least, we now needed fronts other than Google +in order to have any claim of effective circumvention in China. +For that reason, I set up additional backends: +Amazon CloudFront and Microsoft Azure. +When meek made its debut in Tor Browser, +it would offer three modes: meek-google, meek-amazon, and meek azure. + +Google sponsored a summit of circumvention researchers +in June 2014. +I presented domain fronting there. +(By this time I had started using the term ``domain fronting,'' +realizing that what I had been working on needed a specific name. +I tried to separate the idea ``domain fronting'' +from the implementation ``meek,'' +but the terms have sometimes gotten confused in discourse.) +Developers from Lantern and Psiphon where there---I was +pleased to learn that Psiphon had already implemented and deployed +domain fronting, after reading my mailing list posts. +The meeting started a fruitful collaboration: +Percy Wegmann from Lantern +and Rod Hynes from Psiphon +would later be among my coauthors on the paper +on domain fronting~\cite{Fifield2015a-local}. + +Chang, Vern, and I submitted a paper on domain fronting to the +Network and Distributed System Security Symposium (NDSS) +in August 2014, whence it was rejected. + +The first public release of Tor Browser that had +a built-in easy-to-use meek client was version +4.0-alpha-1 on August 12, 2014~\cite{tor-blog-tor-browser-364-and-40-alpha-1-are-released}. +This was an alpha release, used by fewer users than the stable release. +I made a blog post explaining how to use it a few days later~\cite{tor-blog-how-use-meek-pluggable-transport}. +The release and blog post had a positive effect on the number of users, +however the absolute numbers are uncertain, +because of a configuration error I had made on the meek bridge. +I was running the meek bridge and the flash proxy bridge +on the same instance of Tor; +and because of how Tor's statistics are aggregated, +the counts were spuriously correlated~\cite{tor-dev-user-counting-confusion}. +I switched the meek bridge to a separate instance of Tor +on September 15; +numbers after that date are more trustworthy. +In any case, the usage before this first release was tiny: +the App Engine bill (\$0.12/GB, with one~GB free each day) +was less than \$1.00 per month for the first seven months of 2014~\cite[\S~Costs]{meek-wiki}. +In August, the cost started to be nonzero every day, +and would continue to rise from there. + +Tor Browser 4.0~\cite{tor-blog-tor-browser-40-released} +was released on October 15, 2014. +It was the first stable (not alpha) release to have meek, +and it had an immediate effect on the number of users: +the estimate jumped from 50 to~500 within a week. +(The increase was partially conflated with +a failure of the meek-amazon bridge to publish statistics +before that date, but the other bridge, +servicing meek-google and meek-azure, +individually showed the same increase.) +It was a lesson in user behavior: +although there had been a working implementation +in the alpha release for two months already, +evidently a large number of users did not know of it +or chose not to try it. +At that time, the other transports available were +obfs3, FTE, ScrambleSuit, and flash proxy. + + +\subsection{2015} + +Through the first part of 2015, the estimated number of +simultaneous users continued to grow, reaching about 2,000, +as we fixed bugs and Tor Browser had further releases. + +We submitted a revised version of the domain fronting~\cite{Fifield2015a-local}, +now with contributions from Psiphon and Lantern, +to the Privacy Enhancing Technologies Symposium, +where it was accepted +and appeared on June~30 at the symposium. + +The increasing use of domain fronting by various +circumvention tools begain to attract more attention. +A March 2015 article by Eva Dou and Alistair Barr +in the Wall Street Journal~\cite{DouBarrWallStreetJournal} +described domain fronting and ``collateral freedom'' in general, +depicting cloud service providers as being caught in the crossfire +between censors and circumventors. +The journalists had contacted me but I declined to be interviewed. +The CEO of CloudFlare, through whose service Lantern had been fronting, +said that recently they had altered their systems to prevent domain fronting +by enforcing a match between SNI and Host header~\cite{PrinceCloudflareHackerNews}. +GreatFire, an anticensorship organization that had also been mentioned, +shortly thereafter experienced a new type of denial-of-service attack~\cite{greatfire-we-are-under-attack}, +caused by a Chinese network attack system later called the ``Great Cannon''~\cite{citizenlab-great-cannon}. +They blamed the attack on the attention brought by the news article. + +Since initial deployment, the Azure backend +had been slower, with fewer users, than the other two options, +App Engine and CloudFront. +For months I had chalked it up to limitations of the platform. +In April 2015, though, I found the real source of the problem: +the code I had written to run on Azure, +the code that receives domain-fronted HTTP requests and forwards them +to the meek bridge, +was not reusing TCP connections. +For every outgoing request, +the Azure code was doing a fresh TCP and TLS handshake---causing a bottleneck +at the CPU of the bridge, coping with all the incoming TLS. +When I fixed the Azure code to reuse connections~\cite{tor-dev-meek-azure-persistent}, +the number of users (overall, not only for Azure) +had a sudden jump, reaching 6,000 in less than a week. +Evidently, we had been leaving users on the table +by having one of the backends not run as fast as possible. + +The deployment of domain fronting was being partly supported by a +\$500/month grant from Google. +Already the February 2015, the monthly cost for App Engine alone +began to exceed that amount~\cite[\S~Costs]{meek-wiki}. +In an effort to control costs, in May 2015 we began to rate-limit the +App Engine and CloudFront bridges, +deliberately slowing the service +so that fewer would use it. +Until October 2015, the Azure bridge was on a research grant +provided by Microsoft, so we allowed it to run as fast as possible, +but when the grant expired, we rate-limited the Azure bridge as well. +The rate-limiting explains the relative flatness of the user graph +from May to the end of~2015. + +Google changed the terms of service governing App Engine in 2015, +adding a paragraph that seemed to prohibit running a proxy service~\cite{google-cloud-service-terms-20150326000133}: +% I and Yawning got notice of the change on 2015-05-20. The notice said: +% • Add a restriction against using the Google Cloud Platform services to +% provide network transport or sell bandwidth +% Yawning quoted paragraph 10.2, while for me it was paragraph 13.2. +% However the Wayback Machine version of 2015-03-26 already has the new paragraph (numbere 10.2). +% The version of 2015-01-14 does not have it. https://web.archive.org/web/20150114140104/https://cloud.google.com/terms/service-terms +% The later version of 2015-04-06 also has it as paragraph 10.2. https://web.archive.org/web/20150406172008/https://cloud.google.com/terms/service-terms +% The still later version of 2015-07-14 has it as paragraph 13.2. https://web.archive.org/web/20150714102030/https://cloud.google.com/terms/service-terms +\begin{quote} +\emph{Networking}. Customer will not, and will not allow third parties +under its control to: (i)~use the Services to provide a service, +Application, or functionality of network transport or transmission +(including, but not limited to, IP transit, virtual private networks, +or content delivery networks); or (ii)~sell bandwidth from the +Services. +\end{quote} +This was an uncomfortable time: +we seemed to have the support of Google, +but the terms of service said otherwise. +I contacted Google and asked for clarification or guidance, +in the meantime leaving meek-google running; +however I never got an answer to my questions. +The point became moot a year later, +when Google shut down our App Engine project, +for another reason altogether. + +By this time we had not received any reports of any type of blocking of domain fronting. +We did, however, suffer a few accidental outages +(which look just like blocking, from a user's point of view). +Between July~20 and August~14, an account transition error +left the Azure configuration broken~\cite{tor-dev-meek-azure-outage-201508}. +I set up another configuration on Azure and published instructions on how to use it, +but it would not be available to the majority of users until the next release of Tor Browser, +which happened on August~11. +Between September~30 and October~9, the CloudFront-fronted bridge +was effectively down because of an expired TLS certificate. +When it rebooted on October~9, an adminstrative oversight +caused its Tor relay identity fingerprint changed---meaning +that clients expecting the former fingerprint would refuse +to connect to it~\cite{tor-trac-17473}. +The situation was not fully resolved until November~4 +with the next release of Tor Browser: +cascading failured led to over a month of downtime. + +One of the benefits of building a circumvention system for Tor +is the easy integration with Tor Metrics---the source of the user +number estimates in this section. +Since the beginning of meek's deployment, we had known about a problem +with the way it integrates with Tor Metrics' data collection. +Tor pluggable transports geolocate the client's IP address +in order to aggregate statistics by country. +But when a meek bridge receives a connection, the ``client IP address'' +it sees is not that of the true client, but rather is +some cloud server, the intermediary through which the domain-fronted +traffic passes. +So the total counts were fine, but the per-country counts were meaningless. +For example, because App Engine's servers were located in the U.S., +every meek-google connection was being counted in the U.S. bucket. +By the end of 2015, meek users were a large enough fraction (about~20\%) +of all bridge users, that they were really starting to skew the overall per-country counts. +I wrote a patch to have the client's true IP address forwarded +through the network intermediary in a special HTTP header, +which fixed the per-country counts from then on. + + +\subsection{2016} + +In mid-January 2016 the Tor Project asked me to raise +the rate limits on the meek bridges, in anticipation +of rumored attempts to block Tor in Egypt. +(The blocking attempts were in turn rumored to be caused +by Facebook's integration of Tor into their mobile application.) +I had the bridge operators raise the rate limits +from approximately 1~MB/s to 3~MB/s. +The effect of the relaxed rate limits was immediate: +the count shot up as high 15,000 simultaneous users, +briefly becoming Tor's most-used pluggable transport, +before settling in around 10,000. + +The first action that may have been a deliberate +attempt to block domain fronting came on January~29, 2016, +when the Great Firewall of China blocked one of the edge servers +of the Azure CDN. +The blocking was by IP address, a severe method: +not only the domain name we were using for domain fronting, +but also thousands of other names, +became inaccessible. +The block lasted about four days. +On February~2, the server changed its IP address, +incrementing the final octet from\ .200 to~.201, +causing it to become unblocked. +I am aware of no similar incidents before or since. + +The next surprise was on May~13, 2016. +meek's App Engine backend stopped working and I got a notice saying: +\begin{quote} +We've recently detected some activity on your Google Cloud Platform/API Project ID meek-reflect that appears to violate our Terms of Service. Please take a moment to review the Google Cloud Platform Terms of Service or the applicable Terms of Service for the specific Google API you are using. + +Your project is being suspended for committing a general terms of service violation. + +We will delete your project unless you correct the violation by filling in the appeals form available on the project page of Developers Console to get in touch with our team so that we can provide you with more details. +\end{quote} +My first thought was that it had to do with the changes +to the terms of service that had happened the previous year---but +the true cause was unexpected. +I tried repeatedly to contact Google and learn the nature +of the ``general'' violation, but was stonewalled. +None of my inquiries received so much as an acknowlegement. +It as not until June~18 that I got some insight +as to what happened, +through an unofficial channel. +Some botnet had apparently been misusing meek for command and control purposes; +and its operators hadn't even bothered to set up their own App Engine project. +They were using the service that we had been operating for the public. +Although we may have been able to reinstate the meek-google service, +seeing as the suspension was the result of someone else's botnet, +with the already uncertain standing with regard to the terms of service +I didn't have the heart to pursue it. +meek-google remained off and users migrated to +meek-amazon or meek-azure. +It turned out, later, that it had been no common botnet +misusing meek-google, but an organized political hacker group, +known as Cozy Bear\index{Cozy Bear}\index{APT29|see {Cozy Bear}} +or APT29. +Matthew Dunwoody presented observations to that effect +in a FireEye blog post~\cite{fireeye-apt29_domain_frontin} +in March 2017. +He and Nick Carr had presented those findings at DerbyCon +in September 2016, but I was not aware of them until the blog post. +Malware would install a backdoor that operated over a Tor +onion service, and used meek for camouflage. + +TLS fingerprinting +Cyberoam May 2016 \url{https://groups.google.com/d/topic/traffic-obf/BpFSCVgi5rs} \url{https://lists.torproject.org/pipermail/tor-talk/2016-May/040898.html} +FortiGuard July 2016 \url{https://groups.google.com/d/topic/traffic-obf/fwAN-WWz2Bk} +Kazakhstan < Dec 2016 \url{https://trac.torproject.org/projects/tor/ticket/20348#comment:142} + +Brazil + + +\bigskip + + +Cert reload by Yawning Angel, +Let's Encrypt\index{Let's Encrypt} support based on a patch by George Tankersley. + + +GAEuploader + +Funding sources + +% ||2015-12-25 || || ||meek ||Established an unthrottled bridge [https://atlas.torproject.org/#details/C20658946DD706A7A2181159A1A04CD838570D04 C20658946DD706A7A2181159A1A04CD838570D04] for people who set up their own meek CDN configuration. || || +% ||2016-07-21 ||2017-03-03 ||br ||meek ||Sustained increase in meek users in Brazil. Locals believe that they are not actual users, rather bots or something like that. End date coincides with shutdown of meek-azure before migration. ||[https://metrics.torproject.org/userstats-bridge-combined.html?start=2016-06-01&end=2017-04-01&country=br graph] || +% ||2016-10-19 ||2016-11-10 || ||meek ||Large decrease in meek users, perhaps caused by problems in Orbot 15.0.2 BETA 1 that were fixed in Orbot 15.2.0 RC8. ||[https://bugs.torproject.org/20495 ticket] [https://lists.torproject.org/pipermail/tor-project/2016-October/000764.html initial email] [https://lists.torproject.org/pipermail/tor-project/2016-November/000778.html followup email] [https://groups.google.com/d/msg/traffic-obf/CSJLt3t-_OI/FnAqWqquAwAJ Orbot mail] || +% ||2016-11-22 || || ||meek ||Decreased the rate limit on the meek-amazon bridge to 2.0 MB/s, from 3.0 MB/s. || || +% ||2017-01-09 || || ||meek ||Decreased the rate limit on the meek-azure bridge to 2.0 MB/s, from 3.0 MB/s. || || +% ||2017-03-03 17:32:00 || || ||meek ||Stopped the meek-azure CDN endpoint az668014.vo.msecnd.net. ||[https://lists.torproject.org/pipermail/tor-project/2017-March/000981.html mailing list post] || +% ||2017-03-03 17:34:00 || || ||meek ||Stopped the meek-azure CDN endpoint az786092.vo.msecnd.net. ||[https://lists.torproject.org/pipermail/tor-project/2017-March/000981.html mailing list post] || +% ||2017-03-03 17:36:00 || || ||meek ||Stopped the (unused) meek-azure CDN endpoint meek-reflect.azureedge.net. ||[https://lists.torproject.org/pipermail/tor-project/2017-March/000981.html mailing list post] || +% ||2017-03-07 18:32:11 || || ||meek scramblesuit ||Tor Browser 6.5.1 is released, containing the new meek-azure CDN configuration, and removing the last remaining scramblesuit bridge. ||[https://blog.torproject.org/blog/tor-browser-651-released blog post] [https://bugs.torproject.org/21342 meek ticket] [https://bugs.torproject.org/21536 scramblesuit ticket] || +% ||2017-03-22 || || ||meek ||Orbot 15.4.0 beta-2 multi is released, containing the new meek-azure CDN configuration. ||[https://lists.mayfirst.org/pipermail/guardian-dev/2017-March/005220.html mailing list post] [https://github.com/n8fr8/orbot/commit/6496cb11d61e0e42c48569c9eae303e0cd625e85 commit] || +% ||2017-06-20 00:09:50 ||2017-06-20 00:32:21 || ||meek ||Outage of meek-azure bridge. ||[https://lists.torproject.org/pipermail/tor-project/2017-June/001209.html start] [https://lists.torproject.org/pipermail/tor-project/2017-June/001209.html end] || +% ||2017-07-29 07:19:00 ||2017-08-17 03:32:00 || ||meek ||Outage of meek-amazon bridge, caused by an expired certificate. ||[https://atlas.torproject.org/#details/F4AD82B2032EDEF6C02C5A529C42CFAFE516564D Atlas] [https://crt.sh/?id=130970041 expired certificate] [https://crt.sh/?id=192217077 new certificate] || + \chapter{Building circumvention systems}