From 343bdd526674ffed3b1d25604dd05f96197d2295 Mon Sep 17 00:00:00 2001 From: Barna Csorogi Date: Sat, 28 Mar 2015 23:07:40 +0100 Subject: [PATCH] initial commit --- LICENSE | 13 +++++ README | 21 ++++++++ compy.go | 65 ++++++++++++++++++++++++ proxy/certfaker.go | 35 +++++++++++++ proxy/mitmlistener.go | 45 +++++++++++++++++ proxy/proxy.go | 111 +++++++++++++++++++++++++++++++++++++++++ proxy/response.go | 82 ++++++++++++++++++++++++++++++ transcoder/gif.go | 19 +++++++ transcoder/gzip.go | 39 +++++++++++++++ transcoder/identity.go | 13 +++++ transcoder/jpeg.go | 32 ++++++++++++ transcoder/minify.go | 34 +++++++++++++ transcoder/png.go | 19 +++++++ transcoder/text.go | 34 +++++++++++++ 14 files changed, 562 insertions(+) create mode 100644 LICENSE create mode 100644 README create mode 100644 compy.go create mode 100644 proxy/certfaker.go create mode 100644 proxy/mitmlistener.go create mode 100644 proxy/proxy.go create mode 100644 proxy/response.go create mode 100644 transcoder/gif.go create mode 100644 transcoder/gzip.go create mode 100644 transcoder/identity.go create mode 100644 transcoder/jpeg.go create mode 100644 transcoder/minify.go create mode 100644 transcoder/png.go create mode 100644 transcoder/text.go diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..b32e0bc --- /dev/null +++ b/LICENSE @@ -0,0 +1,13 @@ +Copyright (c) 2015, Barna Csorogi + +Permission to use, copy, modify, and/or distribute this software for any +purpose with or without fee is hereby granted, provided that the above +copyright notice and this permission notice appear in all copies. + +THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES +WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF +MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR +ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES +WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN +ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF +OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. diff --git a/README b/README new file mode 100644 index 0000000..7d66dc9 --- /dev/null +++ b/README @@ -0,0 +1,21 @@ +1, Compy +Compy is an HTTP/HTTPS proxy with mitm support and basic content compression/transcoding capabilities. + +2, Features: +- HTTPS proxy (encrypted connection between client and proxy) +- man in the middle support +- gzip compression +- transcode animated gif to static image +- transcode jpeg to desired quality using libjpeg +- transcode png +- html/css/js minification + +3, Usage +See compy --help + +4, Credits +https://github.com/pixiv/go-libjpeg +https://github.com/tdewolff/minify + +5, License +See LICENSE diff --git a/compy.go b/compy.go new file mode 100644 index 0000000..aaf87ea --- /dev/null +++ b/compy.go @@ -0,0 +1,65 @@ +package main + +import ( + "flag" + "fmt" + "github.com/barnacs/compy/proxy" + tc "github.com/barnacs/compy/transcoder" + "log" +) + +var ( + host = flag.String("host", ":9999", "") + cert = flag.String("cert", "", "proxy cert path") + key = flag.String("key", "", "proxy cert key path") + ca = flag.String("ca", "", "CA path") + caKey = flag.String("cakey", "", "CA key path") + + jpeg = flag.Int("jpeg", 50, "jpeg quality (1-100, 0 to disable)") + gif = flag.Bool("gif", true, "transcode gifs into static images") + png = flag.Bool("png", true, "transcode png") + minify = flag.Bool("minify", false, "minify css/html/js - WARNING: tends to break the web") +) + +func main() { + flag.Parse() + + p := proxy.New() + + if *ca != "" { + if err := p.EnableMitm(*ca, *caKey); err != nil { + fmt.Println("not using mitm:", err) + } + } + + if *jpeg != 0 { + p.AddTranscoder("image/jpeg", tc.NewJpeg(*jpeg)) + } + if *gif { + p.AddTranscoder("image/gif", &tc.Gif{}) + } + if *png { + p.AddTranscoder("image/png", &tc.Png{}) + } + + var ttc proxy.Transcoder + if *minify { + ttc = &tc.Gzip{tc.NewMinifier(), false} + } else { + ttc = &tc.Gzip{&tc.Identity{}, true} + } + + p.AddTranscoder("text/css", ttc) + p.AddTranscoder("text/html", ttc) + p.AddTranscoder("text/javascript", ttc) + p.AddTranscoder("application/javascript", ttc) + p.AddTranscoder("application/x-javascript", ttc) + + var err error + if *cert != "" { + err = p.StartTLS(*host, *cert, *key) + } else { + err = p.Start(*host) + } + log.Fatalln(err) +} diff --git a/proxy/certfaker.go b/proxy/certfaker.go new file mode 100644 index 0000000..be53ce0 --- /dev/null +++ b/proxy/certfaker.go @@ -0,0 +1,35 @@ +package proxy + +import ( + "crypto" + "crypto/tls" + "crypto/x509" +) + +type certFaker struct { + ca *x509.Certificate + key crypto.PrivateKey +} + +func newCertFaker(caPath, keyPath string) (*certFaker, error) { + certs, err := tls.LoadX509KeyPair(caPath, keyPath) + if err != nil { + return nil, err + } + ca, err := x509.ParseCertificate(certs.Certificate[0]) + if err != nil { + return nil, err + } + return &certFaker{ + ca: ca, + key: certs.PrivateKey, + }, nil +} + +func (cf *certFaker) FakeCert(original *x509.Certificate) (*tls.Certificate, error) { + fakeCertData, err := x509.CreateCertificate(nil, original, cf.ca, cf.ca.PublicKey, cf.key) + return &tls.Certificate{ + Certificate: [][]byte{fakeCertData}, + PrivateKey: cf.key, + }, err +} diff --git a/proxy/mitmlistener.go b/proxy/mitmlistener.go new file mode 100644 index 0000000..519274c --- /dev/null +++ b/proxy/mitmlistener.go @@ -0,0 +1,45 @@ +package proxy + +import ( + "crypto/tls" + "net" +) + +type mitmListener struct { + c chan net.Conn + cf *certFaker +} + +func newMitmListener(cf *certFaker) *mitmListener { + return &mitmListener{ + c: make(chan net.Conn), + cf: cf, + } +} + +func (l *mitmListener) Accept() (net.Conn, error) { + return <-l.c, nil +} + +func (l *mitmListener) Close() error { + return nil +} + +func (l *mitmListener) Addr() net.Addr { + return nil +} + +func (l *mitmListener) Serve(conn net.Conn, host string) (net.Conn, error) { + sconn, err := tls.Dial("tcp", host, nil) + if err != nil { + return nil, err + } + fakeCert, err := l.cf.FakeCert(sconn.ConnectionState().PeerCertificates[0]) + if err != nil { + sconn.Close() + return nil, err + } + tlsconf := &tls.Config{Certificates: []tls.Certificate{*fakeCert}} + l.c <- tls.Server(conn, tlsconf) + return sconn, nil +} diff --git a/proxy/proxy.go b/proxy/proxy.go new file mode 100644 index 0000000..c183215 --- /dev/null +++ b/proxy/proxy.go @@ -0,0 +1,111 @@ +package proxy + +import ( + "fmt" + "log" + "net/http" +) + +type Proxy struct { + transcoders map[string]Transcoder + ml *mitmListener +} + +type Transcoder interface { + Transcode(*ResponseWriter, *ResponseReader) error +} + +func New() *Proxy { + p := &Proxy{ + transcoders: make(map[string]Transcoder), + ml: nil, + } + return p +} + +func (p *Proxy) EnableMitm(ca, key string) error { + cf, err := newCertFaker(ca, key) + if err != nil { + return err + } + p.ml = newMitmListener(cf) + go http.Serve(p.ml, p) + return nil +} + +func (p *Proxy) AddTranscoder(contentType string, transcoder Transcoder) { + p.transcoders[contentType] = transcoder +} + +func (p *Proxy) Start(host string) error { + return http.ListenAndServe(host, p) +} + +func (p *Proxy) StartTLS(host, cert, key string) error { + return http.ListenAndServeTLS(host, cert, key, p) +} + +func (p *Proxy) ServeHTTP(w http.ResponseWriter, r *http.Request) { + if err := p.handle(w, r); err != nil { + log.Printf("%s while serving request: %s", err, r.URL) + } +} + +func (p *Proxy) handle(w http.ResponseWriter, r *http.Request) error { + if r.Method == "CONNECT" { + return p.handleConnect(w, r.Host) + } + resp, err := forward(r) + if err != nil { + w.WriteHeader(http.StatusInternalServerError) + return fmt.Errorf("error forwarding request: %s", err) + } + defer resp.Body.Close() + return p.proxyResponse(newResponseWriter(w), newResponseReader(resp)) +} + +func forward(r *http.Request) (*http.Response, error) { + if r.URL.Scheme == "" { + r.URL.Scheme = "https" + } + if r.URL.Host == "" { + r.URL.Host = r.Host + } + r.RequestURI = "" + return http.DefaultTransport.RoundTrip(r) +} + +func (p *Proxy) proxyResponse(w *ResponseWriter, r *ResponseReader) error { + w.takeHeaders(r) + transcoder, found := p.transcoders[r.ContentType()] + if !found { + return w.writeFrom(r) + } + w.setChunked() + if err := transcoder.Transcode(w, r); err != nil { + return fmt.Errorf("transcoding error: %s", err) + } + return nil +} + +func (p *Proxy) handleConnect(w http.ResponseWriter, host string) error { + if p.ml == nil { + return fmt.Errorf("CONNECT received but mitm is not enabled") + } + h, ok := w.(http.Hijacker) + if !ok { + return fmt.Errorf("connection cannot be hijacked") + } + w.WriteHeader(http.StatusOK) + conn, _, err := h.Hijack() + if err != nil { + return err + } + sconn, err := p.ml.Serve(conn, host) + if err != nil { + conn.Close() + return err + } + sconn.Close() // TODO: reuse this connection for https requests + return nil +} diff --git a/proxy/response.go b/proxy/response.go new file mode 100644 index 0000000..8f6510d --- /dev/null +++ b/proxy/response.go @@ -0,0 +1,82 @@ +package proxy + +import ( + "io" + "mime" + "net/http" +) + +type ResponseReader struct { + io.Reader + r *http.Response +} + +func newResponseReader(r *http.Response) *ResponseReader { + return &ResponseReader{ + Reader: r.Body, + r: r, + } +} + +func (r *ResponseReader) ContentType() string { + cth := r.Header().Get("Content-Type") + ct, _, _ := mime.ParseMediaType(cth) + return ct +} + +func (r *ResponseReader) Header() http.Header { + return r.r.Header +} + +func (r *ResponseReader) Request() *http.Request { + return r.r.Request +} + +type ResponseWriter struct { + io.Writer + rw http.ResponseWriter + statusCode int + headersDone bool +} + +func newResponseWriter(w http.ResponseWriter) *ResponseWriter { + return &ResponseWriter{ + Writer: w, + rw: w, + } +} + +func (w *ResponseWriter) takeHeaders(r *ResponseReader) { + for k, v := range r.Header() { + for _, v := range v { + w.Header().Add(k, v) + } + } + w.WriteHeader(r.r.StatusCode) +} + +func (w *ResponseWriter) WriteHeader(s int) { + w.statusCode = s +} + +func (w *ResponseWriter) Header() http.Header { + return w.rw.Header() +} + +func (w *ResponseWriter) writeFrom(r *ResponseReader) error { + w.rw.WriteHeader(r.r.StatusCode) + _, err := io.Copy(w.rw, r) + return err +} + +func (w *ResponseWriter) setChunked() { + w.Header().Del("Content-Length") +} + +func (w *ResponseWriter) Write(b []byte) (int, error) { + if !w.headersDone { + w.rw.WriteHeader(w.statusCode) + w.headersDone = true + } + return w.Writer.Write(b) +} diff --git a/transcoder/gif.go b/transcoder/gif.go new file mode 100644 index 0000000..db0f2b9 --- /dev/null +++ b/transcoder/gif.go @@ -0,0 +1,19 @@ +package transcoder + +import ( + "github.com/barnacs/compy/proxy" + "image/gif" +) + +type Gif struct{} + +func (t *Gif) Transcode(w *proxy.ResponseWriter, r *proxy.ResponseReader) error { + img, err := gif.Decode(r) + if err != nil { + return err + } + if err = gif.Encode(w, img, nil); err != nil { + return err + } + return nil +} diff --git a/transcoder/gzip.go b/transcoder/gzip.go new file mode 100644 index 0000000..550f2f4 --- /dev/null +++ b/transcoder/gzip.go @@ -0,0 +1,39 @@ +package transcoder + +import ( + "compress/gzip" + "github.com/barnacs/compy/proxy" +) + +type Gzip struct { + proxy.Transcoder + SkipGzipped bool +} + +func (t *Gzip) Transcode(w *proxy.ResponseWriter, r *proxy.ResponseReader) error { + if t.decompress(r) { + gzr, err := gzip.NewReader(r.Reader) + if err != nil { + return err + } + defer gzr.Close() + r.Reader = gzr + r.Header().Del("Content-Encoding") + w.Header().Del("Content-Encoding") + } + if compress(r) { + gzw := gzip.NewWriter(w.Writer) + defer gzw.Flush() + w.Writer = gzw + w.Header().Set("Content-Encoding", "gzip") + } + return t.Transcoder.Transcode(w, r) +} + +func (t *Gzip) decompress(r *proxy.ResponseReader) bool { + return !t.SkipGzipped && r.Header().Get("Content-Encoding") == "gzip" +} + +func compress(r *proxy.ResponseReader) bool { + return r.Header().Get("Content-Encoding") == "" +} diff --git a/transcoder/identity.go b/transcoder/identity.go new file mode 100644 index 0000000..cc659ea --- /dev/null +++ b/transcoder/identity.go @@ -0,0 +1,13 @@ +package transcoder + +import ( + "github.com/barnacs/compy/proxy" + "io" +) + +type Identity struct{} + +func (i *Identity) Transcode(w *proxy.ResponseWriter, r *proxy.ResponseReader) error { + _, err := io.Copy(w, r) + return err +} diff --git a/transcoder/jpeg.go b/transcoder/jpeg.go new file mode 100644 index 0000000..5366788 --- /dev/null +++ b/transcoder/jpeg.go @@ -0,0 +1,32 @@ +package transcoder + +import ( + "github.com/barnacs/compy/proxy" + "github.com/pixiv/go-libjpeg/jpeg" +) + +type Jpeg struct { + decOptions *jpeg.DecoderOptions + encOptions *jpeg.EncoderOptions +} + +func NewJpeg(quality int) *Jpeg { + return &Jpeg{ + decOptions: &jpeg.DecoderOptions{}, + encOptions: &jpeg.EncoderOptions{ + Quality: quality, + OptimizeCoding: true, + }, + } +} + +func (t *Jpeg) Transcode(w *proxy.ResponseWriter, r *proxy.ResponseReader) error { + img, err := jpeg.Decode(r, t.decOptions) + if err != nil { + return err + } + if err = jpeg.Encode(w, img, t.encOptions); err != nil { + return err + } + return nil +} diff --git a/transcoder/minify.go b/transcoder/minify.go new file mode 100644 index 0000000..31bbcee --- /dev/null +++ b/transcoder/minify.go @@ -0,0 +1,34 @@ +package transcoder + +import ( + "github.com/barnacs/compy/proxy" + "github.com/tdewolff/minify" + "github.com/tdewolff/minify/css" + "github.com/tdewolff/minify/html" + "github.com/tdewolff/minify/js" + "github.com/tdewolff/parse" +) + +func init() { + parse.MaxBuf *= 8 +} + +type Minifier struct { + m minify.Minify +} + +func NewMinifier() *Minifier { + m := minify.New() + m.AddFunc("text/html", html.Minify) + m.AddFunc("text/css", css.Minify) + m.AddFunc("text/javascript", js.Minify) + m.AddFunc("application/javascript", js.Minify) + m.AddFunc("application/x-javascript", js.Minify) + return &Minifier{ + m: m, + } +} + +func (t *Minifier) Transcode(w *proxy.ResponseWriter, r *proxy.ResponseReader) error { + return t.m.Minify(r.ContentType(), w, r) +} diff --git a/transcoder/png.go b/transcoder/png.go new file mode 100644 index 0000000..7249076 --- /dev/null +++ b/transcoder/png.go @@ -0,0 +1,19 @@ +package transcoder + +import ( + "github.com/barnacs/compy/proxy" + "image/png" +) + +type Png struct{} + +func (t *Png) Transcode(w *proxy.ResponseWriter, r *proxy.ResponseReader) error { + img, err := png.Decode(r) + if err != nil { + return err + } + if err = png.Encode(w, img); err != nil { + return err + } + return nil +} diff --git a/transcoder/text.go b/transcoder/text.go new file mode 100644 index 0000000..8a96ce7 --- /dev/null +++ b/transcoder/text.go @@ -0,0 +1,34 @@ +package transcoder + +import ( + "github.com/barnacs/compy/proxy" + "github.com/tdewolff/minify" + "github.com/tdewolff/minify/css" + "github.com/tdewolff/minify/html" + "github.com/tdewolff/minify/js" + "github.com/tdewolff/parse" +) + +func init() { + parse.MaxBuf *= 8 +} + +type Text struct { + m minify.Minify +} + +func NewText() *Text { + m := minify.New() + m.AddFunc("text/html", html.Minify) + m.AddFunc("text/css", css.Minify) + m.AddFunc("text/javascript", js.Minify) + m.AddFunc("application/javascript", js.Minify) + m.AddFunc("application/x-javascript", js.Minify) + return &Text{ + m: m, + } +} + +func (t *Text) Transcode(w *proxy.ResponseWriter, r *proxy.ResponseReader) error { + return t.m.Minify(r.ContentType(), w, r) +}