From e00cf383a2258af146f1ee47e9234d55cc737d63 Mon Sep 17 00:00:00 2001 From: kataras Date: Sat, 12 Aug 2017 08:49:00 +0300 Subject: [PATCH] Vol2 : https://github.com/kataras/iris/issues/717, worked Former-commit-id: f4a19eb83a28279782b8a75ee298b38c9e180157 --- _examples/file-server/basic/main.go | 11 ++++ context/context.go | 7 +++ context/gzip_response_writer.go | 41 ++++++++----- core/router/api_builder.go | 13 ++-- core/router/fs.go | 94 +++++++++++++++++++---------- iris.go | 5 ++ 6 files changed, 119 insertions(+), 52 deletions(-) diff --git a/_examples/file-server/basic/main.go b/_examples/file-server/basic/main.go index 34c0008b..09b4faf8 100644 --- a/_examples/file-server/basic/main.go +++ b/_examples/file-server/basic/main.go @@ -9,6 +9,17 @@ func main() { app.Favicon("./assets/favicon.ico") + // enable gzip, optionally: + // if used before the `StaticXXX` handlers then + // the content byte range feature is gone. + // recommend: turn off for large files especially + // when server has low memory, + // turn on for medium-sized files + // or for large-sized files if they are zipped already, + // i.e "zippedDir/file.gz" + // + // app.Use(iris.Gzip) + // first parameter is the request path // second is the system directory // diff --git a/context/context.go b/context/context.go index 33d2956c..5793c87c 100644 --- a/context/context.go +++ b/context/context.go @@ -767,6 +767,13 @@ var LimitRequestBodySize = func(maxRequestBodySizeBytes int64) Handler { } } +// Gzip is a middleware which enables writing +// using gzip compression, if client supports. +var Gzip = func(ctx Context) { + ctx.Gzip(true) + ctx.Next() +} + // Map is just a shortcut of the map[string]interface{}. type Map map[string]interface{} diff --git a/context/gzip_response_writer.go b/context/gzip_response_writer.go index 7a60360c..6a44c63d 100644 --- a/context/gzip_response_writer.go +++ b/context/gzip_response_writer.go @@ -99,31 +99,40 @@ func (w *GzipResponseWriter) EndResponse() { w.ResponseWriter.EndResponse() } -// Write compresses and writes that data to the underline response writer +// Write prepares the data write to the gzip writer and finally to its +// underline response writer, returns the uncompressed len(contents). func (w *GzipResponseWriter) Write(contents []byte) (int, error) { // save the contents to serve them (only gzip data here) w.chunks = append(w.chunks, contents...) return len(w.chunks), nil } +// WriteNow compresses and writes that data to the underline response writer, +// returns the compressed written len. +// +// Use `WriteNow` instead of `Write` +// when you need to know the compressed written size before +// the `FlushResponse`, note that you can't post any new headers +// after that, so that information is not closed to the handler anymore. +func (w *GzipResponseWriter) WriteNow(contents []byte) (int, error) { + if w.disabled { + return w.ResponseWriter.Write(contents) + } + + w.ResponseWriter.Header().Add(varyHeaderKey, "Accept-Encoding") + w.ResponseWriter.Header().Set(contentEncodingHeaderKey, "gzip") + // if not `WriteNow` but "Content-Length" header + // is exists, then delete it before `.Write` + // Content-Length should not be there. + // no, for now at least: w.ResponseWriter.Header().Del(contentLengthHeaderKey) + + return w.gzipWriter.Write(contents) +} + // FlushResponse validates the response headers in order to be compatible with the gzip written data // and writes the data to the underline ResponseWriter. func (w *GzipResponseWriter) FlushResponse() { - if w.disabled { - w.ResponseWriter.Write(w.chunks) - // remove gzip headers: no need, we just add two of them if gzip was enabled, below - // headers := w.ResponseWriter.Header() - // headers[contentType] = nil - // headers["X-Content-Type-Options"] = nil - // headers[varyHeader] = nil - // headers[contentEncodingHeader] = nil - // headers[contentLength] = nil - } else { - // if it's not disable write all chunks gzip compressed with the correct response headers. - w.ResponseWriter.Header().Add(varyHeaderKey, "Accept-Encoding") - w.ResponseWriter.Header().Set(contentEncodingHeaderKey, "gzip") - w.gzipWriter.Write(w.chunks) // it writes to the underline ResponseWriter. - } + w.WriteNow(w.chunks) w.ResponseWriter.FlushResponse() } diff --git a/core/router/api_builder.go b/core/router/api_builder.go index bc51f9ce..dfa608e6 100644 --- a/core/router/api_builder.go +++ b/core/router/api_builder.go @@ -645,15 +645,18 @@ func (rb *APIBuilder) StaticWeb(requestPath string, systemPath string) *Route { handler := func(ctx context.Context) { h(ctx) - // re-check the content type here for any case, - // although the new code does it automatically but it's good to have it here. if ctx.GetStatusCode() >= 200 && ctx.GetStatusCode() < 400 { - if fname := ctx.Params().Get(paramName); fname != "" { - cType := TypeByFilename(fname) - ctx.ContentType(cType) + // re-check the content type here for any case, + // although the new code does it automatically but it's good to have it here. + if _, exists := ctx.ResponseWriter().Header()["Content-Type"]; !exists { + if fname := ctx.Params().Get(paramName); fname != "" { + cType := TypeByFilename(fname) + ctx.ContentType(cType) + } } } } + requestPath = joinPath(fullpath, WildcardParam(paramName)) return rb.registerResourceRoute(requestPath, handler) } diff --git a/core/router/fs.go b/core/router/fs.go index 8ff26213..8885f4d5 100644 --- a/core/router/fs.go +++ b/core/router/fs.go @@ -4,6 +4,7 @@ import ( "errors" "fmt" "io" + "io/ioutil" "mime/multipart" "net/http" "net/textproto" @@ -378,12 +379,38 @@ var errNoOverlap = errors.New("invalid range: failed to overlap") // The algorithm uses at most sniffLen bytes to make its decision. const sniffLen = 512 +func detectOrWriteContentType(ctx context.Context, name string, content io.ReadSeeker) (string, error) { + // If Content-Type isn't set, use the file's extension to find it, but + // if the Content-Type is unset explicitly, do not sniff the type. + ctypes, haveType := ctx.ResponseWriter().Header()["Content-Type"] + var ctype string + + if !haveType { + ctype = TypeByExtension(filepath.Ext(name)) + if ctype == "" { + // read a chunk to decide between utf-8 text and binary + var buf [sniffLen]byte + n, _ := io.ReadFull(content, buf[:]) + ctype = http.DetectContentType(buf[:n]) + _, err := content.Seek(0, io.SeekStart) // rewind to output whole file + if err != nil { + return "", err + } + } + + ctx.ContentType(ctype) + } else if len(ctypes) > 0 { + ctype = ctypes[0] + } + + return ctype, nil +} + // if name is empty, filename is unknown. (used for mime type, before sniffing) // if modtime.IsZero(), modtime is unknown. // content must be seeked to the beginning of the file. // The sizeFunc is called at most once. Its error, if any, is sent in the HTTP response. -func serveContent(ctx context.Context, name string, modtime time.Time, sizeFunc func() (int64, error), content io.ReadSeeker, gzip bool) (string, int) /* we could use the TransactionErrResult but prefer not to create new objects for each of the errors on static file handlers*/ { - +func serveContent(ctx context.Context, name string, modtime time.Time, sizeFunc func() (int64, error), content io.ReadSeeker) (string, int) /* we could use the TransactionErrResult but prefer not to create new objects for each of the errors on static file handlers*/ { setLastModified(ctx, modtime) done, rangeReq := checkPreconditions(ctx, modtime) if done { @@ -394,27 +421,9 @@ func serveContent(ctx context.Context, name string, modtime time.Time, sizeFunc // If Content-Type isn't set, use the file's extension to find it, but // if the Content-Type is unset explicitly, do not sniff the type. - ctypes, haveType := ctx.ResponseWriter().Header()["Content-Type"] - var ctype string - - if !haveType { - ctype = TypeByExtension(filepath.Ext(name)) - if ctype == "" { - - // read a chunk to decide between utf-8 text and binary - var buf [sniffLen]byte - n, _ := io.ReadFull(content, buf[:]) - ctype = http.DetectContentType(buf[:n]) - _, err := content.Seek(0, io.SeekStart) // rewind to output whole file - if err != nil { - return "seeker can't seek", http.StatusInternalServerError - - } - } - - ctx.ContentType(ctype) - } else if len(ctypes) > 0 { - ctype = ctypes[0] + ctype, err := detectOrWriteContentType(ctx, name, content) + if err != nil { + return "while seeking", http.StatusInternalServerError } size, err := sizeFunc() @@ -426,9 +435,6 @@ func serveContent(ctx context.Context, name string, modtime time.Time, sizeFunc sendSize := size var sendContent io.Reader = content - if gzip { - _ = ctx.GzipResponseWriter() - } if size >= 0 { ranges, err := parseRange(rangeReq, size) if err != nil { @@ -496,7 +502,6 @@ func serveContent(ctx context.Context, name string, modtime time.Time, sizeFunc } ctx.Header("Accept-Ranges", "bytes") if ctx.ResponseWriter().Header().Get(contentEncodingHeaderKey) == "" { - ctx.Header(contentLengthHeaderKey, strconv.FormatInt(sendSize, 10)) } } @@ -827,12 +832,39 @@ func serveFile(ctx context.Context, fs http.FileSystem, name string, redirect bo } ctx.Header("Last-Modified", d.ModTime().UTC().Format(ctx.Application().ConfigurationReadOnly().GetTimeFormat())) return dirList(ctx, f) - } - // serveContent will check modification time - sizeFunc := func() (int64, error) { return d.Size(), nil } - return serveContent(ctx, d.Name(), d.ModTime(), sizeFunc, f, gzip) + // if gzip disabled then continue using content byte ranges + if !gzip { + // serveContent will check modification time + sizeFunc := func() (int64, error) { return d.Size(), nil } + return serveContent(ctx, d.Name(), d.ModTime(), sizeFunc, f) + } + + // else, set the last modified as "serveContent" does. + setLastModified(ctx, d.ModTime()) + + // write the file to the response writer. + contents, err := ioutil.ReadAll(f) + if err != nil { + ctx.Application().Logger().Debugf("err reading file: %v", err) + return "error reading the file", http.StatusInternalServerError + } + + // Use `WriteNow` instead of `Write` + // because we need to know the compressed written size before + // the `FlushResponse`. + _, err = ctx.GzipResponseWriter().Write(contents) + if err != nil { + ctx.Application().Logger().Debugf("short write: %v", err) + return "short write", http.StatusInternalServerError + } + + // try to find and send the correct content type based on the filename + // and the binary data inside "f". + detectOrWriteContentType(ctx, d.Name(), f) + + return "", 200 } // toHTTPError returns a non-specific HTTP error message and status code diff --git a/iris.go b/iris.go index e660cc8c..80101857 100644 --- a/iris.go +++ b/iris.go @@ -303,6 +303,11 @@ var ( // // A shortcut for the `context#LimitRequestBodySize`. LimitRequestBodySize = context.LimitRequestBodySize + // Gzip is a middleware which enables writing + // using gzip compression, if client supports. + // + // A shortcut for the `context#Gzip`. + Gzip = context.Gzip // FromStd converts native http.Handler, http.HandlerFunc & func(w, r, next) to context.Handler. // // Supported form types: