From a6fc0072ffdcff62e05d74b04f1526a9204d2b55 Mon Sep 17 00:00:00 2001 From: "Gerasimos (Makis) Maropoulos" Date: Wed, 1 Mar 2017 19:17:32 +0200 Subject: [PATCH] Organising kataras/go-fs. No api changes for these changes don't worry. See previous commit's description for more info. Former-commit-id: 8af960e5e4e5f7c8816140ac912328b9c524370b --- compression.go | 70 ++++++++++++++ context.go | 24 +++-- fs.go | 67 ++++++++++++++ iris.go | 46 +--------- iris/get.go | 16 +++- response_writer.go | 5 +- router.go | 17 ++-- updater.go | 221 +++++++++++++++++++++++++++++++++++++++++++++ 8 files changed, 392 insertions(+), 74 deletions(-) create mode 100644 compression.go create mode 100644 updater.go diff --git a/compression.go b/compression.go new file mode 100644 index 00000000..b440b746 --- /dev/null +++ b/compression.go @@ -0,0 +1,70 @@ +package iris + +import ( + "io" + "sync" + + "github.com/klauspost/compress/gzip" +) + +// compressionPool is a wrapper of sync.Pool, to initialize a new compression writer pool +type compressionPool struct { + sync.Pool + Level int +} + +// +------------------------------------------------------------+ +// | | +// | GZIP | +// | | +// +------------------------------------------------------------+ + +// writes gzip compressed content to an underline io.Writer. It uses sync.Pool to reduce memory allocations. +// Better performance through klauspost/compress package which provides us a gzip.Writer which is faster than Go standard's gzip package's writer. + +// These constants are copied from the standard flate package +// available Compressors +const ( + NoCompressionLevel = 0 + BestSpeedLevel = 1 + BestCompressionLevel = 9 + DefaultCompressionLevel = -1 + ConstantCompressionLevel = -2 // Does only Huffman encoding +) + +// default writer pool with Compressor's level setted to DefaultCompressionLevel +var gzipPool = &compressionPool{Level: DefaultCompressionLevel} + +// AcquireGzipWriter prepares a gzip writer and returns it +// +// see ReleaseGzipWriter +func acquireGzipWriter(w io.Writer) *gzip.Writer { + v := gzipPool.Get() + if v == nil { + gzipWriter, err := gzip.NewWriterLevel(w, gzipPool.Level) + if err != nil { + return nil + } + return gzipWriter + } + gzipWriter := v.(*gzip.Writer) + gzipWriter.Reset(w) + return gzipWriter +} + +// ReleaseGzipWriter called when flush/close and put the gzip writer back to the pool +// +// see AcquireGzipWriter +func releaseGzipWriter(gzipWriter *gzip.Writer) { + gzipWriter.Close() + gzipPool.Put(gzipWriter) +} + +// WriteGzip writes a compressed form of p to the underlying io.Writer. The +// compressed bytes are not necessarily flushed until the Writer is closed +func writeGzip(w io.Writer, b []byte) (int, error) { + gzipWriter := acquireGzipWriter(w) + n, err := gzipWriter.Write(b) + releaseGzipWriter(gzipWriter) + return n, err +} diff --git a/context.go b/context.go index c49bedcb..4368b8c3 100644 --- a/context.go +++ b/context.go @@ -22,8 +22,6 @@ import ( "github.com/iris-contrib/formBinder" "github.com/kataras/go-errors" - "github.com/kataras/go-fs" - "github.com/kataras/go-template" ) const ( @@ -775,7 +773,7 @@ func (ctx *Context) EmitError(statusCode int) { // ------------------------------------------------------------------------------------- // ------------------------------------------------------------------------------------- // -------------------------Context's gzip inline response writer ---------------------- -// ---------------------Look template.go & iris.go for more options--------------------- +// ---------------------Look adaptors/view & iris.go for more options------------------- // ------------------------------------------------------------------------------------- var ( @@ -799,9 +797,9 @@ func (ctx *Context) WriteGzip(b []byte) (int, error) { if ctx.clientAllowsGzip() { ctx.ResponseWriter.Header().Add(varyHeader, acceptEncodingHeader) - gzipWriter := fs.AcquireGzipWriter(ctx.ResponseWriter) + gzipWriter := acquireGzipWriter(ctx.ResponseWriter) n, err := gzipWriter.Write(b) - fs.ReleaseGzipWriter(gzipWriter) + releaseGzipWriter(gzipWriter) if err == nil { ctx.SetHeader(contentEncodingHeader, "gzip") @@ -834,7 +832,7 @@ func (ctx *Context) TryWriteGzip(b []byte) (int, error) { const ( // NoLayout to disable layout for a particular template file - NoLayout = template.NoLayout + NoLayout = "@.|.@no_layout@.|.@" // TemplateLayoutContextKey is the name of the user values which can be used to set a template layout from a middleware and override the parent's TemplateLayoutContextKey = "templateLayout" ) @@ -876,8 +874,8 @@ func (ctx *Context) fastRenderWithStatus(status int, cType string, data []byte) ctx.ResponseWriter.Header().Add(varyHeader, acceptEncodingHeader) ctx.SetHeader(contentEncodingHeader, "gzip") - gzipWriter := fs.AcquireGzipWriter(ctx.ResponseWriter) - defer fs.ReleaseGzipWriter(gzipWriter) + gzipWriter := acquireGzipWriter(ctx.ResponseWriter) + defer releaseGzipWriter(gzipWriter) out = gzipWriter } else { out = ctx.ResponseWriter @@ -943,8 +941,8 @@ func (ctx *Context) RenderWithStatus(status int, name string, binding interface{ ctx.ResponseWriter.Header().Add(varyHeader, acceptEncodingHeader) ctx.SetHeader(contentEncodingHeader, "gzip") - gzipWriter := fs.AcquireGzipWriter(ctx.ResponseWriter) - defer fs.ReleaseGzipWriter(gzipWriter) + gzipWriter := acquireGzipWriter(ctx.ResponseWriter) + defer releaseGzipWriter(gzipWriter) out = gzipWriter } else { out = ctx.ResponseWriter @@ -1097,7 +1095,7 @@ func (ctx *Context) ServeContent(content io.ReadSeeker, filename string, modtime return nil } - ctx.ResponseWriter.Header().Set(contentType, fs.TypeByExtension(filename)) + ctx.ResponseWriter.Header().Set(contentType, typeByExtension(filename)) ctx.ResponseWriter.Header().Set(lastModified, modtime.UTC().Format(ctx.framework.Config.TimeFormat)) ctx.SetStatusCode(StatusOK) var out io.Writer @@ -1105,8 +1103,8 @@ func (ctx *Context) ServeContent(content io.ReadSeeker, filename string, modtime ctx.ResponseWriter.Header().Add(varyHeader, acceptEncodingHeader) ctx.SetHeader(contentEncodingHeader, "gzip") - gzipWriter := fs.AcquireGzipWriter(ctx.ResponseWriter) - defer fs.ReleaseGzipWriter(gzipWriter) + gzipWriter := acquireGzipWriter(ctx.ResponseWriter) + defer releaseGzipWriter(gzipWriter) out = gzipWriter } else { out = ctx.ResponseWriter diff --git a/fs.go b/fs.go index a1183eb2..e54b7f05 100644 --- a/fs.go +++ b/fs.go @@ -1,6 +1,7 @@ package iris import ( + "mime" "net/http" "os" "path/filepath" @@ -19,6 +20,12 @@ type StaticHandlerBuilder interface { Build() HandlerFunc } +// +------------------------------------------------------------+ +// | | +// | Static Builder | +// | | +// +------------------------------------------------------------+ + type fsHandler struct { // user options, only directory is required. directory http.Dir @@ -201,3 +208,63 @@ func StripPrefix(prefix string, h HandlerFunc) HandlerFunc { } } } + +// typeByExtension returns the MIME type associated with the file extension ext. +// The extension ext should begin with a leading dot, as in ".html". +// When ext has no associated type, typeByExtension returns "". +// +// Extensions are looked up first case-sensitively, then case-insensitively. +// +// The built-in table is small but on unix it is augmented by the local +// system's mime.types file(s) if available under one or more of these +// names: +// +// /etc/mime.types +// /etc/apache2/mime.types +// /etc/apache/mime.types +// +// On Windows, MIME types are extracted from the registry. +// +// Text types have the charset parameter set to "utf-8" by default. +func typeByExtension(fullfilename string) (t string) { + ext := filepath.Ext(fullfilename) + //these should be found by the windows(registry) and unix(apache) but on windows some machines have problems on this part. + if t = mime.TypeByExtension(ext); t == "" { + // no use of map here because we will have to lock/unlock it, by hand is better, no problem: + if ext == ".json" { + t = "application/json" + } else if ext == ".js" { + t = "application/javascript" + } else if ext == ".zip" { + t = "application/zip" + } else if ext == ".3gp" { + t = "video/3gpp" + } else if ext == ".7z" { + t = "application/x-7z-compressed" + } else if ext == ".ace" { + t = "application/x-ace-compressed" + } else if ext == ".aac" { + t = "audio/x-aac" + } else if ext == ".ico" { // for any case + t = "image/x-icon" + } else if ext == ".png" { + t = "image/png" + } else { + t = "application/octet-stream" + } + // mime.TypeByExtension returns as text/plain; | charset=utf-8 the static .js (not always) + } else if t == "text/plain" || t == "text/plain; charset=utf-8" { + if ext == ".js" { + t = "application/javascript" + } + } + return +} + +// directoryExists returns true if a directory(or file) exists, otherwise false +func directoryExists(dir string) bool { + if _, err := os.Stat(dir); os.IsNotExist(err) { + return false + } + return true +} diff --git a/iris.go b/iris.go index e4dde7ec..83d5765f 100644 --- a/iris.go +++ b/iris.go @@ -27,7 +27,6 @@ import ( "github.com/geekypanda/httpcache" "github.com/kataras/go-errors" - "github.com/kataras/go-fs" ) const ( @@ -353,7 +352,7 @@ func New(setters ...OptionSetter) *Framework { // On Build: local repository updates s.Adapt(EventPolicy{Build: func(*Framework) { if s.Config.CheckForUpdates { - go s.CheckForUpdates(false) + go CheckForUpdates(false) } }}) } @@ -671,49 +670,6 @@ func (s *Framework) ServeHTTP(w http.ResponseWriter, r *http.Request) { s.Router.ServeHTTP(w, r) } -// global once because is not necessary to check for updates on more than one iris station* -var updateOnce sync.Once - -const ( - githubOwner = "kataras" - githubRepo = "iris" -) - -// CheckForUpdates will try to search for newer version of Iris based on the https://github.com/kataras/iris/releases -// If a newer version found then the app will ask the he dev/user if want to update the 'x' version -// if 'y' is pressed then the updater will try to install the latest version -// the updater, will notify the dev/user that the update is finished and should restart the App manually. -// Note: exported func CheckForUpdates exists because of the reason that an update can be executed while Iris is running -func (s *Framework) CheckForUpdates(force bool) { - updated := false - checker := func() { - - fs.DefaultUpdaterAlreadyInstalledMessage = "Updater: Running with the latest version(%s)\n" - updater, err := fs.GetUpdater(githubOwner, githubRepo, Version) - - if err != nil { - // ignore writer's error - s.Log(DevMode, "update failed: "+err.Error()) - return - } - - updated = updater.Run(fs.Stdout(s.policies.LoggerPolicy), fs.Stderr(s.policies.LoggerPolicy), fs.Silent(false)) - - } - - if force { - checker() - } else { - updateOnce.Do(checker) - } - - if updated { // if updated, then do not run the web server - s.Log(DevMode, "exiting now...") - os.Exit(1) - } - -} - // Adapt adapds a policy to the Framework. // It accepts single or more objects that implements the iris.Policy. // Iris provides some of them but you can build your own based on one or more of these: diff --git a/iris/get.go b/iris/get.go index 7c978e52..4d0a80ac 100644 --- a/iris/get.go +++ b/iris/get.go @@ -7,7 +7,6 @@ import ( "strings" "github.com/kataras/cli" - "github.com/kataras/go-fs" "github.com/skratchdot/open-golang/open" ) @@ -30,6 +29,14 @@ var ( } ) +// DirectoryExists returns true if a directory(or file) exists, otherwise false +func DirectoryExists(dir string) bool { + if _, err := os.Stat(dir); os.IsNotExist(err) { + return false + } + return true +} + // dir returns the supposed local directory for this project func (p project) dir() string { return join(getGoPath(), p.remote) @@ -41,11 +48,12 @@ func (p project) mainfile() string { func (p project) download() { // first, check if the repo exists locally in gopath - if fs.DirectoryExists(p.dir()) { + if DirectoryExists(p.dir()) { return } app.Printf("Downloading... ") - finish := fs.ShowIndicator(cli.Output, false) + + finish := cli.ShowIndicator(false) defer func() { @@ -72,7 +80,7 @@ func (p project) run() { // the source and change the import paths from there too // so here just let run and watch it mainFile := p.mainfile() - if !fs.DirectoryExists(mainFile) { // or file exists, same thing + if !DirectoryExists(mainFile) { // or file exists, same thing p.download() } installedDir := p.dir() diff --git a/response_writer.go b/response_writer.go index df9f7698..ad88e3dd 100644 --- a/response_writer.go +++ b/response_writer.go @@ -8,7 +8,6 @@ import ( "sync" "github.com/kataras/go-errors" - "github.com/kataras/go-fs" "github.com/klauspost/compress/gzip" ) @@ -22,12 +21,12 @@ var gzpool = sync.Pool{New: func() interface{} { return &gzipResponseWriter{} }} func acquireGzipResponseWriter(underline ResponseWriter) *gzipResponseWriter { w := gzpool.Get().(*gzipResponseWriter) w.ResponseWriter = underline - w.gzipWriter = fs.AcquireGzipWriter(w.ResponseWriter) + w.gzipWriter = acquireGzipWriter(w.ResponseWriter) return w } func releaseGzipResponseWriter(w *gzipResponseWriter) { - fs.ReleaseGzipWriter(w.gzipWriter) + releaseGzipWriter(w.gzipWriter) gzpool.Put(w) } diff --git a/router.go b/router.go index 6165ff84..df0b0ed1 100644 --- a/router.go +++ b/router.go @@ -8,7 +8,6 @@ import ( "time" "github.com/kataras/go-errors" - "github.com/kataras/go-fs" ) const ( @@ -374,9 +373,9 @@ func (router *Router) StaticServe(systemPath string, requestPath ...string) Rout var reqPath string if len(requestPath) == 0 { - reqPath = strings.Replace(systemPath, fs.PathSeparator, slash, -1) // replaces any \ to / - reqPath = strings.Replace(reqPath, "//", slash, -1) // for any case, replaces // to / - reqPath = strings.Replace(reqPath, ".", "", -1) // replace any dots (./mypath -> /mypath) + reqPath = strings.Replace(systemPath, string(os.PathSeparator), slash, -1) // replaces any \ to / + reqPath = strings.Replace(reqPath, "//", slash, -1) // for any case, replaces // to / + reqPath = strings.Replace(reqPath, ".", "", -1) // replace any dots (./mypath -> /mypath) } else { reqPath = requestPath[0] } @@ -384,10 +383,10 @@ func (router *Router) StaticServe(systemPath string, requestPath ...string) Rout return router.Get(reqPath+"/*file", func(ctx *Context) { filepath := ctx.Param("file") - spath := strings.Replace(filepath, "/", fs.PathSeparator, -1) + spath := strings.Replace(filepath, "/", string(os.PathSeparator), -1) spath = path.Join(systemPath, spath) - if !fs.DirectoryExists(spath) { + if !directoryExists(spath) { ctx.NotFound() return } @@ -471,7 +470,7 @@ func (router *Router) StaticEmbedded(requestPath string, vdir string, assetFn fu continue } - cType := fs.TypeByExtension(path) + cType := typeByExtension(path) fullpath := vdir + path buf, err := assetFn(fullpath) @@ -530,7 +529,7 @@ func (router *Router) Favicon(favPath string, requestPath ...string) RouteInfo { fi, _ = f.Stat() } - cType := fs.TypeByExtension(favPath) + cType := typeByExtension(favPath) // copy the bytes here in order to cache and not read the ico on each request. cacheFav := make([]byte, fi.Size()) if _, err = f.Read(cacheFav); err != nil { @@ -628,7 +627,7 @@ func (router *Router) StaticWeb(reqPath string, systemPath string, exceptRoutes handler := func(ctx *Context) { h(ctx) if fname := ctx.Param(paramName); fname != "" { - cType := fs.TypeByExtension(fname) + cType := typeByExtension(fname) if cType != contentBinary && !strings.Contains(cType, "charset") { cType += "; charset=" + ctx.framework.Config.Charset } diff --git a/updater.go b/updater.go new file mode 100644 index 00000000..a5fbbc20 --- /dev/null +++ b/updater.go @@ -0,0 +1,221 @@ +package iris + +import ( + "bufio" + "context" + "fmt" + "io" + "os" + "os/exec" + "sync" + "time" + + "github.com/google/go-github/github" + "github.com/hashicorp/go-version" + "github.com/kataras/go-errors" +) + +// global once because is not necessary to check for updates on more than one iris station* +var updateOnce sync.Once + +// CheckForUpdates will try to search for newer version of Iris based on the https://github.com/kataras/iris/releases +// If a newer version found then the app will ask the he dev/user if want to update the 'x' version +// if 'y' is pressed then the updater will try to install the latest version +// the updater, will notify the dev/user that the update is finished and should restart the App manually. +// Note: exported func CheckForUpdates exists because of the reason that an update can be executed while Iris is running +func CheckForUpdates(force bool) { + var ( + updated bool + err error + ) + + checker := func() { + updated, err = update(os.Stdin, os.Stdout, false) + if err != nil { + // ignore writer's error + os.Stdout.Write([]byte("update failed: " + err.Error())) + return + } + } + + if force { + checker() + } else { + updateOnce.Do(checker) + } + + if updated { // if updated, then do not run the web server + os.Stdout.Write([]byte("exiting now...")) + os.Exit(1) + } + +} + +// +------------------------------------------------------------+ +// | | +// | Updater based on github repository's releases | +// | | +// +------------------------------------------------------------+ + +// showIndicator shows a silly terminal indicator for a process, close of the finish channel is done here. +func showIndicator(wr io.Writer, newLine bool) chan bool { + finish := make(chan bool) + waitDur := 500 * time.Millisecond + go func() { + if newLine { + wr.Write([]byte("\n")) + } + wr.Write([]byte("|")) + wr.Write([]byte("_")) + wr.Write([]byte("|")) + + for { + select { + case v := <-finish: + { + if v { + wr.Write([]byte("\010\010\010")) //remove the loading chars + close(finish) + return + } + } + default: + wr.Write([]byte("\010\010-")) + time.Sleep(waitDur) + wr.Write([]byte("\010\\")) + time.Sleep(waitDur) + wr.Write([]byte("\010|")) + time.Sleep(waitDur) + wr.Write([]byte("\010/")) + time.Sleep(waitDur) + wr.Write([]byte("\010-")) + time.Sleep(waitDur) + wr.Write([]byte("|")) + } + } + + }() + + return finish +} + +var updaterYesInput = [...]string{"y", "yes", "nai", "si"} + +func shouldProceedUpdate(sc *bufio.Scanner) bool { + silent := sc == nil + + inputText := "" + if !silent { + if sc.Scan() { + inputText = sc.Text() + } + } + + for _, s := range updaterYesInput { + if inputText == s { + return true + } + } + // if silent, then return 'yes/true' always + return silent +} + +var ( + errUpdaterUnknown = errors.New("updater: Unknown error: %s") + errCantFetchRepo = errors.New("updater: Error while trying to fetch the repository: %s. Trace: %s") + errAccessRepo = errors.New("updater: Couldn't access to the github repository, please make sure you're connected to the internet") + + // lastVersionAlreadyInstalledMessage "\nThe latest version '%s' was already installed." + lastVersionAlreadyInstalledMessage = "the latest version '%s' is already installed." +) + +// update runs the updater, returns true if update has been found and installed, otherwise false +func update(in io.Reader, out io.Writer, silent bool) (bool, error) { + + const ( + owner = "kataras" + repo = "iris" + ) + + client := github.NewClient(nil) // unuthenticated client, 60 req/hour + ///TODO: rate limit error catching( impossible to same client checks 60 times for github updates, but we should do that check) + + ctx := context.TODO() + + // get the latest release, delay depends on the user's internet connection's download speed + latestRelease, response, err := client.Repositories.GetLatestRelease(ctx, owner, repo) + if err != nil { + return false, errCantFetchRepo.Format(owner+":"+repo, err) + } + + if c := response.StatusCode; c != 200 && c != 201 && c != 202 && c != 301 && c != 302 && c == 304 { + return false, errAccessRepo + } + + currentVersion, err := version.NewVersion(Version) + if err != nil { + return false, err + } + + latestVersion, err := version.NewVersion(*latestRelease.TagName) + if err != nil { + return false, err + } + + writef := func(s string, a ...interface{}) { + if !silent { + out.Write([]byte(fmt.Sprintf(s, a...))) + } + } + + has, v := currentVersion.LessThan(latestVersion), latestVersion.String() + if has { + + var scanner *bufio.Scanner + if in != nil { + scanner = bufio.NewScanner(in) + } + + shouldProceedUpdate := func() bool { + return shouldProceedUpdate(scanner) + } + + writef("A newer version has been found[%s > %s].\n"+ + "Release notes: %s\n"+ + "Update now?[%s]: ", + latestVersion.String(), currentVersion.String(), + fmt.Sprintf("https://github.com/%s/%s/releases/latest", owner, repo), + updaterYesInput[0]+"/n") + + if shouldProceedUpdate() { + if !silent { + finish := showIndicator(out, true) + + defer func() { + finish <- true + }() + } + // go get -u github.com/:owner/:repo + cmd := exec.Command("go", "get", "-u", fmt.Sprintf("github.com/%s/%s", owner, repo)) + cmd.Stdout = out + cmd.Stderr = out + + if err := cmd.Run(); err != nil { + return false, fmt.Errorf("error while trying to get the package: %s", err.Error()) + } + + writef("\010\010\010") // remove the loading bars + writef("Update has been installed, current version: %s. Please re-start your App.\n", latestVersion.String()) + + // TODO: normally, this should be in dev-mode machine, so a 'go build' and' & './$executable' on the current working path should be ok + // for now just log a message to re-run the app manually + //writef("\nUpdater was not able to re-build and re-run your updated App.\nPlease run your App again, by yourself.") + return true, nil + } + + } else { + writef(fmt.Sprintf(lastVersionAlreadyInstalledMessage, v)) + } + + return false, nil +}