From 4e9a6be9aafe3e1c0cc647ccc5ba6f1eb33f47ea Mon Sep 17 00:00:00 2001 From: "Gerasimos (Makis) Maropoulos" Date: Fri, 22 Nov 2019 02:57:26 +0200 Subject: [PATCH] i18n several improvements trigger #1369 Former-commit-id: af16dd8de1a0096d33c4e4c97f29ec12a73302f4 --- _examples/miscellaneous/i18n/main.go | 10 +- _examples/miscellaneous/i18n/main_test.go | 20 +- go.mod | 2 +- go.sum | 3 +- middleware/i18n/i18n.go | 379 ++++++++++++++-------- 5 files changed, 273 insertions(+), 141 deletions(-) diff --git a/_examples/miscellaneous/i18n/main.go b/_examples/miscellaneous/i18n/main.go index b0aaabde..0e42e50d 100644 --- a/_examples/miscellaneous/i18n/main.go +++ b/_examples/miscellaneous/i18n/main.go @@ -13,11 +13,11 @@ var i18nConfig = i18n.Config{ "zh-CN": "./locales/locale_zh-CN.ini", // maps to zh-CN, zh-cn and zh. }, // Optionals. - Alternatives: map[string]string{ // optional. - "english": "en-US", // now english maps to en-US - "greek": "el-GR", // and greek to el-GR - "chinese": "zh-CN", // and chinese to zh-CN too. - }, + // LanguagesMap: i18n.Map{ + // "en": "en-US", // now en maps to en-US + // "el": "el-GR", + // "zh": "zh-CN", + // } or a custom i18n.MapFunc, defaults to accept all lowercase and [en] as [en-US] and e.t.c. URLParameter: "lang", Subdomain: true, // Cookie: "lang", diff --git a/_examples/miscellaneous/i18n/main_test.go b/_examples/miscellaneous/i18n/main_test.go index 809b8a5c..55594c86 100644 --- a/_examples/miscellaneous/i18n/main_test.go +++ b/_examples/miscellaneous/i18n/main_test.go @@ -11,12 +11,19 @@ import ( func TestI18n(t *testing.T) { app := newApp() - expectedf := "From the language %s translated output: %s" + const ( + expectedf = "From the language %s translated output: %s" + + enUS = "hello, iris" + elGR = "γεια, iris" + zhCN = "您好，iris" + ) + var ( tests = map[string]string{ - "en-US": fmt.Sprintf(expectedf, "en-US", "hello, iris"), - "el-GR": fmt.Sprintf(expectedf, "el-GR", "γεια, iris"), - "zh-CN": fmt.Sprintf(expectedf, "zh-CN", "您好，iris"), + "en-US": fmt.Sprintf(expectedf, "en-US", enUS), + "el-GR": fmt.Sprintf(expectedf, "el-GR", elGR), + "zh-CN": fmt.Sprintf(expectedf, "zh-CN", zhCN), } elgrMulti = fmt.Sprintf("From the language: %s, translated output:\n%s=%s\n%s=%s", "el-GR", @@ -71,4 +78,9 @@ func TestI18n(t *testing.T) { Body().Equal(elgrMulti) e.GET("/en/multi").Expect().Status(httptest.StatusOK). Body().Equal(enusMulti) + + e.GET("/el-GRtemplates").Expect().Status(httptest.StatusNotFound) + e.GET("/el-templates").Expect().Status(httptest.StatusNotFound) + + e.GET("/el/templates").Expect().Status(httptest.StatusOK).Body().Contains(elGR).Contains(zhCN) } diff --git a/go.mod b/go.mod index 7fc34e73..fda5dd86 100644 --- a/go.mod +++ b/go.mod @@ -19,7 +19,6 @@ require ( github.com/hashicorp/go-version v1.2.0 // indirect github.com/iris-contrib/blackfriday v2.0.0+incompatible github.com/iris-contrib/go.uuid v2.0.0+incompatible - github.com/iris-contrib/i18n v0.0.0-20171121225848-987a633949d0 github.com/iris-contrib/schema v0.0.1 github.com/json-iterator/go v1.1.6 github.com/kataras/golog v0.0.9 @@ -32,5 +31,6 @@ require ( github.com/ryanuber/columnize v2.1.0+incompatible github.com/shurcooL/sanitized_anchor_name v1.0.0 // indirect golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2 + gopkg.in/ini.v1 v1.51.0 gopkg.in/yaml.v2 v2.2.2 ) diff --git a/go.sum b/go.sum index 58d2ecf8..9d2fd181 100644 --- a/go.sum +++ b/go.sum @@ -46,7 +46,6 @@ github.com/iris-contrib/blackfriday v2.0.0+incompatible h1:o5sHQHHm0ToHUlAJSTjW9 github.com/iris-contrib/blackfriday v2.0.0+incompatible/go.mod h1:UzZ2bDEoaSGPbkg6SAB4att1aAwTmVIx/5gCVqeyUdI= github.com/iris-contrib/go.uuid v2.0.0+incompatible h1:XZubAYg61/JwnJNbZilGjf3b3pB80+OQg2qf6c8BfWE= github.com/iris-contrib/go.uuid v2.0.0+incompatible/go.mod h1:iz2lgM/1UnEf1kP0L/+fafWORmlnuysV2EMP8MW+qe0= -github.com/iris-contrib/i18n v0.0.0-20171121225848-987a633949d0/go.mod h1:pMCz62A0xJL6I+umB2YTlFRwWXaDFA0jy+5HzGiJjqI= github.com/iris-contrib/schema v0.0.1 h1:10g/WnoRR+U+XXHWKBHeNy/+tZmM2kcAVGLOsz+yaDA= github.com/iris-contrib/schema v0.0.1/go.mod h1:urYA3uvUNG1TIIjOSCzHr9/LmbQo8LrOcOqfqxa4hXw= github.com/json-iterator/go v1.1.6 h1:MrUvLMLTMxbqFJ9kzlvat/rYZqZnW3u4wkLzWTaFwKs= @@ -116,6 +115,8 @@ golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/tools v0.0.0-20181221001348-537d06c36207/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190327201419-c70d86f8b7cf/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/ini.v1 v1.51.0 h1:AQvPpx3LzTDM0AjnIRlVFwFFGC+npRopjZxLJj6gdno= +gopkg.in/ini.v1 v1.51.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/mgo.v2 v2.0.0-20180705113604-9856a29383ce/go.mod h1:yeKp02qBN3iKW1OzL3MGk2IdtZzaj7SFntXj72NppTA= gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= diff --git a/middleware/i18n/i18n.go b/middleware/i18n/i18n.go index 6d95d7f3..260ac83d 100644 --- a/middleware/i18n/i18n.go +++ b/middleware/i18n/i18n.go @@ -7,15 +7,11 @@ import ( "net/http" "reflect" "strings" - "sync" - "github.com/iris-contrib/i18n" "github.com/kataras/iris/v12/context" -) -// If `Config.Default` is missing and `Config.Languages` or `Config.Alternatives` contains this key then it will set as the default locale, -// no need to be exported(see `Config.Default`). -const defLang = "en-US" + "gopkg.in/ini.v1" +) // Config the i18n options. type Config struct { @@ -39,7 +35,7 @@ type Config struct { SetCookie bool // If Subdomain is true then it will try to map a subdomain - // with a valid language from the language list or alternatives. + // with a valid language from the language list or a valid map to a language. Subdomain bool // Indentifier is a function which the language can be indentified if the above URLParameter and Cookie failed to. @@ -50,160 +46,288 @@ type Config struct { // Example of key is: 'en-US'. // Example of value is: './locales/en-US.ini'. Languages map[string]string - // Alternatives is a language map which if it's filled, - // it tries to associate its keys with a value of "Languages" field when a possible value of "Language" was not present. - // Example of - // Languages: map[string]string{"en-US": "./locales/en-US.ini"} set - // Alternatives: map[string]string{ "en":"en-US", "english": "en-US"}. - Alternatives map[string]string + // LanguagesMap is a language map which if it's filled, + // it tries to associate an incoming possible language code to a key of "Languages" field + // when the value of "Language" was not present as it is at serve-time. + // + // Defaults to a non-nil LanguagesMap which accepts all lowercase and [en] as [en-US] and e.t.c. + LanguagesMap LanguagesMap } -// Exists returns true if the language, or something similar -// exists (e.g. en-US maps to en or Alternatives[key] == lang). -// it returns the found name and whether it was able to match something. -func (c *Config) Exists(lang string) (string, bool) { - for k, v := range c.Alternatives { - if k == lang { - lang = v - break - } - } - - return i18n.IsExistSimilar(lang) +// LanguagesMap the type for mapping an incoming word to a locale. +type LanguagesMap interface { + Map(lang string) (locale string, found bool) } -// all locale files passed, we keep them in order -// to check if a file is already passed by `New` or `NewWrapper`, -// because we don't have a way to check before the appending of -// a locale file and the same locale code can be used more than one to register different file names (at runtime too). -var ( - localeFilesSet = make(map[string]struct{}) - localesMutex sync.RWMutex - once sync.Once -) +// Map is a Go map[string]string type which is a LanguagesMap that +// matches literal key with value as the found locale. +type Map map[string]string -func (c *Config) loadLanguages() { - if len(c.Languages) == 0 { - panic("field Languages is empty") - } +// Map loops through its registered alternative language codes +// and reports if it is valid registered locale one. +func (m Map) Map(lang string) (string, bool) { + locale, ok := m[lang] + return locale, ok +} - for k, v := range c.Alternatives { - if _, ok := c.Languages[v]; !ok { - panic(fmt.Sprintf("language alternative '%s' does not map to a valid language '%s'", k, v)) - } - } +// MapFunc is a function shortcut for the LanguagesMap. +type MapFunc func(lang string) (locale string, found bool) - // load the files - for k, langFileOrFiles := range c.Languages { - // remove all spaces. - langFileOrFiles = strings.Replace(langFileOrFiles, " ", "", -1) - // note: if only one, then the first element is the "v". - languages := strings.Split(langFileOrFiles, ",") +// Map should report if a given "lang" is valid registered locale. +func (m MapFunc) Map(lang string) (string, bool) { + return m(lang) +} - for _, v := range languages { // loop each of the files separated by comma, if any. - if !strings.HasSuffix(v, ".ini") { - v += ".ini" +func makeDefaultLanguagesMap(languages map[string]string) MapFunc { + return func(lang string) (string, bool) { + lang = strings.ToLower(lang) + for locale := range languages { + if lang == strings.ToLower(locale) { + return locale, true } - localesMutex.RLock() - _, exists := localeFilesSet[v] - localesMutex.RUnlock() - if !exists { - localesMutex.Lock() - err := i18n.SetMessage(k, v) - // fmt.Printf("add %s = %s\n", k, v) - if err != nil && err != i18n.ErrLangAlreadyExist { - panic(fmt.Sprintf("Failed to set locale file' %s' with error: %v", k, err)) + // this matches "en-anything" too, which can be accepted too on some cases, but not here. + // if sep := strings.IndexRune(lang, '-'); sep > 0 { + // lang = lang[0:sep] + // } + + if len(lang) == 2 { + if strings.Contains(locale, lang) { + return locale, true } - - localeFilesSet[v] = struct{}{} - localesMutex.Unlock() } - } - } - if c.Default == "" { - if lang, ok := c.Exists(defLang); ok { - c.Default = lang - } + return "", false } - - once.Do(func() { // set global default lang once. - // fmt.Printf("set default language: %s\n", c.Default) - i18n.SetDefaultLang(c.Default) - }) } // I18n is the structure which keeps the i18n configuration and implement all Iris i18n features. type I18n struct { config Config + + locales map[string][]*ini.File } +// If `Config.Default` is missing and `Config.Languages` or `Config.Map` contains this key then it will set as the default locale, +// no need to be exported(see `Config.Default`). +const defLangCode = "en-US" + // NewI18n returns a new i18n middleware which contains // the middleware itself and a router wrapper. -func NewI18n(config Config) *I18n { - config.loadLanguages() - return &I18n{config} +func NewI18n(c Config) *I18n { + if len(c.Languages) == 0 { + panic("field Languages is empty") + } + + // check and validate (if possible) languages map. + if c.LanguagesMap == nil { + c.LanguagesMap = makeDefaultLanguagesMap(c.Languages) + } + + if mTyp, ok := c.LanguagesMap.(Map); ok { + for k, v := range mTyp { + if _, ok := c.Languages[v]; !ok { + panic(fmt.Sprintf("language alternative '%s' does not map to a valid language '%s'", k, v)) + } + } + } + + i := new(I18n) + + // load messages. + i.locales = make(map[string][]*ini.File) + for locale, src := range c.Languages { + if err := i.AddSource(locale, src); err != nil { + panic(err) + } + } + + // validate and set default lang code. + if c.Default == "" { + c.Default = defLangCode + } + + if locale, _, ok := i.Exists(c.Default); !ok { + panic(fmt.Sprintf("default language '%s' does not match any of the registered language", c.Default)) + } else { + c.Default = locale + } + + i.config = c + + return i } +// AddSource adds a source file to the lang locale. +// It is called on NewI18n, New and NewWrapper. +// +// If you wish to use this at serve-time please protect the process with a mutex. +func (i *I18n) AddSource(locale, src string) error { + // remove all spaces. + src = strings.Replace(src, " ", "", -1) + // note: if only one, then the first element is the "v". + languageFiles := strings.Split(src, ",") + + for _, fileName := range languageFiles { + if !strings.HasSuffix(fileName, ".ini") { + fileName += ".ini" + } + + f, err := ini.Load(fileName) + if err != nil { + return err + } + + i.locales[locale] = append(i.locales[locale], f) + } + + return nil +} + +// GetMessage returns a message from a locale, locale is case-sensitivity and languages map does not playing its part here. +func (i *I18n) GetMessage(locale, section, format string, args ...interface{}) (string, bool) { + files, ok := i.locales[locale] + if !ok { + return "", false + } + + return i.getMessage(files, section, format, args) +} + +func (i *I18n) getMessage(files []*ini.File, section, format string, args []interface{}) (string, bool) { + for _, f := range files { + // returns the first available. + // section is the same for both files if key(format) exists. + s, err := f.GetSection(section) + if err != nil { + return "", false + } + + k, err := s.GetKey(format) + if err != nil { + continue + } + + format = k.Value() + if len(args) > 0 { + return fmt.Sprintf(format, args...), true + } + + return format, true + } + + return "", false +} + +// Translate translates and returns a message based on any language code +// and its key(format) with any optional arguments attached to it. +func (i *I18n) Translate(lang, format string, args ...interface{}) string { + if _, files, ok := i.Exists(lang); ok { + return i.translate(files, format, args) + } + + return "" +} + +func (i *I18n) translate(files []*ini.File, format string, args []interface{}) string { + section := "" + + if idx := strings.IndexRune(format, '.'); idx > 0 { + section = format[:idx] + format = format[idx+1:] + } + + msg, ok := i.getMessage(files, section, format, args) + if !ok { + return fmt.Sprintf(format, args...) + } + + return msg +} + +// Exists reports whether a language code is a valid registered locale through its Languages list and Languages mapping. +func (i *I18n) Exists(lang string) (string, []*ini.File, bool) { + if lang == "" { + return "", nil, false + } + + files, ok := i.locales[lang] + if ok { + return lang, files, true + } + + for locale, files := range i.locales { + if locale == lang { + return locale, files, true + } + } + + if i.config.LanguagesMap != nil { + if locale, ok := i.config.LanguagesMap.Map(lang); ok { + if files, ok := i.locales[locale]; ok { + return locale, files, true + } + } + } + + return "", nil, false +} + +func (i *I18n) newTranslateLanguageFunc(files []*ini.File) func(format string, args ...interface{}) string { + return func(format string, args ...interface{}) string { + return i.translate(files, format, args) + } +} + +const acceptLanguageHeaderKey = "Accept-Language" + // Handler returns the middleware handler. func (i *I18n) Handler() context.Handler { return func(ctx context.Context) { wasByCookie := false langKey := ctx.Application().ConfigurationReadOnly().GetTranslateLanguageContextKey() - language := ctx.Values().GetString(langKey) - if language == "" { + language, files, ok := i.Exists(ctx.Values().GetString(langKey)) + + if !ok { if i.config.URLParameter != "" { - // try to get by url parameter - language = ctx.URLParam(i.config.URLParameter) + language, files, ok = i.Exists(ctx.URLParam(i.config.URLParameter)) } - if language == "" { + if !ok { + // then try to take the lang field from the cookie if i.config.Cookie != "" { - // then try to take the lang field from the cookie - language = ctx.GetCookie(i.config.Cookie) - wasByCookie = language != "" - } - - if language == "" && i.config.Subdomain { - if subdomain := ctx.Subdomain(); subdomain != "" { - if lang, ok := i.config.Exists(subdomain); ok { - language = lang - } + if language, files, ok = i.Exists(ctx.GetCookie(i.config.Cookie)); ok { + wasByCookie = true } } - if language == "" { + if !ok && i.config.Subdomain { + language, files, ok = i.Exists(ctx.Subdomain()) + } + + if !ok { // try to get by the request headers. - langHeader := ctx.GetHeader("Accept-Language") - if len(langHeader) > 0 { - for _, langEntry := range strings.Split(langHeader, ",") { - lc := strings.Split(langEntry, ";")[0] - if lang, ok := i.config.Exists(lc); ok { - language = lang - break - } + if langHeader := ctx.GetHeader(acceptLanguageHeaderKey); langHeader != "" { + idx := strings.IndexRune(langHeader, ';') + if idx > 0 { + langHeader = langHeader[:idx] } + + language, files, ok = i.Exists(langHeader) } } - if language == "" && i.config.Indentifier != nil { - language = i.config.Indentifier(ctx) + if !ok && i.config.Indentifier != nil { + language, files, ok = i.Exists(i.config.Indentifier(ctx)) } } } - if language == "" { - language = i.config.Default - } - - // returns the original key of the language and true - // when the language, or something similar exists (e.g. en-US maps to en). - if lc, ok := i.config.Exists(language); ok { - language = lc + if !ok { + language, files, ok = i.Exists(i.config.Default) } // if it was not taken by the cookie, then set the cookie in order to have it. @@ -214,9 +338,8 @@ func (i *I18n) Handler() context.Handler { ctx.Values().Set(langKey, language) // Set iris.translate and iris.translateLang functions (they can be passed to templates as they are later on). - ctx.Values().Set(ctx.Application().ConfigurationReadOnly().GetTranslateFunctionContextKey(), getTranslateFunction(language)) - // Note: translate (global) language function input argument should match exactly, case-sensitive and "Alternatives" field is not part of the fetch progress. - ctx.Values().Set(ctx.Application().ConfigurationReadOnly().GetTranslateLangFunctionContextKey(), i18n.Tr) + ctx.Values().Set(ctx.Application().ConfigurationReadOnly().GetTranslateFunctionContextKey(), i.newTranslateLanguageFunc(files)) + ctx.Values().Set(ctx.Application().ConfigurationReadOnly().GetTranslateLangFunctionContextKey(), i.Translate) ctx.Next() } @@ -232,21 +355,23 @@ func (i *I18n) Handler() context.Handler { func (i *I18n) Wrapper() func(http.ResponseWriter, *http.Request, http.HandlerFunc) { return func(w http.ResponseWriter, r *http.Request, routerHandler http.HandlerFunc) { found := false - path := r.URL.Path[1:] + reqPath := r.URL.Path[1:] + path := reqPath if idx := strings.IndexByte(path, '/'); idx > 0 { path = path[:idx] } if path != "" { - if lang, ok := i.config.Exists(path); ok { + if lang, _, ok := i.Exists(path); ok { path = r.URL.Path[len(path)+1:] if path == "" { path = "/" } + r.RequestURI = path r.URL.Path = path - r.Header.Set("Accept-Language", lang) + r.Header.Set(acceptLanguageHeaderKey, lang) found = true } } @@ -256,11 +381,11 @@ func (i *I18n) Wrapper() func(http.ResponseWriter, *http.Request, http.HandlerFu if dotIdx := strings.IndexByte(host, '.'); dotIdx > 0 { subdomain := host[0:dotIdx] if subdomain != "" { - if lang, ok := i.config.Exists(subdomain); ok { + if lang, _, ok := i.Exists(subdomain); ok { host = host[dotIdx+1:] r.URL.Host = host r.Host = host - r.Header.Set("Accept-Language", lang) + r.Header.Set(acceptLanguageHeaderKey, lang) } } } @@ -270,15 +395,9 @@ func (i *I18n) Wrapper() func(http.ResponseWriter, *http.Request, http.HandlerFu } } -func getTranslateFunction(lang string) func(string, ...interface{}) string { - return func(format string, args ...interface{}) string { - return i18n.Tr(lang, format, args...) - } -} - // New returns a new i18n middleware. -func New(config Config) context.Handler { - return NewI18n(config).Handler() +func New(c Config) context.Handler { + return NewI18n(c).Handler() } // NewWrapper accepts a Config and returns a new router wrapper. @@ -288,8 +407,8 @@ func New(config Config) context.Handler { // // In order this to work as expected, it should be combined with `Application.Use(New)` // which registers the i18n middleware itself. -func NewWrapper(config Config) func(http.ResponseWriter, *http.Request, http.HandlerFunc) { - return NewI18n(config).Wrapper() +func NewWrapper(c Config) func(http.ResponseWriter, *http.Request, http.HandlerFunc) { + return NewI18n(c).Wrapper() } // Translate returns the translated word from a context based on the current selected locale.