go-i18n: translations embedding

.idea/runConfigurations/Kurzd_generate_translations.xml

@@ -0,0 +1,14 @@
+<component name="ProjectRunConfigurationManager">
+  <configuration default="false" name="Kurzd generate-translations" type="GoApplicationRunConfiguration" factoryName="Go Application" show_console_on_std_err="true" show_console_on_std_out="true">
+    <module name="kurz" />
+    <working_directory value="$PROJECT_DIR$" />
+    <go_parameters value="-o kurzd" />
+    <parameters value="generate-translations" />
+    <kind value="PACKAGE" />
+    <filePath value="$PROJECT_DIR$/" />
+    <package value="code.osinet.fr/fgm/kurz/cmd/kurzd" />
+    <directory value="$PROJECT_DIR$/cmd/kurzd" />
+    <output_directory value="$PROJECT_DIR$/cmd/kurzd" />
+    <method v="2" />
+  </configuration>
+</component>

cmd/kurzd/generate_translations.go

@@ -0,0 +1,205 @@
+package main
+
+import (
+	"io/ioutil"
+	"log"
+	"os"
+	"path/filepath"
+	"regexp"
+	"strings"
+	"text/template"
+	"time"
+
+	"code.osinet.fr/fgm/kurz/translations"
+
+	"github.com/nicksnyder/go-i18n/v2/i18n"
+	"github.com/spf13/cobra"
+	"golang.org/x/text/language"
+	"gopkg.in/yaml.v2"
+)
+
+var cmdGenerateTranslations = &cobra.Command{
+	Args:  cobra.NoArgs,
+	Long:  "Generate the translations package",
+	Run:   generateTranslationsHandler,
+	Short: "Converts the messages.*.yaml files to a compilable i18n messages embed package. Only useful during Kurz development",
+	Use:   "generate-translations",
+}
+
+func init() {
+	cmd.AddCommand(cmdGenerateTranslations)
+}
+
+/*
+dumpMessages is a debug helper, logging a messageMap in YAML format for readability.
+*/
+func dumpMessages(messageMap translations.MessageMap) {
+	out, err := yaml.Marshal(messageMap)
+	if err != nil {
+		panic(err)
+	}
+	log.Println("Messages:\n" + string(out))
+}
+
+func generateTranslationsHandler(cmd *cobra.Command, args []string) {
+	cwd, err := os.Getwd()
+	if err != nil {
+		panic(err)
+	}
+	t0 := time.Now()
+	log.Printf("Finding message catalogs below %s\n", cwd)
+
+	tags := []string{
+		language.English.String(),
+		language.French.String(),
+	}
+
+	messageMap := make(translations.MessageMap)
+	for _, tag := range tags {
+		messageMap[language.Make(tag)] = [](*i18n.Message){}
+	}
+	// Build the regex matching message file names.
+	re := regexp.MustCompile("messages\\.(?:" + strings.Join(tags, "|") + ")\\.yaml")
+	scanned := 0
+	catalogCount := 0
+	walkErr := filepath.Walk(cwd, scanCatalog(messageMap, re, &scanned, &catalogCount))
+	if walkErr != nil {
+		panic(walkErr)
+	}
+
+	t1 := time.Now()
+	diff := t1.Sub(t0)
+	log.Printf("%d items scanned, %d catalogs loaded in %d msec\n", scanned, catalogCount, diff/time.Millisecond)
+	// dumpMessages(messageMap)
+
+	pwd, err := os.Getwd()
+	if err != nil {
+		panic(err)
+	}
+	println(pwd)
+	translationsDir := pwd + "/translations"
+	tpl := template.Must(template.ParseFiles(translationsDir + "/template.go.gotext"))
+	gen, err := os.Create(translationsDir + "/generated.go")
+	if err != nil {
+		panic(err)
+	}
+	err = tpl.ExecuteTemplate(gen, "template.go.gotext", messageMap)
+	if err != nil {
+		panic(err)
+	}
+}
+
+/*
+parseMessageFileBytes merges the messages found in "buf" into the messageMap.
+
+See scanCatalog().
+*/
+func parseMessages(messagesMap translations.MessageMap, buf []byte, tag language.Tag) error {
+	if len(buf) == 0 {
+		return nil
+	}
+
+	raw := make(map[string]interface{})
+	if err := yaml.Unmarshal(buf, &raw); err != nil {
+		return err
+	}
+	for id, item := range raw {
+		switch item.(type) {
+		// Degenerate case: "key: one"
+		case string:
+			message := &i18n.Message{
+				ID:  id,
+				Other: item.(string),
+			}
+			messagesMap[tag] = append(messagesMap[tag], message)
+
+		// Normal case:	 key: { ID: ..., one: ... }.
+		case map[interface{}]interface{}:
+			serial, err := yaml.Marshal(item)
+			if err != nil {
+				panic(err)
+			}
+			message := &i18n.Message{
+				ID: id,
+			}
+			err = yaml.Unmarshal(serial, message)
+			if err != nil {
+				panic(err)
+			}
+			messagesMap[tag] = append(messagesMap[tag], message)
+
+		default:
+			log.Fatalf("Unsupported message format %#v", item)
+		}
+	}
+	return nil
+}
+
+// parsePath is a copy of go-i18n.internal.parsePath
+//
+// Used under its MIT license.
+func parsePath(path string) (langTag, format string) {
+	formatStartIdx := -1
+	for i := len(path) - 1; i >= 0; i-- {
+		c := path[i]
+		if os.IsPathSeparator(c) {
+			if formatStartIdx != -1 {
+				langTag = path[i+1 : formatStartIdx]
+			}
+			return
+		}
+		if path[i] == '.' {
+			if formatStartIdx != -1 {
+				langTag = path[i+1 : formatStartIdx]
+				return
+			}
+			if formatStartIdx == -1 {
+				format = path[i+1:]
+				formatStartIdx = i
+			}
+		}
+	}
+	if formatStartIdx != -1 {
+		langTag = path[:formatStartIdx]
+	}
+	return
+}
+
+/*
+scanCatalog builds a WalkFn callback parsing each message file as it is found, and
+merging its values into messageMap.
+
+See generateTranslationsHandler
+*/
+func scanCatalog(messageMap translations.MessageMap, re *regexp.Regexp, scanned *int, catalogCount *int) filepath.WalkFunc {
+	return func(path string, info os.FileInfo, err error) error {
+		*scanned++
+		if err != nil {
+			panic(err)
+		}
+		if info.IsDir() {
+			return nil
+		}
+
+		if !re.MatchString(info.Name()) {
+			return nil
+		}
+
+		langid, _ := parsePath(path)
+		tag := language.Make(langid)
+		_, ok := messageMap[tag]
+		if !ok {
+			log.Printf("Found message catalog %s in unexpected language %s, skipping.",
+				path, tag.String())
+			return nil
+		}
+
+		buf, err := ioutil.ReadFile(path)
+		if err != nil {
+			panic(err)
+		}
+		err = parseMessages(messageMap, buf, tag)
+		*catalogCount++
+		return err
+	}
+}

cmd/kurzd/rootCmd.go

@@ -2,18 +2,13 @@ package main
 
 import (
 	"fmt"
-	"log"
 	"os"
-	"path/filepath"
-	"regexp"
-	"strings"
-	"time"
 
+	"code.osinet.fr/fgm/kurz/translations"
 	"code.osinet.fr/fgm/kurz/web/i18n"
 
 	"github.com/spf13/cobra"
 	"github.com/spf13/viper"
-	"golang.org/x/text/language"
 )
 
 var cmd = &cobra.Command{
@@ -39,45 +34,5 @@ func initConfig() {
 func init() {
 	cobra.OnInitialize(initConfig)
 	verbose = *cmd.PersistentFlags().BoolP("verbose", "v", false, "Add -v for verbose operation")
-	setupI18n()
-}
-
-func setupI18n() {
-	cwd, err := os.Getwd()
-	if err != nil {
-		panic(err)
-	}
-	t0 := time.Now()
-	log.Printf("Finding message catalogs below %s\n", cwd)
-
-	tags := []string{language.English.String(), language.French.String()}
-
-	// Build the regex matching message file names.
-	re := regexp.MustCompile("messages\\.(?:" + strings.Join(tags, "|") + ")\\.yaml")
-
-	scanned := 0
-	catalogCount := 0
-	walkErr := filepath.Walk(cwd, func(path string, info os.FileInfo, err error) error {
-		scanned++
-		if err != nil {
-			panic(err)
-		}
-		if info.IsDir() {
-			return nil
-		}
-
-		if !re.MatchString(info.Name()) {
-			return nil
-		}
-
-		_, err = i18n.Bundle.LoadMessageFile(path)
-		catalogCount++
-		return err
-	})
-	if walkErr != nil {
-		panic(walkErr)
-	}
-	t1 := time.Now()
-	diff := t1.Sub(t0)
-	log.Printf("%d items scanned, %d catalogs loaded in %d msec\n", scanned, catalogCount, diff/time.Millisecond)
+	translations.MustLoadMessages(i18n.Bundle)
 }

domain/messages.en.yaml

@@ -2,8 +2,8 @@
 Zerror.error_format:
   description: The format for errors
   one: '{.Kind}: {.Detail}'
-  leftDelim: '{'
-  rightDelim: '}'
+  leftdelim: '{'
+  rightdelim: '}'
 error.none: No error
 error.short_not_created: Short URL not created for a new target
 error.short_not_found: Short URL not defined

domain/messages.fr.yaml

@@ -2,8 +2,8 @@
 Zerror.error_format:
   description: Le format des erreurs
   one: '{.Kind}: {.Detail}'
-  leftDelim: '{'
-  rightDelim: '}'
+  leftdelim: '{'
+  rightdelim: '}'
 error.none: Aucune erreur
 error.short_not_created: URL court non créé pour une nouvelle cible
 error.short_not_found: URL court non défini

translations/generated.go

@@ -0,0 +1,133 @@
+// This file is generated by the "kurzd generate-translations" command. DO NOT EDIT.
+
+package translations
+
+import (
+	"github.com/nicksnyder/go-i18n/v2/i18n"
+	"golang.org/x/text/language"
+)
+
+func init() {
+	loadMessages = func(b *i18n.Bundle) error {
+		// These are the messages extracted from all the messages.<lang>.yaml files.
+		m := make(MessageMap)
+		m[language.Make("en")] = []*i18n.Message{
+			{
+				ID:				"error.none",
+				Other:			"No error",
+			},
+			{
+				ID:				"error.short_not_created",
+				Other:			"Short URL not created for a new target",
+			},
+			{
+				ID:				"error.storage_unspecified",
+				Other:			"Storage unspecified error",
+			},
+			{
+				ID:				"error.target_blocked",
+				Other:			"Target blocked",
+			},
+			{
+				ID:				"error.target_censored",
+				Other:			"Target unavailable for legal reasons",
+			},
+			{
+				ID:				"Zerror.error_format",
+				Description:	"The format for errors",
+				LeftDelim:		"{",
+				RightDelim:		"}",
+				One:			"{.Kind}: {.Detail}",
+			},
+			{
+				ID:				"error.short_not_found",
+				Other:			"Short URL not defined",
+			},
+			{
+				ID:				"error.storage_read",
+				Other:			"Storage read error",
+			},
+			{
+				ID:				"error.storage_write",
+				Other:			"Storage write error",
+			},
+			{
+				ID:				"error.target_invalid",
+				Other:			"Target invalid",
+			},
+			{
+				ID:				"error.unimplemented",
+				Other:			"Not yet implemented",
+			},
+			{
+				ID:				"web.ui.empty.target",
+				Other:			"empty target",
+			},
+		}
+		m[language.Make("fr")] = []*i18n.Message{
+			{
+				ID:				"Zerror.error_format",
+				Description:	"Le format des erreurs",
+				LeftDelim:		"{",
+				RightDelim:		"}",
+				One:			"{.Kind}: {.Detail}",
+			},
+			{
+				ID:				"error.short_not_created",
+				Other:			"URL court non créé pour une nouvelle cible",
+			},
+			{
+				ID:				"error.storage_unspecified",
+				Other:			"Erreur de stockage non spécifique",
+			},
+			{
+				ID:				"error.storage_write",
+				Other:			"Erreur d'écriture sur le stockage",
+			},
+			{
+				ID:				"error.target_blocked",
+				Other:			"Cible bloquée",
+			},
+			{
+				ID:				"error.target_censored",
+				Other:			"Cible censurée",
+			},
+			{
+				ID:				"error.target_invalid",
+				Other:			"Cible invalide",
+			},
+			{
+				ID:				"error.none",
+				Other:			"Aucune erreur",
+			},
+			{
+				ID:				"error.short_not_found",
+				Other:			"URL court non défini",
+			},
+			{
+				ID:				"error.storage_read",
+				Other:			"Erreur de lecture depuis le stockage",
+			},
+			{
+				ID:				"error.unimplemented",
+				Other:			"Pas encore réalisé",
+			},
+			{
+				ID:				"web.ui.empty.target",
+				Other:			"cible vide",
+			},
+		}
+
+		for tag, messagesByTag := range m {
+			err := b.AddMessages(tag, messagesByTag...)
+			if err != nil {
+				b = &i18n.Bundle{}
+				return err
+			}
+		}
+
+		return nil
+	}
+}
+
+/* vim: set tabstop=4 : */

translations/template.go.gotext

@@ -0,0 +1,56 @@
+// This file is generated by the "kurzd generate-translations" command. DO NOT EDIT.
+
+package translations
+
+import (
+	"github.com/nicksnyder/go-i18n/v2/i18n"
+	"golang.org/x/text/language"
+)
+
+func init() {
+	loadMessages = func(b *i18n.Bundle) error {
+		// These are the messages extracted from all the messages.<lang>.yaml files.
+		m := make(MessageMap)
+
+		{{- range $tag, $messagesByTag :=  . }}
+		m[language.Make("{{$tag}}")] = []*i18n.Message{
+			{{- range $message := $messagesByTag }}
+			{
+				{{- if ne $message.ID "" }}
+				ID:				"{{$message.ID}}",{{ end }}
+				{{- if ne $message.Hash "" }}
+				Hash:				"{{$message.Hash}}",{{end }}
+				{{- if ne $message.Description "" }}
+				Description:	"{{$message.Description}}",{{ end }}
+				{{- if ne $message.LeftDelim "" }}
+				LeftDelim:		"{{$message.LeftDelim}}",{{ end }}
+				{{- if ne $message.RightDelim "" }}
+				RightDelim:		"{{$message.RightDelim}}",{{ end }}
+				{{- if ne $message.Zero "" }}
+				Zero:			"{{$message.Zero}}",{{ end }}
+				{{- if ne $message.One "" }}
+				One:			"{{$message.One}}",{{ end }}
+				{{- if ne $message.Two "" }}
+				Two:			"{{$message.Two}}",{{ end }}
+				{{- if ne $message.Few "" }}
+				Few:			"{{$message.Few}}",{{ end }}
+				{{- if ne $message.Many "" }}
+				Many:			"{{$message.Many}}",{{ end }}
+				{{- if ne $message.Other "" }}
+				Other:			"{{$message.Other}}",{{ end }}
+			},{{ end }}
+		}{{ end }}
+
+		for tag, messagesByTag := range m {
+			err := b.AddMessages(tag, messagesByTag...)
+			if err != nil {
+				b = &i18n.Bundle{}
+				return err
+			}
+		}
+
+		return nil
+	}
+}
+
+/* vim: set tabstop=4 : */

translations/translations.go

@@ -0,0 +1,29 @@
+/*
+Package translations contains the generated Go file built by the "kurzd generate-translations"
+command.
+
+It MUST NOT be edited by hand.
+*/
+package translations
+
+import (
+	"github.com/nicksnyder/go-i18n/v2/i18n"
+	"golang.org/x/text/language"
+	"log"
+)
+
+type MessageMap map[language.Tag][](*i18n.Message)
+
+// Provide a stub loader to allow compilation before the first translations generation.
+var loadMessages func(bundle *i18n.Bundle) error = func(bundle *i18n.Bundle) error {
+	log.Println("Running the stub translations loader. Only useful to run generate-translations.")
+	return nil
+}
+
+func MustLoadMessages(b *i18n.Bundle) {
+	// This function is created when running kurzd generate-translations.
+	err := loadMessages(b)
+	if err != nil {
+		panic(err)
+	}
+}

web/api/api.go

@@ -76,4 +76,3 @@ func SetupRoutes(router *mux.Router) {
 func URLFromRoute(name string, args map[string]string) url.URL {
 	return url.URL{}
 }
-

web/ui/post_target.go

@@ -95,7 +95,7 @@ func validateTarget(raw string, localizer *i18n.Localizer) (string, error) {
 	if raw == "" {
 		detail := localizer.MustLocalize(&i18n.LocalizeConfig{
 			DefaultMessage: &i18n.Message{
-				ID:  "web.ui.empty.target",
+				ID:    "web.ui.empty.target",
 				Other: "empty target",
 			},
 		})