maketables.go 4.4 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161
  1. // Copyright 2013 The Go Authors. All rights reserved.
  2. // Use of this source code is governed by a BSD-style
  3. // license that can be found in the LICENSE file.
  4. // +build ignore
  5. package main
  6. // This program generates tables.go:
  7. // go run maketables.go | gofmt > tables.go
  8. // TODO: Emoji extensions?
  9. // http://www.unicode.org/faq/emoji_dingbats.html
  10. // http://www.unicode.org/Public/UNIDATA/EmojiSources.txt
  11. import (
  12. "bufio"
  13. "fmt"
  14. "log"
  15. "net/http"
  16. "sort"
  17. "strings"
  18. )
  19. type entry struct {
  20. jisCode, table int
  21. }
  22. func main() {
  23. fmt.Printf("// generated by go run maketables.go; DO NOT EDIT\n\n")
  24. fmt.Printf("// Package japanese provides Japanese encodings such as EUC-JP and Shift JIS.\n")
  25. fmt.Printf(`package japanese // import "golang.org/x/text/encoding/japanese"` + "\n\n")
  26. reverse := [65536]entry{}
  27. for i := range reverse {
  28. reverse[i].table = -1
  29. }
  30. tables := []struct {
  31. url string
  32. name string
  33. }{
  34. {"http://encoding.spec.whatwg.org/index-jis0208.txt", "0208"},
  35. {"http://encoding.spec.whatwg.org/index-jis0212.txt", "0212"},
  36. }
  37. for i, table := range tables {
  38. res, err := http.Get(table.url)
  39. if err != nil {
  40. log.Fatalf("%q: Get: %v", table.url, err)
  41. }
  42. defer res.Body.Close()
  43. mapping := [65536]uint16{}
  44. scanner := bufio.NewScanner(res.Body)
  45. for scanner.Scan() {
  46. s := strings.TrimSpace(scanner.Text())
  47. if s == "" || s[0] == '#' {
  48. continue
  49. }
  50. x, y := 0, uint16(0)
  51. if _, err := fmt.Sscanf(s, "%d 0x%x", &x, &y); err != nil {
  52. log.Fatalf("%q: could not parse %q", table.url, s)
  53. }
  54. if x < 0 || 120*94 <= x {
  55. log.Fatalf("%q: JIS code %d is out of range", table.url, x)
  56. }
  57. mapping[x] = y
  58. if reverse[y].table == -1 {
  59. reverse[y] = entry{jisCode: x, table: i}
  60. }
  61. }
  62. if err := scanner.Err(); err != nil {
  63. log.Fatalf("%q: scanner error: %v", table.url, err)
  64. }
  65. fmt.Printf("// jis%sDecode is the decoding table from JIS %s code to Unicode.\n// It is defined at %s\n",
  66. table.name, table.name, table.url)
  67. fmt.Printf("var jis%sDecode = [...]uint16{\n", table.name)
  68. for i, m := range mapping {
  69. if m != 0 {
  70. fmt.Printf("\t%d: 0x%04X,\n", i, m)
  71. }
  72. }
  73. fmt.Printf("}\n\n")
  74. }
  75. // Any run of at least separation continuous zero entries in the reverse map will
  76. // be a separate encode table.
  77. const separation = 1024
  78. intervals := []interval(nil)
  79. low, high := -1, -1
  80. for i, v := range reverse {
  81. if v.table == -1 {
  82. continue
  83. }
  84. if low < 0 {
  85. low = i
  86. } else if i-high >= separation {
  87. if high >= 0 {
  88. intervals = append(intervals, interval{low, high})
  89. }
  90. low = i
  91. }
  92. high = i + 1
  93. }
  94. if high >= 0 {
  95. intervals = append(intervals, interval{low, high})
  96. }
  97. sort.Sort(byDecreasingLength(intervals))
  98. fmt.Printf("const (\n")
  99. fmt.Printf("\tjis0208 = 1\n")
  100. fmt.Printf("\tjis0212 = 2\n")
  101. fmt.Printf("\tcodeMask = 0x7f\n")
  102. fmt.Printf("\tcodeShift = 7\n")
  103. fmt.Printf("\ttableShift = 14\n")
  104. fmt.Printf(")\n\n")
  105. fmt.Printf("const numEncodeTables = %d\n\n", len(intervals))
  106. fmt.Printf("// encodeX are the encoding tables from Unicode to JIS code,\n")
  107. fmt.Printf("// sorted by decreasing length.\n")
  108. for i, v := range intervals {
  109. fmt.Printf("// encode%d: %5d entries for runes in [%5d, %5d).\n", i, v.len(), v.low, v.high)
  110. }
  111. fmt.Printf("//\n")
  112. fmt.Printf("// The high two bits of the value record whether the JIS code comes from the\n")
  113. fmt.Printf("// JIS0208 table (high bits == 1) or the JIS0212 table (high bits == 2).\n")
  114. fmt.Printf("// The low 14 bits are two 7-bit unsigned integers j1 and j2 that form the\n")
  115. fmt.Printf("// JIS code (94*j1 + j2) within that table.\n")
  116. fmt.Printf("\n")
  117. for i, v := range intervals {
  118. fmt.Printf("const encode%dLow, encode%dHigh = %d, %d\n\n", i, i, v.low, v.high)
  119. fmt.Printf("var encode%d = [...]uint16{\n", i)
  120. for j := v.low; j < v.high; j++ {
  121. x := reverse[j]
  122. if x.table == -1 {
  123. continue
  124. }
  125. fmt.Printf("\t%d - %d: jis%s<<14 | 0x%02X<<7 | 0x%02X,\n",
  126. j, v.low, tables[x.table].name, x.jisCode/94, x.jisCode%94)
  127. }
  128. fmt.Printf("}\n\n")
  129. }
  130. }
  131. // interval is a half-open interval [low, high).
  132. type interval struct {
  133. low, high int
  134. }
  135. func (i interval) len() int { return i.high - i.low }
  136. // byDecreasingLength sorts intervals by decreasing length.
  137. type byDecreasingLength []interval
  138. func (b byDecreasingLength) Len() int { return len(b) }
  139. func (b byDecreasingLength) Less(i, j int) bool { return b[i].len() > b[j].len() }
  140. func (b byDecreasingLength) Swap(i, j int) { b[i], b[j] = b[j], b[i] }