maketables.go 4.1 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161
  1. // Copyright 2013 The Go Authors. All rights reserved.
  2. // Use of this source code is governed by a BSD-style
  3. // license that can be found in the LICENSE file.
  4. // +build ignore
  5. package main
  6. // This program generates tables.go:
  7. // go run maketables.go | gofmt > tables.go
  8. import (
  9. "bufio"
  10. "fmt"
  11. "log"
  12. "net/http"
  13. "sort"
  14. "strings"
  15. )
  16. func main() {
  17. fmt.Printf("// generated by go run maketables.go; DO NOT EDIT\n\n")
  18. fmt.Printf("// Package simplifiedchinese provides Simplified Chinese encodings such as GBK.\n")
  19. fmt.Printf(`package simplifiedchinese // import "golang.org/x/text/encoding/simplifiedchinese"` + "\n\n")
  20. printGB18030()
  21. printGBK()
  22. }
  23. func printGB18030() {
  24. res, err := http.Get("http://encoding.spec.whatwg.org/index-gb18030.txt")
  25. if err != nil {
  26. log.Fatalf("Get: %v", err)
  27. }
  28. defer res.Body.Close()
  29. fmt.Printf("// gb18030 is the table from http://encoding.spec.whatwg.org/index-gb18030.txt\n")
  30. fmt.Printf("var gb18030 = [...][2]uint16{\n")
  31. scanner := bufio.NewScanner(res.Body)
  32. for scanner.Scan() {
  33. s := strings.TrimSpace(scanner.Text())
  34. if s == "" || s[0] == '#' {
  35. continue
  36. }
  37. x, y := uint32(0), uint32(0)
  38. if _, err := fmt.Sscanf(s, "%d 0x%x", &x, &y); err != nil {
  39. log.Fatalf("could not parse %q", s)
  40. }
  41. if x < 0x10000 && y < 0x10000 {
  42. fmt.Printf("\t{0x%04x, 0x%04x},\n", x, y)
  43. }
  44. }
  45. fmt.Printf("}\n\n")
  46. }
  47. func printGBK() {
  48. res, err := http.Get("http://encoding.spec.whatwg.org/index-gbk.txt")
  49. if err != nil {
  50. log.Fatalf("Get: %v", err)
  51. }
  52. defer res.Body.Close()
  53. mapping := [65536]uint16{}
  54. reverse := [65536]uint16{}
  55. scanner := bufio.NewScanner(res.Body)
  56. for scanner.Scan() {
  57. s := strings.TrimSpace(scanner.Text())
  58. if s == "" || s[0] == '#' {
  59. continue
  60. }
  61. x, y := uint16(0), uint16(0)
  62. if _, err := fmt.Sscanf(s, "%d 0x%x", &x, &y); err != nil {
  63. log.Fatalf("could not parse %q", s)
  64. }
  65. if x < 0 || 126*190 <= x {
  66. log.Fatalf("GBK code %d is out of range", x)
  67. }
  68. mapping[x] = y
  69. if reverse[y] == 0 {
  70. c0, c1 := x/190, x%190
  71. if c1 >= 0x3f {
  72. c1++
  73. }
  74. reverse[y] = (0x81+c0)<<8 | (0x40 + c1)
  75. }
  76. }
  77. if err := scanner.Err(); err != nil {
  78. log.Fatalf("scanner error: %v", err)
  79. }
  80. fmt.Printf("// decode is the decoding table from GBK code to Unicode.\n")
  81. fmt.Printf("// It is defined at http://encoding.spec.whatwg.org/index-gbk.txt\n")
  82. fmt.Printf("var decode = [...]uint16{\n")
  83. for i, v := range mapping {
  84. if v != 0 {
  85. fmt.Printf("\t%d: 0x%04X,\n", i, v)
  86. }
  87. }
  88. fmt.Printf("}\n\n")
  89. // Any run of at least separation continuous zero entries in the reverse map will
  90. // be a separate encode table.
  91. const separation = 1024
  92. intervals := []interval(nil)
  93. low, high := -1, -1
  94. for i, v := range reverse {
  95. if v == 0 {
  96. continue
  97. }
  98. if low < 0 {
  99. low = i
  100. } else if i-high >= separation {
  101. if high >= 0 {
  102. intervals = append(intervals, interval{low, high})
  103. }
  104. low = i
  105. }
  106. high = i + 1
  107. }
  108. if high >= 0 {
  109. intervals = append(intervals, interval{low, high})
  110. }
  111. sort.Sort(byDecreasingLength(intervals))
  112. fmt.Printf("const numEncodeTables = %d\n\n", len(intervals))
  113. fmt.Printf("// encodeX are the encoding tables from Unicode to GBK code,\n")
  114. fmt.Printf("// sorted by decreasing length.\n")
  115. for i, v := range intervals {
  116. fmt.Printf("// encode%d: %5d entries for runes in [%5d, %5d).\n", i, v.len(), v.low, v.high)
  117. }
  118. fmt.Printf("\n")
  119. for i, v := range intervals {
  120. fmt.Printf("const encode%dLow, encode%dHigh = %d, %d\n\n", i, i, v.low, v.high)
  121. fmt.Printf("var encode%d = [...]uint16{\n", i)
  122. for j := v.low; j < v.high; j++ {
  123. x := reverse[j]
  124. if x == 0 {
  125. continue
  126. }
  127. fmt.Printf("\t%d-%d: 0x%04X,\n", j, v.low, x)
  128. }
  129. fmt.Printf("}\n\n")
  130. }
  131. }
  132. // interval is a half-open interval [low, high).
  133. type interval struct {
  134. low, high int
  135. }
  136. func (i interval) len() int { return i.high - i.low }
  137. // byDecreasingLength sorts intervals by decreasing length.
  138. type byDecreasingLength []interval
  139. func (b byDecreasingLength) Len() int { return len(b) }
  140. func (b byDecreasingLength) Less(i, j int) bool { return b[i].len() > b[j].len() }
  141. func (b byDecreasingLength) Swap(i, j int) { b[i], b[j] = b[j], b[i] }