maketables.go 3.9 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140
  1. // Copyright 2013 The Go Authors. All rights reserved.
  2. // Use of this source code is governed by a BSD-style
  3. // license that can be found in the LICENSE file.
  4. // +build ignore
  5. package main
  6. // This program generates tables.go:
  7. // go run maketables.go | gofmt > tables.go
  8. import (
  9. "bufio"
  10. "fmt"
  11. "log"
  12. "net/http"
  13. "sort"
  14. "strings"
  15. )
  16. func main() {
  17. fmt.Printf("// generated by go run maketables.go; DO NOT EDIT\n\n")
  18. fmt.Printf("// Package traditionalchinese provides Traditional Chinese encodings such as Big5.\n")
  19. fmt.Printf(`package traditionalchinese // import "golang.org/x/text/encoding/traditionalchinese"` + "\n\n")
  20. res, err := http.Get("http://encoding.spec.whatwg.org/index-big5.txt")
  21. if err != nil {
  22. log.Fatalf("Get: %v", err)
  23. }
  24. defer res.Body.Close()
  25. mapping := [65536]uint32{}
  26. reverse := [65536 * 4]uint16{}
  27. scanner := bufio.NewScanner(res.Body)
  28. for scanner.Scan() {
  29. s := strings.TrimSpace(scanner.Text())
  30. if s == "" || s[0] == '#' {
  31. continue
  32. }
  33. x, y := uint16(0), uint32(0)
  34. if _, err := fmt.Sscanf(s, "%d 0x%x", &x, &y); err != nil {
  35. log.Fatalf("could not parse %q", s)
  36. }
  37. if x < 0 || 126*157 <= x {
  38. log.Fatalf("Big5 code %d is out of range", x)
  39. }
  40. mapping[x] = y
  41. // The WHATWG spec http://encoding.spec.whatwg.org/#indexes says that
  42. // "The index pointer for code point in index is the first pointer
  43. // corresponding to code point in index", which would normally mean
  44. // that the code below should be guarded by "if reverse[y] == 0", but
  45. // last instead of first seems to match the behavior of
  46. // "iconv -f UTF-8 -t BIG5". For example, U+8005 者 occurs twice in
  47. // http://encoding.spec.whatwg.org/index-big5.txt, as index 2148
  48. // (encoded as "\x8e\xcd") and index 6543 (encoded as "\xaa\xcc")
  49. // and "echo 者 | iconv -f UTF-8 -t BIG5 | xxd" gives "\xaa\xcc".
  50. c0, c1 := x/157, x%157
  51. if c1 < 0x3f {
  52. c1 += 0x40
  53. } else {
  54. c1 += 0x62
  55. }
  56. reverse[y] = (0x81+c0)<<8 | c1
  57. }
  58. if err := scanner.Err(); err != nil {
  59. log.Fatalf("scanner error: %v", err)
  60. }
  61. fmt.Printf("// decode is the decoding table from Big5 code to Unicode.\n")
  62. fmt.Printf("// It is defined at http://encoding.spec.whatwg.org/index-big5.txt\n")
  63. fmt.Printf("var decode = [...]uint32{\n")
  64. for i, v := range mapping {
  65. if v != 0 {
  66. fmt.Printf("\t%d: 0x%08X,\n", i, v)
  67. }
  68. }
  69. fmt.Printf("}\n\n")
  70. // Any run of at least separation continuous zero entries in the reverse map will
  71. // be a separate encode table.
  72. const separation = 1024
  73. intervals := []interval(nil)
  74. low, high := -1, -1
  75. for i, v := range reverse {
  76. if v == 0 {
  77. continue
  78. }
  79. if low < 0 {
  80. low = i
  81. } else if i-high >= separation {
  82. if high >= 0 {
  83. intervals = append(intervals, interval{low, high})
  84. }
  85. low = i
  86. }
  87. high = i + 1
  88. }
  89. if high >= 0 {
  90. intervals = append(intervals, interval{low, high})
  91. }
  92. sort.Sort(byDecreasingLength(intervals))
  93. fmt.Printf("const numEncodeTables = %d\n\n", len(intervals))
  94. fmt.Printf("// encodeX are the encoding tables from Unicode to Big5 code,\n")
  95. fmt.Printf("// sorted by decreasing length.\n")
  96. for i, v := range intervals {
  97. fmt.Printf("// encode%d: %5d entries for runes in [%6d, %6d).\n", i, v.len(), v.low, v.high)
  98. }
  99. fmt.Printf("\n")
  100. for i, v := range intervals {
  101. fmt.Printf("const encode%dLow, encode%dHigh = %d, %d\n\n", i, i, v.low, v.high)
  102. fmt.Printf("var encode%d = [...]uint16{\n", i)
  103. for j := v.low; j < v.high; j++ {
  104. x := reverse[j]
  105. if x == 0 {
  106. continue
  107. }
  108. fmt.Printf("\t%d-%d: 0x%04X,\n", j, v.low, x)
  109. }
  110. fmt.Printf("}\n\n")
  111. }
  112. }
  113. // interval is a half-open interval [low, high).
  114. type interval struct {
  115. low, high int
  116. }
  117. func (i interval) len() int { return i.high - i.low }
  118. // byDecreasingLength sorts intervals by decreasing length.
  119. type byDecreasingLength []interval
  120. func (b byDecreasingLength) Len() int { return len(b) }
  121. func (b byDecreasingLength) Less(i, j int) bool { return b[i].len() > b[j].len() }
  122. func (b byDecreasingLength) Swap(i, j int) { b[i], b[j] = b[j], b[i] }