This commit is contained in:
215
vendor/github.com/rivo/uniseg/gen_breaktest.go
generated
vendored
Normal file
215
vendor/github.com/rivo/uniseg/gen_breaktest.go
generated
vendored
Normal file
@ -0,0 +1,215 @@
|
||||
//go:build generate
|
||||
|
||||
// This program generates a Go containing a slice of test cases based on the
|
||||
// Unicode Character Database auxiliary data files. The command line arguments
|
||||
// are as follows:
|
||||
//
|
||||
// 1. The name of the Unicode data file (just the filename, without extension).
|
||||
// 2. The name of the locally generated Go file.
|
||||
// 3. The name of the slice containing the test cases.
|
||||
// 4. The name of the generator, for logging purposes.
|
||||
//
|
||||
//go:generate go run gen_breaktest.go GraphemeBreakTest graphemebreak_test.go graphemeBreakTestCases graphemes
|
||||
//go:generate go run gen_breaktest.go WordBreakTest wordbreak_test.go wordBreakTestCases words
|
||||
//go:generate go run gen_breaktest.go SentenceBreakTest sentencebreak_test.go sentenceBreakTestCases sentences
|
||||
//go:generate go run gen_breaktest.go LineBreakTest linebreak_test.go lineBreakTestCases lines
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"bytes"
|
||||
"errors"
|
||||
"fmt"
|
||||
"go/format"
|
||||
"io/ioutil"
|
||||
"log"
|
||||
"net/http"
|
||||
"os"
|
||||
"time"
|
||||
)
|
||||
|
||||
// We want to test against a specific version rather than the latest. When the
|
||||
// package is upgraded to a new version, change these to generate new tests.
|
||||
const (
|
||||
testCaseURL = `https://www.unicode.org/Public/15.0.0/ucd/auxiliary/%s.txt`
|
||||
)
|
||||
|
||||
func main() {
|
||||
if len(os.Args) < 5 {
|
||||
fmt.Println("Not enough arguments, see code for details")
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
log.SetPrefix("gen_breaktest (" + os.Args[4] + "): ")
|
||||
log.SetFlags(0)
|
||||
|
||||
// Read text of testcases and parse into Go source code.
|
||||
src, err := parse(fmt.Sprintf(testCaseURL, os.Args[1]))
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
|
||||
// Format the Go code.
|
||||
formatted, err := format.Source(src)
|
||||
if err != nil {
|
||||
log.Fatalln("gofmt:", err)
|
||||
}
|
||||
|
||||
// Write it out.
|
||||
log.Print("Writing to ", os.Args[2])
|
||||
if err := ioutil.WriteFile(os.Args[2], formatted, 0644); err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
// parse reads a break text file, either from a local file or from a URL. It
|
||||
// parses the file data into Go source code representing the test cases.
|
||||
func parse(url string) ([]byte, error) {
|
||||
log.Printf("Parsing %s", url)
|
||||
res, err := http.Get(url)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
body := res.Body
|
||||
defer body.Close()
|
||||
|
||||
buf := new(bytes.Buffer)
|
||||
buf.Grow(120 << 10)
|
||||
buf.WriteString(`// Code generated via go generate from gen_breaktest.go. DO NOT EDIT.
|
||||
|
||||
package uniseg
|
||||
|
||||
// ` + os.Args[3] + ` are Grapheme testcases taken from
|
||||
// ` + url + `
|
||||
// on ` + time.Now().Format("January 2, 2006") + `. See
|
||||
// https://www.unicode.org/license.html for the Unicode license agreement.
|
||||
var ` + os.Args[3] + ` = []testCase {
|
||||
`)
|
||||
|
||||
sc := bufio.NewScanner(body)
|
||||
num := 1
|
||||
var line []byte
|
||||
original := make([]byte, 0, 64)
|
||||
expected := make([]byte, 0, 64)
|
||||
for sc.Scan() {
|
||||
num++
|
||||
line = sc.Bytes()
|
||||
if len(line) == 0 || line[0] == '#' {
|
||||
continue
|
||||
}
|
||||
var comment []byte
|
||||
if i := bytes.IndexByte(line, '#'); i >= 0 {
|
||||
comment = bytes.TrimSpace(line[i+1:])
|
||||
line = bytes.TrimSpace(line[:i])
|
||||
}
|
||||
original, expected, err := parseRuneSequence(line, original[:0], expected[:0])
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf(`line %d: %v: %q`, num, err, line)
|
||||
}
|
||||
fmt.Fprintf(buf, "\t{original: \"%s\", expected: %s}, // %s\n", original, expected, comment)
|
||||
}
|
||||
if err := sc.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Check for final "# EOF", useful check if we're streaming via HTTP
|
||||
if !bytes.Equal(line, []byte("# EOF")) {
|
||||
return nil, fmt.Errorf(`line %d: exected "# EOF" as final line, got %q`, num, line)
|
||||
}
|
||||
buf.WriteString("}\n")
|
||||
return buf.Bytes(), nil
|
||||
}
|
||||
|
||||
// Used by parseRuneSequence to match input via bytes.HasPrefix.
|
||||
var (
|
||||
prefixBreak = []byte("÷ ")
|
||||
prefixDontBreak = []byte("× ")
|
||||
breakOk = []byte("÷")
|
||||
breakNo = []byte("×")
|
||||
)
|
||||
|
||||
// parseRuneSequence parses a rune + breaking opportunity sequence from b
|
||||
// and appends the Go code for testcase.original to orig
|
||||
// and appends the Go code for testcase.expected to exp.
|
||||
// It retuns the new orig and exp slices.
|
||||
//
|
||||
// E.g. for the input b="÷ 0020 × 0308 ÷ 1F1E6 ÷"
|
||||
// it will append
|
||||
//
|
||||
// "\u0020\u0308\U0001F1E6"
|
||||
//
|
||||
// and "[][]rune{{0x0020,0x0308},{0x1F1E6},}"
|
||||
// to orig and exp respectively.
|
||||
//
|
||||
// The formatting of exp is expected to be cleaned up by gofmt or format.Source.
|
||||
// Note we explicitly require the sequence to start with ÷ and we implicitly
|
||||
// require it to end with ÷.
|
||||
func parseRuneSequence(b, orig, exp []byte) ([]byte, []byte, error) {
|
||||
// Check for and remove first ÷ or ×.
|
||||
if !bytes.HasPrefix(b, prefixBreak) && !bytes.HasPrefix(b, prefixDontBreak) {
|
||||
return nil, nil, errors.New("expected ÷ or × as first character")
|
||||
}
|
||||
if bytes.HasPrefix(b, prefixBreak) {
|
||||
b = b[len(prefixBreak):]
|
||||
} else {
|
||||
b = b[len(prefixDontBreak):]
|
||||
}
|
||||
|
||||
boundary := true
|
||||
exp = append(exp, "[][]rune{"...)
|
||||
for len(b) > 0 {
|
||||
if boundary {
|
||||
exp = append(exp, '{')
|
||||
}
|
||||
exp = append(exp, "0x"...)
|
||||
// Find end of hex digits.
|
||||
var i int
|
||||
for i = 0; i < len(b) && b[i] != ' '; i++ {
|
||||
if d := b[i]; ('0' <= d || d <= '9') ||
|
||||
('A' <= d || d <= 'F') ||
|
||||
('a' <= d || d <= 'f') {
|
||||
continue
|
||||
}
|
||||
return nil, nil, errors.New("bad hex digit")
|
||||
}
|
||||
switch i {
|
||||
case 4:
|
||||
orig = append(orig, "\\u"...)
|
||||
case 5:
|
||||
orig = append(orig, "\\U000"...)
|
||||
default:
|
||||
return nil, nil, errors.New("unsupport code point hex length")
|
||||
}
|
||||
orig = append(orig, b[:i]...)
|
||||
exp = append(exp, b[:i]...)
|
||||
b = b[i:]
|
||||
|
||||
// Check for space between hex and ÷ or ×.
|
||||
if len(b) < 1 || b[0] != ' ' {
|
||||
return nil, nil, errors.New("bad input")
|
||||
}
|
||||
b = b[1:]
|
||||
|
||||
// Check for next boundary.
|
||||
switch {
|
||||
case bytes.HasPrefix(b, breakOk):
|
||||
boundary = true
|
||||
b = b[len(breakOk):]
|
||||
case bytes.HasPrefix(b, breakNo):
|
||||
boundary = false
|
||||
b = b[len(breakNo):]
|
||||
default:
|
||||
return nil, nil, errors.New("missing ÷ or ×")
|
||||
}
|
||||
if boundary {
|
||||
exp = append(exp, '}')
|
||||
}
|
||||
exp = append(exp, ',')
|
||||
if len(b) > 0 && b[0] == ' ' {
|
||||
b = b[1:]
|
||||
}
|
||||
}
|
||||
exp = append(exp, '}')
|
||||
return orig, exp, nil
|
||||
}
|
Reference in New Issue
Block a user