gtsocial-umbx

runes.go (8677B)

---

 // Copyright 2014 The Go Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.
 
 // Package runes provide transforms for UTF-8 encoded text.
 package runes // import "golang.org/x/text/runes"
 
 import (
 	"unicode"
 	"unicode/utf8"
 
 	"golang.org/x/text/transform"
 )
 
 // A Set is a collection of runes.
 type Set interface {
 	// Contains returns true if r is contained in the set.
 	Contains(r rune) bool
 }
 
 type setFunc func(rune) bool
 
 func (s setFunc) Contains(r rune) bool {
 	return s(r)
 }
 
 // Note: using funcs here instead of wrapping types result in cleaner
 // documentation and a smaller API.
 
 // In creates a Set with a Contains method that returns true for all runes in
 // the given RangeTable.
 func In(rt *unicode.RangeTable) Set {
 	return setFunc(func(r rune) bool { return unicode.Is(rt, r) })
 }
 
 // NotIn creates a Set with a Contains method that returns true for all runes not
 // in the given RangeTable.
 func NotIn(rt *unicode.RangeTable) Set {
 	return setFunc(func(r rune) bool { return !unicode.Is(rt, r) })
 }
 
 // Predicate creates a Set with a Contains method that returns f(r).
 func Predicate(f func(rune) bool) Set {
 	return setFunc(f)
 }
 
 // Transformer implements the transform.Transformer interface.
 type Transformer struct {
 	t transform.SpanningTransformer
 }
 
 func (t Transformer) Transform(dst, src []byte, atEOF bool) (nDst, nSrc int, err error) {
 	return t.t.Transform(dst, src, atEOF)
 }
 
 func (t Transformer) Span(b []byte, atEOF bool) (n int, err error) {
 	return t.t.Span(b, atEOF)
 }
 
 func (t Transformer) Reset() { t.t.Reset() }
 
 // Bytes returns a new byte slice with the result of converting b using t.  It
 // calls Reset on t. It returns nil if any error was found. This can only happen
 // if an error-producing Transformer is passed to If.
 func (t Transformer) Bytes(b []byte) []byte {
 	b, _, err := transform.Bytes(t, b)
 	if err != nil {
 		return nil
 	}
 	return b
 }
 
 // String returns a string with the result of converting s using t. It calls
 // Reset on t. It returns the empty string if any error was found. This can only
 // happen if an error-producing Transformer is passed to If.
 func (t Transformer) String(s string) string {
 	s, _, err := transform.String(t, s)
 	if err != nil {
 		return ""
 	}
 	return s
 }
 
 // TODO:
 // - Copy: copying strings and bytes in whole-rune units.
 // - Validation (maybe)
 // - Well-formed-ness (maybe)
 
 const runeErrorString = string(utf8.RuneError)
 
 // Remove returns a Transformer that removes runes r for which s.Contains(r).
 // Illegal input bytes are replaced by RuneError before being passed to f.
 func Remove(s Set) Transformer {
 	if f, ok := s.(setFunc); ok {
 		// This little trick cuts the running time of BenchmarkRemove for sets
 		// created by Predicate roughly in half.
 		// TODO: special-case RangeTables as well.
 		return Transformer{remove(f)}
 	}
 	return Transformer{remove(s.Contains)}
 }
 
 // TODO: remove transform.RemoveFunc.
 
 type remove func(r rune) bool
 
 func (remove) Reset() {}
 
 // Span implements transform.Spanner.
 func (t remove) Span(src []byte, atEOF bool) (n int, err error) {
 	for r, size := rune(0), 0; n < len(src); {
 		if r = rune(src[n]); r < utf8.RuneSelf {
 			size = 1
 		} else if r, size = utf8.DecodeRune(src[n:]); size == 1 {
 			// Invalid rune.
 			if !atEOF && !utf8.FullRune(src[n:]) {
 				err = transform.ErrShortSrc
 			} else {
 				err = transform.ErrEndOfSpan
 			}
 			break
 		}
 		if t(r) {
 			err = transform.ErrEndOfSpan
 			break
 		}
 		n += size
 	}
 	return
 }
 
 // Transform implements transform.Transformer.
 func (t remove) Transform(dst, src []byte, atEOF bool) (nDst, nSrc int, err error) {
 	for r, size := rune(0), 0; nSrc < len(src); {
 		if r = rune(src[nSrc]); r < utf8.RuneSelf {
 			size = 1
 		} else if r, size = utf8.DecodeRune(src[nSrc:]); size == 1 {
 			// Invalid rune.
 			if !atEOF && !utf8.FullRune(src[nSrc:]) {
 				err = transform.ErrShortSrc
 				break
 			}
 			// We replace illegal bytes with RuneError. Not doing so might
 			// otherwise turn a sequence of invalid UTF-8 into valid UTF-8.
 			// The resulting byte sequence may subsequently contain runes
 			// for which t(r) is true that were passed unnoticed.
 			if !t(utf8.RuneError) {
 				if nDst+3 > len(dst) {
 					err = transform.ErrShortDst
 					break
 				}
 				dst[nDst+0] = runeErrorString[0]
 				dst[nDst+1] = runeErrorString[1]
 				dst[nDst+2] = runeErrorString[2]
 				nDst += 3
 			}
 			nSrc++
 			continue
 		}
 		if t(r) {
 			nSrc += size
 			continue
 		}
 		if nDst+size > len(dst) {
 			err = transform.ErrShortDst
 			break
 		}
 		for i := 0; i < size; i++ {
 			dst[nDst] = src[nSrc]
 			nDst++
 			nSrc++
 		}
 	}
 	return
 }
 
 // Map returns a Transformer that maps the runes in the input using the given
 // mapping. Illegal bytes in the input are converted to utf8.RuneError before
 // being passed to the mapping func.
 func Map(mapping func(rune) rune) Transformer {
 	return Transformer{mapper(mapping)}
 }
 
 type mapper func(rune) rune
 
 func (mapper) Reset() {}
 
 // Span implements transform.Spanner.
 func (t mapper) Span(src []byte, atEOF bool) (n int, err error) {
 	for r, size := rune(0), 0; n < len(src); n += size {
 		if r = rune(src[n]); r < utf8.RuneSelf {
 			size = 1
 		} else if r, size = utf8.DecodeRune(src[n:]); size == 1 {
 			// Invalid rune.
 			if !atEOF && !utf8.FullRune(src[n:]) {
 				err = transform.ErrShortSrc
 			} else {
 				err = transform.ErrEndOfSpan
 			}
 			break
 		}
 		if t(r) != r {
 			err = transform.ErrEndOfSpan
 			break
 		}
 	}
 	return n, err
 }
 
 // Transform implements transform.Transformer.
 func (t mapper) Transform(dst, src []byte, atEOF bool) (nDst, nSrc int, err error) {
 	var replacement rune
 	var b [utf8.UTFMax]byte
 
 	for r, size := rune(0), 0; nSrc < len(src); {
 		if r = rune(src[nSrc]); r < utf8.RuneSelf {
 			if replacement = t(r); replacement < utf8.RuneSelf {
 				if nDst == len(dst) {
 					err = transform.ErrShortDst
 					break
 				}
 				dst[nDst] = byte(replacement)
 				nDst++
 				nSrc++
 				continue
 			}
 			size = 1
 		} else if r, size = utf8.DecodeRune(src[nSrc:]); size == 1 {
 			// Invalid rune.
 			if !atEOF && !utf8.FullRune(src[nSrc:]) {
 				err = transform.ErrShortSrc
 				break
 			}
 
 			if replacement = t(utf8.RuneError); replacement == utf8.RuneError {
 				if nDst+3 > len(dst) {
 					err = transform.ErrShortDst
 					break
 				}
 				dst[nDst+0] = runeErrorString[0]
 				dst[nDst+1] = runeErrorString[1]
 				dst[nDst+2] = runeErrorString[2]
 				nDst += 3
 				nSrc++
 				continue
 			}
 		} else if replacement = t(r); replacement == r {
 			if nDst+size > len(dst) {
 				err = transform.ErrShortDst
 				break
 			}
 			for i := 0; i < size; i++ {
 				dst[nDst] = src[nSrc]
 				nDst++
 				nSrc++
 			}
 			continue
 		}
 
 		n := utf8.EncodeRune(b[:], replacement)
 
 		if nDst+n > len(dst) {
 			err = transform.ErrShortDst
 			break
 		}
 		for i := 0; i < n; i++ {
 			dst[nDst] = b[i]
 			nDst++
 		}
 		nSrc += size
 	}
 	return
 }
 
 // ReplaceIllFormed returns a transformer that replaces all input bytes that are
 // not part of a well-formed UTF-8 code sequence with utf8.RuneError.
 func ReplaceIllFormed() Transformer {
 	return Transformer{&replaceIllFormed{}}
 }
 
 type replaceIllFormed struct{ transform.NopResetter }
 
 func (t replaceIllFormed) Span(src []byte, atEOF bool) (n int, err error) {
 	for n < len(src) {
 		// ASCII fast path.
 		if src[n] < utf8.RuneSelf {
 			n++
 			continue
 		}
 
 		r, size := utf8.DecodeRune(src[n:])
 
 		// Look for a valid non-ASCII rune.
 		if r != utf8.RuneError || size != 1 {
 			n += size
 			continue
 		}
 
 		// Look for short source data.
 		if !atEOF && !utf8.FullRune(src[n:]) {
 			err = transform.ErrShortSrc
 			break
 		}
 
 		// We have an invalid rune.
 		err = transform.ErrEndOfSpan
 		break
 	}
 	return n, err
 }
 
 func (t replaceIllFormed) Transform(dst, src []byte, atEOF bool) (nDst, nSrc int, err error) {
 	for nSrc < len(src) {
 		// ASCII fast path.
 		if r := src[nSrc]; r < utf8.RuneSelf {
 			if nDst == len(dst) {
 				err = transform.ErrShortDst
 				break
 			}
 			dst[nDst] = r
 			nDst++
 			nSrc++
 			continue
 		}
 
 		// Look for a valid non-ASCII rune.
 		if _, size := utf8.DecodeRune(src[nSrc:]); size != 1 {
 			if size != copy(dst[nDst:], src[nSrc:nSrc+size]) {
 				err = transform.ErrShortDst
 				break
 			}
 			nDst += size
 			nSrc += size
 			continue
 		}
 
 		// Look for short source data.
 		if !atEOF && !utf8.FullRune(src[nSrc:]) {
 			err = transform.ErrShortSrc
 			break
 		}
 
 		// We have an invalid rune.
 		if nDst+3 > len(dst) {
 			err = transform.ErrShortDst
 			break
 		}
 		dst[nDst+0] = runeErrorString[0]
 		dst[nDst+1] = runeErrorString[1]
 		dst[nDst+2] = runeErrorString[2]
 		nDst += 3
 		nSrc++
 	}
 	return nDst, nSrc, err
 }