bloc);
Struct_type* type_descriptor_type =
- Type::make_builtin_struct_type(9,
+ Type::make_builtin_struct_type(10,
"Kind", uint8_type,
"align", uint8_type,
"fieldAlign", uint8_type,
"hashfn", hashfn_type,
"equalfn", equalfn_type,
"string", pointer_string_type,
- "", pointer_uncommon_type);
+ "", pointer_uncommon_type,
+ "ptrToThis",
+ pointer_type_descriptor_type);
Named_type* named = Type::make_builtin_named_type("commonType",
type_descriptor_type);
only_value_methods));
}
+ ++p;
+ gcc_assert(p->field_name() == "ptrToThis");
+ if (name == NULL)
+ vals->push_back(Expression::make_nil(bloc));
+ else
+ {
+ Type* pt = Type::make_pointer_type(name);
+ vals->push_back(Expression::make_type_descriptor(pt, bloc));
+ }
+
++p;
gcc_assert(p == fields->end());
-559f12e8fcd5
+94d654be2064
The first line of this file holds the Mercurial revision number of the
last merge done from the master library sources.
bufio.gox \
bytes.gox \
cmath.gox \
+ crypto.gox \
ebnf.gox \
exec.gox \
expvar.gox \
toolexeclibgocompressdir = $(toolexeclibgodir)/compress
toolexeclibgocompress_DATA = \
+ compress/bzip2.gox \
compress/flate.gox \
compress/gzip.gox \
+ compress/lzw.gox \
compress/zlib.gox
toolexeclibgocontainerdir = $(toolexeclibgodir)/container
crypto/blowfish.gox \
crypto/cast5.gox \
crypto/cipher.gox \
+ crypto/dsa.gox \
crypto/elliptic.gox \
crypto/hmac.gox \
crypto/md4.gox \
crypto/md5.gox \
crypto/ocsp.gox \
+ crypto/openpgp.gox \
crypto/rand.gox \
crypto/rc4.gox \
crypto/ripemd160.gox \
toolexeclibgocryptoopenpgp_DATA = \
crypto/openpgp/armor.gox \
crypto/openpgp/error.gox \
+ crypto/openpgp/packet.gox \
crypto/openpgp/s2k.gox
toolexeclibgodebugdir = $(toolexeclibgodir)/debug
runtime/debug.gox \
runtime/pprof.gox
+toolexeclibgosyncdir = $(toolexeclibgodir)/sync
+
+toolexeclibgosync_DATA = \
+ sync/atomic.gox
+
toolexeclibgotestingdir = $(toolexeclibgodir)/testing
toolexeclibgotesting_DATA = \
runtime/mfixalloc.c \
runtime/mgc0.c \
runtime/mheap.c \
- runtime/mheapmap32.c \
- runtime/mheapmap64.c \
runtime/msize.c \
runtime/proc.c \
runtime/thread.c \
go/cmath/sqrt.go \
go/cmath/tan.go
+go_crypto_files = \
+ go/crypto/crypto.go
+
go_ebnf_files = \
go/ebnf/ebnf.go \
go/ebnf/parser.go
go/http/client.go \
go/http/dump.go \
go/http/fs.go \
+ go/http/header.go \
go/http/lex.go \
go/http/persist.go \
go/http/request.go \
go/http/server.go \
go/http/status.go \
go/http/transfer.go \
+ go/http/transport.go \
go/http/url.go
go_image_files = \
go/os/env_unix.go \
go/os/error.go \
go/os/exec.go \
+ go/os/exec_unix.go \
go/os/file.go \
go/os/file_unix.go \
go/os/getwd.go \
go/runtime/debug.go \
go/runtime/error.go \
go/runtime/extern.go \
- go/runtime/malloc_defs.go \
- go/runtime/runtime_defs.go \
go/runtime/sig.go \
go/runtime/softfloat64.go \
go/runtime/type.go \
go/strings/strings.go
go_sync_files = \
+ go/sync/cond.go \
go/sync/mutex.go \
go/sync/once.go \
- go/sync/rwmutex.go
-go_sync_c_files = \
- go/sync/cas.c
+ go/sync/rwmutex.go \
+ go/sync/waitgroup.go
if LIBGO_IS_SOLARIS
go_syslog_file = go/syslog/syslog_solaris.go
go/archive/zip/reader.go \
go/archive/zip/struct.go
+go_compress_bzip2_files = \
+ go/compress/bzip2/bit_reader.go \
+ go/compress/bzip2/bzip2.go \
+ go/compress/bzip2/huffman.go \
+ go/compress/bzip2/move_to_front.go
+
go_compress_flate_files = \
go/compress/flate/deflate.go \
go/compress/flate/huffman_bit_writer.go \
go/compress/gzip/gzip.go \
go/compress/gzip/gunzip.go
+go_compress_lzw_files = \
+ go/compress/lzw/reader.go \
+ go/compress/lzw/writer.go
+
go_compress_zlib_files = \
go/compress/zlib/reader.go \
go/compress/zlib/writer.go
go/crypto/cipher/io.go \
go/crypto/cipher/ocfb.go \
go/crypto/cipher/ofb.go
+go_crypto_dsa_files = \
+ go/crypto/dsa/dsa.go
go_crypto_elliptic_files = \
go/crypto/elliptic/elliptic.go
go_crypto_hmac_files = \
go/crypto/md5/md5block.go
go_crypto_ocsp_files = \
go/crypto/ocsp/ocsp.go
+go_crypto_openpgp_files = \
+ go/crypto/openpgp/canonical_text.go \
+ go/crypto/openpgp/keys.go \
+ go/crypto/openpgp/read.go \
+ go/crypto/openpgp/write.go
go_crypto_rand_files = \
go/crypto/rand/rand.go \
go/crypto/rand/rand_unix.go
go/crypto/openpgp/armor/encode.go
go_crypto_openpgp_error_files = \
go/crypto/openpgp/error/error.go
+go_crypto_openpgp_packet_files = \
+ go/crypto/openpgp/packet/compressed.go \
+ go/crypto/openpgp/packet/encrypted_key.go \
+ go/crypto/openpgp/packet/literal.go \
+ go/crypto/openpgp/packet/one_pass_signature.go \
+ go/crypto/openpgp/packet/packet.go \
+ go/crypto/openpgp/packet/private_key.go \
+ go/crypto/openpgp/packet/public_key.go \
+ go/crypto/openpgp/packet/reader.go \
+ go/crypto/openpgp/packet/signature.go \
+ go/crypto/openpgp/packet/symmetric_key_encrypted.go \
+ go/crypto/openpgp/packet/symmetrically_encrypted.go \
+ go/crypto/openpgp/packet/userid.go
go_crypto_openpgp_s2k_files = \
go/crypto/openpgp/s2k/s2k.go
go/net/dict/dict.go
go_net_textproto_files = \
+ go/net/textproto/header.go \
go/net/textproto/pipeline.go \
go/net/textproto/reader.go \
go/net/textproto/textproto.go \
go_runtime_pprof_files = \
go/runtime/pprof/pprof.go
+go_sync_atomic_files = \
+ go/sync/atomic/doc.go
+go_sync_atomic_c_files = \
+ go/sync/atomic/atomic.c
+
go_testing_iotest_files = \
go/testing/iotest/logger.go \
go/testing/iotest/reader.go \
bytes/bytes.lo \
bytes/index.lo \
cmath/cmath.lo \
+ crypto/crypto.lo \
ebnf/ebnf.lo \
exec/exec.lo \
expvar/expvar.lo \
sort/sort.lo \
strconv/strconv.lo \
strings/strings.lo \
- sync/mutex.lo \
- sync/cas.lo \
+ sync/sync.lo \
syslog/syslog.lo \
syslog/syslog_c.lo \
tabwriter/tabwriter.lo \
xml/xml.lo \
archive/tar.lo \
archive/zip.lo \
+ compress/bzip2.lo \
compress/flate.lo \
compress/gzip.lo \
+ compress/lzw.lo \
compress/zlib.lo \
container/heap.lo \
container/list.lo \
crypto/blowfish.lo \
crypto/cast5.lo \
crypto/cipher.lo \
+ crypto/dsa.lo \
crypto/elliptic.lo \
crypto/hmac.lo \
crypto/md4.lo \
crypto/md5.lo \
crypto/ocsp.lo \
+ crypto/openpgp.lo \
crypto/rand.lo \
crypto/rc4.lo \
crypto/ripemd160.lo \
crypto/xtea.lo \
crypto/openpgp/armor.lo \
crypto/openpgp/error.lo \
+ crypto/openpgp/packet.lo \
crypto/openpgp/s2k.lo \
debug/dwarf.lo \
debug/elf.lo \
rpc/jsonrpc.lo \
runtime/debug.lo \
runtime/pprof.lo \
+ sync/atomic.lo \
+ sync/atomic_c.lo \
syscalls/syscall.lo \
syscalls/errno.lo \
testing/testing.lo \
$(CHECK)
.PHONY: cmath/check
+crypto/crypto.lo: $(go_crypto_files) hash.gox
+ $(BUILDPACKAGE)
+crypto/check: $(CHECK_DEPS)
+ $(CHECK)
+.PHONY: crypto/check
+
ebnf/ebnf.lo: $(go_ebnf_files) container/vector.gox go/scanner.gox \
go/token.gox os.gox strconv.gox unicode.gox utf8.gox
$(BUILDPACKAGE)
$(CHECK)
.PHONY: ebnf/check
-exec/exec.lo: $(go_exec_files) os.gox strings.gox
+exec/exec.lo: $(go_exec_files) os.gox strconv.gox strings.gox
$(BUILDPACKAGE)
exec/check: $(CHECK_DEPS)
$(CHECK)
$(CHECK)
.PHONY: flag/check
-fmt/fmt.lo: $(go_fmt_files) bytes.gox io.gox os.gox reflect.gox strconv.gox \
- strings.gox unicode.gox utf8.gox
+fmt/fmt.lo: $(go_fmt_files) bytes.gox io.gox math.gox os.gox reflect.gox \
+ strconv.gox strings.gox unicode.gox utf8.gox
$(BUILDPACKAGE)
fmt/check: $(CHECK_DEPS)
$(CHECK)
$(CHECK)
.PHONY: html/check
-http/http.lo: $(go_http_files) bufio.gox bytes.gox container/list.gox \
- container/vector.gox crypto/rand.gox crypto/tls.gox \
- encoding/base64.gox fmt.gox io.gox io/ioutil.gox log.gox \
- mime.gox mime/multipart.gox net.gox os.gox path.gox sort.gox \
+http/http.lo: $(go_http_files) bufio.gox bytes.gox container/vector.gox \
+ crypto/rand.gox crypto/tls.gox encoding/base64.gox fmt.gox \
+ io.gox io/ioutil.gox log.gox mime.gox mime/multipart.gox \
+ net.gox net/textproto.gox os.gox path.gox sort.gox \
strconv.gox strings.gox sync.gox time.gox utf8.gox
$(BUILDPACKAGE)
http/check: $(CHECK_DEPS)
$(CHECK)
.PHONY: io/check
-json/json.lo: $(go_json_files) bytes.gox container/vector.gox fmt.gox io.gox \
- math.gox os.gox reflect.gox runtime.gox strconv.gox \
- strings.gox unicode.gox utf16.gox utf8.gox
+json/json.lo: $(go_json_files) bytes.gox container/vector.gox \
+ encoding/base64.gox fmt.gox io.gox math.gox os.gox \
+ reflect.gox runtime.gox strconv.gox strings.gox unicode.gox \
+ utf16.gox utf8.gox
$(BUILDPACKAGE)
json/check: $(CHECK_DEPS)
$(CHECK)
$(CHECK)
.PHONY: net/check
-netchan/netchan.lo: $(go_netchan_files) gob.gox log.gox net.gox os.gox \
+netchan/netchan.lo: $(go_netchan_files) gob.gox io.gox log.gox net.gox os.gox \
reflect.gox strconv.gox sync.gox time.gox
$(BUILDPACKAGE)
netchan/check: $(CHECK_DEPS)
$(CHECK)
.PHONY: netchan/check
-os/os.lo: $(go_os_files) sync.gox syscall.gox
+os/os.lo: $(go_os_files) runtime.gox sync.gox syscall.gox
$(BUILDPACKAGE)
os/check: $(CHECK_DEPS)
$(CHECK)
$(CHECK)
.PHONY: strings/check
-sync/mutex.lo: $(go_sync_files) runtime.gox
+sync/sync.lo: $(go_sync_files) runtime.gox sync/atomic.gox
$(BUILDPACKAGE)
-sync/cas.lo: $(go_sync_c_files) sync/mutex.lo
- $(LTCOMPILE) -c -o sync/cas.lo $(srcdir)/go/sync/cas.c
sync/check: $(CHECK_DEPS)
$(CHECK)
.PHONY: sync/check
$(CHECK)
.PHONY: archive/zip/check
+compress/bzip2.lo: $(go_compress_bzip2_files) bufio.gox io.gox os.gox sort.gox
+ $(BUILDPACKAGE)
+compress/bzip2/check: $(CHECK_DEPS)
+ @$(MKDIR_P) compress/bzip2
+ $(CHECK)
+.PHONY: compress/bzip2/check
+
compress/flate.lo: $(go_compress_flate_files) bufio.gox io.gox math.gox \
os.gox sort.gox strconv.gox
$(BUILDPACKAGE)
$(CHECK)
.PHONY: compress/gzip/check
+compress/lzw.lo: $(go_compress_lzw_files) bufio.gox fmt.gox io.gox os.gox
+ $(BUILDPACKAGE)
+compress/lzw/check: $(CHECK_DEPS)
+ @$(MKDIR_P) compress/lzw
+ $(CHECK)
+.PHONY: compress/lzw/check
+
compress/zlib.lo: $(go_compress_zlib_files) bufio.gox compress/flate.gox \
hash.gox hash/adler32.gox io.gox os.gox
$(BUILDPACKAGE)
$(CHECK)
.PHONY: crypto/cipher/check
+crypto/dsa.lo: $(go_crypto_dsa_files) big.gox io.gox os.gox
+ $(BUILDPACKAGE)
+crypto/dsa/check: $(CHECK_DEPS)
+ @$(MKDIR_P) crypto/dsa
+ $(CHECK)
+.PHONY: crypto/dsa/check
+
crypto/elliptic.lo: $(go_crypto_elliptic_files) big.gox io.gox os.gox sync.gox
$(BUILDPACKAGE)
crypto/elliptic/check: $(CHECK_DEPS)
$(CHECK)
.PHONY: crypto/hmac/check
-crypto/md4.lo: $(go_crypto_md4_files) hash.gox os.gox
+crypto/md4.lo: $(go_crypto_md4_files) crypto.gox hash.gox os.gox
$(BUILDPACKAGE)
crypto/md4/check: $(CHECK_DEPS)
@$(MKDIR_P) crypto/md4
$(CHECK)
.PHONY: crypto/md4/check
-crypto/md5.lo: $(go_crypto_md5_files) hash.gox os.gox
+crypto/md5.lo: $(go_crypto_md5_files) crypto.gox hash.gox os.gox
$(BUILDPACKAGE)
crypto/md5/check: $(CHECK_DEPS)
@$(MKDIR_P) crypto/md5
$(CHECK)
.PHONY: crypto/md5/check
-crypto/ocsp.lo: $(go_crypto_ocsp_files) asn1.gox crypto/rsa.gox \
+crypto/ocsp.lo: $(go_crypto_ocsp_files) asn1.gox crypto.gox crypto/rsa.gox \
crypto/sha1.gox crypto/x509.gox os.gox time.gox
$(BUILDPACKAGE)
crypto/ocsp/check: $(CHECK_DEPS)
$(CHECK)
.PHONY: crypto/ocsp/check
-crypto/rand.lo: $(go_crypto_rand_files) crypto/aes.gox io.gox os.gox sync.gox \
- time.gox
+crypto/openpgp.lo: $(go_crypto_openpgp_files) crypto.gox \
+ crypto/openpgp/armor.gox crypto/openpgp/error.gox \
+ crypto/openpgp/packet.gox crypto/rsa.gox crypto/sha256.gox \
+ hash.gox io.gox os.gox strconv.gox time.gox
+ $(BUILDPACKAGE)
+crypto/openpgp/check: $(CHECK_DEPS)
+ @$(MKDIR_P) crypto/openpgp
+ $(CHECK)
+.PHONY: crypto/openpgp/check
+
+crypto/rand.lo: $(go_crypto_rand_files) bufio.gox crypto/aes.gox io.gox \
+ os.gox sync.gox time.gox
$(BUILDPACKAGE)
crypto/rand/check: $(CHECK_DEPS)
@$(MKDIR_P) crypto/rand
$(CHECK)
.PHONY: crypto/rc4/check
-crypto/ripemd160.lo: $(go_crypto_ripemd160_files) hash.gox os.gox
+crypto/ripemd160.lo: $(go_crypto_ripemd160_files) crypto.gox hash.gox os.gox
$(BUILDPACKAGE)
crypto/ripemd160/check: $(CHECK_DEPS)
@$(MKDIR_P) crypto/ripemd160
$(CHECK)
.PHONY: crypto/ripemd160/check
-crypto/rsa.lo: $(go_crypto_rsa_files) big.gox crypto/sha1.gox \
+crypto/rsa.lo: $(go_crypto_rsa_files) big.gox crypto.gox crypto/sha1.gox \
crypto/subtle.gox encoding/hex.gox hash.gox io.gox os.gox
$(BUILDPACKAGE)
crypto/rsa/check: $(CHECK_DEPS)
$(CHECK)
.PHONY: crypto/rsa/check
-crypto/sha1.lo: $(go_crypto_sha1_files) hash.gox os.gox
+crypto/sha1.lo: $(go_crypto_sha1_files) crypto.gox hash.gox os.gox
$(BUILDPACKAGE)
crypto/sha1/check: $(CHECK_DEPS)
@$(MKDIR_P) crypto/sha1
$(CHECK)
.PHONY: crypto/sha1/check
-crypto/sha256.lo: $(go_crypto_sha256_files) hash.gox os.gox
+crypto/sha256.lo: $(go_crypto_sha256_files) crypto.gox hash.gox os.gox
$(BUILDPACKAGE)
crypto/sha256/check: $(CHECK_DEPS)
@$(MKDIR_P) crypto/sha256
$(CHECK)
.PHONY: crypto/sha256/check
-crypto/sha512.lo: $(go_crypto_sha512_files) hash.gox os.gox
+crypto/sha512.lo: $(go_crypto_sha512_files) crypto.gox hash.gox os.gox
$(BUILDPACKAGE)
crypto/sha512/check: $(CHECK_DEPS)
@$(MKDIR_P) crypto/sha512
.PHONY: crypto/subtle/check
crypto/tls.lo: $(go_crypto_tls_files) big.gox bufio.gox bytes.gox \
- container/list.gox crypto/aes.gox crypto/cipher.gox \
+ container/list.gox crypto.gox crypto/aes.gox crypto/cipher.gox \
crypto/elliptic.gox crypto/hmac.gox crypto/md5.gox \
crypto/rc4.gox crypto/rand.gox crypto/rsa.gox crypto/sha1.gox \
crypto/subtle.gox crypto/rsa.gox crypto/sha1.gox \
.PHONY: crypto/twofish/check
crypto/x509.lo: $(go_crypto_x509_files) asn1.gox big.gox container/vector.gox \
- crypto/rsa.gox crypto/sha1.gox hash.gox os.gox strings.gox \
- time.gox
+ crypto.gox crypto/rsa.gox crypto/sha1.gox hash.gox os.gox \
+ strings.gox time.gox
$(BUILDPACKAGE)
crypto/x509/check: $(CHECK_DEPS)
@$(MKDIR_P) crypto/x509
$(CHECK)
.PHONY: crypto/openpgp/armor/check
-crypto/openpgp/error.lo: $(go_crypto_openpgp_error_files)
+crypto/openpgp/error.lo: $(go_crypto_openpgp_error_files) strconv.gox
$(BUILDPACKAGE)
crypto/openpgp/error/check: $(CHECK_DEPS)
@$(MKDIR_P) crypto/openpgp/error
$(CHECK)
.PHONY: crypto/openpgp/error/check
-crypto/openpgp/s2k.lo: $(go_crypto_openpgp_s2k_files) crypto/md5.gox \
- crypto/openpgp/error.gox crypto/ripemd160.gox crypto/sha1.gox \
- crypto/sha256.gox crypto/sha512.gox hash.gox io.gox os.gox
+crypto/openpgp/packet.lo: $(go_crypto_openpgp_packet_files) big.gox bytes.gox \
+ compress/flate.gox compress/zlib.gox crypto.gox \
+ crypto/aes.gox crypto/cast5.gox crypto/cipher.gox \
+ crypto/openpgp/error.gox crypto/openpgp/s2k.gox \
+ crypto/rand.gox crypto/rsa.gox crypto/sha1.gox \
+ crypto/subtle.gox encoding/binary.gox hash.gox io.gox \
+ io/ioutil.gox os.gox strconv.gox strings.gox
+ $(BUILDPACKAGE)
+crypto/openpgp/packet/check: $(CHECK_DEPS)
+ @$(MKDIR_P) crypto/openpgp/packet
+ $(CHECK)
+.PHONY: crypto/openpgp/packet/check
+
+crypto/openpgp/s2k.lo: $(go_crypto_openpgp_s2k_files) crypto.gox \
+ crypto/md5.gox crypto/openpgp/error.gox crypto/ripemd160.gox \
+ crypto/sha1.gox crypto/sha256.gox crypto/sha512.gox hash.gox \
+ io.gox os.gox
$(BUILDPACKAGE)
crypto/openpgp/s2k/check: $(CHECK_DEPS)
@$(MKDIR_P) crypto/openpgp/s2k
$(CHECK)
.PHONY: runtime/pprof/check
+sync/atomic.lo: $(go_sync_atomic_files)
+ $(BUILDPACKAGE)
+sync/atomic_c.lo: $(go_sync_atomic_c_files) sync/atomic.lo
+ $(LTCOMPILE) -c -o $@ $(srcdir)/go/sync/atomic/atomic.c
+sync/atomic/check: $(CHECK_DEPS)
+ @$(MKDIR_P) sync/atomic
+ $(CHECK)
+.PHONY: sync/atomic/check
+
testing/iotest.lo: $(go_testing_iotest_files) io.gox log.gox os.gox
$(BUILDPACKAGE)
testing/iotest/check: $(CHECK_DEPS)
$(BUILDGOX)
cmath.gox: cmath/cmath.lo
$(BUILDGOX)
+crypto.gox: crypto/crypto.lo
+ $(BUILDGOX)
ebnf.gox: ebnf/ebnf.lo
$(BUILDGOX)
exec.gox: exec/exec.lo
$(BUILDGOX)
strings.gox: strings/strings.lo
$(BUILDGOX)
-sync.gox: sync/mutex.lo
+sync.gox: sync/sync.lo
$(BUILDGOX)
syslog.gox: syslog/syslog.lo
$(BUILDGOX)
archive/zip.gox: archive/zip.lo
$(BUILDGOX)
+compress/bzip2.gox: compress/bzip2.lo
+ $(BUILDGOX)
compress/flate.gox: compress/flate.lo
$(BUILDGOX)
compress/gzip.gox: compress/gzip.lo
$(BUILDGOX)
+compress/lzw.gox: compress/lzw.lo
+ $(BUILDGOX)
compress/zlib.gox: compress/zlib.lo
$(BUILDGOX)
$(BUILDGOX)
crypto/cipher.gox: crypto/cipher.lo
$(BUILDGOX)
+crypto/dsa.gox: crypto/dsa.lo
+ $(BUILDGOX)
crypto/elliptic.gox: crypto/elliptic.lo
$(BUILDGOX)
crypto/hmac.gox: crypto/hmac.lo
$(BUILDGOX)
crypto/ocsp.gox: crypto/ocsp.lo
$(BUILDGOX)
+crypto/openpgp.gox: crypto/openpgp.lo
+ $(BUILDGOX)
crypto/rand.gox: crypto/rand.lo
$(BUILDGOX)
crypto/rc4.gox: crypto/rc4.lo
$(BUILDGOX)
crypto/openpgp/error.gox: crypto/openpgp/error.lo
$(BUILDGOX)
+crypto/openpgp/packet.gox: crypto/openpgp/packet.lo
+ $(BUILDGOX)
crypto/openpgp/s2k.gox: crypto/openpgp/s2k.lo
$(BUILDGOX)
runtime/pprof.gox: runtime/pprof.lo
$(BUILDGOX)
+sync/atomic.gox: sync/atomic.lo
+ $(BUILDGOX)
+
testing/iotest.gox: testing/iotest.lo
$(BUILDGOX)
testing/quick.gox: testing/quick.lo
xml/check \
archive/tar/check \
archive/zip/check \
+ compress/bzip2/check \
compress/flate/check \
compress/gzip/check \
+ compress/lzw/check \
compress/zlib/check \
container/heap/check \
container/list/check \
crypto/blowfish/check \
crypto/cast5/check \
crypto/cipher/check \
+ crypto/dsa/check \
crypto/elliptic/check \
crypto/hmac/check \
crypto/md4/check \
crypto/md5/check \
crypto/ocsp/check \
+ crypto/openpgp/check \
crypto/rand/check \
crypto/rc4/check \
crypto/ripemd160/check \
crypto/x509/check \
crypto/xtea/check \
crypto/openpgp/armor/check \
+ crypto/openpgp/packet/check \
crypto/openpgp/s2k/check \
debug/dwarf/check \
debug/elf/check \
$(os_inotify_check) \
os/signal/check \
rpc/jsonrpc/check \
+ sync/atomic/check \
testing/quick/check \
testing/script/check
"$(DESTDIR)$(toolexeclibgoosdir)" \
"$(DESTDIR)$(toolexeclibgorpcdir)" \
"$(DESTDIR)$(toolexeclibgoruntimedir)" \
+ "$(DESTDIR)$(toolexeclibgosyncdir)" \
"$(DESTDIR)$(toolexeclibgotestingdir)"
LIBRARIES = $(toolexeclib_LIBRARIES)
ARFLAGS = cru
LTLIBRARIES = $(toolexeclib_LTLIBRARIES)
am__DEPENDENCIES_1 =
am__DEPENDENCIES_2 = asn1/asn1.lo big/big.lo bufio/bufio.lo \
- bytes/bytes.lo bytes/index.lo cmath/cmath.lo ebnf/ebnf.lo \
- exec/exec.lo expvar/expvar.lo flag/flag.lo fmt/fmt.lo \
- gob/gob.lo hash/hash.lo html/html.lo http/http.lo \
+ bytes/bytes.lo bytes/index.lo cmath/cmath.lo crypto/crypto.lo \
+ ebnf/ebnf.lo exec/exec.lo expvar/expvar.lo flag/flag.lo \
+ fmt/fmt.lo gob/gob.lo hash/hash.lo html/html.lo http/http.lo \
image/image.lo io/io.lo json/json.lo log/log.lo math/math.lo \
mime/mime.lo net/net.lo netchan/netchan.lo os/os.lo \
patch/patch.lo path/path.lo rand/rand.lo reflect/reflect.lo \
regexp/regexp.lo rpc/rpc.lo runtime/runtime.lo \
scanner/scanner.lo smtp/smtp.lo sort/sort.lo \
- strconv/strconv.lo strings/strings.lo sync/mutex.lo \
- sync/cas.lo syslog/syslog.lo syslog/syslog_c.lo \
- tabwriter/tabwriter.lo template/template.lo time/time.lo \
- try/try.lo unicode/unicode.lo utf16/utf16.lo utf8/utf8.lo \
+ strconv/strconv.lo strings/strings.lo sync/sync.lo \
+ syslog/syslog.lo syslog/syslog_c.lo tabwriter/tabwriter.lo \
+ template/template.lo time/time.lo try/try.lo \
+ unicode/unicode.lo utf16/utf16.lo utf8/utf8.lo \
websocket/websocket.lo xml/xml.lo archive/tar.lo \
- archive/zip.lo compress/flate.lo compress/gzip.lo \
- compress/zlib.lo container/heap.lo container/list.lo \
- container/ring.lo container/vector.lo crypto/aes.lo \
- crypto/block.lo crypto/blowfish.lo crypto/cast5.lo \
- crypto/cipher.lo crypto/elliptic.lo crypto/hmac.lo \
- crypto/md4.lo crypto/md5.lo crypto/ocsp.lo crypto/rand.lo \
+ archive/zip.lo compress/bzip2.lo compress/flate.lo \
+ compress/gzip.lo compress/lzw.lo compress/zlib.lo \
+ container/heap.lo container/list.lo container/ring.lo \
+ container/vector.lo crypto/aes.lo crypto/block.lo \
+ crypto/blowfish.lo crypto/cast5.lo crypto/cipher.lo \
+ crypto/dsa.lo crypto/elliptic.lo crypto/hmac.lo crypto/md4.lo \
+ crypto/md5.lo crypto/ocsp.lo crypto/openpgp.lo crypto/rand.lo \
crypto/rc4.lo crypto/ripemd160.lo crypto/rsa.lo crypto/sha1.lo \
crypto/sha256.lo crypto/sha512.lo crypto/subtle.lo \
crypto/tls.lo crypto/twofish.lo crypto/x509.lo crypto/xtea.lo \
crypto/openpgp/armor.lo crypto/openpgp/error.lo \
- crypto/openpgp/s2k.lo debug/dwarf.lo debug/elf.lo \
- debug/gosym.lo debug/macho.lo debug/pe.lo debug/proc.lo \
- encoding/ascii85.lo encoding/base32.lo encoding/base64.lo \
- encoding/binary.lo encoding/git85.lo encoding/hex.lo \
- encoding/line.lo encoding/pem.lo exp/datafmt.lo exp/draw.lo \
- exp/eval.lo go/ast.lo go/doc.lo go/parser.lo go/printer.lo \
- go/scanner.lo go/token.lo go/typechecker.lo hash/adler32.lo \
- hash/crc32.lo hash/crc64.lo http/pprof.lo image/jpeg.lo \
- image/png.lo index/suffixarray.lo io/ioutil.lo \
- mime/multipart.lo net/dict.lo net/textproto.lo \
+ crypto/openpgp/packet.lo crypto/openpgp/s2k.lo debug/dwarf.lo \
+ debug/elf.lo debug/gosym.lo debug/macho.lo debug/pe.lo \
+ debug/proc.lo encoding/ascii85.lo encoding/base32.lo \
+ encoding/base64.lo encoding/binary.lo encoding/git85.lo \
+ encoding/hex.lo encoding/line.lo encoding/pem.lo \
+ exp/datafmt.lo exp/draw.lo exp/eval.lo go/ast.lo go/doc.lo \
+ go/parser.lo go/printer.lo go/scanner.lo go/token.lo \
+ go/typechecker.lo hash/adler32.lo hash/crc32.lo hash/crc64.lo \
+ http/pprof.lo image/jpeg.lo image/png.lo index/suffixarray.lo \
+ io/ioutil.lo mime/multipart.lo net/dict.lo net/textproto.lo \
$(am__DEPENDENCIES_1) os/signal.lo rpc/jsonrpc.lo \
- runtime/debug.lo runtime/pprof.lo syscalls/syscall.lo \
- syscalls/errno.lo testing/testing.lo testing/iotest.lo \
- testing/quick.lo testing/script.lo
+ runtime/debug.lo runtime/pprof.lo sync/atomic.lo \
+ sync/atomic_c.lo syscalls/syscall.lo syscalls/errno.lo \
+ testing/testing.lo testing/iotest.lo testing/quick.lo \
+ testing/script.lo
libgo_la_DEPENDENCIES = $(am__DEPENDENCIES_2) $(am__DEPENDENCIES_1) \
$(am__DEPENDENCIES_1) $(am__DEPENDENCIES_1) \
$(am__DEPENDENCIES_1)
runtime/mcache.c runtime/mcentral.c \
runtime/mem_posix_memalign.c runtime/mem.c runtime/mfinal.c \
runtime/mfixalloc.c runtime/mgc0.c runtime/mheap.c \
- runtime/mheapmap32.c runtime/mheapmap64.c runtime/msize.c \
- runtime/proc.c runtime/thread.c \
+ runtime/msize.c runtime/proc.c runtime/thread.c \
runtime/rtems-task-variable-add.c chan.c iface.c malloc.c \
map.c mprof.c reflect.c sigqueue.c string.c
@HAVE_SYS_MMAN_H_FALSE@am__objects_1 = mem_posix_memalign.lo
go-unreflect.lo go-unsafe-new.lo go-unsafe-newarray.lo \
go-unsafe-pointer.lo go-unwind.lo mcache.lo mcentral.lo \
$(am__objects_1) mfinal.lo mfixalloc.lo mgc0.lo mheap.lo \
- mheapmap32.lo mheapmap64.lo msize.lo proc.lo thread.lo \
- $(am__objects_2) chan.lo iface.lo malloc.lo map.lo mprof.lo \
- reflect.lo sigqueue.lo string.lo
+ msize.lo proc.lo thread.lo $(am__objects_2) chan.lo iface.lo \
+ malloc.lo map.lo mprof.lo reflect.lo sigqueue.lo string.lo
am_libgo_la_OBJECTS = $(am__objects_3)
libgo_la_OBJECTS = $(am_libgo_la_OBJECTS)
DEFAULT_INCLUDES = -I.@am__isrc@
$(toolexeclibgoio_DATA) $(toolexeclibgomime_DATA) \
$(toolexeclibgonet_DATA) $(toolexeclibgoos_DATA) \
$(toolexeclibgorpc_DATA) $(toolexeclibgoruntime_DATA) \
- $(toolexeclibgotesting_DATA)
+ $(toolexeclibgosync_DATA) $(toolexeclibgotesting_DATA)
RECURSIVE_CLEAN_TARGETS = mostlyclean-recursive clean-recursive \
distclean-recursive maintainer-clean-recursive
AM_RECURSIVE_TARGETS = $(RECURSIVE_TARGETS:-recursive=) \
bufio.gox \
bytes.gox \
cmath.gox \
+ crypto.gox \
ebnf.gox \
exec.gox \
expvar.gox \
toolexeclibgocompressdir = $(toolexeclibgodir)/compress
toolexeclibgocompress_DATA = \
+ compress/bzip2.gox \
compress/flate.gox \
compress/gzip.gox \
+ compress/lzw.gox \
compress/zlib.gox
toolexeclibgocontainerdir = $(toolexeclibgodir)/container
crypto/blowfish.gox \
crypto/cast5.gox \
crypto/cipher.gox \
+ crypto/dsa.gox \
crypto/elliptic.gox \
crypto/hmac.gox \
crypto/md4.gox \
crypto/md5.gox \
crypto/ocsp.gox \
+ crypto/openpgp.gox \
crypto/rand.gox \
crypto/rc4.gox \
crypto/ripemd160.gox \
toolexeclibgocryptoopenpgp_DATA = \
crypto/openpgp/armor.gox \
crypto/openpgp/error.gox \
+ crypto/openpgp/packet.gox \
crypto/openpgp/s2k.gox
toolexeclibgodebugdir = $(toolexeclibgodir)/debug
runtime/debug.gox \
runtime/pprof.gox
+toolexeclibgosyncdir = $(toolexeclibgodir)/sync
+toolexeclibgosync_DATA = \
+ sync/atomic.gox
+
toolexeclibgotestingdir = $(toolexeclibgodir)/testing
toolexeclibgotesting_DATA = \
testing/iotest.gox \
runtime/mfixalloc.c \
runtime/mgc0.c \
runtime/mheap.c \
- runtime/mheapmap32.c \
- runtime/mheapmap64.c \
runtime/msize.c \
runtime/proc.c \
runtime/thread.c \
go/cmath/sqrt.go \
go/cmath/tan.go
+go_crypto_files = \
+ go/crypto/crypto.go
+
go_ebnf_files = \
go/ebnf/ebnf.go \
go/ebnf/parser.go
go/http/client.go \
go/http/dump.go \
go/http/fs.go \
+ go/http/header.go \
go/http/lex.go \
go/http/persist.go \
go/http/request.go \
go/http/server.go \
go/http/status.go \
go/http/transfer.go \
+ go/http/transport.go \
go/http/url.go
go_image_files = \
go/os/env_unix.go \
go/os/error.go \
go/os/exec.go \
+ go/os/exec_unix.go \
go/os/file.go \
go/os/file_unix.go \
go/os/getwd.go \
go/runtime/debug.go \
go/runtime/error.go \
go/runtime/extern.go \
- go/runtime/malloc_defs.go \
- go/runtime/runtime_defs.go \
go/runtime/sig.go \
go/runtime/softfloat64.go \
go/runtime/type.go \
go/strings/strings.go
go_sync_files = \
+ go/sync/cond.go \
go/sync/mutex.go \
go/sync/once.go \
- go/sync/rwmutex.go
-
-go_sync_c_files = \
- go/sync/cas.c
+ go/sync/rwmutex.go \
+ go/sync/waitgroup.go
@LIBGO_IS_SOLARIS_FALSE@go_syslog_file = go/syslog/syslog_unix.go
@LIBGO_IS_SOLARIS_TRUE@go_syslog_file = go/syslog/syslog_solaris.go
go/archive/zip/reader.go \
go/archive/zip/struct.go
+go_compress_bzip2_files = \
+ go/compress/bzip2/bit_reader.go \
+ go/compress/bzip2/bzip2.go \
+ go/compress/bzip2/huffman.go \
+ go/compress/bzip2/move_to_front.go
+
go_compress_flate_files = \
go/compress/flate/deflate.go \
go/compress/flate/huffman_bit_writer.go \
go/compress/gzip/gzip.go \
go/compress/gzip/gunzip.go
+go_compress_lzw_files = \
+ go/compress/lzw/reader.go \
+ go/compress/lzw/writer.go
+
go_compress_zlib_files = \
go/compress/zlib/reader.go \
go/compress/zlib/writer.go
go/crypto/cipher/ocfb.go \
go/crypto/cipher/ofb.go
+go_crypto_dsa_files = \
+ go/crypto/dsa/dsa.go
+
go_crypto_elliptic_files = \
go/crypto/elliptic/elliptic.go
go_crypto_ocsp_files = \
go/crypto/ocsp/ocsp.go
+go_crypto_openpgp_files = \
+ go/crypto/openpgp/canonical_text.go \
+ go/crypto/openpgp/keys.go \
+ go/crypto/openpgp/read.go \
+ go/crypto/openpgp/write.go
+
go_crypto_rand_files = \
go/crypto/rand/rand.go \
go/crypto/rand/rand_unix.go
go_crypto_openpgp_error_files = \
go/crypto/openpgp/error/error.go
+go_crypto_openpgp_packet_files = \
+ go/crypto/openpgp/packet/compressed.go \
+ go/crypto/openpgp/packet/encrypted_key.go \
+ go/crypto/openpgp/packet/literal.go \
+ go/crypto/openpgp/packet/one_pass_signature.go \
+ go/crypto/openpgp/packet/packet.go \
+ go/crypto/openpgp/packet/private_key.go \
+ go/crypto/openpgp/packet/public_key.go \
+ go/crypto/openpgp/packet/reader.go \
+ go/crypto/openpgp/packet/signature.go \
+ go/crypto/openpgp/packet/symmetric_key_encrypted.go \
+ go/crypto/openpgp/packet/symmetrically_encrypted.go \
+ go/crypto/openpgp/packet/userid.go
+
go_crypto_openpgp_s2k_files = \
go/crypto/openpgp/s2k/s2k.go
go/net/dict/dict.go
go_net_textproto_files = \
+ go/net/textproto/header.go \
go/net/textproto/pipeline.go \
go/net/textproto/reader.go \
go/net/textproto/textproto.go \
go_runtime_pprof_files = \
go/runtime/pprof/pprof.go
+go_sync_atomic_files = \
+ go/sync/atomic/doc.go
+
+go_sync_atomic_c_files = \
+ go/sync/atomic/atomic.c
+
go_testing_iotest_files = \
go/testing/iotest/logger.go \
go/testing/iotest/reader.go \
bytes/bytes.lo \
bytes/index.lo \
cmath/cmath.lo \
+ crypto/crypto.lo \
ebnf/ebnf.lo \
exec/exec.lo \
expvar/expvar.lo \
sort/sort.lo \
strconv/strconv.lo \
strings/strings.lo \
- sync/mutex.lo \
- sync/cas.lo \
+ sync/sync.lo \
syslog/syslog.lo \
syslog/syslog_c.lo \
tabwriter/tabwriter.lo \
xml/xml.lo \
archive/tar.lo \
archive/zip.lo \
+ compress/bzip2.lo \
compress/flate.lo \
compress/gzip.lo \
+ compress/lzw.lo \
compress/zlib.lo \
container/heap.lo \
container/list.lo \
crypto/blowfish.lo \
crypto/cast5.lo \
crypto/cipher.lo \
+ crypto/dsa.lo \
crypto/elliptic.lo \
crypto/hmac.lo \
crypto/md4.lo \
crypto/md5.lo \
crypto/ocsp.lo \
+ crypto/openpgp.lo \
crypto/rand.lo \
crypto/rc4.lo \
crypto/ripemd160.lo \
crypto/xtea.lo \
crypto/openpgp/armor.lo \
crypto/openpgp/error.lo \
+ crypto/openpgp/packet.lo \
crypto/openpgp/s2k.lo \
debug/dwarf.lo \
debug/elf.lo \
rpc/jsonrpc.lo \
runtime/debug.lo \
runtime/pprof.lo \
+ sync/atomic.lo \
+ sync/atomic_c.lo \
syscalls/syscall.lo \
syscalls/errno.lo \
testing/testing.lo \
xml/check \
archive/tar/check \
archive/zip/check \
+ compress/bzip2/check \
compress/flate/check \
compress/gzip/check \
+ compress/lzw/check \
compress/zlib/check \
container/heap/check \
container/list/check \
crypto/blowfish/check \
crypto/cast5/check \
crypto/cipher/check \
+ crypto/dsa/check \
crypto/elliptic/check \
crypto/hmac/check \
crypto/md4/check \
crypto/md5/check \
crypto/ocsp/check \
+ crypto/openpgp/check \
crypto/rand/check \
crypto/rc4/check \
crypto/ripemd160/check \
crypto/x509/check \
crypto/xtea/check \
crypto/openpgp/armor/check \
+ crypto/openpgp/packet/check \
crypto/openpgp/s2k/check \
debug/dwarf/check \
debug/elf/check \
$(os_inotify_check) \
os/signal/check \
rpc/jsonrpc/check \
+ sync/atomic/check \
testing/quick/check \
testing/script/check
@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/mfixalloc.Plo@am__quote@
@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/mgc0.Plo@am__quote@
@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/mheap.Plo@am__quote@
-@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/mheapmap32.Plo@am__quote@
-@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/mheapmap64.Plo@am__quote@
@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/mprof.Plo@am__quote@
@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/msize.Plo@am__quote@
@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/proc.Plo@am__quote@
@AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
@am__fastdepCC_FALSE@ $(LIBTOOL) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o mheap.lo `test -f 'runtime/mheap.c' || echo '$(srcdir)/'`runtime/mheap.c
-mheapmap32.lo: runtime/mheapmap32.c
-@am__fastdepCC_TRUE@ $(LIBTOOL) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT mheapmap32.lo -MD -MP -MF $(DEPDIR)/mheapmap32.Tpo -c -o mheapmap32.lo `test -f 'runtime/mheapmap32.c' || echo '$(srcdir)/'`runtime/mheapmap32.c
-@am__fastdepCC_TRUE@ $(am__mv) $(DEPDIR)/mheapmap32.Tpo $(DEPDIR)/mheapmap32.Plo
-@AMDEP_TRUE@@am__fastdepCC_FALSE@ source='runtime/mheapmap32.c' object='mheapmap32.lo' libtool=yes @AMDEPBACKSLASH@
-@AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
-@am__fastdepCC_FALSE@ $(LIBTOOL) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o mheapmap32.lo `test -f 'runtime/mheapmap32.c' || echo '$(srcdir)/'`runtime/mheapmap32.c
-
-mheapmap64.lo: runtime/mheapmap64.c
-@am__fastdepCC_TRUE@ $(LIBTOOL) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT mheapmap64.lo -MD -MP -MF $(DEPDIR)/mheapmap64.Tpo -c -o mheapmap64.lo `test -f 'runtime/mheapmap64.c' || echo '$(srcdir)/'`runtime/mheapmap64.c
-@am__fastdepCC_TRUE@ $(am__mv) $(DEPDIR)/mheapmap64.Tpo $(DEPDIR)/mheapmap64.Plo
-@AMDEP_TRUE@@am__fastdepCC_FALSE@ source='runtime/mheapmap64.c' object='mheapmap64.lo' libtool=yes @AMDEPBACKSLASH@
-@AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
-@am__fastdepCC_FALSE@ $(LIBTOOL) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o mheapmap64.lo `test -f 'runtime/mheapmap64.c' || echo '$(srcdir)/'`runtime/mheapmap64.c
-
msize.lo: runtime/msize.c
@am__fastdepCC_TRUE@ $(LIBTOOL) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT msize.lo -MD -MP -MF $(DEPDIR)/msize.Tpo -c -o msize.lo `test -f 'runtime/msize.c' || echo '$(srcdir)/'`runtime/msize.c
@am__fastdepCC_TRUE@ $(am__mv) $(DEPDIR)/msize.Tpo $(DEPDIR)/msize.Plo
test -n "$$files" || exit 0; \
echo " ( cd '$(DESTDIR)$(toolexeclibgoruntimedir)' && rm -f" $$files ")"; \
cd "$(DESTDIR)$(toolexeclibgoruntimedir)" && rm -f $$files
+install-toolexeclibgosyncDATA: $(toolexeclibgosync_DATA)
+ @$(NORMAL_INSTALL)
+ test -z "$(toolexeclibgosyncdir)" || $(MKDIR_P) "$(DESTDIR)$(toolexeclibgosyncdir)"
+ @list='$(toolexeclibgosync_DATA)'; test -n "$(toolexeclibgosyncdir)" || list=; \
+ for p in $$list; do \
+ if test -f "$$p"; then d=; else d="$(srcdir)/"; fi; \
+ echo "$$d$$p"; \
+ done | $(am__base_list) | \
+ while read files; do \
+ echo " $(INSTALL_DATA) $$files '$(DESTDIR)$(toolexeclibgosyncdir)'"; \
+ $(INSTALL_DATA) $$files "$(DESTDIR)$(toolexeclibgosyncdir)" || exit $$?; \
+ done
+
+uninstall-toolexeclibgosyncDATA:
+ @$(NORMAL_UNINSTALL)
+ @list='$(toolexeclibgosync_DATA)'; test -n "$(toolexeclibgosyncdir)" || list=; \
+ files=`for p in $$list; do echo $$p; done | sed -e 's|^.*/||'`; \
+ test -n "$$files" || exit 0; \
+ echo " ( cd '$(DESTDIR)$(toolexeclibgosyncdir)' && rm -f" $$files ")"; \
+ cd "$(DESTDIR)$(toolexeclibgosyncdir)" && rm -f $$files
install-toolexeclibgotestingDATA: $(toolexeclibgotesting_DATA)
@$(NORMAL_INSTALL)
test -z "$(toolexeclibgotestingdir)" || $(MKDIR_P) "$(DESTDIR)$(toolexeclibgotestingdir)"
config.h
installdirs: installdirs-recursive
installdirs-am:
- for dir in "$(DESTDIR)$(toolexeclibdir)" "$(DESTDIR)$(toolexeclibdir)" "$(DESTDIR)$(toolexeclibgodir)" "$(DESTDIR)$(toolexeclibgoarchivedir)" "$(DESTDIR)$(toolexeclibgocompressdir)" "$(DESTDIR)$(toolexeclibgocontainerdir)" "$(DESTDIR)$(toolexeclibgocryptodir)" "$(DESTDIR)$(toolexeclibgocryptoopenpgpdir)" "$(DESTDIR)$(toolexeclibgodebugdir)" "$(DESTDIR)$(toolexeclibgoencodingdir)" "$(DESTDIR)$(toolexeclibgoexpdir)" "$(DESTDIR)$(toolexeclibgogodir)" "$(DESTDIR)$(toolexeclibgohashdir)" "$(DESTDIR)$(toolexeclibgohttpdir)" "$(DESTDIR)$(toolexeclibgoimagedir)" "$(DESTDIR)$(toolexeclibgoindexdir)" "$(DESTDIR)$(toolexeclibgoiodir)" "$(DESTDIR)$(toolexeclibgomimedir)" "$(DESTDIR)$(toolexeclibgonetdir)" "$(DESTDIR)$(toolexeclibgoosdir)" "$(DESTDIR)$(toolexeclibgorpcdir)" "$(DESTDIR)$(toolexeclibgoruntimedir)" "$(DESTDIR)$(toolexeclibgotestingdir)"; do \
+ for dir in "$(DESTDIR)$(toolexeclibdir)" "$(DESTDIR)$(toolexeclibdir)" "$(DESTDIR)$(toolexeclibgodir)" "$(DESTDIR)$(toolexeclibgoarchivedir)" "$(DESTDIR)$(toolexeclibgocompressdir)" "$(DESTDIR)$(toolexeclibgocontainerdir)" "$(DESTDIR)$(toolexeclibgocryptodir)" "$(DESTDIR)$(toolexeclibgocryptoopenpgpdir)" "$(DESTDIR)$(toolexeclibgodebugdir)" "$(DESTDIR)$(toolexeclibgoencodingdir)" "$(DESTDIR)$(toolexeclibgoexpdir)" "$(DESTDIR)$(toolexeclibgogodir)" "$(DESTDIR)$(toolexeclibgohashdir)" "$(DESTDIR)$(toolexeclibgohttpdir)" "$(DESTDIR)$(toolexeclibgoimagedir)" "$(DESTDIR)$(toolexeclibgoindexdir)" "$(DESTDIR)$(toolexeclibgoiodir)" "$(DESTDIR)$(toolexeclibgomimedir)" "$(DESTDIR)$(toolexeclibgonetdir)" "$(DESTDIR)$(toolexeclibgoosdir)" "$(DESTDIR)$(toolexeclibgorpcdir)" "$(DESTDIR)$(toolexeclibgoruntimedir)" "$(DESTDIR)$(toolexeclibgosyncdir)" "$(DESTDIR)$(toolexeclibgotestingdir)"; do \
test -z "$$dir" || $(MKDIR_P) "$$dir"; \
done
install: install-recursive
install-toolexeclibgoindexDATA install-toolexeclibgoioDATA \
install-toolexeclibgomimeDATA install-toolexeclibgonetDATA \
install-toolexeclibgoosDATA install-toolexeclibgorpcDATA \
- install-toolexeclibgoruntimeDATA \
+ install-toolexeclibgoruntimeDATA install-toolexeclibgosyncDATA \
install-toolexeclibgotestingDATA
install-html: install-html-recursive
uninstall-toolexeclibgomimeDATA uninstall-toolexeclibgonetDATA \
uninstall-toolexeclibgoosDATA uninstall-toolexeclibgorpcDATA \
uninstall-toolexeclibgoruntimeDATA \
+ uninstall-toolexeclibgosyncDATA \
uninstall-toolexeclibgotestingDATA
.MAKE: $(RECURSIVE_CLEAN_TARGETS) $(RECURSIVE_TARGETS) all all-multi \
install-toolexeclibgoindexDATA install-toolexeclibgoioDATA \
install-toolexeclibgomimeDATA install-toolexeclibgonetDATA \
install-toolexeclibgoosDATA install-toolexeclibgorpcDATA \
- install-toolexeclibgoruntimeDATA \
+ install-toolexeclibgoruntimeDATA install-toolexeclibgosyncDATA \
install-toolexeclibgotestingDATA installcheck installcheck-am \
installdirs installdirs-am maintainer-clean \
maintainer-clean-generic maintainer-clean-multi mostlyclean \
uninstall-toolexeclibgomimeDATA uninstall-toolexeclibgonetDATA \
uninstall-toolexeclibgoosDATA uninstall-toolexeclibgorpcDATA \
uninstall-toolexeclibgoruntimeDATA \
+ uninstall-toolexeclibgosyncDATA \
uninstall-toolexeclibgotestingDATA
$(CHECK)
.PHONY: cmath/check
+crypto/crypto.lo: $(go_crypto_files) hash.gox
+ $(BUILDPACKAGE)
+crypto/check: $(CHECK_DEPS)
+ $(CHECK)
+.PHONY: crypto/check
+
ebnf/ebnf.lo: $(go_ebnf_files) container/vector.gox go/scanner.gox \
go/token.gox os.gox strconv.gox unicode.gox utf8.gox
$(BUILDPACKAGE)
$(CHECK)
.PHONY: ebnf/check
-exec/exec.lo: $(go_exec_files) os.gox strings.gox
+exec/exec.lo: $(go_exec_files) os.gox strconv.gox strings.gox
$(BUILDPACKAGE)
exec/check: $(CHECK_DEPS)
$(CHECK)
$(CHECK)
.PHONY: flag/check
-fmt/fmt.lo: $(go_fmt_files) bytes.gox io.gox os.gox reflect.gox strconv.gox \
- strings.gox unicode.gox utf8.gox
+fmt/fmt.lo: $(go_fmt_files) bytes.gox io.gox math.gox os.gox reflect.gox \
+ strconv.gox strings.gox unicode.gox utf8.gox
$(BUILDPACKAGE)
fmt/check: $(CHECK_DEPS)
$(CHECK)
$(CHECK)
.PHONY: html/check
-http/http.lo: $(go_http_files) bufio.gox bytes.gox container/list.gox \
- container/vector.gox crypto/rand.gox crypto/tls.gox \
- encoding/base64.gox fmt.gox io.gox io/ioutil.gox log.gox \
- mime.gox mime/multipart.gox net.gox os.gox path.gox sort.gox \
+http/http.lo: $(go_http_files) bufio.gox bytes.gox container/vector.gox \
+ crypto/rand.gox crypto/tls.gox encoding/base64.gox fmt.gox \
+ io.gox io/ioutil.gox log.gox mime.gox mime/multipart.gox \
+ net.gox net/textproto.gox os.gox path.gox sort.gox \
strconv.gox strings.gox sync.gox time.gox utf8.gox
$(BUILDPACKAGE)
http/check: $(CHECK_DEPS)
$(CHECK)
.PHONY: io/check
-json/json.lo: $(go_json_files) bytes.gox container/vector.gox fmt.gox io.gox \
- math.gox os.gox reflect.gox runtime.gox strconv.gox \
- strings.gox unicode.gox utf16.gox utf8.gox
+json/json.lo: $(go_json_files) bytes.gox container/vector.gox \
+ encoding/base64.gox fmt.gox io.gox math.gox os.gox \
+ reflect.gox runtime.gox strconv.gox strings.gox unicode.gox \
+ utf16.gox utf8.gox
$(BUILDPACKAGE)
json/check: $(CHECK_DEPS)
$(CHECK)
$(CHECK)
.PHONY: net/check
-netchan/netchan.lo: $(go_netchan_files) gob.gox log.gox net.gox os.gox \
+netchan/netchan.lo: $(go_netchan_files) gob.gox io.gox log.gox net.gox os.gox \
reflect.gox strconv.gox sync.gox time.gox
$(BUILDPACKAGE)
netchan/check: $(CHECK_DEPS)
$(CHECK)
.PHONY: netchan/check
-os/os.lo: $(go_os_files) sync.gox syscall.gox
+os/os.lo: $(go_os_files) runtime.gox sync.gox syscall.gox
$(BUILDPACKAGE)
os/check: $(CHECK_DEPS)
$(CHECK)
$(CHECK)
.PHONY: strings/check
-sync/mutex.lo: $(go_sync_files) runtime.gox
+sync/sync.lo: $(go_sync_files) runtime.gox sync/atomic.gox
$(BUILDPACKAGE)
-sync/cas.lo: $(go_sync_c_files) sync/mutex.lo
- $(LTCOMPILE) -c -o sync/cas.lo $(srcdir)/go/sync/cas.c
sync/check: $(CHECK_DEPS)
$(CHECK)
.PHONY: sync/check
$(CHECK)
.PHONY: archive/zip/check
+compress/bzip2.lo: $(go_compress_bzip2_files) bufio.gox io.gox os.gox sort.gox
+ $(BUILDPACKAGE)
+compress/bzip2/check: $(CHECK_DEPS)
+ @$(MKDIR_P) compress/bzip2
+ $(CHECK)
+.PHONY: compress/bzip2/check
+
compress/flate.lo: $(go_compress_flate_files) bufio.gox io.gox math.gox \
os.gox sort.gox strconv.gox
$(BUILDPACKAGE)
$(CHECK)
.PHONY: compress/gzip/check
+compress/lzw.lo: $(go_compress_lzw_files) bufio.gox fmt.gox io.gox os.gox
+ $(BUILDPACKAGE)
+compress/lzw/check: $(CHECK_DEPS)
+ @$(MKDIR_P) compress/lzw
+ $(CHECK)
+.PHONY: compress/lzw/check
+
compress/zlib.lo: $(go_compress_zlib_files) bufio.gox compress/flate.gox \
hash.gox hash/adler32.gox io.gox os.gox
$(BUILDPACKAGE)
$(CHECK)
.PHONY: crypto/cipher/check
+crypto/dsa.lo: $(go_crypto_dsa_files) big.gox io.gox os.gox
+ $(BUILDPACKAGE)
+crypto/dsa/check: $(CHECK_DEPS)
+ @$(MKDIR_P) crypto/dsa
+ $(CHECK)
+.PHONY: crypto/dsa/check
+
crypto/elliptic.lo: $(go_crypto_elliptic_files) big.gox io.gox os.gox sync.gox
$(BUILDPACKAGE)
crypto/elliptic/check: $(CHECK_DEPS)
$(CHECK)
.PHONY: crypto/hmac/check
-crypto/md4.lo: $(go_crypto_md4_files) hash.gox os.gox
+crypto/md4.lo: $(go_crypto_md4_files) crypto.gox hash.gox os.gox
$(BUILDPACKAGE)
crypto/md4/check: $(CHECK_DEPS)
@$(MKDIR_P) crypto/md4
$(CHECK)
.PHONY: crypto/md4/check
-crypto/md5.lo: $(go_crypto_md5_files) hash.gox os.gox
+crypto/md5.lo: $(go_crypto_md5_files) crypto.gox hash.gox os.gox
$(BUILDPACKAGE)
crypto/md5/check: $(CHECK_DEPS)
@$(MKDIR_P) crypto/md5
$(CHECK)
.PHONY: crypto/md5/check
-crypto/ocsp.lo: $(go_crypto_ocsp_files) asn1.gox crypto/rsa.gox \
+crypto/ocsp.lo: $(go_crypto_ocsp_files) asn1.gox crypto.gox crypto/rsa.gox \
crypto/sha1.gox crypto/x509.gox os.gox time.gox
$(BUILDPACKAGE)
crypto/ocsp/check: $(CHECK_DEPS)
$(CHECK)
.PHONY: crypto/ocsp/check
-crypto/rand.lo: $(go_crypto_rand_files) crypto/aes.gox io.gox os.gox sync.gox \
- time.gox
+crypto/openpgp.lo: $(go_crypto_openpgp_files) crypto.gox \
+ crypto/openpgp/armor.gox crypto/openpgp/error.gox \
+ crypto/openpgp/packet.gox crypto/rsa.gox crypto/sha256.gox \
+ hash.gox io.gox os.gox strconv.gox time.gox
+ $(BUILDPACKAGE)
+crypto/openpgp/check: $(CHECK_DEPS)
+ @$(MKDIR_P) crypto/openpgp
+ $(CHECK)
+.PHONY: crypto/openpgp/check
+
+crypto/rand.lo: $(go_crypto_rand_files) bufio.gox crypto/aes.gox io.gox \
+ os.gox sync.gox time.gox
$(BUILDPACKAGE)
crypto/rand/check: $(CHECK_DEPS)
@$(MKDIR_P) crypto/rand
$(CHECK)
.PHONY: crypto/rc4/check
-crypto/ripemd160.lo: $(go_crypto_ripemd160_files) hash.gox os.gox
+crypto/ripemd160.lo: $(go_crypto_ripemd160_files) crypto.gox hash.gox os.gox
$(BUILDPACKAGE)
crypto/ripemd160/check: $(CHECK_DEPS)
@$(MKDIR_P) crypto/ripemd160
$(CHECK)
.PHONY: crypto/ripemd160/check
-crypto/rsa.lo: $(go_crypto_rsa_files) big.gox crypto/sha1.gox \
+crypto/rsa.lo: $(go_crypto_rsa_files) big.gox crypto.gox crypto/sha1.gox \
crypto/subtle.gox encoding/hex.gox hash.gox io.gox os.gox
$(BUILDPACKAGE)
crypto/rsa/check: $(CHECK_DEPS)
$(CHECK)
.PHONY: crypto/rsa/check
-crypto/sha1.lo: $(go_crypto_sha1_files) hash.gox os.gox
+crypto/sha1.lo: $(go_crypto_sha1_files) crypto.gox hash.gox os.gox
$(BUILDPACKAGE)
crypto/sha1/check: $(CHECK_DEPS)
@$(MKDIR_P) crypto/sha1
$(CHECK)
.PHONY: crypto/sha1/check
-crypto/sha256.lo: $(go_crypto_sha256_files) hash.gox os.gox
+crypto/sha256.lo: $(go_crypto_sha256_files) crypto.gox hash.gox os.gox
$(BUILDPACKAGE)
crypto/sha256/check: $(CHECK_DEPS)
@$(MKDIR_P) crypto/sha256
$(CHECK)
.PHONY: crypto/sha256/check
-crypto/sha512.lo: $(go_crypto_sha512_files) hash.gox os.gox
+crypto/sha512.lo: $(go_crypto_sha512_files) crypto.gox hash.gox os.gox
$(BUILDPACKAGE)
crypto/sha512/check: $(CHECK_DEPS)
@$(MKDIR_P) crypto/sha512
.PHONY: crypto/subtle/check
crypto/tls.lo: $(go_crypto_tls_files) big.gox bufio.gox bytes.gox \
- container/list.gox crypto/aes.gox crypto/cipher.gox \
+ container/list.gox crypto.gox crypto/aes.gox crypto/cipher.gox \
crypto/elliptic.gox crypto/hmac.gox crypto/md5.gox \
crypto/rc4.gox crypto/rand.gox crypto/rsa.gox crypto/sha1.gox \
crypto/subtle.gox crypto/rsa.gox crypto/sha1.gox \
.PHONY: crypto/twofish/check
crypto/x509.lo: $(go_crypto_x509_files) asn1.gox big.gox container/vector.gox \
- crypto/rsa.gox crypto/sha1.gox hash.gox os.gox strings.gox \
- time.gox
+ crypto.gox crypto/rsa.gox crypto/sha1.gox hash.gox os.gox \
+ strings.gox time.gox
$(BUILDPACKAGE)
crypto/x509/check: $(CHECK_DEPS)
@$(MKDIR_P) crypto/x509
$(CHECK)
.PHONY: crypto/openpgp/armor/check
-crypto/openpgp/error.lo: $(go_crypto_openpgp_error_files)
+crypto/openpgp/error.lo: $(go_crypto_openpgp_error_files) strconv.gox
$(BUILDPACKAGE)
crypto/openpgp/error/check: $(CHECK_DEPS)
@$(MKDIR_P) crypto/openpgp/error
$(CHECK)
.PHONY: crypto/openpgp/error/check
-crypto/openpgp/s2k.lo: $(go_crypto_openpgp_s2k_files) crypto/md5.gox \
- crypto/openpgp/error.gox crypto/ripemd160.gox crypto/sha1.gox \
- crypto/sha256.gox crypto/sha512.gox hash.gox io.gox os.gox
+crypto/openpgp/packet.lo: $(go_crypto_openpgp_packet_files) big.gox bytes.gox \
+ compress/flate.gox compress/zlib.gox crypto.gox \
+ crypto/aes.gox crypto/cast5.gox crypto/cipher.gox \
+ crypto/openpgp/error.gox crypto/openpgp/s2k.gox \
+ crypto/rand.gox crypto/rsa.gox crypto/sha1.gox \
+ crypto/subtle.gox encoding/binary.gox hash.gox io.gox \
+ io/ioutil.gox os.gox strconv.gox strings.gox
+ $(BUILDPACKAGE)
+crypto/openpgp/packet/check: $(CHECK_DEPS)
+ @$(MKDIR_P) crypto/openpgp/packet
+ $(CHECK)
+.PHONY: crypto/openpgp/packet/check
+
+crypto/openpgp/s2k.lo: $(go_crypto_openpgp_s2k_files) crypto.gox \
+ crypto/md5.gox crypto/openpgp/error.gox crypto/ripemd160.gox \
+ crypto/sha1.gox crypto/sha256.gox crypto/sha512.gox hash.gox \
+ io.gox os.gox
$(BUILDPACKAGE)
crypto/openpgp/s2k/check: $(CHECK_DEPS)
@$(MKDIR_P) crypto/openpgp/s2k
$(CHECK)
.PHONY: runtime/pprof/check
+sync/atomic.lo: $(go_sync_atomic_files)
+ $(BUILDPACKAGE)
+sync/atomic_c.lo: $(go_sync_atomic_c_files) sync/atomic.lo
+ $(LTCOMPILE) -c -o $@ $(srcdir)/go/sync/atomic/atomic.c
+sync/atomic/check: $(CHECK_DEPS)
+ @$(MKDIR_P) sync/atomic
+ $(CHECK)
+.PHONY: sync/atomic/check
+
testing/iotest.lo: $(go_testing_iotest_files) io.gox log.gox os.gox
$(BUILDPACKAGE)
testing/iotest/check: $(CHECK_DEPS)
$(BUILDGOX)
cmath.gox: cmath/cmath.lo
$(BUILDGOX)
+crypto.gox: crypto/crypto.lo
+ $(BUILDGOX)
ebnf.gox: ebnf/ebnf.lo
$(BUILDGOX)
exec.gox: exec/exec.lo
$(BUILDGOX)
strings.gox: strings/strings.lo
$(BUILDGOX)
-sync.gox: sync/mutex.lo
+sync.gox: sync/sync.lo
$(BUILDGOX)
syslog.gox: syslog/syslog.lo
$(BUILDGOX)
archive/zip.gox: archive/zip.lo
$(BUILDGOX)
+compress/bzip2.gox: compress/bzip2.lo
+ $(BUILDGOX)
compress/flate.gox: compress/flate.lo
$(BUILDGOX)
compress/gzip.gox: compress/gzip.lo
$(BUILDGOX)
+compress/lzw.gox: compress/lzw.lo
+ $(BUILDGOX)
compress/zlib.gox: compress/zlib.lo
$(BUILDGOX)
$(BUILDGOX)
crypto/cipher.gox: crypto/cipher.lo
$(BUILDGOX)
+crypto/dsa.gox: crypto/dsa.lo
+ $(BUILDGOX)
crypto/elliptic.gox: crypto/elliptic.lo
$(BUILDGOX)
crypto/hmac.gox: crypto/hmac.lo
$(BUILDGOX)
crypto/ocsp.gox: crypto/ocsp.lo
$(BUILDGOX)
+crypto/openpgp.gox: crypto/openpgp.lo
+ $(BUILDGOX)
crypto/rand.gox: crypto/rand.lo
$(BUILDGOX)
crypto/rc4.gox: crypto/rc4.lo
$(BUILDGOX)
crypto/openpgp/error.gox: crypto/openpgp/error.lo
$(BUILDGOX)
+crypto/openpgp/packet.gox: crypto/openpgp/packet.lo
+ $(BUILDGOX)
crypto/openpgp/s2k.gox: crypto/openpgp/s2k.lo
$(BUILDGOX)
runtime/pprof.gox: runtime/pprof.lo
$(BUILDGOX)
+sync/atomic.gox: sync/atomic.lo
+ $(BUILDGOX)
+
testing/iotest.gox: testing/iotest.lo
$(BUILDGOX)
testing/quick.gox: testing/quick.lo
bodyOffset int64
}
+func (f *File) hasDataDescriptor() bool {
+ return f.Flags&0x8 != 0
+}
+
// OpenReader will open the Zip file specified by name and return a Reader.
func OpenReader(name string) (*Reader, os.Error) {
f, err := os.Open(name, os.O_RDONLY, 0644)
return
}
}
- r := io.NewSectionReader(f.zipr, off+f.bodyOffset, int64(f.CompressedSize))
+ size := int64(f.CompressedSize)
+ if f.hasDataDescriptor() {
+ if size == 0 {
+ // permit SectionReader to see the rest of the file
+ size = f.zipsize - (off + f.bodyOffset)
+ } else {
+ size += dataDescriptorLen
+ }
+ }
+ r := io.NewSectionReader(f.zipr, off+f.bodyOffset, size)
switch f.Method {
case 0: // store (no compression)
rc = nopCloser{r}
err = UnsupportedMethod
}
if rc != nil {
- rc = &checksumReader{rc, crc32.NewIEEE(), f.CRC32}
+ rc = &checksumReader{rc, crc32.NewIEEE(), f, r}
}
return
}
type checksumReader struct {
rc io.ReadCloser
hash hash.Hash32
- sum uint32
+ f *File
+ zipr io.Reader // for reading the data descriptor
}
func (r *checksumReader) Read(b []byte) (n int, err os.Error) {
if err != os.EOF {
return
}
- if r.hash.Sum32() != r.sum {
+ if r.f.hasDataDescriptor() {
+ if err = readDataDescriptor(r.zipr, r.f); err != nil {
+ return
+ }
+ }
+ if r.hash.Sum32() != r.f.CRC32 {
err = ChecksumError
}
return
return
}
+func readDataDescriptor(r io.Reader, f *File) (err os.Error) {
+ defer func() {
+ if rerr, ok := recover().(os.Error); ok {
+ err = rerr
+ }
+ }()
+ read(r, &f.CRC32)
+ read(r, &f.CompressedSize)
+ read(r, &f.UncompressedSize)
+ return
+}
+
func readDirectoryEnd(r io.ReaderAt, size int64) (d *directoryEnd, err os.Error) {
// look for directoryEndSignature in the last 1k, then in the last 65k
var b []byte
},
{Name: "readme.zip"},
{Name: "readme.notzip", Error: FormatError},
+ {
+ Name: "dd.zip",
+ File: []ZipTestFile{
+ {
+ Name: "filename",
+ Content: []byte("This is a test textfile.\n"),
+ },
+ },
+ },
}
func TestReader(t *testing.T) {
}
// test invalid checksum
- z.File[0].CRC32++ // invalidate
- r, err := z.File[0].Open()
- if err != nil {
- t.Error(err)
- return
- }
- var b bytes.Buffer
- _, err = io.Copy(&b, r)
- if err != ChecksumError {
- t.Errorf("%s: copy error=%v, want %v", z.File[0].Name, err, ChecksumError)
+ if !z.File[0].hasDataDescriptor() { // skip test when crc32 in dd
+ z.File[0].CRC32++ // invalidate
+ r, err := z.File[0].Open()
+ if err != nil {
+ t.Error(err)
+ return
+ }
+ var b bytes.Buffer
+ _, err = io.Copy(&b, r)
+ if err != ChecksumError {
+ t.Errorf("%s: copy error=%v, want %v", z.File[0].Name, err, ChecksumError)
+ }
}
}
fileHeaderSignature = 0x04034b50
directoryHeaderSignature = 0x02014b50
directoryEndSignature = 0x06054b50
+ dataDescriptorLen = 12
)
type FileHeader struct {
switch v := value.(type) {
case *reflect.BoolValue:
if v.Get() {
- return out.WriteByte(1)
+ return out.WriteByte(255)
} else {
return out.WriteByte(0)
}
// returning a slice containing the data up to and including the delimiter.
// If ReadBytes encounters an error before finding a delimiter,
// it returns the data read before the error and the error itself (often os.EOF).
-// ReadBytes returns err != nil if and only if line does not end in delim.
+// ReadBytes returns err != nil if and only if the returned data does not end in
+// delim.
func (b *Reader) ReadBytes(delim byte) (line []byte, err os.Error) {
// Use ReadSlice to look for array,
// accumulating full buffers.
// returning a string containing the data up to and including the delimiter.
// If ReadString encounters an error before finding a delimiter,
// it returns the data read before the error and the error itself (often os.EOF).
-// ReadString returns err != nil if and only if line does not end in delim.
+// ReadString returns err != nil if and only if the returned data does not end in
+// delim.
func (b *Reader) ReadString(delim byte) (line string, err os.Error) {
bytes, e := b.ReadBytes(delim)
return string(bytes), e
if b.err != nil {
return b.err
}
+ if b.n == 0 {
+ return nil
+ }
n, e := b.wr.Write(b.buf[0:b.n])
if n < b.n && e == nil {
e = io.ErrShortWrite
}
// WriteTo writes data to w until the buffer is drained or an error
-// occurs. The return value n is the number of bytes written.
+// occurs. The return value n is the number of bytes written; it always
+// fits into an int, but it is int64 to match the io.WriterTo interface.
// Any error encountered during the write is also returned.
func (b *Buffer) WriteTo(w io.Writer) (n int64, err os.Error) {
b.lastRead = opInvalid
- for b.off < len(b.buf) {
+ if b.off < len(b.buf) {
m, e := w.Write(b.buf[b.off:])
- n += int64(m)
b.off += m
+ n = int64(m)
if e != nil {
return n, e
}
+ // otherwise all bytes were written, by definition of
+ // Write method in io.Writer
}
// Buffer is now empty; reset.
b.Truncate(0)
return nil
}
+// ReadBytes reads until the first occurrence of delim in the input,
+// returning a slice containing the data up to and including the delimiter.
+// If ReadBytes encounters an error before finding a delimiter,
+// it returns the data read before the error and the error itself (often os.EOF).
+// ReadBytes returns err != nil if and only if the returned data does not end in
+// delim.
+func (b *Buffer) ReadBytes(delim byte) (line []byte, err os.Error) {
+ i := IndexByte(b.buf[b.off:], delim)
+ size := i + 1
+ if i < 0 {
+ size = len(b.buf) - b.off
+ err = os.EOF
+ }
+ line = make([]byte, size)
+ copy(line, b.buf[b.off:])
+ b.off += size
+ return
+}
+
+// ReadString reads until the first occurrence of delim in the input,
+// returning a string containing the data up to and including the delimiter.
+// If ReadString encounters an error before finding a delimiter,
+// it returns the data read before the error and the error itself (often os.EOF).
+// ReadString returns err != nil if and only if the returned data does not end
+// in delim.
+func (b *Buffer) ReadString(delim byte) (line string, err os.Error) {
+ bytes, err := b.ReadBytes(delim)
+ return string(bytes), err
+}
+
// NewBuffer creates and initializes a new Buffer using buf as its initial
// contents. It is intended to prepare a Buffer to read existing data. It
// can also be used to size the internal buffer for writing. To do that,
import (
. "bytes"
+ "os"
"rand"
"testing"
"utf8"
func TestNil(t *testing.T) {
var b *Buffer
if b.String() != "<nil>" {
- t.Errorf("expcted <nil>; got %q", b.String())
+ t.Errorf("expected <nil>; got %q", b.String())
}
}
}
}
}
+
+var readBytesTests = []struct {
+ buffer string
+ delim byte
+ expected []string
+ err os.Error
+}{
+ {"", 0, []string{""}, os.EOF},
+ {"a\x00", 0, []string{"a\x00"}, nil},
+ {"abbbaaaba", 'b', []string{"ab", "b", "b", "aaab"}, nil},
+ {"hello\x01world", 1, []string{"hello\x01"}, nil},
+ {"foo\nbar", 0, []string{"foo\nbar"}, os.EOF},
+ {"alpha\nbeta\ngamma\n", '\n', []string{"alpha\n", "beta\n", "gamma\n"}, nil},
+ {"alpha\nbeta\ngamma", '\n', []string{"alpha\n", "beta\n", "gamma"}, os.EOF},
+}
+
+func TestReadBytes(t *testing.T) {
+ for _, test := range readBytesTests {
+ buf := NewBufferString(test.buffer)
+ var err os.Error
+ for _, expected := range test.expected {
+ var bytes []byte
+ bytes, err = buf.ReadBytes(test.delim)
+ if string(bytes) != expected {
+ t.Errorf("expected %q, got %q", expected, bytes)
+ }
+ if err != nil {
+ break
+ }
+ }
+ if err != test.err {
+ t.Errorf("expected error %v, got %v", test.err, err)
+ }
+ }
+}
--- /dev/null
+// Copyright 2011 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package bzip2
+
+import (
+ "bufio"
+ "io"
+ "os"
+)
+
+// bitReader wraps an io.Reader and provides the ability to read values,
+// bit-by-bit, from it. Its Read* methods don't return the usual os.Error
+// because the error handling was verbose. Instead, any error is kept and can
+// be checked afterwards.
+type bitReader struct {
+ r byteReader
+ n uint64
+ bits uint
+ err os.Error
+}
+
+// bitReader needs to read bytes from an io.Reader. We attempt to cast the
+// given io.Reader to this interface and, if it doesn't already fit, we wrap in
+// a bufio.Reader.
+type byteReader interface {
+ ReadByte() (byte, os.Error)
+}
+
+func newBitReader(r io.Reader) bitReader {
+ byter, ok := r.(byteReader)
+ if !ok {
+ byter = bufio.NewReader(r)
+ }
+ return bitReader{r: byter}
+}
+
+// ReadBits64 reads the given number of bits and returns them in the
+// least-significant part of a uint64. In the event of an error, it returns 0
+// and the error can be obtained by calling Error().
+func (br *bitReader) ReadBits64(bits uint) (n uint64) {
+ for bits > br.bits {
+ b, err := br.r.ReadByte()
+ if err == os.EOF {
+ err = io.ErrUnexpectedEOF
+ }
+ if err != nil {
+ br.err = err
+ return 0
+ }
+ br.n <<= 8
+ br.n |= uint64(b)
+ br.bits += 8
+ }
+
+ // br.n looks like this (assuming that br.bits = 14 and bits = 6):
+ // Bit: 111111
+ // 5432109876543210
+ //
+ // (6 bits, the desired output)
+ // |-----|
+ // V V
+ // 0101101101001110
+ // ^ ^
+ // |------------|
+ // br.bits (num valid bits)
+ //
+ // This the next line right shifts the desired bits into the
+ // least-significant places and masks off anything above.
+ n = (br.n >> (br.bits - bits)) & ((1 << bits) - 1)
+ br.bits -= bits
+ return
+}
+
+func (br *bitReader) ReadBits(bits uint) (n int) {
+ n64 := br.ReadBits64(bits)
+ return int(n64)
+}
+
+func (br *bitReader) ReadBit() bool {
+ n := br.ReadBits(1)
+ return n != 0
+}
+
+func (br *bitReader) Error() os.Error {
+ return br.err
+}
--- /dev/null
+// Copyright 2011 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Package bzip2 implements bzip2 decompression.
+package bzip2
+
+import (
+ "io"
+ "os"
+)
+
+// There's no RFC for bzip2. I used the Wikipedia page for reference and a lot
+// of guessing: http://en.wikipedia.org/wiki/Bzip2
+// The source code to pyflate was useful for debugging:
+// http://www.paul.sladen.org/projects/pyflate
+
+// A StructuralError is returned when the bzip2 data is found to be
+// syntactically invalid.
+type StructuralError string
+
+func (s StructuralError) String() string {
+ return "bzip2 data invalid: " + string(s)
+}
+
+// A reader decompresses bzip2 compressed data.
+type reader struct {
+ br bitReader
+ setupDone bool // true if we have parsed the bzip2 header.
+ blockSize int // blockSize in bytes, i.e. 900 * 1024.
+ eof bool
+ buf []byte // stores Burrows-Wheeler transformed data.
+ c [256]uint // the `C' array for the inverse BWT.
+ tt []uint32 // mirrors the `tt' array in the bzip2 source and contains the P array in the upper 24 bits.
+ tPos uint32 // Index of the next output byte in tt.
+
+ preRLE []uint32 // contains the RLE data still to be processed.
+ preRLEUsed int // number of entries of preRLE used.
+ lastByte int // the last byte value seen.
+ byteRepeats uint // the number of repeats of lastByte seen.
+ repeats uint // the number of copies of lastByte to output.
+}
+
+// NewReader returns an io.Reader which decompresses bzip2 data from r.
+func NewReader(r io.Reader) io.Reader {
+ bz2 := new(reader)
+ bz2.br = newBitReader(r)
+ return bz2
+}
+
+const bzip2FileMagic = 0x425a // "BZ"
+const bzip2BlockMagic = 0x314159265359
+const bzip2FinalMagic = 0x177245385090
+
+// setup parses the bzip2 header.
+func (bz2 *reader) setup() os.Error {
+ br := &bz2.br
+
+ magic := br.ReadBits(16)
+ if magic != bzip2FileMagic {
+ return StructuralError("bad magic value")
+ }
+
+ t := br.ReadBits(8)
+ if t != 'h' {
+ return StructuralError("non-Huffman entropy encoding")
+ }
+
+ level := br.ReadBits(8)
+ if level < '1' || level > '9' {
+ return StructuralError("invalid compression level")
+ }
+
+ bz2.blockSize = 100 * 1024 * (int(level) - '0')
+ bz2.tt = make([]uint32, bz2.blockSize)
+ return nil
+}
+
+func (bz2 *reader) Read(buf []byte) (n int, err os.Error) {
+ if bz2.eof {
+ return 0, os.EOF
+ }
+
+ if !bz2.setupDone {
+ err = bz2.setup()
+ brErr := bz2.br.Error()
+ if brErr != nil {
+ err = brErr
+ }
+ if err != nil {
+ return 0, err
+ }
+ bz2.setupDone = true
+ }
+
+ n, err = bz2.read(buf)
+ brErr := bz2.br.Error()
+ if brErr != nil {
+ err = brErr
+ }
+ return
+}
+
+func (bz2 *reader) read(buf []byte) (n int, err os.Error) {
+ // bzip2 is a block based compressor, except that it has a run-length
+ // preprocessing step. The block based nature means that we can
+ // preallocate fixed-size buffers and reuse them. However, the RLE
+ // preprocessing would require allocating huge buffers to store the
+ // maximum expansion. Thus we process blocks all at once, except for
+ // the RLE which we decompress as required.
+
+ for (bz2.repeats > 0 || bz2.preRLEUsed < len(bz2.preRLE)) && n < len(buf) {
+ // We have RLE data pending.
+
+ // The run-length encoding works like this:
+ // Any sequence of four equal bytes is followed by a length
+ // byte which contains the number of repeats of that byte to
+ // include. (The number of repeats can be zero.) Because we are
+ // decompressing on-demand our state is kept in the reader
+ // object.
+
+ if bz2.repeats > 0 {
+ buf[n] = byte(bz2.lastByte)
+ n++
+ bz2.repeats--
+ if bz2.repeats == 0 {
+ bz2.lastByte = -1
+ }
+ continue
+ }
+
+ bz2.tPos = bz2.preRLE[bz2.tPos]
+ b := byte(bz2.tPos)
+ bz2.tPos >>= 8
+ bz2.preRLEUsed++
+
+ if bz2.byteRepeats == 3 {
+ bz2.repeats = uint(b)
+ bz2.byteRepeats = 0
+ continue
+ }
+
+ if bz2.lastByte == int(b) {
+ bz2.byteRepeats++
+ } else {
+ bz2.byteRepeats = 0
+ }
+ bz2.lastByte = int(b)
+
+ buf[n] = b
+ n++
+ }
+
+ if n > 0 {
+ return
+ }
+
+ // No RLE data is pending so we need to read a block.
+
+ br := &bz2.br
+ magic := br.ReadBits64(48)
+ if magic == bzip2FinalMagic {
+ br.ReadBits64(32) // ignored CRC
+ bz2.eof = true
+ return 0, os.EOF
+ } else if magic != bzip2BlockMagic {
+ return 0, StructuralError("bad magic value found")
+ }
+
+ err = bz2.readBlock()
+ if err != nil {
+ return 0, err
+ }
+
+ return bz2.read(buf)
+}
+
+// readBlock reads a bzip2 block. The magic number should already have been consumed.
+func (bz2 *reader) readBlock() (err os.Error) {
+ br := &bz2.br
+ br.ReadBits64(32) // skip checksum. TODO: check it if we can figure out what it is.
+ randomized := br.ReadBits(1)
+ if randomized != 0 {
+ return StructuralError("deprecated randomized files")
+ }
+ origPtr := uint(br.ReadBits(24))
+
+ // If not every byte value is used in the block (i.e., it's text) then
+ // the symbol set is reduced. The symbols used are stored as a
+ // two-level, 16x16 bitmap.
+ symbolRangeUsedBitmap := br.ReadBits(16)
+ symbolPresent := make([]bool, 256)
+ numSymbols := 0
+ for symRange := uint(0); symRange < 16; symRange++ {
+ if symbolRangeUsedBitmap&(1<<(15-symRange)) != 0 {
+ bits := br.ReadBits(16)
+ for symbol := uint(0); symbol < 16; symbol++ {
+ if bits&(1<<(15-symbol)) != 0 {
+ symbolPresent[16*symRange+symbol] = true
+ numSymbols++
+ }
+ }
+ }
+ }
+
+ // A block uses between two and six different Huffman trees.
+ numHuffmanTrees := br.ReadBits(3)
+ if numHuffmanTrees < 2 || numHuffmanTrees > 6 {
+ return StructuralError("invalid number of Huffman trees")
+ }
+
+ // The Huffman tree can switch every 50 symbols so there's a list of
+ // tree indexes telling us which tree to use for each 50 symbol block.
+ numSelectors := br.ReadBits(15)
+ treeIndexes := make([]uint8, numSelectors)
+
+ // The tree indexes are move-to-front transformed and stored as unary
+ // numbers.
+ mtfTreeDecoder := newMTFDecoderWithRange(numHuffmanTrees)
+ for i := range treeIndexes {
+ c := 0
+ for {
+ inc := br.ReadBits(1)
+ if inc == 0 {
+ break
+ }
+ c++
+ }
+ if c >= numHuffmanTrees {
+ return StructuralError("tree index too large")
+ }
+ treeIndexes[i] = uint8(mtfTreeDecoder.Decode(c))
+ }
+
+ // The list of symbols for the move-to-front transform is taken from
+ // the previously decoded symbol bitmap.
+ symbols := make([]byte, numSymbols)
+ nextSymbol := 0
+ for i := 0; i < 256; i++ {
+ if symbolPresent[i] {
+ symbols[nextSymbol] = byte(i)
+ nextSymbol++
+ }
+ }
+ mtf := newMTFDecoder(symbols)
+
+ numSymbols += 2 // to account for RUNA and RUNB symbols
+ huffmanTrees := make([]huffmanTree, numHuffmanTrees)
+
+ // Now we decode the arrays of code-lengths for each tree.
+ lengths := make([]uint8, numSymbols)
+ for i := 0; i < numHuffmanTrees; i++ {
+ // The code lengths are delta encoded from a 5-bit base value.
+ length := br.ReadBits(5)
+ for j := 0; j < numSymbols; j++ {
+ for {
+ if !br.ReadBit() {
+ break
+ }
+ if br.ReadBit() {
+ length--
+ } else {
+ length++
+ }
+ }
+ if length < 0 || length > 20 {
+ return StructuralError("Huffman length out of range")
+ }
+ lengths[j] = uint8(length)
+ }
+ huffmanTrees[i], err = newHuffmanTree(lengths)
+ if err != nil {
+ return err
+ }
+ }
+
+ selectorIndex := 1 // the next tree index to use
+ currentHuffmanTree := huffmanTrees[treeIndexes[0]]
+ bufIndex := 0 // indexes bz2.buf, the output buffer.
+ // The output of the move-to-front transform is run-length encoded and
+ // we merge the decoding into the Huffman parsing loop. These two
+ // variables accumulate the repeat count. See the Wikipedia page for
+ // details.
+ repeat := 0
+ repeat_power := 0
+
+ // The `C' array (used by the inverse BWT) needs to be zero initialised.
+ for i := range bz2.c {
+ bz2.c[i] = 0
+ }
+
+ decoded := 0 // counts the number of symbols decoded by the current tree.
+ for {
+ if decoded == 50 {
+ currentHuffmanTree = huffmanTrees[treeIndexes[selectorIndex]]
+ selectorIndex++
+ decoded = 0
+ }
+
+ v := currentHuffmanTree.Decode(br)
+ decoded++
+
+ if v < 2 {
+ // This is either the RUNA or RUNB symbol.
+ if repeat == 0 {
+ repeat_power = 1
+ }
+ repeat += repeat_power << v
+ repeat_power <<= 1
+
+ // This limit of 2 million comes from the bzip2 source
+ // code. It prevents repeat from overflowing.
+ if repeat > 2*1024*1024 {
+ return StructuralError("repeat count too large")
+ }
+ continue
+ }
+
+ if repeat > 0 {
+ // We have decoded a complete run-length so we need to
+ // replicate the last output symbol.
+ for i := 0; i < repeat; i++ {
+ b := byte(mtf.First())
+ bz2.tt[bufIndex] = uint32(b)
+ bz2.c[b]++
+ bufIndex++
+ }
+ repeat = 0
+ }
+
+ if int(v) == numSymbols-1 {
+ // This is the EOF symbol. Because it's always at the
+ // end of the move-to-front list, and nevers gets moved
+ // to the front, it has this unique value.
+ break
+ }
+
+ // Since two metasymbols (RUNA and RUNB) have values 0 and 1,
+ // one would expect |v-2| to be passed to the MTF decoder.
+ // However, the front of the MTF list is never referenced as 0,
+ // it's always referenced with a run-length of 1. Thus 0
+ // doesn't need to be encoded and we have |v-1| in the next
+ // line.
+ b := byte(mtf.Decode(int(v - 1)))
+ bz2.tt[bufIndex] = uint32(b)
+ bz2.c[b]++
+ bufIndex++
+ }
+
+ if origPtr >= uint(bufIndex) {
+ return StructuralError("origPtr out of bounds")
+ }
+
+ // We have completed the entropy decoding. Now we can perform the
+ // inverse BWT and setup the RLE buffer.
+ bz2.preRLE = bz2.tt[:bufIndex]
+ bz2.preRLEUsed = 0
+ bz2.tPos = inverseBWT(bz2.preRLE, origPtr, bz2.c[:])
+ bz2.lastByte = -1
+ bz2.byteRepeats = 0
+ bz2.repeats = 0
+
+ return nil
+}
+
+// inverseBWT implements the inverse Burrows-Wheeler transform as described in
+// http://www.hpl.hp.com/techreports/Compaq-DEC/SRC-RR-124.pdf, section 4.2.
+// In that document, origPtr is called `I' and c is the `C' array after the
+// first pass over the data. It's an argument here because we merge the first
+// pass with the Huffman decoding.
+//
+// This also implements the `single array' method from the bzip2 source code
+// which leaves the output, still shuffled, in the bottom 8 bits of tt with the
+// index of the next byte in the top 24-bits. The index of the first byte is
+// returned.
+func inverseBWT(tt []uint32, origPtr uint, c []uint) uint32 {
+ sum := uint(0)
+ for i := 0; i < 256; i++ {
+ sum += c[i]
+ c[i] = sum - c[i]
+ }
+
+ for i := range tt {
+ b := tt[i] & 0xff
+ tt[c[b]] |= uint32(i) << 8
+ c[b]++
+ }
+
+ return tt[origPtr] >> 8
+}
--- /dev/null
+// Copyright 2011 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package bzip2
+
+import (
+ "bytes"
+ "encoding/hex"
+ "io"
+ "io/ioutil"
+ "os"
+ "testing"
+)
+
+func TestBitReader(t *testing.T) {
+ buf := bytes.NewBuffer([]byte{0xaa})
+ br := newBitReader(buf)
+ if n := br.ReadBits(1); n != 1 {
+ t.Errorf("read 1 wrong")
+ }
+ if n := br.ReadBits(1); n != 0 {
+ t.Errorf("read 2 wrong")
+ }
+ if n := br.ReadBits(1); n != 1 {
+ t.Errorf("read 3 wrong")
+ }
+ if n := br.ReadBits(1); n != 0 {
+ t.Errorf("read 4 wrong")
+ }
+}
+
+func TestBitReaderLarge(t *testing.T) {
+ buf := bytes.NewBuffer([]byte{0x12, 0x34, 0x56, 0x78})
+ br := newBitReader(buf)
+ if n := br.ReadBits(32); n != 0x12345678 {
+ t.Errorf("got: %x want: %x", n, 0x12345678)
+ }
+}
+
+func readerFromHex(s string) io.Reader {
+ data, err := hex.DecodeString(s)
+ if err != nil {
+ panic("readerFromHex: bad input")
+ }
+ return bytes.NewBuffer(data)
+}
+
+func decompressHex(s string) (out []byte, err os.Error) {
+ r := NewReader(readerFromHex(s))
+ return ioutil.ReadAll(r)
+}
+
+func TestHelloWorldBZ2(t *testing.T) {
+ out, err := decompressHex(helloWorldBZ2Hex)
+ if err != nil {
+ t.Errorf("error from Read: %s", err)
+ return
+ }
+
+ if !bytes.Equal(helloWorld, out) {
+ t.Errorf("got %x, want %x", out, helloWorld)
+ }
+}
+
+func testZeros(t *testing.T, inHex string, n int) {
+ out, err := decompressHex(inHex)
+ if err != nil {
+ t.Errorf("error from Read: %s", err)
+ return
+ }
+
+ expected := make([]byte, n)
+
+ if !bytes.Equal(expected, out) {
+ allZeros := true
+ for _, b := range out {
+ if b != 0 {
+ allZeros = false
+ break
+ }
+ }
+ t.Errorf("incorrect result, got %d bytes (allZeros: %t)", len(out), allZeros)
+ }
+}
+
+func Test32Zeros(t *testing.T) {
+ testZeros(t, thirtyTwoZerosBZ2Hex, 32)
+}
+
+func Test1MBZeros(t *testing.T) {
+ testZeros(t, oneMBZerosBZ2Hex, 1024*1024)
+}
+
+func testRandomData(t *testing.T, compressedHex, uncompressedHex string) {
+ out, err := decompressHex(compressedHex)
+ if err != nil {
+ t.Errorf("error from Read: %s", err)
+ return
+ }
+
+ expected, _ := hex.DecodeString(uncompressedHex)
+
+ if !bytes.Equal(out, expected) {
+ t.Errorf("incorrect result\ngot: %x\nwant: %x", out, expected)
+ }
+}
+
+func TestRandomData1(t *testing.T) {
+ testRandomData(t, randBZ2Hex, randHex)
+}
+
+func TestRandomData2(t *testing.T) {
+ // This test involves several repeated bytes in the output, but they
+ // should trigger RLE decoding.
+ testRandomData(t, rand2BZ2Hex, rand2Hex)
+}
+
+func TestRandomData3(t *testing.T) {
+ // This test uses the full range of symbols.
+ testRandomData(t, rand3BZ2Hex, rand3Hex)
+}
+
+func Test1MBSawtooth(t *testing.T) {
+ out, err := decompressHex(oneMBSawtoothBZ2Hex)
+ if err != nil {
+ t.Errorf("error from Read: %s", err)
+ return
+ }
+
+ expected := make([]byte, 1024*1024)
+
+ for i := range expected {
+ expected[i] = byte(i)
+ }
+
+ if !bytes.Equal(out, expected) {
+ t.Error("incorrect result")
+ }
+}
+
+const helloWorldBZ2Hex = "425a68393141592653594eece83600000251800010400006449080200031064c4101a7a9a580bb9431f8bb9229c28482776741b0"
+
+var helloWorld = []byte("hello world\n")
+
+const thirtyTwoZerosBZ2Hex = "425a6839314159265359b5aa5098000000600040000004200021008283177245385090b5aa5098"
+const oneMBZerosBZ2Hex = "425a683931415926535938571ce50008084000c0040008200030cc0529a60806c4201e2ee48a70a12070ae39ca"
+
+const randBZ2Hex = "425a6839314159265359905d990d0001957fffffffffffafffffffffffffffffbfff6fffdfffffffffffffffffffffffffffffc002b6dd75676ed5b77720098320d11a64626981323d4da47a83131a13d09e8040f534cd4f4d27a464d193008cd09804601347a980026350c9886234d36864193d1351b44c136919e90340d26127a4cd264c32023009898981310c0344c340027a8303427a99a04c00003534c230d034f5006468d268cf54d36a3009a69a62626261311b40026013d34201a6934c9a604c98ca6c8460989fa9346234d30d3469a2604fd4131a7aa6d0046043d4c62098479269e89e835190d018d4c046001a11e801a0264792321932308c43a130688c260d46686804cd01a9e80981193684c6a68c00000004c4c20c04627a4c0000260003400d04c0681a01334026009a6f48041466132581ec5212b081d96b0effc16543e2228b052fcd30f2567ee8d970e0f10aabca68dd8270591c376cfc1baae0dba00aaff2d6caf6b211322c997cc18eaee5927f75185336bf907021324c71626c1dd20e22b9b0977f05d0f901eaa51db9fbaf7c603b4c87bc82890e6dd7e61d0079e27ec050dd788fd958152061cd01e222f9547cb9efc465d775b6fc98bac7d387bffd151ae09dadf19494f7a638e2eae58e550faba5fe6820ea520eb986096de4e527d80def3ba625e71fbefdcf7e7844e0a25d29b52dcd1344fca083737d42692aab38d230485f3c8ed54c2ed31f15cf0270c8143765b10b92157233fa1dfe0d7ce8ffe70b8b8f7250071701dfe9f1c94de362c9031455951c93eb098a6b50ee45c6131fefc3b6f9643e21f4adc59497138e246f5c57d834aa67c4f10d8bd8b3908d8130dd7388409c299a268eab3664fa4907c5c31574874bd8d388a4ab22b339660804e53e1b8d05867d40e3082560608d35d5d2c6054e8bab23da28f61f83efd41d25529ad6ea15fb50505cacfabb0902166427354ca3830a2c8415f21b19e592690fbe447020d685a4bcd16ecc4ff1a1c0e572627d0ef6265c008a43fc243240541061ed7840606be466d1c0dac2c53250ed567507d926c844154560d631960c65e15157829b2c7f16859f111a3a8cb72bf24ffa57a680c3be67b1be67c8dd8aea73ac2437a78df5b686d427080ebc01bd30b71a49f6ea31dc0f08e4849e38face96717690239538bc08b6cc5aa8d467cb9c36aa83d40ac7e58bddbfa185b22065e89a86c0145569d9e23726651aec49e31588d70f40fe9a4449dcf4f89eac220171e9c938e803dc195679651004b79ad33cc0c13aeeba5941b33ffeeb8fbe16e76c7811445c67b4269c90479433ddf9e8ed1d00c166b6c17217fb22c3ef1b0c1c7e28e185446a111c37f1ea6c07a59fbcc6546ecc6968d36ba58bc5489a5640647e426b0c39350cb6f07d5dc7a717648c4ec7f841467597ae1f65f408fd2d9940a4b1b860b3c9ae351dcae0b4425f7e8538710f2e40b7f70d13b51ac05ccc6ecda8264a88cad2d721d18132a9b9110a9e759c2483c77dcefc7e464ec88588174cb0c9abff93230ea0bed8decdd8ed8bfe2b5df0a253803678df04fab44c03b9ab7cc97d6e6d6fd0c4c840ce0efc498436f453bbb181603459471f2b588724592b222ec990614db530e10cadd84705621cfdd9261fa44a5f5806a2d74b575056b3c915255c65678f9c16e6dc00a99180fef1a840aff0e842ac02731080cc92782538360a60a727991013984da4fad95f79d5030677b7528d076b2483685fca4429edf804682fdc110dfc2f7c30e23e20a72e039108a0ad6fdee2f76985a4b4be4f5afc6101bf9d5042b657a05dc914e1424241766434"
+const randHex = "c95138082bdf2b9bfa5b1072b23f729735d42c785eeb94320fb14c265b9c2ca421d01a3db986df1ac2acde5a0e6bf955d6f95e61261540905928e195f1a66644cc7f37281744fff4dc6df35566a494c41a8167151950eb74f5fc45f85ad0e5ed28b49adfe218aa7ec1707e8e1d55825f61f72beda3b4c006b8c9188d7336a5d875329b1b58c27cc4e89ecbae02c7712400c39dd131d2c6de82e2863da51d472bdfb21ecce62cc9cf769ed28aedc7583d755da45a0d90874bda269dd53283a9bdfd05f95fc8e9a304bb338ea1a2111894678c18134f17d31a15d9bfc1237894650f3e715e2548639ecbddb845cfe4a46a7b3a3c540f48629488e8c869f1e9f3f4c552243a8105b20eb8e264994214349dae83b165fd6c2a5b8e83fce09fc0a80d3281c8d53a9a08095bd19cbc1388df23975646ed259e003d39261ee68cbece8bcf32971f7fe7e588e8ba8f5e8597909abaea693836a79a1964050ed910a45a0f13a58cd2d3ae18992c5b23082407fd920d0bf01e33118a017bb5e39f44931346845af52128f7965206759433a346034ea481671f501280067567619f5ecef6cded077f92ed7f3b3ce8e308c80f34ba06939e9303f91b4318c8c1dd4cc223c1f057ac0c91211c629cd30e46ee9ec1d9fd493086b7bc2bc83e33f08749a5d430b0ed4f79d70f481940c9b0930b16321886a0df4fa5a1465d5208c7d3494a7987d9a5e42aa256f0c9523947f8318d0ef0af3d59a45cfc2418d0785c9a548b32b81e7de18be7d55a69a4c156bbb3d7579c0ac8e9c72b24646e54b0d0e8725f8f49fb44ae3c6b9d0287be118586255a90a4a83483ed0328518037e52aa959c5748ed83e13023e532306be98b8288da306bbb040bcf5d92176f84a9306dc6b274b040370b61d71fde58dd6d20e6fee348eae0c54bd0a5a487b2d005f329794f2a902c296af0a4c1f638f63292a1fa18e006c1b1838636f4de71c73635b25660d32e88a0917e1a5677f6a02ca65585b82cbd99fb4badbfa97a585da1e6cadf6737b4ec6ca33f245d66ee6a9fae6785d69b003c17b9fc6ec34fe5824ab8caae5e8e14dc6f9e116e7bf4a60c04388783c8ae929e1b46b3ef3bbe81b38f2fa6da771bf39dfba2374d3d2ed356b8e2c42081d885a91a3afb2f31986d2f9873354c48cf5448492c32e62385af423aa4f83db6d1b2669650379a1134b0a04cbca0862d6f9743c791cbb527d36cd5d1f0fc7f503831c8bd1b7a0ef8ae1a5ed1155dfdd9e32b6bb33138112d3d476b802179cb85a2a6c354ccfed2f31604fbd8d6ec4baf9f1c8454f72c6588c06a7df3178c43a6970bfa02dd6f74cb5ec3b63f9eddaa17db5cbf27fac6de8e57c384afd0954179f7b5690c3bee42abc4fa79b4b12101a9cf5f0b9aecdda945def0bd04163237247d3539850e123fe18139f316fa0256d5bd2faa8"
+
+const oneMBSawtoothBZ2Hex = "425a683931415926535971931ea00006ddffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe007de00000000000000024c00130001300000000000000000000000000000000000000000000000000000000126000980009800000000000000000000000000000000000000000000000000000000930004c0004c000000000000000000000000000000000000000000000000000000004980026000260000000000000000000000000000000000000000000000000000000009aaaaa0000000000000000000000000000000000000000000000000000000000000000498002600026000000000000000000000000000000000000000000000000000000007fc42271980d044c0a822607411304a08982d044c1a82260f411308a08984d044c2a82261741130ca08986d044c3a82261f411310a08988d044c4a822627411314a0898ad044c5a82262f411318a0898cd044c6a82263741131ca0898ed044c7a82263f411320a08990d044c8a822647411324a08992d044c9a82264f411328a08994d044caa82265741132ca08996d044cba82265f411330a08998d044cca822667411334a0899ad044cda82266f411338a0899cd044cea82267741133ca0899ed044cfa82267f411340a089a0d044d0a822687411344a089a2d044d1a82268f411348a089a4d044d2a82269741134ca089a6d044d3a82269f411350a089a8d044d4a8226a7411354a089aad044d5a8226af411358a089acd044d6a8226b741135ca089aed044d7a8226bf411360a089b0d044d8a8226c7411364a089b2d044d9a8226cf411368a089b4d044daa8226d741136ca089b6d044dba8226df411370a089b8d044dca8226e7411374a089bad044dda8226ef411378a089bcd044dea8226f741137ca089bed044dfa8226ff411380a089c0d044e0a822707411384a089c2d044e1a82270f411388a089c4d044e2a82271741138ca089c59089c69089c71089c79089c81089c89089c91089c99089ca1089ca9089cb1089cb9089cc1089cc9089cd1089cd9089ce1089ce9089cf1089cf9089d01089d09089d11089d19089d21089d29089d31089d39089d41089d49089d51089d59089d61089d69089d71089d79089d81089d89089d91089d99089da1089da9089db1089db9089dc1089dc9089dd1089dd9089de1089de9089df1089df9089e01089e09089e11089e19089e21089e29089e31089e39089e41089e49089e51089e59089e61089e69089e71089e79089e81089e89089e91089e99089ea1089ea9089eb1089eb9089ec1089ec9089ed1089ed9089ee1089ee9089ef1089ef9089f01089f09089f11089f19089f21089f29089f31089f39089f41089f49089f51089f59089f61089f69089f71089f79089f81089f89089f91089f99089fa1089fa9089fb1089fb9089fc1089fc9089fd1089fd9089fe1089fe9089ff1089ff98a0ac9329acf23ba884804fdd3ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff0034f800000000000024c00130001300000000000000000000000000000000000000000000000000000000126000980009800000000000000000000000000000000000000000000000000000000930004c0004c000000000000000000000000000000000000000000000000000000004980026000260000000000000000000000000000000000000000000000000000000024c0013000130000000000000000000000000000000000000000000000000000000002955540000000000000000000000000000000000000000000000000000000000000001ff108c00846024230221181908c108460a4230621183908c20846124230a21185908c308461a4230e21187908c40846224231221189908c508462a423162118b908c60846324231a2118d908c708463a4231e2118f908c80846424232221191908c908464a4232621193908ca0846524232a21195908cb08465a4232e21197908cc0846624233221199908cd08466a423362119b908ce0846724233a2119d908cf08467a4233e2119f908d008468242342211a1908d108468a42346211a3908d20846924234a211a5908d308469a4234e211a7908d40846a242352211a9908d50846aa42356211ab908d60846b24235a211ad908d70846ba4235e211af908d80846c242362211b1908d90846ca42366211b3908da0846d24236a211b5908db0846da4236e211b7908dc0846e242372211b9908dd0846ea42376211bb908de0846f24237a211bd908df0846fa4237e211bf908e008470242382211c1908e108470a42386211c3908e20847124238a211c5908e2f8c211c6c8471d211c7c84721211c8c84725211c9c84729211cac8472d211cbc84731211ccc84735211cdc84739211cec8473d211cfc84741211d0c84745211d1c84749211d2c8474d211d3c84751211d4c84755211d5c84759211d6c8475d211d7c84761211d8c84765211d9c84769211dac8476d211dbc84771211dcc84775211ddc84779211dec8477d211dfc84781211e0c84785211e1c84789211e2c8478d211e3c84791211e4c84795211e5c84799211e6c8479d211e7c847a1211e8c847a5211e9c847a9211eac847ad211ebc847b1211ecc847b5211edc847b9211eec847bd211efc847c1211f0c847c5211f1c847c9211f2c847cd211f3c847d1211f4c847d5211f5c847d9211f6c847dd211f7c847e1211f8c847e5211f9c847e9211fac847ed211fbc847f1211fcc847f5211fdc847f9211fec847fd211ff8bb9229c284803a8b6248"
+
+const rand2BZ2Hex = "425a6839314159265359d992d0f60000137dfe84020310091c1e280e100e042801099210094806c0110002e70806402000546034000034000000f2830000032000d3403264049270eb7a9280d308ca06ad28f6981bee1bf8160727c7364510d73a1e123083421b63f031f63993a0f40051fbf177245385090d992d0f60"
+const rand2Hex = "92d5652616ac444a4a04af1a8a3964aca0450d43d6cf233bd03233f4ba92f8719e6c2a2bd4f5f88db07ecd0da3a33b263483db9b2c158786ad6363be35d17335ba"
+
+const rand3BZ2Hex = "425a68393141592653593be669d00000327ffffffffffffffffffffffffffffffffffff7ffffffffffffffffffffffffffffffc002b3b2b1b6e2bae400004c00132300004c0d268c004c08c0130026001a008683234c0684c34008c230261a04c0260064d07a8d00034000d27a1268c9931a8d327a3427a41faa69ea0da264c1a34219326869b51b49a6469a3268c689fa53269a62794687a9a68f5189994c9e487a8f534fd49a3d34043629e8c93d04da4f4648d30d4f44d3234c4d3023d0840680984d309934c234d3131a000640984f536a6132601300130130c8d00d04d1841ea7a8d31a02609b40023460010c01a34d4c1a0d04d3069306810034d0d0d4c0046130d034d0131a9a64d321804c68003400098344c13000991808c0001a00000000098004d3d4da4604c47a13012140aadf8d673c922c607ef6212a8c0403adea4b28aee578900e653b9cdeb8d11e6b838815f3ebaad5a01c5408d84a332170aff8734d4e06612d3c2889f31925fb89e33561f5100ae89b1f7047102e729373d3667e58d73aaa80fa7be368a1cc2dadd81d81ec8e1b504bd772ca31d03649269b01ceddaca07bf3d4eba24de141be3f86f93601e03714c0f64654671684f9f9528626fd4e1b76753dc0c54b842486b8d59d8ab314e86ca818e7a1f079463cbbd70d9b79b283c7edc419406311022e4be98c2c1374df9cdde2d008ce1d00e5f06ad1024baf555631f70831fc1023034e62be7c4bcb648caf276963ffa20e96bb50377fe1c113da0db4625b50741c35a058edb009c6ee5dbf93b8a6b060eec568180e8db791b82aab96cbf4326ca98361461379425ba8dcc347be670bdba7641883e5526ae3d833f6e9cb9bac9557747c79e206151072f7f0071dff3880411846f66bf4075c7462f302b53cb3400a74cf35652ad5641ed33572fd54e7ed7f85f58a0acba89327e7c6be5c58cb71528b99df2431f1d0358f8d28d81d95292da631fb06701decabb205fac59ff0fb1df536afc681eece6ea658c4d9eaa45f1342aa1ff70bdaff2ddaf25ec88c22f12829a0553db1ec2505554cb17d7b282e213a5a2aa30431ded2bce665bb199d023840832fedb2c0c350a27291407ff77440792872137df281592e82076a05c64c345ffb058c64f7f7c207ef78420b7010520610f17e302cc4dfcfaef72a0ed091aab4b541eb0531bbe941ca2f792bf7b31ca6162882b68054a8470115bc2c19f2df2023f7800432b39b04d3a304e8085ba3f1f0ca5b1ba4d38d339e6084de979cdea6d0e244c6c9fa0366bd890621e3d30846f5e8497e21597b8f29bbf52c961a485dfbea647600da0fc1f25ce4d203a8352ece310c39073525044e7ac46acf2ed9120bae1b4f6f02364abfe343f80b290983160c103557af1c68416480d024cc31b6c06cfec011456f1e95c420a12b48b1c3fe220c2879a982fb099948ac440db844b9a112a5188c7783fd3b19593290785f908d95c9db4b280bafe89c1313aeec24772046d9bc089645f0d182a21184e143823c5f52de50e5d7e98d3d7ab56f5413bbccd1415c9bcff707def475b643fb7f29842582104d4cc1dbaaca8f10a2f44273c339e0984f2b1e06ab2f0771db01fafa8142298345f3196f23e5847bda024034b6f59b11c29e981c881456e40d211929fd4f766200258aad8212016322bd5c605790dcfdf1bd2a93d99c9b8f498722d311d7eae7ff420496a31804c55f4759a7b13aaaf5f7ce006c3a8a998897d5e0a504398c2b627852545baf440798bcc5cc049357cf3f17d9771e4528a1af3d77dc794a11346e1bdf5efe37a405b127b4c43b616d61fbc5dc914e14240ef99a7400"
+const rand3Hex = "1744b384d68c042371244e13500d4bfb98c6244e3d71a5b700224420b59c593553f33bd786e3d0ce31626f511bc985f59d1a88aa38ba8ad6218d306abee60dd9172540232b95be1af146c69e72e5fde667a090dc3f93bdc5c5af0ab80acdbaa7a505f628c59dc0247b31a439cacf5010a94376d71521df08c178b02fb96fdb1809144ea38c68536187c53201fea8631fb0a880b4451ccdca7cc61f6aafca21cc7449d920599db61789ac3b1e164b3390124f95022aeea39ccca3ec1053f4fa10de2978e2861ea58e477085c2220021a0927aa94c5d0006b5055abba340e4f9eba22e969978dfd18e278a8b89d877328ae34268bc0174cfe211954c0036f078025217d1269fac1932a03b05a0b616012271bbe1fb554171c7a59b196d8a4479f45a77931b5d97aaf6c0c673cbe597b79b96e2a0c1eae2e66e46ccc8c85798e23ffe972ebdaa3f6caea243c004e60321eb47cd79137d78fd0613be606feacc5b3637bdc96a89c13746db8cad886f3ccf912b2178c823bcac395f06d28080269bdca2debf3419c66c690fd1adcfbd53e32e79443d7a42511a84cb22ca94fffad9149275a075b2f8ae0b021dcde9bf62b102db920733b897560518b06e1ad7f4b03458493ddaa7f4fa2c1609f7a1735aeeb1b3e2cea3ab45fc376323cc91873b7e9c90d07c192e38d3f5dfc9bfab1fd821c854da9e607ea596c391c7ec4161c6c4493929a8176badaa5a5af7211c623f29643a937677d3df0da9266181b7c4da5dd40376db677fe8f4a1dc456adf6f33c1e37cec471dd318c2647644fe52f93707a77da7d1702380a80e14cc0fdce7bf2eed48a529090bae0388ee277ce6c7018c5fb00b88362554362205c641f0d0fab94fd5b8357b5ff08b207fee023709bc126ec90cfb17c006754638f8186aaeb1265e80be0c1189ec07d01d5f6f96cb9ce82744147d18490de7dc72862f42f024a16968891a356f5e7e0e695d8c933ba5b5e43ad4c4ade5399bc2cae9bb6189b7870d7f22956194d277f28b10e01c10c6ffe3e065f7e2d6d056aa790db5649ca84dc64c35566c0af1b68c32b5b7874aaa66467afa44f40e9a0846a07ae75360a641dd2acc69d93219b2891f190621511e62a27f5e4fbe641ece1fa234fc7e9a74f48d2a760d82160d9540f649256b169d1fed6fbefdc491126530f3cbad7913e19fbd7aa53b1e243fbf28d5f38c10ebd77c8b986775975cc1d619efb27cdcd733fa1ca36cffe9c0a33cc9f02463c91a886601fd349efee85ef1462065ef9bd2c8f533220ad93138b8382d5938103ab25b2d9af8ae106e1211eb9b18793fba033900c809c02cd6d17e2f3e6fc84dae873411f8e87c3f0a8f1765b7825d185ce3730f299c3028d4a62da9ee95c2b870fb70c79370d485f9d5d9acb78926d20444033d960524d2776dc31988ec7c0dbf23b9905d"
--- /dev/null
+// Copyright 2011 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package bzip2
+
+import (
+ "os"
+ "sort"
+)
+
+// A huffmanTree is a binary tree which is navigated, bit-by-bit to reach a
+// symbol.
+type huffmanTree struct {
+ // nodes contains all the non-leaf nodes in the tree. nodes[0] is the
+ // root of the tree and nextNode contains the index of the next element
+ // of nodes to use when the tree is being constructed.
+ nodes []huffmanNode
+ nextNode int
+}
+
+// A huffmanNode is a node in the tree. left and right contain indexes into the
+// nodes slice of the tree. If left or right is invalidNodeValue then the child
+// is a left node and its value is in leftValue/rightValue.
+//
+// The symbols are uint16s because bzip2 encodes not only MTF indexes in the
+// tree, but also two magic values for run-length encoding and an EOF symbol.
+// Thus there are more than 256 possible symbols.
+type huffmanNode struct {
+ left, right uint16
+ leftValue, rightValue uint16
+}
+
+// invalidNodeValue is an invalid index which marks a leaf node in the tree.
+const invalidNodeValue = 0xffff
+
+// Decode reads bits from the given bitReader and navigates the tree until a
+// symbol is found.
+func (t huffmanTree) Decode(br *bitReader) (v uint16) {
+ nodeIndex := uint16(0) // node 0 is the root of the tree.
+
+ for {
+ node := &t.nodes[nodeIndex]
+ bit := br.ReadBit()
+ // bzip2 encodes left as a true bit.
+ if bit {
+ // left
+ if node.left == invalidNodeValue {
+ return node.leftValue
+ }
+ nodeIndex = node.left
+ } else {
+ // right
+ if node.right == invalidNodeValue {
+ return node.rightValue
+ }
+ nodeIndex = node.right
+ }
+ }
+
+ panic("unreachable")
+}
+
+// newHuffmanTree builds a Huffman tree from a slice containing the code
+// lengths of each symbol. The maximum code length is 32 bits.
+func newHuffmanTree(lengths []uint8) (huffmanTree, os.Error) {
+ // There are many possible trees that assign the same code length to
+ // each symbol (consider reflecting a tree down the middle, for
+ // example). Since the code length assignments determine the
+ // efficiency of the tree, each of these trees is equally good. In
+ // order to minimise the amount of information needed to build a tree
+ // bzip2 uses a canonical tree so that it can be reconstructed given
+ // only the code length assignments.
+
+ if len(lengths) < 2 {
+ panic("newHuffmanTree: too few symbols")
+ }
+
+ var t huffmanTree
+
+ // First we sort the code length assignments by ascending code length,
+ // using the symbol value to break ties.
+ pairs := huffmanSymbolLengthPairs(make([]huffmanSymbolLengthPair, len(lengths)))
+ for i, length := range lengths {
+ pairs[i].value = uint16(i)
+ pairs[i].length = length
+ }
+
+ sort.Sort(pairs)
+
+ // Now we assign codes to the symbols, starting with the longest code.
+ // We keep the codes packed into a uint32, at the most-significant end.
+ // So branches are taken from the MSB downwards. This makes it easy to
+ // sort them later.
+ code := uint32(0)
+ length := uint8(32)
+
+ codes := huffmanCodes(make([]huffmanCode, len(lengths)))
+ for i := len(pairs) - 1; i >= 0; i-- {
+ if length > pairs[i].length {
+ // If the code length decreases we shift in order to
+ // zero any bits beyond the end of the code.
+ length >>= 32 - pairs[i].length
+ length <<= 32 - pairs[i].length
+ length = pairs[i].length
+ }
+ codes[i].code = code
+ codes[i].codeLen = length
+ codes[i].value = pairs[i].value
+ // We need to 'increment' the code, which means treating |code|
+ // like a |length| bit number.
+ code += 1 << (32 - length)
+ }
+
+ // Now we can sort by the code so that the left half of each branch are
+ // grouped together, recursively.
+ sort.Sort(codes)
+
+ t.nodes = make([]huffmanNode, len(codes))
+ _, err := buildHuffmanNode(&t, codes, 0)
+ return t, err
+}
+
+// huffmanSymbolLengthPair contains a symbol and its code length.
+type huffmanSymbolLengthPair struct {
+ value uint16
+ length uint8
+}
+
+// huffmanSymbolLengthPair is used to provide an interface for sorting.
+type huffmanSymbolLengthPairs []huffmanSymbolLengthPair
+
+func (h huffmanSymbolLengthPairs) Len() int {
+ return len(h)
+}
+
+func (h huffmanSymbolLengthPairs) Less(i, j int) bool {
+ if h[i].length < h[j].length {
+ return true
+ }
+ if h[i].length > h[j].length {
+ return false
+ }
+ if h[i].value < h[j].value {
+ return true
+ }
+ return false
+}
+
+func (h huffmanSymbolLengthPairs) Swap(i, j int) {
+ h[i], h[j] = h[j], h[i]
+}
+
+// huffmanCode contains a symbol, its code and code length.
+type huffmanCode struct {
+ code uint32
+ codeLen uint8
+ value uint16
+}
+
+// huffmanCodes is used to provide an interface for sorting.
+type huffmanCodes []huffmanCode
+
+func (n huffmanCodes) Len() int {
+ return len(n)
+}
+
+func (n huffmanCodes) Less(i, j int) bool {
+ return n[i].code < n[j].code
+}
+
+func (n huffmanCodes) Swap(i, j int) {
+ n[i], n[j] = n[j], n[i]
+}
+
+// buildHuffmanNode takes a slice of sorted huffmanCodes and builds a node in
+// the Huffman tree at the given level. It returns the index of the newly
+// constructed node.
+func buildHuffmanNode(t *huffmanTree, codes []huffmanCode, level uint32) (nodeIndex uint16, err os.Error) {
+ test := uint32(1) << (31 - level)
+
+ // We have to search the list of codes to find the divide between the left and right sides.
+ firstRightIndex := len(codes)
+ for i, code := range codes {
+ if code.code&test != 0 {
+ firstRightIndex = i
+ break
+ }
+ }
+
+ left := codes[:firstRightIndex]
+ right := codes[firstRightIndex:]
+
+ if len(left) == 0 || len(right) == 0 {
+ return 0, StructuralError("superfluous level in Huffman tree")
+ }
+
+ nodeIndex = uint16(t.nextNode)
+ node := &t.nodes[t.nextNode]
+ t.nextNode++
+
+ if len(left) == 1 {
+ // leaf node
+ node.left = invalidNodeValue
+ node.leftValue = left[0].value
+ } else {
+ node.left, err = buildHuffmanNode(t, left, level+1)
+ }
+
+ if err != nil {
+ return
+ }
+
+ if len(right) == 1 {
+ // leaf node
+ node.right = invalidNodeValue
+ node.rightValue = right[0].value
+ } else {
+ node.right, err = buildHuffmanNode(t, right, level+1)
+ }
+
+ return
+}
--- /dev/null
+// Copyright 2011 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package bzip2
+
+// moveToFrontDecoder implements a move-to-front list. Such a list is an
+// efficient way to transform a string with repeating elements into one with
+// many small valued numbers, which is suitable for entropy encoding. It works
+// by starting with an initial list of symbols and references symbols by their
+// index into that list. When a symbol is referenced, it's moved to the front
+// of the list. Thus, a repeated symbol ends up being encoded with many zeros,
+// as the symbol will be at the front of the list after the first access.
+type moveToFrontDecoder struct {
+ // Rather than actually keep the list in memory, the symbols are stored
+ // as a circular, double linked list with the symbol indexed by head
+ // at the front of the list.
+ symbols []byte
+ next []uint8
+ prev []uint8
+ head uint8
+}
+
+// newMTFDecoder creates a move-to-front decoder with an explicit initial list
+// of symbols.
+func newMTFDecoder(symbols []byte) *moveToFrontDecoder {
+ if len(symbols) > 256 {
+ panic("too many symbols")
+ }
+
+ m := &moveToFrontDecoder{
+ symbols: symbols,
+ next: make([]uint8, len(symbols)),
+ prev: make([]uint8, len(symbols)),
+ }
+
+ m.threadLinkedList()
+ return m
+}
+
+// newMTFDecoderWithRange creates a move-to-front decoder with an initial
+// symbol list of 0...n-1.
+func newMTFDecoderWithRange(n int) *moveToFrontDecoder {
+ if n > 256 {
+ panic("newMTFDecoderWithRange: cannot have > 256 symbols")
+ }
+
+ m := &moveToFrontDecoder{
+ symbols: make([]uint8, n),
+ next: make([]uint8, n),
+ prev: make([]uint8, n),
+ }
+
+ for i := 0; i < n; i++ {
+ m.symbols[i] = byte(i)
+ }
+
+ m.threadLinkedList()
+ return m
+}
+
+// threadLinkedList creates the initial linked-list pointers.
+func (m *moveToFrontDecoder) threadLinkedList() {
+ if len(m.symbols) == 0 {
+ return
+ }
+
+ m.prev[0] = uint8(len(m.symbols) - 1)
+
+ for i := 0; i < len(m.symbols)-1; i++ {
+ m.next[i] = uint8(i + 1)
+ m.prev[i+1] = uint8(i)
+ }
+
+ m.next[len(m.symbols)-1] = 0
+}
+
+func (m *moveToFrontDecoder) Decode(n int) (b byte) {
+ // Most of the time, n will be zero so it's worth dealing with this
+ // simple case.
+ if n == 0 {
+ return m.symbols[m.head]
+ }
+
+ i := m.head
+ for j := 0; j < n; j++ {
+ i = m.next[i]
+ }
+ b = m.symbols[i]
+
+ m.next[m.prev[i]] = m.next[i]
+ m.prev[m.next[i]] = m.prev[i]
+ m.next[i] = m.head
+ m.prev[i] = m.prev[m.head]
+ m.next[m.prev[m.head]] = i
+ m.prev[m.head] = i
+ m.head = i
+
+ return
+}
+
+// First returns the symbol at the front of the list.
+func (m *moveToFrontDecoder) First() byte {
+ return m.symbols[m.head]
+}
panic("unreachable")
}
+func (b *syncBuffer) signal() {
+ select {
+ case b.ready <- true:
+ default:
+ }
+}
+
func (b *syncBuffer) Write(p []byte) (n int, err os.Error) {
n, err = b.buf.Write(p)
- _ = b.ready <- true
+ b.signal()
return
}
func (b *syncBuffer) ReadMode() {
b.mu.Unlock()
- _ = b.ready <- true
+ b.signal()
}
func (b *syncBuffer) Close() os.Error {
b.closed = true
- _ = b.ready <- true
+ b.signal()
return nil
}
}
func TestDeflateInflateString(t *testing.T) {
- gold := bytes.NewBufferString(getEdata()).Bytes()
+ gold, err := ioutil.ReadFile("../testdata/e.txt")
+ if err != nil {
+ t.Error(err)
+ }
testToFromWithLevel(t, 1, gold, "2.718281828...")
}
-
-func getEdata() string {
- return "2.718281828459045235360287471352662497757247093699959574966967627724076630353547" +
- "59457138217852516642742746639193200305992181741359662904357290033429526059563073" +
- "81323286279434907632338298807531952510190115738341879307021540891499348841675092" +
- "44761460668082264800168477411853742345442437107539077744992069551702761838606261" +
- "33138458300075204493382656029760673711320070932870912744374704723069697720931014" +
- "16928368190255151086574637721112523897844250569536967707854499699679468644549059" +
- "87931636889230098793127736178215424999229576351482208269895193668033182528869398" +
- "49646510582093923982948879332036250944311730123819706841614039701983767932068328" +
- "23764648042953118023287825098194558153017567173613320698112509961818815930416903" +
- "51598888519345807273866738589422879228499892086805825749279610484198444363463244" +
- "96848756023362482704197862320900216099023530436994184914631409343173814364054625" +
- "31520961836908887070167683964243781405927145635490613031072085103837505101157477" +
- "04171898610687396965521267154688957035035402123407849819334321068170121005627880" +
- "23519303322474501585390473041995777709350366041699732972508868769664035557071622" +
- "68447162560798826517871341951246652010305921236677194325278675398558944896970964" +
- "09754591856956380236370162112047742722836489613422516445078182442352948636372141" +
- "74023889344124796357437026375529444833799801612549227850925778256209262264832627" +
- "79333865664816277251640191059004916449982893150566047258027786318641551956532442" +
- "58698294695930801915298721172556347546396447910145904090586298496791287406870504" +
- "89585867174798546677575732056812884592054133405392200011378630094556068816674001" +
- "69842055804033637953764520304024322566135278369511778838638744396625322498506549" +
- "95886234281899707733276171783928034946501434558897071942586398772754710962953741" +
- "52111513683506275260232648472870392076431005958411661205452970302364725492966693" +
- "81151373227536450988890313602057248176585118063036442812314965507047510254465011" +
- "72721155519486685080036853228183152196003735625279449515828418829478761085263981" +
- "39559900673764829224437528718462457803619298197139914756448826260390338144182326" +
- "25150974827987779964373089970388867782271383605772978824125611907176639465070633" +
- "04527954661855096666185664709711344474016070462621568071748187784437143698821855" +
- "96709591025968620023537185887485696522000503117343920732113908032936344797273559" +
- "55277349071783793421637012050054513263835440001863239914907054797780566978533580" +
- "48966906295119432473099587655236812859041383241160722602998330535370876138939639" +
- "17795745401613722361878936526053815584158718692553860616477983402543512843961294" +
- "60352913325942794904337299085731580290958631382683291477116396337092400316894586" +
- "36060645845925126994655724839186564209752685082307544254599376917041977780085362" +
- "73094171016343490769642372229435236612557250881477922315197477806056967253801718" +
- "07763603462459278778465850656050780844211529697521890874019660906651803516501792" +
- "50461950136658543663271254963990854914420001457476081930221206602433009641270489" +
- "43903971771951806990869986066365832322787093765022601492910115171776359446020232" +
- "49300280401867723910288097866605651183260043688508817157238669842242201024950551" +
- "88169480322100251542649463981287367765892768816359831247788652014117411091360116" +
- "49950766290779436460058519419985601626479076153210387275571269925182756879893027" +
- "61761146162549356495903798045838182323368612016243736569846703785853305275833337" +
- "93990752166069238053369887956513728559388349989470741618155012539706464817194670" +
- "83481972144888987906765037959036696724949925452790337296361626589760394985767413" +
- "97359441023744329709355477982629614591442936451428617158587339746791897571211956" +
- "18738578364475844842355558105002561149239151889309946342841393608038309166281881" +
- "15037152849670597416256282360921680751501777253874025642534708790891372917228286" +
- "11515915683725241630772254406337875931059826760944203261924285317018781772960235" +
- "41306067213604600038966109364709514141718577701418060644363681546444005331608778" +
- "31431744408119494229755993140118886833148328027065538330046932901157441475631399" +
- "97221703804617092894579096271662260740718749975359212756084414737823303270330168" +
- "23719364800217328573493594756433412994302485023573221459784328264142168487872167" +
- "33670106150942434569844018733128101079451272237378861260581656680537143961278887" +
- "32527373890392890506865324138062796025930387727697783792868409325365880733988457" +
- "21874602100531148335132385004782716937621800490479559795929059165547050577751430" +
- "81751126989851884087185640260353055837378324229241856256442550226721559802740126" +
- "17971928047139600689163828665277009752767069777036439260224372841840883251848770" +
- "47263844037953016690546593746161932384036389313136432713768884102681121989127522" +
- "30562567562547017250863497653672886059667527408686274079128565769963137897530346" +
- "60616669804218267724560530660773899624218340859882071864682623215080288286359746" +
- "83965435885668550377313129658797581050121491620765676995065971534476347032085321" +
- "56036748286083786568030730626576334697742956346437167093971930608769634953288468" +
- "33613038829431040800296873869117066666146800015121143442256023874474325250769387" +
- "07777519329994213727721125884360871583483562696166198057252661220679754062106208" +
- "06498829184543953015299820925030054982570433905535701686531205264956148572492573" +
- "86206917403695213533732531666345466588597286659451136441370331393672118569553952" +
- "10845840724432383558606310680696492485123263269951460359603729725319836842336390" +
- "46321367101161928217111502828016044880588023820319814930963695967358327420249882" +
- "45684941273860566491352526706046234450549227581151709314921879592718001940968866" +
- "98683703730220047531433818109270803001720593553052070070607223399946399057131158" +
- "70996357773590271962850611465148375262095653467132900259943976631145459026858989" +
- "79115837093419370441155121920117164880566945938131183843765620627846310490346293" +
- "95002945834116482411496975832601180073169943739350696629571241027323913874175492" +
- "30718624545432220395527352952402459038057445028922468862853365422138157221311632" +
- "88112052146489805180092024719391710555390113943316681515828843687606961102505171" +
- "00739276238555338627255353883096067164466237092264680967125406186950214317621166" +
- "81400975952814939072226011126811531083873176173232352636058381731510345957365382" +
- "23534992935822836851007810884634349983518404451704270189381994243410090575376257" +
- "76757111809008816418331920196262341628816652137471732547772778348877436651882875" +
- "21566857195063719365653903894493664217640031215278702223664636357555035655769488" +
- "86549500270853923617105502131147413744106134445544192101336172996285694899193369" +
- "18472947858072915608851039678195942983318648075608367955149663644896559294818785" +
- "17840387733262470519450504198477420141839477312028158868457072905440575106012852" +
- "58056594703046836344592652552137008068752009593453607316226118728173928074623094" +
- "68536782310609792159936001994623799343421068781349734695924646975250624695861690" +
- "91785739765951993929939955675427146549104568607020990126068187049841780791739240" +
- "71945996323060254707901774527513186809982284730860766536866855516467702911336827" +
- "56310722334672611370549079536583453863719623585631261838715677411873852772292259" +
- "47433737856955384562468010139057278710165129666367644518724656537304024436841408" +
- "14488732957847348490003019477888020460324660842875351848364959195082888323206522" +
- "12810419044804724794929134228495197002260131043006241071797150279343326340799596" +
- "05314460532304885289729176598760166678119379323724538572096075822771784833616135" +
- "82612896226118129455927462767137794487586753657544861407611931125958512655759734" +
- "57301533364263076798544338576171533346232527057200530398828949903425956623297578" +
- "24887350292591668258944568946559926584547626945287805165017206747854178879822768" +
- "06536650641910973434528878338621726156269582654478205672987756426325321594294418" +
- "03994321700009054265076309558846589517170914760743713689331946909098190450129030" +
- "70995662266203031826493657336984195557769637876249188528656866076005660256054457" +
- "11337286840205574416030837052312242587223438854123179481388550075689381124935386" +
- "31863528708379984569261998179452336408742959118074745341955142035172618420084550" +
- "91708456823682008977394558426792142734775608796442792027083121501564063413416171" +
- "66448069815483764491573900121217041547872591998943825364950514771379399147205219" +
- "52907939613762110723849429061635760459623125350606853765142311534966568371511660" +
- "42207963944666211632551577290709784731562782775987881364919512574833287937715714" +
- "59091064841642678309949723674420175862269402159407924480541255360431317992696739" +
- "15754241929660731239376354213923061787675395871143610408940996608947141834069836" +
- "29936753626215452472984642137528910798843813060955526227208375186298370667872244" +
- "30195793793786072107254277289071732854874374355781966511716618330881129120245204" +
- "04868220007234403502544820283425418788465360259150644527165770004452109773558589" +
- "76226554849416217149895323834216001140629507184904277892585527430352213968356790" +
- "18076406042138307308774460170842688272261177180842664333651780002171903449234264" +
- "26629226145600433738386833555534345300426481847398921562708609565062934040526494" +
- "32442614456659212912256488935696550091543064261342526684725949143142393988454324" +
- "86327461842846655985332312210466259890141712103446084271616619001257195870793217" +
- "56969854401339762209674945418540711844643394699016269835160784892451405894094639" +
- "52678073545797003070511636825194877011897640028276484141605872061841852971891540" +
- "19688253289309149665345753571427318482016384644832499037886069008072709327673127" +
- "58196656394114896171683298045513972950668760474091542042842999354102582911350224" +
- "16907694316685742425225090269390348148564513030699251995904363840284292674125734" +
- "22447765584177886171737265462085498294498946787350929581652632072258992368768457" +
- "01782303809656788311228930580914057261086588484587310165815116753332767488701482" +
- "91674197015125597825727074064318086014281490241467804723275976842696339357735429" +
- "30186739439716388611764209004068663398856841681003872389214483176070116684503887" +
- "21236436704331409115573328018297798873659091665961240202177855885487617616198937" +
- "07943800566633648843650891448055710397652146960276625835990519870423001794655367" +
- "9"
-}
--- /dev/null
+// Copyright 2011 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// The lzw package implements the Lempel-Ziv-Welch compressed data format,
+// described in T. A. Welch, ``A Technique for High-Performance Data
+// Compression'', Computer, 17(6) (June 1984), pp 8-19.
+//
+// In particular, it implements LZW as used by the GIF, TIFF and PDF file
+// formats, which means variable-width codes up to 12 bits and the first
+// two non-literal codes are a clear code and an EOF code.
+package lzw
+
+// TODO(nigeltao): check that TIFF and PDF use LZW in the same way as GIF,
+// modulo LSB/MSB packing order.
+
+import (
+ "bufio"
+ "fmt"
+ "io"
+ "os"
+)
+
+// Order specifies the bit ordering in an LZW data stream.
+type Order int
+
+const (
+ // LSB means Least Significant Bits first, as used in the GIF file format.
+ LSB Order = iota
+ // MSB means Most Significant Bits first, as used in the TIFF and PDF
+ // file formats.
+ MSB
+)
+
+// decoder is the state from which the readXxx method converts a byte
+// stream into a code stream.
+type decoder struct {
+ r io.ByteReader
+ bits uint32
+ nBits uint
+ width uint
+}
+
+// readLSB returns the next code for "Least Significant Bits first" data.
+func (d *decoder) readLSB() (uint16, os.Error) {
+ for d.nBits < d.width {
+ x, err := d.r.ReadByte()
+ if err != nil {
+ return 0, err
+ }
+ d.bits |= uint32(x) << d.nBits
+ d.nBits += 8
+ }
+ code := uint16(d.bits & (1<<d.width - 1))
+ d.bits >>= d.width
+ d.nBits -= d.width
+ return code, nil
+}
+
+// readMSB returns the next code for "Most Significant Bits first" data.
+func (d *decoder) readMSB() (uint16, os.Error) {
+ for d.nBits < d.width {
+ x, err := d.r.ReadByte()
+ if err != nil {
+ return 0, err
+ }
+ d.bits |= uint32(x) << (24 - d.nBits)
+ d.nBits += 8
+ }
+ code := uint16(d.bits >> (32 - d.width))
+ d.bits <<= d.width
+ d.nBits -= d.width
+ return code, nil
+}
+
+// decode decompresses bytes from r and writes them to pw.
+// read specifies how to decode bytes into codes.
+// litWidth is the width in bits of literal codes.
+func decode(r io.Reader, read func(*decoder) (uint16, os.Error), litWidth int, pw *io.PipeWriter) {
+ br, ok := r.(io.ByteReader)
+ if !ok {
+ br = bufio.NewReader(r)
+ }
+ pw.CloseWithError(decode1(pw, br, read, uint(litWidth)))
+}
+
+func decode1(pw *io.PipeWriter, r io.ByteReader, read func(*decoder) (uint16, os.Error), litWidth uint) os.Error {
+ const (
+ maxWidth = 12
+ invalidCode = 0xffff
+ )
+ d := decoder{r, 0, 0, 1 + litWidth}
+ w := bufio.NewWriter(pw)
+ // The first 1<<litWidth codes are literal codes.
+ // The next two codes mean clear and EOF.
+ // Other valid codes are in the range [lo, hi] where lo := clear + 2,
+ // with the upper bound incrementing on each code seen.
+ clear := uint16(1) << litWidth
+ eof, hi := clear+1, clear+1
+ // overflow is the code at which hi overflows the code width.
+ overflow := uint16(1) << d.width
+ var (
+ // Each code c in [lo, hi] expands to two or more bytes. For c != hi:
+ // suffix[c] is the last of these bytes.
+ // prefix[c] is the code for all but the last byte.
+ // This code can either be a literal code or another code in [lo, c).
+ // The c == hi case is a special case.
+ suffix [1 << maxWidth]uint8
+ prefix [1 << maxWidth]uint16
+ // buf is a scratch buffer for reconstituting the bytes that a code expands to.
+ // Code suffixes are written right-to-left from the end of the buffer.
+ buf [1 << maxWidth]byte
+ )
+
+ // Loop over the code stream, converting codes into decompressed bytes.
+ last := uint16(invalidCode)
+ for {
+ code, err := read(&d)
+ if err != nil {
+ if err == os.EOF {
+ err = io.ErrUnexpectedEOF
+ }
+ return err
+ }
+ switch {
+ case code < clear:
+ // We have a literal code.
+ if err := w.WriteByte(uint8(code)); err != nil {
+ return err
+ }
+ if last != invalidCode {
+ // Save what the hi code expands to.
+ suffix[hi] = uint8(code)
+ prefix[hi] = last
+ }
+ case code == clear:
+ d.width = 1 + litWidth
+ hi = eof
+ overflow = 1 << d.width
+ last = invalidCode
+ continue
+ case code == eof:
+ return w.Flush()
+ case code <= hi:
+ c, i := code, len(buf)-1
+ if code == hi {
+ // code == hi is a special case which expands to the last expansion
+ // followed by the head of the last expansion. To find the head, we walk
+ // the prefix chain until we find a literal code.
+ c = last
+ for c >= clear {
+ c = prefix[c]
+ }
+ buf[i] = uint8(c)
+ i--
+ c = last
+ }
+ // Copy the suffix chain into buf and then write that to w.
+ for c >= clear {
+ buf[i] = suffix[c]
+ i--
+ c = prefix[c]
+ }
+ buf[i] = uint8(c)
+ if _, err := w.Write(buf[i:]); err != nil {
+ return err
+ }
+ // Save what the hi code expands to.
+ suffix[hi] = uint8(c)
+ prefix[hi] = last
+ default:
+ return os.NewError("lzw: invalid code")
+ }
+ last, hi = code, hi+1
+ if hi == overflow {
+ if d.width == maxWidth {
+ return os.NewError("lzw: missing clear code")
+ }
+ d.width++
+ overflow <<= 1
+ }
+ }
+ panic("unreachable")
+}
+
+// NewReader creates a new io.ReadCloser that satisfies reads by decompressing
+// the data read from r.
+// It is the caller's responsibility to call Close on the ReadCloser when
+// finished reading.
+// The number of bits to use for literal codes, litWidth, must be in the
+// range [2,8] and is typically 8.
+func NewReader(r io.Reader, order Order, litWidth int) io.ReadCloser {
+ pr, pw := io.Pipe()
+ var read func(*decoder) (uint16, os.Error)
+ switch order {
+ case LSB:
+ read = (*decoder).readLSB
+ case MSB:
+ read = (*decoder).readMSB
+ default:
+ pw.CloseWithError(os.NewError("lzw: unknown order"))
+ return pr
+ }
+ if litWidth < 2 || 8 < litWidth {
+ pw.CloseWithError(fmt.Errorf("lzw: litWidth %d out of range", litWidth))
+ return pr
+ }
+ go decode(r, read, litWidth, pw)
+ return pr
+}
--- /dev/null
+// Copyright 2011 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package lzw
+
+import (
+ "bytes"
+ "io"
+ "io/ioutil"
+ "os"
+ "strconv"
+ "strings"
+ "testing"
+)
+
+type lzwTest struct {
+ desc string
+ raw string
+ compressed string
+ err os.Error
+}
+
+var lzwTests = []lzwTest{
+ {
+ "empty;LSB;8",
+ "",
+ "\x01\x01",
+ nil,
+ },
+ {
+ "empty;MSB;8",
+ "",
+ "\x80\x80",
+ nil,
+ },
+ {
+ "tobe;LSB;7",
+ "TOBEORNOTTOBEORTOBEORNOT",
+ "\x54\x4f\x42\x45\x4f\x52\x4e\x4f\x54\x82\x84\x86\x8b\x85\x87\x89\x81",
+ nil,
+ },
+ {
+ "tobe;LSB;8",
+ "TOBEORNOTTOBEORTOBEORNOT",
+ "\x54\x9e\x08\x29\xf2\x44\x8a\x93\x27\x54\x04\x12\x34\xb8\xb0\xe0\xc1\x84\x01\x01",
+ nil,
+ },
+ {
+ "tobe;MSB;7",
+ "TOBEORNOTTOBEORTOBEORNOT",
+ "\x54\x4f\x42\x45\x4f\x52\x4e\x4f\x54\x82\x84\x86\x8b\x85\x87\x89\x81",
+ nil,
+ },
+ {
+ "tobe;MSB;8",
+ "TOBEORNOTTOBEORTOBEORNOT",
+ "\x2a\x13\xc8\x44\x52\x79\x48\x9c\x4f\x2a\x40\xa0\x90\x68\x5c\x16\x0f\x09\x80\x80",
+ nil,
+ },
+ {
+ "tobe-truncated;LSB;8",
+ "TOBEORNOTTOBEORTOBEORNOT",
+ "\x54\x9e\x08\x29\xf2\x44\x8a\x93\x27\x54\x04",
+ io.ErrUnexpectedEOF,
+ },
+ // This example comes from http://en.wikipedia.org/wiki/Graphics_Interchange_Format.
+ {
+ "gif;LSB;8",
+ "\x28\xff\xff\xff\x28\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff",
+ "\x00\x51\xfc\x1b\x28\x70\xa0\xc1\x83\x01\x01",
+ nil,
+ },
+ // This example comes from http://compgroups.net/comp.lang.ruby/Decompressing-LZW-compression-from-PDF-file
+ {
+ "pdf;MSB;8",
+ "-----A---B",
+ "\x80\x0b\x60\x50\x22\x0c\x0c\x85\x01",
+ nil,
+ },
+}
+
+func TestReader(t *testing.T) {
+ b := bytes.NewBuffer(nil)
+ for _, tt := range lzwTests {
+ d := strings.Split(tt.desc, ";", -1)
+ var order Order
+ switch d[1] {
+ case "LSB":
+ order = LSB
+ case "MSB":
+ order = MSB
+ default:
+ t.Errorf("%s: bad order %q", tt.desc, d[1])
+ }
+ litWidth, _ := strconv.Atoi(d[2])
+ rc := NewReader(strings.NewReader(tt.compressed), order, litWidth)
+ defer rc.Close()
+ b.Reset()
+ n, err := io.Copy(b, rc)
+ if err != nil {
+ if err != tt.err {
+ t.Errorf("%s: io.Copy: %v want %v", tt.desc, err, tt.err)
+ }
+ continue
+ }
+ s := b.String()
+ if s != tt.raw {
+ t.Errorf("%s: got %d-byte %q want %d-byte %q", tt.desc, n, s, len(tt.raw), tt.raw)
+ }
+ }
+}
+
+type devNull struct{}
+
+func (devNull) Write(p []byte) (int, os.Error) {
+ return len(p), nil
+}
+
+func BenchmarkDecoder(b *testing.B) {
+ b.StopTimer()
+ buf0, _ := ioutil.ReadFile("../testdata/e.txt")
+ compressed := bytes.NewBuffer(nil)
+ w := NewWriter(compressed, LSB, 8)
+ io.Copy(w, bytes.NewBuffer(buf0))
+ w.Close()
+ buf1 := compressed.Bytes()
+ b.StartTimer()
+ for i := 0; i < b.N; i++ {
+ io.Copy(devNull{}, NewReader(bytes.NewBuffer(buf1), LSB, 8))
+ }
+}
--- /dev/null
+// Copyright 2011 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package lzw
+
+import (
+ "bufio"
+ "fmt"
+ "io"
+ "os"
+)
+
+// A writer is a buffered, flushable writer.
+type writer interface {
+ WriteByte(byte) os.Error
+ Flush() os.Error
+}
+
+// An errWriteCloser is an io.WriteCloser that always returns a given error.
+type errWriteCloser struct {
+ err os.Error
+}
+
+func (e *errWriteCloser) Write([]byte) (int, os.Error) {
+ return 0, e.err
+}
+
+func (e *errWriteCloser) Close() os.Error {
+ return e.err
+}
+
+const (
+ // A code is a 12 bit value, stored as a uint32 when encoding to avoid
+ // type conversions when shifting bits.
+ maxCode = 1<<12 - 1
+ invalidCode = 1<<32 - 1
+ // There are 1<<12 possible codes, which is an upper bound on the number of
+ // valid hash table entries at any given point in time. tableSize is 4x that.
+ tableSize = 4 * 1 << 12
+ tableMask = tableSize - 1
+ // A hash table entry is a uint32. Zero is an invalid entry since the
+ // lower 12 bits of a valid entry must be a non-literal code.
+ invalidEntry = 0
+)
+
+// encoder is LZW compressor.
+type encoder struct {
+ // w is the writer that compressed bytes are written to.
+ w writer
+ // write, bits, nBits and width are the state for converting a code stream
+ // into a byte stream.
+ write func(*encoder, uint32) os.Error
+ bits uint32
+ nBits uint
+ width uint
+ // litWidth is the width in bits of literal codes.
+ litWidth uint
+ // hi is the code implied by the next code emission.
+ // overflow is the code at which hi overflows the code width.
+ hi, overflow uint32
+ // savedCode is the accumulated code at the end of the most recent Write
+ // call. It is equal to invalidCode if there was no such call.
+ savedCode uint32
+ // err is the first error encountered during writing. Closing the encoder
+ // will make any future Write calls return os.EINVAL.
+ err os.Error
+ // table is the hash table from 20-bit keys to 12-bit values. Each table
+ // entry contains key<<12|val and collisions resolve by linear probing.
+ // The keys consist of a 12-bit code prefix and an 8-bit byte suffix.
+ // The values are a 12-bit code.
+ table [tableSize]uint32
+}
+
+// writeLSB writes the code c for "Least Significant Bits first" data.
+func (e *encoder) writeLSB(c uint32) os.Error {
+ e.bits |= c << e.nBits
+ e.nBits += e.width
+ for e.nBits >= 8 {
+ if err := e.w.WriteByte(uint8(e.bits)); err != nil {
+ return err
+ }
+ e.bits >>= 8
+ e.nBits -= 8
+ }
+ return nil
+}
+
+// writeMSB writes the code c for "Most Significant Bits first" data.
+func (e *encoder) writeMSB(c uint32) os.Error {
+ e.bits |= c << (32 - e.width - e.nBits)
+ e.nBits += e.width
+ for e.nBits >= 8 {
+ if err := e.w.WriteByte(uint8(e.bits >> 24)); err != nil {
+ return err
+ }
+ e.bits <<= 8
+ e.nBits -= 8
+ }
+ return nil
+}
+
+// errOutOfCodes is an internal error that means that the encoder has run out
+// of unused codes and a clear code needs to be sent next.
+var errOutOfCodes = os.NewError("lzw: out of codes")
+
+// incHi increments e.hi and checks for both overflow and running out of
+// unused codes. In the latter case, incHi sends a clear code, resets the
+// encoder state and returns errOutOfCodes.
+func (e *encoder) incHi() os.Error {
+ e.hi++
+ if e.hi == e.overflow {
+ e.width++
+ e.overflow <<= 1
+ }
+ if e.hi == maxCode {
+ clear := uint32(1) << e.litWidth
+ if err := e.write(e, clear); err != nil {
+ return err
+ }
+ e.width = uint(e.litWidth) + 1
+ e.hi = clear + 1
+ e.overflow = clear << 1
+ for i := range e.table {
+ e.table[i] = invalidEntry
+ }
+ return errOutOfCodes
+ }
+ return nil
+}
+
+// Write writes a compressed representation of p to e's underlying writer.
+func (e *encoder) Write(p []byte) (int, os.Error) {
+ if e.err != nil {
+ return 0, e.err
+ }
+ if len(p) == 0 {
+ return 0, nil
+ }
+ litMask := uint32(1<<e.litWidth - 1)
+ code := e.savedCode
+ if code == invalidCode {
+ // The first code sent is always a literal code.
+ code, p = uint32(p[0])&litMask, p[1:]
+ }
+loop:
+ for _, x := range p {
+ literal := uint32(x) & litMask
+ key := code<<8 | literal
+ // If there is a hash table hit for this key then we continue the loop
+ // and do not emit a code yet.
+ hash := (key>>12 ^ key) & tableMask
+ for h, t := hash, e.table[hash]; t != invalidEntry; {
+ if key == t>>12 {
+ code = t & maxCode
+ continue loop
+ }
+ h = (h + 1) & tableMask
+ t = e.table[h]
+ }
+ // Otherwise, write the current code, and literal becomes the start of
+ // the next emitted code.
+ if e.err = e.write(e, code); e.err != nil {
+ return 0, e.err
+ }
+ code = literal
+ // Increment e.hi, the next implied code. If we run out of codes, reset
+ // the encoder state (including clearing the hash table) and continue.
+ if err := e.incHi(); err != nil {
+ if err == errOutOfCodes {
+ continue
+ }
+ e.err = err
+ return 0, e.err
+ }
+ // Otherwise, insert key -> e.hi into the map that e.table represents.
+ for {
+ if e.table[hash] == invalidEntry {
+ e.table[hash] = (key << 12) | e.hi
+ break
+ }
+ hash = (hash + 1) & tableMask
+ }
+ }
+ e.savedCode = code
+ return len(p), nil
+}
+
+// Close closes the encoder, flushing any pending output. It does not close or
+// flush e's underlying writer.
+func (e *encoder) Close() os.Error {
+ if e.err != nil {
+ if e.err == os.EINVAL {
+ return nil
+ }
+ return e.err
+ }
+ // Make any future calls to Write return os.EINVAL.
+ e.err = os.EINVAL
+ // Write the savedCode if valid.
+ if e.savedCode != invalidCode {
+ if err := e.write(e, e.savedCode); err != nil {
+ return err
+ }
+ if err := e.incHi(); err != nil && err != errOutOfCodes {
+ return err
+ }
+ }
+ // Write the eof code.
+ eof := uint32(1)<<e.litWidth + 1
+ if err := e.write(e, eof); err != nil {
+ return err
+ }
+ // Write the final bits.
+ if e.nBits > 0 {
+ if e.write == (*encoder).writeMSB {
+ e.bits >>= 24
+ }
+ if err := e.w.WriteByte(uint8(e.bits)); err != nil {
+ return err
+ }
+ }
+ return e.w.Flush()
+}
+
+// NewWriter creates a new io.WriteCloser that satisfies writes by compressing
+// the data and writing it to w.
+// It is the caller's responsibility to call Close on the WriteCloser when
+// finished writing.
+// The number of bits to use for literal codes, litWidth, must be in the
+// range [2,8] and is typically 8.
+func NewWriter(w io.Writer, order Order, litWidth int) io.WriteCloser {
+ var write func(*encoder, uint32) os.Error
+ switch order {
+ case LSB:
+ write = (*encoder).writeLSB
+ case MSB:
+ write = (*encoder).writeMSB
+ default:
+ return &errWriteCloser{os.NewError("lzw: unknown order")}
+ }
+ if litWidth < 2 || 8 < litWidth {
+ return &errWriteCloser{fmt.Errorf("lzw: litWidth %d out of range", litWidth)}
+ }
+ bw, ok := w.(writer)
+ if !ok {
+ bw = bufio.NewWriter(w)
+ }
+ lw := uint(litWidth)
+ return &encoder{
+ w: bw,
+ write: write,
+ width: 1 + lw,
+ litWidth: lw,
+ hi: 1<<lw + 1,
+ overflow: 1 << (lw + 1),
+ savedCode: invalidCode,
+ }
+}
--- /dev/null
+// Copyright 2011 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package lzw
+
+import (
+ "io"
+ "io/ioutil"
+ "os"
+ "testing"
+)
+
+var filenames = []string{
+ "../testdata/e.txt",
+ "../testdata/pi.txt",
+}
+
+// testFile tests that compressing and then decompressing the given file with
+// the given options yields equivalent bytes to the original file.
+func testFile(t *testing.T, fn string, order Order, litWidth int) {
+ // Read the file, as golden output.
+ golden, err := os.Open(fn, os.O_RDONLY, 0400)
+ if err != nil {
+ t.Errorf("%s (order=%d litWidth=%d): %v", fn, order, litWidth, err)
+ return
+ }
+ defer golden.Close()
+
+ // Read the file again, and push it through a pipe that compresses at the write end, and decompresses at the read end.
+ raw, err := os.Open(fn, os.O_RDONLY, 0400)
+ if err != nil {
+ t.Errorf("%s (order=%d litWidth=%d): %v", fn, order, litWidth, err)
+ return
+ }
+
+ piper, pipew := io.Pipe()
+ defer piper.Close()
+ go func() {
+ defer raw.Close()
+ defer pipew.Close()
+ lzww := NewWriter(pipew, order, litWidth)
+ defer lzww.Close()
+ var b [4096]byte
+ for {
+ n, err0 := raw.Read(b[:])
+ if err0 != nil && err0 != os.EOF {
+ t.Errorf("%s (order=%d litWidth=%d): %v", fn, order, litWidth, err0)
+ return
+ }
+ _, err1 := lzww.Write(b[:n])
+ if err1 == os.EPIPE {
+ // Fail, but do not report the error, as some other (presumably reportable) error broke the pipe.
+ return
+ }
+ if err1 != nil {
+ t.Errorf("%s (order=%d litWidth=%d): %v", fn, order, litWidth, err1)
+ return
+ }
+ if err0 == os.EOF {
+ break
+ }
+ }
+ }()
+ lzwr := NewReader(piper, order, litWidth)
+ defer lzwr.Close()
+
+ // Compare the two.
+ b0, err0 := ioutil.ReadAll(golden)
+ b1, err1 := ioutil.ReadAll(lzwr)
+ if err0 != nil {
+ t.Errorf("%s (order=%d litWidth=%d): %v", fn, order, litWidth, err0)
+ return
+ }
+ if err1 != nil {
+ t.Errorf("%s (order=%d litWidth=%d): %v", fn, order, litWidth, err1)
+ return
+ }
+ if len(b0) != len(b1) {
+ t.Errorf("%s (order=%d litWidth=%d): length mismatch %d versus %d", fn, order, litWidth, len(b0), len(b1))
+ return
+ }
+ for i := 0; i < len(b0); i++ {
+ if b0[i] != b1[i] {
+ t.Errorf("%s (order=%d litWidth=%d): mismatch at %d, 0x%02x versus 0x%02x\n", fn, order, litWidth, i, b0[i], b1[i])
+ return
+ }
+ }
+}
+
+func TestWriter(t *testing.T) {
+ for _, filename := range filenames {
+ for _, order := range [...]Order{LSB, MSB} {
+ // The test data "2.71828 etcetera" is ASCII text requiring at least 6 bits.
+ for _, litWidth := range [...]int{6, 7, 8} {
+ testFile(t, filename, order, litWidth)
+ }
+ }
+ }
+}
+
+func BenchmarkEncoder(b *testing.B) {
+ b.StopTimer()
+ buf, _ := ioutil.ReadFile("../testdata/e.txt")
+ b.StartTimer()
+ for i := 0; i < b.N; i++ {
+ w := NewWriter(devNull{}, LSB, 8)
+ w.Write(buf)
+ w.Close()
+ }
+}
--- /dev/null
+2.7182818284590452353602874713526624977572470936999595749669676277240766303535475945713821785251664274274663919320030599218174135966290435729003342952605956307381323286279434907632338298807531952510190115738341879307021540891499348841675092447614606680822648001684774118537423454424371075390777449920695517027618386062613313845830007520449338265602976067371132007093287091274437470472306969772093101416928368190255151086574637721112523897844250569536967707854499699679468644549059879316368892300987931277361782154249992295763514822082698951936680331825288693984964651058209392398294887933203625094431173012381970684161403970198376793206832823764648042953118023287825098194558153017567173613320698112509961818815930416903515988885193458072738667385894228792284998920868058257492796104841984443634632449684875602336248270419786232090021609902353043699418491463140934317381436405462531520961836908887070167683964243781405927145635490613031072085103837505101157477041718986106873969655212671546889570350354021234078498193343210681701210056278802351930332247450158539047304199577770935036604169973297250886876966403555707162268447162560798826517871341951246652010305921236677194325278675398558944896970964097545918569563802363701621120477427228364896134225164450781824423529486363721417402388934412479635743702637552944483379980161254922785092577825620926226483262779333865664816277251640191059004916449982893150566047258027786318641551956532442586982946959308019152987211725563475463964479101459040905862984967912874068705048958586717479854667757573205681288459205413340539220001137863009455606881667400169842055804033637953764520304024322566135278369511778838638744396625322498506549958862342818997077332761717839280349465014345588970719425863987727547109629537415211151368350627526023264847287039207643100595841166120545297030236472549296669381151373227536450988890313602057248176585118063036442812314965507047510254465011727211555194866850800368532281831521960037356252794495158284188294787610852639813955990067376482922443752871846245780361929819713991475644882626039033814418232625150974827987779964373089970388867782271383605772978824125611907176639465070633045279546618550966661856647097113444740160704626215680717481877844371436988218559670959102596862002353718588748569652200050311734392073211390803293634479727355955277349071783793421637012050054513263835440001863239914907054797780566978533580489669062951194324730995876552368128590413832411607226029983305353708761389396391779574540161372236187893652605381558415871869255386061647798340254351284396129460352913325942794904337299085731580290958631382683291477116396337092400316894586360606458459251269946557248391865642097526850823075442545993769170419777800853627309417101634349076964237222943523661255725088147792231519747780605696725380171807763603462459278778465850656050780844211529697521890874019660906651803516501792504619501366585436632712549639908549144200014574760819302212066024330096412704894390397177195180699086998606636583232278709376502260149291011517177635944602023249300280401867723910288097866605651183260043688508817157238669842242201024950551881694803221002515426494639812873677658927688163598312477886520141174110913601164995076629077943646005851941998560162647907615321038727557126992518275687989302761761146162549356495903798045838182323368612016243736569846703785853305275833337939907521660692380533698879565137285593883499894707416181550125397064648171946708348197214488898790676503795903669672494992545279033729636162658976039498576741397359441023744329709355477982629614591442936451428617158587339746791897571211956187385783644758448423555581050025611492391518893099463428413936080383091662818811503715284967059741625628236092168075150177725387402564253470879089137291722828611515915683725241630772254406337875931059826760944203261924285317018781772960235413060672136046000389661093647095141417185777014180606443636815464440053316087783143174440811949422975599314011888683314832802706553833004693290115744147563139997221703804617092894579096271662260740718749975359212756084414737823303270330168237193648002173285734935947564334129943024850235732214597843282641421684878721673367010615094243456984401873312810107945127223737886126058165668053714396127888732527373890392890506865324138062796025930387727697783792868409325365880733988457218746021005311483351323850047827169376218004904795597959290591655470505777514308175112698985188408718564026035305583737832422924185625644255022672155980274012617971928047139600689163828665277009752767069777036439260224372841840883251848770472638440379530166905465937461619323840363893131364327137688841026811219891275223056256756254701725086349765367288605966752740868627407912856576996313789753034660616669804218267724560530660773899624218340859882071864682623215080288286359746839654358856685503773131296587975810501214916207656769950659715344763470320853215603674828608378656803073062657633469774295634643716709397193060876963495328846833613038829431040800296873869117066666146800015121143442256023874474325250769387077775193299942137277211258843608715834835626961661980572526612206797540621062080649882918454395301529982092503005498257043390553570168653120526495614857249257386206917403695213533732531666345466588597286659451136441370331393672118569553952108458407244323835586063106806964924851232632699514603596037297253198368423363904632136710116192821711150282801604488058802382031981493096369596735832742024988245684941273860566491352526706046234450549227581151709314921879592718001940968866986837037302200475314338181092708030017205935530520700706072233999463990571311587099635777359027196285061146514837526209565346713290025994397663114545902685898979115837093419370441155121920117164880566945938131183843765620627846310490346293950029458341164824114969758326011800731699437393506966295712410273239138741754923071862454543222039552735295240245903805744502892246886285336542213815722131163288112052146489805180092024719391710555390113943316681515828843687606961102505171007392762385553386272553538830960671644662370922646809671254061869502143176211668140097595281493907222601112681153108387317617323235263605838173151034595736538223534992935822836851007810884634349983518404451704270189381994243410090575376257767571118090088164183319201962623416288166521374717325477727783488774366518828752156685719506371936565390389449366421764003121527870222366463635755503565576948886549500270853923617105502131147413744106134445544192101336172996285694899193369184729478580729156088510396781959429833186480756083679551496636448965592948187851784038773326247051945050419847742014183947731202815886845707290544057510601285258056594703046836344592652552137008068752009593453607316226118728173928074623094685367823106097921599360019946237993434210687813497346959246469752506246958616909178573976595199392993995567542714654910456860702099012606818704984178079173924071945996323060254707901774527513186809982284730860766536866855516467702911336827563107223346726113705490795365834538637196235856312618387156774118738527722922594743373785695538456246801013905727871016512966636764451872465653730402443684140814488732957847348490003019477888020460324660842875351848364959195082888323206522128104190448047247949291342284951970022601310430062410717971502793433263407995960531446053230488528972917659876016667811937932372453857209607582277178483361613582612896226118129455927462767137794487586753657544861407611931125958512655759734573015333642630767985443385761715333462325270572005303988289499034259566232975782488735029259166825894456894655992658454762694528780516501720674785417887982276806536650641910973434528878338621726156269582654478205672987756426325321594294418039943217000090542650763095588465895171709147607437136893319469090981904501290307099566226620303182649365733698419555776963787624918852865686607600566025605445711337286840205574416030837052312242587223438854123179481388550075689381124935386318635287083799845692619981794523364087429591180747453419551420351726184200845509170845682368200897739455842679214273477560879644279202708312150156406341341617166448069815483764491573900121217041547872591998943825364950514771379399147205219529079396137621107238494290616357604596231253506068537651423115349665683715116604220796394466621163255157729070978473156278277598788136491951257483328793771571459091064841642678309949723674420175862269402159407924480541255360431317992696739157542419296607312393763542139230617876753958711436104089409966089471418340698362993675362621545247298464213752891079884381306095552622720837518629837066787224430195793793786072107254277289071732854874374355781966511716618330881129120245204048682200072344035025448202834254187884653602591506445271657700044521097735585897622655484941621714989532383421600114062950718490427789258552743035221396835679018076406042138307308774460170842688272261177180842664333651780002171903449234264266292261456004337383868335555343453004264818473989215627086095650629340405264943244261445665921291225648893569655009154306426134252668472594914314239398845432486327461842846655985332312210466259890141712103446084271616619001257195870793217569698544013397622096749454185407118446433946990162698351607848924514058940946395267807354579700307051163682519487701189764002827648414160587206184185297189154019688253289309149665345753571427318482016384644832499037886069008072709327673127581966563941148961716832980455139729506687604740915420428429993541025829113502241690769431668574242522509026939034814856451303069925199590436384028429267412573422447765584177886171737265462085498294498946787350929581652632072258992368768457017823038096567883112289305809140572610865884845873101658151167533327674887014829167419701512559782572707406431808601428149024146780472327597684269633935773542930186739439716388611764209004068663398856841681003872389214483176070116684503887212364367043314091155733280182977988736590916659612402021778558854876176161989370794380056663364884365089144805571039765214696027662583599051987042300179465536788
--- /dev/null
+3.1415926535897932384626433832795028841971693993751058209749445923078164062862089986280348253421170679821480865132823066470938446095505822317253594081284811174502841027019385211055596446229489549303819644288109756659334461284756482337867831652712019091456485669234603486104543266482133936072602491412737245870066063155881748815209209628292540917153643678925903600113305305488204665213841469519415116094330572703657595919530921861173819326117931051185480744623799627495673518857527248912279381830119491298336733624406566430860213949463952247371907021798609437027705392171762931767523846748184676694051320005681271452635608277857713427577896091736371787214684409012249534301465495853710507922796892589235420199561121290219608640344181598136297747713099605187072113499999983729780499510597317328160963185950244594553469083026425223082533446850352619311881710100031378387528865875332083814206171776691473035982534904287554687311595628638823537875937519577818577805321712268066130019278766111959092164201989380952572010654858632788659361533818279682303019520353018529689957736225994138912497217752834791315155748572424541506959508295331168617278558890750983817546374649393192550604009277016711390098488240128583616035637076601047101819429555961989467678374494482553797747268471040475346462080466842590694912933136770289891521047521620569660240580381501935112533824300355876402474964732639141992726042699227967823547816360093417216412199245863150302861829745557067498385054945885869269956909272107975093029553211653449872027559602364806654991198818347977535663698074265425278625518184175746728909777727938000816470600161452491921732172147723501414419735685481613611573525521334757418494684385233239073941433345477624168625189835694855620992192221842725502542568876717904946016534668049886272327917860857843838279679766814541009538837863609506800642251252051173929848960841284886269456042419652850222106611863067442786220391949450471237137869609563643719172874677646575739624138908658326459958133904780275900994657640789512694683983525957098258226205224894077267194782684826014769909026401363944374553050682034962524517493996514314298091906592509372216964615157098583874105978859597729754989301617539284681382686838689427741559918559252459539594310499725246808459872736446958486538367362226260991246080512438843904512441365497627807977156914359977001296160894416948685558484063534220722258284886481584560285060168427394522674676788952521385225499546667278239864565961163548862305774564980355936345681743241125150760694794510965960940252288797108931456691368672287489405601015033086179286809208747609178249385890097149096759852613655497818931297848216829989487226588048575640142704775551323796414515237462343645428584447952658678210511413547357395231134271661021359695362314429524849371871101457654035902799344037420073105785390621983874478084784896833214457138687519435064302184531910484810053706146806749192781911979399520614196634287544406437451237181921799983910159195618146751426912397489409071864942319615679452080951465502252316038819301420937621378559566389377870830390697920773467221825625996615014215030680384477345492026054146659252014974428507325186660021324340881907104863317346496514539057962685610055081066587969981635747363840525714591028970641401109712062804390397595156771577004203378699360072305587631763594218731251471205329281918261861258673215791984148488291644706095752706957220917567116722910981690915280173506712748583222871835209353965725121083579151369882091444210067510334671103141267111369908658516398315019701651511685171437657618351556508849099898599823873455283316355076479185358932261854896321329330898570642046752590709154814165498594616371802709819943099244889575712828905923233260972997120844335732654893823911932597463667305836041428138830320382490375898524374417029132765618093773444030707469211201913020330380197621101100449293215160842444859637669838952286847831235526582131449576857262433441893039686426243410773226978028073189154411010446823252716201052652272111660396665573092547110557853763466820653109896526918620564769312570586356620185581007293606598764861179104533488503461136576867532494416680396265797877185560845529654126654085306143444318586769751456614068007002378776591344017127494704205622305389945613140711270004078547332699390814546646458807972708266830634328587856983052358089330657574067954571637752542021149557615814002501262285941302164715509792592309907965473761255176567513575178296664547791745011299614890304639947132962107340437518957359614589019389713111790429782856475032031986915140287080859904801094121472213179476477726224142548545403321571853061422881375850430633217518297986622371721591607716692547487389866549494501146540628433663937900397692656721463853067360965712091807638327166416274888800786925602902284721040317211860820419000422966171196377921337575114959501566049631862947265473642523081770367515906735023507283540567040386743513622224771589150495309844489333096340878076932599397805419341447377441842631298608099888687413260472156951623965864573021631598193195167353812974167729478672422924654366800980676928238280689964004824354037014163149658979409243237896907069779422362508221688957383798623001593776471651228935786015881617557829735233446042815126272037343146531977774160319906655418763979293344195215413418994854447345673831624993419131814809277771038638773431772075456545322077709212019051660962804909263601975988281613323166636528619326686336062735676303544776280350450777235547105859548702790814356240145171806246436267945612753181340783303362542327839449753824372058353114771199260638133467768796959703098339130771098704085913374641442822772634659470474587847787201927715280731767907707157213444730605700733492436931138350493163128404251219256517980694113528013147013047816437885185290928545201165839341965621349143415956258658655705526904965209858033850722426482939728584783163057777560688876446248246857926039535277348030480290058760758251047470916439613626760449256274204208320856611906254543372131535958450687724602901618766795240616342522577195429162991930645537799140373404328752628889639958794757291746426357455254079091451357111369410911939325191076020825202618798531887705842972591677813149699009019211697173727847684726860849003377024242916513005005168323364350389517029893922334517220138128069650117844087451960121228599371623130171144484640903890644954440061986907548516026327505298349187407866808818338510228334508504860825039302133219715518430635455007668282949304137765527939751754613953984683393638304746119966538581538420568533862186725233402830871123282789212507712629463229563989898935821167456270102183564622013496715188190973038119800497340723961036854066431939509790190699639552453005450580685501956730229219139339185680344903982059551002263535361920419947455385938102343955449597783779023742161727111723643435439478221818528624085140066604433258885698670543154706965747458550332323342107301545940516553790686627333799585115625784322988273723198987571415957811196358330059408730681216028764962867446047746491599505497374256269010490377819868359381465741268049256487985561453723478673303904688383436346553794986419270563872931748723320837601123029911367938627089438799362016295154133714248928307220126901475466847653576164773794675200490757155527819653621323926406160136358155907422020203187277605277219005561484255518792530343513984425322341576233610642506390497500865627109535919465897514131034822769306247435363256916078154781811528436679570611086153315044521274739245449454236828860613408414863776700961207151249140430272538607648236341433462351897576645216413767969031495019108575984423919862916421939949072362346468441173940326591840443780513338945257423995082965912285085558215725031071257012668302402929525220118726767562204154205161841634847565169998116141010029960783869092916030288400269104140792886215078424516709087000699282120660418371806535567252532567532861291042487761825829765157959847035622262934860034158722980534989650226291748788202734209222245339856264766914905562842503912757710284027998066365825488926488025456610172967026640765590429099456815065265305371829412703369313785178609040708667114965583434347693385781711386455873678123014587687126603489139095620099393610310291616152881384379099042317473363948045759314931405297634757481193567091101377517210080315590248530906692037671922033229094334676851422144773793937517034436619910403375111735471918550464490263655128162288244625759163330391072253837421821408835086573917715096828874782656995995744906617583441375223970968340800535598491754173818839994469748676265516582765848358845314277568790029095170283529716344562129640435231176006651012412006597558512761785838292041974844236080071930457618932349229279650198751872127267507981255470958904556357921221033346697499235630254947802490114195212382815309114079073860251522742995818072471625916685451333123948049470791191532673430282441860414263639548000448002670496248201792896476697583183271314251702969234889627668440323260927524960357996469256504936818360900323809293459588970695365349406034021665443755890045632882250545255640564482465151875471196218443965825337543885690941130315095261793780029741207665147939425902989695946995565761218656196733786236256125216320862869222103274889218654364802296780705765615144632046927906821207388377814233562823608963208068222468012248261177185896381409183903673672220888321513755600372798394004152970028783076670944474560134556417254370906979396122571429894671543578468788614445812314593571984922528471605049221242470141214780573455105008019086996033027634787081081754501193071412233908663938339529425786905076431006383519834389341596131854347546495569781038293097164651438407007073604112373599843452251610507027056235266012764848308407611830130527932054274628654036036745328651057065874882256981579367897669742205750596834408697350201410206723585020072452256326513410559240190274216248439140359989535394590944070469120914093870012645600162374288021092764579310657922955249887275846101264836999892256959688159205600101655256375678
)
var filenames = []string{
- "testdata/e.txt",
- "testdata/pi.txt",
+ "../testdata/e.txt",
+ "../testdata/pi.txt",
}
// Tests that compressing and then decompressing the given file at the given compression level
}
-func (r *Ring) Iter() <-chan interface{} {
- c := make(chan interface{})
- go func() {
- if r != nil {
- c <- r.Value
- for p := r.Next(); p != r; p = p.next {
- c <- p.Value
- }
+// Do calls function f on each element of the ring, in forward order.
+// The behavior of Do is undefined if f changes *r.
+func (r *Ring) Do(f func(interface{})) {
+ if r != nil {
+ f(r.Value)
+ for p := r.Next(); p != r; p = p.next {
+ f(p.Value)
}
- close(c)
- }()
- return c
+ }
}
// iteration
n = 0
s := 0
- for p := range r.Iter() {
+ r.Do(func(p interface{}) {
n++
if p != nil {
s += p.(int)
}
- }
+ })
if n != N {
t.Errorf("number of forward iterations == %d; expected %d", n, N)
}
return r
}
-
-func sum(r *Ring) int {
- s := 0
- for p := range r.Iter() {
- s += p.(int)
- }
- return s
-}
-
-
func sumN(n int) int { return (n*n + n) / 2 }
outUsed int
}
+// An OCFBResyncOption determines if the "resynchronization step" of OCFB is
+// performed.
+type OCFBResyncOption bool
+
+const (
+ OCFBResync OCFBResyncOption = true
+ OCFBNoResync OCFBResyncOption = false
+)
+
// NewOCFBEncrypter returns a Stream which encrypts data with OpenPGP's cipher
// feedback mode using the given Block, and an initial amount of ciphertext.
// randData must be random bytes and be the same length as the Block's block
-// size.
-func NewOCFBEncrypter(block Block, randData []byte) (Stream, []byte) {
+// size. Resync determines if the "resynchronization step" from RFC 4880, 13.9
+// step 7 is performed. Different parts of OpenPGP vary on this point.
+func NewOCFBEncrypter(block Block, randData []byte, resync OCFBResyncOption) (Stream, []byte) {
blockSize := block.BlockSize()
if len(randData) != blockSize {
return nil, nil
prefix[blockSize] = x.fre[0] ^ randData[blockSize-2]
prefix[blockSize+1] = x.fre[1] ^ randData[blockSize-1]
- block.Encrypt(x.fre, prefix[2:])
+ if resync {
+ block.Encrypt(x.fre, prefix[2:])
+ } else {
+ x.fre[0] = prefix[blockSize]
+ x.fre[1] = prefix[blockSize+1]
+ x.outUsed = 2
+ }
return x, prefix
}
// NewOCFBDecrypter returns a Stream which decrypts data with OpenPGP's cipher
// feedback mode using the given Block. Prefix must be the first blockSize + 2
// bytes of the ciphertext, where blockSize is the Block's block size. If an
-// incorrect key is detected then nil is returned.
-func NewOCFBDecrypter(block Block, prefix []byte) Stream {
+// incorrect key is detected then nil is returned. Resync determines if the
+// "resynchronization step" from RFC 4880, 13.9 step 7 is performed. Different
+// parts of OpenPGP vary on this point.
+func NewOCFBDecrypter(block Block, prefix []byte, resync OCFBResyncOption) Stream {
blockSize := block.BlockSize()
if len(prefix) != blockSize+2 {
return nil
return nil
}
- block.Encrypt(x.fre, prefix[2:])
+ if resync {
+ block.Encrypt(x.fre, prefix[2:])
+ } else {
+ x.fre[0] = prefix[blockSize]
+ x.fre[1] = prefix[blockSize+1]
+ x.outUsed = 2
+ }
return x
}
"testing"
)
-func TestOCFB(t *testing.T) {
+func testOCFB(t *testing.T, resync OCFBResyncOption) {
block, err := aes.NewCipher(commonKey128)
if err != nil {
t.Error(err)
return
}
- plaintext := []byte("this is the plaintext")
+ plaintext := []byte("this is the plaintext, which is long enough to span several blocks.")
randData := make([]byte, block.BlockSize())
rand.Reader.Read(randData)
- ocfb, prefix := NewOCFBEncrypter(block, randData)
+ ocfb, prefix := NewOCFBEncrypter(block, randData, resync)
ciphertext := make([]byte, len(plaintext))
ocfb.XORKeyStream(ciphertext, plaintext)
- ocfbdec := NewOCFBDecrypter(block, prefix)
+ ocfbdec := NewOCFBDecrypter(block, prefix, resync)
if ocfbdec == nil {
- t.Error("NewOCFBDecrypter failed")
+ t.Errorf("NewOCFBDecrypter failed (resync: %t)", resync)
return
}
plaintextCopy := make([]byte, len(plaintext))
ocfbdec.XORKeyStream(plaintextCopy, ciphertext)
if !bytes.Equal(plaintextCopy, plaintext) {
- t.Errorf("got: %x, want: %x", plaintextCopy, plaintext)
+ t.Errorf("got: %x, want: %x (resync: %t)", plaintextCopy, plaintext, resync)
}
}
+
+func TestOCFB(t *testing.T) {
+ testOCFB(t, OCFBNoResync)
+ testOCFB(t, OCFBResync)
+}
--- /dev/null
+// Copyright 2011 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// The crypto package collects common cryptographic constants.
+package crypto
+
+import (
+ "hash"
+)
+
+// Hash identifies a cryptographic hash function that is implemented in another
+// package.
+type Hash uint
+
+const (
+ MD4 Hash = 1 + iota // in package crypto/md4
+ MD5 // in package crypto/md5
+ SHA1 // in package crypto/sha1
+ SHA224 // in package crypto/sha256
+ SHA256 // in package crypto/sha256
+ SHA384 // in package crypto/sha512
+ SHA512 // in package crypto/sha512
+ MD5SHA1 // no implementation; MD5+SHA1 used for TLS RSA
+ RIPEMD160 // in package crypto/ripemd160
+ maxHash
+)
+
+var digestSizes = []uint8{
+ MD4: 16,
+ MD5: 16,
+ SHA1: 20,
+ SHA224: 28,
+ SHA256: 32,
+ SHA384: 48,
+ SHA512: 64,
+ MD5SHA1: 36,
+ RIPEMD160: 20,
+}
+
+// Size returns the length, in bytes, of a digest resulting from the given hash
+// function. It doesn't require that the hash function in question be linked
+// into the program.
+func (h Hash) Size() int {
+ if h > 0 && h < maxHash {
+ return int(digestSizes[h])
+ }
+ panic("crypto: Size of unknown hash function")
+}
+
+var hashes = make([]func() hash.Hash, maxHash)
+
+// New returns a new hash.Hash calculating the given hash function. If the
+// hash function is not linked into the binary, New returns nil.
+func (h Hash) New() hash.Hash {
+ if h > 0 && h < maxHash {
+ f := hashes[h]
+ if f != nil {
+ return f()
+ }
+ }
+ return nil
+}
+
+// RegisterHash registers a function that returns a new instance of the given
+// hash function. This is intended to be called from the init function in
+// packages that implement hash functions.
+func RegisterHash(h Hash, f func() hash.Hash) {
+ if h >= maxHash {
+ panic("crypto: RegisterHash of unknown hash function")
+ }
+ hashes[h] = f
+}
--- /dev/null
+// Copyright 2011 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Package dsa implements the Digital Signature Algorithm, as defined in FIPS 186-3
+package dsa
+
+import (
+ "big"
+ "io"
+ "os"
+)
+
+// Parameters represents the domain parameters for a key. These parameters can
+// be shared across many keys. The bit length of Q must be a multiple of 8.
+type Parameters struct {
+ P, Q, G *big.Int
+}
+
+// PublicKey represents a DSA public key.
+type PublicKey struct {
+ Parameters
+ Y *big.Int
+}
+
+// PrivateKey represents a DSA private key.
+type PrivateKey struct {
+ PublicKey
+ X *big.Int
+}
+
+type invalidPublicKeyError int
+
+func (invalidPublicKeyError) String() string {
+ return "crypto/dsa: invalid public key"
+}
+
+// InvalidPublicKeyError results when a public key is not usable by this code.
+// FIPS is quite strict about the format of DSA keys, but other code may be
+// less so. Thus, when using keys which may have been generated by other code,
+// this error must be handled.
+var InvalidPublicKeyError = invalidPublicKeyError(0)
+
+// ParameterSizes is a enumeration of the acceptable bit lengths of the primes
+// in a set of DSA parameters. See FIPS 186-3, section 4.2.
+type ParameterSizes int
+
+const (
+ L1024N160 ParameterSizes = iota
+ L2048N224
+ L2048N256
+ L3072N256
+)
+
+// numMRTests is the number of Miller-Rabin primality tests that we perform. We
+// pick the largest recommended number from table C.1 of FIPS 186-3.
+const numMRTests = 64
+
+// GenerateParameters puts a random, valid set of DSA parameters into params.
+// This function takes many seconds, even on fast machines.
+func GenerateParameters(params *Parameters, rand io.Reader, sizes ParameterSizes) (err os.Error) {
+ // This function doesn't follow FIPS 186-3 exactly in that it doesn't
+ // use a verification seed to generate the primes. The verification
+ // seed doesn't appear to be exported or used by other code and
+ // omitting it makes the code cleaner.
+
+ var L, N int
+ switch sizes {
+ case L1024N160:
+ L = 1024
+ N = 160
+ case L2048N224:
+ L = 2048
+ N = 224
+ case L2048N256:
+ L = 2048
+ N = 256
+ case L3072N256:
+ L = 3072
+ N = 256
+ default:
+ return os.ErrorString("crypto/dsa: invalid ParameterSizes")
+ }
+
+ qBytes := make([]byte, N/8)
+ pBytes := make([]byte, L/8)
+
+ q := new(big.Int)
+ p := new(big.Int)
+ rem := new(big.Int)
+ one := new(big.Int)
+ one.SetInt64(1)
+
+GeneratePrimes:
+ for {
+ _, err = io.ReadFull(rand, qBytes)
+ if err != nil {
+ return
+ }
+
+ qBytes[len(qBytes)-1] |= 1
+ qBytes[0] |= 0x80
+ q.SetBytes(qBytes)
+
+ if !big.ProbablyPrime(q, numMRTests) {
+ continue
+ }
+
+ for i := 0; i < 4*L; i++ {
+ _, err = io.ReadFull(rand, pBytes)
+ if err != nil {
+ return
+ }
+
+ pBytes[len(pBytes)-1] |= 1
+ pBytes[0] |= 0x80
+
+ p.SetBytes(pBytes)
+ rem.Mod(p, q)
+ rem.Sub(rem, one)
+ p.Sub(p, rem)
+ if p.BitLen() < L {
+ continue
+ }
+
+ if !big.ProbablyPrime(p, numMRTests) {
+ continue
+ }
+
+ params.P = p
+ params.Q = q
+ break GeneratePrimes
+ }
+ }
+
+ h := new(big.Int)
+ h.SetInt64(2)
+ g := new(big.Int)
+
+ pm1 := new(big.Int).Sub(p, one)
+ e := new(big.Int).Div(pm1, q)
+
+ for {
+ g.Exp(h, e, p)
+ if g.Cmp(one) == 0 {
+ h.Add(h, one)
+ continue
+ }
+
+ params.G = g
+ return
+ }
+
+ panic("unreachable")
+}
+
+// GenerateKey generates a public&private key pair. The Parameters of the
+// PrivateKey must already be valid (see GenerateParameters).
+func GenerateKey(priv *PrivateKey, rand io.Reader) os.Error {
+ if priv.P == nil || priv.Q == nil || priv.G == nil {
+ return os.ErrorString("crypto/dsa: parameters not set up before generating key")
+ }
+
+ x := new(big.Int)
+ xBytes := make([]byte, priv.Q.BitLen()/8)
+
+ for {
+ _, err := io.ReadFull(rand, xBytes)
+ if err != nil {
+ return err
+ }
+ x.SetBytes(xBytes)
+ if x.Sign() != 0 && x.Cmp(priv.Q) < 0 {
+ break
+ }
+ }
+
+ priv.X = x
+ priv.Y = new(big.Int)
+ priv.Y.Exp(priv.G, x, priv.P)
+ return nil
+}
+
+// Sign signs an arbitrary length hash (which should be the result of hashing a
+// larger message) using the private key, priv. It returns the signature as a
+// pair of integers. The security of the private key depends on the entropy of
+// rand.
+func Sign(rand io.Reader, priv *PrivateKey, hash []byte) (r, s *big.Int, err os.Error) {
+ // FIPS 186-3, section 4.6
+
+ n := priv.Q.BitLen()
+ if n&7 != 0 {
+ err = InvalidPublicKeyError
+ return
+ }
+ n >>= 3
+
+ for {
+ k := new(big.Int)
+ buf := make([]byte, n)
+ for {
+ _, err = io.ReadFull(rand, buf)
+ if err != nil {
+ return
+ }
+ k.SetBytes(buf)
+ if k.Sign() > 0 && k.Cmp(priv.Q) < 0 {
+ break
+ }
+ }
+
+ kInv := new(big.Int).ModInverse(k, priv.Q)
+
+ r = new(big.Int).Exp(priv.G, k, priv.P)
+ r.Mod(r, priv.Q)
+
+ if r.Sign() == 0 {
+ continue
+ }
+
+ if n > len(hash) {
+ n = len(hash)
+ }
+ z := k.SetBytes(hash[:n])
+
+ s = new(big.Int).Mul(priv.X, r)
+ s.Add(s, z)
+ s.Mod(s, priv.Q)
+ s.Mul(s, kInv)
+ s.Mod(s, priv.Q)
+
+ if s.Sign() != 0 {
+ break
+ }
+ }
+
+ return
+}
+
+// Verify verifies the signature in r, s of hash using the public key, pub. It
+// returns true iff the signature is valid.
+func Verify(pub *PublicKey, hash []byte, r, s *big.Int) bool {
+ // FIPS 186-3, section 4.7
+
+ if r.Sign() < 1 || r.Cmp(pub.Q) >= 0 {
+ return false
+ }
+ if s.Sign() < 1 || s.Cmp(pub.Q) >= 0 {
+ return false
+ }
+
+ w := new(big.Int).ModInverse(s, pub.Q)
+
+ n := pub.Q.BitLen()
+ if n&7 != 0 {
+ return false
+ }
+ n >>= 3
+
+ if n > len(hash) {
+ n = len(hash)
+ }
+ z := new(big.Int).SetBytes(hash[:n])
+
+ u1 := new(big.Int).Mul(z, w)
+ u1.Mod(u1, pub.Q)
+ u2 := w.Mul(r, w)
+ u2.Mod(u2, pub.Q)
+ v := u1.Exp(pub.G, u1, pub.P)
+ u2.Exp(pub.Y, u2, pub.P)
+ v.Mul(v, u2)
+ v.Mod(v, pub.P)
+ v.Mod(v, pub.Q)
+
+ return v.Cmp(r) == 0
+}
--- /dev/null
+// Copyright 2011 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package dsa
+
+import (
+ "big"
+ "crypto/rand"
+ "testing"
+)
+
+func testSignAndVerify(t *testing.T, i int, priv *PrivateKey) {
+ hashed := []byte("testing")
+ r, s, err := Sign(rand.Reader, priv, hashed)
+ if err != nil {
+ t.Errorf("%d: error signing: %s", i, err)
+ return
+ }
+
+ if !Verify(&priv.PublicKey, hashed, r, s) {
+ t.Errorf("%d: Verify failed", i)
+ }
+}
+
+func testParameterGeneration(t *testing.T, sizes ParameterSizes, L, N int) {
+ var priv PrivateKey
+ params := &priv.Parameters
+
+ err := GenerateParameters(params, rand.Reader, sizes)
+ if err != nil {
+ t.Errorf("%d: %s", int(sizes), err)
+ return
+ }
+
+ if params.P.BitLen() != L {
+ t.Errorf("%d: params.BitLen got:%d want:%d", int(sizes), params.P.BitLen(), L)
+ }
+
+ if params.Q.BitLen() != N {
+ t.Errorf("%d: q.BitLen got:%d want:%d", int(sizes), params.Q.BitLen(), L)
+ }
+
+ one := new(big.Int)
+ one.SetInt64(1)
+ pm1 := new(big.Int).Sub(params.P, one)
+ quo, rem := new(big.Int).DivMod(pm1, params.Q, new(big.Int))
+ if rem.Sign() != 0 {
+ t.Errorf("%d: p-1 mod q != 0", int(sizes))
+ }
+ x := new(big.Int).Exp(params.G, quo, params.P)
+ if x.Cmp(one) == 0 {
+ t.Errorf("%d: invalid generator", int(sizes))
+ }
+
+ err = GenerateKey(&priv, rand.Reader)
+ if err != nil {
+ t.Errorf("error generating key: %s", err)
+ return
+ }
+
+ testSignAndVerify(t, int(sizes), &priv)
+}
+
+func TestParameterGeneration(t *testing.T) {
+ // This test is too slow to run all the time.
+ return
+
+ testParameterGeneration(t, L1024N160, 1024, 160)
+ testParameterGeneration(t, L2048N224, 2048, 224)
+ testParameterGeneration(t, L2048N256, 2048, 256)
+ testParameterGeneration(t, L3072N256, 3072, 256)
+}
+
+func TestSignAndVerify(t *testing.T) {
+ var priv PrivateKey
+ priv.P, _ = new(big.Int).SetString("A9B5B793FB4785793D246BAE77E8FF63CA52F442DA763C440259919FE1BC1D6065A9350637A04F75A2F039401D49F08E066C4D275A5A65DA5684BC563C14289D7AB8A67163BFBF79D85972619AD2CFF55AB0EE77A9002B0EF96293BDD0F42685EBB2C66C327079F6C98000FBCB79AACDE1BC6F9D5C7B1A97E3D9D54ED7951FEF", 16)
+ priv.Q, _ = new(big.Int).SetString("E1D3391245933D68A0714ED34BBCB7A1F422B9C1", 16)
+ priv.G, _ = new(big.Int).SetString("634364FC25248933D01D1993ECABD0657CC0CB2CEED7ED2E3E8AECDFCDC4A25C3B15E9E3B163ACA2984B5539181F3EFF1A5E8903D71D5B95DA4F27202B77D2C44B430BB53741A8D59A8F86887525C9F2A6A5980A195EAA7F2FF910064301DEF89D3AA213E1FAC7768D89365318E370AF54A112EFBA9246D9158386BA1B4EEFDA", 16)
+ priv.Y, _ = new(big.Int).SetString("32969E5780CFE1C849A1C276D7AEB4F38A23B591739AA2FE197349AEEBD31366AEE5EB7E6C6DDB7C57D02432B30DB5AA66D9884299FAA72568944E4EEDC92EA3FBC6F39F53412FBCC563208F7C15B737AC8910DBC2D9C9B8C001E72FDC40EB694AB1F06A5A2DBD18D9E36C66F31F566742F11EC0A52E9F7B89355C02FB5D32D2", 16)
+ priv.X, _ = new(big.Int).SetString("5078D4D29795CBE76D3AACFE48C9AF0BCDBEE91A", 16)
+
+ testSignAndVerify(t, 0, &priv)
+}
package md4
import (
+ "crypto"
"hash"
"os"
)
+func init() {
+ crypto.RegisterHash(crypto.MD4, New)
+}
+
// The size of an MD4 checksum in bytes.
const Size = 16
package md5
import (
+ "crypto"
"hash"
"os"
)
+func init() {
+ crypto.RegisterHash(crypto.MD5, New)
+}
+
// The size of an MD5 checksum in bytes.
const Size = 16
import (
"asn1"
+ "crypto"
"crypto/rsa"
- "crypto/sha1"
+ _ "crypto/sha1"
"crypto/x509"
"os"
"time"
return nil, x509.UnsupportedAlgorithmError{}
}
- h := sha1.New()
- hashType := rsa.HashSHA1
+ hashType := crypto.SHA1
+ h := hashType.New()
pub := ret.Certificate.PublicKey.(*rsa.PublicKey)
h.Write(basicResp.TBSResponseData.Raw)
return 0, os.EOF
}
- if len(line) != 64 {
+ if len(line) > 64 {
return 0, ArmorCorrupt
}
t.Error(err)
}
- if adler32.Checksum(contents) != 0x789d7f00 {
+ if adler32.Checksum(contents) != 0x27b144be {
t.Errorf("contents: got: %x", contents)
}
const armorExample1 = `-----BEGIN PGP SIGNATURE-----
Version: GnuPG v1.4.10 (GNU/Linux)
-iQEcBAABAgAGBQJMtFESAAoJEKsQXJGvOPsVj40H/1WW6jaMXv4BW+1ueDSMDwM8
-kx1fLOXbVM5/Kn5LStZNt1jWWnpxdz7eq3uiqeCQjmqUoRde3YbB2EMnnwRbAhpp
-cacnAvy9ZQ78OTxUdNW1mhX5bS6q1MTEJnl+DcyigD70HG/yNNQD7sOPMdYQw0TA
-byQBwmLwmTsuZsrYqB68QyLHI+DUugn+kX6Hd2WDB62DKa2suoIUIHQQCd/ofwB3
-WfCYInXQKKOSxu2YOg2Eb4kLNhSMc1i9uKUWAH+sdgJh7NBgdoE4MaNtBFkHXRvv
-okWuf3+xA9ksp1npSY/mDvgHijmjvtpRDe6iUeqfCn8N9u9CBg8geANgaG8+QA4=
-=wfQG
+iJwEAAECAAYFAk1Fv/0ACgkQo01+GMIMMbsYTwQAiAw+QAaNfY6WBdplZ/uMAccm
+4g+81QPmTSGHnetSb6WBiY13kVzK4HQiZH8JSkmmroMLuGeJwsRTEL4wbjRyUKEt
+p1xwUZDECs234F1xiG5enc5SGlRtP7foLBz9lOsjx+LEcA4sTl5/2eZR9zyFZqWW
+TxRjs+fJCIFuo71xb1g=
+=/teI
-----END PGP SIGNATURE-----`
const armorLongLine = `-----BEGIN PGP SIGNATURE-----
if err != nil {
return
}
+ e.breaker.Close()
var checksumBytes [3]byte
checksumBytes[0] = byte(e.crc >> 16)
}
}
- if len(headers) > 0 {
- _, err := out.Write(newline)
- if err != nil {
- return
- }
+ _, err = out.Write(newline)
+ if err != nil {
+ return
}
e := &encoding{
--- /dev/null
+// Copyright 2011 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package openpgp
+
+import (
+ "hash"
+ "os"
+)
+
+// NewCanonicalTextHash reformats text written to it into the canonical
+// form and then applies the hash h. See RFC 4880, section 5.2.1.
+func NewCanonicalTextHash(h hash.Hash) hash.Hash {
+ return &canonicalTextHash{h, 0}
+}
+
+type canonicalTextHash struct {
+ h hash.Hash
+ s int
+}
+
+var newline = []byte{'\r', '\n'}
+
+func (cth *canonicalTextHash) Write(buf []byte) (int, os.Error) {
+ start := 0
+
+ for i, c := range buf {
+ switch cth.s {
+ case 0:
+ if c == '\r' {
+ cth.s = 1
+ } else if c == '\n' {
+ cth.h.Write(buf[start:i])
+ cth.h.Write(newline)
+ start = i + 1
+ }
+ case 1:
+ cth.s = 0
+ }
+ }
+
+ cth.h.Write(buf[start:])
+ return len(buf), nil
+}
+
+func (cth *canonicalTextHash) Sum() []byte {
+ return cth.h.Sum()
+}
+
+func (cth *canonicalTextHash) Reset() {
+ cth.h.Reset()
+ cth.s = 0
+}
+
+func (cth *canonicalTextHash) Size() int {
+ return cth.h.Size()
+}
--- /dev/null
+// Copyright 2011 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package openpgp
+
+import (
+ "bytes"
+ "os"
+ "testing"
+)
+
+type recordingHash struct {
+ buf *bytes.Buffer
+}
+
+func (r recordingHash) Write(b []byte) (n int, err os.Error) {
+ return r.buf.Write(b)
+}
+
+func (r recordingHash) Sum() []byte {
+ return r.buf.Bytes()
+}
+
+func (r recordingHash) Reset() {
+ panic("shouldn't be called")
+}
+
+func (r recordingHash) Size() int {
+ panic("shouldn't be called")
+}
+
+
+func testCanonicalText(t *testing.T, input, expected string) {
+ r := recordingHash{bytes.NewBuffer(nil)}
+ c := NewCanonicalTextHash(r)
+ c.Write([]byte(input))
+ result := c.Sum()
+ if expected != string(result) {
+ t.Errorf("input: %x got: %x want: %x", input, result, expected)
+ }
+}
+
+func TestCanonicalText(t *testing.T) {
+ testCanonicalText(t, "foo\n", "foo\r\n")
+ testCanonicalText(t, "foo", "foo")
+ testCanonicalText(t, "foo\r\n", "foo\r\n")
+ testCanonicalText(t, "foo\r\nbar", "foo\r\nbar")
+ testCanonicalText(t, "foo\r\nbar\n\n", "foo\r\nbar\r\n\r\n")
+}
// This package contains common error types for the OpenPGP packages.
package error
+import (
+ "strconv"
+)
+
// A StructuralError is returned when OpenPGP data is found to be syntactically
// invalid.
type StructuralError string
}
var KeyIncorrectError = keyIncorrect(0)
+
+type unknownIssuer int
+
+func (unknownIssuer) String() string {
+ return "signature make by unknown entity"
+}
+
+var UnknownIssuerError = unknownIssuer(0)
+
+type UnknownPacketTypeError uint8
+
+func (upte UnknownPacketTypeError) String() string {
+ return "unknown OpenPGP packet type: " + strconv.Itoa(int(upte))
+}
--- /dev/null
+// Copyright 2011 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package openpgp
+
+import (
+ "crypto/openpgp/error"
+ "crypto/openpgp/packet"
+ "io"
+ "os"
+)
+
+// PublicKeyType is the armor type for a PGP public key.
+var PublicKeyType = "PGP PUBLIC KEY BLOCK"
+
+// An Entity represents the components of an OpenPGP key: a primary public key
+// (which must be a signing key), one or more identities claimed by that key,
+// and zero or more subkeys, which may be encryption keys.
+type Entity struct {
+ PrimaryKey *packet.PublicKey
+ PrivateKey *packet.PrivateKey
+ Identities map[string]*Identity // indexed by Identity.Name
+ Subkeys []Subkey
+}
+
+// An Identity represents an identity claimed by an Entity and zero or more
+// assertions by other entities about that claim.
+type Identity struct {
+ Name string // by convention, has the form "Full Name (comment) <email@example.com>"
+ UserId *packet.UserId
+ SelfSignature *packet.Signature
+ Signatures []*packet.Signature
+}
+
+// A Subkey is an additional public key in an Entity. Subkeys can be used for
+// encryption.
+type Subkey struct {
+ PublicKey *packet.PublicKey
+ PrivateKey *packet.PrivateKey
+ Sig *packet.Signature
+}
+
+// A Key identifies a specific public key in an Entity. This is either the
+// Entity's primary key or a subkey.
+type Key struct {
+ Entity *Entity
+ PublicKey *packet.PublicKey
+ PrivateKey *packet.PrivateKey
+ SelfSignature *packet.Signature
+}
+
+// A KeyRing provides access to public and private keys.
+type KeyRing interface {
+ // KeysById returns the set of keys that have the given key id.
+ KeysById(id uint64) []Key
+ // DecryptionKeys returns all private keys that are valid for
+ // decryption.
+ DecryptionKeys() []Key
+}
+
+// An EntityList contains one or more Entities.
+type EntityList []*Entity
+
+// KeysById returns the set of keys that have the given key id.
+func (el EntityList) KeysById(id uint64) (keys []Key) {
+ for _, e := range el {
+ if e.PrimaryKey.KeyId == id {
+ var selfSig *packet.Signature
+ for _, ident := range e.Identities {
+ if selfSig == nil {
+ selfSig = ident.SelfSignature
+ } else if ident.SelfSignature.IsPrimaryId != nil && *ident.SelfSignature.IsPrimaryId {
+ selfSig = ident.SelfSignature
+ break
+ }
+ }
+ keys = append(keys, Key{e, e.PrimaryKey, e.PrivateKey, selfSig})
+ }
+
+ for _, subKey := range e.Subkeys {
+ if subKey.PublicKey.KeyId == id {
+ keys = append(keys, Key{e, subKey.PublicKey, subKey.PrivateKey, subKey.Sig})
+ }
+ }
+ }
+ return
+}
+
+// DecryptionKeys returns all private keys that are valid for decryption.
+func (el EntityList) DecryptionKeys() (keys []Key) {
+ for _, e := range el {
+ for _, subKey := range e.Subkeys {
+ if subKey.PrivateKey != nil && (!subKey.Sig.FlagsValid || subKey.Sig.FlagEncryptStorage || subKey.Sig.FlagEncryptCommunications) {
+ keys = append(keys, Key{e, subKey.PublicKey, subKey.PrivateKey, subKey.Sig})
+ }
+ }
+ }
+ return
+}
+
+// ReadArmoredKeyRing reads one or more public/private keys from an armor keyring file.
+func ReadArmoredKeyRing(r io.Reader) (EntityList, os.Error) {
+ body, err := readArmored(r, PublicKeyType)
+ if err != nil {
+ return nil, err
+ }
+
+ return ReadKeyRing(body)
+}
+
+// ReadKeyRing reads one or more public/private keys, ignoring unsupported keys.
+func ReadKeyRing(r io.Reader) (el EntityList, err os.Error) {
+ packets := packet.NewReader(r)
+
+ for {
+ var e *Entity
+ e, err = readEntity(packets)
+ if err != nil {
+ if _, ok := err.(error.UnsupportedError); ok {
+ err = readToNextPublicKey(packets)
+ }
+ if err == os.EOF {
+ err = nil
+ return
+ }
+ if err != nil {
+ el = nil
+ return
+ }
+ } else {
+ el = append(el, e)
+ }
+ }
+ return
+}
+
+// readToNextPublicKey reads packets until the start of the entity and leaves
+// the first packet of the new entity in the Reader.
+func readToNextPublicKey(packets *packet.Reader) (err os.Error) {
+ var p packet.Packet
+ for {
+ p, err = packets.Next()
+ if err == os.EOF {
+ return
+ } else if err != nil {
+ if _, ok := err.(error.UnsupportedError); ok {
+ err = nil
+ continue
+ }
+ return
+ }
+
+ if pk, ok := p.(*packet.PublicKey); ok && !pk.IsSubkey {
+ packets.Unread(p)
+ return
+ }
+ }
+
+ panic("unreachable")
+}
+
+// readEntity reads an entity (public key, identities, subkeys etc) from the
+// given Reader.
+func readEntity(packets *packet.Reader) (*Entity, os.Error) {
+ e := new(Entity)
+ e.Identities = make(map[string]*Identity)
+
+ p, err := packets.Next()
+ if err != nil {
+ return nil, err
+ }
+
+ var ok bool
+ if e.PrimaryKey, ok = p.(*packet.PublicKey); !ok {
+ if e.PrivateKey, ok = p.(*packet.PrivateKey); !ok {
+ packets.Unread(p)
+ return nil, error.StructuralError("first packet was not a public/private key")
+ } else {
+ e.PrimaryKey = &e.PrivateKey.PublicKey
+ }
+ }
+
+ var current *Identity
+EachPacket:
+ for {
+ p, err := packets.Next()
+ if err == os.EOF {
+ break
+ } else if err != nil {
+ return nil, err
+ }
+
+ switch pkt := p.(type) {
+ case *packet.UserId:
+ current = new(Identity)
+ current.Name = pkt.Id
+ current.UserId = pkt
+ e.Identities[pkt.Id] = current
+ p, err = packets.Next()
+ if err == os.EOF {
+ err = io.ErrUnexpectedEOF
+ }
+ if err != nil {
+ if _, ok := err.(error.UnsupportedError); ok {
+ return nil, err
+ }
+ return nil, error.StructuralError("identity self-signature invalid: " + err.String())
+ }
+ current.SelfSignature, ok = p.(*packet.Signature)
+ if !ok {
+ return nil, error.StructuralError("user ID packet not followed by self signature")
+ }
+ if current.SelfSignature.SigType != packet.SigTypePositiveCert {
+ return nil, error.StructuralError("user ID self-signature with wrong type")
+ }
+ if err = e.PrimaryKey.VerifyUserIdSignature(pkt.Id, current.SelfSignature); err != nil {
+ return nil, error.StructuralError("user ID self-signature invalid: " + err.String())
+ }
+ case *packet.Signature:
+ if current == nil {
+ return nil, error.StructuralError("signature packet found before user id packet")
+ }
+ current.Signatures = append(current.Signatures, pkt)
+ case *packet.PrivateKey:
+ if pkt.IsSubkey == false {
+ packets.Unread(p)
+ break EachPacket
+ }
+ err = addSubkey(e, packets, &pkt.PublicKey, pkt)
+ if err != nil {
+ return nil, err
+ }
+ case *packet.PublicKey:
+ if pkt.IsSubkey == false {
+ packets.Unread(p)
+ break EachPacket
+ }
+ err = addSubkey(e, packets, pkt, nil)
+ if err != nil {
+ return nil, err
+ }
+ default:
+ // we ignore unknown packets
+ }
+ }
+
+ if len(e.Identities) == 0 {
+ return nil, error.StructuralError("entity without any identities")
+ }
+
+ return e, nil
+}
+
+func addSubkey(e *Entity, packets *packet.Reader, pub *packet.PublicKey, priv *packet.PrivateKey) os.Error {
+ var subKey Subkey
+ subKey.PublicKey = pub
+ subKey.PrivateKey = priv
+ p, err := packets.Next()
+ if err == os.EOF {
+ return io.ErrUnexpectedEOF
+ }
+ if err != nil {
+ return error.StructuralError("subkey signature invalid: " + err.String())
+ }
+ var ok bool
+ subKey.Sig, ok = p.(*packet.Signature)
+ if !ok {
+ return error.StructuralError("subkey packet not followed by signature")
+ }
+ if subKey.Sig.SigType != packet.SigTypeSubkeyBinding {
+ return error.StructuralError("subkey signature with wrong type")
+ }
+ err = e.PrimaryKey.VerifyKeySignature(subKey.PublicKey, subKey.Sig)
+ if err != nil {
+ return error.StructuralError("subkey signature invalid: " + err.String())
+ }
+ e.Subkeys = append(e.Subkeys, subKey)
+ return nil
+}
--- /dev/null
+// Copyright 2011 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package packet
+
+import (
+ "compress/flate"
+ "compress/zlib"
+ "crypto/openpgp/error"
+ "io"
+ "os"
+ "strconv"
+)
+
+// Compressed represents a compressed OpenPGP packet. The decompressed contents
+// will contain more OpenPGP packets. See RFC 4880, section 5.6.
+type Compressed struct {
+ Body io.Reader
+}
+
+func (c *Compressed) parse(r io.Reader) os.Error {
+ var buf [1]byte
+ _, err := readFull(r, buf[:])
+ if err != nil {
+ return err
+ }
+
+ switch buf[0] {
+ case 1:
+ c.Body = flate.NewReader(r)
+ case 2:
+ c.Body, err = zlib.NewReader(r)
+ default:
+ err = error.UnsupportedError("unknown compression algorithm: " + strconv.Itoa(int(buf[0])))
+ }
+
+ return err
+}
--- /dev/null
+// Copyright 2011 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package packet
+
+import (
+ "bytes"
+ "encoding/hex"
+ "os"
+ "io/ioutil"
+ "testing"
+)
+
+func TestCompressed(t *testing.T) {
+ packet, err := Read(readerFromHex(compressedHex))
+ if err != nil {
+ t.Errorf("failed to read Compressed: %s", err)
+ return
+ }
+
+ c, ok := packet.(*Compressed)
+ if !ok {
+ t.Error("didn't find Compressed packet")
+ return
+ }
+
+ contents, err := ioutil.ReadAll(c.Body)
+ if err != nil && err != os.EOF {
+ t.Error(err)
+ return
+ }
+
+ expected, _ := hex.DecodeString(compressedExpectedHex)
+ if !bytes.Equal(expected, contents) {
+ t.Errorf("got:%x want:%x", contents, expected)
+ }
+}
+
+const compressedHex = "a3013b2d90c4e02b72e25f727e5e496a5e49b11e1700"
+const compressedExpectedHex = "cb1062004d14c8fe636f6e74656e74732e0a"
--- /dev/null
+// Copyright 2011 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package packet
+
+import (
+ "crypto/openpgp/error"
+ "crypto/rand"
+ "crypto/rsa"
+ "encoding/binary"
+ "io"
+ "os"
+ "strconv"
+)
+
+// EncryptedKey represents a public-key encrypted session key. See RFC 4880,
+// section 5.1.
+type EncryptedKey struct {
+ KeyId uint64
+ Algo PublicKeyAlgorithm
+ Encrypted []byte
+ CipherFunc CipherFunction // only valid after a successful Decrypt
+ Key []byte // only valid after a successful Decrypt
+}
+
+func (e *EncryptedKey) parse(r io.Reader) (err os.Error) {
+ var buf [10]byte
+ _, err = readFull(r, buf[:])
+ if err != nil {
+ return
+ }
+ if buf[0] != 3 {
+ return error.UnsupportedError("unknown EncryptedKey version " + strconv.Itoa(int(buf[0])))
+ }
+ e.KeyId = binary.BigEndian.Uint64(buf[1:9])
+ e.Algo = PublicKeyAlgorithm(buf[9])
+ if e.Algo == PubKeyAlgoRSA || e.Algo == PubKeyAlgoRSAEncryptOnly {
+ e.Encrypted, _, err = readMPI(r)
+ }
+ _, err = consumeAll(r)
+ return
+}
+
+// DecryptRSA decrypts an RSA encrypted session key with the given private key.
+func (e *EncryptedKey) DecryptRSA(priv *rsa.PrivateKey) (err os.Error) {
+ if e.Algo != PubKeyAlgoRSA && e.Algo != PubKeyAlgoRSAEncryptOnly {
+ return error.InvalidArgumentError("EncryptedKey not RSA encrypted")
+ }
+ b, err := rsa.DecryptPKCS1v15(rand.Reader, priv, e.Encrypted)
+ if err != nil {
+ return
+ }
+ e.CipherFunc = CipherFunction(b[0])
+ e.Key = b[1 : len(b)-2]
+ expectedChecksum := uint16(b[len(b)-2])<<8 | uint16(b[len(b)-1])
+ var checksum uint16
+ for _, v := range e.Key {
+ checksum += uint16(v)
+ }
+ if checksum != expectedChecksum {
+ return error.StructuralError("EncryptedKey checksum incorrect")
+ }
+
+ return
+}
--- /dev/null
+// Copyright 2011 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package packet
+
+import (
+ "big"
+ "crypto/rsa"
+ "fmt"
+ "testing"
+)
+
+func bigFromBase10(s string) *big.Int {
+ b, ok := new(big.Int).SetString(s, 10)
+ if !ok {
+ panic("bigFromBase10 failed")
+ }
+ return b
+}
+
+func TestEncryptedKey(t *testing.T) {
+ p, err := Read(readerFromHex(encryptedKeyHex))
+ if err != nil {
+ t.Errorf("error from Read: %s", err)
+ return
+ }
+ ek, ok := p.(*EncryptedKey)
+ if !ok {
+ t.Errorf("didn't parse an EncryptedKey, got %#v", p)
+ return
+ }
+
+ if ek.KeyId != 0x2a67d68660df41c7 || ek.Algo != PubKeyAlgoRSA {
+ t.Errorf("unexpected EncryptedKey contents: %#v", ek)
+ return
+ }
+
+ pub := rsa.PublicKey{
+ E: 65537,
+ N: bigFromBase10("115804063926007623305902631768113868327816898845124614648849934718568541074358183759250136204762053879858102352159854352727097033322663029387610959884180306668628526686121021235757016368038585212410610742029286439607686208110250133174279811431933746643015923132833417396844716207301518956640020862630546868823"),
+ }
+
+ priv := &rsa.PrivateKey{
+ PublicKey: pub,
+ D: bigFromBase10("32355588668219869544751561565313228297765464314098552250409557267371233892496951383426602439009993875125222579159850054973310859166139474359774543943714622292329487391199285040721944491839695981199720170366763547754915493640685849961780092241140181198779299712578774460837139360803883139311171713302987058393"),
+ }
+
+ err = ek.DecryptRSA(priv)
+ if err != nil {
+ t.Errorf("error from DecryptRSA: %s", err)
+ return
+ }
+
+ if ek.CipherFunc != CipherAES256 {
+ t.Errorf("unexpected EncryptedKey contents: %#v", ek)
+ return
+ }
+
+ keyHex := fmt.Sprintf("%x", ek.Key)
+ if keyHex != expectedKeyHex {
+ t.Errorf("bad key, got %s want %x", keyHex, expectedKeyHex)
+ }
+}
+
+const encryptedKeyHex = "c18c032a67d68660df41c70104005789d0de26b6a50c985a02a13131ca829c413a35d0e6fa8d6842599252162808ac7439c72151c8c6183e76923fe3299301414d0c25a2f06a2257db3839e7df0ec964773f6e4c4ac7ff3b48c444237166dd46ba8ff443a5410dc670cb486672fdbe7c9dfafb75b4fea83af3a204fe2a7dfa86bd20122b4f3d2646cbeecb8f7be8"
+const expectedKeyHex = "d930363f7e0308c333b9618617ea728963d8df993665ae7be1092d4926fd864b"
--- /dev/null
+// Copyright 2011 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package packet
+
+import (
+ "encoding/binary"
+ "io"
+ "os"
+)
+
+// LiteralData represents an encrypted file. See RFC 4880, section 5.9.
+type LiteralData struct {
+ IsBinary bool
+ FileName string
+ Time uint32 // Unix epoch time. Either creation time or modification time. 0 means undefined.
+ Body io.Reader
+}
+
+// ForEyesOnly returns whether the contents of the LiteralData have been marked
+// as especially sensitive.
+func (l *LiteralData) ForEyesOnly() bool {
+ return l.FileName == "_CONSOLE"
+}
+
+func (l *LiteralData) parse(r io.Reader) (err os.Error) {
+ var buf [256]byte
+
+ _, err = readFull(r, buf[:2])
+ if err != nil {
+ return
+ }
+
+ l.IsBinary = buf[0] == 'b'
+ fileNameLen := int(buf[1])
+
+ _, err = readFull(r, buf[:fileNameLen])
+ if err != nil {
+ return
+ }
+
+ l.FileName = string(buf[:fileNameLen])
+
+ _, err = readFull(r, buf[:4])
+ if err != nil {
+ return
+ }
+
+ l.Time = binary.BigEndian.Uint32(buf[:4])
+ l.Body = r
+ return
+}
--- /dev/null
+// Copyright 2011 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package packet
+
+import (
+ "crypto"
+ "crypto/openpgp/error"
+ "crypto/openpgp/s2k"
+ "encoding/binary"
+ "io"
+ "os"
+ "strconv"
+)
+
+// OnePassSignature represents a one-pass signature packet. See RFC 4880,
+// section 5.4.
+type OnePassSignature struct {
+ SigType SignatureType
+ Hash crypto.Hash
+ PubKeyAlgo PublicKeyAlgorithm
+ KeyId uint64
+ IsLast bool
+}
+
+func (ops *OnePassSignature) parse(r io.Reader) (err os.Error) {
+ var buf [13]byte
+
+ _, err = readFull(r, buf[:])
+ if err != nil {
+ return
+ }
+ if buf[0] != 3 {
+ err = error.UnsupportedError("one-pass-signature packet version " + strconv.Itoa(int(buf[0])))
+ }
+
+ var ok bool
+ ops.Hash, ok = s2k.HashIdToHash(buf[2])
+ if !ok {
+ return error.UnsupportedError("hash function: " + strconv.Itoa(int(buf[2])))
+ }
+
+ ops.SigType = SignatureType(buf[1])
+ ops.PubKeyAlgo = PublicKeyAlgorithm(buf[3])
+ ops.KeyId = binary.BigEndian.Uint64(buf[4:12])
+ ops.IsLast = buf[12] != 0
+ return
+}
--- /dev/null
+// Copyright 2011 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// This package implements parsing and serialisation of OpenPGP packets, as
+// specified in RFC 4880.
+package packet
+
+import (
+ "crypto/aes"
+ "crypto/cast5"
+ "crypto/cipher"
+ "crypto/openpgp/error"
+ "io"
+ "os"
+)
+
+// readFull is the same as io.ReadFull except that reading zero bytes returns
+// ErrUnexpectedEOF rather than EOF.
+func readFull(r io.Reader, buf []byte) (n int, err os.Error) {
+ n, err = io.ReadFull(r, buf)
+ if err == os.EOF {
+ err = io.ErrUnexpectedEOF
+ }
+ return
+}
+
+// readLength reads an OpenPGP length from r. See RFC 4880, section 4.2.2.
+func readLength(r io.Reader) (length int64, isPartial bool, err os.Error) {
+ var buf [4]byte
+ _, err = readFull(r, buf[:1])
+ if err != nil {
+ return
+ }
+ switch {
+ case buf[0] < 192:
+ length = int64(buf[0])
+ case buf[0] < 224:
+ length = int64(buf[0]-192) << 8
+ _, err = readFull(r, buf[0:1])
+ if err != nil {
+ return
+ }
+ length += int64(buf[0]) + 192
+ case buf[0] < 255:
+ length = int64(1) << (buf[0] & 0x1f)
+ isPartial = true
+ default:
+ _, err = readFull(r, buf[0:4])
+ if err != nil {
+ return
+ }
+ length = int64(buf[0])<<24 |
+ int64(buf[1])<<16 |
+ int64(buf[2])<<8 |
+ int64(buf[3])
+ }
+ return
+}
+
+// partialLengthReader wraps an io.Reader and handles OpenPGP partial lengths.
+// The continuation lengths are parsed and removed from the stream and EOF is
+// returned at the end of the packet. See RFC 4880, section 4.2.2.4.
+type partialLengthReader struct {
+ r io.Reader
+ remaining int64
+ isPartial bool
+}
+
+func (r *partialLengthReader) Read(p []byte) (n int, err os.Error) {
+ for r.remaining == 0 {
+ if !r.isPartial {
+ return 0, os.EOF
+ }
+ r.remaining, r.isPartial, err = readLength(r.r)
+ if err != nil {
+ return 0, err
+ }
+ }
+
+ toRead := int64(len(p))
+ if toRead > r.remaining {
+ toRead = r.remaining
+ }
+
+ n, err = r.r.Read(p[:int(toRead)])
+ r.remaining -= int64(n)
+ if n < int(toRead) && err == os.EOF {
+ err = io.ErrUnexpectedEOF
+ }
+ return
+}
+
+// A spanReader is an io.LimitReader, but it returns ErrUnexpectedEOF if the
+// underlying Reader returns EOF before the limit has been reached.
+type spanReader struct {
+ r io.Reader
+ n int64
+}
+
+func (l *spanReader) Read(p []byte) (n int, err os.Error) {
+ if l.n <= 0 {
+ return 0, os.EOF
+ }
+ if int64(len(p)) > l.n {
+ p = p[0:l.n]
+ }
+ n, err = l.r.Read(p)
+ l.n -= int64(n)
+ if l.n > 0 && err == os.EOF {
+ err = io.ErrUnexpectedEOF
+ }
+ return
+}
+
+// readHeader parses a packet header and returns an io.Reader which will return
+// the contents of the packet. See RFC 4880, section 4.2.
+func readHeader(r io.Reader) (tag packetType, length int64, contents io.Reader, err os.Error) {
+ var buf [4]byte
+ _, err = io.ReadFull(r, buf[:1])
+ if err != nil {
+ return
+ }
+ if buf[0]&0x80 == 0 {
+ err = error.StructuralError("tag byte does not have MSB set")
+ return
+ }
+ if buf[0]&0x40 == 0 {
+ // Old format packet
+ tag = packetType((buf[0] & 0x3f) >> 2)
+ lengthType := buf[0] & 3
+ if lengthType == 3 {
+ length = -1
+ contents = r
+ return
+ }
+ lengthBytes := 1 << lengthType
+ _, err = readFull(r, buf[0:lengthBytes])
+ if err != nil {
+ return
+ }
+ for i := 0; i < lengthBytes; i++ {
+ length <<= 8
+ length |= int64(buf[i])
+ }
+ contents = &spanReader{r, length}
+ return
+ }
+
+ // New format packet
+ tag = packetType(buf[0] & 0x3f)
+ length, isPartial, err := readLength(r)
+ if err != nil {
+ return
+ }
+ if isPartial {
+ contents = &partialLengthReader{
+ remaining: length,
+ isPartial: true,
+ r: r,
+ }
+ length = -1
+ } else {
+ contents = &spanReader{r, length}
+ }
+ return
+}
+
+// serialiseHeader writes an OpenPGP packet header to w. See RFC 4880, section
+// 4.2.
+func serialiseHeader(w io.Writer, ptype packetType, length int) (err os.Error) {
+ var buf [5]byte
+ var n int
+
+ buf[0] = 0x80 | 0x40 | byte(ptype)
+ if length < 192 {
+ buf[1] = byte(length)
+ n = 2
+ } else if length < 8384 {
+ length -= 192
+ buf[1] = byte(length >> 8)
+ buf[2] = byte(length)
+ n = 3
+ } else {
+ buf[0] = 255
+ buf[1] = byte(length >> 24)
+ buf[2] = byte(length >> 16)
+ buf[3] = byte(length >> 8)
+ buf[4] = byte(length)
+ n = 5
+ }
+
+ _, err = w.Write(buf[:n])
+ return
+}
+
+// Packet represents an OpenPGP packet. Users are expected to try casting
+// instances of this interface to specific packet types.
+type Packet interface {
+ parse(io.Reader) os.Error
+}
+
+// consumeAll reads from the given Reader until error, returning the number of
+// bytes read.
+func consumeAll(r io.Reader) (n int64, err os.Error) {
+ var m int
+ var buf [1024]byte
+
+ for {
+ m, err = r.Read(buf[:])
+ n += int64(m)
+ if err == os.EOF {
+ err = nil
+ return
+ }
+ if err != nil {
+ return
+ }
+ }
+
+ panic("unreachable")
+}
+
+// packetType represents the numeric ids of the different OpenPGP packet types. See
+// http://www.iana.org/assignments/pgp-parameters/pgp-parameters.xhtml#pgp-parameters-2
+type packetType uint8
+
+const (
+ packetTypeEncryptedKey packetType = 1
+ packetTypeSignature packetType = 2
+ packetTypeSymmetricKeyEncrypted packetType = 3
+ packetTypeOnePassSignature packetType = 4
+ packetTypePrivateKey packetType = 5
+ packetTypePublicKey packetType = 6
+ packetTypePrivateSubkey packetType = 7
+ packetTypeCompressed packetType = 8
+ packetTypeSymmetricallyEncrypted packetType = 9
+ packetTypeLiteralData packetType = 11
+ packetTypeUserId packetType = 13
+ packetTypePublicSubkey packetType = 14
+ packetTypeSymmetricallyEncryptedMDC packetType = 18
+)
+
+// Read reads a single OpenPGP packet from the given io.Reader. If there is an
+// error parsing a packet, the whole packet is consumed from the input.
+func Read(r io.Reader) (p Packet, err os.Error) {
+ tag, _, contents, err := readHeader(r)
+ if err != nil {
+ return
+ }
+
+ switch tag {
+ case packetTypeEncryptedKey:
+ p = new(EncryptedKey)
+ case packetTypeSignature:
+ p = new(Signature)
+ case packetTypeSymmetricKeyEncrypted:
+ p = new(SymmetricKeyEncrypted)
+ case packetTypeOnePassSignature:
+ p = new(OnePassSignature)
+ case packetTypePrivateKey, packetTypePrivateSubkey:
+ pk := new(PrivateKey)
+ if tag == packetTypePrivateSubkey {
+ pk.IsSubkey = true
+ }
+ p = pk
+ case packetTypePublicKey, packetTypePublicSubkey:
+ pk := new(PublicKey)
+ if tag == packetTypePublicSubkey {
+ pk.IsSubkey = true
+ }
+ p = pk
+ case packetTypeCompressed:
+ p = new(Compressed)
+ case packetTypeSymmetricallyEncrypted:
+ p = new(SymmetricallyEncrypted)
+ case packetTypeLiteralData:
+ p = new(LiteralData)
+ case packetTypeUserId:
+ p = new(UserId)
+ case packetTypeSymmetricallyEncryptedMDC:
+ se := new(SymmetricallyEncrypted)
+ se.MDC = true
+ p = se
+ default:
+ err = error.UnknownPacketTypeError(tag)
+ }
+ if p != nil {
+ err = p.parse(contents)
+ }
+ if err != nil {
+ consumeAll(contents)
+ }
+ return
+}
+
+// SignatureType represents the different semantic meanings of an OpenPGP
+// signature. See RFC 4880, section 5.2.1.
+type SignatureType uint8
+
+const (
+ SigTypeBinary SignatureType = 0
+ SigTypeText = 1
+ SigTypeGenericCert = 0x10
+ SigTypePersonaCert = 0x11
+ SigTypeCasualCert = 0x12
+ SigTypePositiveCert = 0x13
+ SigTypeSubkeyBinding = 0x18
+)
+
+// PublicKeyAlgorithm represents the different public key system specified for
+// OpenPGP. See
+// http://www.iana.org/assignments/pgp-parameters/pgp-parameters.xhtml#pgp-parameters-12
+type PublicKeyAlgorithm uint8
+
+const (
+ PubKeyAlgoRSA PublicKeyAlgorithm = 1
+ PubKeyAlgoRSAEncryptOnly PublicKeyAlgorithm = 2
+ PubKeyAlgoRSASignOnly PublicKeyAlgorithm = 3
+ PubKeyAlgoElgamal PublicKeyAlgorithm = 16
+ PubKeyAlgoDSA PublicKeyAlgorithm = 17
+)
+
+// CipherFunction represents the different block ciphers specified for OpenPGP. See
+// http://www.iana.org/assignments/pgp-parameters/pgp-parameters.xhtml#pgp-parameters-13
+type CipherFunction uint8
+
+const (
+ CipherCAST5 = 3
+ CipherAES128 = 7
+ CipherAES192 = 8
+ CipherAES256 = 9
+)
+
+// keySize returns the key size, in bytes, of cipher.
+func (cipher CipherFunction) keySize() int {
+ switch cipher {
+ case CipherCAST5:
+ return cast5.KeySize
+ case CipherAES128:
+ return 16
+ case CipherAES192:
+ return 24
+ case CipherAES256:
+ return 32
+ }
+ return 0
+}
+
+// blockSize returns the block size, in bytes, of cipher.
+func (cipher CipherFunction) blockSize() int {
+ switch cipher {
+ case CipherCAST5:
+ return 8
+ case CipherAES128, CipherAES192, CipherAES256:
+ return 16
+ }
+ return 0
+}
+
+// new returns a fresh instance of the given cipher.
+func (cipher CipherFunction) new(key []byte) (block cipher.Block) {
+ switch cipher {
+ case CipherCAST5:
+ block, _ = cast5.NewCipher(key)
+ case CipherAES128, CipherAES192, CipherAES256:
+ block, _ = aes.NewCipher(key)
+ }
+ return
+}
+
+// readMPI reads a big integer from r. The bit length returned is the bit
+// length that was specified in r. This is preserved so that the integer can be
+// reserialised exactly.
+func readMPI(r io.Reader) (mpi []byte, bitLength uint16, err os.Error) {
+ var buf [2]byte
+ _, err = readFull(r, buf[0:])
+ if err != nil {
+ return
+ }
+ bitLength = uint16(buf[0])<<8 | uint16(buf[1])
+ numBytes := (int(bitLength) + 7) / 8
+ mpi = make([]byte, numBytes)
+ _, err = readFull(r, mpi)
+ return
+}
+
+// writeMPI serialises a big integer to r.
+func writeMPI(w io.Writer, bitLength uint16, mpiBytes []byte) (err os.Error) {
+ _, err = w.Write([]byte{byte(bitLength >> 8), byte(bitLength)})
+ if err == nil {
+ _, err = w.Write(mpiBytes)
+ }
+ return
+}
--- /dev/null
+// Copyright 2011 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package packet
+
+import (
+ "bytes"
+ "crypto/openpgp/error"
+ "encoding/hex"
+ "fmt"
+ "io"
+ "io/ioutil"
+ "os"
+ "testing"
+)
+
+func TestReadFull(t *testing.T) {
+ var out [4]byte
+
+ b := bytes.NewBufferString("foo")
+ n, err := readFull(b, out[:3])
+ if n != 3 || err != nil {
+ t.Errorf("full read failed n:%d err:%s", n, err)
+ }
+
+ b = bytes.NewBufferString("foo")
+ n, err = readFull(b, out[:4])
+ if n != 3 || err != io.ErrUnexpectedEOF {
+ t.Errorf("partial read failed n:%d err:%s", n, err)
+ }
+
+ b = bytes.NewBuffer(nil)
+ n, err = readFull(b, out[:3])
+ if n != 0 || err != io.ErrUnexpectedEOF {
+ t.Errorf("empty read failed n:%d err:%s", n, err)
+ }
+}
+
+func readerFromHex(s string) io.Reader {
+ data, err := hex.DecodeString(s)
+ if err != nil {
+ panic("readerFromHex: bad input")
+ }
+ return bytes.NewBuffer(data)
+}
+
+var readLengthTests = []struct {
+ hexInput string
+ length int64
+ isPartial bool
+ err os.Error
+}{
+ {"", 0, false, io.ErrUnexpectedEOF},
+ {"1f", 31, false, nil},
+ {"c0", 0, false, io.ErrUnexpectedEOF},
+ {"c101", 256 + 1 + 192, false, nil},
+ {"e0", 1, true, nil},
+ {"e1", 2, true, nil},
+ {"e2", 4, true, nil},
+ {"ff", 0, false, io.ErrUnexpectedEOF},
+ {"ff00", 0, false, io.ErrUnexpectedEOF},
+ {"ff0000", 0, false, io.ErrUnexpectedEOF},
+ {"ff000000", 0, false, io.ErrUnexpectedEOF},
+ {"ff00000000", 0, false, nil},
+ {"ff01020304", 16909060, false, nil},
+}
+
+func TestReadLength(t *testing.T) {
+ for i, test := range readLengthTests {
+ length, isPartial, err := readLength(readerFromHex(test.hexInput))
+ if test.err != nil {
+ if err != test.err {
+ t.Errorf("%d: expected different error got:%s want:%s", i, err, test.err)
+ }
+ continue
+ }
+ if err != nil {
+ t.Errorf("%d: unexpected error: %s", i, err)
+ continue
+ }
+ if length != test.length || isPartial != test.isPartial {
+ t.Errorf("%d: bad result got:(%d,%t) want:(%d,%t)", i, length, isPartial, test.length, test.isPartial)
+ }
+ }
+}
+
+var partialLengthReaderTests = []struct {
+ hexInput string
+ err os.Error
+ hexOutput string
+}{
+ {"e0", io.ErrUnexpectedEOF, ""},
+ {"e001", io.ErrUnexpectedEOF, ""},
+ {"e0010102", nil, "0102"},
+ {"ff00000000", nil, ""},
+ {"e10102e1030400", nil, "01020304"},
+ {"e101", io.ErrUnexpectedEOF, ""},
+}
+
+func TestPartialLengthReader(t *testing.T) {
+ for i, test := range partialLengthReaderTests {
+ r := &partialLengthReader{readerFromHex(test.hexInput), 0, true}
+ out, err := ioutil.ReadAll(r)
+ if test.err != nil {
+ if err != test.err {
+ t.Errorf("%d: expected different error got:%s want:%s", i, err, test.err)
+ }
+ continue
+ }
+ if err != nil {
+ t.Errorf("%d: unexpected error: %s", i, err)
+ continue
+ }
+
+ got := fmt.Sprintf("%x", out)
+ if got != test.hexOutput {
+ t.Errorf("%d: got:%s want:%s", i, test.hexOutput, got)
+ }
+ }
+}
+
+var readHeaderTests = []struct {
+ hexInput string
+ structuralError bool
+ unexpectedEOF bool
+ tag int
+ length int64
+ hexOutput string
+}{
+ {"", false, false, 0, 0, ""},
+ {"7f", true, false, 0, 0, ""},
+
+ // Old format headers
+ {"80", false, true, 0, 0, ""},
+ {"8001", false, true, 0, 1, ""},
+ {"800102", false, false, 0, 1, "02"},
+ {"81000102", false, false, 0, 1, "02"},
+ {"820000000102", false, false, 0, 1, "02"},
+ {"860000000102", false, false, 1, 1, "02"},
+ {"83010203", false, false, 0, -1, "010203"},
+
+ // New format headers
+ {"c0", false, true, 0, 0, ""},
+ {"c000", false, false, 0, 0, ""},
+ {"c00102", false, false, 0, 1, "02"},
+ {"c0020203", false, false, 0, 2, "0203"},
+ {"c00202", false, true, 0, 2, ""},
+ {"c3020203", false, false, 3, 2, "0203"},
+}
+
+func TestReadHeader(t *testing.T) {
+ for i, test := range readHeaderTests {
+ tag, length, contents, err := readHeader(readerFromHex(test.hexInput))
+ if test.structuralError {
+ if _, ok := err.(error.StructuralError); ok {
+ continue
+ }
+ t.Errorf("%d: expected StructuralError, got:%s", i, err)
+ continue
+ }
+ if err != nil {
+ if len(test.hexInput) == 0 && err == os.EOF {
+ continue
+ }
+ if !test.unexpectedEOF || err != io.ErrUnexpectedEOF {
+ t.Errorf("%d: unexpected error from readHeader: %s", i, err)
+ }
+ continue
+ }
+ if int(tag) != test.tag || length != test.length {
+ t.Errorf("%d: got:(%d,%d) want:(%d,%d)", i, int(tag), length, test.tag, test.length)
+ continue
+ }
+
+ body, err := ioutil.ReadAll(contents)
+ if err != nil {
+ if !test.unexpectedEOF || err != io.ErrUnexpectedEOF {
+ t.Errorf("%d: unexpected error from contents: %s", i, err)
+ }
+ continue
+ }
+ if test.unexpectedEOF {
+ t.Errorf("%d: expected ErrUnexpectedEOF from contents but got no error", i)
+ continue
+ }
+ got := fmt.Sprintf("%x", body)
+ if got != test.hexOutput {
+ t.Errorf("%d: got:%s want:%s", i, got, test.hexOutput)
+ }
+ }
+}
--- /dev/null
+// Copyright 2011 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package packet
+
+import (
+ "big"
+ "bytes"
+ "crypto/cipher"
+ "crypto/openpgp/error"
+ "crypto/openpgp/s2k"
+ "crypto/rsa"
+ "crypto/sha1"
+ "io"
+ "io/ioutil"
+ "os"
+ "strconv"
+)
+
+// PrivateKey represents a possibly encrypted private key. See RFC 4880,
+// section 5.5.3.
+type PrivateKey struct {
+ PublicKey
+ Encrypted bool // if true then the private key is unavailable until Decrypt has been called.
+ encryptedData []byte
+ cipher CipherFunction
+ s2k func(out, in []byte)
+ PrivateKey interface{} // An *rsa.PrivateKey.
+ sha1Checksum bool
+ iv []byte
+}
+
+func (pk *PrivateKey) parse(r io.Reader) (err os.Error) {
+ err = (&pk.PublicKey).parse(r)
+ if err != nil {
+ return
+ }
+ var buf [1]byte
+ _, err = readFull(r, buf[:])
+ if err != nil {
+ return
+ }
+
+ s2kType := buf[0]
+
+ switch s2kType {
+ case 0:
+ pk.s2k = nil
+ pk.Encrypted = false
+ case 254, 255:
+ _, err = readFull(r, buf[:])
+ if err != nil {
+ return
+ }
+ pk.cipher = CipherFunction(buf[0])
+ pk.Encrypted = true
+ pk.s2k, err = s2k.Parse(r)
+ if err != nil {
+ return
+ }
+ if s2kType == 254 {
+ pk.sha1Checksum = true
+ }
+ default:
+ return error.UnsupportedError("deprecated s2k function in private key")
+ }
+
+ if pk.Encrypted {
+ blockSize := pk.cipher.blockSize()
+ if blockSize == 0 {
+ return error.UnsupportedError("unsupported cipher in private key: " + strconv.Itoa(int(pk.cipher)))
+ }
+ pk.iv = make([]byte, blockSize)
+ _, err = readFull(r, pk.iv)
+ if err != nil {
+ return
+ }
+ }
+
+ pk.encryptedData, err = ioutil.ReadAll(r)
+ if err != nil {
+ return
+ }
+
+ if !pk.Encrypted {
+ return pk.parsePrivateKey(pk.encryptedData)
+ }
+
+ return
+}
+
+// Decrypt decrypts an encrypted private key using a passphrase.
+func (pk *PrivateKey) Decrypt(passphrase []byte) os.Error {
+ if !pk.Encrypted {
+ return nil
+ }
+
+ key := make([]byte, pk.cipher.keySize())
+ pk.s2k(key, passphrase)
+ block := pk.cipher.new(key)
+ cfb := cipher.NewCFBDecrypter(block, pk.iv)
+
+ data := pk.encryptedData
+ cfb.XORKeyStream(data, data)
+
+ if pk.sha1Checksum {
+ if len(data) < sha1.Size {
+ return error.StructuralError("truncated private key data")
+ }
+ h := sha1.New()
+ h.Write(data[:len(data)-sha1.Size])
+ sum := h.Sum()
+ if !bytes.Equal(sum, data[len(data)-sha1.Size:]) {
+ return error.StructuralError("private key checksum failure")
+ }
+ data = data[:len(data)-sha1.Size]
+ } else {
+ if len(data) < 2 {
+ return error.StructuralError("truncated private key data")
+ }
+ var sum uint16
+ for i := 0; i < len(data)-2; i++ {
+ sum += uint16(data[i])
+ }
+ if data[len(data)-2] != uint8(sum>>8) ||
+ data[len(data)-1] != uint8(sum) {
+ return error.StructuralError("private key checksum failure")
+ }
+ data = data[:len(data)-2]
+ }
+
+ return pk.parsePrivateKey(data)
+}
+
+func (pk *PrivateKey) parsePrivateKey(data []byte) (err os.Error) {
+ // TODO(agl): support DSA and ECDSA private keys.
+ rsaPub := pk.PublicKey.PublicKey.(*rsa.PublicKey)
+ rsaPriv := new(rsa.PrivateKey)
+ rsaPriv.PublicKey = *rsaPub
+
+ buf := bytes.NewBuffer(data)
+ d, _, err := readMPI(buf)
+ if err != nil {
+ return
+ }
+ p, _, err := readMPI(buf)
+ if err != nil {
+ return
+ }
+ q, _, err := readMPI(buf)
+ if err != nil {
+ return
+ }
+
+ rsaPriv.D = new(big.Int).SetBytes(d)
+ rsaPriv.P = new(big.Int).SetBytes(p)
+ rsaPriv.Q = new(big.Int).SetBytes(q)
+ pk.PrivateKey = rsaPriv
+ pk.Encrypted = false
+ pk.encryptedData = nil
+
+ return nil
+}
--- /dev/null
+// Copyright 2011 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package packet
+
+import (
+ "testing"
+)
+
+func TestPrivateKeyRead(t *testing.T) {
+ packet, err := Read(readerFromHex(privKeyHex))
+ if err != nil {
+ t.Error(err)
+ return
+ }
+
+ privKey := packet.(*PrivateKey)
+
+ if !privKey.Encrypted {
+ t.Error("private key isn't encrypted")
+ return
+ }
+
+ err = privKey.Decrypt([]byte("testing"))
+ if err != nil {
+ t.Error(err)
+ return
+ }
+
+ if privKey.CreationTime != 0x4cc349a8 || privKey.Encrypted {
+ t.Errorf("failed to parse, got: %#v", privKey)
+ }
+}
+
+// Generated with `gpg --export-secret-keys "Test Key 2"`
+const privKeyHex = "9501fe044cc349a8010400b70ca0010e98c090008d45d1ee8f9113bd5861fd57b88bacb7c68658747663f1e1a3b5a98f32fda6472373c024b97359cd2efc88ff60f77751adfbf6af5e615e6a1408cfad8bf0cea30b0d5f53aa27ad59089ba9b15b7ebc2777a25d7b436144027e3bcd203909f147d0e332b240cf63d3395f5dfe0df0a6c04e8655af7eacdf0011010001fe0303024a252e7d475fd445607de39a265472aa74a9320ba2dac395faa687e9e0336aeb7e9a7397e511b5afd9dc84557c80ac0f3d4d7bfec5ae16f20d41c8c84a04552a33870b930420e230e179564f6d19bb153145e76c33ae993886c388832b0fa042ddda7f133924f3854481533e0ede31d51278c0519b29abc3bf53da673e13e3e1214b52413d179d7f66deee35cac8eacb060f78379d70ef4af8607e68131ff529439668fc39c9ce6dfef8a5ac234d234802cbfb749a26107db26406213ae5c06d4673253a3cbee1fcbae58d6ab77e38d6e2c0e7c6317c48e054edadb5a40d0d48acb44643d998139a8a66bb820be1f3f80185bc777d14b5954b60effe2448a036d565c6bc0b915fcea518acdd20ab07bc1529f561c58cd044f723109b93f6fd99f876ff891d64306b5d08f48bab59f38695e9109c4dec34013ba3153488ce070268381ba923ee1eb77125b36afcb4347ec3478c8f2735b06ef17351d872e577fa95d0c397c88c71b59629a36aec"
--- /dev/null
+// Copyright 2011 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package packet
+
+import (
+ "big"
+ "crypto/dsa"
+ "crypto/openpgp/error"
+ "crypto/rsa"
+ "crypto/sha1"
+ "encoding/binary"
+ "hash"
+ "io"
+ "os"
+)
+
+// PublicKey represents an OpenPGP public key. See RFC 4880, section 5.5.2.
+type PublicKey struct {
+ CreationTime uint32 // seconds since the epoch
+ PubKeyAlgo PublicKeyAlgorithm
+ PublicKey interface{} // Either a *rsa.PublicKey or *dsa.PublicKey
+ Fingerprint [20]byte
+ KeyId uint64
+ IsSubkey bool
+
+ n, e, p, q, g, y parsedMPI
+}
+
+func (pk *PublicKey) parse(r io.Reader) (err os.Error) {
+ // RFC 4880, section 5.5.2
+ var buf [6]byte
+ _, err = readFull(r, buf[:])
+ if err != nil {
+ return
+ }
+ if buf[0] != 4 {
+ return error.UnsupportedError("public key version")
+ }
+ pk.CreationTime = uint32(buf[1])<<24 | uint32(buf[2])<<16 | uint32(buf[3])<<8 | uint32(buf[4])
+ pk.PubKeyAlgo = PublicKeyAlgorithm(buf[5])
+ switch pk.PubKeyAlgo {
+ case PubKeyAlgoRSA, PubKeyAlgoRSAEncryptOnly, PubKeyAlgoRSASignOnly:
+ err = pk.parseRSA(r)
+ case PubKeyAlgoDSA:
+ err = pk.parseDSA(r)
+ default:
+ err = error.UnsupportedError("public key type")
+ }
+ if err != nil {
+ return
+ }
+
+ // RFC 4880, section 12.2
+ fingerPrint := sha1.New()
+ pk.SerializeSignaturePrefix(fingerPrint)
+ pk.Serialize(fingerPrint)
+ copy(pk.Fingerprint[:], fingerPrint.Sum())
+ pk.KeyId = binary.BigEndian.Uint64(pk.Fingerprint[12:20])
+
+ return
+}
+
+// parseRSA parses RSA public key material from the given Reader. See RFC 4880,
+// section 5.5.2.
+func (pk *PublicKey) parseRSA(r io.Reader) (err os.Error) {
+ pk.n.bytes, pk.n.bitLength, err = readMPI(r)
+ if err != nil {
+ return
+ }
+ pk.e.bytes, pk.e.bitLength, err = readMPI(r)
+ if err != nil {
+ return
+ }
+
+ if len(pk.e.bytes) > 3 {
+ err = error.UnsupportedError("large public exponent")
+ return
+ }
+ rsa := &rsa.PublicKey{
+ N: new(big.Int).SetBytes(pk.n.bytes),
+ E: 0,
+ }
+ for i := 0; i < len(pk.e.bytes); i++ {
+ rsa.E <<= 8
+ rsa.E |= int(pk.e.bytes[i])
+ }
+ pk.PublicKey = rsa
+ return
+}
+
+// parseRSA parses DSA public key material from the given Reader. See RFC 4880,
+// section 5.5.2.
+func (pk *PublicKey) parseDSA(r io.Reader) (err os.Error) {
+ pk.p.bytes, pk.p.bitLength, err = readMPI(r)
+ if err != nil {
+ return
+ }
+ pk.q.bytes, pk.q.bitLength, err = readMPI(r)
+ if err != nil {
+ return
+ }
+ pk.g.bytes, pk.g.bitLength, err = readMPI(r)
+ if err != nil {
+ return
+ }
+ pk.y.bytes, pk.y.bitLength, err = readMPI(r)
+ if err != nil {
+ return
+ }
+
+ dsa := new(dsa.PublicKey)
+ dsa.P = new(big.Int).SetBytes(pk.p.bytes)
+ dsa.Q = new(big.Int).SetBytes(pk.q.bytes)
+ dsa.G = new(big.Int).SetBytes(pk.g.bytes)
+ dsa.Y = new(big.Int).SetBytes(pk.y.bytes)
+ pk.PublicKey = dsa
+ return
+}
+
+// SerializeSignaturePrefix writes the prefix for this public key to the given Writer.
+// The prefix is used when calculating a signature over this public key. See
+// RFC 4880, section 5.2.4.
+func (pk *PublicKey) SerializeSignaturePrefix(h hash.Hash) {
+ var pLength uint16
+ switch pk.PubKeyAlgo {
+ case PubKeyAlgoRSA, PubKeyAlgoRSAEncryptOnly, PubKeyAlgoRSASignOnly:
+ pLength += 2 + uint16(len(pk.n.bytes))
+ pLength += 2 + uint16(len(pk.e.bytes))
+ case PubKeyAlgoDSA:
+ pLength += 2 + uint16(len(pk.p.bytes))
+ pLength += 2 + uint16(len(pk.q.bytes))
+ pLength += 2 + uint16(len(pk.g.bytes))
+ pLength += 2 + uint16(len(pk.y.bytes))
+ default:
+ panic("unknown public key algorithm")
+ }
+ pLength += 6
+ h.Write([]byte{0x99, byte(pLength >> 8), byte(pLength)})
+ return
+}
+
+// Serialize marshals the PublicKey to w in the form of an OpenPGP public key
+// packet, not including the packet header.
+func (pk *PublicKey) Serialize(w io.Writer) (err os.Error) {
+ var buf [6]byte
+ buf[0] = 4
+ buf[1] = byte(pk.CreationTime >> 24)
+ buf[2] = byte(pk.CreationTime >> 16)
+ buf[3] = byte(pk.CreationTime >> 8)
+ buf[4] = byte(pk.CreationTime)
+ buf[5] = byte(pk.PubKeyAlgo)
+
+ _, err = w.Write(buf[:])
+ if err != nil {
+ return
+ }
+
+ switch pk.PubKeyAlgo {
+ case PubKeyAlgoRSA, PubKeyAlgoRSAEncryptOnly, PubKeyAlgoRSASignOnly:
+ return writeMPIs(w, pk.n, pk.e)
+ case PubKeyAlgoDSA:
+ return writeMPIs(w, pk.p, pk.q, pk.g, pk.y)
+ }
+ return error.InvalidArgumentError("bad public-key algorithm")
+}
+
+// CanSign returns true iff this public key can generate signatures
+func (pk *PublicKey) CanSign() bool {
+ return pk.PubKeyAlgo != PubKeyAlgoRSAEncryptOnly && pk.PubKeyAlgo != PubKeyAlgoElgamal
+}
+
+// VerifySignature returns nil iff sig is a valid signature, made by this
+// public key, of the data hashed into signed. signed is mutated by this call.
+func (pk *PublicKey) VerifySignature(signed hash.Hash, sig *Signature) (err os.Error) {
+ if !pk.CanSign() {
+ return error.InvalidArgumentError("public key cannot generate signatures")
+ }
+
+ rsaPublicKey, ok := pk.PublicKey.(*rsa.PublicKey)
+ if !ok {
+ // TODO(agl): support DSA and ECDSA keys.
+ return error.UnsupportedError("non-RSA public key")
+ }
+
+ signed.Write(sig.HashSuffix)
+ hashBytes := signed.Sum()
+
+ if hashBytes[0] != sig.HashTag[0] || hashBytes[1] != sig.HashTag[1] {
+ return error.SignatureError("hash tag doesn't match")
+ }
+
+ err = rsa.VerifyPKCS1v15(rsaPublicKey, sig.Hash, hashBytes, sig.Signature)
+ if err != nil {
+ return error.SignatureError("RSA verification failure")
+ }
+ return nil
+}
+
+// VerifyKeySignature returns nil iff sig is a valid signature, make by this
+// public key, of the public key in signed.
+func (pk *PublicKey) VerifyKeySignature(signed *PublicKey, sig *Signature) (err os.Error) {
+ h := sig.Hash.New()
+ if h == nil {
+ return error.UnsupportedError("hash function")
+ }
+
+ // RFC 4880, section 5.2.4
+ pk.SerializeSignaturePrefix(h)
+ pk.Serialize(h)
+ signed.SerializeSignaturePrefix(h)
+ signed.Serialize(h)
+
+ return pk.VerifySignature(h, sig)
+}
+
+// VerifyUserIdSignature returns nil iff sig is a valid signature, make by this
+// public key, of the given user id.
+func (pk *PublicKey) VerifyUserIdSignature(id string, sig *Signature) (err os.Error) {
+ h := sig.Hash.New()
+ if h == nil {
+ return error.UnsupportedError("hash function")
+ }
+
+ // RFC 4880, section 5.2.4
+ pk.SerializeSignaturePrefix(h)
+ pk.Serialize(h)
+
+ var buf [5]byte
+ buf[0] = 0xb4
+ buf[1] = byte(len(id) >> 24)
+ buf[2] = byte(len(id) >> 16)
+ buf[3] = byte(len(id) >> 8)
+ buf[4] = byte(len(id))
+ h.Write(buf[:])
+ h.Write([]byte(id))
+
+ return pk.VerifySignature(h, sig)
+}
+
+// A parsedMPI is used to store the contents of a big integer, along with the
+// bit length that was specified in the original input. This allows the MPI to
+// be reserialised exactly.
+type parsedMPI struct {
+ bytes []byte
+ bitLength uint16
+}
+
+// writeMPIs is a utility function for serialising several big integers to the
+// given Writer.
+func writeMPIs(w io.Writer, mpis ...parsedMPI) (err os.Error) {
+ for _, mpi := range mpis {
+ err = writeMPI(w, mpi.bitLength, mpi.bytes)
+ if err != nil {
+ return
+ }
+ }
+ return
+}
--- /dev/null
+// Copyright 2011 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package packet
+
+import (
+ "bytes"
+ "encoding/hex"
+ "testing"
+)
+
+var pubKeyTests = []struct {
+ hexData string
+ hexFingerprint string
+ creationTime uint32
+ pubKeyAlgo PublicKeyAlgorithm
+ keyId uint64
+}{
+ {rsaPkDataHex, rsaFingerprintHex, 0x4d3c5c10, PubKeyAlgoRSA, 0xa34d7e18c20c31bb},
+ {dsaPkDataHex, dsaFingerprintHex, 0x4d432f89, PubKeyAlgoDSA, 0x8e8fbe54062f19ed},
+}
+
+func TestPublicKeyRead(t *testing.T) {
+ for i, test := range pubKeyTests {
+ packet, err := Read(readerFromHex(test.hexData))
+ if err != nil {
+ t.Errorf("#%d: Read error: %s", i, err)
+ return
+ }
+ pk, ok := packet.(*PublicKey)
+ if !ok {
+ t.Errorf("#%d: failed to parse, got: %#v", i, packet)
+ return
+ }
+ if pk.PubKeyAlgo != test.pubKeyAlgo {
+ t.Errorf("#%d: bad public key algorithm got:%x want:%x", i, pk.PubKeyAlgo, test.pubKeyAlgo)
+ }
+ if pk.CreationTime != test.creationTime {
+ t.Errorf("#%d: bad creation time got:%x want:%x", i, pk.CreationTime, test.creationTime)
+ }
+ expectedFingerprint, _ := hex.DecodeString(test.hexFingerprint)
+ if !bytes.Equal(expectedFingerprint, pk.Fingerprint[:]) {
+ t.Errorf("#%d: bad fingerprint got:%x want:%x", i, pk.Fingerprint[:], expectedFingerprint)
+ }
+ if pk.KeyId != test.keyId {
+ t.Errorf("#%d: bad keyid got:%x want:%x", i, pk.KeyId, test.keyId)
+ }
+ }
+}
+
+const rsaFingerprintHex = "5fb74b1d03b1e3cb31bc2f8aa34d7e18c20c31bb"
+
+const rsaPkDataHex = "988d044d3c5c10010400b1d13382944bd5aba23a4312968b5095d14f947f600eb478e14a6fcb16b0e0cac764884909c020bc495cfcc39a935387c661507bdb236a0612fb582cac3af9b29cc2c8c70090616c41b662f4da4c1201e195472eb7f4ae1ccbcbf9940fe21d985e379a5563dde5b9a23d35f1cfaa5790da3b79db26f23695107bfaca8e7b5bcd0011010001"
+
+const dsaFingerprintHex = "eece4c094db002103714c63c8e8fbe54062f19ed"
+
+const dsaPkDataHex = "9901a2044d432f89110400cd581334f0d7a1e1bdc8b9d6d8c0baf68793632735d2bb0903224cbaa1dfbf35a60ee7a13b92643421e1eb41aa8d79bea19a115a677f6b8ba3c7818ce53a6c2a24a1608bd8b8d6e55c5090cbde09dd26e356267465ae25e69ec8bdd57c7bbb2623e4d73336f73a0a9098f7f16da2e25252130fd694c0e8070c55a812a423ae7f00a0ebf50e70c2f19c3520a551bd4b08d30f23530d3d03ff7d0bf4a53a64a09dc5e6e6e35854b7d70c882b0c60293401958b1bd9e40abec3ea05ba87cf64899299d4bd6aa7f459c201d3fbbd6c82004bdc5e8a9eb8082d12054cc90fa9d4ec251a843236a588bf49552441817436c4f43326966fe85447d4e6d0acf8fa1ef0f014730770603ad7634c3088dc52501c237328417c31c89ed70400b2f1a98b0bf42f11fefc430704bebbaa41d9f355600c3facee1e490f64208e0e094ea55e3a598a219a58500bf78ac677b670a14f4e47e9cf8eab4f368cc1ddcaa18cc59309d4cc62dd4f680e73e6cc3e1ce87a84d0925efbcb26c575c093fc42eecf45135fabf6403a25c2016e1774c0484e440a18319072c617cc97ac0a3bb0"
--- /dev/null
+// Copyright 2011 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package packet
+
+import (
+ "crypto/openpgp/error"
+ "io"
+ "os"
+)
+
+// Reader reads packets from an io.Reader and allows packets to be 'unread' so
+// that they result from the next call to Next.
+type Reader struct {
+ q []Packet
+ readers []io.Reader
+}
+
+// Next returns the most recently unread Packet, or reads another packet from
+// the top-most io.Reader. Unknown packet types are skipped.
+func (r *Reader) Next() (p Packet, err os.Error) {
+ if len(r.q) > 0 {
+ p = r.q[len(r.q)-1]
+ r.q = r.q[:len(r.q)-1]
+ return
+ }
+
+ for len(r.readers) > 0 {
+ p, err = Read(r.readers[len(r.readers)-1])
+ if err == nil {
+ return
+ }
+ if err == os.EOF {
+ r.readers = r.readers[:len(r.readers)-1]
+ continue
+ }
+ if _, ok := err.(error.UnknownPacketTypeError); !ok {
+ return nil, err
+ }
+ }
+
+ return nil, os.EOF
+}
+
+// Push causes the Reader to start reading from a new io.Reader. When an EOF
+// error is seen from the new io.Reader, it is popped and the Reader continues
+// to read from the next most recent io.Reader.
+func (r *Reader) Push(reader io.Reader) {
+ r.readers = append(r.readers, reader)
+}
+
+// Unread causes the given Packet to be returned from the next call to Next.
+func (r *Reader) Unread(p Packet) {
+ r.q = append(r.q, p)
+}
+
+func NewReader(r io.Reader) *Reader {
+ return &Reader{
+ q: nil,
+ readers: []io.Reader{r},
+ }
+}
--- /dev/null
+// Copyright 2011 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package packet
+
+import (
+ "crypto"
+ "crypto/openpgp/error"
+ "crypto/openpgp/s2k"
+ "crypto/rand"
+ "crypto/rsa"
+ "encoding/binary"
+ "hash"
+ "io"
+ "os"
+ "strconv"
+)
+
+// Signature represents a signature. See RFC 4880, section 5.2.
+type Signature struct {
+ SigType SignatureType
+ PubKeyAlgo PublicKeyAlgorithm
+ Hash crypto.Hash
+
+ // HashSuffix is extra data that is hashed in after the signed data.
+ HashSuffix []byte
+ // HashTag contains the first two bytes of the hash for fast rejection
+ // of bad signed data.
+ HashTag [2]byte
+ CreationTime uint32 // Unix epoch time
+ Signature []byte
+
+ // The following are optional so are nil when not included in the
+ // signature.
+
+ SigLifetimeSecs, KeyLifetimeSecs *uint32
+ PreferredSymmetric, PreferredHash, PreferredCompression []uint8
+ IssuerKeyId *uint64
+ IsPrimaryId *bool
+
+ // FlagsValid is set if any flags were given. See RFC 4880, section
+ // 5.2.3.21 for details.
+ FlagsValid bool
+ FlagCertify, FlagSign, FlagEncryptCommunications, FlagEncryptStorage bool
+
+ outSubpackets []outputSubpacket
+}
+
+func (sig *Signature) parse(r io.Reader) (err os.Error) {
+ // RFC 4880, section 5.2.3
+ var buf [5]byte
+ _, err = readFull(r, buf[:1])
+ if err != nil {
+ return
+ }
+ if buf[0] != 4 {
+ err = error.UnsupportedError("signature packet version " + strconv.Itoa(int(buf[0])))
+ return
+ }
+
+ _, err = readFull(r, buf[:5])
+ if err != nil {
+ return
+ }
+ sig.SigType = SignatureType(buf[0])
+ sig.PubKeyAlgo = PublicKeyAlgorithm(buf[1])
+ switch sig.PubKeyAlgo {
+ case PubKeyAlgoRSA, PubKeyAlgoRSASignOnly:
+ default:
+ err = error.UnsupportedError("public key algorithm " + strconv.Itoa(int(sig.PubKeyAlgo)))
+ return
+ }
+
+ var ok bool
+ sig.Hash, ok = s2k.HashIdToHash(buf[2])
+ if !ok {
+ return error.UnsupportedError("hash function " + strconv.Itoa(int(buf[2])))
+ }
+
+ hashedSubpacketsLength := int(buf[3])<<8 | int(buf[4])
+ l := 6 + hashedSubpacketsLength
+ sig.HashSuffix = make([]byte, l+6)
+ sig.HashSuffix[0] = 4
+ copy(sig.HashSuffix[1:], buf[:5])
+ hashedSubpackets := sig.HashSuffix[6:l]
+ _, err = readFull(r, hashedSubpackets)
+ if err != nil {
+ return
+ }
+ // See RFC 4880, section 5.2.4
+ trailer := sig.HashSuffix[l:]
+ trailer[0] = 4
+ trailer[1] = 0xff
+ trailer[2] = uint8(l >> 24)
+ trailer[3] = uint8(l >> 16)
+ trailer[4] = uint8(l >> 8)
+ trailer[5] = uint8(l)
+
+ err = parseSignatureSubpackets(sig, hashedSubpackets, true)
+ if err != nil {
+ return
+ }
+
+ _, err = readFull(r, buf[:2])
+ if err != nil {
+ return
+ }
+ unhashedSubpacketsLength := int(buf[0])<<8 | int(buf[1])
+ unhashedSubpackets := make([]byte, unhashedSubpacketsLength)
+ _, err = readFull(r, unhashedSubpackets)
+ if err != nil {
+ return
+ }
+ err = parseSignatureSubpackets(sig, unhashedSubpackets, false)
+ if err != nil {
+ return
+ }
+
+ _, err = readFull(r, sig.HashTag[:2])
+ if err != nil {
+ return
+ }
+
+ // We have already checked that the public key algorithm is RSA.
+ sig.Signature, _, err = readMPI(r)
+ return
+}
+
+// parseSignatureSubpackets parses subpackets of the main signature packet. See
+// RFC 4880, section 5.2.3.1.
+func parseSignatureSubpackets(sig *Signature, subpackets []byte, isHashed bool) (err os.Error) {
+ for len(subpackets) > 0 {
+ subpackets, err = parseSignatureSubpacket(sig, subpackets, isHashed)
+ if err != nil {
+ return
+ }
+ }
+
+ if sig.CreationTime == 0 {
+ err = error.StructuralError("no creation time in signature")
+ }
+
+ return
+}
+
+type signatureSubpacketType uint8
+
+const (
+ creationTimeSubpacket signatureSubpacketType = 2
+ signatureExpirationSubpacket signatureSubpacketType = 3
+ keyExpirySubpacket signatureSubpacketType = 9
+ prefSymmetricAlgosSubpacket signatureSubpacketType = 11
+ issuerSubpacket signatureSubpacketType = 16
+ prefHashAlgosSubpacket signatureSubpacketType = 21
+ prefCompressionSubpacket signatureSubpacketType = 22
+ primaryUserIdSubpacket signatureSubpacketType = 25
+ keyFlagsSubpacket signatureSubpacketType = 27
+)
+
+// parseSignatureSubpacket parses a single subpacket. len(subpacket) is >= 1.
+func parseSignatureSubpacket(sig *Signature, subpacket []byte, isHashed bool) (rest []byte, err os.Error) {
+ // RFC 4880, section 5.2.3.1
+ var length uint32
+ switch {
+ case subpacket[0] < 192:
+ length = uint32(subpacket[0])
+ subpacket = subpacket[1:]
+ case subpacket[0] < 255:
+ if len(subpacket) < 2 {
+ goto Truncated
+ }
+ length = uint32(subpacket[0]-192)<<8 + uint32(subpacket[1]) + 192
+ subpacket = subpacket[2:]
+ default:
+ if len(subpacket) < 5 {
+ goto Truncated
+ }
+ length = uint32(subpacket[1])<<24 |
+ uint32(subpacket[2])<<16 |
+ uint32(subpacket[3])<<8 |
+ uint32(subpacket[4])
+ subpacket = subpacket[5:]
+ }
+ if length > uint32(len(subpacket)) {
+ goto Truncated
+ }
+ rest = subpacket[length:]
+ subpacket = subpacket[:length]
+ if len(subpacket) == 0 {
+ err = error.StructuralError("zero length signature subpacket")
+ return
+ }
+ packetType := subpacket[0] & 0x7f
+ isCritial := subpacket[0]&0x80 == 0x80
+ subpacket = subpacket[1:]
+ switch signatureSubpacketType(packetType) {
+ case creationTimeSubpacket:
+ if !isHashed {
+ err = error.StructuralError("signature creation time in non-hashed area")
+ return
+ }
+ if len(subpacket) != 4 {
+ err = error.StructuralError("signature creation time not four bytes")
+ return
+ }
+ sig.CreationTime = binary.BigEndian.Uint32(subpacket)
+ case signatureExpirationSubpacket:
+ // Signature expiration time, section 5.2.3.10
+ if !isHashed {
+ return
+ }
+ if len(subpacket) != 4 {
+ err = error.StructuralError("expiration subpacket with bad length")
+ return
+ }
+ sig.SigLifetimeSecs = new(uint32)
+ *sig.SigLifetimeSecs = binary.BigEndian.Uint32(subpacket)
+ case keyExpirySubpacket:
+ // Key expiration time, section 5.2.3.6
+ if !isHashed {
+ return
+ }
+ if len(subpacket) != 4 {
+ err = error.StructuralError("key expiration subpacket with bad length")
+ return
+ }
+ sig.KeyLifetimeSecs = new(uint32)
+ *sig.KeyLifetimeSecs = binary.BigEndian.Uint32(subpacket)
+ case prefSymmetricAlgosSubpacket:
+ // Preferred symmetric algorithms, section 5.2.3.7
+ if !isHashed {
+ return
+ }
+ sig.PreferredSymmetric = make([]byte, len(subpacket))
+ copy(sig.PreferredSymmetric, subpacket)
+ case issuerSubpacket:
+ // Issuer, section 5.2.3.5
+ if len(subpacket) != 8 {
+ err = error.StructuralError("issuer subpacket with bad length")
+ return
+ }
+ sig.IssuerKeyId = new(uint64)
+ *sig.IssuerKeyId = binary.BigEndian.Uint64(subpacket)
+ case prefHashAlgosSubpacket:
+ // Preferred hash algorithms, section 5.2.3.8
+ if !isHashed {
+ return
+ }
+ sig.PreferredHash = make([]byte, len(subpacket))
+ copy(sig.PreferredHash, subpacket)
+ case prefCompressionSubpacket:
+ // Preferred compression algorithms, section 5.2.3.9
+ if !isHashed {
+ return
+ }
+ sig.PreferredCompression = make([]byte, len(subpacket))
+ copy(sig.PreferredCompression, subpacket)
+ case primaryUserIdSubpacket:
+ // Primary User ID, section 5.2.3.19
+ if !isHashed {
+ return
+ }
+ if len(subpacket) != 1 {
+ err = error.StructuralError("primary user id subpacket with bad length")
+ return
+ }
+ sig.IsPrimaryId = new(bool)
+ if subpacket[0] > 0 {
+ *sig.IsPrimaryId = true
+ }
+ case keyFlagsSubpacket:
+ // Key flags, section 5.2.3.21
+ if !isHashed {
+ return
+ }
+ if len(subpacket) == 0 {
+ err = error.StructuralError("empty key flags subpacket")
+ return
+ }
+ sig.FlagsValid = true
+ if subpacket[0]&1 != 0 {
+ sig.FlagCertify = true
+ }
+ if subpacket[0]&2 != 0 {
+ sig.FlagSign = true
+ }
+ if subpacket[0]&4 != 0 {
+ sig.FlagEncryptCommunications = true
+ }
+ if subpacket[0]&8 != 0 {
+ sig.FlagEncryptStorage = true
+ }
+
+ default:
+ if isCritial {
+ err = error.UnsupportedError("unknown critical signature subpacket type " + strconv.Itoa(int(packetType)))
+ return
+ }
+ }
+ return
+
+Truncated:
+ err = error.StructuralError("signature subpacket truncated")
+ return
+}
+
+// subpacketLengthLength returns the length, in bytes, of an encoded length value.
+func subpacketLengthLength(length int) int {
+ if length < 192 {
+ return 1
+ }
+ if length < 16320 {
+ return 2
+ }
+ return 5
+}
+
+// serialiseSubpacketLength marshals the given length into to.
+func serialiseSubpacketLength(to []byte, length int) int {
+ if length < 192 {
+ to[0] = byte(length)
+ return 1
+ }
+ if length < 16320 {
+ length -= 192
+ to[0] = byte(length >> 8)
+ to[1] = byte(length)
+ return 2
+ }
+ to[0] = 255
+ to[1] = byte(length >> 24)
+ to[2] = byte(length >> 16)
+ to[3] = byte(length >> 8)
+ to[4] = byte(length)
+ return 5
+}
+
+// subpacketsLength returns the serialised length, in bytes, of the given
+// subpackets.
+func subpacketsLength(subpackets []outputSubpacket, hashed bool) (length int) {
+ for _, subpacket := range subpackets {
+ if subpacket.hashed == hashed {
+ length += subpacketLengthLength(len(subpacket.contents) + 1)
+ length += 1 // type byte
+ length += len(subpacket.contents)
+ }
+ }
+ return
+}
+
+// serialiseSubpackets marshals the given subpackets into to.
+func serialiseSubpackets(to []byte, subpackets []outputSubpacket, hashed bool) {
+ for _, subpacket := range subpackets {
+ if subpacket.hashed == hashed {
+ n := serialiseSubpacketLength(to, len(subpacket.contents)+1)
+ to[n] = byte(subpacket.subpacketType)
+ to = to[1+n:]
+ n = copy(to, subpacket.contents)
+ to = to[n:]
+ }
+ }
+ return
+}
+
+// buildHashSuffix constructs the HashSuffix member of sig in preparation for signing.
+func (sig *Signature) buildHashSuffix() (err os.Error) {
+ sig.outSubpackets = sig.buildSubpackets()
+ hashedSubpacketsLen := subpacketsLength(sig.outSubpackets, true)
+
+ var ok bool
+ l := 6 + hashedSubpacketsLen
+ sig.HashSuffix = make([]byte, l+6)
+ sig.HashSuffix[0] = 4
+ sig.HashSuffix[1] = uint8(sig.SigType)
+ sig.HashSuffix[2] = uint8(sig.PubKeyAlgo)
+ sig.HashSuffix[3], ok = s2k.HashToHashId(sig.Hash)
+ if !ok {
+ sig.HashSuffix = nil
+ return error.InvalidArgumentError("hash cannot be repesented in OpenPGP: " + strconv.Itoa(int(sig.Hash)))
+ }
+ sig.HashSuffix[4] = byte(hashedSubpacketsLen >> 8)
+ sig.HashSuffix[5] = byte(hashedSubpacketsLen)
+ serialiseSubpackets(sig.HashSuffix[6:l], sig.outSubpackets, true)
+ trailer := sig.HashSuffix[l:]
+ trailer[0] = 4
+ trailer[1] = 0xff
+ trailer[2] = byte(l >> 24)
+ trailer[3] = byte(l >> 16)
+ trailer[4] = byte(l >> 8)
+ trailer[5] = byte(l)
+ return
+}
+
+// SignRSA signs a message with an RSA private key. The hash, h, must contain
+// the hash of message to be signed and will be mutated by this function.
+func (sig *Signature) SignRSA(h hash.Hash, priv *rsa.PrivateKey) (err os.Error) {
+ err = sig.buildHashSuffix()
+ if err != nil {
+ return
+ }
+
+ h.Write(sig.HashSuffix)
+ digest := h.Sum()
+ copy(sig.HashTag[:], digest)
+ sig.Signature, err = rsa.SignPKCS1v15(rand.Reader, priv, sig.Hash, digest)
+ return
+}
+
+// Serialize marshals sig to w. SignRSA must have been called first.
+func (sig *Signature) Serialize(w io.Writer) (err os.Error) {
+ if sig.Signature == nil {
+ return error.InvalidArgumentError("Signature: need to call SignRSA before Serialize")
+ }
+
+ unhashedSubpacketsLen := subpacketsLength(sig.outSubpackets, false)
+ length := len(sig.HashSuffix) - 6 /* trailer not included */ +
+ 2 /* length of unhashed subpackets */ + unhashedSubpacketsLen +
+ 2 /* hash tag */ + 2 /* length of signature MPI */ + len(sig.Signature)
+ err = serialiseHeader(w, packetTypeSignature, length)
+ if err != nil {
+ return
+ }
+
+ _, err = w.Write(sig.HashSuffix[:len(sig.HashSuffix)-6])
+ if err != nil {
+ return
+ }
+
+ unhashedSubpackets := make([]byte, 2+unhashedSubpacketsLen)
+ unhashedSubpackets[0] = byte(unhashedSubpacketsLen >> 8)
+ unhashedSubpackets[1] = byte(unhashedSubpacketsLen)
+ serialiseSubpackets(unhashedSubpackets[2:], sig.outSubpackets, false)
+
+ _, err = w.Write(unhashedSubpackets)
+ if err != nil {
+ return
+ }
+ _, err = w.Write(sig.HashTag[:])
+ if err != nil {
+ return
+ }
+ return writeMPI(w, 8*uint16(len(sig.Signature)), sig.Signature)
+}
+
+// outputSubpacket represents a subpacket to be marshaled.
+type outputSubpacket struct {
+ hashed bool // true if this subpacket is in the hashed area.
+ subpacketType signatureSubpacketType
+ contents []byte
+}
+
+func (sig *Signature) buildSubpackets() (subpackets []outputSubpacket) {
+ creationTime := make([]byte, 4)
+ creationTime[0] = byte(sig.CreationTime >> 24)
+ creationTime[1] = byte(sig.CreationTime >> 16)
+ creationTime[2] = byte(sig.CreationTime >> 8)
+ creationTime[3] = byte(sig.CreationTime)
+ subpackets = append(subpackets, outputSubpacket{true, creationTimeSubpacket, creationTime})
+
+ if sig.IssuerKeyId != nil {
+ keyId := make([]byte, 8)
+ binary.BigEndian.PutUint64(keyId, *sig.IssuerKeyId)
+ subpackets = append(subpackets, outputSubpacket{true, issuerSubpacket, keyId})
+ }
+
+ return
+}
--- /dev/null
+// Copyright 2011 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package packet
+
+import (
+ "bytes"
+ "crypto"
+ "encoding/hex"
+ "testing"
+)
+
+func TestSignatureRead(t *testing.T) {
+ signatureData, _ := hex.DecodeString(signatureDataHex)
+ buf := bytes.NewBuffer(signatureData)
+ packet, err := Read(buf)
+ if err != nil {
+ t.Error(err)
+ return
+ }
+ sig, ok := packet.(*Signature)
+ if !ok || sig.SigType != SigTypeBinary || sig.PubKeyAlgo != PubKeyAlgoRSA || sig.Hash != crypto.SHA1 {
+ t.Errorf("failed to parse, got: %#v", packet)
+ }
+}
+
+const signatureDataHex = "89011c04000102000605024cb45112000a0910ab105c91af38fb158f8d07ff5596ea368c5efe015bed6e78348c0f033c931d5f2ce5db54ce7f2a7e4b4ad64db758d65a7a71773edeab7ba2a9e0908e6a94a1175edd86c1d843279f045b021a6971a72702fcbd650efc393c5474d5b59a15f96d2eaad4c4c426797e0dcca2803ef41c6ff234d403eec38f31d610c344c06f2401c262f0993b2e66cad8a81ebc4322c723e0d4ba09fe917e8777658307ad8329adacba821420741009dfe87f007759f0982275d028a392c6ed983a0d846f890b36148c7358bdb8a516007fac760261ecd06076813831a36d0459075d1befa245ae7f7fb103d92ca759e9498fe60ef8078a39a3beda510deea251ea9f0a7f0df6ef42060f20780360686f3e400e"
--- /dev/null
+// Copyright 2011 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package packet
+
+import (
+ "crypto/cipher"
+ "crypto/openpgp/error"
+ "crypto/openpgp/s2k"
+ "io"
+ "os"
+ "strconv"
+)
+
+// This is the largest session key that we'll support. Since no 512-bit cipher
+// has even been seriously used, this is comfortably large.
+const maxSessionKeySizeInBytes = 64
+
+// SymmetricKeyEncrypted represents a passphrase protected session key. See RFC
+// 4880, section 5.3.
+type SymmetricKeyEncrypted struct {
+ CipherFunc CipherFunction
+ Encrypted bool
+ Key []byte // Empty unless Encrypted is false.
+ s2k func(out, in []byte)
+ encryptedKey []byte
+}
+
+func (ske *SymmetricKeyEncrypted) parse(r io.Reader) (err os.Error) {
+ // RFC 4880, section 5.3.
+ var buf [2]byte
+ _, err = readFull(r, buf[:])
+ if err != nil {
+ return
+ }
+ if buf[0] != 4 {
+ return error.UnsupportedError("SymmetricKeyEncrypted version")
+ }
+ ske.CipherFunc = CipherFunction(buf[1])
+
+ if ske.CipherFunc.keySize() == 0 {
+ return error.UnsupportedError("unknown cipher: " + strconv.Itoa(int(buf[1])))
+ }
+
+ ske.s2k, err = s2k.Parse(r)
+ if err != nil {
+ return
+ }
+
+ encryptedKey := make([]byte, maxSessionKeySizeInBytes)
+ // The session key may follow. We just have to try and read to find
+ // out. If it exists then we limit it to maxSessionKeySizeInBytes.
+ n, err := readFull(r, encryptedKey)
+ if err != nil && err != io.ErrUnexpectedEOF {
+ return
+ }
+ err = nil
+ if n != 0 {
+ if n == maxSessionKeySizeInBytes {
+ return error.UnsupportedError("oversized encrypted session key")
+ }
+ ske.encryptedKey = encryptedKey[:n]
+ }
+
+ ske.Encrypted = true
+
+ return
+}
+
+// Decrypt attempts to decrypt an encrypted session key. If it returns nil,
+// ske.Key will contain the session key.
+func (ske *SymmetricKeyEncrypted) Decrypt(passphrase []byte) os.Error {
+ if !ske.Encrypted {
+ return nil
+ }
+
+ key := make([]byte, ske.CipherFunc.keySize())
+ ske.s2k(key, passphrase)
+
+ if len(ske.encryptedKey) == 0 {
+ ske.Key = key
+ } else {
+ // the IV is all zeros
+ iv := make([]byte, ske.CipherFunc.blockSize())
+ c := cipher.NewCFBDecrypter(ske.CipherFunc.new(key), iv)
+ c.XORKeyStream(ske.encryptedKey, ske.encryptedKey)
+ ske.CipherFunc = CipherFunction(ske.encryptedKey[0])
+ if ske.CipherFunc.blockSize() == 0 {
+ return error.UnsupportedError("unknown cipher: " + strconv.Itoa(int(ske.CipherFunc)))
+ }
+ ske.CipherFunc = CipherFunction(ske.encryptedKey[0])
+ ske.Key = ske.encryptedKey[1:]
+ if len(ske.Key)%ske.CipherFunc.blockSize() != 0 {
+ ske.Key = nil
+ return error.StructuralError("length of decrypted key not a multiple of block size")
+ }
+ }
+
+ ske.Encrypted = false
+ return nil
+}
--- /dev/null
+// Copyright 2011 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package packet
+
+import (
+ "bytes"
+ "encoding/hex"
+ "io/ioutil"
+ "os"
+ "testing"
+)
+
+func TestSymmetricKeyEncrypted(t *testing.T) {
+ buf := readerFromHex(symmetricallyEncryptedHex)
+ packet, err := Read(buf)
+ if err != nil {
+ t.Errorf("failed to read SymmetricKeyEncrypted: %s", err)
+ return
+ }
+ ske, ok := packet.(*SymmetricKeyEncrypted)
+ if !ok {
+ t.Error("didn't find SymmetricKeyEncrypted packet")
+ return
+ }
+ err = ske.Decrypt([]byte("password"))
+ if err != nil {
+ t.Error(err)
+ return
+ }
+
+ packet, err = Read(buf)
+ if err != nil {
+ t.Errorf("failed to read SymmetricallyEncrypted: %s", err)
+ return
+ }
+ se, ok := packet.(*SymmetricallyEncrypted)
+ if !ok {
+ t.Error("didn't find SymmetricallyEncrypted packet")
+ return
+ }
+ r, err := se.Decrypt(ske.CipherFunc, ske.Key)
+ if err != nil {
+ t.Error(err)
+ return
+ }
+
+ contents, err := ioutil.ReadAll(r)
+ if err != nil && err != os.EOF {
+ t.Error(err)
+ return
+ }
+
+ expectedContents, _ := hex.DecodeString(symmetricallyEncryptedContentsHex)
+ if !bytes.Equal(expectedContents, contents) {
+ t.Errorf("bad contents got:%x want:%x", contents, expectedContents)
+ }
+}
+
+const symmetricallyEncryptedHex = "8c0d04030302371a0b38d884f02060c91cf97c9973b8e58e028e9501708ccfe618fb92afef7fa2d80ddadd93cf"
+const symmetricallyEncryptedContentsHex = "cb1062004d14c4df636f6e74656e74732e0a"
--- /dev/null
+// Copyright 2011 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package packet
+
+import (
+ "crypto/cipher"
+ "crypto/openpgp/error"
+ "crypto/sha1"
+ "crypto/subtle"
+ "hash"
+ "io"
+ "os"
+ "strconv"
+)
+
+// SymmetricallyEncrypted represents a symmetrically encrypted byte string. The
+// encrypted contents will consist of more OpenPGP packets. See RFC 4880,
+// sections 5.7 and 5.13.
+type SymmetricallyEncrypted struct {
+ MDC bool // true iff this is a type 18 packet and thus has an embedded MAC.
+ contents io.Reader
+ prefix []byte
+}
+
+func (se *SymmetricallyEncrypted) parse(r io.Reader) os.Error {
+ if se.MDC {
+ // See RFC 4880, section 5.13.
+ var buf [1]byte
+ _, err := readFull(r, buf[:])
+ if err != nil {
+ return err
+ }
+ if buf[0] != 1 {
+ return error.UnsupportedError("unknown SymmetricallyEncrypted version")
+ }
+ }
+ se.contents = r
+ return nil
+}
+
+// Decrypt returns a ReadCloser, from which the decrypted contents of the
+// packet can be read. An incorrect key can, with high probability, be detected
+// immediately and this will result in a KeyIncorrect error being returned.
+func (se *SymmetricallyEncrypted) Decrypt(c CipherFunction, key []byte) (io.ReadCloser, os.Error) {
+ keySize := c.keySize()
+ if keySize == 0 {
+ return nil, error.UnsupportedError("unknown cipher: " + strconv.Itoa(int(c)))
+ }
+ if len(key) != keySize {
+ return nil, error.InvalidArgumentError("SymmetricallyEncrypted: incorrect key length")
+ }
+
+ if se.prefix == nil {
+ se.prefix = make([]byte, c.blockSize()+2)
+ _, err := readFull(se.contents, se.prefix)
+ if err != nil {
+ return nil, err
+ }
+ } else if len(se.prefix) != c.blockSize()+2 {
+ return nil, error.InvalidArgumentError("can't try ciphers with different block lengths")
+ }
+
+ ocfbResync := cipher.OCFBResync
+ if se.MDC {
+ // MDC packets use a different form of OCFB mode.
+ ocfbResync = cipher.OCFBNoResync
+ }
+
+ s := cipher.NewOCFBDecrypter(c.new(key), se.prefix, ocfbResync)
+ if s == nil {
+ return nil, error.KeyIncorrectError
+ }
+
+ plaintext := cipher.StreamReader{S: s, R: se.contents}
+
+ if se.MDC {
+ // MDC packets have an embedded hash that we need to check.
+ h := sha1.New()
+ h.Write(se.prefix)
+ return &seMDCReader{in: plaintext, h: h}, nil
+ }
+
+ // Otherwise, we just need to wrap plaintext so that it's a valid ReadCloser.
+ return seReader{plaintext}, nil
+}
+
+// seReader wraps an io.Reader with a no-op Close method.
+type seReader struct {
+ in io.Reader
+}
+
+func (ser seReader) Read(buf []byte) (int, os.Error) {
+ return ser.in.Read(buf)
+}
+
+func (ser seReader) Close() os.Error {
+ return nil
+}
+
+const mdcTrailerSize = 1 /* tag byte */ + 1 /* length byte */ + sha1.Size
+
+// An seMDCReader wraps an io.Reader, maintains a running hash and keeps hold
+// of the most recent 22 bytes (mdcTrailerSize). Upon EOF, those bytes form an
+// MDC packet containing a hash of the previous contents which is checked
+// against the running hash. See RFC 4880, section 5.13.
+type seMDCReader struct {
+ in io.Reader
+ h hash.Hash
+ trailer [mdcTrailerSize]byte
+ scratch [mdcTrailerSize]byte
+ trailerUsed int
+ error bool
+ eof bool
+}
+
+func (ser *seMDCReader) Read(buf []byte) (n int, err os.Error) {
+ if ser.error {
+ err = io.ErrUnexpectedEOF
+ return
+ }
+ if ser.eof {
+ err = os.EOF
+ return
+ }
+
+ // If we haven't yet filled the trailer buffer then we must do that
+ // first.
+ for ser.trailerUsed < mdcTrailerSize {
+ n, err = ser.in.Read(ser.trailer[ser.trailerUsed:])
+ ser.trailerUsed += n
+ if err == os.EOF {
+ if ser.trailerUsed != mdcTrailerSize {
+ n = 0
+ err = io.ErrUnexpectedEOF
+ ser.error = true
+ return
+ }
+ ser.eof = true
+ n = 0
+ return
+ }
+
+ if err != nil {
+ n = 0
+ return
+ }
+ }
+
+ // If it's a short read then we read into a temporary buffer and shift
+ // the data into the caller's buffer.
+ if len(buf) <= mdcTrailerSize {
+ n, err = readFull(ser.in, ser.scratch[:len(buf)])
+ copy(buf, ser.trailer[:n])
+ ser.h.Write(buf[:n])
+ copy(ser.trailer[:], ser.trailer[n:])
+ copy(ser.trailer[mdcTrailerSize-n:], ser.scratch[:])
+ if n < len(buf) {
+ ser.eof = true
+ err = os.EOF
+ }
+ return
+ }
+
+ n, err = ser.in.Read(buf[mdcTrailerSize:])
+ copy(buf, ser.trailer[:])
+ ser.h.Write(buf[:n])
+ copy(ser.trailer[:], buf[n:])
+
+ if err == os.EOF {
+ ser.eof = true
+ }
+ return
+}
+
+func (ser *seMDCReader) Close() os.Error {
+ if ser.error {
+ return error.SignatureError("error during reading")
+ }
+
+ for !ser.eof {
+ // We haven't seen EOF so we need to read to the end
+ var buf [1024]byte
+ _, err := ser.Read(buf[:])
+ if err == os.EOF {
+ break
+ }
+ if err != nil {
+ return error.SignatureError("error during reading")
+ }
+ }
+
+ // This is a new-format packet tag byte for a type 19 (MDC) packet.
+ const mdcPacketTagByte = byte(0x80) | 0x40 | 19
+ if ser.trailer[0] != mdcPacketTagByte || ser.trailer[1] != sha1.Size {
+ return error.SignatureError("MDC packet not found")
+ }
+ ser.h.Write(ser.trailer[:2])
+
+ final := ser.h.Sum()
+ if subtle.ConstantTimeCompare(final, ser.trailer[2:]) == 1 {
+ return error.SignatureError("hash mismatch")
+ }
+ return nil
+}
--- /dev/null
+// Copyright 2011 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package packet
+
+import (
+ "bytes"
+ "crypto/openpgp/error"
+ "crypto/sha1"
+ "encoding/hex"
+ "io/ioutil"
+ "os"
+ "testing"
+)
+
+// TestReader wraps a []byte and returns reads of a specific length.
+type testReader struct {
+ data []byte
+ stride int
+}
+
+func (t *testReader) Read(buf []byte) (n int, err os.Error) {
+ n = t.stride
+ if n > len(t.data) {
+ n = len(t.data)
+ }
+ if n > len(buf) {
+ n = len(buf)
+ }
+ copy(buf, t.data)
+ t.data = t.data[n:]
+ if len(t.data) == 0 {
+ err = os.EOF
+ }
+ return
+}
+
+func testMDCReader(t *testing.T) {
+ mdcPlaintext, _ := hex.DecodeString(mdcPlaintextHex)
+
+ for stride := 1; stride < len(mdcPlaintext)/2; stride++ {
+ r := &testReader{data: mdcPlaintext, stride: stride}
+ mdcReader := &seMDCReader{in: r, h: sha1.New()}
+ body, err := ioutil.ReadAll(mdcReader)
+ if err != nil {
+ t.Errorf("stride: %d, error: %s", stride, err)
+ continue
+ }
+ if !bytes.Equal(body, mdcPlaintext[:len(mdcPlaintext)-22]) {
+ t.Errorf("stride: %d: bad contents %x", stride, body)
+ continue
+ }
+
+ err = mdcReader.Close()
+ if err != nil {
+ t.Errorf("stride: %d, error on Close: %s", stride, err)
+ }
+ }
+
+ mdcPlaintext[15] ^= 80
+
+ r := &testReader{data: mdcPlaintext, stride: 2}
+ mdcReader := &seMDCReader{in: r, h: sha1.New()}
+ _, err := ioutil.ReadAll(mdcReader)
+ if err != nil {
+ t.Errorf("corruption test, error: %s", err)
+ return
+ }
+ err = mdcReader.Close()
+ if err == nil {
+ t.Error("corruption: no error")
+ } else if _, ok := err.(*error.SignatureError); !ok {
+ t.Errorf("corruption: expected SignatureError, got: %s", err)
+ }
+}
+
+const mdcPlaintextHex = "a302789c3b2d93c4e0eb9aba22283539b3203335af44a134afb800c849cb4c4de10200aff40b45d31432c80cb384299a0655966d6939dfdeed1dddf980"
--- /dev/null
+// Copyright 2011 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package packet
+
+import (
+ "io"
+ "io/ioutil"
+ "os"
+ "strings"
+)
+
+// UserId contains text that is intended to represent the name and email
+// address of the key holder. See RFC 4880, section 5.11. By convention, this
+// takes the form "Full Name (Comment) <email@example.com>"
+type UserId struct {
+ Id string // By convention, this takes the form "Full Name (Comment) <email@example.com>" which is split out in the fields below.
+
+ Name, Comment, Email string
+}
+
+func (uid *UserId) parse(r io.Reader) (err os.Error) {
+ // RFC 4880, section 5.11
+ b, err := ioutil.ReadAll(r)
+ if err != nil {
+ return
+ }
+ uid.Id = string(b)
+ uid.Name, uid.Comment, uid.Email = parseUserId(uid.Id)
+ return
+}
+
+// parseUserId extracts the name, comment and email from a user id string that
+// is formatted as "Full Name (Comment) <email@example.com>".
+func parseUserId(id string) (name, comment, email string) {
+ var n, c, e struct {
+ start, end int
+ }
+ var state int
+
+ for offset, rune := range id {
+ switch state {
+ case 0:
+ // Entering name
+ n.start = offset
+ state = 1
+ fallthrough
+ case 1:
+ // In name
+ if rune == '(' {
+ state = 2
+ n.end = offset
+ } else if rune == '<' {
+ state = 5
+ n.end = offset
+ }
+ case 2:
+ // Entering comment
+ c.start = offset
+ state = 3
+ fallthrough
+ case 3:
+ // In comment
+ if rune == ')' {
+ state = 4
+ c.end = offset
+ }
+ case 4:
+ // Between comment and email
+ if rune == '<' {
+ state = 5
+ }
+ case 5:
+ // Entering email
+ e.start = offset
+ state = 6
+ fallthrough
+ case 6:
+ // In email
+ if rune == '>' {
+ state = 7
+ e.end = offset
+ }
+ default:
+ // After email
+ }
+ }
+ switch state {
+ case 1:
+ // ended in the name
+ n.end = len(id)
+ case 3:
+ // ended in comment
+ c.end = len(id)
+ case 6:
+ // ended in email
+ e.end = len(id)
+ }
+
+ name = strings.TrimSpace(id[n.start:n.end])
+ comment = strings.TrimSpace(id[c.start:c.end])
+ email = strings.TrimSpace(id[e.start:e.end])
+ return
+}
--- /dev/null
+// Copyright 2011 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package packet
+
+import (
+ "testing"
+)
+
+var userIdTests = []struct {
+ id string
+ name, comment, email string
+}{
+ {"", "", "", ""},
+ {"John Smith", "John Smith", "", ""},
+ {"John Smith ()", "John Smith", "", ""},
+ {"John Smith () <>", "John Smith", "", ""},
+ {"(comment", "", "comment", ""},
+ {"(comment)", "", "comment", ""},
+ {"<email", "", "", "email"},
+ {"<email> sdfk", "", "", "email"},
+ {" John Smith ( Comment ) asdkflj < email > lksdfj", "John Smith", "Comment", "email"},
+ {" John Smith < email > lksdfj", "John Smith", "", "email"},
+ {"(<foo", "", "<foo", ""},
+ {"René Descartes (العربي)", "René Descartes", "العربي", ""},
+}
+
+func TestParseUserId(t *testing.T) {
+ for i, test := range userIdTests {
+ name, comment, email := parseUserId(test.id)
+ if name != test.name {
+ t.Errorf("%d: name mismatch got:%s want:%s", i, name, test.name)
+ }
+ if comment != test.comment {
+ t.Errorf("%d: comment mismatch got:%s want:%s", i, comment, test.comment)
+ }
+ if email != test.email {
+ t.Errorf("%d: email mismatch got:%s want:%s", i, email, test.email)
+ }
+ }
+}
--- /dev/null
+// Copyright 2011 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// This openpgp package implements high level operations on OpenPGP messages.
+package openpgp
+
+import (
+ "crypto"
+ "crypto/openpgp/armor"
+ "crypto/openpgp/error"
+ "crypto/openpgp/packet"
+ "crypto/rsa"
+ _ "crypto/sha256"
+ "hash"
+ "io"
+ "os"
+ "strconv"
+)
+
+// SignatureType is the armor type for a PGP signature.
+var SignatureType = "PGP SIGNATURE"
+
+// readArmored reads an armored block with the given type.
+func readArmored(r io.Reader, expectedType string) (body io.Reader, err os.Error) {
+ block, err := armor.Decode(r)
+ if err != nil {
+ return
+ }
+
+ if block.Type != expectedType {
+ return nil, error.InvalidArgumentError("expected '" + expectedType + "', got: " + block.Type)
+ }
+
+ return block.Body, nil
+}
+
+// MessageDetails contains the result of parsing an OpenPGP encrypted and/or
+// signed message.
+type MessageDetails struct {
+ IsEncrypted bool // true if the message was encrypted.
+ EncryptedToKeyIds []uint64 // the list of recipient key ids.
+ IsSymmetricallyEncrypted bool // true if a passphrase could have decrypted the message.
+ DecryptedWith Key // the private key used to decrypt the message, if any.
+ IsSigned bool // true if the message is signed.
+ SignedByKeyId uint64 // the key id of the signer, if any.
+ SignedBy *Key // the key of the signer, if availible.
+ LiteralData *packet.LiteralData // the metadata of the contents
+ UnverifiedBody io.Reader // the contents of the message.
+
+ // If IsSigned is true and SignedBy is non-zero then the signature will
+ // be verified as UnverifiedBody is read. The signature cannot be
+ // checked until the whole of UnverifiedBody is read so UnverifiedBody
+ // must be consumed until EOF before the data can trusted. Even if a
+ // message isn't signed (or the signer is unknown) the data may contain
+ // an authentication code that is only checked once UnverifiedBody has
+ // been consumed. Once EOF has been seen, the following fields are
+ // valid. (An authentication code failure is reported as a
+ // SignatureError error when reading from UnverifiedBody.)
+
+ SignatureError os.Error // nil if the signature is good.
+ Signature *packet.Signature // the signature packet itself.
+
+ decrypted io.ReadCloser
+}
+
+// A PromptFunction is used as a callback by functions that may need to decrypt
+// a private key, or prompt for a passphrase. It is called with a list of
+// acceptable, encrypted private keys and a boolean that indicates whether a
+// passphrase is usable. It should either decrypt a private key or return a
+// passphrase to try. If the decrypted private key or given passphrase isn't
+// correct, the function will be called again, forever. Any error returned will
+// be passed up.
+type PromptFunction func(keys []Key, symmetric bool) ([]byte, os.Error)
+
+// A keyEnvelopePair is used to store a private key with the envelope that
+// contains a symmetric key, encrypted with that key.
+type keyEnvelopePair struct {
+ key Key
+ encryptedKey *packet.EncryptedKey
+}
+
+// ReadMessage parses an OpenPGP message that may be signed and/or encrypted.
+// The given KeyRing should contain both public keys (for signature
+// verification) and, possibly encrypted, private keys for decrypting.
+func ReadMessage(r io.Reader, keyring KeyRing, prompt PromptFunction) (md *MessageDetails, err os.Error) {
+ var p packet.Packet
+
+ var symKeys []*packet.SymmetricKeyEncrypted
+ var pubKeys []keyEnvelopePair
+ var se *packet.SymmetricallyEncrypted
+
+ packets := packet.NewReader(r)
+ md = new(MessageDetails)
+ md.IsEncrypted = true
+
+ // The message, if encrypted, starts with a number of packets
+ // containing an encrypted decryption key. The decryption key is either
+ // encrypted to a public key, or with a passphrase. This loop
+ // collects these packets.
+ParsePackets:
+ for {
+ p, err = packets.Next()
+ if err != nil {
+ return nil, err
+ }
+ switch p := p.(type) {
+ case *packet.SymmetricKeyEncrypted:
+ // This packet contains the decryption key encrypted with a passphrase.
+ md.IsSymmetricallyEncrypted = true
+ symKeys = append(symKeys, p)
+ case *packet.EncryptedKey:
+ // This packet contains the decryption key encrypted to a public key.
+ md.EncryptedToKeyIds = append(md.EncryptedToKeyIds, p.KeyId)
+ if p.Algo != packet.PubKeyAlgoRSA && p.Algo != packet.PubKeyAlgoRSAEncryptOnly {
+ continue
+ }
+ var keys []Key
+ if p.KeyId == 0 {
+ keys = keyring.DecryptionKeys()
+ } else {
+ keys = keyring.KeysById(p.KeyId)
+ }
+ for _, k := range keys {
+ pubKeys = append(pubKeys, keyEnvelopePair{k, p})
+ }
+ case *packet.SymmetricallyEncrypted:
+ se = p
+ break ParsePackets
+ case *packet.Compressed, *packet.LiteralData, *packet.OnePassSignature:
+ // This message isn't encrypted.
+ if len(symKeys) != 0 || len(pubKeys) != 0 {
+ return nil, error.StructuralError("key material not followed by encrypted message")
+ }
+ packets.Unread(p)
+ return readSignedMessage(packets, nil, keyring)
+ }
+ }
+
+ var candidates []Key
+ var decrypted io.ReadCloser
+
+ // Now that we have the list of encrypted keys we need to decrypt at
+ // least one of them or, if we cannot, we need to call the prompt
+ // function so that it can decrypt a key or give us a passphrase.
+FindKey:
+ for {
+ // See if any of the keys already have a private key availible
+ candidates = candidates[:0]
+ candidateFingerprints := make(map[string]bool)
+
+ for _, pk := range pubKeys {
+ if pk.key.PrivateKey == nil {
+ continue
+ }
+ if !pk.key.PrivateKey.Encrypted {
+ if len(pk.encryptedKey.Key) == 0 {
+ pk.encryptedKey.DecryptRSA(pk.key.PrivateKey.PrivateKey.(*rsa.PrivateKey))
+ }
+ if len(pk.encryptedKey.Key) == 0 {
+ continue
+ }
+ decrypted, err = se.Decrypt(pk.encryptedKey.CipherFunc, pk.encryptedKey.Key)
+ if err != nil && err != error.KeyIncorrectError {
+ return nil, err
+ }
+ if decrypted != nil {
+ md.DecryptedWith = pk.key
+ break FindKey
+ }
+ } else {
+ fpr := string(pk.key.PublicKey.Fingerprint[:])
+ if v := candidateFingerprints[fpr]; v {
+ continue
+ }
+ candidates = append(candidates, pk.key)
+ candidateFingerprints[fpr] = true
+ }
+ }
+
+ if len(candidates) == 0 && len(symKeys) == 0 {
+ return nil, error.KeyIncorrectError
+ }
+
+ if prompt == nil {
+ return nil, error.KeyIncorrectError
+ }
+
+ passphrase, err := prompt(candidates, len(symKeys) != 0)
+ if err != nil {
+ return nil, err
+ }
+
+ // Try the symmetric passphrase first
+ if len(symKeys) != 0 && passphrase != nil {
+ for _, s := range symKeys {
+ err = s.Decrypt(passphrase)
+ if err == nil && !s.Encrypted {
+ decrypted, err = se.Decrypt(s.CipherFunc, s.Key)
+ if err != nil && err != error.KeyIncorrectError {
+ return nil, err
+ }
+ if decrypted != nil {
+ break FindKey
+ }
+ }
+
+ }
+ }
+ }
+
+ md.decrypted = decrypted
+ packets.Push(decrypted)
+ return readSignedMessage(packets, md, keyring)
+}
+
+// readSignedMessage reads a possibily signed message if mdin is non-zero then
+// that structure is updated and returned. Otherwise a fresh MessageDetails is
+// used.
+func readSignedMessage(packets *packet.Reader, mdin *MessageDetails, keyring KeyRing) (md *MessageDetails, err os.Error) {
+ if mdin == nil {
+ mdin = new(MessageDetails)
+ }
+ md = mdin
+
+ var p packet.Packet
+ var h hash.Hash
+ var wrappedHash hash.Hash
+FindLiteralData:
+ for {
+ p, err = packets.Next()
+ if err != nil {
+ return nil, err
+ }
+ switch p := p.(type) {
+ case *packet.Compressed:
+ packets.Push(p.Body)
+ case *packet.OnePassSignature:
+ if !p.IsLast {
+ return nil, error.UnsupportedError("nested signatures")
+ }
+
+ h, wrappedHash, err = hashForSignature(p.Hash, p.SigType)
+ if err != nil {
+ md = nil
+ return
+ }
+
+ md.IsSigned = true
+ md.SignedByKeyId = p.KeyId
+ keys := keyring.KeysById(p.KeyId)
+ for _, key := range keys {
+ if key.SelfSignature.FlagsValid && !key.SelfSignature.FlagSign {
+ continue
+ }
+ md.SignedBy = &key
+ }
+ case *packet.LiteralData:
+ md.LiteralData = p
+ break FindLiteralData
+ }
+ }
+
+ if md.SignedBy != nil {
+ md.UnverifiedBody = &signatureCheckReader{packets, h, wrappedHash, md}
+ } else if md.decrypted != nil {
+ md.UnverifiedBody = checkReader{md}
+ } else {
+ md.UnverifiedBody = md.LiteralData.Body
+ }
+
+ return md, nil
+}
+
+// hashForSignature returns a pair of hashes that can be used to verify a
+// signature. The signature may specify that the contents of the signed message
+// should be preprocessed (i.e. to normalise line endings). Thus this function
+// returns two hashes. The second should be used to hash the message itself and
+// performs any needed preprocessing.
+func hashForSignature(hashId crypto.Hash, sigType packet.SignatureType) (hash.Hash, hash.Hash, os.Error) {
+ h := hashId.New()
+ if h == nil {
+ return nil, nil, error.UnsupportedError("hash not availible: " + strconv.Itoa(int(hashId)))
+ }
+
+ switch sigType {
+ case packet.SigTypeBinary:
+ return h, h, nil
+ case packet.SigTypeText:
+ return h, NewCanonicalTextHash(h), nil
+ }
+
+ return nil, nil, error.UnsupportedError("unsupported signature type: " + strconv.Itoa(int(sigType)))
+}
+
+// checkReader wraps an io.Reader from a LiteralData packet. When it sees EOF
+// it closes the ReadCloser from any SymmetricallyEncrypted packet to trigger
+// MDC checks.
+type checkReader struct {
+ md *MessageDetails
+}
+
+func (cr checkReader) Read(buf []byte) (n int, err os.Error) {
+ n, err = cr.md.LiteralData.Body.Read(buf)
+ if err == os.EOF {
+ mdcErr := cr.md.decrypted.Close()
+ if mdcErr != nil {
+ err = mdcErr
+ }
+ }
+ return
+}
+
+// signatureCheckReader wraps an io.Reader from a LiteralData packet and hashes
+// the data as it is read. When it sees an EOF from the underlying io.Reader
+// it parses and checks a trailing Signature packet and triggers any MDC checks.
+type signatureCheckReader struct {
+ packets *packet.Reader
+ h, wrappedHash hash.Hash
+ md *MessageDetails
+}
+
+func (scr *signatureCheckReader) Read(buf []byte) (n int, err os.Error) {
+ n, err = scr.md.LiteralData.Body.Read(buf)
+ scr.wrappedHash.Write(buf[:n])
+ if err == os.EOF {
+ var p packet.Packet
+ p, scr.md.SignatureError = scr.packets.Next()
+ if scr.md.SignatureError != nil {
+ return
+ }
+
+ var ok bool
+ if scr.md.Signature, ok = p.(*packet.Signature); !ok {
+ scr.md.SignatureError = error.StructuralError("LiteralData not followed by Signature")
+ return
+ }
+
+ scr.md.SignatureError = scr.md.SignedBy.PublicKey.VerifySignature(scr.h, scr.md.Signature)
+
+ // The SymmetricallyEncrypted packet, if any, might have an
+ // unsigned hash of its own. In order to check this we need to
+ // close that Reader.
+ if scr.md.decrypted != nil {
+ mdcErr := scr.md.decrypted.Close()
+ if mdcErr != nil {
+ err = mdcErr
+ }
+ }
+ }
+ return
+}
+
+// CheckDetachedSignature takes a signed file and a detached signature and
+// returns the signer if the signature is valid. If the signer isn't know,
+// UnknownIssuerError is returned.
+func CheckDetachedSignature(keyring KeyRing, signed, signature io.Reader) (signer *Entity, err os.Error) {
+ p, err := packet.Read(signature)
+ if err != nil {
+ return
+ }
+
+ sig, ok := p.(*packet.Signature)
+ if !ok {
+ return nil, error.StructuralError("non signature packet found")
+ }
+
+ if sig.IssuerKeyId == nil {
+ return nil, error.StructuralError("signature doesn't have an issuer")
+ }
+
+ keys := keyring.KeysById(*sig.IssuerKeyId)
+ if len(keys) == 0 {
+ return nil, error.UnknownIssuerError
+ }
+
+ h, wrappedHash, err := hashForSignature(sig.Hash, sig.SigType)
+ if err != nil {
+ return
+ }
+
+ _, err = io.Copy(wrappedHash, signed)
+ if err != nil && err != os.EOF {
+ return
+ }
+
+ for _, key := range keys {
+ if key.SelfSignature.FlagsValid && !key.SelfSignature.FlagSign {
+ continue
+ }
+ err = key.PublicKey.VerifySignature(h, sig)
+ if err == nil {
+ return key.Entity, nil
+ }
+ }
+
+ if err != nil {
+ return
+ }
+
+ return nil, error.UnknownIssuerError
+}
+
+// CheckArmoredDetachedSignature performs the same actions as
+// CheckDetachedSignature but expects the signature to be armored.
+func CheckArmoredDetachedSignature(keyring KeyRing, signed, signature io.Reader) (signer *Entity, err os.Error) {
+ body, err := readArmored(signature, SignatureType)
+ if err != nil {
+ return
+ }
+
+ return CheckDetachedSignature(keyring, signed, body)
+}
--- /dev/null
+// Copyright 2011 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package openpgp
+
+import (
+ "bytes"
+ "crypto/openpgp/error"
+ "encoding/hex"
+ "io"
+ "io/ioutil"
+ "os"
+ "testing"
+)
+
+func readerFromHex(s string) io.Reader {
+ data, err := hex.DecodeString(s)
+ if err != nil {
+ panic("readerFromHex: bad input")
+ }
+ return bytes.NewBuffer(data)
+}
+
+func TestReadKeyRing(t *testing.T) {
+ kring, err := ReadKeyRing(readerFromHex(testKeys1And2Hex))
+ if err != nil {
+ t.Error(err)
+ return
+ }
+ if len(kring) != 2 || uint32(kring[0].PrimaryKey.KeyId) != 0xC20C31BB || uint32(kring[1].PrimaryKey.KeyId) != 0x1E35246B {
+ t.Errorf("bad keyring: %#v", kring)
+ }
+}
+
+func TestReadPrivateKeyRing(t *testing.T) {
+ kring, err := ReadKeyRing(readerFromHex(testKeys1And2PrivateHex))
+ if err != nil {
+ t.Error(err)
+ return
+ }
+ if len(kring) != 2 || uint32(kring[0].PrimaryKey.KeyId) != 0xC20C31BB || uint32(kring[1].PrimaryKey.KeyId) != 0x1E35246B || kring[0].PrimaryKey == nil {
+ t.Errorf("bad keyring: %#v", kring)
+ }
+}
+
+func TestGetKeyById(t *testing.T) {
+ kring, _ := ReadKeyRing(readerFromHex(testKeys1And2Hex))
+
+ keys := kring.KeysById(0xa34d7e18c20c31bb)
+ if len(keys) != 1 || keys[0].Entity != kring[0] {
+ t.Errorf("bad result for 0xa34d7e18c20c31bb: %#v", keys)
+ }
+
+ keys = kring.KeysById(0xfd94408d4543314f)
+ if len(keys) != 1 || keys[0].Entity != kring[0] {
+ t.Errorf("bad result for 0xa34d7e18c20c31bb: %#v", keys)
+ }
+}
+
+func checkSignedMessage(t *testing.T, signedHex, expected string) {
+ kring, _ := ReadKeyRing(readerFromHex(testKeys1And2Hex))
+
+ md, err := ReadMessage(readerFromHex(signedHex), kring, nil)
+ if err != nil {
+ t.Error(err)
+ return
+ }
+
+ if !md.IsSigned || md.SignedByKeyId != 0xa34d7e18c20c31bb || md.SignedBy == nil || md.IsEncrypted || md.IsSymmetricallyEncrypted || len(md.EncryptedToKeyIds) != 0 || md.IsSymmetricallyEncrypted {
+ t.Errorf("bad MessageDetails: %#v", md)
+ }
+
+ contents, err := ioutil.ReadAll(md.UnverifiedBody)
+ if err != nil {
+ t.Errorf("error reading UnverifiedBody: %s", err)
+ }
+ if string(contents) != expected {
+ t.Errorf("bad UnverifiedBody got:%s want:%s", string(contents), expected)
+ }
+ if md.SignatureError != nil || md.Signature == nil {
+ t.Errorf("failed to validate: %s", md.SignatureError)
+ }
+}
+
+func TestSignedMessage(t *testing.T) {
+ checkSignedMessage(t, signedMessageHex, signedInput)
+}
+
+func TestTextSignedMessage(t *testing.T) {
+ checkSignedMessage(t, signedTextMessageHex, signedTextInput)
+}
+
+func TestSignedEncryptedMessage(t *testing.T) {
+ expected := "Signed and encrypted message\n"
+ kring, _ := ReadKeyRing(readerFromHex(testKeys1And2PrivateHex))
+ prompt := func(keys []Key, symmetric bool) ([]byte, os.Error) {
+ if symmetric {
+ t.Errorf("prompt: message was marked as symmetrically encrypted")
+ return nil, error.KeyIncorrectError
+ }
+
+ if len(keys) == 0 {
+ t.Error("prompt: no keys requested")
+ return nil, error.KeyIncorrectError
+ }
+
+ err := keys[0].PrivateKey.Decrypt([]byte("passphrase"))
+ if err != nil {
+ t.Errorf("prompt: error decrypting key: %s", err)
+ return nil, error.KeyIncorrectError
+ }
+
+ return nil, nil
+ }
+
+ md, err := ReadMessage(readerFromHex(signedEncryptedMessageHex), kring, prompt)
+ if err != nil {
+ t.Errorf("error reading message: %s", err)
+ return
+ }
+
+ if !md.IsSigned || md.SignedByKeyId != 0xa34d7e18c20c31bb || md.SignedBy == nil || !md.IsEncrypted || md.IsSymmetricallyEncrypted || len(md.EncryptedToKeyIds) == 0 || md.EncryptedToKeyIds[0] != 0x2a67d68660df41c7 {
+ t.Errorf("bad MessageDetails: %#v", md)
+ }
+
+ contents, err := ioutil.ReadAll(md.UnverifiedBody)
+ if err != nil {
+ t.Errorf("error reading UnverifiedBody: %s", err)
+ }
+ if string(contents) != expected {
+ t.Errorf("bad UnverifiedBody got:%s want:%s", string(contents), expected)
+ }
+
+ if md.SignatureError != nil || md.Signature == nil {
+ t.Errorf("failed to validate: %s", md.SignatureError)
+ }
+}
+
+func TestUnspecifiedRecipient(t *testing.T) {
+ expected := "Recipient unspecified\n"
+ kring, _ := ReadKeyRing(readerFromHex(testKeys1And2PrivateHex))
+
+ md, err := ReadMessage(readerFromHex(recipientUnspecifiedHex), kring, nil)
+ if err != nil {
+ t.Errorf("error reading message: %s", err)
+ return
+ }
+
+ contents, err := ioutil.ReadAll(md.UnverifiedBody)
+ if err != nil {
+ t.Errorf("error reading UnverifiedBody: %s", err)
+ }
+ if string(contents) != expected {
+ t.Errorf("bad UnverifiedBody got:%s want:%s", string(contents), expected)
+ }
+}
+
+func TestSymmetricallyEncrypted(t *testing.T) {
+ expected := "Symmetrically encrypted.\n"
+
+ prompt := func(keys []Key, symmetric bool) ([]byte, os.Error) {
+ if len(keys) != 0 {
+ t.Errorf("prompt: len(keys) = %d (want 0)", len(keys))
+ }
+
+ if !symmetric {
+ t.Errorf("symmetric is not set")
+ }
+
+ return []byte("password"), nil
+ }
+
+ md, err := ReadMessage(readerFromHex(symmetricallyEncryptedCompressedHex), nil, prompt)
+ if err != nil {
+ t.Errorf("ReadMessage: %s", err)
+ return
+ }
+
+ contents, err := ioutil.ReadAll(md.UnverifiedBody)
+ if err != nil {
+ t.Errorf("ReadAll: %s", err)
+ }
+
+ expectedCreatationTime := uint32(1295992998)
+ if md.LiteralData.Time != expectedCreatationTime {
+ t.Errorf("LiteralData.Time is %d, want %d", md.LiteralData.Time, expectedCreatationTime)
+ }
+
+ if string(contents) != expected {
+ t.Errorf("contents got: %s want: %s", string(contents), expected)
+ }
+}
+
+func testDetachedSignature(t *testing.T, kring KeyRing, signature io.Reader, sigInput, tag string) {
+ signed := bytes.NewBufferString(sigInput)
+ signer, err := CheckDetachedSignature(kring, signed, signature)
+ if err != nil {
+ t.Errorf("%s: signature error: %s", tag, err)
+ return
+ }
+ if signer == nil {
+ t.Errorf("%s: signer is nil", tag)
+ return
+ }
+ expectedSignerKeyId := uint64(0xa34d7e18c20c31bb)
+ if signer.PrimaryKey.KeyId != expectedSignerKeyId {
+ t.Errorf("%s: wrong signer got:%x want:%x", tag, signer.PrimaryKey.KeyId, expectedSignerKeyId)
+ }
+}
+
+func TestDetachedSignature(t *testing.T) {
+ kring, _ := ReadKeyRing(readerFromHex(testKeys1And2Hex))
+ testDetachedSignature(t, kring, readerFromHex(detachedSignatureHex), signedInput, "binary")
+ testDetachedSignature(t, kring, readerFromHex(detachedSignatureTextHex), signedInput, "text")
+}
+
+const signedInput = "Signed message\nline 2\nline 3\n"
+const signedTextInput = "Signed message\r\nline 2\r\nline 3\r\n"
+
+const recipientUnspecifiedHex = "848c0300000000000000000103ff62d4d578d03cf40c3da998dfe216c074fa6ddec5e31c197c9666ba292830d91d18716a80f699f9d897389a90e6d62d0238f5f07a5248073c0f24920e4bc4a30c2d17ee4e0cae7c3d4aaa4e8dced50e3010a80ee692175fa0385f62ecca4b56ee6e9980aa3ec51b61b077096ac9e800edaf161268593eedb6cc7027ff5cb32745d250010d407a6221ae22ef18469b444f2822478c4d190b24d36371a95cb40087cdd42d9399c3d06a53c0673349bfb607927f20d1e122bde1e2bf3aa6cae6edf489629bcaa0689539ae3b718914d88ededc3b"
+
+const detachedSignatureHex = "889c04000102000605024d449cd1000a0910a34d7e18c20c31bb167603ff57718d09f28a519fdc7b5a68b6a3336da04df85e38c5cd5d5bd2092fa4629848a33d85b1729402a2aab39c3ac19f9d573f773cc62c264dc924c067a79dfd8a863ae06c7c8686120760749f5fd9b1e03a64d20a7df3446ddc8f0aeadeaeba7cbaee5c1e366d65b6a0c6cc749bcb912d2f15013f812795c2e29eb7f7b77f39ce77"
+
+const detachedSignatureTextHex = "889c04010102000605024d449d21000a0910a34d7e18c20c31bbc8c60400a24fbef7342603a41cb1165767bd18985d015fb72fe05db42db36cfb2f1d455967f1e491194fbf6cf88146222b23bf6ffbd50d17598d976a0417d3192ff9cc0034fd00f287b02e90418bbefe609484b09231e4e7a5f3562e199bf39909ab5276c4d37382fe088f6b5c3426fc1052865da8b3ab158672d58b6264b10823dc4b39"
+
+const testKeys1And2Hex = "988d044d3c5c10010400b1d13382944bd5aba23a4312968b5095d14f947f600eb478e14a6fcb16b0e0cac764884909c020bc495cfcc39a935387c661507bdb236a0612fb582cac3af9b29cc2c8c70090616c41b662f4da4c1201e195472eb7f4ae1ccbcbf9940fe21d985e379a5563dde5b9a23d35f1cfaa5790da3b79db26f23695107bfaca8e7b5bcd0011010001b41054657374204b6579203120285253412988b804130102002205024d3c5c10021b03060b090807030206150802090a0b0416020301021e01021780000a0910a34d7e18c20c31bbb5b304009cc45fe610b641a2c146331be94dade0a396e73ca725e1b25c21708d9cab46ecca5ccebc23055879df8f99eea39b377962a400f2ebdc36a7c99c333d74aeba346315137c3ff9d0a09b0273299090343048afb8107cf94cbd1400e3026f0ccac7ecebbc4d78588eb3e478fe2754d3ca664bcf3eac96ca4a6b0c8d7df5102f60f6b0020003b88d044d3c5c10010400b201df61d67487301f11879d514f4248ade90c8f68c7af1284c161098de4c28c2850f1ec7b8e30f959793e571542ffc6532189409cb51c3d30dad78c4ad5165eda18b20d9826d8707d0f742e2ab492103a85bbd9ddf4f5720f6de7064feb0d39ee002219765bb07bcfb8b877f47abe270ddeda4f676108cecb6b9bb2ad484a4f0011010001889f04180102000905024d3c5c10021b0c000a0910a34d7e18c20c31bb1a03040085c8d62e16d05dc4e9dad64953c8a2eed8b6c12f92b1575eeaa6dcf7be9473dd5b24b37b6dffbb4e7c99ed1bd3cb11634be19b3e6e207bed7505c7ca111ccf47cb323bf1f8851eb6360e8034cbff8dd149993c959de89f8f77f38e7e98b8e3076323aa719328e2b408db5ec0d03936efd57422ba04f925cdc7b4c1af7590e40ab0020003988d044d3c5c33010400b488c3e5f83f4d561f317817538d9d0397981e9aef1321ca68ebfae1cf8b7d388e19f4b5a24a82e2fbbf1c6c26557a6c5845307a03d815756f564ac7325b02bc83e87d5480a8fae848f07cb891f2d51ce7df83dcafdc12324517c86d472cc0ee10d47a68fd1d9ae49a6c19bbd36d82af597a0d88cc9c49de9df4e696fc1f0b5d0011010001b42754657374204b6579203220285253412c20656e637279707465642070726976617465206b65792988b804130102002205024d3c5c33021b03060b090807030206150802090a0b0416020301021e01021780000a0910d4984f961e35246b98940400908a73b6a6169f700434f076c6c79015a49bee37130eaf23aaa3cfa9ce60bfe4acaa7bc95f1146ada5867e0079babb38804891f4f0b8ebca57a86b249dee786161a755b7a342e68ccf3f78ed6440a93a6626beb9a37aa66afcd4f888790cb4bb46d94a4ae3eb3d7d3e6b00f6bfec940303e89ec5b32a1eaaacce66497d539328b0020003b88d044d3c5c33010400a4e913f9442abcc7f1804ccab27d2f787ffa592077ca935a8bb23165bd8d57576acac647cc596b2c3f814518cc8c82953c7a4478f32e0cf645630a5ba38d9618ef2bc3add69d459ae3dece5cab778938d988239f8c5ae437807075e06c828019959c644ff05ef6a5a1dab72227c98e3a040b0cf219026640698d7a13d8538a570011010001889f04180102000905024d3c5c33021b0c000a0910d4984f961e35246b26c703ff7ee29ef53bc1ae1ead533c408fa136db508434e233d6e62be621e031e5940bbd4c08142aed0f82217e7c3e1ec8de574bc06ccf3c36633be41ad78a9eacd209f861cae7b064100758545cc9dd83db71806dc1cfd5fb9ae5c7474bba0c19c44034ae61bae5eca379383339dece94ff56ff7aa44a582f3e5c38f45763af577c0934b0020003"
+
+const testKeys1And2PrivateHex = "9501d8044d3c5c10010400b1d13382944bd5aba23a4312968b5095d14f947f600eb478e14a6fcb16b0e0cac764884909c020bc495cfcc39a935387c661507bdb236a0612fb582cac3af9b29cc2c8c70090616c41b662f4da4c1201e195472eb7f4ae1ccbcbf9940fe21d985e379a5563dde5b9a23d35f1cfaa5790da3b79db26f23695107bfaca8e7b5bcd00110100010003ff4d91393b9a8e3430b14d6209df42f98dc927425b881f1209f319220841273a802a97c7bdb8b3a7740b3ab5866c4d1d308ad0d3a79bd1e883aacf1ac92dfe720285d10d08752a7efe3c609b1d00f17f2805b217be53999a7da7e493bfc3e9618fd17018991b8128aea70a05dbce30e4fbe626aa45775fa255dd9177aabf4df7cf0200c1ded12566e4bc2bb590455e5becfb2e2c9796482270a943343a7835de41080582c2be3caf5981aa838140e97afa40ad652a0b544f83eb1833b0957dce26e47b0200eacd6046741e9ce2ec5beb6fb5e6335457844fb09477f83b050a96be7da043e17f3a9523567ed40e7a521f818813a8b8a72209f1442844843ccc7eb9805442570200bdafe0438d97ac36e773c7162028d65844c4d463e2420aa2228c6e50dc2743c3d6c72d0d782a5173fe7be2169c8a9f4ef8a7cf3e37165e8c61b89c346cdc6c1799d2b41054657374204b6579203120285253412988b804130102002205024d3c5c10021b03060b090807030206150802090a0b0416020301021e01021780000a0910a34d7e18c20c31bbb5b304009cc45fe610b641a2c146331be94dade0a396e73ca725e1b25c21708d9cab46ecca5ccebc23055879df8f99eea39b377962a400f2ebdc36a7c99c333d74aeba346315137c3ff9d0a09b0273299090343048afb8107cf94cbd1400e3026f0ccac7ecebbc4d78588eb3e478fe2754d3ca664bcf3eac96ca4a6b0c8d7df5102f60f6b00200009d01d8044d3c5c10010400b201df61d67487301f11879d514f4248ade90c8f68c7af1284c161098de4c28c2850f1ec7b8e30f959793e571542ffc6532189409cb51c3d30dad78c4ad5165eda18b20d9826d8707d0f742e2ab492103a85bbd9ddf4f5720f6de7064feb0d39ee002219765bb07bcfb8b877f47abe270ddeda4f676108cecb6b9bb2ad484a4f00110100010003fd17a7490c22a79c59281fb7b20f5e6553ec0c1637ae382e8adaea295f50241037f8997cf42c1ce26417e015091451b15424b2c59eb8d4161b0975630408e394d3b00f88d4b4e18e2cc85e8251d4753a27c639c83f5ad4a571c4f19d7cd460b9b73c25ade730c99df09637bd173d8e3e981ac64432078263bb6dc30d3e974150dd0200d0ee05be3d4604d2146fb0457f31ba17c057560785aa804e8ca5530a7cd81d3440d0f4ba6851efcfd3954b7e68908fc0ba47f7ac37bf559c6c168b70d3a7c8cd0200da1c677c4bce06a068070f2b3733b0a714e88d62aa3f9a26c6f5216d48d5c2b5624144f3807c0df30be66b3268eeeca4df1fbded58faf49fc95dc3c35f134f8b01fd1396b6c0fc1b6c4f0eb8f5e44b8eace1e6073e20d0b8bc5385f86f1cf3f050f66af789f3ef1fc107b7f4421e19e0349c730c68f0a226981f4e889054fdb4dc149e8e889f04180102000905024d3c5c10021b0c000a0910a34d7e18c20c31bb1a03040085c8d62e16d05dc4e9dad64953c8a2eed8b6c12f92b1575eeaa6dcf7be9473dd5b24b37b6dffbb4e7c99ed1bd3cb11634be19b3e6e207bed7505c7ca111ccf47cb323bf1f8851eb6360e8034cbff8dd149993c959de89f8f77f38e7e98b8e3076323aa719328e2b408db5ec0d03936efd57422ba04f925cdc7b4c1af7590e40ab00200009501fe044d3c5c33010400b488c3e5f83f4d561f317817538d9d0397981e9aef1321ca68ebfae1cf8b7d388e19f4b5a24a82e2fbbf1c6c26557a6c5845307a03d815756f564ac7325b02bc83e87d5480a8fae848f07cb891f2d51ce7df83dcafdc12324517c86d472cc0ee10d47a68fd1d9ae49a6c19bbd36d82af597a0d88cc9c49de9df4e696fc1f0b5d0011010001fe030302e9030f3c783e14856063f16938530e148bc57a7aa3f3e4f90df9dceccdc779bc0835e1ad3d006e4a8d7b36d08b8e0de5a0d947254ecfbd22037e6572b426bcfdc517796b224b0036ff90bc574b5509bede85512f2eefb520fb4b02aa523ba739bff424a6fe81c5041f253f8d757e69a503d3563a104d0d49e9e890b9d0c26f96b55b743883b472caa7050c4acfd4a21f875bdf1258d88bd61224d303dc9df77f743137d51e6d5246b88c406780528fd9a3e15bab5452e5b93970d9dcc79f48b38651b9f15bfbcf6da452837e9cc70683d1bdca94507870f743e4ad902005812488dd342f836e72869afd00ce1850eea4cfa53ce10e3608e13d3c149394ee3cbd0e23d018fcbcb6e2ec5a1a22972d1d462ca05355d0d290dd2751e550d5efb38c6c89686344df64852bf4ff86638708f644e8ec6bd4af9b50d8541cb91891a431326ab2e332faa7ae86cfb6e0540aa63160c1e5cdd5a4add518b303fff0a20117c6bc77f7cfbaf36b04c865c6c2b42754657374204b6579203220285253412c20656e637279707465642070726976617465206b65792988b804130102002205024d3c5c33021b03060b090807030206150802090a0b0416020301021e01021780000a0910d4984f961e35246b98940400908a73b6a6169f700434f076c6c79015a49bee37130eaf23aaa3cfa9ce60bfe4acaa7bc95f1146ada5867e0079babb38804891f4f0b8ebca57a86b249dee786161a755b7a342e68ccf3f78ed6440a93a6626beb9a37aa66afcd4f888790cb4bb46d94a4ae3eb3d7d3e6b00f6bfec940303e89ec5b32a1eaaacce66497d539328b00200009d01fe044d3c5c33010400a4e913f9442abcc7f1804ccab27d2f787ffa592077ca935a8bb23165bd8d57576acac647cc596b2c3f814518cc8c82953c7a4478f32e0cf645630a5ba38d9618ef2bc3add69d459ae3dece5cab778938d988239f8c5ae437807075e06c828019959c644ff05ef6a5a1dab72227c98e3a040b0cf219026640698d7a13d8538a570011010001fe030302e9030f3c783e148560f936097339ae381d63116efcf802ff8b1c9360767db5219cc987375702a4123fd8657d3e22700f23f95020d1b261eda5257e9a72f9a918e8ef22dd5b3323ae03bbc1923dd224db988cadc16acc04b120a9f8b7e84da9716c53e0334d7b66586ddb9014df604b41be1e960dcfcbc96f4ed150a1a0dd070b9eb14276b9b6be413a769a75b519a53d3ecc0c220e85cd91ca354d57e7344517e64b43b6e29823cbd87eae26e2b2e78e6dedfbb76e3e9f77bcb844f9a8932eb3db2c3f9e44316e6f5d60e9e2a56e46b72abe6b06dc9a31cc63f10023d1f5e12d2a3ee93b675c96f504af0001220991c88db759e231b3320dcedf814dcf723fd9857e3d72d66a0f2af26950b915abdf56c1596f46a325bf17ad4810d3535fb02a259b247ac3dbd4cc3ecf9c51b6c07cebb009c1506fba0a89321ec8683e3fd009a6e551d50243e2d5092fefb3321083a4bad91320dc624bd6b5dddf93553e3d53924c05bfebec1fb4bd47e89a1a889f04180102000905024d3c5c33021b0c000a0910d4984f961e35246b26c703ff7ee29ef53bc1ae1ead533c408fa136db508434e233d6e62be621e031e5940bbd4c08142aed0f82217e7c3e1ec8de574bc06ccf3c36633be41ad78a9eacd209f861cae7b064100758545cc9dd83db71806dc1cfd5fb9ae5c7474bba0c19c44034ae61bae5eca379383339dece94ff56ff7aa44a582f3e5c38f45763af577c0934b0020000"
+
+const signedMessageHex = "a3019bc0cbccc0c4b8d8b74ee2108fe16ec6d3ca490cbe362d3f8333d3f352531472538b8b13d353b97232f352158c20943157c71c16064626063656269052062e4e01987e9b6fccff4b7df3a34c534b23e679cbec3bc0f8f6e64dfb4b55fe3f8efa9ce110ddb5cd79faf1d753c51aecfa669f7e7aa043436596cccc3359cb7dd6bbe9ecaa69e5989d9e57209571edc0b2fa7f57b9b79a64ee6e99ce1371395fee92fec2796f7b15a77c386ff668ee27f6d38f0baa6c438b561657377bf6acff3c5947befd7bf4c196252f1d6e5c524d0300"
+
+const signedTextMessageHex = "a3019bc0cbccc8c4b8d8b74ee2108fe16ec6d36a250cbece0c178233d3f352531472538b8b13d35379b97232f352158ca0b4312f57c71c1646462606365626906a062e4e019811591798ff99bf8afee860b0d8a8c2a85c3387e3bcf0bb3b17987f2bbcfab2aa526d930cbfd3d98757184df3995c9f3e7790e36e3e9779f06089d4c64e9e47dd6202cb6e9bc73c5d11bb59fbaf89d22d8dc7cf199ddf17af96e77c5f65f9bbed56f427bd8db7af37f6c9984bf9385efaf5f184f986fb3e6adb0ecfe35bbf92d16a7aa2a344fb0bc52fb7624f0200"
+
+const signedEncryptedMessageHex = "848c032a67d68660df41c70103ff5789d0de26b6a50c985a02a13131ca829c413a35d0e6fa8d6842599252162808ac7439c72151c8c6183e76923fe3299301414d0c25a2f06a2257db3839e7df0ec964773f6e4c4ac7ff3b48c444237166dd46ba8ff443a5410dc670cb486672fdbe7c9dfafb75b4fea83af3a204fe2a7dfa86bd20122b4f3d2646cbeecb8f7be8d2c03b018bd210b1d3791e1aba74b0f1034e122ab72e760492c192383cf5e20b5628bd043272d63df9b923f147eb6091cd897553204832aba48fec54aa447547bb16305a1024713b90e77fd0065f1918271947549205af3c74891af22ee0b56cd29bfec6d6e351901cd4ab3ece7c486f1e32a792d4e474aed98ee84b3f591c7dff37b64e0ecd68fd036d517e412dcadf85840ce184ad7921ad446c4ee28db80447aea1ca8d4f574db4d4e37688158ddd19e14ee2eab4873d46947d65d14a23e788d912cf9a19624ca7352469b72a83866b7c23cb5ace3deab3c7018061b0ba0f39ed2befe27163e5083cf9b8271e3e3d52cc7ad6e2a3bd81d4c3d7022f8d"
+
+const symmetricallyEncryptedCompressedHex = "8c0d04030302eb4a03808145d0d260c92f714339e13de5a79881216431925bf67ee2898ea61815f07894cd0703c50d0a76ef64d482196f47a8bc729af9b80bb6"
package s2k
import (
- "crypto/md5"
+ "crypto"
"crypto/openpgp/error"
- "crypto/ripemd160"
- "crypto/sha1"
- "crypto/sha256"
- "crypto/sha512"
"hash"
"io"
"os"
+ "strconv"
)
// Simple writes to out the result of computing the Simple S2K function (RFC
return
}
- h := hashFuncFromType(buf[1])
+ hash, ok := HashIdToHash(buf[1])
+ if !ok {
+ return nil, error.UnsupportedError("hash for S2K function: " + strconv.Itoa(int(buf[1])))
+ }
+ h := hash.New()
if h == nil {
- return nil, error.UnsupportedError("hash for S2K function")
+ return nil, error.UnsupportedError("hash not availible: " + strconv.Itoa(int(hash)))
}
switch buf[0] {
return nil, error.UnsupportedError("S2K function")
}
-// hashFuncFromType returns a hash.Hash which corresponds to the given hash
-// type byte. See RFC 4880, section 9.4.
-func hashFuncFromType(hashType byte) hash.Hash {
- switch hashType {
- case 1:
- return md5.New()
- case 2:
- return sha1.New()
- case 3:
- return ripemd160.New()
- case 8:
- return sha256.New()
- case 9:
- return sha512.New384()
- case 10:
- return sha512.New()
- case 11:
- return sha256.New224()
+// hashToHashIdMapping contains pairs relating OpenPGP's hash identifier with
+// Go's crypto.Hash type. See RFC 4880, section 9.4.
+var hashToHashIdMapping = []struct {
+ id byte
+ hash crypto.Hash
+}{
+ {1, crypto.MD5},
+ {2, crypto.SHA1},
+ {3, crypto.RIPEMD160},
+ {8, crypto.SHA256},
+ {9, crypto.SHA384},
+ {10, crypto.SHA512},
+ {11, crypto.SHA224},
+}
+
+// HashIdToHash returns a crypto.Hash which corresponds to the given OpenPGP
+// hash id.
+func HashIdToHash(id byte) (h crypto.Hash, ok bool) {
+ for _, m := range hashToHashIdMapping {
+ if m.id == id {
+ return m.hash, true
+ }
}
+ return 0, false
+}
- return nil
+// HashIdToHash returns an OpenPGP hash id which corresponds the given Hash.
+func HashToHashId(h crypto.Hash) (id byte, ok bool) {
+ for _, m := range hashToHashIdMapping {
+ if m.hash == h {
+ return m.id, true
+ }
+ }
+ return 0, false
}
--- /dev/null
+// Copyright 2011 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package openpgp
+
+import (
+ "crypto"
+ "crypto/openpgp/armor"
+ "crypto/openpgp/error"
+ "crypto/openpgp/packet"
+ "crypto/rsa"
+ _ "crypto/sha256"
+ "io"
+ "os"
+ "strconv"
+ "time"
+)
+
+// DetachSign signs message with the private key from signer (which must
+// already have been decrypted) and writes the signature to w.
+func DetachSign(w io.Writer, signer *Entity, message io.Reader) os.Error {
+ return detachSign(w, signer, message, packet.SigTypeBinary)
+}
+
+// ArmoredDetachSign signs message with the private key from signer (which
+// must already have been decrypted) and writes an armored signature to w.
+func ArmoredDetachSign(w io.Writer, signer *Entity, message io.Reader) (err os.Error) {
+ return armoredDetachSign(w, signer, message, packet.SigTypeBinary)
+}
+
+// DetachSignText signs message (after canonicalising the line endings) with
+// the private key from signer (which must already have been decrypted) and
+// writes the signature to w.
+func DetachSignText(w io.Writer, signer *Entity, message io.Reader) os.Error {
+ return detachSign(w, signer, message, packet.SigTypeText)
+}
+
+// ArmoredDetachSignText signs message (after canonicalising the line endings)
+// with the private key from signer (which must already have been decrypted)
+// and writes an armored signature to w.
+func SignTextDetachedArmored(w io.Writer, signer *Entity, message io.Reader) os.Error {
+ return armoredDetachSign(w, signer, message, packet.SigTypeText)
+}
+
+func armoredDetachSign(w io.Writer, signer *Entity, message io.Reader, sigType packet.SignatureType) (err os.Error) {
+ out, err := armor.Encode(w, SignatureType, nil)
+ if err != nil {
+ return
+ }
+ err = detachSign(out, signer, message, sigType)
+ if err != nil {
+ return
+ }
+ return out.Close()
+}
+
+func detachSign(w io.Writer, signer *Entity, message io.Reader, sigType packet.SignatureType) (err os.Error) {
+ if signer.PrivateKey == nil {
+ return error.InvalidArgumentError("signing key doesn't have a private key")
+ }
+ if signer.PrivateKey.Encrypted {
+ return error.InvalidArgumentError("signing key is encrypted")
+ }
+
+ sig := new(packet.Signature)
+ sig.SigType = sigType
+ sig.PubKeyAlgo = signer.PrivateKey.PubKeyAlgo
+ sig.Hash = crypto.SHA256
+ sig.CreationTime = uint32(time.Seconds())
+ sig.IssuerKeyId = &signer.PrivateKey.KeyId
+
+ h, wrappedHash, err := hashForSignature(sig.Hash, sig.SigType)
+ if err != nil {
+ return
+ }
+ io.Copy(wrappedHash, message)
+
+ switch signer.PrivateKey.PubKeyAlgo {
+ case packet.PubKeyAlgoRSA, packet.PubKeyAlgoRSASignOnly:
+ priv := signer.PrivateKey.PrivateKey.(*rsa.PrivateKey)
+ err = sig.SignRSA(h, priv)
+ default:
+ err = error.UnsupportedError("public key algorithm: " + strconv.Itoa(int(sig.PubKeyAlgo)))
+ }
+
+ if err != nil {
+ return
+ }
+
+ return sig.Serialize(w)
+}
--- /dev/null
+// Copyright 2011 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package openpgp
+
+import (
+ "bytes"
+ "testing"
+)
+
+func TestSignDetached(t *testing.T) {
+ kring, _ := ReadKeyRing(readerFromHex(testKeys1And2PrivateHex))
+ out := bytes.NewBuffer(nil)
+ message := bytes.NewBufferString(signedInput)
+ err := DetachSign(out, kring[0], message)
+ if err != nil {
+ t.Error(err)
+ }
+
+ testDetachedSignature(t, kring, out, signedInput, "check")
+}
+
+func TestSignTextDetached(t *testing.T) {
+ kring, _ := ReadKeyRing(readerFromHex(testKeys1And2PrivateHex))
+ out := bytes.NewBuffer(nil)
+ message := bytes.NewBufferString(signedInput)
+ err := DetachSignText(out, kring[0], message)
+ if err != nil {
+ t.Error(err)
+ }
+
+ testDetachedSignature(t, kring, out, signedInput, "check")
+}
package rand
import (
+ "bufio"
"crypto/aes"
"io"
"os"
// A devReader satisfies reads by reading the file named name.
type devReader struct {
name string
- f *os.File
+ f io.Reader
mu sync.Mutex
}
func (r *devReader) Read(b []byte) (n int, err os.Error) {
r.mu.Lock()
+ defer r.mu.Unlock()
if r.f == nil {
f, err := os.Open(r.name, os.O_RDONLY, 0)
if f == nil {
- r.mu.Unlock()
return 0, err
}
- r.f = f
+ r.f = bufio.NewReader(f)
}
- r.mu.Unlock()
return r.f.Read(b)
}
if r.prov == 0 {
const provType = syscall.PROV_RSA_FULL
const flags = syscall.CRYPT_VERIFYCONTEXT | syscall.CRYPT_SILENT
- ok, errno := syscall.CryptAcquireContext(&r.prov, nil, nil, provType, flags)
- if !ok {
+ errno := syscall.CryptAcquireContext(&r.prov, nil, nil, provType, flags)
+ if errno != 0 {
r.mu.Unlock()
return 0, os.NewSyscallError("CryptAcquireContext", errno)
}
}
r.mu.Unlock()
- ok, errno := syscall.CryptGenRandom(r.prov, uint32(len(b)), &b[0])
- if !ok {
+ errno := syscall.CryptGenRandom(r.prov, uint32(len(b)), &b[0])
+ if errno != 0 {
return 0, os.NewSyscallError("CryptGenRandom", errno)
}
return len(b), nil
// http://homes.esat.kuleuven.be/~cosicart/pdf/AB-9601/AB-9601.pdf.
import (
+ "crypto"
"hash"
"os"
)
+func init() {
+ crypto.RegisterHash(crypto.RIPEMD160, New)
+}
+
// The size of the checksum in bytes.
const Size = 20
import (
"big"
+ "crypto"
"crypto/subtle"
"io"
"os"
return
}
-// Due to the design of PKCS#1 v1.5, we need to know the exact hash function in
-// use. A generic hash.Hash will not do.
-type PKCS1v15Hash int
-
-const (
- HashMD5 PKCS1v15Hash = iota
- HashSHA1
- HashSHA256
- HashSHA384
- HashSHA512
- HashMD5SHA1 // combined MD5 and SHA1 hash used for RSA signing in TLS.
-)
-
// These are ASN1 DER structures:
// DigestInfo ::= SEQUENCE {
// digestAlgorithm AlgorithmIdentifier,
// For performance, we don't use the generic ASN1 encoder. Rather, we
// precompute a prefix of the digest value that makes a valid ASN1 DER string
// with the correct contents.
-var hashPrefixes = [][]byte{
- // HashMD5
- {0x30, 0x20, 0x30, 0x0c, 0x06, 0x08, 0x2a, 0x86, 0x48, 0x86, 0xf7, 0x0d, 0x02, 0x05, 0x05, 0x00, 0x04, 0x10},
- // HashSHA1
- {0x30, 0x21, 0x30, 0x09, 0x06, 0x05, 0x2b, 0x0e, 0x03, 0x02, 0x1a, 0x05, 0x00, 0x04, 0x14},
- // HashSHA256
- {0x30, 0x31, 0x30, 0x0d, 0x06, 0x09, 0x60, 0x86, 0x48, 0x01, 0x65, 0x03, 0x04, 0x02, 0x01, 0x05, 0x00, 0x04, 0x20},
- // HashSHA384
- {0x30, 0x41, 0x30, 0x0d, 0x06, 0x09, 0x60, 0x86, 0x48, 0x01, 0x65, 0x03, 0x04, 0x02, 0x02, 0x05, 0x00, 0x04, 0x30},
- // HashSHA512
- {0x30, 0x51, 0x30, 0x0d, 0x06, 0x09, 0x60, 0x86, 0x48, 0x01, 0x65, 0x03, 0x04, 0x02, 0x03, 0x05, 0x00, 0x04, 0x40},
- // HashMD5SHA1
- {}, // A special TLS case which doesn't use an ASN1 prefix.
+var hashPrefixes = map[crypto.Hash][]byte{
+ crypto.MD5: []byte{0x30, 0x20, 0x30, 0x0c, 0x06, 0x08, 0x2a, 0x86, 0x48, 0x86, 0xf7, 0x0d, 0x02, 0x05, 0x05, 0x00, 0x04, 0x10},
+ crypto.SHA1: []byte{0x30, 0x21, 0x30, 0x09, 0x06, 0x05, 0x2b, 0x0e, 0x03, 0x02, 0x1a, 0x05, 0x00, 0x04, 0x14},
+ crypto.SHA256: []byte{0x30, 0x31, 0x30, 0x0d, 0x06, 0x09, 0x60, 0x86, 0x48, 0x01, 0x65, 0x03, 0x04, 0x02, 0x01, 0x05, 0x00, 0x04, 0x20},
+ crypto.SHA384: []byte{0x30, 0x41, 0x30, 0x0d, 0x06, 0x09, 0x60, 0x86, 0x48, 0x01, 0x65, 0x03, 0x04, 0x02, 0x02, 0x05, 0x00, 0x04, 0x30},
+ crypto.SHA512: {0x30, 0x51, 0x30, 0x0d, 0x06, 0x09, 0x60, 0x86, 0x48, 0x01, 0x65, 0x03, 0x04, 0x02, 0x03, 0x05, 0x00, 0x04, 0x40},
+ crypto.MD5SHA1: {}, // A special TLS case which doesn't use an ASN1 prefix.
+ crypto.RIPEMD160: {0x30, 0x20, 0x30, 0x08, 0x06, 0x06, 0x28, 0xcf, 0x06, 0x03, 0x00, 0x31, 0x04, 0x14},
}
-// SignPKCS1v15 calcuates the signature of hashed using RSASSA-PSS-SIGN from RSA PKCS#1 v1.5.
+// SignPKCS1v15 calculates the signature of hashed using RSASSA-PKCS1-V1_5-SIGN from RSA PKCS#1 v1.5.
// Note that hashed must be the result of hashing the input message using the
// given hash function.
-func SignPKCS1v15(rand io.Reader, priv *PrivateKey, hash PKCS1v15Hash, hashed []byte) (s []byte, err os.Error) {
+func SignPKCS1v15(rand io.Reader, priv *PrivateKey, hash crypto.Hash, hashed []byte) (s []byte, err os.Error) {
hashLen, prefix, err := pkcs1v15HashInfo(hash, len(hashed))
if err != nil {
return
// hashed is the result of hashing the input message using the given hash
// function and sig is the signature. A valid signature is indicated by
// returning a nil error.
-func VerifyPKCS1v15(pub *PublicKey, hash PKCS1v15Hash, hashed []byte, sig []byte) (err os.Error) {
+func VerifyPKCS1v15(pub *PublicKey, hash crypto.Hash, hashed []byte, sig []byte) (err os.Error) {
hashLen, prefix, err := pkcs1v15HashInfo(hash, len(hashed))
if err != nil {
return
return nil
}
-func pkcs1v15HashInfo(hash PKCS1v15Hash, inLen int) (hashLen int, prefix []byte, err os.Error) {
- switch hash {
- case HashMD5:
- hashLen = 16
- case HashSHA1:
- hashLen = 20
- case HashSHA256:
- hashLen = 32
- case HashSHA384:
- hashLen = 48
- case HashSHA512:
- hashLen = 64
- case HashMD5SHA1:
- hashLen = 36
- default:
- return 0, nil, os.ErrorString("unknown hash function")
- }
-
+func pkcs1v15HashInfo(hash crypto.Hash, inLen int) (hashLen int, prefix []byte, err os.Error) {
+ hashLen = hash.Size()
if inLen != hashLen {
return 0, nil, os.ErrorString("input must be hashed message")
}
-
- prefix = hashPrefixes[int(hash)]
+ prefix, ok := hashPrefixes[hash]
+ if !ok {
+ return 0, nil, os.ErrorString("unsupported hash function")
+ }
return
}
import (
"big"
"bytes"
+ "crypto"
"crypto/rand"
"crypto/sha1"
"encoding/base64"
h.Write([]byte(test.in))
digest := h.Sum()
- s, err := SignPKCS1v15(nil, rsaPrivateKey, HashSHA1, digest)
+ s, err := SignPKCS1v15(nil, rsaPrivateKey, crypto.SHA1, digest)
if err != nil {
t.Errorf("#%d %s", i, err)
}
sig, _ := hex.DecodeString(test.out)
- err := VerifyPKCS1v15(&rsaPrivateKey.PublicKey, HashSHA1, digest, sig)
+ err := VerifyPKCS1v15(&rsaPrivateKey.PublicKey, crypto.SHA1, digest, sig)
if err != nil {
t.Errorf("#%d %s", i, err)
}
m.SetBytes(em)
c := encrypt(new(big.Int), pub, m)
out = c.Bytes()
+
+ if len(out) < k {
+ // If the output is too small, we need to left-pad with zeros.
+ t := make([]byte, k)
+ copy(t[k-len(out):], out)
+ out = t
+ }
+
return
}
t.Errorf("#%d,%d error: %s", i, j, err)
}
if bytes.Compare(out, message.out) != 0 {
- t.Errorf("#%d,%d bad result: %s (want %s)", i, j, out, message.out)
+ t.Errorf("#%d,%d bad result: %x (want %x)", i, j, out, message.out)
}
}
}
package sha1
import (
+ "crypto"
"hash"
"os"
)
+func init() {
+ crypto.RegisterHash(crypto.SHA1, New)
+}
+
// The size of a SHA1 checksum in bytes.
const Size = 20
package sha256
import (
+ "crypto"
"hash"
"os"
)
+func init() {
+ crypto.RegisterHash(crypto.SHA224, New224)
+ crypto.RegisterHash(crypto.SHA256, New)
+}
+
// The size of a SHA256 checksum in bytes.
const Size = 32
package sha512
import (
+ "crypto"
"hash"
"os"
)
+func init() {
+ crypto.RegisterHash(crypto.SHA384, New384)
+ crypto.RegisterHash(crypto.SHA512, New)
+}
+
// The size of a SHA512 checksum in bytes.
const Size = 64
package tls
import (
+ "crypto"
"crypto/rsa"
"crypto/subtle"
"crypto/x509"
vers, ok := mutualVersion(serverHello.vers)
if !ok {
- c.sendAlert(alertProtocolVersion)
+ return c.sendAlert(alertProtocolVersion)
}
c.vers = vers
c.haveVers = true
var digest [36]byte
copy(digest[0:16], finishedHash.serverMD5.Sum())
copy(digest[16:36], finishedHash.serverSHA1.Sum())
- signed, err := rsa.SignPKCS1v15(c.config.rand(), c.config.Certificates[0].PrivateKey, rsa.HashMD5SHA1, digest[0:])
+ signed, err := rsa.SignPKCS1v15(c.config.rand(), c.config.Certificates[0].PrivateKey, crypto.MD5SHA1, digest[0:])
if err != nil {
return c.sendAlert(alertInternalError)
}
// Script of interaction with gnutls implementation.
// The values for this test are obtained by building and running in client mode:
-// % gotest -match "TestRunClient" -connect
+// % gotest -test.run "TestRunClient" -connect
// and then:
// % gnutls-serv -p 10443 --debug 100 --x509keyfile key.pem --x509certfile cert.pem -a > /tmp/log 2>&1
// % python parse-gnutls-cli-debug-log.py < /tmp/log
package tls
import (
+ "crypto"
"crypto/rsa"
"crypto/subtle"
"crypto/x509"
var suite *cipherSuite
var suiteId uint16
+FindCipherSuite:
for _, id := range clientHello.cipherSuites {
for _, supported := range config.cipherSuites() {
if id == supported {
continue
}
suiteId = id
- break
+ break FindCipherSuite
}
}
}
digest := make([]byte, 36)
copy(digest[0:16], finishedHash.serverMD5.Sum())
copy(digest[16:36], finishedHash.serverSHA1.Sum())
- err = rsa.VerifyPKCS1v15(pub, rsa.HashMD5SHA1, digest, certVerify.signature)
+ err = rsa.VerifyPKCS1v15(pub, crypto.MD5SHA1, digest, certVerify.signature)
if err != nil {
c.sendAlert(alertBadCertificate)
return os.ErrorString("could not validate signature of connection nonces: " + err.String())
// Script of interaction with gnutls implementation.
// The values for this test are obtained by building and running in server mode:
-// % gotest -match "TestRunServer" -serve
+// % gotest -test.run "TestRunServer" -serve
// and then:
// % gnutls-cli --insecure --debug 100 -p 10443 localhost > /tmp/log 2>&1
// % python parse-gnutls-cli-debug-log.py < /tmp/log
import (
"big"
+ "crypto"
"crypto/elliptic"
"crypto/md5"
"crypto/rsa"
copy(serverECDHParams[4:], ecdhePublic)
md5sha1 := md5SHA1Hash(clientHello.random, hello.random, serverECDHParams)
- sig, err := rsa.SignPKCS1v15(config.rand(), config.Certificates[0].PrivateKey, rsa.HashMD5SHA1, md5sha1)
+ sig, err := rsa.SignPKCS1v15(config.rand(), config.Certificates[0].PrivateKey, crypto.MD5SHA1, md5sha1)
if err != nil {
return nil, os.ErrorString("failed to sign ECDHE parameters: " + err.String())
}
sig = sig[2:]
md5sha1 := md5SHA1Hash(clientHello.random, serverHello.random, serverECDHParams)
- return rsa.VerifyPKCS1v15(cert.PublicKey.(*rsa.PublicKey), rsa.HashMD5SHA1, md5sha1, sig)
+ return rsa.VerifyPKCS1v15(cert.PublicKey.(*rsa.PublicKey), crypto.MD5SHA1, md5sha1, sig)
Error:
return os.ErrorString("invalid ServerKeyExchange")
return
}
- certDERBlock, _ := pem.Decode(certPEMBlock)
- if certDERBlock == nil {
+ var certDERBlock *pem.Block
+ for {
+ certDERBlock, certPEMBlock = pem.Decode(certPEMBlock)
+ if certDERBlock == nil {
+ break
+ }
+ if certDERBlock.Type == "CERTIFICATE" {
+ cert.Certificate = append(cert.Certificate, certDERBlock.Bytes)
+ }
+ }
+
+ if len(cert.Certificate) == 0 {
err = os.ErrorString("crypto/tls: failed to parse certificate PEM data")
return
}
- cert.Certificate = [][]byte{certDERBlock.Bytes}
-
keyPEMBlock, err := ioutil.ReadFile(keyFile)
if err != nil {
return
// We don't need to parse the public key for TLS, but we so do anyway
// to check that it looks sane and matches the private key.
- x509Cert, err := x509.ParseCertificate(certDERBlock.Bytes)
+ x509Cert, err := x509.ParseCertificate(cert.Certificate[0])
if err != nil {
return
}
"asn1"
"big"
"container/vector"
+ "crypto"
"crypto/rsa"
"crypto/sha1"
"hash"
DNSNames []string
EmailAddresses []string
+ // Name constraints
+ PermittedDNSDomainsCritical bool // if true then the name constraints are marked critical.
+ PermittedDNSDomains []string
+
PolicyIdentifiers []asn1.ObjectIdentifier
}
// TODO(agl): don't ignore the path length constraint.
var h hash.Hash
- var hashType rsa.PKCS1v15Hash
+ var hashType crypto.Hash
switch c.SignatureAlgorithm {
case SHA1WithRSA:
h = sha1.New()
- hashType = rsa.HashSHA1
+ hashType = crypto.SHA1
default:
return UnsupportedAlgorithmError{}
}
// policyQualifiers omitted
}
+// RFC 5280, 4.2.1.10
+type nameConstraints struct {
+ Permitted []generalSubtree "optional,tag:0"
+ Excluded []generalSubtree "optional,tag:1"
+}
+
+type generalSubtree struct {
+ Name string "tag:2,optional,ia5"
+ Min int "optional,tag:0"
+ Max int "optional,tag:1"
+}
+
func parsePublicKey(algo PublicKeyAlgorithm, asn1Data []byte) (interface{}, os.Error) {
switch algo {
case RSA:
// If we didn't parse any of the names then we
// fall through to the critical check below.
+ case 30:
+ // RFC 5280, 4.2.1.10
+
+ // NameConstraints ::= SEQUENCE {
+ // permittedSubtrees [0] GeneralSubtrees OPTIONAL,
+ // excludedSubtrees [1] GeneralSubtrees OPTIONAL }
+ //
+ // GeneralSubtrees ::= SEQUENCE SIZE (1..MAX) OF GeneralSubtree
+ //
+ // GeneralSubtree ::= SEQUENCE {
+ // base GeneralName,
+ // minimum [0] BaseDistance DEFAULT 0,
+ // maximum [1] BaseDistance OPTIONAL }
+ //
+ // BaseDistance ::= INTEGER (0..MAX)
+
+ var constraints nameConstraints
+ _, err := asn1.Unmarshal(e.Value, &constraints)
+ if err != nil {
+ return nil, err
+ }
+
+ if len(constraints.Excluded) > 0 && e.Critical {
+ return out, UnhandledCriticalExtension{}
+ }
+
+ for _, subtree := range constraints.Permitted {
+ if subtree.Min > 0 || subtree.Max > 0 || len(subtree.Name) == 0 {
+ if e.Critical {
+ return out, UnhandledCriticalExtension{}
+ }
+ continue
+ }
+ out.PermittedDNSDomains = append(out.PermittedDNSDomains, subtree.Name)
+ }
+ continue
+
case 35:
// RFC 5280, 4.2.1.1
var a authKeyId
oidExtensionBasicConstraints = []int{2, 5, 29, 19}
oidExtensionSubjectAltName = []int{2, 5, 29, 17}
oidExtensionCertificatePolicies = []int{2, 5, 29, 32}
+ oidExtensionNameConstraints = []int{2, 5, 29, 30}
)
func buildExtensions(template *Certificate) (ret []extension, err os.Error) {
- ret = make([]extension, 6 /* maximum number of elements. */ )
+ ret = make([]extension, 7 /* maximum number of elements. */ )
n := 0
if template.KeyUsage != 0 {
n++
}
+ if len(template.PermittedDNSDomains) > 0 {
+ ret[n].Id = oidExtensionNameConstraints
+ ret[n].Critical = template.PermittedDNSDomainsCritical
+
+ var out nameConstraints
+ out.Permitted = make([]generalSubtree, len(template.PermittedDNSDomains))
+ for i, permitted := range template.PermittedDNSDomains {
+ out.Permitted[i] = generalSubtree{Name: permitted}
+ }
+ ret[n].Value, err = asn1.Marshal(out)
+ if err != nil {
+ return
+ }
+ n++
+ }
+
// Adding another extension here? Remember to update the maximum number
// of elements in the make() at the top of the function.
// CreateSelfSignedCertificate creates a new certificate based on
// a template. The following members of template are used: SerialNumber,
// Subject, NotBefore, NotAfter, KeyUsage, BasicConstraintsValid, IsCA,
-// MaxPathLen, SubjectKeyId, DNSNames.
+// MaxPathLen, SubjectKeyId, DNSNames, PermittedDNSDomainsCritical,
+// PermittedDNSDomains.
//
// The certificate is signed by parent. If parent is equal to template then the
// certificate is self-signed. The parameter pub is the public key of the
h.Write(tbsCertContents)
digest := h.Sum()
- signature, err := rsa.SignPKCS1v15(rand, priv, rsa.HashSHA1, digest)
+ signature, err := rsa.SignPKCS1v15(rand, priv, crypto.SHA1, digest)
if err != nil {
return
}
IsCA: true,
DNSNames: []string{"test.example.com"},
- PolicyIdentifiers: []asn1.ObjectIdentifier{[]int{1, 2, 3}},
+ PolicyIdentifiers: []asn1.ObjectIdentifier{[]int{1, 2, 3}},
+ PermittedDNSDomains: []string{".example.com", "example.com"},
}
derBytes, err := CreateCertificate(random, &template, &template, &priv.PublicKey, priv)
t.Errorf("Failed to parse policy identifiers: got:%#v want:%#v", cert.PolicyIdentifiers, template.PolicyIdentifiers)
}
+ if len(cert.PermittedDNSDomains) != 2 || cert.PermittedDNSDomains[0] != ".example.com" || cert.PermittedDNSDomains[1] != "example.com" {
+ t.Errorf("Failed to parse name constraints: %#v", cert.PermittedDNSDomains)
+ }
+
err = cert.CheckSignatureFrom(cert)
if err != nil {
t.Errorf("Signature verification failed: %s", err)
FirstThunk uint32
dll string
- rva []uint32
}
// Data reads and returns the contents of the PE section.
}
ida = append(ida, dt)
}
- for i, _ := range ida {
+ names, _ := ds.Data()
+ var all []string
+ for _, dt := range ida {
+ dt.dll, _ = getString(names, int(dt.Name-ds.VirtualAddress))
+ d, _ = ds.Data()
+ // seek to OriginalFirstThunk
+ d = d[dt.OriginalFirstThunk-ds.VirtualAddress:]
for len(d) > 0 {
va := binary.LittleEndian.Uint32(d[0:4])
d = d[4:]
if va == 0 {
break
}
- ida[i].rva = append(ida[i].rva, va)
- }
- }
- for _, _ = range ida {
- for len(d) > 0 {
- va := binary.LittleEndian.Uint32(d[0:4])
- d = d[4:]
- if va == 0 {
- break
+ if va&0x80000000 > 0 { // is Ordinal
+ // TODO add dynimport ordinal support.
+ //ord := va&0x0000FFFF
+ } else {
+ fn, _ := getString(names, int(va-ds.VirtualAddress+2))
+ all = append(all, fn+":"+dt.dll)
}
}
}
- names, _ := ds.Data()
- var all []string
- for _, dt := range ida {
- dt.dll, _ = getString(names, int(dt.Name-ds.VirtualAddress))
- for _, va := range dt.rva {
- fn, _ := getString(names, int(va-ds.VirtualAddress+2))
- all = append(all, fn+":"+dt.dll)
- }
- }
return all, nil
}
var grammars = []string{
- `Program = .
- `,
-
- `Program = foo .
- foo = "foo" .
- `,
-
- `Program = "a" | "b" "c" .
- `,
-
- `Program = "a" ... "z" .
- `,
-
- `Program = Song .
- Song = { Note } .
- Note = Do | (Re | Mi | Fa | So | La) | Ti .
- Do = "c" .
- Re = "d" .
- Mi = "e" .
- Fa = "f" .
- So = "g" .
- La = "a" .
- Ti = ti .
- ti = "b" .
- `,
+`Program = .
+`,
+
+`Program = foo .
+foo = "foo" .
+`,
+
+`Program = "a" | "b" "c" .
+`,
+
+`Program = "a" ... "z" .
+`,
+
+`Program = Song .
+ Song = { Note } .
+ Note = Do | (Re | Mi | Fa | So | La) | Ti .
+ Do = "c" .
+ Re = "d" .
+ Mi = "e" .
+ Fa = "f" .
+ So = "g" .
+ La = "a" .
+ Ti = ti .
+ ti = "b" .
+`,
}
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
-// This package implements translation between
-// unsigned integer values and byte sequences.
+// Package binary implements translation between
+// unsigned integer values and byte sequences
+// and the reading and writing of fixed-size values.
package binary
import (
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
-// This package implements a Reader which handles reading \r and \r\n
-// deliminated lines.
+// The line package implements a Reader that reads lines delimited by '\n' or ' \r\n'.
package line
import (
"os"
)
-// Reader reads lines from an io.Reader (which may use either '\n' or
-// '\r\n').
+// Reader reads lines, delimited by '\n' or \r\n', from an io.Reader.
type Reader struct {
buf []byte
consumed int
err os.Error
}
-func NewReader(in io.Reader, maxLineLength int) *Reader {
+// NewReader returns a new Reader that will read successive
+// lines from the input Reader.
+func NewReader(input io.Reader, maxLineLength int) *Reader {
return &Reader{
buf: make([]byte, 0, maxLineLength),
consumed: 0,
- in: in,
+ in: input,
+ }
+}
+
+// Read reads from any buffered data past the last line read, or from the underlying
+// io.Reader if the buffer is empty.
+func (l *Reader) Read(p []byte) (n int, err os.Error) {
+ l.removeConsumedFromBuffer()
+ if len(l.buf) > 0 {
+ n = copy(p, l.buf)
+ l.consumed += n
+ return
+ }
+ return l.in.Read(p)
+}
+
+func (l *Reader) removeConsumedFromBuffer() {
+ if l.consumed > 0 {
+ n := copy(l.buf, l.buf[l.consumed:])
+ l.buf = l.buf[:n]
+ l.consumed = 0
}
}
// the Reader and is only valid until the next call to ReadLine. ReadLine
// either returns a non-nil line or it returns an error, never both.
func (l *Reader) ReadLine() (line []byte, isPrefix bool, err os.Error) {
- if l.consumed > 0 {
- n := copy(l.buf, l.buf[l.consumed:])
- l.buf = l.buf[:n]
- l.consumed = 0
- }
+ l.removeConsumedFromBuffer()
if len(l.buf) == 0 && l.err != nil {
err = l.err
l.buf = l.buf[:oldLen+n]
if readErr != nil {
l.err = readErr
+ if len(l.buf) == 0 {
+ return nil, false, readErr
+ }
}
}
panic("unreachable")
import (
"bytes"
+ "io"
+ "io/ioutil"
"os"
"testing"
)
t.Errorf("bad result for third line: %x", line)
}
}
+
+func TestReadAfterLines(t *testing.T) {
+ line1 := "line1"
+ restData := "line2\nline 3\n"
+ inbuf := bytes.NewBuffer([]byte(line1 + "\n" + restData))
+ outbuf := new(bytes.Buffer)
+ maxLineLength := len(line1) + len(restData)/2
+ l := NewReader(inbuf, maxLineLength)
+ line, isPrefix, err := l.ReadLine()
+ if isPrefix || err != nil || string(line) != line1 {
+ t.Errorf("bad result for first line: isPrefix=%v err=%v line=%q", isPrefix, err, string(line))
+ }
+ n, err := io.Copy(outbuf, l)
+ if int(n) != len(restData) || err != nil {
+ t.Errorf("bad result for Read: n=%d err=%v", n, err)
+ }
+ if outbuf.String() != restData {
+ t.Errorf("bad result for Read: got %q; expected %q", outbuf.String(), restData)
+ }
+}
+
+func TestReadEmptyBuffer(t *testing.T) {
+ l := NewReader(bytes.NewBuffer(nil), 10)
+ line, isPrefix, err := l.ReadLine()
+ if err != os.EOF {
+ t.Errorf("expected EOF from ReadLine, got '%s' %t %s", line, isPrefix, err)
+ }
+}
+
+func TestLinesAfterRead(t *testing.T) {
+ l := NewReader(bytes.NewBuffer([]byte("foo")), 10)
+ _, err := ioutil.ReadAll(l)
+ if err != nil {
+ t.Error(err)
+ return
+ }
+
+ line, isPrefix, err := l.ReadLine()
+ if err != os.EOF {
+ t.Errorf("expected EOF from ReadLine, got '%s' %t %s", line, isPrefix, err)
+ }
+}
import (
"os"
+ "strconv"
)
// Arguments to Run.
// Stdin, Stdout, and Stderr are Files representing pipes
// connected to the running command's standard input, output, and error,
// or else nil, depending on the arguments to Run.
-// Pid is the running command's operating system process ID.
+// Process represents the underlying operating system process.
type Cmd struct {
- Stdin *os.File
- Stdout *os.File
- Stderr *os.File
- Pid int
+ Stdin *os.File
+ Stdout *os.File
+ Stderr *os.File
+ Process *os.Process
+}
+
+// PathError records the name of a binary that was not
+// found on the current $PATH.
+type PathError struct {
+ Name string
+}
+
+func (e *PathError) String() string {
+ return "command " + strconv.Quote(e.Name) + " not found in $PATH"
}
// Given mode (DevNull, etc), return file for child
// If a parameter is Pipe, then the corresponding field (Stdin, Stdout, Stderr)
// of the returned Cmd is the other end of the pipe.
// Otherwise the field in Cmd is nil.
-func Run(name string, argv, envv []string, dir string, stdin, stdout, stderr int) (p *Cmd, err os.Error) {
- p = new(Cmd)
+func Run(name string, argv, envv []string, dir string, stdin, stdout, stderr int) (c *Cmd, err os.Error) {
+ c = new(Cmd)
var fd [3]*os.File
- if fd[0], p.Stdin, err = modeToFiles(stdin, 0); err != nil {
+ if fd[0], c.Stdin, err = modeToFiles(stdin, 0); err != nil {
goto Error
}
- if fd[1], p.Stdout, err = modeToFiles(stdout, 1); err != nil {
+ if fd[1], c.Stdout, err = modeToFiles(stdout, 1); err != nil {
goto Error
}
if stderr == MergeWithStdout {
fd[2] = fd[1]
- } else if fd[2], p.Stderr, err = modeToFiles(stderr, 2); err != nil {
+ } else if fd[2], c.Stderr, err = modeToFiles(stderr, 2); err != nil {
goto Error
}
// Run command.
- p.Pid, err = os.ForkExec(name, argv, envv, dir, fd[0:])
+ c.Process, err = os.StartProcess(name, argv, envv, dir, fd[0:])
if err != nil {
goto Error
}
if fd[2] != os.Stderr && fd[2] != fd[1] {
fd[2].Close()
}
- return p, nil
+ return c, nil
Error:
if fd[0] != os.Stdin && fd[0] != nil {
if fd[2] != os.Stderr && fd[2] != nil && fd[2] != fd[1] {
fd[2].Close()
}
- if p.Stdin != nil {
- p.Stdin.Close()
+ if c.Stdin != nil {
+ c.Stdin.Close()
+ }
+ if c.Stdout != nil {
+ c.Stdout.Close()
}
- if p.Stdout != nil {
- p.Stdout.Close()
+ if c.Stderr != nil {
+ c.Stderr.Close()
}
- if p.Stderr != nil {
- p.Stderr.Close()
+ if c.Process != nil {
+ c.Process.Release()
}
return nil, err
}
-// Wait waits for the running command p,
-// returning the Waitmsg returned by os.Wait and an error.
-// The options are passed through to os.Wait.
-// Setting options to 0 waits for p to exit;
+// Wait waits for the running command c,
+// returning the Waitmsg returned when the process exits.
+// The options are passed to the process's Wait method.
+// Setting options to 0 waits for c to exit;
// other options cause Wait to return for other
// process events; see package os for details.
-func (p *Cmd) Wait(options int) (*os.Waitmsg, os.Error) {
- if p.Pid <= 0 {
+func (c *Cmd) Wait(options int) (*os.Waitmsg, os.Error) {
+ if c.Process == nil {
return nil, os.ErrorString("exec: invalid use of Cmd.Wait")
}
- w, err := os.Wait(p.Pid, options)
+ w, err := c.Process.Wait(options)
if w != nil && (w.Exited() || w.Signaled()) {
- p.Pid = -1
+ c.Process.Release()
+ c.Process = nil
}
return w, err
}
-// Close waits for the running command p to exit,
+// Close waits for the running command c to exit,
// if it hasn't already, and then closes the non-nil file descriptors
-// p.Stdin, p.Stdout, and p.Stderr.
-func (p *Cmd) Close() os.Error {
- if p.Pid > 0 {
+// c.Stdin, c.Stdout, and c.Stderr.
+func (c *Cmd) Close() os.Error {
+ if c.Process != nil {
// Loop on interrupt, but
// ignore other errors -- maybe
// caller has already waited for pid.
- _, err := p.Wait(0)
+ _, err := c.Wait(0)
for err == os.EINTR {
- _, err = p.Wait(0)
+ _, err = c.Wait(0)
}
}
// Close the FDs that are still open.
var err os.Error
- if p.Stdin != nil && p.Stdin.Fd() >= 0 {
- if err1 := p.Stdin.Close(); err1 != nil {
+ if c.Stdin != nil && c.Stdin.Fd() >= 0 {
+ if err1 := c.Stdin.Close(); err1 != nil {
err = err1
}
}
- if p.Stdout != nil && p.Stdout.Fd() >= 0 {
- if err1 := p.Stdout.Close(); err1 != nil && err != nil {
+ if c.Stdout != nil && c.Stdout.Fd() >= 0 {
+ if err1 := c.Stdout.Close(); err1 != nil && err != nil {
err = err1
}
}
- if p.Stderr != nil && p.Stderr != p.Stdout && p.Stderr.Fd() >= 0 {
- if err1 := p.Stderr.Close(); err1 != nil && err != nil {
+ if c.Stderr != nil && c.Stderr != c.Stdout && c.Stderr.Fd() >= 0 {
+ if err1 := c.Stderr.Close(); err1 != nil && err != nil {
err = err1
}
}
--- /dev/null
+// Copyright 2011 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package exec
+
+import (
+ "testing"
+)
+
+var nonExistentPaths = []string{
+ "some-non-existent-path",
+ "non-existent-path/slashed",
+}
+
+func TestLookPathNotFound(t *testing.T) {
+ for _, name := range nonExistentPaths {
+ path, err := LookPath(name)
+ if err == nil {
+ t.Fatalf("LookPath found %q in $PATH", name)
+ }
+ if path != "" {
+ t.Fatalf("LookPath path == %q when err != nil", path)
+ }
+ perr, ok := err.(*PathError)
+ if !ok {
+ t.Fatal("LookPath error is not a PathError")
+ }
+ if perr.Name != name {
+ t.Fatalf("want PathError name %q, got %q", name, perr.Name)
+ }
+ }
+}
if canExec(file) {
return file, nil
}
- return "", &os.PathError{"lookpath", file, os.ENOENT}
+ return "", &PathError{file}
}
pathenv := os.Getenv("PATH")
for _, dir := range strings.Split(pathenv, ":", -1) {
return dir + "/" + file, nil
}
}
- return "", &os.PathError{"lookpath", file, os.ENOENT}
+ return "", &PathError{file}
}
if f, ok := canExec(file, exts); ok {
return f, nil
}
- return ``, &os.PathError{"lookpath", file, os.ENOENT}
+ return ``, &PathError{file}
}
if pathenv := os.Getenv(`PATH`); pathenv == `` {
if f, ok := canExec(`.\`+file, exts); ok {
}
}
}
- return ``, &os.PathError{"lookpath", file, os.ENOENT}
+ return ``, &PathError{file}
}
func (c *conn) Screen() draw.Image { return c.img }
func (c *conn) FlushImage() {
- // We do the send (the <- operator) in an expression context, rather than in
- // a statement context, so that it does not block, and fails if the buffered
- // channel is full (in which case there already is a flush request pending).
- _ = c.flush <- false
+ select {
+ case c.flush <- false:
+ // Flush notification sent.
+ default:
+ // Could not send.
+ // Flush notification must be pending already.
+ }
}
func (c *conn) Close() os.Error {
return
default:
- log.Panic("Unexpected branch token %v", s.Tok)
+ log.Panicf("Unexpected branch token %v", s.Tok)
}
a.flow.put1(false, pc)
Val2("if false { i = 2 } else { i = 3 }; i2 = 4", "i", 3, "i2", 4),
Val2("if i == i2 { i = 2 } else { i = 3 }; i2 = 4", "i", 3, "i2", 4),
// Omit optional parts
- Val2("if { i = 2 } else { i = 3 }; i2 = 4", "i", 2, "i2", 4),
+ Val2("if true { i = 2 } else { i = 3 }; i2 = 4", "i", 2, "i2", 4),
Val2("if true { i = 2 }; i2 = 4", "i", 2, "i2", 4),
Val2("if false { i = 2 }; i2 = 4", "i", 1, "i2", 4),
// Init
CErr("fn1 := func() int { if true { return 1 } }", "return"),
CErr("fn1 := func() int { if true { } }", "return"),
Run("fn1 := func() int { if true { }; return 1 }"),
- CErr("fn1 := func() int { if { } }", "return"),
- CErr("fn1 := func() int { if { } else { return 2 } }", "return"),
- Run("fn1 := func() int { if { return 1 } }"),
- Run("fn1 := func() int { if { return 1 } else { } }"),
- Run("fn1 := func() int { if { return 1 } else { } }"),
+ CErr("fn1 := func() int { if true { } }", "return"),
+ CErr("fn1 := func() int { if true { } else { return 2 } }", "return"),
+ Run("fn1 := func() int { if true { return 1 }; return 0 }"),
+ Run("fn1 := func() int { if true { return 1 } else { }; return 0 }"),
+ Run("fn1 := func() int { if true { return 1 } else { }; return 0 }"),
// Switch
Val1("switch { case false: i += 2; case true: i += 4; default: i += 8 }", "i", 1+4),
--- /dev/null
+// Copyright 2011 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package main
+
+import (
+ "fmt"
+ "syscall"
+ "os"
+ "unsafe"
+)
+
+// some help functions
+
+func abortf(format string, a ...interface{}) {
+ fmt.Fprintf(os.Stdout, format, a...)
+ os.Exit(1)
+}
+
+func abortErrNo(funcname string, err int) {
+ abortf("%s failed: %d %s\n", funcname, err, syscall.Errstr(err))
+}
+
+// global vars
+
+var (
+ mh uint32
+ bh uint32
+)
+
+// WinProc called by windows to notify us of all windows events we might be interested in.
+func WndProc(hwnd, msg uint32, wparam, lparam int32) uintptr {
+ var rc int32
+ switch msg {
+ case WM_CREATE:
+ var e int
+ // CreateWindowEx
+ bh, e = CreateWindowEx(
+ 0,
+ syscall.StringToUTF16Ptr("button"),
+ syscall.StringToUTF16Ptr("Quit"),
+ WS_CHILD|WS_VISIBLE|BS_DEFPUSHBUTTON,
+ 75, 70, 140, 25,
+ hwnd, 1, mh, 0)
+ if e != 0 {
+ abortErrNo("CreateWindowEx", e)
+ }
+ fmt.Printf("button handle is %x\n", bh)
+ rc = DefWindowProc(hwnd, msg, wparam, lparam)
+ case WM_COMMAND:
+ switch uint32(lparam) {
+ case bh:
+ e := PostMessage(hwnd, WM_CLOSE, 0, 0)
+ if e != 0 {
+ abortErrNo("PostMessage", e)
+ }
+ default:
+ rc = DefWindowProc(hwnd, msg, wparam, lparam)
+ }
+ case WM_CLOSE:
+ DestroyWindow(hwnd)
+ case WM_DESTROY:
+ PostQuitMessage(0)
+ default:
+ rc = DefWindowProc(hwnd, msg, wparam, lparam)
+ }
+ //fmt.Printf("WndProc(0x%08x, %d, 0x%08x, 0x%08x) (%d)\n", hwnd, msg, wparam, lparam, rc)
+ return uintptr(rc)
+}
+
+func rungui() int {
+ var e int
+
+ // GetModuleHandle
+ mh, e = GetModuleHandle(nil)
+ if e != 0 {
+ abortErrNo("GetModuleHandle", e)
+ }
+
+ // Get icon we're going to use.
+ myicon, e := LoadIcon(0, IDI_APPLICATION)
+ if e != 0 {
+ abortErrNo("LoadIcon", e)
+ }
+
+ // Get cursor we're going to use.
+ mycursor, e := LoadCursor(0, IDC_ARROW)
+ if e != 0 {
+ abortErrNo("LoadCursor", e)
+ }
+
+ // Create callback
+ wproc := syscall.NewCallback(WndProc)
+
+ // RegisterClassEx
+ wcname := syscall.StringToUTF16Ptr("myWindowClass")
+ var wc Wndclassex
+ wc.Size = uint32(unsafe.Sizeof(wc))
+ wc.WndProc = wproc
+ wc.Instance = mh
+ wc.Icon = myicon
+ wc.Cursor = mycursor
+ wc.Background = COLOR_BTNFACE + 1
+ wc.MenuName = nil
+ wc.ClassName = wcname
+ wc.IconSm = myicon
+ if _, e := RegisterClassEx(&wc); e != 0 {
+ abortErrNo("RegisterClassEx", e)
+ }
+
+ // CreateWindowEx
+ wh, e := CreateWindowEx(
+ WS_EX_CLIENTEDGE,
+ wcname,
+ syscall.StringToUTF16Ptr("My window"),
+ WS_OVERLAPPEDWINDOW,
+ CW_USEDEFAULT, CW_USEDEFAULT, 300, 200,
+ 0, 0, mh, 0)
+ if e != 0 {
+ abortErrNo("CreateWindowEx", e)
+ }
+ fmt.Printf("main window handle is %x\n", wh)
+
+ // ShowWindow
+ ShowWindow(wh, SW_SHOWDEFAULT)
+
+ // UpdateWindow
+ if e := UpdateWindow(wh); e != 0 {
+ abortErrNo("UpdateWindow", e)
+ }
+
+ // Process all windows messages until WM_QUIT.
+ var m Msg
+ for {
+ r, e := GetMessage(&m, 0, 0, 0)
+ if e != 0 {
+ abortErrNo("GetMessage", e)
+ }
+ if r == 0 {
+ // WM_QUIT received -> get out
+ break
+ }
+ TranslateMessage(&m)
+ DispatchMessage(&m)
+ }
+ return int(m.Wparam)
+}
+
+func main() {
+ rc := rungui()
+ os.Exit(rc)
+}
--- /dev/null
+// Copyright 2011 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package main
+
+import (
+ "syscall"
+ "unsafe"
+)
+
+func loadDll(fname string) uint32 {
+ h, e := syscall.LoadLibrary(fname)
+ if e != 0 {
+ abortf("LoadLibrary(%s) failed with err=%d.\n", fname, e)
+ }
+ return h
+}
+
+func getSysProcAddr(m uint32, pname string) uintptr {
+ p, e := syscall.GetProcAddress(m, pname)
+ if e != 0 {
+ abortf("GetProcAddress(%s) failed with err=%d.\n", pname, e)
+ }
+ return uintptr(p)
+}
+
+type Wndclassex struct {
+ Size uint32
+ Style uint32
+ WndProc uintptr
+ ClsExtra int32
+ WndExtra int32
+ Instance uint32
+ Icon uint32
+ Cursor uint32
+ Background uint32
+ MenuName *uint16
+ ClassName *uint16
+ IconSm uint32
+}
+
+type Point struct {
+ X int32
+ Y int32
+}
+
+type Msg struct {
+ Hwnd uint32
+ Message uint32
+ Wparam int32
+ Lparam int32
+ Time uint32
+ Pt Point
+}
+
+const (
+ // Window styles
+ WS_OVERLAPPED = 0
+ WS_POPUP = 0x80000000
+ WS_CHILD = 0x40000000
+ WS_MINIMIZE = 0x20000000
+ WS_VISIBLE = 0x10000000
+ WS_DISABLED = 0x8000000
+ WS_CLIPSIBLINGS = 0x4000000
+ WS_CLIPCHILDREN = 0x2000000
+ WS_MAXIMIZE = 0x1000000
+ WS_CAPTION = WS_BORDER | WS_DLGFRAME
+ WS_BORDER = 0x800000
+ WS_DLGFRAME = 0x400000
+ WS_VSCROLL = 0x200000
+ WS_HSCROLL = 0x100000
+ WS_SYSMENU = 0x80000
+ WS_THICKFRAME = 0x40000
+ WS_GROUP = 0x20000
+ WS_TABSTOP = 0x10000
+ WS_MINIMIZEBOX = 0x20000
+ WS_MAXIMIZEBOX = 0x10000
+ WS_TILED = WS_OVERLAPPED
+ WS_ICONIC = WS_MINIMIZE
+ WS_SIZEBOX = WS_THICKFRAME
+ // Common Window Styles
+ WS_OVERLAPPEDWINDOW = WS_OVERLAPPED | WS_CAPTION | WS_SYSMENU | WS_THICKFRAME | WS_MINIMIZEBOX | WS_MAXIMIZEBOX
+ WS_TILEDWINDOW = WS_OVERLAPPEDWINDOW
+ WS_POPUPWINDOW = WS_POPUP | WS_BORDER | WS_SYSMENU
+ WS_CHILDWINDOW = WS_CHILD
+
+ WS_EX_CLIENTEDGE = 0x200
+
+ // Some windows messages
+ WM_CREATE = 1
+ WM_DESTROY = 2
+ WM_CLOSE = 16
+ WM_COMMAND = 273
+
+ // Some button control styles
+ BS_DEFPUSHBUTTON = 1
+
+ // Some colour constants
+ COLOR_WINDOW = 5
+ COLOR_BTNFACE = 15
+
+ // Default window position
+ CW_USEDEFAULT = 0x80000000 - 0x100000000
+
+ // Show window default style
+ SW_SHOWDEFAULT = 10
+)
+
+var (
+ // Some globaly known cusrors
+ IDC_ARROW = MakeIntResource(32512)
+ IDC_IBEAM = MakeIntResource(32513)
+ IDC_WAIT = MakeIntResource(32514)
+ IDC_CROSS = MakeIntResource(32515)
+
+ // Some globaly known icons
+ IDI_APPLICATION = MakeIntResource(32512)
+ IDI_HAND = MakeIntResource(32513)
+ IDI_QUESTION = MakeIntResource(32514)
+ IDI_EXCLAMATION = MakeIntResource(32515)
+ IDI_ASTERISK = MakeIntResource(32516)
+ IDI_WINLOGO = MakeIntResource(32517)
+ IDI_WARNING = IDI_EXCLAMATION
+ IDI_ERROR = IDI_HAND
+ IDI_INFORMATION = IDI_ASTERISK
+)
+
+//sys GetModuleHandle(modname *uint16) (handle uint32, errno int) = GetModuleHandleW
+//sys RegisterClassEx(wndclass *Wndclassex) (atom uint16, errno int) = user32.RegisterClassExW
+//sys CreateWindowEx(exstyle uint32, classname *uint16, windowname *uint16, style uint32, x int32, y int32, width int32, height int32, wndparent uint32, menu uint32, instance uint32, param uintptr) (hwnd uint32, errno int) = user32.CreateWindowExW
+//sys DefWindowProc(hwnd uint32, msg uint32, wparam int32, lparam int32) (lresult int32) = user32.DefWindowProcW
+//sys DestroyWindow(hwnd uint32) (errno int) = user32.DestroyWindow
+//sys PostQuitMessage(exitcode int32) = user32.PostQuitMessage
+//sys ShowWindow(hwnd uint32, cmdshow int32) (wasvisible bool) = user32.ShowWindow
+//sys UpdateWindow(hwnd uint32) (errno int) = user32.UpdateWindow
+//sys GetMessage(msg *Msg, hwnd uint32, MsgFilterMin uint32, MsgFilterMax uint32) (ret int32, errno int) [failretval==-1] = user32.GetMessageW
+//sys TranslateMessage(msg *Msg) (done bool) = user32.TranslateMessage
+//sys DispatchMessage(msg *Msg) (ret int32) = user32.DispatchMessageW
+//sys LoadIcon(instance uint32, iconname *uint16) (icon uint32, errno int) = user32.LoadIconW
+//sys LoadCursor(instance uint32, cursorname *uint16) (cursor uint32, errno int) = user32.LoadCursorW
+//sys SetCursor(cursor uint32) (precursor uint32, errno int) = user32.SetCursor
+//sys SendMessage(hwnd uint32, msg uint32, wparam int32, lparam int32) (lresult int32) = user32.SendMessageW
+//sys PostMessage(hwnd uint32, msg uint32, wparam int32, lparam int32) (errno int) = user32.PostMessageW
+
+func MakeIntResource(id uint16) *uint16 {
+ return (*uint16)(unsafe.Pointer(uintptr(id)))
+}
--- /dev/null
+// mksyscall_windows.sh winapi.go
+// MACHINE GENERATED BY THE COMMAND ABOVE; DO NOT EDIT
+
+package main
+
+import "unsafe"
+import "syscall"
+
+var (
+ modkernel32 = loadDll("kernel32.dll")
+ moduser32 = loadDll("user32.dll")
+
+ procGetModuleHandleW = getSysProcAddr(modkernel32, "GetModuleHandleW")
+ procRegisterClassExW = getSysProcAddr(moduser32, "RegisterClassExW")
+ procCreateWindowExW = getSysProcAddr(moduser32, "CreateWindowExW")
+ procDefWindowProcW = getSysProcAddr(moduser32, "DefWindowProcW")
+ procDestroyWindow = getSysProcAddr(moduser32, "DestroyWindow")
+ procPostQuitMessage = getSysProcAddr(moduser32, "PostQuitMessage")
+ procShowWindow = getSysProcAddr(moduser32, "ShowWindow")
+ procUpdateWindow = getSysProcAddr(moduser32, "UpdateWindow")
+ procGetMessageW = getSysProcAddr(moduser32, "GetMessageW")
+ procTranslateMessage = getSysProcAddr(moduser32, "TranslateMessage")
+ procDispatchMessageW = getSysProcAddr(moduser32, "DispatchMessageW")
+ procLoadIconW = getSysProcAddr(moduser32, "LoadIconW")
+ procLoadCursorW = getSysProcAddr(moduser32, "LoadCursorW")
+ procSetCursor = getSysProcAddr(moduser32, "SetCursor")
+ procSendMessageW = getSysProcAddr(moduser32, "SendMessageW")
+ procPostMessageW = getSysProcAddr(moduser32, "PostMessageW")
+)
+
+func GetModuleHandle(modname *uint16) (handle uint32, errno int) {
+ r0, _, e1 := syscall.Syscall(procGetModuleHandleW, 1, uintptr(unsafe.Pointer(modname)), 0, 0)
+ handle = uint32(r0)
+ if handle == 0 {
+ if e1 != 0 {
+ errno = int(e1)
+ } else {
+ errno = syscall.EINVAL
+ }
+ } else {
+ errno = 0
+ }
+ return
+}
+
+func RegisterClassEx(wndclass *Wndclassex) (atom uint16, errno int) {
+ r0, _, e1 := syscall.Syscall(procRegisterClassExW, 1, uintptr(unsafe.Pointer(wndclass)), 0, 0)
+ atom = uint16(r0)
+ if atom == 0 {
+ if e1 != 0 {
+ errno = int(e1)
+ } else {
+ errno = syscall.EINVAL
+ }
+ } else {
+ errno = 0
+ }
+ return
+}
+
+func CreateWindowEx(exstyle uint32, classname *uint16, windowname *uint16, style uint32, x int32, y int32, width int32, height int32, wndparent uint32, menu uint32, instance uint32, param uintptr) (hwnd uint32, errno int) {
+ r0, _, e1 := syscall.Syscall12(procCreateWindowExW, 12, uintptr(exstyle), uintptr(unsafe.Pointer(classname)), uintptr(unsafe.Pointer(windowname)), uintptr(style), uintptr(x), uintptr(y), uintptr(width), uintptr(height), uintptr(wndparent), uintptr(menu), uintptr(instance), uintptr(param))
+ hwnd = uint32(r0)
+ if hwnd == 0 {
+ if e1 != 0 {
+ errno = int(e1)
+ } else {
+ errno = syscall.EINVAL
+ }
+ } else {
+ errno = 0
+ }
+ return
+}
+
+func DefWindowProc(hwnd uint32, msg uint32, wparam int32, lparam int32) (lresult int32) {
+ r0, _, _ := syscall.Syscall6(procDefWindowProcW, 4, uintptr(hwnd), uintptr(msg), uintptr(wparam), uintptr(lparam), 0, 0)
+ lresult = int32(r0)
+ return
+}
+
+func DestroyWindow(hwnd uint32) (errno int) {
+ r1, _, e1 := syscall.Syscall(procDestroyWindow, 1, uintptr(hwnd), 0, 0)
+ if int(r1) == 0 {
+ if e1 != 0 {
+ errno = int(e1)
+ } else {
+ errno = syscall.EINVAL
+ }
+ } else {
+ errno = 0
+ }
+ return
+}
+
+func PostQuitMessage(exitcode int32) {
+ syscall.Syscall(procPostQuitMessage, 1, uintptr(exitcode), 0, 0)
+ return
+}
+
+func ShowWindow(hwnd uint32, cmdshow int32) (wasvisible bool) {
+ r0, _, _ := syscall.Syscall(procShowWindow, 2, uintptr(hwnd), uintptr(cmdshow), 0)
+ wasvisible = bool(r0 != 0)
+ return
+}
+
+func UpdateWindow(hwnd uint32) (errno int) {
+ r1, _, e1 := syscall.Syscall(procUpdateWindow, 1, uintptr(hwnd), 0, 0)
+ if int(r1) == 0 {
+ if e1 != 0 {
+ errno = int(e1)
+ } else {
+ errno = syscall.EINVAL
+ }
+ } else {
+ errno = 0
+ }
+ return
+}
+
+func GetMessage(msg *Msg, hwnd uint32, MsgFilterMin uint32, MsgFilterMax uint32) (ret int32, errno int) {
+ r0, _, e1 := syscall.Syscall6(procGetMessageW, 4, uintptr(unsafe.Pointer(msg)), uintptr(hwnd), uintptr(MsgFilterMin), uintptr(MsgFilterMax), 0, 0)
+ ret = int32(r0)
+ if ret == -1 {
+ if e1 != 0 {
+ errno = int(e1)
+ } else {
+ errno = syscall.EINVAL
+ }
+ } else {
+ errno = 0
+ }
+ return
+}
+
+func TranslateMessage(msg *Msg) (done bool) {
+ r0, _, _ := syscall.Syscall(procTranslateMessage, 1, uintptr(unsafe.Pointer(msg)), 0, 0)
+ done = bool(r0 != 0)
+ return
+}
+
+func DispatchMessage(msg *Msg) (ret int32) {
+ r0, _, _ := syscall.Syscall(procDispatchMessageW, 1, uintptr(unsafe.Pointer(msg)), 0, 0)
+ ret = int32(r0)
+ return
+}
+
+func LoadIcon(instance uint32, iconname *uint16) (icon uint32, errno int) {
+ r0, _, e1 := syscall.Syscall(procLoadIconW, 2, uintptr(instance), uintptr(unsafe.Pointer(iconname)), 0)
+ icon = uint32(r0)
+ if icon == 0 {
+ if e1 != 0 {
+ errno = int(e1)
+ } else {
+ errno = syscall.EINVAL
+ }
+ } else {
+ errno = 0
+ }
+ return
+}
+
+func LoadCursor(instance uint32, cursorname *uint16) (cursor uint32, errno int) {
+ r0, _, e1 := syscall.Syscall(procLoadCursorW, 2, uintptr(instance), uintptr(unsafe.Pointer(cursorname)), 0)
+ cursor = uint32(r0)
+ if cursor == 0 {
+ if e1 != 0 {
+ errno = int(e1)
+ } else {
+ errno = syscall.EINVAL
+ }
+ } else {
+ errno = 0
+ }
+ return
+}
+
+func SetCursor(cursor uint32) (precursor uint32, errno int) {
+ r0, _, e1 := syscall.Syscall(procSetCursor, 1, uintptr(cursor), 0, 0)
+ precursor = uint32(r0)
+ if precursor == 0 {
+ if e1 != 0 {
+ errno = int(e1)
+ } else {
+ errno = syscall.EINVAL
+ }
+ } else {
+ errno = 0
+ }
+ return
+}
+
+func SendMessage(hwnd uint32, msg uint32, wparam int32, lparam int32) (lresult int32) {
+ r0, _, _ := syscall.Syscall6(procSendMessageW, 4, uintptr(hwnd), uintptr(msg), uintptr(wparam), uintptr(lparam), 0, 0)
+ lresult = int32(r0)
+ return
+}
+
+func PostMessage(hwnd uint32, msg uint32, wparam int32, lparam int32) (errno int) {
+ r1, _, e1 := syscall.Syscall6(procPostMessageW, 4, uintptr(hwnd), uintptr(msg), uintptr(wparam), uintptr(lparam), 0, 0)
+ if int(r1) == 0 {
+ if e1 != 0 {
+ errno = int(e1)
+ } else {
+ errno = syscall.EINVAL
+ }
+ } else {
+ errno = 0
+ }
+ return
+}
}
func (i *intValue) Set(s string) bool {
- v, err := strconv.Atoi(s)
+ v, err := strconv.Btoi64(s, 0)
*i = intValue(v)
return err == nil
}
}
func (i *int64Value) Set(s string) bool {
- v, err := strconv.Atoi64(s)
+ v, err := strconv.Btoi64(s, 0)
*i = int64Value(v)
return err == nil
}
}
func (i *uintValue) Set(s string) bool {
- v, err := strconv.Atoui(s)
+ v, err := strconv.Btoui64(s, 0)
*i = uintValue(v)
return err == nil
}
}
func (i *uint64Value) Set(s string) bool {
- v, err := strconv.Atoui64(s)
+ v, err := strconv.Btoui64(s, 0)
*i = uint64Value(v)
return err == nil
}
"-bool",
"-bool2=true",
"--int", "22",
- "--int64", "23",
+ "--int64", "0x23",
"-uint", "24",
"--uint64", "25",
"-string", "hello",
if *intFlag != 22 {
t.Error("int flag should be 22, is ", *intFlag)
}
- if *int64Flag != 23 {
- t.Error("int64 flag should be 23, is ", *int64Flag)
+ if *int64Flag != 0x23 {
+ t.Error("int64 flag should be 0x23, is ", *int64Flag)
}
if *uintFlag != 24 {
t.Error("uint flag should be 24, is ", *uintFlag)
when printing structs, the plus flag (%+v) adds field names
%#v a Go-syntax representation of the value
%T a Go-syntax representation of the type of the value
+ %% a literal percent sign; consumes no value
Boolean:
%t the word true or false
%o base 8
%x base 16, with lower-case letters for a-f
%X base 16, with upper-case letters for A-F
- %U unicode format: U+1234; same as "U+%x" with 4 digits default
+ %U Unicode format: U+1234; same as "U+%x" with 4 digits default
Floating-point and complex constituents:
+ %b decimalless scientific notation with exponent a power
+ of two, in the manner of strconv.Ftoa32, e.g. -123456p-78
%e scientific notation, e.g. -1234.456e+78
%E scientific notation, e.g. -1234.456E+78
%f decimal point but no exponent, e.g. 123.456
There is no 'u' flag. Integers are printed unsigned if they have unsigned type.
Similarly, there is no need to specify the size of the operand (int8, int64).
- For numeric values, the width and precision flags control
- formatting; width sets the width of the field, precision the
- number of places after the decimal, if appropriate. The
- format %6.2f prints 123.45. The width of a field is the number
- of Unicode code points in the string. This differs from C's printf where
- the field width is the number of bytes. Either or both of the
- flags may be replaced with the character '*', causing their values
- to be obtained from the next operand, which must be of type int.
+ The width and precision control formatting and are in units of Unicode
+ code points. (This differs from C's printf where the units are numbers
+ of bytes.) Either or both of the flags may be replaced with the
+ character '*', causing their values to be obtained from the next
+ operand, which must be of type int.
+
+ For numeric values, width sets the width of the field and precision
+ sets the number of places after the decimal, if appropriate. For
+ example, the format %6.2f prints 123.45.
+
+ For strings, width is the minimum number of characters to output,
+ padding with spaces if necessary, and precision is the maximum
+ number of characters to output, truncating if necessary.
Other flags:
+ always print a sign for numeric values
An analogous set of functions scans formatted text to yield
values. Scan, Scanf and Scanln read from os.Stdin; Fscan,
Fscanf and Fscanln read from a specified os.Reader; Sscan,
- Sscanf and Sscanln read from an argument string. Sscanln,
+ Sscanf and Sscanln read from an argument string. Scanln,
Fscanln and Sscanln stop scanning at a newline and require that
the items be followed by one; Sscanf, Fscanf and Sscanf require
newlines in the input to match newlines in the format; the other
%e %E %f %F %g %g are all equivalent and scan any floating point or complex value
%s and %v on strings scan a space-delimited token
+ The familiar base-setting prefixes 0 (octal) and 0x
+ (hexadecimal) are accepted when scanning integers without a
+ format or with the %v verb.
+
Width is interpreted in the input text (%5s means at most
five runes of input will be read to scan a string) but there
is no syntax for scanning with a precision (no %5.2f, just
All arguments to be scanned must be either pointers to basic
types or implementations of the Scanner interface.
- Note: Fscan etc. can read one character (rune) past the
- input they return, which means that a loop calling a scan
- routine may skip some of the input. This is usually a
- problem only when there is no space between input values.
- However, if the reader provided to Fscan implements UnreadRune,
+ Note: Fscan etc. can read one character (rune) past the input
+ they return, which means that a loop calling a scan routine
+ may skip some of the input. This is usually a problem only
+ when there is no space between input values. If the reader
+ provided to Fscan implements ReadRune, that method will be used
+ to read characters. If the reader also implements UnreadRune,
that method will be used to save the character and successive
- calls will not lose data. To attach an UnreadRune method
- to a reader without that capability, use bufio.NewReader.
+ calls will not lose data. To attach ReadRune and UnreadRune
+ methods to a reader without that capability, use
+ bufio.NewReader.
*/
package fmt
// go syntax
{"%#v", A{1, 2, "a", []int{1, 2}}, `fmt_test.A{i:1, j:0x2, s:"a", x:[]int{1, 2}}`},
- {"%#v", &b, "(*uint8)(PTR)"},
- {"%#v", TestFmtInterface, "(func(*testing.T))(PTR)"},
- {"%#v", make(chan int), "(chan int)(PTR)"},
+ {"%#v", &b, "(*uint8)(0xPTR)"},
+ {"%#v", TestFmtInterface, "(func(*testing.T))(0xPTR)"},
+ {"%#v", make(chan int), "(chan int)(0xPTR)"},
{"%#v", uint64(1<<64 - 1), "0xffffffffffffffff"},
{"%#v", 1000000000, "1000000000"},
{"%#v", map[string]int{"a": 1, "b": 2}, `map[string] int{"a":1, "b":2}`},
{"%6T", &intVal, " *int"},
// %p
- {"p0=%p", new(int), "p0=PTR"},
+ {"p0=%p", new(int), "p0=0xPTR"},
{"p1=%s", &pValue, "p1=String(p)"}, // String method...
- {"p2=%p", &pValue, "p2=PTR"}, // ... not called with %p
+ {"p2=%p", &pValue, "p2=0xPTR"}, // ... not called with %p
+ {"p4=%#p", new(int), "p4=PTR"},
// %p on non-pointers
- {"%p", make(chan int), "PTR"},
- {"%p", make(map[int]int), "PTR"},
- {"%p", make([]int, 1), "PTR"},
+ {"%p", make(chan int), "0xPTR"},
+ {"%p", make(map[int]int), "0xPTR"},
+ {"%p", make([]int, 1), "0xPTR"},
{"%p", 27, "%!p(int=27)"}, // not a pointer at all
// erroneous things
func TestSprintf(t *testing.T) {
for _, tt := range fmttests {
s := Sprintf(tt.fmt, tt.val)
- if i := strings.Index(s, "0x"); i >= 0 && strings.Contains(tt.out, "PTR") {
- j := i + 2
+ if i := strings.Index(tt.out, "PTR"); i >= 0 {
+ j := i
for ; j < len(s); j++ {
c := s[j]
if (c < '0' || c > '9') && (c < 'a' || c > 'f') && (c < 'A' || c > 'F') {
fmt fmt
}
-// A leaky bucket of reusable pp structures.
-var ppFree = make(chan *pp, 100)
+// A cache holds a set of reusable objects.
+// The buffered channel holds the currently available objects.
+// If more are needed, the cache creates them by calling new.
+type cache struct {
+ saved chan interface{}
+ new func() interface{}
+}
+
+func (c *cache) put(x interface{}) {
+ select {
+ case c.saved <- x:
+ // saved in cache
+ default:
+ // discard
+ }
+}
-// Allocate a new pp struct. Probably can grab the previous one from ppFree.
-func newPrinter() *pp {
- p, ok := <-ppFree
- if !ok {
- p = new(pp)
+func (c *cache) get() interface{} {
+ select {
+ case x := <-c.saved:
+ return x // reused from cache
+ default:
+ return c.new()
}
+ panic("not reached")
+}
+
+func newCache(f func() interface{}) *cache {
+ return &cache{make(chan interface{}, 100), f}
+}
+
+var ppFree = newCache(func() interface{} { return new(pp) })
+
+// Allocate a new pp struct or grab a cached one.
+func newPrinter() *pp {
+ p := ppFree.get().(*pp)
p.fmt.init(&p.buf)
return p
}
return
}
p.buf.Reset()
- _ = ppFree <- p
+ ppFree.put(p)
}
func (p *pp) Width() (wid int, ok bool) { return p.fmt.wid, p.fmt.widPresent }
}
}
-// fmt0x64 formats a uint64 in hexadecimal and prefixes it with 0x by
-// temporarily turning on the sharp flag.
-func (p *pp) fmt0x64(v uint64) {
+// fmt0x64 formats a uint64 in hexadecimal and prefixes it with 0x or
+// not, as requested, by temporarily setting the sharp flag.
+func (p *pp) fmt0x64(v uint64, leading0x bool) {
sharp := p.fmt.sharp
- p.fmt.sharp = true // turn on 0x
+ p.fmt.sharp = leading0x
p.fmt.integer(int64(v), 16, unsigned, ldigits)
p.fmt.sharp = sharp
}
p.fmt.integer(int64(v), 10, unsigned, ldigits)
case 'v':
if goSyntax {
- p.fmt0x64(v)
+ p.fmt0x64(v, true)
} else {
p.fmt.integer(int64(v), 10, unsigned, ldigits)
}
if u == 0 {
p.buf.Write(nilBytes)
} else {
- p.fmt0x64(uint64(v.Get()))
+ p.fmt0x64(uint64(v.Get()), true)
}
p.add(')')
} else {
- p.fmt0x64(uint64(u))
+ p.fmt0x64(uint64(u), !p.fmt.sharp)
}
}
if v == 0 {
p.buf.Write(nilBytes)
} else {
- p.fmt0x64(uint64(v))
+ p.fmt0x64(uint64(v), true)
}
p.buf.WriteByte(')')
break
p.buf.Write(nilAngleBytes)
break
}
- p.fmt0x64(uint64(v))
+ p.fmt0x64(uint64(v), true)
case uintptrGetter:
p.fmtPointer(field, value, verb, goSyntax)
default:
import (
"bytes"
"io"
+ "math"
"os"
"reflect"
"strconv"
"utf8"
)
-// readRuner is the interface to something that can read runes. If
-// the object provided to Scan does not satisfy this interface, the
-// object will be wrapped by a readRune object.
-type readRuner interface {
- ReadRune() (rune int, size int, err os.Error)
-}
-
-// unreadRuner is the interface to something that can unread runes.
+// runeUnreader is the interface to something that can unread runes.
// If the object provided to Scan does not satisfy this interface,
// a local buffer will be used to back up the input, but its contents
// will be lost when Scan returns.
-type unreadRuner interface {
+type runeUnreader interface {
UnreadRune() os.Error
}
// Scanners may do rune-at-a-time scanning or ask the ScanState
// to discover the next space-delimited token.
type ScanState interface {
- // GetRune reads the next rune (Unicode code point) from the input.
- GetRune() (rune int, err os.Error)
- // UngetRune causes the next call to GetRune to return the rune.
- UngetRune()
- // Width returns the value of the width option and whether it has been set.
- // The unit is Unicode code points.
- Width() (wid int, ok bool)
+ // ReadRune reads the next rune (Unicode code point) from the input.
+ // If invoked during Scanln, Fscanln, or Sscanln, ReadRune() will
+ // return EOF after returning the first '\n' or when reading beyond
+ // the specified width.
+ ReadRune() (rune int, size int, err os.Error)
+ // UnreadRune causes the next call to ReadRune to return the same rune.
+ UnreadRune() os.Error
// Token returns the next space-delimited token from the input. If
// a width has been specified, the returned token will be no longer
// than the width.
Token() (token string, err os.Error)
+ // Width returns the value of the width option and whether it has been set.
+ // The unit is Unicode code points.
+ Width() (wid int, ok bool)
+ // Because ReadRune is implemented by the interface, Read should never be
+ // called by the scanning routines and a valid implementation of
+ // ScanState may choose always to return an error from Read.
+ Read(buf []byte) (n int, err os.Error)
}
// Scanner is implemented by any value that has a Scan method, which scans
// the input for the representation of a value and stores the result in the
// receiver, which must be a pointer to be useful. The Scan method is called
-// for any argument to Scan or Scanln that implements it.
+// for any argument to Scan, Scanf, or Scanln that implements it.
type Scanner interface {
Scan(state ScanState, verb int) os.Error
}
// returns the number of items successfully scanned. If that is less
// than the number of arguments, err will report why.
func Fscan(r io.Reader, a ...interface{}) (n int, err os.Error) {
- s := newScanState(r, true)
+ s, old := newScanState(r, true, false)
n, err = s.doScan(a)
- s.free()
+ s.free(old)
return
}
// Fscanln is similar to Fscan, but stops scanning at a newline and
// after the final item there must be a newline or EOF.
func Fscanln(r io.Reader, a ...interface{}) (n int, err os.Error) {
- s := newScanState(r, false)
+ s, old := newScanState(r, false, true)
n, err = s.doScan(a)
- s.free()
+ s.free(old)
return
}
// values into successive arguments as determined by the format. It
// returns the number of items successfully parsed.
func Fscanf(r io.Reader, format string, a ...interface{}) (n int, err os.Error) {
- s := newScanState(r, false)
+ s, old := newScanState(r, false, false)
n, err = s.doScanf(format, a)
- s.free()
+ s.free(old)
return
}
// ss is the internal implementation of ScanState.
type ss struct {
- rr readRuner // where to read input
- buf bytes.Buffer // token accumulator
- nlIsSpace bool // whether newline counts as white space
- peekRune int // one-rune lookahead
- prevRune int // last rune returned by GetRune
- atEOF bool // already read EOF
- maxWid int // max width of field, in runes
- widPresent bool // width was specified
- wid int // width consumed so far; used in accept()
-}
-
-func (s *ss) GetRune() (rune int, err os.Error) {
+ rr io.RuneReader // where to read input
+ buf bytes.Buffer // token accumulator
+ peekRune int // one-rune lookahead
+ prevRune int // last rune returned by ReadRune
+ count int // runes consumed so far.
+ atEOF bool // already read EOF
+ ssave
+}
+
+// ssave holds the parts of ss that need to be
+// saved and restored on recursive scans.
+type ssave struct {
+ validSave bool // is or was a part of an actual ss.
+ nlIsEnd bool // whether newline terminates scan
+ nlIsSpace bool // whether newline counts as white space
+ fieldLimit int // max value of ss.count for this field; fieldLimit <= limit
+ limit int // max value of ss.count.
+ maxWid int // width of this field.
+}
+
+// The Read method is only in ScanState so that ScanState
+// satisfies io.Reader. It will never be called when used as
+// intended, so there is no need to make it actually work.
+func (s *ss) Read(buf []byte) (n int, err os.Error) {
+ return 0, os.ErrorString("ScanState's Read should not be called. Use ReadRune")
+}
+
+func (s *ss) ReadRune() (rune int, size int, err os.Error) {
if s.peekRune >= 0 {
+ s.count++
rune = s.peekRune
+ size = utf8.RuneLen(rune)
s.prevRune = rune
s.peekRune = -1
return
}
- rune, _, err = s.rr.ReadRune()
+ if s.atEOF || s.nlIsEnd && s.prevRune == '\n' || s.count >= s.fieldLimit {
+ err = os.EOF
+ return
+ }
+
+ rune, size, err = s.rr.ReadRune()
if err == nil {
+ s.count++
s.prevRune = rune
+ } else if err == os.EOF {
+ s.atEOF = true
}
return
}
func (s *ss) Width() (wid int, ok bool) {
- return s.maxWid, s.widPresent
+ if s.maxWid == hugeWid {
+ return 0, false
+ }
+ return s.maxWid, true
}
// The public method returns an error; this private one panics.
// If getRune reaches EOF, the return value is EOF (-1).
func (s *ss) getRune() (rune int) {
- if s.atEOF {
- return EOF
- }
- if s.peekRune >= 0 {
- rune = s.peekRune
- s.prevRune = rune
- s.peekRune = -1
- return
- }
- rune, _, err := s.rr.ReadRune()
- if err == nil {
- s.prevRune = rune
- } else if err != nil {
+ rune, _, err := s.ReadRune()
+ if err != nil {
if err == os.EOF {
- s.atEOF = true
return EOF
}
s.error(err)
return
}
-// mustGetRune turns os.EOF into a panic(io.ErrUnexpectedEOF).
+// mustReadRune turns os.EOF into a panic(io.ErrUnexpectedEOF).
// It is called in cases such as string scanning where an EOF is a
// syntax error.
-func (s *ss) mustGetRune() (rune int) {
- if s.atEOF {
+func (s *ss) mustReadRune() (rune int) {
+ rune = s.getRune()
+ if rune == EOF {
s.error(io.ErrUnexpectedEOF)
}
- if s.peekRune >= 0 {
- rune = s.peekRune
- s.peekRune = -1
- return
- }
- rune, _, err := s.rr.ReadRune()
- if err != nil {
- if err == os.EOF {
- err = io.ErrUnexpectedEOF
- }
- s.error(err)
- }
return
}
-
-func (s *ss) UngetRune() {
- if u, ok := s.rr.(unreadRuner); ok {
+func (s *ss) UnreadRune() os.Error {
+ if u, ok := s.rr.(runeUnreader); ok {
u.UnreadRune()
} else {
s.peekRune = s.prevRune
}
+ s.count--
+ return nil
}
func (s *ss) error(err os.Error) {
// readRune is a structure to enable reading UTF-8 encoded code points
// from an io.Reader. It is used if the Reader given to the scanner does
-// not already implement ReadRuner.
+// not already implement io.RuneReader.
type readRune struct {
reader io.Reader
buf [utf8.UTFMax]byte // used only inside ReadRune
}
-// A leaky bucket of reusable ss structures.
-var ssFree = make(chan *ss, 100)
+var ssFree = newCache(func() interface{} { return new(ss) })
-// Allocate a new ss struct. Probably can grab the previous one from ssFree.
-func newScanState(r io.Reader, nlIsSpace bool) *ss {
- s, ok := <-ssFree
- if !ok {
- s = new(ss)
+// Allocate a new ss struct or grab a cached one.
+func newScanState(r io.Reader, nlIsSpace, nlIsEnd bool) (s *ss, old ssave) {
+ // If the reader is a *ss, then we've got a recursive
+ // call to Scan, so re-use the scan state.
+ s, ok := r.(*ss)
+ if ok {
+ old = s.ssave
+ s.limit = s.fieldLimit
+ s.nlIsEnd = nlIsEnd || s.nlIsEnd
+ s.nlIsSpace = nlIsSpace
+ return
}
- if rr, ok := r.(readRuner); ok {
+
+ s = ssFree.get().(*ss)
+ if rr, ok := r.(io.RuneReader); ok {
s.rr = rr
} else {
s.rr = &readRune{reader: r}
}
s.nlIsSpace = nlIsSpace
+ s.nlIsEnd = nlIsEnd
+ s.prevRune = -1
s.peekRune = -1
s.atEOF = false
- s.maxWid = 0
- s.widPresent = false
- return s
+ s.limit = hugeWid
+ s.fieldLimit = hugeWid
+ s.maxWid = hugeWid
+ s.validSave = true
+ return
}
// Save used ss structs in ssFree; avoid an allocation per invocation.
-func (s *ss) free() {
+func (s *ss) free(old ssave) {
+ // If it was used recursively, just restore the old state.
+ if old.validSave {
+ s.ssave = old
+ return
+ }
// Don't hold on to ss structs with large buffers.
if cap(s.buf.Bytes()) > 1024 {
return
}
s.buf.Reset()
s.rr = nil
- _ = ssFree <- s
+ ssFree.put(s)
}
// skipSpace skips spaces and maybe newlines.
return
}
if !unicode.IsSpace(rune) {
- s.UngetRune()
+ s.UnreadRune()
break
}
}
func (s *ss) token() string {
s.skipSpace(false)
// read until white space or newline
- for nrunes := 0; !s.widPresent || nrunes < s.maxWid; nrunes++ {
+ for {
rune := s.getRune()
if rune == EOF {
break
}
if unicode.IsSpace(rune) {
- s.UngetRune()
+ s.UnreadRune()
break
}
s.buf.WriteRune(rune)
// consume reads the next rune in the input and reports whether it is in the ok string.
// If accept is true, it puts the character into the input token.
func (s *ss) consume(ok string, accept bool) bool {
- if s.wid >= s.maxWid {
- return false
- }
rune := s.getRune()
if rune == EOF {
return false
}
- for i := 0; i < len(ok); i++ {
- if int(ok[i]) == rune {
- if accept {
- s.buf.WriteRune(rune)
- s.wid++
- }
- return true
+ if strings.IndexRune(ok, rune) >= 0 {
+ if accept {
+ s.buf.WriteRune(rune)
}
+ return true
}
if rune != EOF && accept {
- s.UngetRune()
+ s.UnreadRune()
}
return false
}
+// peek reports whether the next character is in the ok string, without consuming it.
+func (s *ss) peek(ok string) bool {
+ rune := s.getRune()
+ if rune != EOF {
+ s.UnreadRune()
+ }
+ return strings.IndexRune(ok, rune) >= 0
+}
+
// accept checks the next rune in the input. If it's a byte (sic) in the string, it puts it in the
// buffer and returns true. Otherwise it return false.
func (s *ss) accept(ok string) bool {
return false
}
// Syntax-checking a boolean is annoying. We're not fastidious about case.
- switch s.mustGetRune() {
+ switch s.mustReadRune() {
case '0':
return false
case '1':
hexadecimalDigits = "0123456789aAbBcCdDeEfF"
sign = "+-"
period = "."
- exponent = "eE"
+ exponent = "eEp"
)
// getBase returns the numeric base represented by the verb and its digit string.
}
// scanNumber returns the numerical string with specified digits starting here.
-func (s *ss) scanNumber(digits string) string {
- if !s.accept(digits) {
+func (s *ss) scanNumber(digits string, haveDigits bool) string {
+ if !haveDigits && !s.accept(digits) {
s.errorString("expected integer")
}
for s.accept(digits) {
// scanRune returns the next rune value in the input.
func (s *ss) scanRune(bitSize int) int64 {
- rune := int64(s.mustGetRune())
+ rune := int64(s.mustReadRune())
n := uint(bitSize)
x := (rune << (64 - n)) >> (64 - n)
if x != rune {
return rune
}
+// scanBasePrefix reports whether the integer begins with a 0 or 0x,
+// and returns the base, digit string, and whether a zero was found.
+// It is called only if the verb is %v.
+func (s *ss) scanBasePrefix() (base int, digits string, found bool) {
+ if !s.peek("0") {
+ return 10, decimalDigits, false
+ }
+ s.accept("0")
+ found = true // We've put a digit into the token buffer.
+ // Special cases for '0' && '0x'
+ base, digits = 8, octalDigits
+ if s.peek("xX") {
+ s.consume("xX", false)
+ base, digits = 16, hexadecimalDigits
+ }
+ return
+}
+
// scanInt returns the value of the integer represented by the next
// token, checking for overflow. Any error is stored in s.err.
func (s *ss) scanInt(verb int, bitSize int) int64 {
if verb == 'c' {
return s.scanRune(bitSize)
}
- base, digits := s.getBase(verb)
s.skipSpace(false)
+ base, digits := s.getBase(verb)
+ haveDigits := false
if verb == 'U' {
if !s.consume("U", false) || !s.consume("+", false) {
s.errorString("bad unicode format ")
}
} else {
s.accept(sign) // If there's a sign, it will be left in the token buffer.
+ if verb == 'v' {
+ base, digits, haveDigits = s.scanBasePrefix()
+ }
}
- tok := s.scanNumber(digits)
+ tok := s.scanNumber(digits, haveDigits)
i, err := strconv.Btoi64(tok, base)
if err != nil {
s.error(err)
if verb == 'c' {
return uint64(s.scanRune(bitSize))
}
- base, digits := s.getBase(verb)
s.skipSpace(false)
+ base, digits := s.getBase(verb)
+ haveDigits := false
if verb == 'U' {
if !s.consume("U", false) || !s.consume("+", false) {
s.errorString("bad unicode format ")
}
+ } else if verb == 'v' {
+ base, digits, haveDigits = s.scanBasePrefix()
}
- tok := s.scanNumber(digits)
+ tok := s.scanNumber(digits, haveDigits)
i, err := strconv.Btoui64(tok, base)
if err != nil {
s.error(err)
// convertFloat converts the string to a float64value.
func (s *ss) convertFloat(str string, n int) float64 {
+ if p := strings.Index(str, "p"); p >= 0 {
+ // Atof doesn't handle power-of-2 exponents,
+ // but they're easy to evaluate.
+ f, err := strconv.AtofN(str[:p], n)
+ if err != nil {
+ // Put full string into error.
+ if e, ok := err.(*strconv.NumError); ok {
+ e.Num = str
+ }
+ s.error(err)
+ }
+ n, err := strconv.Atoi(str[p+1:])
+ if err != nil {
+ // Put full string into error.
+ if e, ok := err.(*strconv.NumError); ok {
+ e.Num = str
+ }
+ s.error(err)
+ }
+ return math.Ldexp(f, n)
+ }
f, err := strconv.AtofN(str, n)
if err != nil {
s.error(err)
// quotedString returns the double- or back-quoted string represented by the next input characters.
func (s *ss) quotedString() string {
- quote := s.mustGetRune()
+ quote := s.mustReadRune()
switch quote {
case '`':
// Back-quoted: Anything goes until EOF or back quote.
for {
- rune := s.mustGetRune()
+ rune := s.mustReadRune()
if rune == quote {
break
}
// Double-quoted: Include the quotes and let strconv.Unquote do the backslash escapes.
s.buf.WriteRune(quote)
for {
- rune := s.mustGetRune()
+ rune := s.mustReadRune()
s.buf.WriteRune(rune)
if rune == '\\' {
// In a legal backslash escape, no matter how long, only the character
// immediately after the escape can itself be a backslash or quote.
// Thus we only need to protect the first character after the backslash.
- rune := s.mustGetRune()
+ rune := s.mustReadRune()
s.buf.WriteRune(rune)
} else if rune == '"' {
break
return
}
if unicode.IsSpace(rune1) {
- s.UngetRune()
+ s.UnreadRune()
return
}
- rune2 := s.mustGetRune()
+ rune2 := s.mustReadRune()
return byte(s.hexDigit(rune1)<<4 | s.hexDigit(rune2)), true
}
return s.buf.String()
}
-const floatVerbs = "eEfFgGv"
+const floatVerbs = "beEfFgGv"
+
+const hugeWid = 1 << 30
// scanOne scans a single value, deriving the scanner from the type of the argument.
func (s *ss) scanOne(verb int, field interface{}) {
if v, ok := field.(Scanner); ok {
err = v.Scan(s, verb)
if err != nil {
+ if err == os.EOF {
+ err = io.ErrUnexpectedEOF
+ }
s.error(err)
}
return
}
- if !s.widPresent {
- s.maxWid = 1 << 30 // Huge
- }
- s.wid = 0
switch v := field.(type) {
case *bool:
*v = s.scanBool(verb)
}
// doScan does the real work for scanning without a format string.
-// At the moment, it handles only pointers to basic types.
func (s *ss) doScan(a []interface{}) (numProcessed int, err os.Error) {
defer errorHandler(&err)
for _, field := range a {
s.skipSpace(true)
continue
}
- inputc := s.mustGetRune()
+ inputc := s.mustReadRune()
if fmtc != inputc {
- s.UngetRune()
+ s.UnreadRune()
return -1
}
i += w
i++ // % is one byte
// do we have 20 (width)?
- s.maxWid, s.widPresent, i = parsenum(format, i, end)
+ var widPresent bool
+ s.maxWid, widPresent, i = parsenum(format, i, end)
+ if !widPresent {
+ s.maxWid = hugeWid
+ }
+ s.fieldLimit = s.limit
+ if f := s.count + s.maxWid; f < s.fieldLimit {
+ s.fieldLimit = f
+ }
c, w := utf8.DecodeRuneInString(format[i:])
i += w
s.scanOne(c, field)
numProcessed++
+ s.fieldLimit = s.limit
}
if numProcessed < len(a) {
s.errorString("too many operands")
import (
"bufio"
+ "bytes"
. "fmt"
"io"
"math"
type Xs string
func (x *Xs) Scan(state ScanState, verb int) os.Error {
- var tok string
- var c int
- var err os.Error
- wid, present := state.Width()
- if !present {
- tok, err = state.Token()
- } else {
- for i := 0; i < wid; i++ {
- c, err = state.GetRune()
- if err != nil {
- break
- }
- tok += string(c)
- }
- }
+ tok, err := state.Token()
if err != nil {
return err
}
var xVal Xs
+// IntString accepts an integer followed immediately by a string.
+// It tests the embedding of a scan within a scan.
+type IntString struct {
+ i int
+ s string
+}
+
+func (s *IntString) Scan(state ScanState, verb int) os.Error {
+ if _, err := Fscan(state, &s.i); err != nil {
+ return err
+ }
+
+ if _, err := Fscan(state, &s.s); err != nil {
+ return err
+ }
+ return nil
+}
+
+var intStringVal IntString
+
// myStringReader implements Read but not ReadRune, allowing us to test our readRune wrapper
// type that creates something that can read runes given only Read().
type myStringReader struct {
}
var scanTests = []ScanTest{
- // Numbers
+ // Basic types
{"T\n", &boolVal, true}, // boolean test vals toggle to be sure they are written
{"F\n", &boolVal, false}, // restored to zero value
{"21\n", &intVal, 21},
+ {"0\n", &intVal, 0},
+ {"000\n", &intVal, 0},
+ {"0x10\n", &intVal, 0x10},
+ {"-0x10\n", &intVal, -0x10},
+ {"0377\n", &intVal, 0377},
+ {"-0377\n", &intVal, -0377},
+ {"0\n", &uintVal, uint(0)},
+ {"000\n", &uintVal, uint(0)},
+ {"0x10\n", &uintVal, uint(0x10)},
+ {"0377\n", &uintVal, uint(0377)},
{"22\n", &int8Val, int8(22)},
{"23\n", &int16Val, int16(23)},
{"24\n", &int32Val, int32(24)},
{"2.3\n", &float64Val, 2.3},
{"2.3e1\n", &float32Val, float32(2.3e1)},
{"2.3e2\n", &float64Val, 2.3e2},
+ {"2.3p2\n", &float64Val, 2.3 * 4},
+ {"2.3p+2\n", &float64Val, 2.3 * 4},
+ {"2.3p+66\n", &float64Val, 2.3 * (1 << 32) * (1 << 32) * 4},
+ {"2.3p-66\n", &float64Val, 2.3 / ((1 << 32) * (1 << 32) * 4)},
{"2.35\n", &stringVal, "2.35"},
{"2345678\n", &bytesVal, []byte("2345678")},
{"(3.4e1-2i)\n", &complex128Val, 3.4e1 - 2i},
{"114\n", &renamedStringVal, renamedString("114")},
{"115\n", &renamedBytesVal, renamedBytes([]byte("115"))},
- // Custom scanner.
+ // Custom scanners.
{" vvv ", &xVal, Xs("vvv")},
+ {" 1234hello", &intStringVal, IntString{1234, "hello"}},
// Fixed bugs
{"2147483648\n", &int64Val, int64(2147483648)}, // was: integer overflow
{"%v", "TRUE\n", &boolVal, true},
{"%t", "false\n", &boolVal, false},
{"%v", "-71\n", &intVal, -71},
+ {"%v", "0377\n", &intVal, 0377},
+ {"%v", "0x44\n", &intVal, 0x44},
{"%d", "72\n", &intVal, 72},
{"%c", "a\n", &intVal, 'a'},
{"%c", "\u5072\n", &intVal, 0x5072},
var s, t string
var c complex128
var x, y Xs
+var z IntString
var multiTests = []ScanfMultiTest{
{"", "", nil, nil, ""},
{"%d%s", "123abc", args(&i, &s), args(123, "abc"), ""},
{"%c%c%c", "2\u50c2X", args(&i, &j, &k), args('2', '\u50c2', 'X'), ""},
- // Custom scanner.
+ // Custom scanners.
{"%2e%f", "eefffff", args(&x, &y), args(Xs("ee"), Xs("fffff")), ""},
+ {"%4v%s", "12abcd", args(&z, &s), args(IntString{12, "ab"}, "cd"), ""},
// Errors
{"%t", "23 18", args(&i), nil, "bad verb"},
}
n, err := scan(r, test.in)
if err != nil {
- t.Errorf("%s got error scanning %q: %s", name, test.text, err)
+ m := ""
+ if n > 0 {
+ m = Sprintf(" (%d fields ok)", n)
+ }
+ t.Errorf("%s got error scanning %q: %s%s", name, test.text, err, m)
continue
}
if n != 1 {
t.Errorf("expected αb; got %q", a)
}
}
+
+type TwoLines string
+
+// Attempt to read two lines into the object. Scanln should prevent this
+// because it stops at newline; Scan and Scanf should be fine.
+func (t *TwoLines) Scan(state ScanState, verb int) os.Error {
+ chars := make([]int, 0, 100)
+ for nlCount := 0; nlCount < 2; {
+ c, _, err := state.ReadRune()
+ if err != nil {
+ return err
+ }
+ chars = append(chars, c)
+ if c == '\n' {
+ nlCount++
+ }
+ }
+ *t = TwoLines(string(chars))
+ return nil
+}
+
+func TestMultiLine(t *testing.T) {
+ input := "abc\ndef\n"
+ // Sscan should work
+ var tscan TwoLines
+ n, err := Sscan(input, &tscan)
+ if n != 1 {
+ t.Errorf("Sscan: expected 1 item; got %d", n)
+ }
+ if err != nil {
+ t.Errorf("Sscan: expected no error; got %s", err)
+ }
+ if string(tscan) != input {
+ t.Errorf("Sscan: expected %q; got %q", input, tscan)
+ }
+ // Sscanf should work
+ var tscanf TwoLines
+ n, err = Sscanf(input, "%s", &tscanf)
+ if n != 1 {
+ t.Errorf("Sscanf: expected 1 item; got %d", n)
+ }
+ if err != nil {
+ t.Errorf("Sscanf: expected no error; got %s", err)
+ }
+ if string(tscanf) != input {
+ t.Errorf("Sscanf: expected %q; got %q", input, tscanf)
+ }
+ // Sscanln should not work
+ var tscanln TwoLines
+ n, err = Sscanln(input, &tscanln)
+ if n != 0 {
+ t.Errorf("Sscanln: expected 0 items; got %d: %q", n, tscanln)
+ }
+ if err == nil {
+ t.Error("Sscanln: expected error; got none")
+ } else if err != io.ErrUnexpectedEOF {
+ t.Errorf("Sscanln: expected io.ErrUnexpectedEOF (ha!); got %s", err)
+ }
+}
+
+// RecursiveInt accepts an string matching %d.%d.%d....
+// and parses it into a linked list.
+// It allows us to benchmark recursive descent style scanners.
+type RecursiveInt struct {
+ i int
+ next *RecursiveInt
+}
+
+func (r *RecursiveInt) Scan(state ScanState, verb int) (err os.Error) {
+ _, err = Fscan(state, &r.i)
+ if err != nil {
+ return
+ }
+ next := new(RecursiveInt)
+ _, err = Fscanf(state, ".%v", next)
+ if err != nil {
+ if err == os.ErrorString("input does not match format") || err == io.ErrUnexpectedEOF {
+ err = nil
+ }
+ return
+ }
+ r.next = next
+ return
+}
+
+// Perform the same scanning task as RecursiveInt.Scan
+// but without recurring through scanner, so we can compare
+// performance more directly.
+func scanInts(r *RecursiveInt, b *bytes.Buffer) (err os.Error) {
+ r.next = nil
+ _, err = Fscan(b, &r.i)
+ if err != nil {
+ return
+ }
+ var c int
+ c, _, err = b.ReadRune()
+ if err != nil {
+ if err == os.EOF {
+ err = nil
+ }
+ return
+ }
+ if c != '.' {
+ return
+ }
+ next := new(RecursiveInt)
+ err = scanInts(next, b)
+ if err == nil {
+ r.next = next
+ }
+ return
+}
+
+func makeInts(n int) []byte {
+ var buf bytes.Buffer
+ Fprintf(&buf, "1")
+ for i := 1; i < n; i++ {
+ Fprintf(&buf, ".%d", i+1)
+ }
+ return buf.Bytes()
+}
+
+func TestScanInts(t *testing.T) {
+ testScanInts(t, scanInts)
+ testScanInts(t, func(r *RecursiveInt, b *bytes.Buffer) (err os.Error) {
+ _, err = Fscan(b, r)
+ return
+ })
+}
+
+const intCount = 1000
+
+func testScanInts(t *testing.T, scan func(*RecursiveInt, *bytes.Buffer) os.Error) {
+ r := new(RecursiveInt)
+ ints := makeInts(intCount)
+ buf := bytes.NewBuffer(ints)
+ err := scan(r, buf)
+ if err != nil {
+ t.Error("unexpected error", err)
+ }
+ i := 1
+ for ; r != nil; r = r.next {
+ if r.i != i {
+ t.Fatal("bad scan: expected %d got %d", i, r.i)
+ }
+ i++
+ }
+ if i-1 != intCount {
+ t.Fatal("bad scan count: expected %d got %d", intCount, i-1)
+ }
+}
+
+func BenchmarkScanInts(b *testing.B) {
+ b.ResetTimer()
+ ints := makeInts(intCount)
+ var r RecursiveInt
+ for i := b.N - 1; i >= 0; i-- {
+ buf := bytes.NewBuffer(ints)
+ b.StartTimer()
+ scanInts(&r, buf)
+ b.StopTimer()
+ }
+}
+
+func BenchmarkScanRecursiveInt(b *testing.B) {
+ b.ResetTimer()
+ ints := makeInts(intCount)
+ var r RecursiveInt
+ for i := b.N - 1; i >= 0; i-- {
+ buf := bytes.NewBuffer(ints)
+ b.StartTimer()
+ Fscan(buf, &r)
+ b.StopTimer()
+ }
+}
X Expr // expression
}
+ // A SendStmt node represents a send statement.
+ SendStmt struct {
+ Chan Expr
+ Arrow token.Pos // position of "<-"
+ Value Expr
+ }
+
// An IncDecStmt node represents an increment or decrement statement.
IncDecStmt struct {
X Expr
IfStmt struct {
If token.Pos // position of "if" keyword
Init Stmt // initalization statement; or nil
- Cond Expr // condition; or nil
+ Cond Expr // condition
Body *BlockStmt
Else Stmt // else branch; or nil
}
// A CommClause node represents a case of a select statement.
CommClause struct {
- Case token.Pos // position of "case" or "default" keyword
- Tok token.Token // ASSIGN or DEFINE (valid only if Lhs != nil)
- Lhs, Rhs Expr // Rhs == nil means default case
- Colon token.Pos // position of ":"
- Body []Stmt // statement list; or nil
+ Case token.Pos // position of "case" or "default" keyword
+ Comm Stmt // send or receive statement; nil means default case
+ Colon token.Pos // position of ":"
+ Body []Stmt // statement list; or nil
}
// An SelectStmt node represents a select statement.
func (s *EmptyStmt) Pos() token.Pos { return s.Semicolon }
func (s *LabeledStmt) Pos() token.Pos { return s.Label.Pos() }
func (s *ExprStmt) Pos() token.Pos { return s.X.Pos() }
+func (s *SendStmt) Pos() token.Pos { return s.Chan.Pos() }
func (s *IncDecStmt) Pos() token.Pos { return s.X.Pos() }
func (s *AssignStmt) Pos() token.Pos { return s.Lhs[0].Pos() }
func (s *GoStmt) Pos() token.Pos { return s.Go }
}
func (s *LabeledStmt) End() token.Pos { return s.Stmt.End() }
func (s *ExprStmt) End() token.Pos { return s.X.End() }
+func (s *SendStmt) End() token.Pos { return s.Value.End() }
func (s *IncDecStmt) End() token.Pos {
return s.TokPos + 2 /* len("++") */
}
func (s *EmptyStmt) stmtNode() {}
func (s *LabeledStmt) stmtNode() {}
func (s *ExprStmt) stmtNode() {}
+func (s *SendStmt) stmtNode() {}
func (s *IncDecStmt) stmtNode() {}
func (s *AssignStmt) stmtNode() {}
func (s *GoStmt) stmtNode() {}
case *ExprStmt:
Walk(v, n.X)
+ case *SendStmt:
+ Walk(v, n.Chan)
+ Walk(v, n.Value)
+
case *IncDecStmt:
Walk(v, n.X)
if n.Init != nil {
Walk(v, n.Init)
}
- if n.Cond != nil {
- Walk(v, n.Cond)
- }
+ Walk(v, n.Cond)
Walk(v, n.Body)
if n.Else != nil {
Walk(v, n.Else)
Walk(v, n.Body)
case *CommClause:
- if n.Lhs != nil {
- Walk(v, n.Lhs)
- }
- if n.Rhs != nil {
- Walk(v, n.Rhs)
+ if n.Comm != nil {
+ Walk(v, n.Comm)
}
walkStmtList(v, n.Body)
x := p.parseExprList()
switch p.tok {
- case token.COLON:
- // labeled statement
- colon := p.pos
- p.next()
- if labelOk && len(x) == 1 {
- if label, isIdent := x[0].(*ast.Ident); isIdent {
- return &ast.LabeledStmt{label, colon, p.parseStmt()}
- }
- }
- p.error(x[0].Pos(), "illegal label declaration")
- return &ast.BadStmt{x[0].Pos(), colon + 1}
-
case
token.DEFINE, token.ASSIGN, token.ADD_ASSIGN,
token.SUB_ASSIGN, token.MUL_ASSIGN, token.QUO_ASSIGN,
}
if len(x) > 1 {
- p.error(x[0].Pos(), "only one expression allowed")
+ p.errorExpected(x[0].Pos(), "1 expression")
// continue with first expression
}
- if p.tok == token.INC || p.tok == token.DEC {
+ switch p.tok {
+ case token.COLON:
+ // labeled statement
+ colon := p.pos
+ p.next()
+ if label, isIdent := x[0].(*ast.Ident); labelOk && isIdent {
+ return &ast.LabeledStmt{label, colon, p.parseStmt()}
+ }
+ p.error(x[0].Pos(), "illegal label declaration")
+ return &ast.BadStmt{x[0].Pos(), colon + 1}
+
+ case token.ARROW:
+ // send statement
+ arrow := p.pos
+ p.next() // consume "<-"
+ y := p.parseExpr()
+ return &ast.SendStmt{x[0], arrow, y}
+
+ case token.INC, token.DEC:
// increment or decrement
s := &ast.IncDecStmt{x[0], p.pos, p.tok}
p.next() // consume "++" or "--"
}
-func (p *parser) parseControlClause(isForStmt bool) (s1, s2, s3 ast.Stmt) {
- if p.tok != token.LBRACE {
+func (p *parser) parseIfStmt() *ast.IfStmt {
+ if p.trace {
+ defer un(trace(p, "IfStmt"))
+ }
+
+ pos := p.expect(token.IF)
+
+ var s ast.Stmt
+ var x ast.Expr
+ {
prevLev := p.exprLev
p.exprLev = -1
-
- if p.tok != token.SEMICOLON {
- s1 = p.parseSimpleStmt(false)
- }
if p.tok == token.SEMICOLON {
p.next()
- if p.tok != token.LBRACE && p.tok != token.SEMICOLON {
- s2 = p.parseSimpleStmt(false)
- }
- if isForStmt {
- // for statements have a 3rd section
- p.expectSemi()
- if p.tok != token.LBRACE {
- s3 = p.parseSimpleStmt(false)
- }
- }
+ x = p.parseExpr()
} else {
- s1, s2 = nil, s1
+ s = p.parseSimpleStmt(false)
+ if p.tok == token.SEMICOLON {
+ p.next()
+ x = p.parseExpr()
+ } else {
+ x = p.makeExpr(s)
+ s = nil
+ }
}
-
p.exprLev = prevLev
}
- return s1, s2, s3
-}
-
-
-func (p *parser) parseIfStmt() *ast.IfStmt {
- if p.trace {
- defer un(trace(p, "IfStmt"))
- }
-
- pos := p.expect(token.IF)
- s1, s2, _ := p.parseControlClause(false)
body := p.parseBlockStmt()
var else_ ast.Stmt
if p.tok == token.ELSE {
p.expectSemi()
}
- return &ast.IfStmt{pos, s1, p.makeExpr(s2), body, else_}
+ return &ast.IfStmt{pos, s, x, body, else_}
}
}
pos := p.expect(token.SWITCH)
- s1, s2, _ := p.parseControlClause(false)
+
+ var s1, s2 ast.Stmt
+ if p.tok != token.LBRACE {
+ prevLev := p.exprLev
+ p.exprLev = -1
+ if p.tok != token.SEMICOLON {
+ s2 = p.parseSimpleStmt(false)
+ }
+ if p.tok == token.SEMICOLON {
+ p.next()
+ s1 = s2
+ s2 = nil
+ if p.tok != token.LBRACE {
+ s2 = p.parseSimpleStmt(false)
+ }
+ }
+ p.exprLev = prevLev
+ }
if isExprSwitch(s2) {
lbrace := p.expect(token.LBRACE)
// CommCase
pos := p.pos
- var tok token.Token
- var lhs, rhs ast.Expr
+ var comm ast.Stmt
if p.tok == token.CASE {
p.next()
+ lhs := p.parseExprList()
if p.tok == token.ARROW {
- // RecvExpr without assignment
- rhs = p.parseExpr()
+ // SendStmt
+ if len(lhs) > 1 {
+ p.errorExpected(lhs[0].Pos(), "1 expression")
+ // continue with first expression
+ }
+ arrow := p.pos
+ p.next()
+ rhs := p.parseExpr()
+ comm = &ast.SendStmt{lhs[0], arrow, rhs}
} else {
- // SendExpr or RecvExpr
- rhs = p.parseExpr()
+ // RecvStmt
+ pos := p.pos
+ tok := p.tok
+ var rhs ast.Expr
if p.tok == token.ASSIGN || p.tok == token.DEFINE {
- // RecvExpr with assignment
- tok = p.tok
+ // RecvStmt with assignment
+ if len(lhs) > 2 {
+ p.errorExpected(lhs[0].Pos(), "1 or 2 expressions")
+ // continue with first two expressions
+ lhs = lhs[0:2]
+ }
p.next()
- lhs = rhs
- if p.tok == token.ARROW {
- rhs = p.parseExpr()
- } else {
- p.expect(token.ARROW) // use expect() error handling
+ rhs = p.parseExpr()
+ } else {
+ // rhs must be single receive operation
+ if len(lhs) > 1 {
+ p.errorExpected(lhs[0].Pos(), "1 expression")
+ // continue with first expression
}
+ rhs = lhs[0]
+ lhs = nil // there is no lhs
+ }
+ if x, isUnary := rhs.(*ast.UnaryExpr); !isUnary || x.Op != token.ARROW {
+ p.errorExpected(rhs.Pos(), "send or receive operation")
+ rhs = &ast.BadExpr{rhs.Pos(), rhs.End()}
+ }
+ if lhs != nil {
+ comm = &ast.AssignStmt{lhs, pos, tok, []ast.Expr{rhs}}
+ } else {
+ comm = &ast.ExprStmt{rhs}
}
- // else SendExpr
}
} else {
p.expect(token.DEFAULT)
colon := p.expect(token.COLON)
body := p.parseStmtList()
- return &ast.CommClause{pos, tok, lhs, rhs, colon, body}
+ return &ast.CommClause{pos, comm, colon, body}
}
}
pos := p.expect(token.FOR)
- s1, s2, s3 := p.parseControlClause(true)
+
+ var s1, s2, s3 ast.Stmt
+ if p.tok != token.LBRACE {
+ prevLev := p.exprLev
+ p.exprLev = -1
+ if p.tok != token.SEMICOLON {
+ s2 = p.parseSimpleStmt(false)
+ }
+ if p.tok == token.SEMICOLON {
+ p.next()
+ s1 = s2
+ s2 = nil
+ if p.tok != token.SEMICOLON {
+ s2 = p.parseSimpleStmt(false)
+ }
+ p.expectSemi()
+ if p.tok != token.LBRACE {
+ s3 = p.parseSimpleStmt(false)
+ }
+ }
+ p.exprLev = prevLev
+ }
+
body := p.parseBlockStmt()
p.expectSemi()
}
// check rhs
if len(as.Rhs) != 1 {
- p.errorExpected(as.Rhs[0].Pos(), "1 expressions")
+ p.errorExpected(as.Rhs[0].Pos(), "1 expression")
return &ast.BadStmt{pos, body.End()}
}
if rhs, isUnary := as.Rhs[0].(*ast.UnaryExpr); isUnary && rhs.Op == token.RANGE {
3.14,
[]byte(nil),
"foo!",
+ `package p; func f() { if /* should have condition */ {} };`,
+ `package p; func f() { if ; /* should have condition */ {} };`,
+ `package p; func f() { if f(); /* should have condition */ {} };`,
}
var validPrograms = []interface{}{
- "package main\n",
- `package main;`,
- `package main; import "fmt"; func main() { fmt.Println("Hello, World!") };`,
- `package main; func main() { if f(T{}) {} };`,
- `package main; func main() { _ = (<-chan int)(x) };`,
- `package main; func main() { _ = (<-chan <-chan int)(x) };`,
- `package main; func f(func() func() func());`,
- `package main; func f(...T);`,
- `package main; func f(float, ...int);`,
- `package main; func f(x int, a ...int) { f(0, a...); f(1, a...,) };`,
- `package main; type T []int; var a []bool; func f() { if a[T{42}[0]] {} };`,
- `package main; type T []int; func g(int) bool { return true }; func f() { if g(T{42}[0]) {} };`,
- `package main; type T []int; func f() { for _ = range []int{T{42}[0]} {} };`,
- `package main; var a = T{{1, 2}, {3, 4}}`,
+ "package p\n",
+ `package p;`,
+ `package p; import "fmt"; func f() { fmt.Println("Hello, World!") };`,
+ `package p; func f() { if f(T{}) {} };`,
+ `package p; func f() { _ = (<-chan int)(x) };`,
+ `package p; func f() { _ = (<-chan <-chan int)(x) };`,
+ `package p; func f(func() func() func());`,
+ `package p; func f(...T);`,
+ `package p; func f(float, ...int);`,
+ `package p; func f(x int, a ...int) { f(0, a...); f(1, a...,) };`,
+ `package p; type T []int; var a []bool; func f() { if a[T{42}[0]] {} };`,
+ `package p; type T []int; func g(int) bool { return true }; func f() { if g(T{42}[0]) {} };`,
+ `package p; type T []int; func f() { for _ = range []int{T{42}[0]} {} };`,
+ `package p; var a = T{{1, 2}, {3, 4}}`,
+ `package p; func f() { select { case <- c: case c <- d: case c <- <- d: case <-c <- d: } };`,
+ `package p; func f() { if ; true {} };`,
+ `package p; func f() { switch ; {} };`,
}
)
-func walkBinary(e *ast.BinaryExpr) (has5, has6 bool, maxProblem int) {
+func walkBinary(e *ast.BinaryExpr) (has4, has5 bool, maxProblem int) {
switch e.Op.Precedence() {
+ case 4:
+ has4 = true
case 5:
has5 = true
- case 6:
- has6 = true
}
switch l := e.X.(type) {
// pretend this is an *ast.ParenExpr and do nothing.
break
}
- h5, h6, mp := walkBinary(l)
+ h4, h5, mp := walkBinary(l)
+ has4 = has4 || h4
has5 = has5 || h5
- has6 = has6 || h6
if maxProblem < mp {
maxProblem = mp
}
// pretend this is an *ast.ParenExpr and do nothing.
break
}
- h5, h6, mp := walkBinary(r)
+ h4, h5, mp := walkBinary(r)
+ has4 = has4 || h4
has5 = has5 || h5
- has6 = has6 || h6
if maxProblem < mp {
maxProblem = mp
}
case *ast.StarExpr:
if e.Op.String() == "/" {
- maxProblem = 6
+ maxProblem = 5
}
case *ast.UnaryExpr:
switch e.Op.String() + r.Op.String() {
case "/*", "&&", "&^":
- maxProblem = 6
+ maxProblem = 5
case "++", "--":
- if maxProblem < 5 {
- maxProblem = 5
+ if maxProblem < 4 {
+ maxProblem = 4
}
}
}
func cutoff(e *ast.BinaryExpr, depth int) int {
- has5, has6, maxProblem := walkBinary(e)
+ has4, has5, maxProblem := walkBinary(e)
if maxProblem > 0 {
return maxProblem + 1
}
- if has5 && has6 {
+ if has4 && has5 {
if depth == 1 {
- return 6
+ return 5
}
- return 5
+ return 4
}
if depth == 1 {
- return 7
+ return 6
}
- return 5
+ return 4
}
// (Algorithm suggestion by Russ Cox.)
//
// The precedences are:
-// 6 * / % << >> & &^
-// 5 + - | ^
-// 4 == != < <= > >=
-// 3 <-
+// 5 * / % << >> & &^
+// 4 + - | ^
+// 3 == != < <= > >=
// 2 &&
// 1 ||
//
-// The only decision is whether there will be spaces around levels 5 and 6.
-// There are never spaces at level 7 (unary), and always spaces at levels 4 and below.
+// The only decision is whether there will be spaces around levels 4 and 5.
+// There are never spaces at level 6 (unary), and always spaces at levels 3 and below.
//
// To choose the cutoff, look at the whole expression but excluding primary
// expressions (function calls, parenthesized exprs), and apply these rules:
// 1) If there is a binary operator with a right side unary operand
// that would clash without a space, the cutoff must be (in order):
//
-// /* 7
-// && 7
-// &^ 7
-// ++ 6
-// -- 6
+// /* 6
+// && 6
+// &^ 6
+// ++ 5
+// -- 5
//
// (Comparison operators always have spaces around them.)
//
-// 2) If there is a mix of level 6 and level 5 operators, then the cutoff
-// is 6 (use spaces to distinguish precedence) in Normal mode
-// and 5 (never use spaces) in Compact mode.
+// 2) If there is a mix of level 5 and level 4 operators, then the cutoff
+// is 5 (use spaces to distinguish precedence) in Normal mode
+// and 4 (never use spaces) in Compact mode.
//
-// 3) If there are no level 5 operators or no level 6 operators, then the
-// cutoff is 7 (always use spaces) in Normal mode
-// and 5 (never use spaces) in Compact mode.
+// 3) If there are no level 4 operators or no level 5 operators, then the
+// cutoff is 6 (always use spaces) in Normal mode
+// and 4 (never use spaces) in Compact mode.
//
// Sets multiLine to true if the binary expression spans multiple lines.
func (p *printer) binaryExpr(x *ast.BinaryExpr, prec1, cutoff, depth int, multiLine *bool) {
const depth = 1
p.expr0(s.X, depth, multiLine)
+ case *ast.SendStmt:
+ const depth = 1
+ p.expr0(s.Chan, depth, multiLine)
+ p.print(blank, s.Arrow, token.ARROW, blank)
+ p.expr0(s.Value, depth, multiLine)
+
case *ast.IncDecStmt:
const depth = 1
p.expr0(s.X, depth+1, multiLine)
*multiLine = true
case *ast.CommClause:
- if s.Rhs != nil {
+ if s.Comm != nil {
p.print(token.CASE, blank)
- if s.Lhs != nil {
- p.expr(s.Lhs, multiLine)
- p.print(blank, s.Tok, blank)
- }
- p.expr(s.Rhs, multiLine)
+ p.stmt(s.Comm, false, ignoreMultiLine)
} else {
p.print(token.DEFAULT)
}
)
+const (
+ esc2 = '\xfe' // an escape byte that cannot occur in regular UTF-8
+ _ = 1 / (esc2 - tabwriter.Escape) // cause compiler error if esc2 == tabwriter.Escape
+)
+
+
var (
esc = []byte{tabwriter.Escape}
htab = []byte{'\t'}
htabs = []byte("\t\t\t\t\t\t\t\t")
newlines = []byte("\n\n\n\n\n\n\n\n") // more than the max determined by nlines
formfeeds = []byte("\f\f\f\f\f\f\f\f") // more than the max determined by nlines
-
- esc_quot = []byte(""") // shorter than """
- esc_apos = []byte("'") // shorter than "'"
- esc_amp = []byte("&")
- esc_lt = []byte("<")
- esc_gt = []byte(">")
)
// write0 does not indent after newlines, and does not HTML-escape or update p.pos.
//
func (p *printer) write0(data []byte) {
- n, err := p.output.Write(data)
- p.written += n
- if err != nil {
- p.errors <- err
- runtime.Goexit()
+ if len(data) > 0 {
+ n, err := p.output.Write(data)
+ p.written += n
+ if err != nil {
+ p.errors <- err
+ runtime.Goexit()
+ }
}
}
// write interprets data and writes it to p.output. It inserts indentation
-// after a line break unless in a tabwriter escape sequence, and it HTML-
-// escapes characters if GenHTML is set. It updates p.pos as a side-effect.
+// after a line break unless in a tabwriter escape sequence.
+// It updates p.pos as a side-effect.
//
func (p *printer) write(data []byte) {
i0 := 0
// next segment start
i0 = i + 1
- case '"', '\'', '&', '<', '>':
- if p.Mode&GenHTML != 0 {
- // write segment ending in b
- p.write0(data[i0:i])
-
- // write HTML-escaped b
- var esc []byte
- switch b {
- case '"':
- esc = esc_quot
- case '\'':
- esc = esc_apos
- case '&':
- esc = esc_amp
- case '<':
- esc = esc_lt
- case '>':
- esc = esc_gt
- }
- p.write0(esc)
-
- // update p.pos
- d := i + 1 - i0
- p.pos.Offset += d
- p.pos.Column += d
-
- // next segment start
- i0 = i + 1
- }
-
case tabwriter.Escape:
p.mode ^= inLiteral
}
-func (p *printer) writeTaggedItem(data []byte, tag HTMLTag) {
- // write start tag, if any
- // (no html-escaping and no p.pos update for tags - use write0)
- if tag.Start != "" {
- p.write0([]byte(tag.Start))
- }
- p.write(data)
- // write end tag, if any
- if tag.End != "" {
- p.write0([]byte(tag.End))
- }
-}
-
-
// writeItem writes data at position pos. data is the text corresponding to
// a single lexical token, but may also be comment text. pos is the actual
// (or at least very accurately estimated) position of the data in the original
-// source text. If tags are present and GenHTML is set, the tags are written
-// before and after the data. writeItem updates p.last to the position
-// immediately following the data.
+// source text. writeItem updates p.last to the position immediately following
+// the data.
//
-func (p *printer) writeItem(pos token.Position, data []byte, tag HTMLTag) {
- fileChanged := false
+func (p *printer) writeItem(pos token.Position, data []byte) {
if pos.IsValid() {
// continue with previous position if we don't have a valid pos
if p.last.IsValid() && p.last.Filename != pos.Filename {
p.indent = 0
p.mode = 0
p.buffer = p.buffer[0:0]
- fileChanged = true
}
p.pos = pos
}
_, filename := path.Split(pos.Filename)
p.write0([]byte(fmt.Sprintf("[%s:%d:%d]", filename, pos.Line, pos.Column)))
}
- if p.Mode&GenHTML != 0 {
- // write line tag if on a new line
- // TODO(gri): should write line tags on each line at the start
- // will be more useful (e.g. to show line numbers)
- if p.Styler != nil && (pos.Line != p.lastTaggedLine || fileChanged) {
- p.writeTaggedItem(p.Styler.LineTag(pos.Line))
- p.lastTaggedLine = pos.Line
- }
- p.writeTaggedItem(data, tag)
- } else {
- p.write(data)
- }
+ p.write(data)
p.last = p.pos
}
// If there is any pending whitespace, it consumes as much of
// it as is likely to help position the comment nicely.
// pos is the comment position, next the position of the item
-// after all pending comments, isFirst indicates if this is the
-// first comment in a group of comments, and isKeyword indicates
-// if the next item is a keyword.
+// after all pending comments, prev is the previous comment in
+// a group of comments (or nil), and isKeyword indicates if the
+// next item is a keyword.
//
-func (p *printer) writeCommentPrefix(pos, next token.Position, isFirst, isKeyword bool) {
- if !p.last.IsValid() {
- // there was no preceeding item and the comment is the
- // first item to be printed - don't write any whitespace
+func (p *printer) writeCommentPrefix(pos, next token.Position, prev *ast.Comment, isKeyword bool) {
+ if p.written == 0 {
+ // the comment is the first item to be printed - don't write any whitespace
return
}
return
}
- if pos.IsValid() && pos.Line == p.last.Line {
+ if pos.Line == p.last.Line && (prev == nil || prev.Text[1] != '/') {
// comment on the same line as last item:
// separate with at least one separator
hasSep := false
- if isFirst {
+ if prev == nil {
+ // first comment of a comment group
j := 0
for i, ch := range p.buffer {
switch ch {
} else {
// comment on a different line:
// separate with at least one line break
- if isFirst {
+ if prev == nil {
+ // first comment of a comment group
j := 0
for i, ch := range p.buffer {
switch ch {
}
// use formfeeds to break columns before a comment;
// this is analogous to using formfeeds to separate
- // individual lines of /*-style comments
- // (if !pos.IsValid(), pos.Line == 0, and this will
- // print no newlines)
- p.writeNewlines(pos.Line-p.last.Line, true)
+ // individual lines of /*-style comments - but make
+ // sure there is at least one line break if the previous
+ // comment was a line comment
+ n := pos.Line - p.last.Line // if !pos.IsValid(), pos.Line == 0, and n will be 0
+ if n <= 0 && prev != nil && prev.Text[1] == '/' {
+ n = 1
+ }
+ p.writeNewlines(n, true)
}
}
func (p *printer) writeCommentLine(comment *ast.Comment, pos token.Position, line []byte) {
// line must pass through unchanged, bracket it with tabwriter.Escape
line = bytes.Join([][]byte{esc, line, esc}, nil)
-
- // apply styler, if any
- var tag HTMLTag
- if p.Styler != nil {
- line, tag = p.Styler.Comment(comment, line)
- }
-
- p.writeItem(pos, line, tag)
+ p.writeItem(pos, line)
}
-// TODO(gri): Similar (but not quite identical) functionality for
-// comment processing can be found in go/doc/comment.go.
-// Perhaps this can be factored eventually.
-
// Split comment text into lines
func split(text []byte) [][]byte {
// count lines (comment text never ends in a newline)
var last *ast.Comment
for ; p.commentBefore(next); p.cindex++ {
for _, c := range p.comments[p.cindex].List {
- p.writeCommentPrefix(p.fset.Position(c.Pos()), next, last == nil, tok.IsKeyword())
+ p.writeCommentPrefix(p.fset.Position(c.Pos()), next, last, tok.IsKeyword())
p.writeComment(c)
last = c
}
for _, f := range args {
next := p.pos // estimated position of next item
var data []byte
- var tag HTMLTag
var tok token.Token
switch x := f.(type) {
p.buffer = p.buffer[0 : i+1]
p.buffer[i] = x
case *ast.Ident:
- if p.Styler != nil {
- data, tag = p.Styler.Ident(x)
- } else {
- data = []byte(x.Name)
- }
+ data = []byte(x.Name)
tok = token.IDENT
case *ast.BasicLit:
- if p.Styler != nil {
- data, tag = p.Styler.BasicLit(x)
- } else {
- data = x.Value
- }
// escape all literals so they pass through unchanged
// (note that valid Go programs cannot contain
// tabwriter.Escape bytes since they do not appear in
// legal UTF-8 sequences)
- escData := make([]byte, 0, len(data)+2)
- escData = append(escData, tabwriter.Escape)
- escData = append(escData, data...)
- escData = append(escData, tabwriter.Escape)
- data = escData
+ data = make([]byte, 0, len(x.Value)+2)
+ data = append(data, tabwriter.Escape)
+ data = append(data, x.Value...)
+ data = append(data, tabwriter.Escape)
tok = x.Kind
+ // If we have a raw string that spans multiple lines and
+ // the opening quote (`) is on a line preceded only by
+ // indentation, we don't want to write that indentation
+ // because the following lines of the raw string are not
+ // indented. It's easiest to correct the output at the end
+ // via the trimmer (because of the complex handling of
+ // white space).
+ // Mark multi-line raw strings by replacing the opening
+ // quote with esc2 and have the trimmer take care of fixing
+ // it up. (Do this _after_ making a copy of data!)
+ if data[1] == '`' && bytes.IndexByte(data, '\n') > 0 {
+ data[1] = esc2
+ }
case token.Token:
s := x.String()
if mayCombine(p.lastTok, s[0]) {
p.buffer = p.buffer[0:1]
p.buffer[0] = ' '
}
- if p.Styler != nil {
- data, tag = p.Styler.Token(x)
- } else {
- data = []byte(s)
- }
+ data = []byte(s)
tok = x
case token.Pos:
if x.IsValid() {
// before
p.writeNewlines(next.Line-p.pos.Line, droppedFF)
- p.writeItem(next, data, tag)
+ p.writeItem(next, data)
}
}
}
// through unchanged.
//
type trimmer struct {
- output io.Writer
- space bytes.Buffer
- state int
+ output io.Writer
+ state int
+ space bytes.Buffer
+ hasText bool
}
// trimmer is implemented as a state machine.
// It can be in one of the following states:
const (
- inSpace = iota
- inEscape
- inText
+ inSpace = iota // inside space
+ atEscape // inside space and the last char was an opening tabwriter.Escape
+ inEscape // inside text bracketed by tabwriter.Escapes
+ inText // inside text
)
+var backquote = []byte{'`'}
+
+
// Design note: It is tempting to eliminate extra blanks occurring in
// whitespace in this function as it could simplify some
// of the blanks logic in the node printing functions.
// the tabwriter.
func (p *trimmer) Write(data []byte) (n int, err os.Error) {
- m := 0 // if p.state != inSpace, data[m:n] is unwritten
+ // invariants:
+ // p.state == inSpace, atEscape:
+ // p.space is unwritten
+ // p.hasText indicates if there is any text on this line
+ // p.state == inEscape, inText:
+ // data[m:n] is unwritten
+ m := 0
var b byte
for n, b = range data {
if b == '\v' {
switch b {
case '\t', ' ':
p.space.WriteByte(b) // WriteByte returns no errors
- case '\f', '\n':
+ case '\n', '\f':
p.space.Reset() // discard trailing space
_, err = p.output.Write(newlines[0:1]) // write newline
+ p.hasText = false
case tabwriter.Escape:
- _, err = p.output.Write(p.space.Bytes())
- p.space.Reset()
- p.state = inEscape
- m = n + 1 // drop tabwriter.Escape
+ p.state = atEscape
default:
_, err = p.output.Write(p.space.Bytes())
- p.space.Reset()
p.state = inText
m = n
}
+ case atEscape:
+ // discard indentation if we have a multi-line raw string
+ // (see printer.print for details)
+ if b != esc2 || p.hasText {
+ _, err = p.output.Write(p.space.Bytes())
+ }
+ p.state = inEscape
+ m = n
+ if b == esc2 {
+ _, err = p.output.Write(backquote) // convert back
+ m++
+ }
case inEscape:
if b == tabwriter.Escape {
_, err = p.output.Write(data[m:n])
p.state = inSpace
+ p.space.Reset()
+ p.hasText = true
}
case inText:
switch b {
case '\t', ' ':
_, err = p.output.Write(data[m:n])
p.state = inSpace
+ p.space.Reset()
p.space.WriteByte(b) // WriteByte returns no errors
- case '\f':
- data[n] = '\n' // convert to newline
+ p.hasText = true
+ case '\n', '\f':
+ _, err = p.output.Write(data[m:n])
+ p.state = inSpace
+ p.space.Reset()
+ _, err = p.output.Write(newlines[0:1]) // write newline
+ p.hasText = false
case tabwriter.Escape:
_, err = p.output.Write(data[m:n])
- p.state = inEscape
- m = n + 1 // drop tabwriter.Escape
+ p.state = atEscape
+ p.space.Reset()
+ p.hasText = true
}
}
if err != nil {
}
n = len(data)
- if p.state != inSpace {
+ switch p.state {
+ case inEscape, inText:
_, err = p.output.Write(data[m:n])
p.state = inSpace
+ p.space.Reset()
+ p.hasText = true
}
return
// General printing is controlled with these Config.Mode flags.
const (
- GenHTML uint = 1 << iota // generate HTML
- RawFormat // do not use a tabwriter; if set, UseSpaces is ignored
+ RawFormat uint = 1 << iota // do not use a tabwriter; if set, UseSpaces is ignored
TabIndent // use tabs for indentation independent of UseSpaces
UseSpaces // use spaces instead of tabs for alignment
)
-// An HTMLTag specifies a start and end tag.
-type HTMLTag struct {
- Start, End string // empty if tags are absent
-}
-
-
-// A Styler specifies formatting of line tags and elementary Go words.
-// A format consists of text and a (possibly empty) surrounding HTML tag.
-//
-type Styler interface {
- LineTag(line int) ([]byte, HTMLTag)
- Comment(c *ast.Comment, line []byte) ([]byte, HTMLTag)
- BasicLit(x *ast.BasicLit) ([]byte, HTMLTag)
- Ident(id *ast.Ident) ([]byte, HTMLTag)
- Token(tok token.Token) ([]byte, HTMLTag)
-}
-
-
// A Config node controls the output of Fprint.
type Config struct {
- Mode uint // default: 0
- Tabwidth int // default: 8
- Styler Styler // default: nil
+ Mode uint // default: 0
+ Tabwidth int // default: 8
}
}
twmode := tabwriter.DiscardEmptyColumns
- if cfg.Mode&GenHTML != 0 {
- twmode |= tabwriter.FilterHTML
- }
if cfg.Mode&TabIndent != 0 {
minwidth = 0
twmode |= tabwriter.TabIndent
}
-func Test(t *testing.T) {
+func TestFiles(t *testing.T) {
for _, e := range data {
source := path.Join(dataDir, e.source)
golden := path.Join(dataDir, e.golden)
//check(t, golden, golden, e.mode);
}
}
+
+
+// TestLineComments, using a simple test case, checks that consequtive line
+// comments are properly terminated with a newline even if the AST position
+// information is incorrect.
+//
+func TestLineComments(t *testing.T) {
+ const src = `// comment 1
+ // comment 2
+ // comment 3
+ package main
+ `
+
+ fset := token.NewFileSet()
+ ast1, err1 := parser.ParseFile(fset, "", src, parser.ParseComments)
+ if err1 != nil {
+ panic(err1)
+ }
+
+ var buf bytes.Buffer
+ fset = token.NewFileSet() // use the wrong file set
+ Fprint(&buf, fset, ast1)
+
+ nlines := 0
+ for _, ch := range buf.Bytes() {
+ if ch == '\n' {
+ nlines++
+ }
+ }
+
+ const expected = 3
+ if nlines < expected {
+ t.Errorf("got %d, expected %d\n", nlines, expected)
+ }
+}
}
+func _() {
+ // smart handling of indentation for multi-line raw strings
+ var _ = ``
+ var _ = `foo`
+ var _ = `foo
+bar`
+
+ var _ = ``
+ var _ = `foo`
+ var _ =
+ // the next line should not be indented
+`foo
+bar`
+
+ var _ = // comment
+ ``
+ var _ = // comment
+ `foo`
+ var _ = // comment
+ // the next line should not be indented
+`foo
+bar`
+
+ var _ = /* comment */ ``
+ var _ = /* comment */ `foo`
+ var _ = /* comment */ `foo
+bar`
+
+ var _ = /* comment */
+ ``
+ var _ = /* comment */
+ `foo`
+ var _ = /* comment */
+ // the next line should not be indented
+`foo
+bar`
+
+ var board = []int(
+`...........
+...........
+....●●●....
+....●●●....
+..●●●●●●●..
+..●●●○●●●..
+..●●●●●●●..
+....●●●....
+....●●●....
+...........
+...........
+`)
+
+ var state = S{
+ "foo",
+ // the next line should not be indented
+`...........
+...........
+....●●●....
+....●●●....
+..●●●●●●●..
+..●●●○●●●..
+..●●●●●●●..
+....●●●....
+....●●●....
+...........
+...........
+`,
+ "bar",
+ }
+}
+
+
func _() {
// one-line function literals (body is on a single line)
_ = func() {}
}
+func _() {
+ // smart handling of indentation for multi-line raw strings
+ var _ = ``
+ var _ = `foo`
+ var _ = `foo
+bar`
+
+
+var _ =
+ ``
+var _ =
+ `foo`
+var _ =
+ // the next line should not be indented
+ `foo
+bar`
+
+
+ var _ = // comment
+ ``
+ var _ = // comment
+ `foo`
+ var _ = // comment
+ // the next line should not be indented
+ `foo
+bar`
+
+
+var _ = /* comment */ ``
+var _ = /* comment */ `foo`
+var _ = /* comment */ `foo
+bar`
+
+
+ var _ = /* comment */
+ ``
+ var _ = /* comment */
+ `foo`
+ var _ = /* comment */
+ // the next line should not be indented
+ `foo
+bar`
+
+
+var board = []int(
+ `...........
+...........
+....●●●....
+....●●●....
+..●●●●●●●..
+..●●●○●●●..
+..●●●●●●●..
+....●●●....
+....●●●....
+...........
+...........
+`)
+
+
+ var state = S{
+ "foo",
+ // the next line should not be indented
+ `...........
+...........
+....●●●....
+....●●●....
+..●●●●●●●..
+..●●●○●●●..
+..●●●●●●●..
+....●●●....
+....●●●....
+...........
+...........
+`,
+ "bar",
+ }
+}
+
+
func _() {
// one-line function literals (body is on a single line)
_ = func() {}
_ = `foo
bar`
_ = `three spaces before the end of the line starting here:
-they must not be removed`
+they must not be removed`\f}
+
+
+func _() {
+ // smart handling of indentation for multi-line raw strings
+ var _ = ``
+ var _ = `foo`
+ var _ = `foo
+bar`
+
+ var _ = ``
+ var _ = `foo`
+ var _ =
+ // the next line should not be indented
+`foo
+bar`
+
+ var _ = // comment
+ ``
+ var _ = // comment
+ `foo`
+ var _ = // comment
+ // the next line should not be indented
+`foo
+bar`
+
+ var _ = /* comment */ ``
+ var _ = /* comment */ `foo`
+ var _ = /* comment */ `foo
+bar`
+
+ var _ = /* comment */
+ ``
+ var _ = /* comment */
+ `foo`
+ var _ = /* comment */
+ // the next line should not be indented
+`foo
+bar`
+
+ var board = []int(
+`...........
+...........
+....●●●....
+....●●●....
+..●●●●●●●..
+..●●●○●●●..
+..●●●●●●●..
+....●●●....
+....●●●....
+...........
+...........
+`)
+
+ var state = S{
+ "foo",
+ // the next line should not be indented
+`...........
+...........
+....●●●....
+....●●●....
+..●●●●●●●..
+..●●●○●●●..
+..●●●●●●●..
+....●●●....
+....●●●....
+...........
+...........
+`,
+ "bar",
+ }
}
// Formatting of if-statement headers.
func _() {
- if {
+ if true {
}
- if {
+ if true {
} // no semicolon printed
if expr {
}
} // no parens printed
if expr {
} // no semicolon and parens printed
- if x := expr; {
+ if x := expr; true {
use(x)
}
if x := expr; expr {
func _() {
- if {
+ if true {
_ = 0
}
_ = 0 // the indentation here should not be affected by the long label name
AnOverlongLabel:
_ = 0
- if {
+ if true {
_ = 0
}
_ = 0
// Formatting of if-statement headers.
func _() {
- if {}
- if;{} // no semicolon printed
+ if true {}
+ if; true {} // no semicolon printed
if expr{}
if;expr{} // no semicolon printed
if (expr){} // no parens printed
if;((expr)){} // no semicolon and parens printed
- if x:=expr;{
+ if x:=expr;true{
use(x)}
if x:=expr; expr {use(x)}
}
func _() {
- if {
+ if true {
_ = 0
}
_ = 0 // the indentation here should not be affected by the long label name
AnOverlongLabel:
_ = 0
- if {
+ if true {
_ = 0
}
_ = 0
//
// var s Scanner
// fset := token.NewFileSet() // position information is relative to fset
-// s.Init(fset, filename, src, nil /* no error handler */, 0)
+// file := fset.AddFile(filename, fset.Base(), len(src)) // register file
+// s.Init(file, src, nil /* no error handler */, 0)
// for {
// pos, tok, lit := s.Scan()
// if tok == token.EOF {
for _, e := range tokens {
src += e.lit + whitespace
}
- src_linecount := newlineCount(src) + 1
+ src_linecount := newlineCount(src)
whitespace_linecount := newlineCount(whitespace)
// verify scan
if tok == token.EOF {
lit = "<EOF>"
epos.Line = src_linecount
- epos.Column = 1
+ epos.Column = 2
}
checkPos(t, lit, pos, epos)
if tok != e.tok {
// AddLineInfo adds alternative file and line number information for
// a given file offset. The offset must be larger than the offset for
-// the previously added alternative line info and not larger than the
+// the previously added alternative line info and smaller than the
// file size; otherwise the information is ignored.
//
// AddLineInfo is typically used to register alternative position
//
func (f *File) AddLineInfo(offset int, filename string, line int) {
f.set.mutex.Lock()
- if i := len(f.infos); i == 0 || f.infos[i-1].offset < offset && offset <= f.size {
+ if i := len(f.infos); i == 0 || f.infos[i-1].offset < offset && offset < f.size {
f.infos = append(f.infos, lineInfo{offset, filename, line})
}
f.set.mutex.Unlock()
// AddLine adds the line offset for a new line.
// The line offset must be larger than the offset for the previous line
-// and not larger than the file size; otherwise the line offset is ignored.
+// and smaller than the file size; otherwise the line offset is ignored.
//
func (f *File) AddLine(offset int) {
f.set.mutex.Lock()
- if i := len(f.lines); (i == 0 || f.lines[i-1] < offset) && offset <= f.size {
+ if i := len(f.lines); (i == 0 || f.lines[i-1] < offset) && offset < f.size {
f.lines = append(f.lines, offset)
}
f.set.mutex.Unlock()
}
-// SetLines sets all line offsets for a file and returns true if successful.
+// SetLines sets the line offsets for a file and returns true if successful.
+// The line offsets are the offsets of the first character of each line;
+// for instance for the content "ab\nc\n" the line offsets are {0, 3}.
+// An empty file has an empty line offset table.
// Each line offset must be larger than the offset for the previous line
-// and not larger than the file size; otherwise the SetLines fails and returns
+// and smaller than the file size; otherwise SetLines fails and returns
// false.
//
func (f *File) SetLines(lines []int) bool {
// verify validity of lines table
size := f.size
for i, offset := range lines {
- if i > 0 && offset <= lines[i-1] || size < offset {
+ if i > 0 && offset <= lines[i-1] || size <= offset {
return false
}
}
}
+// SetLinesForContent sets the line offsets for the given file content.
+func (f *File) SetLinesForContent(content []byte) {
+ var lines []int
+ line := 0
+ for offset, b := range content {
+ if line >= 0 {
+ lines = append(lines, line)
+ }
+ line = -1
+ if b == '\n' {
+ line = offset + 1
+ }
+ }
+
+ // set lines table
+ f.set.mutex.Lock()
+ f.lines = lines
+ f.set.mutex.Unlock()
+}
+
+
// Pos returns the Pos value for the given file offset;
// the offset must be <= f.Size().
// f.Pos(f.Offset(p)) == p.
var tests = []struct {
filename string
+ source []byte // may be nil
size int
lines []int
}{
- {"a", 0, []int{}},
- {"b", 5, []int{0}},
- {"c", 10, []int{0, 1, 2, 3, 4, 5, 6, 7, 8, 9}},
- {"d", 100, []int{0, 5, 10, 20, 30, 70, 71, 72, 80, 85, 90, 99}},
- {"e", 777, []int{0, 80, 100, 120, 130, 180, 267, 455, 500, 567, 620}},
+ {"a", []byte{}, 0, []int{}},
+ {"b", []byte("01234"), 5, []int{0}},
+ {"c", []byte("\n\n\n\n\n\n\n\n\n"), 9, []int{0, 1, 2, 3, 4, 5, 6, 7, 8}},
+ {"d", nil, 100, []int{0, 5, 10, 20, 30, 70, 71, 72, 80, 85, 90, 99}},
+ {"e", nil, 777, []int{0, 80, 100, 120, 130, 180, 267, 455, 500, 567, 620}},
+ {"f", []byte("package p\n\nimport \"fmt\""), 23, []int{0, 10, 11}},
+ {"g", []byte("package p\n\nimport \"fmt\"\n"), 24, []int{0, 10, 11}},
+ {"h", []byte("package p\n\nimport \"fmt\"\n "), 25, []int{0, 10, 11, 24}},
}
}
+func makeTestSource(size int, lines []int) []byte {
+ src := make([]byte, size)
+ for _, offs := range lines {
+ if offs > 0 {
+ src[offs-1] = '\n'
+ }
+ }
+ return src
+}
+
+
func TestPositions(t *testing.T) {
const delta = 7 // a non-zero base offset increment
fset := NewFileSet()
for _, test := range tests {
+ // verify consistency of test case
+ if test.source != nil && len(test.source) != test.size {
+ t.Errorf("%s: inconsistent test case: expected file size %d; got %d", test.filename, test.size, len(test.source))
+ }
+
// add file and verify name and size
f := fset.AddFile(test.filename, fset.Base()+delta, test.size)
if f.Name() != test.filename {
verifyPositions(t, fset, f, test.lines[0:i+1])
}
- // add lines at once and verify all positions
- ok := f.SetLines(test.lines)
- if !ok {
+ // add lines with SetLines and verify all positions
+ if ok := f.SetLines(test.lines); !ok {
t.Errorf("%s: SetLines failed", f.Name())
}
if f.LineCount() != len(test.lines) {
t.Errorf("%s, SetLines: expected line count %d; got %d", f.Name(), len(test.lines), f.LineCount())
}
verifyPositions(t, fset, f, test.lines)
+
+ // add lines with SetLinesForContent and verify all positions
+ src := test.source
+ if src == nil {
+ // no test source available - create one from scratch
+ src = makeTestSource(test.size, test.lines)
+ }
+ f.SetLinesForContent(src)
+ if f.LineCount() != len(test.lines) {
+ t.Errorf("%s, SetLinesForContent: expected line count %d; got %d", f.Name(), len(test.lines), f.LineCount())
+ }
+ verifyPositions(t, fset, f, test.lines)
}
}
//
const (
LowestPrec = 0 // non-operators
- UnaryPrec = 7
- HighestPrec = 8
+ UnaryPrec = 6
+ HighestPrec = 7
)
return 1
case LAND:
return 2
- case ARROW:
- return 3
case EQL, NEQ, LSS, LEQ, GTR, GEQ:
- return 4
+ return 3
case ADD, SUB, OR, XOR:
- return 5
+ return 4
case MUL, QUO, REM, SHL, SHR, AND, AND_NOT:
- return 6
+ return 5
}
return LowestPrec
}
t.Errorf("encodeUint: %#x encode: expected % x got % x", tt.x, tt.b, b.Bytes())
}
}
- decState := newDecodeState(nil, &b)
+ decState := newDecodeState(nil, b)
for u := uint64(0); ; u = (u + 1) * 7 {
b.Reset()
encState.encodeUint(u)
var b = new(bytes.Buffer)
encState := newEncoderState(nil, b)
encState.encodeInt(i)
- decState := newDecodeState(nil, &b)
+ decState := newDecodeState(nil, b)
decState.buf = make([]byte, 8)
j := decState.decodeInt()
if i != j {
func newDecodeStateFromData(data []byte) *decodeState {
b := bytes.NewBuffer(data)
- state := newDecodeState(nil, &b)
+ state := newDecodeState(nil, b)
state.fieldnum = -1
return state
}
var data struct {
a int
}
- instr := &decInstr{decOpMap[reflect.Int], 6, 0, 0, ovfl}
+ instr := &decInstr{decOpTable[reflect.Int], 6, 0, 0, ovfl}
state := newDecodeStateFromData(signedResult)
execDec("int", instr, state, t, unsafe.Pointer(&data))
if data.a != 17 {
var data struct {
a uint
}
- instr := &decInstr{decOpMap[reflect.Uint], 6, 0, 0, ovfl}
+ instr := &decInstr{decOpTable[reflect.Uint], 6, 0, 0, ovfl}
state := newDecodeStateFromData(unsignedResult)
execDec("uint", instr, state, t, unsafe.Pointer(&data))
if data.a != 17 {
var data struct {
a uintptr
}
- instr := &decInstr{decOpMap[reflect.Uintptr], 6, 0, 0, ovfl}
+ instr := &decInstr{decOpTable[reflect.Uintptr], 6, 0, 0, ovfl}
state := newDecodeStateFromData(unsignedResult)
execDec("uintptr", instr, state, t, unsafe.Pointer(&data))
if data.a != 17 {
var data struct {
a complex64
}
- instr := &decInstr{decOpMap[reflect.Complex64], 6, 0, 0, ovfl}
+ instr := &decInstr{decOpTable[reflect.Complex64], 6, 0, 0, ovfl}
state := newDecodeStateFromData(complexResult)
execDec("complex", instr, state, t, unsafe.Pointer(&data))
if data.a != 17+19i {
var data struct {
a complex128
}
- instr := &decInstr{decOpMap[reflect.Complex128], 6, 0, 0, ovfl}
+ instr := &decInstr{decOpTable[reflect.Complex128], 6, 0, 0, ovfl}
state := newDecodeStateFromData(complexResult)
execDec("complex", instr, state, t, unsafe.Pointer(&data))
if data.a != 17+19i {
}
}
+
+func TestBadRecursiveType(t *testing.T) {
+ type Rec ***Rec
+ var rec Rec
+ b := new(bytes.Buffer)
+ err := NewEncoder(b).Encode(&rec)
+ if err == nil {
+ t.Error("expected error; got none")
+ } else if strings.Index(err.String(), "recursive") < 0 {
+ t.Error("expected recursive type error; got", err)
+ }
+ // Can't test decode easily because we can't encode one, so we can't pass one to a Decoder.
+}
+
type Bad0 struct {
- ch chan int
- c float64
+ CH chan int
+ C float64
}
-var nilEncoder *Encoder
func TestInvalidField(t *testing.T) {
var bad0 Bad0
- bad0.ch = make(chan int)
+ bad0.CH = make(chan int)
b := new(bytes.Buffer)
- err := nilEncoder.encode(b, reflect.NewValue(&bad0))
+ var nilEncoder *Encoder
+ err := nilEncoder.encode(b, reflect.NewValue(&bad0), userType(reflect.Typeof(&bad0)))
if err == nil {
t.Error("expected error; got none")
} else if strings.Index(err.String(), "type") < 0 {
}
type Point struct {
- a, b int
+ X, Y int
}
func (p Point) Square() int {
- return p.a*p.a + p.b*p.b
+ return p.X*p.X + p.Y*p.Y
}
// A struct with interfaces in it.
}
}
}
-
}
// A struct with all basic types, stored in interfaces.
int(1), int8(1), int16(1), int32(1), int64(1),
uint(1), uint8(1), uint16(1), uint32(1), uint64(1),
float32(1), 1.0,
- complex64(0i), complex128(0i),
+ complex64(1i), complex128(1i),
true,
"hello",
[]byte("sailor"),
}
}
+var singletons = []interface{}{
+ true,
+ 7,
+ 3.2,
+ "hello",
+ [3]int{11, 22, 33},
+ []float32{0.5, 0.25, 0.125},
+ map[string]int{"one": 1, "two": 2},
+}
+
+func TestDebugSingleton(t *testing.T) {
+ if debugFunc == nil {
+ return
+ }
+ b := new(bytes.Buffer)
+ // Accumulate a number of values and print them out all at once.
+ for _, x := range singletons {
+ err := NewEncoder(b).Encode(x)
+ if err != nil {
+ t.Fatal("encode:", err)
+ }
+ }
+ debugFunc(b)
+}
+
// A type that won't be defined in the gob until we send it in an interface value.
type OnTheFly struct {
A int
S []string
}
-func TestDebug(t *testing.T) {
+func TestDebugStruct(t *testing.T) {
if debugFunc == nil {
return
}
dec *Decoder
// The buffer is stored with an extra indirection because it may be replaced
// if we load a type during decode (when reading an interface value).
- b **bytes.Buffer
+ b *bytes.Buffer
fieldnum int // the last field number read.
buf []byte
}
-func newDecodeState(dec *Decoder, b **bytes.Buffer) *decodeState {
+// We pass the bytes.Buffer separately for easier testing of the infrastructure
+// without requiring a full Decoder.
+func newDecodeState(dec *Decoder, buf *bytes.Buffer) *decodeState {
d := new(decodeState)
d.dec = dec
- d.b = b
+ d.b = buf
d.buf = make([]byte, uint64Size)
return d
}
// decodeUintReader reads an encoded unsigned integer from an io.Reader.
// Used only by the Decoder to read the message length.
-func decodeUintReader(r io.Reader, buf []byte) (x uint64, err os.Error) {
- _, err = r.Read(buf[0:1])
+func decodeUintReader(r io.Reader, buf []byte) (x uint64, width int, err os.Error) {
+ width = 1
+ _, err = r.Read(buf[0:width])
if err != nil {
return
}
b := buf[0]
if b <= 0x7f {
- return uint64(b), nil
+ return uint64(b), width, nil
}
nb := -int(int8(b))
if nb > uint64Size {
for i := 0; i < n; i++ {
x <<= 8
x |= uint64(buf[i])
+ width++
}
return
}
return *(*uintptr)(up)
}
-func (dec *Decoder) decodeSingle(engine *decEngine, rtyp reflect.Type, b **bytes.Buffer, p uintptr, indir int) (err os.Error) {
- defer catchError(&err)
- p = allocate(rtyp, p, indir)
- state := newDecodeState(dec, b)
+func (dec *Decoder) decodeSingle(engine *decEngine, ut *userTypeInfo, p uintptr) (err os.Error) {
+ p = allocate(ut.base, p, ut.indir)
+ state := newDecodeState(dec, &dec.buf)
state.fieldnum = singletonField
basep := p
delta := int(state.decodeUint())
return nil
}
-func (dec *Decoder) decodeStruct(engine *decEngine, rtyp *reflect.StructType, b **bytes.Buffer, p uintptr, indir int) (err os.Error) {
- defer catchError(&err)
- p = allocate(rtyp, p, indir)
- state := newDecodeState(dec, b)
+// Indir is for the value, not the type. At the time of the call it may
+// differ from ut.indir, which was computed when the engine was built.
+// This state cannot arise for decodeSingle, which is called directly
+// from the user's value, not from the innards of an engine.
+func (dec *Decoder) decodeStruct(engine *decEngine, ut *userTypeInfo, p uintptr, indir int) (err os.Error) {
+ p = allocate(ut.base.(*reflect.StructType), p, indir)
+ state := newDecodeState(dec, &dec.buf)
state.fieldnum = -1
basep := p
for state.b.Len() > 0 {
return nil
}
-func (dec *Decoder) ignoreStruct(engine *decEngine, b **bytes.Buffer) (err os.Error) {
- defer catchError(&err)
- state := newDecodeState(dec, b)
+func (dec *Decoder) ignoreStruct(engine *decEngine) (err os.Error) {
+ state := newDecodeState(dec, &dec.buf)
state.fieldnum = -1
for state.b.Len() > 0 {
delta := int(state.decodeUint())
return nil
}
+func (dec *Decoder) ignoreSingle(engine *decEngine) (err os.Error) {
+ state := newDecodeState(dec, &dec.buf)
+ state.fieldnum = singletonField
+ delta := int(state.decodeUint())
+ if delta != 0 {
+ errorf("gob decode: corrupted data: non-zero delta for singleton")
+ }
+ instr := &engine.instr[singletonField]
+ instr.op(instr, state, unsafe.Pointer(nil))
+ return nil
+}
+
func (dec *Decoder) decodeArrayHelper(state *decodeState, p uintptr, elemOp decOp, elemWid uintptr, length, elemIndir int, ovfl os.ErrorString) {
instr := &decInstr{elemOp, 0, elemIndir, 0, ovfl}
for i := 0; i < length; i++ {
func decodeIntoValue(state *decodeState, op decOp, indir int, v reflect.Value, ovfl os.ErrorString) reflect.Value {
instr := &decInstr{op, 0, indir, 0, ovfl}
- up := unsafe.Pointer(v.Addr())
+ up := unsafe.Pointer(v.UnsafeAddr())
if indir > 1 {
up = decIndirect(up, indir)
}
if !ok {
errorf("gob: name not registered for interface: %q", name)
}
+ // Read the type id of the concrete value.
+ concreteId := dec.decodeTypeSequence(true)
+ if concreteId < 0 {
+ error(dec.err)
+ }
+ // Byte count of value is next; we don't care what it is (it's there
+ // in case we want to ignore the value by skipping it completely).
+ state.decodeUint()
// Read the concrete value.
value := reflect.MakeZero(typ)
- dec.decodeValueFromBuffer(value, false, true)
+ dec.decodeValue(concreteId, value)
if dec.err != nil {
error(dec.err)
}
if err != nil {
error(err)
}
- dec.decodeValueFromBuffer(nil, true, true)
- if dec.err != nil {
- error(err)
+ id := dec.decodeTypeSequence(true)
+ if id < 0 {
+ error(dec.err)
}
+ // At this point, the decoder buffer contains a delimited value. Just toss it.
+ state.b.Next(int(state.decodeUint()))
}
// Index by Go types.
-var decOpMap = []decOp{
+var decOpTable = [...]decOp{
reflect.Bool: decBool,
reflect.Int8: decInt8,
reflect.Int16: decInt16,
// Return the decoding op for the base type under rt and
// the indirection count to reach it.
-func (dec *Decoder) decOpFor(wireId typeId, rt reflect.Type, name string) (decOp, int) {
- typ, indir := indirect(rt)
+func (dec *Decoder) decOpFor(wireId typeId, rt reflect.Type, name string, inProgress map[reflect.Type]*decOp) (*decOp, int) {
+ ut := userType(rt)
+ // If this type is already in progress, it's a recursive type (e.g. map[string]*T).
+ // Return the pointer to the op we're already building.
+ if opPtr := inProgress[rt]; opPtr != nil {
+ return opPtr, ut.indir
+ }
+ typ := ut.base
+ indir := ut.indir
var op decOp
k := typ.Kind()
- if int(k) < len(decOpMap) {
- op = decOpMap[k]
+ if int(k) < len(decOpTable) {
+ op = decOpTable[k]
}
if op == nil {
+ inProgress[rt] = &op
// Special cases
switch t := typ.(type) {
case *reflect.ArrayType:
name = "element of " + name
elemId := dec.wireType[wireId].ArrayT.Elem
- elemOp, elemIndir := dec.decOpFor(elemId, t.Elem(), name)
+ elemOp, elemIndir := dec.decOpFor(elemId, t.Elem(), name, inProgress)
ovfl := overflow(name)
op = func(i *decInstr, state *decodeState, p unsafe.Pointer) {
- state.dec.decodeArray(t, state, uintptr(p), elemOp, t.Elem().Size(), t.Len(), i.indir, elemIndir, ovfl)
+ state.dec.decodeArray(t, state, uintptr(p), *elemOp, t.Elem().Size(), t.Len(), i.indir, elemIndir, ovfl)
}
case *reflect.MapType:
name = "element of " + name
keyId := dec.wireType[wireId].MapT.Key
elemId := dec.wireType[wireId].MapT.Elem
- keyOp, keyIndir := dec.decOpFor(keyId, t.Key(), name)
- elemOp, elemIndir := dec.decOpFor(elemId, t.Elem(), name)
+ keyOp, keyIndir := dec.decOpFor(keyId, t.Key(), name, inProgress)
+ elemOp, elemIndir := dec.decOpFor(elemId, t.Elem(), name, inProgress)
ovfl := overflow(name)
op = func(i *decInstr, state *decodeState, p unsafe.Pointer) {
up := unsafe.Pointer(p)
- state.dec.decodeMap(t, state, uintptr(up), keyOp, elemOp, i.indir, keyIndir, elemIndir, ovfl)
+ state.dec.decodeMap(t, state, uintptr(up), *keyOp, *elemOp, i.indir, keyIndir, elemIndir, ovfl)
}
case *reflect.SliceType:
} else {
elemId = dec.wireType[wireId].SliceT.Elem
}
- elemOp, elemIndir := dec.decOpFor(elemId, t.Elem(), name)
+ elemOp, elemIndir := dec.decOpFor(elemId, t.Elem(), name, inProgress)
ovfl := overflow(name)
op = func(i *decInstr, state *decodeState, p unsafe.Pointer) {
- state.dec.decodeSlice(t, state, uintptr(p), elemOp, t.Elem().Size(), i.indir, elemIndir, ovfl)
+ state.dec.decodeSlice(t, state, uintptr(p), *elemOp, t.Elem().Size(), i.indir, elemIndir, ovfl)
}
case *reflect.StructType:
error(err)
}
op = func(i *decInstr, state *decodeState, p unsafe.Pointer) {
- // indirect through enginePtr to delay evaluation for recursive structs
- err = dec.decodeStruct(*enginePtr, t, state.b, uintptr(p), i.indir)
+ // indirect through enginePtr to delay evaluation for recursive structs.
+ err = dec.decodeStruct(*enginePtr, userType(typ), uintptr(p), i.indir)
if err != nil {
error(err)
}
if op == nil {
errorf("gob: decode can't handle type %s", rt.String())
}
- return op, indir
+ return &op, indir
}
// Return the decoding op for a field that has no destination.
}
op = func(i *decInstr, state *decodeState, p unsafe.Pointer) {
// indirect through enginePtr to delay evaluation for recursive structs
- state.dec.ignoreStruct(*enginePtr, state.b)
+ state.dec.ignoreStruct(*enginePtr)
}
}
}
// Are these two gob Types compatible?
// Answers the question for basic types, arrays, and slices.
// Structs are considered ok; fields will be checked later.
-func (dec *Decoder) compatibleType(fr reflect.Type, fw typeId) bool {
- fr, _ = indirect(fr)
+func (dec *Decoder) compatibleType(fr reflect.Type, fw typeId, inProgress map[reflect.Type]typeId) bool {
+ if rhs, ok := inProgress[fr]; ok {
+ return rhs == fw
+ }
+ inProgress[fr] = fw
+ fr = userType(fr).base
switch t := fr.(type) {
default:
- // map, chan, etc: cannot handle.
+ // chan, etc: cannot handle.
return false
case *reflect.BoolType:
return fw == tBool
return false
}
array := wire.ArrayT
- return t.Len() == array.Len && dec.compatibleType(t.Elem(), array.Elem)
+ return t.Len() == array.Len && dec.compatibleType(t.Elem(), array.Elem, inProgress)
case *reflect.MapType:
wire, ok := dec.wireType[fw]
if !ok || wire.MapT == nil {
return false
}
MapType := wire.MapT
- return dec.compatibleType(t.Key(), MapType.Key) && dec.compatibleType(t.Elem(), MapType.Elem)
+ return dec.compatibleType(t.Key(), MapType.Key, inProgress) && dec.compatibleType(t.Elem(), MapType.Elem, inProgress)
case *reflect.SliceType:
// Is it an array of bytes?
if t.Elem().Kind() == reflect.Uint8 {
} else {
sw = dec.wireType[fw].SliceT
}
- elem, _ := indirect(t.Elem())
- return sw != nil && dec.compatibleType(elem, sw.Elem)
+ elem := userType(t.Elem()).base
+ return sw != nil && dec.compatibleType(elem, sw.Elem, inProgress)
case *reflect.StructType:
return true
}
engine = new(decEngine)
engine.instr = make([]decInstr, 1) // one item
name := rt.String() // best we can do
- if !dec.compatibleType(rt, remoteId) {
+ if !dec.compatibleType(rt, remoteId, make(map[reflect.Type]typeId)) {
return nil, os.ErrorString("gob: wrong type received for local value " + name + ": " + dec.typeString(remoteId))
}
- op, indir := dec.decOpFor(remoteId, rt, name)
+ op, indir := dec.decOpFor(remoteId, rt, name, make(map[reflect.Type]*decOp))
ovfl := os.ErrorString(`value for "` + name + `" out of range`)
- engine.instr[singletonField] = decInstr{op, singletonField, indir, 0, ovfl}
+ engine.instr[singletonField] = decInstr{*op, singletonField, indir, 0, ovfl}
+ engine.numInstr = 1
+ return
+}
+
+func (dec *Decoder) compileIgnoreSingle(remoteId typeId) (engine *decEngine, err os.Error) {
+ engine = new(decEngine)
+ engine.instr = make([]decInstr, 1) // one item
+ op := dec.decIgnoreOpFor(remoteId)
+ ovfl := overflow(dec.typeString(remoteId))
+ engine.instr[0] = decInstr{op, 0, 0, 0, ovfl}
engine.numInstr = 1
return
}
}
func (dec *Decoder) compileDec(remoteId typeId, rt reflect.Type) (engine *decEngine, err os.Error) {
- defer catchError(&err)
srt, ok := rt.(*reflect.StructType)
if !ok {
return dec.compileSingle(remoteId, rt)
if t, ok := builtinIdToType[remoteId]; ok {
wireStruct, _ = t.(*structType)
} else {
- wireStruct = dec.wireType[remoteId].StructT
+ wire := dec.wireType[remoteId]
+ if wire == nil {
+ error(errBadType)
+ }
+ wireStruct = wire.StructT
}
if wireStruct == nil {
errorf("gob: type mismatch in decoder: want struct type %s; got non-struct", rt.String())
}
engine = new(decEngine)
engine.instr = make([]decInstr, len(wireStruct.Field))
+ seen := make(map[reflect.Type]*decOp)
// Loop over the fields of the wire type.
for fieldnum := 0; fieldnum < len(wireStruct.Field); fieldnum++ {
wireField := wireStruct.Field[fieldnum]
engine.instr[fieldnum] = decInstr{op, fieldnum, 0, 0, ovfl}
continue
}
- if !dec.compatibleType(localField.Type, wireField.Id) {
+ if !dec.compatibleType(localField.Type, wireField.Id, make(map[reflect.Type]typeId)) {
errorf("gob: wrong type (%s) for received field %s.%s", localField.Type, wireStruct.Name, wireField.Name)
}
- op, indir := dec.decOpFor(wireField.Id, localField.Type, localField.Name)
- engine.instr[fieldnum] = decInstr{op, fieldnum, indir, uintptr(localField.Offset), ovfl}
+ op, indir := dec.decOpFor(wireField.Id, localField.Type, localField.Name, seen)
+ engine.instr[fieldnum] = decInstr{*op, fieldnum, indir, uintptr(localField.Offset), ovfl}
engine.numInstr++
}
return
// To handle recursive types, mark this engine as underway before compiling.
enginePtr = new(*decEngine)
dec.ignorerCache[wireId] = enginePtr
- *enginePtr, err = dec.compileDec(wireId, emptyStructType)
+ wire := dec.wireType[wireId]
+ if wire != nil && wire.StructT != nil {
+ *enginePtr, err = dec.compileDec(wireId, emptyStructType)
+ } else {
+ *enginePtr, err = dec.compileIgnoreSingle(wireId)
+ }
if err != nil {
dec.ignorerCache[wireId] = nil, false
}
return
}
-func (dec *Decoder) decode(wireId typeId, val reflect.Value) os.Error {
+func (dec *Decoder) decodeValue(wireId typeId, val reflect.Value) (err os.Error) {
+ defer catchError(&err)
+ // If the value is nil, it means we should just ignore this item.
+ if val == nil {
+ return dec.decodeIgnoredValue(wireId)
+ }
// Dereference down to the underlying struct type.
- rt, indir := indirect(val.Type())
- enginePtr, err := dec.getDecEnginePtr(wireId, rt)
+ ut := userType(val.Type())
+ base := ut.base
+ indir := ut.indir
+ enginePtr, err := dec.getDecEnginePtr(wireId, base)
if err != nil {
return err
}
engine := *enginePtr
- if st, ok := rt.(*reflect.StructType); ok {
+ if st, ok := base.(*reflect.StructType); ok {
if engine.numInstr == 0 && st.NumField() > 0 && len(dec.wireType[wireId].StructT.Field) > 0 {
- name := rt.Name()
+ name := base.Name()
return os.ErrorString("gob: type mismatch: no fields matched compiling decoder for " + name)
}
- return dec.decodeStruct(engine, st, dec.state.b, uintptr(val.Addr()), indir)
+ return dec.decodeStruct(engine, ut, uintptr(val.UnsafeAddr()), indir)
+ }
+ return dec.decodeSingle(engine, ut, uintptr(val.UnsafeAddr()))
+}
+
+func (dec *Decoder) decodeIgnoredValue(wireId typeId) os.Error {
+ enginePtr, err := dec.getIgnoreEnginePtr(wireId)
+ if err != nil {
+ return err
+ }
+ wire := dec.wireType[wireId]
+ if wire != nil && wire.StructT != nil {
+ return dec.ignoreStruct(*enginePtr)
}
- return dec.decodeSingle(engine, rt, dec.state.b, uintptr(val.Addr()), indir)
+ return dec.ignoreSingle(*enginePtr)
}
func init() {
default:
panic("gob: unknown size of int/uint")
}
- decOpMap[reflect.Int] = iop
- decOpMap[reflect.Uint] = uop
+ decOpTable[reflect.Int] = iop
+ decOpTable[reflect.Uint] = uop
// Finally uintptr
switch reflect.Typeof(uintptr(0)).Bits() {
default:
panic("gob: unknown size of uintptr")
}
- decOpMap[reflect.Uintptr] = uop
+ decOpTable[reflect.Uintptr] = uop
}
type Decoder struct {
mutex sync.Mutex // each item must be received atomically
r io.Reader // source of the data
+ buf bytes.Buffer // buffer for more efficient i/o from r
wireType map[typeId]*wireType // map from remote ID to local description
decoderCache map[reflect.Type]map[typeId]**decEngine // cache of compiled engines
ignorerCache map[typeId]**decEngine // ditto for ignored objects
- state *decodeState // reads data from in-memory buffer
countState *decodeState // reads counts from wire
- buf []byte
- countBuf [9]byte // counts may be uint64s (unlikely!), require 9 bytes
- byteBuffer *bytes.Buffer
+ countBuf []byte // used for decoding integers while parsing messages
+ tmp []byte // temporary storage for i/o; saves reallocating
err os.Error
}
dec := new(Decoder)
dec.r = r
dec.wireType = make(map[typeId]*wireType)
- dec.state = newDecodeState(dec, &dec.byteBuffer) // buffer set in Decode()
dec.decoderCache = make(map[reflect.Type]map[typeId]**decEngine)
dec.ignorerCache = make(map[typeId]**decEngine)
+ dec.countBuf = make([]byte, 9) // counts may be uint64s (unlikely!), require 9 bytes
return dec
}
-// recvType loads the definition of a type and reloads the Decoder's buffer.
+// recvType loads the definition of a type.
func (dec *Decoder) recvType(id typeId) {
// Have we already seen this type? That's an error
- if dec.wireType[id] != nil {
+ if id < firstUserId || dec.wireType[id] != nil {
dec.err = os.ErrorString("gob: duplicate type received")
return
}
// Type:
wire := new(wireType)
- dec.err = dec.decode(tWireType, reflect.NewValue(wire))
+ dec.err = dec.decodeValue(tWireType, reflect.NewValue(wire))
if dec.err != nil {
return
}
// Remember we've seen this type.
dec.wireType[id] = wire
-
- // Load the next parcel.
- dec.recv()
}
-// Decode reads the next value from the connection and stores
-// it in the data represented by the empty interface value.
-// The value underlying e must be the correct type for the next
-// data item received, and must be a pointer.
-func (dec *Decoder) Decode(e interface{}) os.Error {
- value := reflect.NewValue(e)
- // If e represents a value as opposed to a pointer, the answer won't
- // get back to the caller. Make sure it's a pointer.
- if value.Type().Kind() != reflect.Ptr {
- dec.err = os.ErrorString("gob: attempt to decode into a non-pointer")
- return dec.err
+// recvMessage reads the next count-delimited item from the input. It is the converse
+// of Encoder.writeMessage. It returns false on EOF or other error reading the message.
+func (dec *Decoder) recvMessage() bool {
+ // Read a count.
+ nbytes, _, err := decodeUintReader(dec.r, dec.countBuf)
+ if err != nil {
+ dec.err = err
+ return false
}
- return dec.DecodeValue(value)
+ dec.readMessage(int(nbytes))
+ return dec.err == nil
}
-// recv reads the next count-delimited item from the input. It is the converse
-// of Encoder.send.
-func (dec *Decoder) recv() {
- // Read a count.
- var nbytes uint64
- nbytes, dec.err = decodeUintReader(dec.r, dec.countBuf[0:])
- if dec.err != nil {
- return
- }
+// readMessage reads the next nbytes bytes from the input.
+func (dec *Decoder) readMessage(nbytes int) {
// Allocate the buffer.
- if nbytes > uint64(len(dec.buf)) {
- dec.buf = make([]byte, nbytes+1000)
+ if cap(dec.tmp) < nbytes {
+ dec.tmp = make([]byte, nbytes+100) // room to grow
}
- dec.byteBuffer = bytes.NewBuffer(dec.buf[0:nbytes])
+ dec.tmp = dec.tmp[:nbytes]
// Read the data
- _, dec.err = io.ReadFull(dec.r, dec.buf[0:nbytes])
+ _, dec.err = io.ReadFull(dec.r, dec.tmp)
if dec.err != nil {
if dec.err == os.EOF {
dec.err = io.ErrUnexpectedEOF
}
return
}
+ dec.buf.Write(dec.tmp)
}
-// decodeValueFromBuffer grabs the next value from the input. The Decoder's
-// buffer already contains data. If the next item in the buffer is a type
-// descriptor, it may be necessary to reload the buffer, but recvType does that.
-func (dec *Decoder) decodeValueFromBuffer(value reflect.Value, ignoreInterfaceValue, countPresent bool) {
- for dec.state.b.Len() > 0 {
- // Receive a type id.
- id := typeId(dec.state.decodeInt())
+// toInt turns an encoded uint64 into an int, according to the marshaling rules.
+func toInt(x uint64) int64 {
+ i := int64(x >> 1)
+ if x&1 != 0 {
+ i = ^i
+ }
+ return i
+}
+
+func (dec *Decoder) nextInt() int64 {
+ n, _, err := decodeUintReader(&dec.buf, dec.countBuf)
+ if err != nil {
+ dec.err = err
+ }
+ return toInt(n)
+}
- // Is it a new type?
- if id < 0 { // 0 is the error state, handled above
- // If the id is negative, we have a type.
- dec.recvType(-id)
- if dec.err != nil {
+func (dec *Decoder) nextUint() uint64 {
+ n, _, err := decodeUintReader(&dec.buf, dec.countBuf)
+ if err != nil {
+ dec.err = err
+ }
+ return n
+}
+
+// decodeTypeSequence parses:
+// TypeSequence
+// (TypeDefinition DelimitedTypeDefinition*)?
+// and returns the type id of the next value. It returns -1 at
+// EOF. Upon return, the remainder of dec.buf is the value to be
+// decoded. If this is an interface value, it can be ignored by
+// simply resetting that buffer.
+func (dec *Decoder) decodeTypeSequence(isInterface bool) typeId {
+ for dec.err == nil {
+ if dec.buf.Len() == 0 {
+ if !dec.recvMessage() {
break
}
- continue
}
-
- // Make sure the type has been defined already or is a builtin type (for
- // top-level singleton values).
- if dec.wireType[id] == nil && builtinIdToType[id] == nil {
- dec.err = errBadType
- break
+ // Receive a type id.
+ id := typeId(dec.nextInt())
+ if id >= 0 {
+ // Value follows.
+ return id
}
- // An interface value is preceded by a byte count.
- if countPresent {
- count := int(dec.state.decodeUint())
- if ignoreInterfaceValue {
- // An interface value is preceded by a byte count. Just skip that many bytes.
- dec.state.b.Next(int(count))
+ // Type definition for (-id) follows.
+ dec.recvType(-id)
+ // When decoding an interface, after a type there may be a
+ // DelimitedValue still in the buffer. Skip its count.
+ // (Alternatively, the buffer is empty and the byte count
+ // will be absorbed by recvMessage.)
+ if dec.buf.Len() > 0 {
+ if !isInterface {
+ dec.err = os.ErrorString("extra data in buffer")
break
}
- // Otherwise fall through and decode it.
+ dec.nextUint()
}
- dec.err = dec.decode(id, value)
- break
}
+ return -1
+}
+
+// Decode reads the next value from the connection and stores
+// it in the data represented by the empty interface value.
+// If e is nil, the value will be discarded. Otherwise,
+// the value underlying e must either be the correct type for the next
+// data item received, and must be a pointer.
+func (dec *Decoder) Decode(e interface{}) os.Error {
+ if e == nil {
+ return dec.DecodeValue(nil)
+ }
+ value := reflect.NewValue(e)
+ // If e represents a value as opposed to a pointer, the answer won't
+ // get back to the caller. Make sure it's a pointer.
+ if value.Type().Kind() != reflect.Ptr {
+ dec.err = os.ErrorString("gob: attempt to decode into a non-pointer")
+ return dec.err
+ }
+ return dec.DecodeValue(value)
}
// DecodeValue reads the next value from the connection and stores
// it in the data represented by the reflection value.
// The value must be the correct type for the next
-// data item received.
+// data item received, or it may be nil, which means the
+// value will be discarded.
func (dec *Decoder) DecodeValue(value reflect.Value) os.Error {
// Make sure we're single-threaded through here.
dec.mutex.Lock()
defer dec.mutex.Unlock()
+ dec.buf.Reset() // In case data lingers from previous invocation.
dec.err = nil
- dec.recv()
- if dec.err != nil {
- return dec.err
+ id := dec.decodeTypeSequence(false)
+ if dec.err == nil {
+ dec.err = dec.decodeValue(id, value)
}
- dec.decodeValueFromBuffer(value, false, false)
return dec.err
}
*/
package gob
+/*
+Grammar:
+
+Tokens starting with a lower case letter are terminals; int(n)
+and uint(n) represent the signed/unsigned encodings of the value n.
+
+GobStream:
+ DelimitedMessage*
+DelimitedMessage:
+ uint(lengthOfMessage) Message
+Message:
+ TypeSequence TypedValue
+TypeSequence
+ (TypeDefinition DelimitedTypeDefinition*)?
+DelimitedTypeDefinition:
+ uint(lengthOfTypeDefinition) TypeDefinition
+TypedValue:
+ int(typeId) Value
+TypeDefinition:
+ int(-typeId) encodingOfWireType
+Value:
+ SingletonValue | StructValue
+SingletonValue:
+ uint(0) FieldValue
+FieldValue:
+ builtinValue | ArrayValue | MapValue | SliceValue | StructValue | InterfaceValue
+InterfaceValue:
+ NilInterfaceValue | NonNilInterfaceValue
+NilInterfaceValue:
+ uint(0)
+NonNilInterfaceValue:
+ ConcreteTypeName TypeSequence InterfaceContents
+ConcreteTypeName:
+ uint(lengthOfName) [already read=n] name
+InterfaceContents:
+ int(concreteTypeId) DelimitedValue
+DelimitedValue:
+ uint(length) Value
+ArrayValue:
+ uint(n) FieldValue*n [n elements]
+MapValue:
+ uint(n) (FieldValue FieldValue)*n [n (key, value) pairs]
+SliceValue:
+ uint(n) FieldValue*n [n elements]
+StructValue:
+ (uint(fieldDelta) FieldValue)*
+*/
+
/*
For implementers and the curious, here is an encoded example. Given
type Point struct {x, y int}
}
}
-func encNoOp(i *encInstr, state *encoderState, p unsafe.Pointer) {
-}
-
// Byte arrays are encoded as an unsigned count followed by the raw bytes.
func encUint8Array(i *encInstr, state *encoderState, p unsafe.Pointer) {
b := *(*[]byte)(p)
if v == nil {
errorf("gob: encodeReflectValue: nil element")
}
- op(nil, state, unsafe.Pointer(v.Addr()))
+ op(nil, state, unsafe.Pointer(v.UnsafeAddr()))
}
func (enc *Encoder) encodeMap(b *bytes.Buffer, mv *reflect.MapValue, keyOp, elemOp encOp, keyIndir, elemIndir int) {
return
}
- typ, _ := indirect(iv.Elem().Type())
- name, ok := concreteTypeToName[typ]
+ ut := userType(iv.Elem().Type())
+ name, ok := concreteTypeToName[ut.base]
if !ok {
- errorf("gob: type not registered for interface: %s", typ)
+ errorf("gob: type not registered for interface: %s", ut.base)
}
// Send the name.
state.encodeUint(uint64(len(name)))
if err != nil {
error(err)
}
- // Send (and maybe first define) the type id.
- enc.sendTypeDescriptor(typ)
- // Encode the value into a new buffer.
+ // Define the type id if necessary.
+ enc.sendTypeDescriptor(enc.writer(), state, ut)
+ // Send the type id.
+ enc.sendTypeId(state, ut)
+ // Encode the value into a new buffer. Any nested type definitions
+ // should be written to b, before the encoded value.
+ enc.pushWriter(b)
data := new(bytes.Buffer)
- err = enc.encode(data, iv.Elem())
+ err = enc.encode(data, iv.Elem(), ut)
if err != nil {
error(err)
}
- state.encodeUint(uint64(data.Len()))
- _, err = state.b.Write(data.Bytes())
- if err != nil {
+ enc.popWriter()
+ enc.writeMessage(b, data)
+ if enc.err != nil {
error(err)
}
}
-var encOpMap = []encOp{
+var encOpTable = [...]encOp{
reflect.Bool: encBool,
reflect.Int: encInt,
reflect.Int8: encInt8,
reflect.String: encString,
}
-// Return the encoding op for the base type under rt and
+// Return (a pointer to) the encoding op for the base type under rt and
// the indirection count to reach it.
-func (enc *Encoder) encOpFor(rt reflect.Type) (encOp, int) {
- typ, indir := indirect(rt)
- var op encOp
+func (enc *Encoder) encOpFor(rt reflect.Type, inProgress map[reflect.Type]*encOp) (*encOp, int) {
+ ut := userType(rt)
+ // If this type is already in progress, it's a recursive type (e.g. map[string]*T).
+ // Return the pointer to the op we're already building.
+ if opPtr := inProgress[rt]; opPtr != nil {
+ return opPtr, ut.indir
+ }
+ typ := ut.base
+ indir := ut.indir
k := typ.Kind()
- if int(k) < len(encOpMap) {
- op = encOpMap[k]
+ var op encOp
+ if int(k) < len(encOpTable) {
+ op = encOpTable[k]
}
if op == nil {
+ inProgress[rt] = &op
// Special cases
switch t := typ.(type) {
case *reflect.SliceType:
break
}
// Slices have a header; we decode it to find the underlying array.
- elemOp, indir := enc.encOpFor(t.Elem())
+ elemOp, indir := enc.encOpFor(t.Elem(), inProgress)
op = func(i *encInstr, state *encoderState, p unsafe.Pointer) {
slice := (*reflect.SliceHeader)(p)
if !state.sendZero && slice.Len == 0 {
return
}
state.update(i)
- state.enc.encodeArray(state.b, slice.Data, elemOp, t.Elem().Size(), indir, int(slice.Len))
+ state.enc.encodeArray(state.b, slice.Data, *elemOp, t.Elem().Size(), indir, int(slice.Len))
}
case *reflect.ArrayType:
// True arrays have size in the type.
- elemOp, indir := enc.encOpFor(t.Elem())
+ elemOp, indir := enc.encOpFor(t.Elem(), inProgress)
op = func(i *encInstr, state *encoderState, p unsafe.Pointer) {
state.update(i)
- state.enc.encodeArray(state.b, uintptr(p), elemOp, t.Elem().Size(), indir, t.Len())
+ state.enc.encodeArray(state.b, uintptr(p), *elemOp, t.Elem().Size(), indir, t.Len())
}
case *reflect.MapType:
- keyOp, keyIndir := enc.encOpFor(t.Key())
- elemOp, elemIndir := enc.encOpFor(t.Elem())
+ keyOp, keyIndir := enc.encOpFor(t.Key(), inProgress)
+ elemOp, elemIndir := enc.encOpFor(t.Elem(), inProgress)
op = func(i *encInstr, state *encoderState, p unsafe.Pointer) {
// Maps cannot be accessed by moving addresses around the way
// that slices etc. can. We must recover a full reflection value for
return
}
state.update(i)
- state.enc.encodeMap(state.b, mv, keyOp, elemOp, keyIndir, elemIndir)
+ state.enc.encodeMap(state.b, mv, *keyOp, *elemOp, keyIndir, elemIndir)
}
case *reflect.StructType:
// Generate a closure that calls out to the engine for the nested type.
if op == nil {
errorf("gob enc: can't happen: encode type %s", rt.String())
}
- return op, indir
+ return &op, indir
}
// The local Type was compiled from the actual value, so we know it's compatible.
func (enc *Encoder) compileEnc(rt reflect.Type) *encEngine {
srt, isStruct := rt.(*reflect.StructType)
engine := new(encEngine)
+ seen := make(map[reflect.Type]*encOp)
if isStruct {
- engine.instr = make([]encInstr, srt.NumField()+1) // +1 for terminator
- for fieldnum := 0; fieldnum < srt.NumField(); fieldnum++ {
- f := srt.Field(fieldnum)
- op, indir := enc.encOpFor(f.Type)
+ for fieldNum := 0; fieldNum < srt.NumField(); fieldNum++ {
+ f := srt.Field(fieldNum)
if !isExported(f.Name) {
- op = encNoOp
+ continue
}
- engine.instr[fieldnum] = encInstr{op, fieldnum, indir, uintptr(f.Offset)}
+ op, indir := enc.encOpFor(f.Type, seen)
+ engine.instr = append(engine.instr, encInstr{*op, fieldNum, indir, uintptr(f.Offset)})
+ }
+ if srt.NumField() > 0 && len(engine.instr) == 0 {
+ errorf("type %s has no exported fields", rt)
}
- engine.instr[srt.NumField()] = encInstr{encStructTerminator, 0, 0, 0}
+ engine.instr = append(engine.instr, encInstr{encStructTerminator, 0, 0, 0})
} else {
engine.instr = make([]encInstr, 1)
- op, indir := enc.encOpFor(rt)
- engine.instr[0] = encInstr{op, singletonField, indir, 0} // offset is zero
+ op, indir := enc.encOpFor(rt, seen)
+ engine.instr[0] = encInstr{*op, singletonField, indir, 0} // offset is zero
}
return engine
}
return enc.getEncEngine(rt)
}
-func (enc *Encoder) encode(b *bytes.Buffer, value reflect.Value) (err os.Error) {
+func (enc *Encoder) encode(b *bytes.Buffer, value reflect.Value, ut *userTypeInfo) (err os.Error) {
defer catchError(&err)
- // Dereference down to the underlying object.
- rt, indir := indirect(value.Type())
- for i := 0; i < indir; i++ {
+ for i := 0; i < ut.indir; i++ {
value = reflect.Indirect(value)
}
- engine := enc.lockAndGetEncEngine(rt)
+ engine := enc.lockAndGetEncEngine(ut.base)
if value.Type().Kind() == reflect.Struct {
- enc.encodeStruct(b, engine, value.Addr())
+ enc.encodeStruct(b, engine, value.UnsafeAddr())
} else {
- enc.encodeSingle(b, engine, value.Addr())
+ enc.encodeSingle(b, engine, value.UnsafeAddr())
}
return nil
}
// other side of a connection.
type Encoder struct {
mutex sync.Mutex // each item must be sent atomically
- w io.Writer // where to send the data
+ w []io.Writer // where to send the data
sent map[reflect.Type]typeId // which types we've already sent
- state *encoderState // so we can encode integers, strings directly
countState *encoderState // stage for writing counts
buf []byte // for collecting the output.
err os.Error
// NewEncoder returns a new encoder that will transmit on the io.Writer.
func NewEncoder(w io.Writer) *Encoder {
enc := new(Encoder)
- enc.w = w
+ enc.w = []io.Writer{w}
enc.sent = make(map[reflect.Type]typeId)
- enc.state = newEncoderState(enc, new(bytes.Buffer))
enc.countState = newEncoderState(enc, new(bytes.Buffer))
return enc
}
+// writer() returns the innermost writer the encoder is using
+func (enc *Encoder) writer() io.Writer {
+ return enc.w[len(enc.w)-1]
+}
+
+// pushWriter adds a writer to the encoder.
+func (enc *Encoder) pushWriter(w io.Writer) {
+ enc.w = append(enc.w, w)
+}
+
+// popWriter pops the innermost writer.
+func (enc *Encoder) popWriter() {
+ enc.w = enc.w[0 : len(enc.w)-1]
+}
+
func (enc *Encoder) badType(rt reflect.Type) {
enc.setError(os.ErrorString("gob: can't encode type " + rt.String()))
}
if enc.err == nil { // remember the first.
enc.err = err
}
- enc.state.b.Reset()
}
-// Send the data item preceded by a unsigned count of its length.
-func (enc *Encoder) send() {
- // Encode the length.
- enc.countState.encodeUint(uint64(enc.state.b.Len()))
+// writeMessage sends the data item preceded by a unsigned count of its length.
+func (enc *Encoder) writeMessage(w io.Writer, b *bytes.Buffer) {
+ enc.countState.encodeUint(uint64(b.Len()))
// Build the buffer.
countLen := enc.countState.b.Len()
- total := countLen + enc.state.b.Len()
+ total := countLen + b.Len()
if total > len(enc.buf) {
enc.buf = make([]byte, total+1000) // extra for growth
}
// TODO(r): avoid the extra copy here.
enc.countState.b.Read(enc.buf[0:countLen])
// Now the data.
- enc.state.b.Read(enc.buf[countLen:total])
+ b.Read(enc.buf[countLen:total])
// Write the data.
- _, err := enc.w.Write(enc.buf[0:total])
+ _, err := w.Write(enc.buf[0:total])
if err != nil {
enc.setError(err)
}
}
-func (enc *Encoder) sendType(origt reflect.Type) (sent bool) {
+func (enc *Encoder) sendType(w io.Writer, state *encoderState, origt reflect.Type) (sent bool) {
// Drill down to the base type.
- rt, _ := indirect(origt)
+ ut := userType(origt)
+ rt := ut.base
switch rt := rt.(type) {
default:
}
// Send the pair (-id, type)
// Id:
- enc.state.encodeInt(-int64(info.id))
+ state.encodeInt(-int64(info.id))
// Type:
- enc.encode(enc.state.b, reflect.NewValue(info.wire))
- enc.send()
+ enc.encode(state.b, reflect.NewValue(info.wire), wireTypeUserInfo)
+ enc.writeMessage(w, state.b)
if enc.err != nil {
return
}
switch st := rt.(type) {
case *reflect.StructType:
for i := 0; i < st.NumField(); i++ {
- enc.sendType(st.Field(i).Type)
+ enc.sendType(w, state, st.Field(i).Type)
}
case reflect.ArrayOrSliceType:
- enc.sendType(st.Elem())
+ enc.sendType(w, state, st.Elem())
}
return true
}
return enc.EncodeValue(reflect.NewValue(e))
}
-// sendTypeId makes sure the remote side knows about this type.
+// sendTypeDescriptor makes sure the remote side knows about this type.
// It will send a descriptor if this is the first time the type has been
-// sent. Regardless, it sends the id.
-func (enc *Encoder) sendTypeDescriptor(rt reflect.Type) {
+// sent.
+func (enc *Encoder) sendTypeDescriptor(w io.Writer, state *encoderState, ut *userTypeInfo) {
// Make sure the type is known to the other side.
- // First, have we already sent this type?
- if _, alreadySent := enc.sent[rt]; !alreadySent {
+ // First, have we already sent this (base) type?
+ base := ut.base
+ if _, alreadySent := enc.sent[base]; !alreadySent {
// No, so send it.
- sent := enc.sendType(rt)
+ sent := enc.sendType(w, state, base)
if enc.err != nil {
return
}
// need to send the type info but we do need to update enc.sent.
if !sent {
typeLock.Lock()
- info, err := getTypeInfo(rt)
+ info, err := getTypeInfo(base)
typeLock.Unlock()
if err != nil {
enc.setError(err)
return
}
- enc.sent[rt] = info.id
+ enc.sent[base] = info.id
}
}
+}
+// sendTypeId sends the id, which must have already been defined.
+func (enc *Encoder) sendTypeId(state *encoderState, ut *userTypeInfo) {
// Identify the type of this top-level value.
- enc.state.encodeInt(int64(enc.sent[rt]))
+ state.encodeInt(int64(enc.sent[ut.base]))
}
// EncodeValue transmits the data item represented by the reflection value,
enc.mutex.Lock()
defer enc.mutex.Unlock()
- enc.err = nil
- rt, _ := indirect(value.Type())
+ // Remove any nested writers remaining due to previous errors.
+ enc.w = enc.w[0:1]
- // Sanity check only: encoder should never come in with data present.
- if enc.state.b.Len() > 0 || enc.countState.b.Len() > 0 {
- enc.err = os.ErrorString("encoder: buffer not empty")
- return enc.err
+ ut, err := validUserType(value.Type())
+ if err != nil {
+ return err
}
- enc.sendTypeDescriptor(rt)
+ enc.err = nil
+ state := newEncoderState(enc, new(bytes.Buffer))
+
+ enc.sendTypeDescriptor(enc.writer(), state, ut)
+ enc.sendTypeId(state, ut)
if enc.err != nil {
return enc.err
}
// Encode the object.
- err := enc.encode(enc.state.b, value)
+ err = enc.encode(state.b, value, ut)
if err != nil {
enc.setError(err)
} else {
- enc.send()
+ enc.writeMessage(enc.writer(), state.b)
}
return enc.err
import (
"bytes"
+ "fmt"
"io"
"os"
"reflect"
dec := NewDecoder(b)
err1 := dec.Decode(new(ET2))
if err1 != err {
- t.Error("expected error", err, "got", err1)
+ t.Errorf("from %q expected error %s; got %s", s, err, err1)
}
}
func TestValueError(t *testing.T) {
// Encode a *T, decode a T
type Type4 struct {
- a int
+ A int
}
t4p := &Type4{3}
var t4 Type4 // note: not a pointer.
}
}
+func TestRecursiveMapType(t *testing.T) {
+ type recursiveMap map[string]recursiveMap
+ r1 := recursiveMap{"A": recursiveMap{"B": nil, "C": nil}, "D": nil}
+ r2 := make(recursiveMap)
+ if err := encAndDec(r1, &r2); err != nil {
+ t.Error(err)
+ }
+}
+
+func TestRecursiveSliceType(t *testing.T) {
+ type recursiveSlice []recursiveSlice
+ r1 := recursiveSlice{0: recursiveSlice{0: nil}, 1: nil}
+ r2 := make(recursiveSlice, 0)
+ if err := encAndDec(r1, &r2); err != nil {
+ t.Error(err)
+ }
+}
+
// Regression test for bug: must send zero values inside arrays
func TestDefaultsInArray(t *testing.T) {
type Type7 struct {
t.Fatal("decode error:", err)
}
}
+
+// Now follow various tests that decode into things that can't represent the
+// encoded value, all of which should be legal.
+
+// Also, when the ignored object contains an interface value, it may define
+// types. Make sure that skipping the value still defines the types by using
+// the encoder/decoder pair to send a value afterwards. If an interface
+// is sent, its type in the test is always NewType0, so this checks that the
+// encoder and decoder don't skew with respect to type definitions.
+
+type Struct0 struct {
+ I interface{}
+}
+
+type NewType0 struct {
+ S string
+}
+
+type ignoreTest struct {
+ in, out interface{}
+}
+
+var ignoreTests = []ignoreTest{
+ // Decode normal struct into an empty struct
+ {&struct{ A int }{23}, &struct{}{}},
+ // Decode normal struct into a nil.
+ {&struct{ A int }{23}, nil},
+ // Decode singleton string into a nil.
+ {"hello, world", nil},
+ // Decode singleton slice into a nil.
+ {[]int{1, 2, 3, 4}, nil},
+ // Decode struct containing an interface into a nil.
+ {&Struct0{&NewType0{"value0"}}, nil},
+ // Decode singleton slice of interfaces into a nil.
+ {[]interface{}{"hi", &NewType0{"value1"}, 23}, nil},
+}
+
+func TestDecodeIntoNothing(t *testing.T) {
+ Register(new(NewType0))
+ for i, test := range ignoreTests {
+ b := new(bytes.Buffer)
+ enc := NewEncoder(b)
+ err := enc.Encode(test.in)
+ if err != nil {
+ t.Errorf("%d: encode error %s:", i, err)
+ continue
+ }
+ dec := NewDecoder(b)
+ err = dec.Decode(test.out)
+ if err != nil {
+ t.Errorf("%d: decode error: %s", i, err)
+ continue
+ }
+ // Now see if the encoder and decoder are in a consistent state.
+ str := fmt.Sprintf("Value %d", i)
+ err = enc.Encode(&NewType0{str})
+ if err != nil {
+ t.Fatalf("%d: NewType0 encode error: %s", i, err)
+ }
+ ns := new(NewType0)
+ err = dec.Decode(ns)
+ if err != nil {
+ t.Fatalf("%d: NewType0 decode error: %s", i, err)
+ }
+ if ns.S != str {
+ t.Fatalf("%d: expected %q got %q", i, str, ns.S)
+ }
+ }
+}
+
+// Another bug from golang-nuts, involving nested interfaces.
+type Bug0Outer struct {
+ Bug0Field interface{}
+}
+
+type Bug0Inner struct {
+ A int
+}
+
+func TestNestedInterfaces(t *testing.T) {
+ var buf bytes.Buffer
+ e := NewEncoder(&buf)
+ d := NewDecoder(&buf)
+ Register(new(Bug0Outer))
+ Register(new(Bug0Inner))
+ f := &Bug0Outer{&Bug0Outer{&Bug0Inner{7}}}
+ var v interface{} = f
+ err := e.Encode(&v)
+ if err != nil {
+ t.Fatal("Encode:", err)
+ }
+ err = d.Decode(&v)
+ if err != nil {
+ t.Fatal("Decode:", err)
+ }
+ // Make sure it decoded correctly.
+ outer1, ok := v.(*Bug0Outer)
+ if !ok {
+ t.Fatalf("v not Bug0Outer: %T", v)
+ }
+ outer2, ok := outer1.Bug0Field.(*Bug0Outer)
+ if !ok {
+ t.Fatalf("v.Bug0Field not Bug0Outer: %T", outer1.Bug0Field)
+ }
+ inner, ok := outer2.Bug0Field.(*Bug0Inner)
+ if !ok {
+ t.Fatalf("v.Bug0Field.Bug0Field not Bug0Inner: %T", outer2.Bug0Field)
+ }
+ if inner.A != 7 {
+ t.Fatalf("final value %d; expected %d", inner.A, 7)
+ }
+}
"sync"
)
-// Reflection types are themselves interface values holding structs
-// describing the type. Each type has a different struct so that struct can
-// be the kind. For example, if typ is the reflect type for an int8, typ is
-// a pointer to a reflect.Int8Type struct; if typ is the reflect type for a
-// function, typ is a pointer to a reflect.FuncType struct; we use the type
-// of that pointer as the kind.
+// userTypeInfo stores the information associated with a type the user has handed
+// to the package. It's computed once and stored in a map keyed by reflection
+// type.
+type userTypeInfo struct {
+ user reflect.Type // the type the user handed us
+ base reflect.Type // the base type after all indirections
+ indir int // number of indirections to reach the base type
+}
+
+var (
+ // Protected by an RWMutex because we read it a lot and write
+ // it only when we see a new type, typically when compiling.
+ userTypeLock sync.RWMutex
+ userTypeCache = make(map[reflect.Type]*userTypeInfo)
+)
+
+// validType returns, and saves, the information associated with user-provided type rt.
+// If the user type is not valid, err will be non-nil. To be used when the error handler
+// is not set up.
+func validUserType(rt reflect.Type) (ut *userTypeInfo, err os.Error) {
+ userTypeLock.RLock()
+ ut = userTypeCache[rt]
+ userTypeLock.RUnlock()
+ if ut != nil {
+ return
+ }
+ // Now set the value under the write lock.
+ userTypeLock.Lock()
+ defer userTypeLock.Unlock()
+ if ut = userTypeCache[rt]; ut != nil {
+ // Lost the race; not a problem.
+ return
+ }
+ ut = new(userTypeInfo)
+ ut.base = rt
+ ut.user = rt
+ // A type that is just a cycle of pointers (such as type T *T) cannot
+ // be represented in gobs, which need some concrete data. We use a
+ // cycle detection algorithm from Knuth, Vol 2, Section 3.1, Ex 6,
+ // pp 539-540. As we step through indirections, run another type at
+ // half speed. If they meet up, there's a cycle.
+ slowpoke := ut.base // walks half as fast as ut.base
+ for {
+ pt, ok := ut.base.(*reflect.PtrType)
+ if !ok {
+ break
+ }
+ ut.base = pt.Elem()
+ if ut.base == slowpoke { // ut.base lapped slowpoke
+ // recursive pointer type.
+ return nil, os.ErrorString("can't represent recursive pointer type " + ut.base.String())
+ }
+ if ut.indir%2 == 0 {
+ slowpoke = slowpoke.(*reflect.PtrType).Elem()
+ }
+ ut.indir++
+ }
+ userTypeCache[rt] = ut
+ return
+}
+// userType returns, and saves, the information associated with user-provided type rt.
+// If the user type is not valid, it calls error.
+func userType(rt reflect.Type) *userTypeInfo {
+ ut, err := validUserType(rt)
+ if err != nil {
+ error(err)
+ }
+ return ut
+}
// A typeId represents a gob Type as an integer that can be passed on the wire.
// Internally, typeIds are used as keys to a map to recover the underlying type info.
type typeId int32
tBool = bootstrapType("bool", false, 1)
tInt = bootstrapType("int", int(0), 2)
tUint = bootstrapType("uint", uint(0), 3)
- tFloat = bootstrapType("float", 0.0, 4)
+ tFloat = bootstrapType("float", float64(0), 4)
tBytes = bootstrapType("bytes", make([]byte, 0), 5)
tString = bootstrapType("string", "", 6)
tComplex = bootstrapType("complex", 0+0i, 7)
// Predefined because it's needed by the Decoder
var tWireType = mustGetTypeInfo(reflect.Typeof(wireType{})).id
+var wireTypeUserInfo *userTypeInfo // userTypeInfo of (*wireType)
func init() {
// Some magic numbers to make sure there are no surprises.
}
nextId = firstUserId
registerBasics()
+ wireTypeUserInfo = userType(reflect.Typeof((*wireType)(nil)))
}
// Array type
Len int
}
-func newArrayType(name string, elem gobType, length int) *arrayType {
- a := &arrayType{CommonType{Name: name}, elem.id(), length}
- setTypeId(a)
+func newArrayType(name string) *arrayType {
+ a := &arrayType{CommonType{Name: name}, 0, 0}
return a
}
+func (a *arrayType) init(elem gobType, len int) {
+ // Set our type id before evaluating the element's, in case it's our own.
+ setTypeId(a)
+ a.Elem = elem.id()
+ a.Len = len
+}
+
func (a *arrayType) safeString(seen map[typeId]bool) string {
if seen[a.Id] {
return a.Name
Elem typeId
}
-func newMapType(name string, key, elem gobType) *mapType {
- m := &mapType{CommonType{Name: name}, key.id(), elem.id()}
- setTypeId(m)
+func newMapType(name string) *mapType {
+ m := &mapType{CommonType{Name: name}, 0, 0}
return m
}
+func (m *mapType) init(key, elem gobType) {
+ // Set our type id before evaluating the element's, in case it's our own.
+ setTypeId(m)
+ m.Key = key.id()
+ m.Elem = elem.id()
+}
+
func (m *mapType) safeString(seen map[typeId]bool) string {
if seen[m.Id] {
return m.Name
Elem typeId
}
-func newSliceType(name string, elem gobType) *sliceType {
- s := &sliceType{CommonType{Name: name}, elem.id()}
- setTypeId(s)
+func newSliceType(name string) *sliceType {
+ s := &sliceType{CommonType{Name: name}, 0}
return s
}
+func (s *sliceType) init(elem gobType) {
+ // Set our type id before evaluating the element's, in case it's our own.
+ setTypeId(s)
+ s.Elem = elem.id()
+}
+
func (s *sliceType) safeString(seen map[typeId]bool) string {
if seen[s.Id] {
return s.Name
func newStructType(name string) *structType {
s := &structType{CommonType{Name: name}, nil}
+ // For historical reasons we set the id here rather than init.
+ // Se the comment in newTypeObject for details.
setTypeId(s)
return s
}
-// Step through the indirections on a type to discover the base type.
-// Return the base type and the number of indirections.
-func indirect(t reflect.Type) (rt reflect.Type, count int) {
- rt = t
- for {
- pt, ok := rt.(*reflect.PtrType)
- if !ok {
- break
- }
- rt = pt.Elem()
- count++
- }
- return
+func (s *structType) init(field []*fieldType) {
+ s.Field = field
}
func newTypeObject(name string, rt reflect.Type) (gobType, os.Error) {
+ var err os.Error
+ var type0, type1 gobType
+ defer func() {
+ if err != nil {
+ types[rt] = nil, false
+ }
+ }()
+ // Install the top-level type before the subtypes (e.g. struct before
+ // fields) so recursive types can be constructed safely.
switch t := rt.(type) {
// All basic types are easy: they are predefined.
case *reflect.BoolType:
return tInterface.gobType(), nil
case *reflect.ArrayType:
- gt, err := getType("", t.Elem())
+ at := newArrayType(name)
+ types[rt] = at
+ type0, err = getType("", t.Elem())
if err != nil {
return nil, err
}
- return newArrayType(name, gt, t.Len()), nil
+ // Historical aside:
+ // For arrays, maps, and slices, we set the type id after the elements
+ // are constructed. This is to retain the order of type id allocation after
+ // a fix made to handle recursive types, which changed the order in
+ // which types are built. Delaying the setting in this way preserves
+ // type ids while allowing recursive types to be described. Structs,
+ // done below, were already handling recursion correctly so they
+ // assign the top-level id before those of the field.
+ at.init(type0, t.Len())
+ return at, nil
case *reflect.MapType:
- kt, err := getType("", t.Key())
+ mt := newMapType(name)
+ types[rt] = mt
+ type0, err = getType("", t.Key())
if err != nil {
return nil, err
}
- vt, err := getType("", t.Elem())
+ type1, err = getType("", t.Elem())
if err != nil {
return nil, err
}
- return newMapType(name, kt, vt), nil
+ mt.init(type0, type1)
+ return mt, nil
case *reflect.SliceType:
// []byte == []uint8 is a special case
if t.Elem().Kind() == reflect.Uint8 {
return tBytes.gobType(), nil
}
- gt, err := getType(t.Elem().Name(), t.Elem())
+ st := newSliceType(name)
+ types[rt] = st
+ type0, err = getType(t.Elem().Name(), t.Elem())
if err != nil {
return nil, err
}
- return newSliceType(name, gt), nil
+ st.init(type0)
+ return st, nil
case *reflect.StructType:
- // Install the struct type itself before the fields so recursive
- // structures can be constructed safely.
- strType := newStructType(name)
- types[rt] = strType
- idToType[strType.id()] = strType
+ st := newStructType(name)
+ types[rt] = st
+ idToType[st.id()] = st
field := make([]*fieldType, t.NumField())
for i := 0; i < t.NumField(); i++ {
f := t.Field(i)
- typ, _ := indirect(f.Type)
+ typ := userType(f.Type).base
tname := typ.Name()
if tname == "" {
- t, _ := indirect(f.Type)
+ t := userType(f.Type).base
tname = t.String()
}
gt, err := getType(tname, f.Type)
}
field[i] = &fieldType{f.Name, gt.id()}
}
- strType.Field = field
- return strType, nil
+ st.init(field)
+ return st, nil
default:
return nil, os.ErrorString("gob NewTypeObject can't handle type: " + rt.String())
// getType returns the Gob type describing the given reflect.Type.
// typeLock must be held.
func getType(name string, rt reflect.Type) (gobType, os.Error) {
- rt, _ = indirect(rt)
+ rt = userType(rt).base
typ, present := types[rt]
if present {
return typ, nil
types[rt] = typ
setTypeId(typ)
checkId(expect, nextId)
+ userType(rt) // might as well cache it now
return nextId
}
// For bootstrapping purposes, we assume that the recipient knows how
// to decode a wireType; it is exactly the wireType struct here, interpreted
// using the gob rules for sending a structure, except that we assume the
-// ids for wireType and structType are known. The relevant pieces
+// ids for wireType and structType etc. are known. The relevant pieces
// are built in encode.go's init() function.
// To maintain binary compatibility, if you extend this type, always put
// the new fields last.
// reserved for nil
panic("attempt to register empty name")
}
- rt, _ := indirect(reflect.Typeof(value))
+ base := userType(reflect.Typeof(value)).base
// Check for incompatible duplicates.
- if t, ok := nameToConcreteType[name]; ok && t != rt {
+ if t, ok := nameToConcreteType[name]; ok && t != base {
panic("gob: registering duplicate types for " + name)
}
- if n, ok := concreteTypeToName[rt]; ok && n != name {
- panic("gob: registering duplicate names for " + rt.String())
+ if n, ok := concreteTypeToName[base]; ok && n != name {
+ panic("gob: registering duplicate names for " + base.String())
}
// Store the name and type provided by the user....
nameToConcreteType[name] = reflect.Typeof(value)
// but the flattened type in the type table, since that's what decode needs.
- concreteTypeToName[rt] = name
+ concreteTypeToName[base] = name
}
// Register records a type, identified by a value for that type, under its
Register(uint32(0))
Register(uint64(0))
Register(float32(0))
- Register(0.0)
+ Register(float64(0))
Register(complex64(0i))
Register(complex128(0i))
Register(false)
}
}
+A Tokenizer typically skips over HTML comments. To return comment tokens, set
+Tokenizer.ReturnComments to true before looping over calls to Next.
+
Parsing is done by calling Parse with an io.Reader, which returns the root of
the parse tree (the document element) as a *Node. It is the caller's
responsibility to ensure that the Reader provides UTF-8 encoded HTML. For
EndTagToken
// A SelfClosingTagToken tag looks like <br/>.
SelfClosingTagToken
+ // A CommentToken looks like <!--x-->.
+ CommentToken
)
// String returns a string representation of the TokenType.
return "EndTag"
case SelfClosingTagToken:
return "SelfClosingTag"
+ case CommentToken:
+ return "Comment"
}
return "Invalid(" + strconv.Itoa(int(t)) + ")"
}
}
// A Token consists of a TokenType and some Data (tag name for start and end
-// tags, content for text). A tag Token may also contain a slice of Attributes.
-// Data is unescaped for both tag and text Tokens (it looks like "a<b" rather
+// tags, content for text and comments). A tag Token may also contain a slice
+// of Attributes. Data is unescaped for all Tokens (it looks like "a<b" rather
// than "a<b").
type Token struct {
Type TokenType
return "</" + t.tagString() + ">"
case SelfClosingTagToken:
return "<" + t.tagString() + "/>"
+ case CommentToken:
+ return "<!--" + EscapeString(t.Data) + "-->"
}
return "Invalid(" + strconv.Itoa(int(t.Type)) + ")"
}
// A Tokenizer returns a stream of HTML Tokens.
type Tokenizer struct {
+ // If ReturnComments is set, Next returns comment tokens;
+ // otherwise it skips over comments (default).
+ ReturnComments bool
+
// r is the source of the HTML text.
r io.Reader
// tt is the TokenType of the most recently read token. If tt == Error
panic("unreachable")
}
+// nextMarkupDeclaration returns the next TokenType starting with "<!".
+func (z *Tokenizer) nextMarkupDeclaration() (TokenType, os.Error) {
+ // TODO: check for <!DOCTYPE ... >, don't just assume that it's a comment.
+ for i := 0; i < 2; i++ {
+ c, err := z.readByte()
+ if err != nil {
+ return TextToken, err
+ }
+ if c != '-' {
+ return z.nextText(), nil
+ }
+ }
+ // <!--> is a valid comment.
+ for dashCount := 2; ; {
+ c, err := z.readByte()
+ if err != nil {
+ return TextToken, err
+ }
+ switch c {
+ case '-':
+ dashCount++
+ case '>':
+ if dashCount >= 2 {
+ return CommentToken, nil
+ }
+ fallthrough
+ default:
+ dashCount = 0
+ }
+ }
+ panic("unreachable")
+}
+
// nextTag returns the next TokenType starting from the tag open state.
func (z *Tokenizer) nextTag() (tt TokenType, err os.Error) {
c, err := z.readByte()
case 'a' <= c && c <= 'z' || 'A' <= c && c <= 'Z':
tt = StartTagToken
case c == '!':
- return ErrorToken, os.NewError("html: TODO(nigeltao): implement comments")
+ return z.nextMarkupDeclaration()
case c == '?':
return ErrorToken, os.NewError("html: TODO(nigeltao): implement XML processing instructions")
default:
panic("unreachable")
}
-// Next scans the next token and returns its type.
-func (z *Tokenizer) Next() TokenType {
- if z.err != nil {
- z.tt = ErrorToken
- return z.tt
- }
- z.p0 = z.p1
- c, err := z.readByte()
- if err != nil {
- z.tt, z.err = ErrorToken, err
- return z.tt
- }
- if c == '<' {
- z.tt, z.err = z.nextTag()
- return z.tt
- }
+// nextText reads all text up until an '<'.
+func (z *Tokenizer) nextText() TokenType {
for {
c, err := z.readByte()
if err != nil {
panic("unreachable")
}
+// Next scans the next token and returns its type.
+func (z *Tokenizer) Next() TokenType {
+ for {
+ if z.err != nil {
+ z.tt = ErrorToken
+ return z.tt
+ }
+ z.p0 = z.p1
+ c, err := z.readByte()
+ if err != nil {
+ z.tt, z.err = ErrorToken, err
+ return z.tt
+ }
+ if c == '<' {
+ z.tt, z.err = z.nextTag()
+ if z.tt == CommentToken && !z.ReturnComments {
+ continue
+ }
+ return z.tt
+ }
+ return z.nextText()
+ }
+ panic("unreachable")
+}
+
// trim returns the largest j such that z.buf[i:j] contains only white space,
// or only white space plus the final ">" or "/>" of the raw data.
func (z *Tokenizer) trim(i int) int {
return z.buf[i0:i], z.trim(i)
}
-// Text returns the raw data after unescaping.
+// Text returns the unescaped text of a TextToken or a CommentToken.
// The contents of the returned slice may change on the next call to Next.
func (z *Tokenizer) Text() []byte {
- s := unescape(z.Raw())
- z.p0 = z.p1
- return s
+ switch z.tt {
+ case TextToken:
+ s := unescape(z.Raw())
+ z.p0 = z.p1
+ return s
+ case CommentToken:
+ // We trim the "<!--" from the left and the "-->" from the right.
+ // "<!-->" is a valid comment, so the adjusted endpoints might overlap.
+ i0 := z.p0 + 4
+ i1 := z.p1 - 3
+ z.p0 = z.p1
+ var s []byte
+ if i0 < i1 {
+ s = unescape(z.buf[i0:i1])
+ }
+ return s
+ }
+ return nil
}
// TagName returns the lower-cased name of a tag token (the `img` out of
-// `<IMG SRC="foo">`), and whether the tag has attributes.
+// `<IMG SRC="foo">`) and whether the tag has attributes.
// The contents of the returned slice may change on the next call to Next.
-func (z *Tokenizer) TagName() (name []byte, remaining bool) {
+func (z *Tokenizer) TagName() (name []byte, hasAttr bool) {
i := z.p0 + 1
if i >= z.p1 {
z.p0 = z.p1
i++
}
name, z.p0 = z.lower(i)
- remaining = z.p0 != z.p1
+ hasAttr = z.p0 != z.p1
return
}
// TagAttr returns the lower-cased key and unescaped value of the next unparsed
-// attribute for the current tag token, and whether there are more attributes.
+// attribute for the current tag token and whether there are more attributes.
// The contents of the returned slices may change on the next call to Next.
-func (z *Tokenizer) TagAttr() (key, val []byte, remaining bool) {
+func (z *Tokenizer) TagAttr() (key, val []byte, moreAttr bool) {
key, i := z.lower(z.p0)
// Get past the "=\"".
if i == z.p1 || z.buf[i] != '=' {
}
}
val, z.p0 = z.buf[i:dst], z.trim(src)
- remaining = z.p0 != z.p1
+ moreAttr = z.p0 != z.p1
return
}
func (z *Tokenizer) Token() Token {
t := Token{Type: z.tt}
switch z.tt {
- case TextToken:
+ case TextToken, CommentToken:
t.Data = string(z.Text())
case StartTagToken, EndTagToken, SelfClosingTagToken:
var attr []Attribute
- name, remaining := z.TagName()
- for remaining {
+ name, moreAttr := z.TagName()
+ for moreAttr {
var key, val []byte
- key, val, remaining = z.TagAttr()
+ key, val, moreAttr = z.TagAttr()
attr = append(attr, Attribute{string(key), string(val)})
}
t.Data = string(name)
import (
"bytes"
"os"
+ "strings"
"testing"
)
desc string
// The HTML to parse.
html string
- // The string representations of the expected tokens.
- tokens []string
+ // The string representations of the expected tokens, joined by '$'.
+ golden string
}
var tokenTests = []tokenTest{
{
"text",
"foo bar",
- []string{
- "foo bar",
- },
+ "foo bar",
},
// An entity.
{
"entity",
"one < two",
- []string{
- "one < two",
- },
+ "one < two",
},
// A start, self-closing and end tag. The tokenizer does not care if the start
// and end tokens don't match; that is the job of the parser.
{
"tags",
"<a>b<c/>d</e>",
- []string{
- "<a>",
- "b",
- "<c/>",
- "d",
- "</e>",
- },
+ "<a>$b$<c/>$d$</e>",
+ },
+ // Comments.
+ {
+ "comment0",
+ "abc<b><!-- skipme --></b>def",
+ "abc$<b>$</b>$def",
+ },
+ {
+ "comment1",
+ "a<!-->z",
+ "a$z",
+ },
+ {
+ "comment2",
+ "a<!--->z",
+ "a$z",
+ },
+ {
+ "comment3",
+ "a<!--x>-->z",
+ "a$z",
+ },
+ {
+ "comment4",
+ "a<!--x->-->z",
+ "a$z",
+ },
+ {
+ "comment5",
+ "a<!>z",
+ "a$<!>z",
+ },
+ {
+ "comment6",
+ "a<!->z",
+ "a$<!->z",
+ },
+ {
+ "comment7",
+ "a<!---<>z",
+ "a$<!---<>z",
+ },
+ {
+ "comment8",
+ "a<!--z",
+ "a$<!--z",
},
// An attribute with a backslash.
{
"backslash",
`<p id="a\"b">`,
- []string{
- `<p id="a"b">`,
- },
+ `<p id="a"b">`,
},
// Entities, tag name and attribute key lower-casing, and whitespace
// normalization within a tag.
{
"tricky",
"<p \t\n iD=\"a"B\" foo=\"bar\"><EM>te<&;xt</em></p>",
- []string{
- `<p id="a"B" foo="bar">`,
- "<em>",
- "te<&;xt",
- "</em>",
- "</p>",
- },
+ `<p id="a"B" foo="bar">$<em>$te<&;xt$</em>$</p>`,
},
// A non-existant entity. Tokenizing and converting back to a string should
// escape the "&" to become "&".
{
"noSuchEntity",
`<a b="c&noSuchEntity;d"><&alsoDoesntExist;&`,
- []string{
- `<a b="c&noSuchEntity;d">`,
- "<&alsoDoesntExist;&",
- },
+ `<a b="c&noSuchEntity;d">$<&alsoDoesntExist;&`,
},
}
loop:
for _, tt := range tokenTests {
z := NewTokenizer(bytes.NewBuffer([]byte(tt.html)))
- for i, s := range tt.tokens {
+ for i, s := range strings.Split(tt.golden, "$", -1) {
if z.Next() == ErrorToken {
t.Errorf("%s token %d: want %q got error %v", tt.desc, i, s, z.Error())
continue loop
package http
import (
- "bufio"
"bytes"
- "crypto/tls"
"encoding/base64"
"fmt"
"io"
- "net"
"os"
"strconv"
"strings"
)
+// A Client is an HTTP client. Its zero value (DefaultClient) is a usable client
+// that uses DefaultTransport.
+// Client is not yet very configurable.
+type Client struct {
+ Transport ClientTransport // if nil, DefaultTransport is used
+}
+
+// DefaultClient is the default Client and is used by Get, Head, and Post.
+var DefaultClient = &Client{}
+
+// ClientTransport is an interface representing the ability to execute a
+// single HTTP transaction, obtaining the Response for a given Request.
+type ClientTransport interface {
+ // Do executes a single HTTP transaction, returning the Response for the
+ // request req. Do should not attempt to interpret the response.
+ // In particular, Do must return err == nil if it obtained a response,
+ // regardless of the response's HTTP status code. A non-nil err should
+ // be reserved for failure to obtain a response. Similarly, Do should
+ // not attempt to handle higher-level protocol details such as redirects,
+ // authentication, or cookies.
+ //
+ // Transports may modify the request. The request Headers field is
+ // guaranteed to be initalized.
+ Do(req *Request) (resp *Response, err os.Error)
+}
+
// Given a string of the form "host", "host:port", or "[ipv6::address]:port",
// return true if the string includes a port.
func hasPort(s string) bool { return strings.LastIndex(s, ":") > strings.LastIndex(s, "]") }
io.Closer
}
-// Send issues an HTTP request. Caller should close resp.Body when done reading it.
+// matchNoProxy returns true if requests to addr should not use a proxy,
+// according to the NO_PROXY or no_proxy environment variable.
+func matchNoProxy(addr string) bool {
+ if len(addr) == 0 {
+ return false
+ }
+ no_proxy := os.Getenv("NO_PROXY")
+ if len(no_proxy) == 0 {
+ no_proxy = os.Getenv("no_proxy")
+ }
+ if no_proxy == "*" {
+ return true
+ }
+
+ addr = strings.ToLower(strings.TrimSpace(addr))
+ if hasPort(addr) {
+ addr = addr[:strings.LastIndex(addr, ":")]
+ }
+
+ for _, p := range strings.Split(no_proxy, ",", -1) {
+ p = strings.ToLower(strings.TrimSpace(p))
+ if len(p) == 0 {
+ continue
+ }
+ if hasPort(p) {
+ p = p[:strings.LastIndex(p, ":")]
+ }
+ if addr == p || (p[0] == '.' && (strings.HasSuffix(addr, p) || addr == p[1:])) {
+ return true
+ }
+ }
+ return false
+}
+
+// Do sends an HTTP request and returns an HTTP response, following
+// policy (e.g. redirects, cookies, auth) as configured on the client.
+//
+// Callers should close resp.Body when done reading from it.
+//
+// Generally Get, Post, or PostForm will be used instead of Do.
+func (c *Client) Do(req *Request) (resp *Response, err os.Error) {
+ return send(req, c.Transport)
+}
+
+
+// send issues an HTTP request. Caller should close resp.Body when done reading from it.
//
// TODO: support persistent connections (multiple requests on a single connection).
// send() method is nonpublic because, when we refactor the code for persistent
// connections, it may no longer make sense to have a method with this signature.
-func send(req *Request) (resp *Response, err os.Error) {
- if req.URL.Scheme != "http" && req.URL.Scheme != "https" {
- return nil, &badStringError{"unsupported protocol scheme", req.URL.Scheme}
+func send(req *Request, t ClientTransport) (resp *Response, err os.Error) {
+ if t == nil {
+ t = DefaultTransport
+ if t == nil {
+ err = os.NewError("no http.Client.Transport or http.DefaultTransport")
+ return
+ }
}
- addr := req.URL.Host
- if !hasPort(addr) {
- addr += ":" + req.URL.Scheme
+ // Most the callers of send (Get, Post, et al) don't need
+ // Headers, leaving it uninitialized. We guarantee to the
+ // ClientTransport that this has been initialized, though.
+ if req.Header == nil {
+ req.Header = Header(make(map[string][]string))
}
+
info := req.URL.RawUserinfo
if len(info) > 0 {
enc := base64.URLEncoding
encoded := make([]byte, enc.EncodedLen(len(info)))
enc.Encode(encoded, []byte(info))
if req.Header == nil {
- req.Header = make(map[string]string)
+ req.Header = make(Header)
}
- req.Header["Authorization"] = "Basic " + string(encoded)
- }
-
- var conn io.ReadWriteCloser
- if req.URL.Scheme == "http" {
- conn, err = net.Dial("tcp", "", addr)
- if err != nil {
- return nil, err
- }
- } else { // https
- conn, err = tls.Dial("tcp", "", addr, nil)
- if err != nil {
- return nil, err
- }
- h := req.URL.Host
- if hasPort(h) {
- h = h[0:strings.LastIndex(h, ":")]
- }
- if err := conn.(*tls.Conn).VerifyHostname(h); err != nil {
- return nil, err
- }
- }
-
- err = req.Write(conn)
- if err != nil {
- conn.Close()
- return nil, err
+ req.Header.Set("Authorization", "Basic "+string(encoded))
}
-
- reader := bufio.NewReader(conn)
- resp, err = ReadResponse(reader, req.Method)
- if err != nil {
- conn.Close()
- return nil, err
- }
-
- resp.Body = readClose{resp.Body, conn}
-
- return
+ return t.Do(req)
}
// True if the specified HTTP status code is one for which the Get utility should
// finalURL is the URL from which the response was fetched -- identical to the
// input URL unless redirects were followed.
//
-// Caller should close r.Body when done reading it.
+// Caller should close r.Body when done reading from it.
+//
+// Get is a convenience wrapper around DefaultClient.Get.
func Get(url string) (r *Response, finalURL string, err os.Error) {
+ return DefaultClient.Get(url)
+}
+
+// Get issues a GET to the specified URL. If the response is one of the following
+// redirect codes, it follows the redirect, up to a maximum of 10 redirects:
+//
+// 301 (Moved Permanently)
+// 302 (Found)
+// 303 (See Other)
+// 307 (Temporary Redirect)
+//
+// finalURL is the URL from which the response was fetched -- identical to the
+// input URL unless redirects were followed.
+//
+// Caller should close r.Body when done reading from it.
+func (c *Client) Get(url string) (r *Response, finalURL string, err os.Error) {
// TODO: if/when we add cookie support, the redirected request shouldn't
// necessarily supply the same cookies as the original.
// TODO: set referrer header on redirects.
var base *URL
+ // TODO: remove this hard-coded 10 and use the Client's policy
+ // (ClientConfig) instead.
for redirect := 0; ; redirect++ {
if redirect >= 10 {
err = os.ErrorString("stopped after 10 redirects")
}
var req Request
+ req.Method = "GET"
+ req.ProtoMajor = 1
+ req.ProtoMinor = 1
if base == nil {
req.URL, err = ParseURL(url)
} else {
break
}
url = req.URL.String()
- if r, err = send(&req); err != nil {
+ if r, err = send(&req, c.Transport); err != nil {
break
}
if shouldRedirect(r.StatusCode) {
r.Body.Close()
- if url = r.GetHeader("Location"); url == "" {
+ if url = r.Header.Get("Location"); url == "" {
err = os.ErrorString(fmt.Sprintf("%d response missing Location header", r.StatusCode))
break
}
// Post issues a POST to the specified URL.
//
-// Caller should close r.Body when done reading it.
+// Caller should close r.Body when done reading from it.
+//
+// Post is a wrapper around DefaultClient.Post
func Post(url string, bodyType string, body io.Reader) (r *Response, err os.Error) {
+ return DefaultClient.Post(url, bodyType, body)
+}
+
+// Post issues a POST to the specified URL.
+//
+// Caller should close r.Body when done reading from it.
+func (c *Client) Post(url string, bodyType string, body io.Reader) (r *Response, err os.Error) {
var req Request
req.Method = "POST"
req.ProtoMajor = 1
req.ProtoMinor = 1
req.Close = true
req.Body = nopCloser{body}
- req.Header = map[string]string{
- "Content-Type": bodyType,
+ req.Header = Header{
+ "Content-Type": {bodyType},
}
req.TransferEncoding = []string{"chunked"}
return nil, err
}
- return send(&req)
+ return send(&req, c.Transport)
}
// PostForm issues a POST to the specified URL,
// with data's keys and values urlencoded as the request body.
//
-// Caller should close r.Body when done reading it.
+// Caller should close r.Body when done reading from it.
+//
+// PostForm is a wrapper around DefaultClient.PostForm
func PostForm(url string, data map[string]string) (r *Response, err os.Error) {
+ return DefaultClient.PostForm(url, data)
+}
+
+// PostForm issues a POST to the specified URL,
+// with data's keys and values urlencoded as the request body.
+//
+// Caller should close r.Body when done reading from it.
+func (c *Client) PostForm(url string, data map[string]string) (r *Response, err os.Error) {
var req Request
req.Method = "POST"
req.ProtoMajor = 1
req.Close = true
body := urlencode(data)
req.Body = nopCloser{body}
- req.Header = map[string]string{
- "Content-Type": "application/x-www-form-urlencoded",
- "Content-Length": strconv.Itoa(body.Len()),
+ req.Header = Header{
+ "Content-Type": {"application/x-www-form-urlencoded"},
+ "Content-Length": {strconv.Itoa(body.Len())},
}
req.ContentLength = int64(body.Len())
return nil, err
}
- return send(&req)
+ return send(&req, c.Transport)
}
// TODO: remove this function when PostForm takes a multimap.
}
// Head issues a HEAD to the specified URL.
+//
+// Head is a wrapper around DefaultClient.Head
func Head(url string) (r *Response, err os.Error) {
+ return DefaultClient.Head(url)
+}
+
+// Head issues a HEAD to the specified URL.
+func (c *Client) Head(url string) (r *Response, err os.Error) {
var req Request
req.Method = "HEAD"
if req.URL, err = ParseURL(url); err != nil {
return
}
- url = req.URL.String()
- if r, err = send(&req); err != nil {
- return
- }
- return
+ return send(&req, c.Transport)
}
type nopCloser struct {
import (
"io/ioutil"
+ "os"
"strings"
"testing"
)
t.Error("Last-Modified header not found.")
}
}
+
+type recordingTransport struct {
+ req *Request
+}
+
+func (t *recordingTransport) Do(req *Request) (resp *Response, err os.Error) {
+ t.req = req
+ return nil, os.NewError("dummy impl")
+}
+
+func TestGetRequestFormat(t *testing.T) {
+ tr := &recordingTransport{}
+ client := &Client{Transport: tr}
+ url := "http://dummy.faketld/"
+ client.Get(url) // Note: doesn't hit network
+ if tr.req.Method != "GET" {
+ t.Errorf("expected method %q; got %q", "GET", tr.req.Method)
+ }
+ if tr.req.URL.String() != url {
+ t.Errorf("expected URL %q; got %q", url, tr.req.URL.String())
+ }
+ if tr.req.Header == nil {
+ t.Errorf("expected non-nil request Header")
+ }
+}
}
}
- if t, _ := time.Parse(TimeFormat, r.Header["If-Modified-Since"]); t != nil && d.Mtime_ns/1e9 <= t.Seconds() {
+ if t, _ := time.Parse(TimeFormat, r.Header.Get("If-Modified-Since")); t != nil && d.Mtime_ns/1e9 <= t.Seconds() {
w.WriteHeader(StatusNotModified)
return
}
// handle Content-Range header.
// TODO(adg): handle multiple ranges
- ranges, err := parseRange(r.Header["Range"], size)
+ ranges, err := parseRange(r.Header.Get("Range"), size)
if err != nil || len(ranges) > 1 {
Error(w, err.String(), StatusRequestedRangeNotSatisfiable)
return
// set up the Request (re-used for all tests)
var req Request
- req.Header = make(map[string]string)
+ req.Header = make(Header)
if req.URL, err = ParseURL("http://" + serverAddr + "/ServeFile"); err != nil {
t.Fatal("ParseURL:", err)
}
// Range tests
for _, rt := range ServeFileRangeTests {
- req.Header["Range"] = "bytes=" + rt.r
+ req.Header.Set("Range", "bytes="+rt.r)
if rt.r == "" {
- req.Header["Range"] = ""
+ req.Header["Range"] = nil
}
r, body := getBody(t, req)
if r.StatusCode != rt.code {
if rt.r == "" {
h = ""
}
- if r.Header["Content-Range"] != h {
- t.Errorf("header mismatch: range=%q: got %q, want %q", rt.r, r.Header["Content-Range"], h)
+ cr := r.Header.Get("Content-Range")
+ if cr != h {
+ t.Errorf("header mismatch: range=%q: got %q, want %q", rt.r, cr, h)
}
if !equal(body, file[rt.start:rt.end]) {
t.Errorf("body mismatch: range=%q: got %q, want %q", rt.r, body, file[rt.start:rt.end])
}
func getBody(t *testing.T, req Request) (*Response, []byte) {
- r, err := send(&req)
+ r, err := send(&req, DefaultTransport)
if err != nil {
t.Fatal(req.URL.String(), "send:", err)
}
--- /dev/null
+// Copyright 2010 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package http
+
+import "net/textproto"
+
+// A Header represents the key-value pairs in an HTTP header.
+type Header map[string][]string
+
+// Add adds the key, value pair to the header.
+// It appends to any existing values associated with key.
+func (h Header) Add(key, value string) {
+ textproto.MIMEHeader(h).Add(key, value)
+}
+
+// Set sets the header entries associated with key to
+// the single element value. It replaces any existing
+// values associated with key.
+func (h Header) Set(key, value string) {
+ textproto.MIMEHeader(h).Set(key, value)
+}
+
+// Get gets the first value associated with the given key.
+// If there are no values associated with the key, Get returns "".
+// Get is a convenience method. For more complex queries,
+// access the map directly.
+func (h Header) Get(key string) string {
+ return textproto.MIMEHeader(h).Get(key)
+}
+
+// Del deletes the values associated with key.
+func (h Header) Del(key string) {
+ textproto.MIMEHeader(h).Del(key)
+}
+
+// CanonicalHeaderKey returns the canonical format of the
+// header key s. The canonicalization converts the first
+// letter and any letter following a hyphen to upper case;
+// the rest are converted to lowercase. For example, the
+// canonical key for "accept-encoding" is "Accept-Encoding".
+func CanonicalHeaderKey(s string) string { return textproto.CanonicalMIMEHeaderKey(s) }
import (
"bufio"
- "container/list"
"io"
"net"
+ "net/textproto"
"os"
"sync"
)
-var ErrPersistEOF = &ProtocolError{"persistent connection closed"}
+var (
+ ErrPersistEOF = &ProtocolError{"persistent connection closed"}
+ ErrPipeline = &ProtocolError{"pipeline error"}
+)
// A ServerConn reads requests and sends responses over an underlying
// connection, until the HTTP keepalive logic commands an end. ServerConn
r *bufio.Reader
clsd bool // indicates a graceful close
re, we os.Error // read/write errors
- lastBody io.ReadCloser
+ lastbody io.ReadCloser
nread, nwritten int
+ pipe textproto.Pipeline
+ pipereq map[*Request]uint
lk sync.Mutex // protected read/write to re,we
}
if r == nil {
r = bufio.NewReader(c)
}
- return &ServerConn{c: c, r: r}
+ return &ServerConn{c: c, r: r, pipereq: make(map[*Request]uint)}
}
// Close detaches the ServerConn and returns the underlying connection as well
// Read returns the next request on the wire. An ErrPersistEOF is returned if
// it is gracefully determined that there are no more requests (e.g. after the
// first request on an HTTP/1.0 connection, or after a Connection:close on a
-// HTTP/1.1 connection). Read can be called concurrently with Write, but not
-// with another Read.
+// HTTP/1.1 connection).
func (sc *ServerConn) Read() (req *Request, err os.Error) {
+ // Ensure ordered execution of Reads and Writes
+ id := sc.pipe.Next()
+ sc.pipe.StartRequest(id)
+ defer func() {
+ sc.pipe.EndRequest(id)
+ if req == nil {
+ sc.pipe.StartResponse(id)
+ sc.pipe.EndResponse(id)
+ } else {
+ // Remember the pipeline id of this request
+ sc.lk.Lock()
+ sc.pipereq[req] = id
+ sc.lk.Unlock()
+ }
+ }()
+
sc.lk.Lock()
if sc.we != nil { // no point receiving if write-side broken or closed
defer sc.lk.Unlock()
sc.lk.Unlock()
// Make sure body is fully consumed, even if user does not call body.Close
- if sc.lastBody != nil {
+ if sc.lastbody != nil {
// body.Close is assumed to be idempotent and multiple calls to
// it should return the error that its first invokation
// returned.
- err = sc.lastBody.Close()
- sc.lastBody = nil
+ err = sc.lastbody.Close()
+ sc.lastbody = nil
if err != nil {
sc.lk.Lock()
defer sc.lk.Unlock()
return
}
}
- sc.lastBody = req.Body
+ sc.lastbody = req.Body
sc.nread++
if req.Close {
sc.lk.Lock()
return sc.nread - sc.nwritten
}
-// Write writes a repsonse. To close the connection gracefully, set the
+// Write writes resp in response to req. To close the connection gracefully, set the
// Response.Close field to true. Write should be considered operational until
// it returns an error, regardless of any errors returned on the Read side.
-// Write can be called concurrently with Read, but not with another Write.
-func (sc *ServerConn) Write(resp *Response) os.Error {
+func (sc *ServerConn) Write(req *Request, resp *Response) os.Error {
+
+ // Retrieve the pipeline ID of this request/response pair
+ sc.lk.Lock()
+ id, ok := sc.pipereq[req]
+ sc.pipereq[req] = 0, false
+ if !ok {
+ sc.lk.Unlock()
+ return ErrPipeline
+ }
+ sc.lk.Unlock()
+
+ // Ensure pipeline order
+ sc.pipe.StartResponse(id)
+ defer sc.pipe.EndResponse(id)
sc.lk.Lock()
if sc.we != nil {
c net.Conn
r *bufio.Reader
re, we os.Error // read/write errors
- lastBody io.ReadCloser
+ lastbody io.ReadCloser
nread, nwritten int
- reqm list.List // request methods in order of execution
- lk sync.Mutex // protects read/write to reqm,re,we
+ pipe textproto.Pipeline
+ pipereq map[*Request]uint
+ lk sync.Mutex // protects read/write to re,we,pipereq,etc.
}
// NewClientConn returns a new ClientConn reading and writing c. If r is not
if r == nil {
r = bufio.NewReader(c)
}
- return &ClientConn{c: c, r: r}
+ return &ClientConn{c: c, r: r, pipereq: make(map[*Request]uint)}
}
// Close detaches the ClientConn and returns the underlying connection as well
r = cc.r
cc.c = nil
cc.r = nil
- cc.reqm.Init()
cc.lk.Unlock()
return
}
// keepalive connection is logically closed after this request and the opposing
// server is informed. An ErrUnexpectedEOF indicates the remote closed the
// underlying TCP connection, which is usually considered as graceful close.
-// Write can be called concurrently with Read, but not with another Write.
-func (cc *ClientConn) Write(req *Request) os.Error {
+func (cc *ClientConn) Write(req *Request) (err os.Error) {
+
+ // Ensure ordered execution of Writes
+ id := cc.pipe.Next()
+ cc.pipe.StartRequest(id)
+ defer func() {
+ cc.pipe.EndRequest(id)
+ if err != nil {
+ cc.pipe.StartResponse(id)
+ cc.pipe.EndResponse(id)
+ } else {
+ // Remember the pipeline id of this request
+ cc.lk.Lock()
+ cc.pipereq[req] = id
+ cc.lk.Unlock()
+ }
+ }()
cc.lk.Lock()
if cc.re != nil { // no point sending if read-side closed or broken
cc.lk.Unlock()
}
- err := req.Write(cc.c)
+ err = req.Write(cc.c)
if err != nil {
cc.lk.Lock()
defer cc.lk.Unlock()
return err
}
cc.nwritten++
- cc.lk.Lock()
- cc.reqm.PushBack(req.Method)
- cc.lk.Unlock()
return nil
}
// returned together with an ErrPersistEOF, which means that the remote
// requested that this be the last request serviced. Read can be called
// concurrently with Write, but not with another Read.
-func (cc *ClientConn) Read() (resp *Response, err os.Error) {
+func (cc *ClientConn) Read(req *Request) (resp *Response, err os.Error) {
+
+ // Retrieve the pipeline ID of this request/response pair
+ cc.lk.Lock()
+ id, ok := cc.pipereq[req]
+ cc.pipereq[req] = 0, false
+ if !ok {
+ cc.lk.Unlock()
+ return nil, ErrPipeline
+ }
+ cc.lk.Unlock()
+
+ // Ensure pipeline order
+ cc.pipe.StartResponse(id)
+ defer cc.pipe.EndResponse(id)
cc.lk.Lock()
if cc.re != nil {
}
cc.lk.Unlock()
- if cc.nread >= cc.nwritten {
- return nil, os.NewError("persist client pipe count")
- }
-
// Make sure body is fully consumed, even if user does not call body.Close
- if cc.lastBody != nil {
+ if cc.lastbody != nil {
// body.Close is assumed to be idempotent and multiple calls to
// it should return the error that its first invokation
// returned.
- err = cc.lastBody.Close()
- cc.lastBody = nil
+ err = cc.lastbody.Close()
+ cc.lastbody = nil
if err != nil {
cc.lk.Lock()
defer cc.lk.Unlock()
}
}
- cc.lk.Lock()
- m := cc.reqm.Front()
- cc.reqm.Remove(m)
- cc.lk.Unlock()
- resp, err = ReadResponse(cc.r, m.Value.(string))
+ resp, err = ReadResponse(cc.r, req.Method)
if err != nil {
cc.lk.Lock()
defer cc.lk.Unlock()
cc.re = err
return
}
- cc.lastBody = resp.Body
+ cc.lastbody = resp.Body
cc.nread++
}
return
}
+
+// Do is convenience method that writes a request and reads a response.
+func (cc *ClientConn) Do(req *Request) (resp *Response, err os.Error) {
+ err = cc.Write(req)
+ if err != nil {
+ return
+ }
+ return cc.Read(req)
+}
--- /dev/null
+// Copyright 2009 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package http
+
+import (
+ "os"
+ "testing"
+)
+
+// TODO(mattn):
+// test ProxyAuth
+
+var MatchNoProxyTests = []struct {
+ host string
+ match bool
+}{
+ {"localhost", true}, // match completely
+ {"barbaz.net", true}, // match as .barbaz.net
+ {"foobar.com:443", true}, // have a port but match
+ {"foofoobar.com", false}, // not match as a part of foobar.com
+ {"baz.com", false}, // not match as a part of barbaz.com
+ {"localhost.net", false}, // not match as suffix of address
+ {"local.localhost", false}, // not match as prefix as address
+ {"barbarbaz.net", false}, // not match because NO_PROXY have a '.'
+ {"www.foobar.com", false}, // not match because NO_PROXY is not .foobar.com
+}
+
+func TestMatchNoProxy(t *testing.T) {
+ oldenv := os.Getenv("NO_PROXY")
+ no_proxy := "foobar.com, .barbaz.net , localhost"
+ os.Setenv("NO_PROXY", no_proxy)
+ defer os.Setenv("NO_PROXY", oldenv)
+
+ for _, test := range MatchNoProxyTests {
+ if matchNoProxy(test.host) != test.match {
+ if test.match {
+ t.Errorf("matchNoProxy(%v) = %v, want %v", test.host, !test.match, test.match)
+ } else {
+ t.Errorf("not expected: '%s' shouldn't match as '%s'", test.host, no_proxy)
+ }
+ }
+ }
+}
Proto: "HTTP/1.1",
ProtoMajor: 1,
ProtoMinor: 1,
- Header: map[string]string{
- "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
- "Accept-Language": "en-us,en;q=0.5",
- "Accept-Encoding": "gzip,deflate",
- "Accept-Charset": "ISO-8859-1,utf-8;q=0.7,*;q=0.7",
- "Keep-Alive": "300",
- "Proxy-Connection": "keep-alive",
- "Content-Length": "7",
+ Header: Header{
+ "Accept": {"text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"},
+ "Accept-Language": {"en-us,en;q=0.5"},
+ "Accept-Encoding": {"gzip,deflate"},
+ "Accept-Charset": {"ISO-8859-1,utf-8;q=0.7,*;q=0.7"},
+ "Keep-Alive": {"300"},
+ "Proxy-Connection": {"keep-alive"},
+ "Content-Length": {"7"},
},
Close: false,
ContentLength: 7,
Proto: "HTTP/1.1",
ProtoMajor: 1,
ProtoMinor: 1,
- Header: map[string]string{},
+ Header: map[string][]string{},
Close: false,
ContentLength: -1,
Host: "test",
import (
"bufio"
- "bytes"
"container/vector"
"fmt"
"io"
"io/ioutil"
"mime"
"mime/multipart"
+ "net/textproto"
"os"
"strconv"
"strings"
// The request parser implements this by canonicalizing the
// name, making the first character and any characters
// following a hyphen uppercase and the rest lowercase.
- Header map[string]string
+ Header Header
// The message body.
Body io.ReadCloser
// Trailer maps trailer keys to values. Like for Header, if the
// response has multiple trailer lines with the same key, they will be
// concatenated, delimited by commas.
- Trailer map[string]string
+ Trailer Header
}
// ProtoAtLeast returns whether the HTTP protocol used
// MultipartReader returns a MIME multipart reader if this is a
// multipart/form-data POST request, else returns nil and an error.
func (r *Request) MultipartReader() (multipart.Reader, os.Error) {
- v, ok := r.Header["Content-Type"]
- if !ok {
+ v := r.Header.Get("Content-Type")
+ if v == "" {
return nil, ErrNotMultipart
}
d, params := mime.ParseMediaType(v)
// If Body is present, Write forces "Transfer-Encoding: chunked" as a header
// and then closes Body when finished sending it.
func (req *Request) Write(w io.Writer) os.Error {
+ return req.write(w, false)
+}
+
+// WriteProxy is like Write but writes the request in the form
+// expected by an HTTP proxy. It includes the scheme and host
+// name in the URI instead of using a separate Host: header line.
+func (req *Request) WriteProxy(w io.Writer) os.Error {
+ return req.write(w, true)
+}
+
+func (req *Request) write(w io.Writer, usingProxy bool) os.Error {
host := req.Host
if host == "" {
host = req.URL.Host
}
}
+ if usingProxy {
+ if uri == "" || uri[0] != '/' {
+ uri = "/" + uri
+ }
+ uri = req.URL.Scheme + "://" + host + uri
+ }
+
fmt.Fprintf(w, "%s %s HTTP/1.1\r\n", valueOrDefault(req.Method, "GET"), uri)
// Header lines
- fmt.Fprintf(w, "Host: %s\r\n", host)
+ if !usingProxy {
+ fmt.Fprintf(w, "Host: %s\r\n", host)
+ }
fmt.Fprintf(w, "User-Agent: %s\r\n", valueOrDefault(req.UserAgent, defaultUserAgent))
if req.Referer != "" {
fmt.Fprintf(w, "Referer: %s\r\n", req.Referer)
return string(p), nil
}
-var colon = []byte{':'}
-
-// Read a key/value pair from b.
-// A key/value has the form Key: Value\r\n
-// and the Value can continue on multiple lines if each continuation line
-// starts with a space.
-func readKeyValue(b *bufio.Reader) (key, value string, err os.Error) {
- line, e := readLineBytes(b)
- if e != nil {
- return "", "", e
- }
- if len(line) == 0 {
- return "", "", nil
- }
-
- // Scan first line for colon.
- i := bytes.Index(line, colon)
- if i < 0 {
- goto Malformed
- }
-
- key = string(line[0:i])
- if strings.Contains(key, " ") {
- // Key field has space - no good.
- goto Malformed
- }
-
- // Skip initial space before value.
- for i++; i < len(line); i++ {
- if line[i] != ' ' {
- break
- }
- }
- value = string(line[i:])
-
- // Look for extension lines, which must begin with space.
- for {
- c, e := b.ReadByte()
- if c != ' ' {
- if e != os.EOF {
- b.UnreadByte()
- }
- break
- }
-
- // Eat leading space.
- for c == ' ' {
- if c, e = b.ReadByte(); e != nil {
- if e == os.EOF {
- e = io.ErrUnexpectedEOF
- }
- return "", "", e
- }
- }
- b.UnreadByte()
-
- // Read the rest of the line and add to value.
- if line, e = readLineBytes(b); e != nil {
- return "", "", e
- }
- value += " " + string(line)
-
- if len(value) >= maxValueLength {
- return "", "", &badStringError{"value too long for key", key}
- }
- }
- return key, value, nil
-
-Malformed:
- return "", "", &badStringError{"malformed header line", string(line)}
-}
-
// Convert decimal at s[i:len(s)] to integer,
// returning value, string position where the digits stopped,
// and whether there was a valid number (digits, not too big).
return n, i, true
}
-// Parse HTTP version: "HTTP/1.2" -> (1, 2, true).
-func parseHTTPVersion(vers string) (int, int, bool) {
+// ParseHTTPVersion parses a HTTP version string.
+// "HTTP/1.0" returns (1, 0, true).
+func ParseHTTPVersion(vers string) (major, minor int, ok bool) {
if len(vers) < 5 || vers[0:5] != "HTTP/" {
return 0, 0, false
}
if !ok || i >= len(vers) || vers[i] != '.' {
return 0, 0, false
}
- var minor int
minor, i, ok = atoi(vers, i+1)
if !ok || i != len(vers) {
return 0, 0, false
return major, minor, true
}
-// CanonicalHeaderKey returns the canonical format of the
-// HTTP header key s. The canonicalization converts the first
-// letter and any letter following a hyphen to upper case;
-// the rest are converted to lowercase. For example, the
-// canonical key for "accept-encoding" is "Accept-Encoding".
-func CanonicalHeaderKey(s string) string {
- // canonicalize: first letter upper case
- // and upper case after each dash.
- // (Host, User-Agent, If-Modified-Since).
- // HTTP headers are ASCII only, so no Unicode issues.
- var a []byte
- upper := true
- for i := 0; i < len(s); i++ {
- v := s[i]
- if upper && 'a' <= v && v <= 'z' {
- if a == nil {
- a = []byte(s)
- }
- a[i] = v + 'A' - 'a'
- }
- if !upper && 'A' <= v && v <= 'Z' {
- if a == nil {
- a = []byte(s)
- }
- a[i] = v + 'a' - 'A'
- }
- upper = false
- if v == '-' {
- upper = true
- }
- }
- if a != nil {
- return string(a)
- }
- return s
-}
-
type chunkedReader struct {
r *bufio.Reader
n uint64 // unread bytes in chunk
// ReadRequest reads and parses a request from b.
func ReadRequest(b *bufio.Reader) (req *Request, err os.Error) {
+
+ tp := textproto.NewReader(b)
req = new(Request)
// First line: GET /index.html HTTP/1.0
var s string
- if s, err = readLine(b); err != nil {
+ if s, err = tp.ReadLine(); err != nil {
+ if err == os.EOF {
+ err = io.ErrUnexpectedEOF
+ }
return nil, err
}
}
req.Method, req.RawURL, req.Proto = f[0], f[1], f[2]
var ok bool
- if req.ProtoMajor, req.ProtoMinor, ok = parseHTTPVersion(req.Proto); !ok {
+ if req.ProtoMajor, req.ProtoMinor, ok = ParseHTTPVersion(req.Proto); !ok {
return nil, &badStringError{"malformed HTTP version", req.Proto}
}
}
// Subsequent lines: Key: value.
- nheader := 0
- req.Header = make(map[string]string)
- for {
- var key, value string
- if key, value, err = readKeyValue(b); err != nil {
- return nil, err
- }
- if key == "" {
- break
- }
- if nheader++; nheader >= maxHeaderLines {
- return nil, ErrHeaderTooLong
- }
-
- key = CanonicalHeaderKey(key)
-
- // RFC 2616 says that if you send the same header key
- // multiple times, it has to be semantically equivalent
- // to concatenating the values separated by commas.
- oldvalue, present := req.Header[key]
- if present {
- req.Header[key] = oldvalue + "," + value
- } else {
- req.Header[key] = value
- }
+ mimeHeader, err := tp.ReadMIMEHeader()
+ if err != nil {
+ return nil, err
}
+ req.Header = Header(mimeHeader)
// RFC2616: Must treat
// GET /index.html HTTP/1.1
// the same. In the second case, any Host line is ignored.
req.Host = req.URL.Host
if req.Host == "" {
- req.Host = req.Header["Host"]
+ req.Host = req.Header.Get("Host")
}
- req.Header["Host"] = "", false
+ req.Header.Del("Host")
fixPragmaCacheControl(req.Header)
// Pull out useful fields as a convenience to clients.
- req.Referer = req.Header["Referer"]
- req.Header["Referer"] = "", false
+ req.Referer = req.Header.Get("Referer")
+ req.Header.Del("Referer")
- req.UserAgent = req.Header["User-Agent"]
- req.Header["User-Agent"] = "", false
+ req.UserAgent = req.Header.Get("User-Agent")
+ req.Header.Del("User-Agent")
// TODO: Parse specific header values:
// Accept
if r.Body == nil {
return os.ErrorString("missing form body")
}
- ct := r.Header["Content-Type"]
+ ct := r.Header.Get("Content-Type")
switch strings.Split(ct, ";", 2)[0] {
case "text/plain", "application/x-www-form-urlencoded", "":
b, e := ioutil.ReadAll(r.Body)
}
func (r *Request) expectsContinue() bool {
- expectation, ok := r.Header["Expect"]
- return ok && strings.ToLower(expectation) == "100-continue"
+ return strings.ToLower(r.Header.Get("Expect")) == "100-continue"
}
func (r *Request) wantsHttp10KeepAlive() bool {
if r.ProtoMajor != 1 || r.ProtoMinor != 0 {
return false
}
- value, exists := r.Header["Connection"]
- if !exists {
- return false
- }
- return strings.Contains(strings.ToLower(value), "keep-alive")
+ return strings.Contains(strings.ToLower(r.Header.Get("Connection")), "keep-alive")
}
func TestPostQuery(t *testing.T) {
req := &Request{Method: "POST"}
req.URL, _ = ParseURL("http://www.google.com/search?q=foo&q=bar&both=x")
- req.Header = map[string]string{"Content-Type": "application/x-www-form-urlencoded; boo!"}
+ req.Header = Header{
+ "Content-Type": {"application/x-www-form-urlencoded; boo!"},
+ }
req.Body = nopCloser{strings.NewReader("z=post&both=y")}
if q := req.FormValue("q"); q != "foo" {
t.Errorf(`req.FormValue("q") = %q, want "foo"`, q)
}
}
-type stringMap map[string]string
+type stringMap map[string][]string
type parseContentTypeTest struct {
contentType stringMap
error bool
}
var parseContentTypeTests = []parseContentTypeTest{
- {contentType: stringMap{"Content-Type": "text/plain"}},
- {contentType: stringMap{"Content-Type": ""}},
- {contentType: stringMap{"Content-Type": "text/plain; boundary="}},
+ {contentType: stringMap{"Content-Type": {"text/plain"}}},
+ {contentType: stringMap{}}, // Non-existent keys are not placed. The value nil is illegal.
+ {contentType: stringMap{"Content-Type": {"text/plain; boundary="}}},
{
- contentType: stringMap{"Content-Type": "application/unknown"},
+ contentType: stringMap{"Content-Type": {"application/unknown"}},
error: true,
},
}
for i, test := range parseContentTypeTests {
req := &Request{
Method: "POST",
- Header: test.contentType,
+ Header: Header(test.contentType),
Body: nopCloser{bytes.NewBufferString("body")},
}
err := req.ParseForm()
func TestMultipartReader(t *testing.T) {
req := &Request{
Method: "POST",
- Header: stringMap{"Content-Type": `multipart/form-data; boundary="foo123"`},
+ Header: Header{"Content-Type": {`multipart/form-data; boundary="foo123"`}},
Body: nopCloser{new(bytes.Buffer)},
}
multipart, err := req.MultipartReader()
t.Errorf("expected multipart; error: %v", err)
}
- req.Header = stringMap{"Content-Type": "text/plain"}
+ req.Header = Header{"Content-Type": {"text/plain"}}
multipart, err = req.MultipartReader()
if multipart != nil {
t.Errorf("unexpected multipart for text/plain")
Proto: "HTTP/1.1",
ProtoMajor: 1,
ProtoMinor: 1,
- Header: map[string]string{
- "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
- "Accept-Charset": "ISO-8859-1,utf-8;q=0.7,*;q=0.7",
- "Accept-Encoding": "gzip,deflate",
- "Accept-Language": "en-us,en;q=0.5",
- "Keep-Alive": "300",
- "Proxy-Connection": "keep-alive",
+ Header: Header{
+ "Accept": {"text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"},
+ "Accept-Charset": {"ISO-8859-1,utf-8;q=0.7,*;q=0.7"},
+ "Accept-Encoding": {"gzip,deflate"},
+ "Accept-Language": {"en-us,en;q=0.5"},
+ "Keep-Alive": {"300"},
+ "Proxy-Connection": {"keep-alive"},
},
Body: nil,
Close: false,
"GET http://www.techcrunch.com/ HTTP/1.1\r\n" +
"Host: www.techcrunch.com\r\n" +
"User-Agent: Fake\r\n" +
+ "Accept: text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8\r\n" +
"Accept-Charset: ISO-8859-1,utf-8;q=0.7,*;q=0.7\r\n" +
"Accept-Encoding: gzip,deflate\r\n" +
"Accept-Language: en-us,en;q=0.5\r\n" +
- "Accept: text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8\r\n" +
"Keep-Alive: 300\r\n" +
"Proxy-Connection: keep-alive\r\n\r\n",
},
},
ProtoMajor: 1,
ProtoMinor: 1,
- Header: map[string]string{},
+ Header: map[string][]string{},
Body: nopCloser{bytes.NewBufferString("abcdef")},
TransferEncoding: []string{"chunked"},
},
},
ProtoMajor: 1,
ProtoMinor: 1,
- Header: map[string]string{},
+ Header: map[string][]string{},
Close: true,
Body: nopCloser{bytes.NewBufferString("abcdef")},
TransferEncoding: []string{"chunked"},
"bufio"
"fmt"
"io"
+ "net/textproto"
"os"
"sort"
"strconv"
// omitted from Header.
//
// Keys in the map are canonicalized (see CanonicalHeaderKey).
- Header map[string]string
+ Header Header
// Body represents the response body.
Body io.ReadCloser
// Trailer maps trailer keys to values. Like for Header, if the
// response has multiple trailer lines with the same key, they will be
// concatenated, delimited by commas.
- Trailer map[string]string
+ Trailer map[string][]string
}
// ReadResponse reads and returns an HTTP response from r. The RequestMethod
// key/value pairs included in the response trailer.
func ReadResponse(r *bufio.Reader, requestMethod string) (resp *Response, err os.Error) {
+ tp := textproto.NewReader(r)
resp = new(Response)
resp.RequestMethod = strings.ToUpper(requestMethod)
// Parse the first line of the response.
- line, err := readLine(r)
+ line, err := tp.ReadLine()
if err != nil {
+ if err == os.EOF {
+ err = io.ErrUnexpectedEOF
+ }
return nil, err
}
f := strings.Split(line, " ", 3)
resp.Proto = f[0]
var ok bool
- if resp.ProtoMajor, resp.ProtoMinor, ok = parseHTTPVersion(resp.Proto); !ok {
+ if resp.ProtoMajor, resp.ProtoMinor, ok = ParseHTTPVersion(resp.Proto); !ok {
return nil, &badStringError{"malformed HTTP version", resp.Proto}
}
// Parse the response headers.
- nheader := 0
- resp.Header = make(map[string]string)
- for {
- key, value, err := readKeyValue(r)
- if err != nil {
- return nil, err
- }
- if key == "" {
- break // end of response header
- }
- if nheader++; nheader >= maxHeaderLines {
- return nil, ErrHeaderTooLong
- }
- resp.AddHeader(key, value)
+ mimeHeader, err := tp.ReadMIMEHeader()
+ if err != nil {
+ return nil, err
}
+ resp.Header = Header(mimeHeader)
fixPragmaCacheControl(resp.Header)
// Pragma: no-cache
// like
// Cache-Control: no-cache
-func fixPragmaCacheControl(header map[string]string) {
- if header["Pragma"] == "no-cache" {
+func fixPragmaCacheControl(header Header) {
+ if hp, ok := header["Pragma"]; ok && len(hp) > 0 && hp[0] == "no-cache" {
if _, presentcc := header["Cache-Control"]; !presentcc {
- header["Cache-Control"] = "no-cache"
+ header["Cache-Control"] = []string{"no-cache"}
}
}
}
-// AddHeader adds a value under the given key. Keys are not case sensitive.
-func (r *Response) AddHeader(key, value string) {
- key = CanonicalHeaderKey(key)
-
- oldValues, oldValuesPresent := r.Header[key]
- if oldValuesPresent {
- r.Header[key] = oldValues + "," + value
- } else {
- r.Header[key] = value
- }
-}
-
-// GetHeader returns the value of the response header with the given key.
-// If there were multiple headers with this key, their values are concatenated,
-// with a comma delimiter. If there were no response headers with the given
-// key, GetHeader returns an empty string. Keys are not case sensitive.
-func (r *Response) GetHeader(key string) (value string) {
- return r.Header[CanonicalHeaderKey(key)]
-}
-
// ProtoAtLeast returns whether the HTTP protocol used
// in the response is at least major.minor.
func (r *Response) ProtoAtLeast(major, minor int) bool {
return nil
}
-func writeSortedKeyValue(w io.Writer, kvm map[string]string, exclude map[string]bool) os.Error {
- kva := make([]string, len(kvm))
- i := 0
- for k, v := range kvm {
+func writeSortedKeyValue(w io.Writer, kvm map[string][]string, exclude map[string]bool) os.Error {
+ keys := make([]string, 0, len(kvm))
+ for k := range kvm {
if !exclude[k] {
- kva[i] = fmt.Sprint(k + ": " + v + "\r\n")
- i++
+ keys = append(keys, k)
}
}
- kva = kva[0:i]
- sort.SortStrings(kva)
- for _, l := range kva {
- if _, err := io.WriteString(w, l); err != nil {
- return err
+ sort.SortStrings(keys)
+ for _, k := range keys {
+ for _, v := range kvm[k] {
+ if _, err := fmt.Fprintf(w, "%s: %s\r\n", k, v); err != nil {
+ return err
+ }
}
}
return nil
ProtoMajor: 1,
ProtoMinor: 0,
RequestMethod: "GET",
- Header: map[string]string{
- "Connection": "close", // TODO(rsc): Delete?
+ Header: Header{
+ "Connection": {"close"}, // TODO(rsc): Delete?
},
Close: true,
ContentLength: -1,
"Body here\n",
},
+ // Unchunked HTTP/1.1 response without Content-Length or
+ // Connection headers.
+ {
+ "HTTP/1.1 200 OK\r\n" +
+ "\r\n" +
+ "Body here\n",
+
+ Response{
+ Status: "200 OK",
+ StatusCode: 200,
+ Proto: "HTTP/1.1",
+ ProtoMajor: 1,
+ ProtoMinor: 1,
+ RequestMethod: "GET",
+ Close: true,
+ ContentLength: -1,
+ },
+
+ "Body here\n",
+ },
+
+ // Unchunked HTTP/1.1 204 response without Content-Length.
+ {
+ "HTTP/1.1 204 No Content\r\n" +
+ "\r\n" +
+ "Body should not be read!\n",
+
+ Response{
+ Status: "204 No Content",
+ StatusCode: 204,
+ Proto: "HTTP/1.1",
+ ProtoMajor: 1,
+ ProtoMinor: 1,
+ RequestMethod: "GET",
+ Close: false,
+ ContentLength: 0,
+ },
+
+ "",
+ },
+
// Unchunked response with Content-Length.
{
"HTTP/1.0 200 OK\r\n" +
ProtoMajor: 1,
ProtoMinor: 0,
RequestMethod: "GET",
- Header: map[string]string{
- "Connection": "close", // TODO(rsc): Delete?
- "Content-Length": "10", // TODO(rsc): Delete?
+ Header: Header{
+ "Connection": {"close"}, // TODO(rsc): Delete?
+ "Content-Length": {"10"}, // TODO(rsc): Delete?
},
Close: true,
ContentLength: 10,
ProtoMajor: 1,
ProtoMinor: 0,
RequestMethod: "GET",
- Header: map[string]string{},
+ Header: Header{},
Close: true,
ContentLength: -1,
TransferEncoding: []string{"chunked"},
ProtoMajor: 1,
ProtoMinor: 0,
RequestMethod: "GET",
- Header: map[string]string{},
+ Header: Header{},
Close: true,
ContentLength: -1, // TODO(rsc): Fix?
TransferEncoding: []string{"chunked"},
ProtoMajor: 1,
ProtoMinor: 0,
RequestMethod: "GET",
- Header: map[string]string{},
+ Header: Header{},
Close: true,
ContentLength: -1,
},
ProtoMajor: 1,
ProtoMinor: 0,
RequestMethod: "GET",
- Header: map[string]string{},
+ Header: Header{},
Close: true,
ContentLength: -1,
},
ProtoMajor: 1,
ProtoMinor: 0,
RequestMethod: "GET",
- Header: map[string]string{},
+ Header: map[string][]string{},
Body: nopCloser{bytes.NewBufferString("abcdef")},
ContentLength: 6,
},
ProtoMajor: 1,
ProtoMinor: 0,
RequestMethod: "GET",
- Header: map[string]string{},
+ Header: map[string][]string{},
Body: nopCloser{bytes.NewBufferString("abcdef")},
ContentLength: -1,
},
ProtoMajor: 1,
ProtoMinor: 1,
RequestMethod: "GET",
- Header: map[string]string{},
+ Header: map[string][]string{},
Body: nopCloser{bytes.NewBufferString("abcdef")},
ContentLength: 6,
TransferEncoding: []string{"chunked"},
import (
"bufio"
"bytes"
+ "fmt"
"io"
+ "io/ioutil"
"os"
"net"
+ "strings"
"testing"
+ "time"
)
type dummyAddr string
}
}
+type stringHandler string
+
+func (s stringHandler) ServeHTTP(w ResponseWriter, r *Request) {
+ w.SetHeader("Result", string(s))
+}
+
+var handlers = []struct {
+ pattern string
+ msg string
+}{
+ {"/", "Default"},
+ {"/someDir/", "someDir"},
+ {"someHost.com/someDir/", "someHost.com/someDir"},
+}
+
+var vtests = []struct {
+ url string
+ expected string
+}{
+ {"http://localhost/someDir/apage", "someDir"},
+ {"http://localhost/otherDir/apage", "Default"},
+ {"http://someHost.com/someDir/apage", "someHost.com/someDir"},
+ {"http://otherHost.com/someDir/apage", "someDir"},
+ {"http://otherHost.com/aDir/apage", "Default"},
+}
+
+func TestHostHandlers(t *testing.T) {
+ for _, h := range handlers {
+ Handle(h.pattern, stringHandler(h.msg))
+ }
+ l, err := net.Listen("tcp", "127.0.0.1:0") // any port
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer l.Close()
+ go Serve(l, nil)
+ conn, err := net.Dial("tcp", "", l.Addr().String())
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer conn.Close()
+ cc := NewClientConn(conn, nil)
+ for _, vt := range vtests {
+ var r *Response
+ var req Request
+ if req.URL, err = ParseURL(vt.url); err != nil {
+ t.Errorf("cannot parse url: %v", err)
+ continue
+ }
+ if err := cc.Write(&req); err != nil {
+ t.Errorf("writing request: %v", err)
+ continue
+ }
+ r, err := cc.Read(&req)
+ if err != nil {
+ t.Errorf("reading response: %v", err)
+ continue
+ }
+ s := r.Header.Get("Result")
+ if s != vt.expected {
+ t.Errorf("Get(%q) = %q, want %q", vt.url, s, vt.expected)
+ }
+ }
+}
+
type responseWriterMethodCall struct {
method string
headerKey, headerValue string // if method == "SetHeader"
}
}
}
+
+func TestServerTimeouts(t *testing.T) {
+ l, err := net.ListenTCP("tcp", &net.TCPAddr{Port: 0})
+ if err != nil {
+ t.Fatalf("listen error: %v", err)
+ }
+ addr, _ := l.Addr().(*net.TCPAddr)
+
+ reqNum := 0
+ handler := HandlerFunc(func(res ResponseWriter, req *Request) {
+ reqNum++
+ fmt.Fprintf(res, "req=%d", reqNum)
+ })
+
+ const second = 1000000000 /* nanos */
+ server := &Server{Handler: handler, ReadTimeout: 0.25 * second, WriteTimeout: 0.25 * second}
+ go server.Serve(l)
+
+ url := fmt.Sprintf("http://localhost:%d/", addr.Port)
+
+ // Hit the HTTP server successfully.
+ r, _, err := Get(url)
+ if err != nil {
+ t.Fatalf("http Get #1: %v", err)
+ }
+ got, _ := ioutil.ReadAll(r.Body)
+ expected := "req=1"
+ if string(got) != expected {
+ t.Errorf("Unexpected response for request #1; got %q; expected %q",
+ string(got), expected)
+ }
+
+ // Slow client that should timeout.
+ t1 := time.Nanoseconds()
+ conn, err := net.Dial("tcp", "", fmt.Sprintf("localhost:%d", addr.Port))
+ if err != nil {
+ t.Fatalf("Dial: %v", err)
+ }
+ buf := make([]byte, 1)
+ n, err := conn.Read(buf)
+ latency := time.Nanoseconds() - t1
+ if n != 0 || err != os.EOF {
+ t.Errorf("Read = %v, %v, wanted %v, %v", n, err, 0, os.EOF)
+ }
+ if latency < second*0.20 /* fudge from 0.25 above */ {
+ t.Errorf("got EOF after %d ns, want >= %d", latency, second*0.20)
+ }
+
+ // Hit the HTTP server successfully again, verifying that the
+ // previous slow connection didn't run our handler. (that we
+ // get "req=2", not "req=3")
+ r, _, err = Get(url)
+ if err != nil {
+ t.Fatalf("http Get #2: %v", err)
+ }
+ got, _ = ioutil.ReadAll(r.Body)
+ expected = "req=2"
+ if string(got) != expected {
+ t.Errorf("Get #2 got %q, want %q", string(got), expected)
+ }
+
+ l.Close()
+}
+
+// TestIdentityResponse verifies that a handler can unset
+func TestIdentityResponse(t *testing.T) {
+ l, err := net.Listen("tcp", "127.0.0.1:0")
+ if err != nil {
+ t.Fatalf("failed to listen on a port: %v", err)
+ }
+ defer l.Close()
+ urlBase := "http://" + l.Addr().String() + "/"
+
+ handler := HandlerFunc(func(rw ResponseWriter, req *Request) {
+ rw.SetHeader("Content-Length", "3")
+ rw.SetHeader("Transfer-Encoding", req.FormValue("te"))
+ switch {
+ case req.FormValue("overwrite") == "1":
+ _, err := rw.Write([]byte("foo TOO LONG"))
+ if err != ErrContentLength {
+ t.Errorf("expected ErrContentLength; got %v", err)
+ }
+ case req.FormValue("underwrite") == "1":
+ rw.SetHeader("Content-Length", "500")
+ rw.Write([]byte("too short"))
+ default:
+ rw.Write([]byte("foo"))
+ }
+ })
+
+ server := &Server{Handler: handler}
+ go server.Serve(l)
+
+ // Note: this relies on the assumption (which is true) that
+ // Get sends HTTP/1.1 or greater requests. Otherwise the
+ // server wouldn't have the choice to send back chunked
+ // responses.
+ for _, te := range []string{"", "identity"} {
+ url := urlBase + "?te=" + te
+ res, _, err := Get(url)
+ if err != nil {
+ t.Fatalf("error with Get of %s: %v", url, err)
+ }
+ if cl, expected := res.ContentLength, int64(3); cl != expected {
+ t.Errorf("for %s expected res.ContentLength of %d; got %d", url, expected, cl)
+ }
+ if cl, expected := res.Header.Get("Content-Length"), "3"; cl != expected {
+ t.Errorf("for %s expected Content-Length header of %q; got %q", url, expected, cl)
+ }
+ if tl, expected := len(res.TransferEncoding), 0; tl != expected {
+ t.Errorf("for %s expected len(res.TransferEncoding) of %d; got %d (%v)",
+ url, expected, tl, res.TransferEncoding)
+ }
+ }
+
+ // Verify that ErrContentLength is returned
+ url := urlBase + "?overwrite=1"
+ _, _, err = Get(url)
+ if err != nil {
+ t.Fatalf("error with Get of %s: %v", url, err)
+ }
+
+ // Verify that the connection is closed when the declared Content-Length
+ // is larger than what the handler wrote.
+ conn, err := net.Dial("tcp", "", l.Addr().String())
+ if err != nil {
+ t.Fatalf("error dialing: %v", err)
+ }
+ _, err = conn.Write([]byte("GET /?underwrite=1 HTTP/1.1\r\nHost: foo\r\n\r\n"))
+ if err != nil {
+ t.Fatalf("error writing: %v", err)
+ }
+ // The next ReadAll will hang for a failing test, so use a Timer instead
+ // to fail more traditionally
+ timer := time.AfterFunc(2e9, func() {
+ t.Fatalf("Timeout expired in ReadAll.")
+ })
+ defer timer.Stop()
+ got, _ := ioutil.ReadAll(conn)
+ expectedSuffix := "\r\n\r\ntoo short"
+ if !strings.HasSuffix(string(got), expectedSuffix) {
+ t.Fatalf("Expected output to end with %q; got response body %q",
+ expectedSuffix, string(got))
+ }
+}
ErrWriteAfterFlush = os.NewError("Conn.Write called after Flush")
ErrBodyNotAllowed = os.NewError("http: response status code does not allow body")
ErrHijacked = os.NewError("Conn has been hijacked")
+ ErrContentLength = os.NewError("Conn.Write wrote more than the declared Content-Length")
)
// Objects implementing the Handler interface can be
//
// Content-Type: text/html; charset=utf-8
//
- // being sent. UTF-8 encoded HTML is the default setting for
+ // being sent. UTF-8 encoded HTML is the default setting for
// Content-Type in this library, so users need not make that
- // particular call. Calls to SetHeader after WriteHeader (or Write)
- // are ignored.
+ // particular call. Calls to SetHeader after WriteHeader (or Write)
+ // are ignored. An empty value removes the header if previously set.
SetHeader(string, string)
// Write writes the data to the connection as part of an HTTP reply.
wroteContinue bool // 100 Continue response was written
header map[string]string // reply header parameters
written int64 // number of bytes written in body
+ contentLength int64 // explicitly-declared Content-Length; or -1
status int // status code passed to WriteHeader
// close connection after this reply. set on request and
w.conn = c
w.req = req
w.header = make(map[string]string)
+ w.contentLength = -1
// Expect 100 Continue support
if req.expectsContinue() && req.ProtoAtLeast(1, 1) {
// Wrap the Body reader with one that replies on the connection
req.Body = &expectContinueReader{readCloser: req.Body, resp: w}
}
-
- // Default output is HTML encoded in UTF-8.
- w.SetHeader("Content-Type", "text/html; charset=utf-8")
- w.SetHeader("Date", time.UTC().Format(TimeFormat))
-
- if req.Method == "HEAD" {
- // do nothing
- } else if req.ProtoAtLeast(1, 1) {
- // HTTP/1.1 or greater: use chunked transfer encoding
- // to avoid closing the connection at EOF.
- w.chunking = true
- w.SetHeader("Transfer-Encoding", "chunked")
- } else {
- // HTTP version < 1.1: cannot do chunked transfer
- // encoding, so signal EOF by closing connection.
- // Will be overridden if the HTTP handler ends up
- // writing a Content-Length and the client requested
- // "Connection: keep-alive"
- w.closeAfterReply = true
- }
-
return w, nil
}
func (w *response) RemoteAddr() string { return w.conn.remoteAddr }
// SetHeader implements the ResponseWriter.SetHeader method
-func (w *response) SetHeader(hdr, val string) { w.header[CanonicalHeaderKey(hdr)] = val }
+// An empty value removes the header from the map.
+func (w *response) SetHeader(hdr, val string) {
+ w.header[CanonicalHeaderKey(hdr)] = val, val != ""
+}
// WriteHeader implements the ResponseWriter.WriteHeader method
func (w *response) WriteHeader(code int) {
w.status = code
if code == StatusNotModified {
// Must not have body.
- w.header["Content-Type"] = "", false
- w.header["Transfer-Encoding"] = "", false
+ for _, header := range []string{"Content-Type", "Content-Length", "Transfer-Encoding"} {
+ if w.header[header] != "" {
+ // TODO: return an error if WriteHeader gets a return parameter
+ // or set a flag on w to make future Writes() write an error page?
+ // for now just log and drop the header.
+ log.Printf("http: StatusNotModified response with header %q defined", header)
+ w.header[header] = "", false
+ }
+ }
+ } else {
+ // Default output is HTML encoded in UTF-8.
+ if w.header["Content-Type"] == "" {
+ w.SetHeader("Content-Type", "text/html; charset=utf-8")
+ }
+ }
+
+ if w.header["Date"] == "" {
+ w.SetHeader("Date", time.UTC().Format(TimeFormat))
+ }
+
+ // Check for a explicit (and valid) Content-Length header.
+ var hasCL bool
+ var contentLength int64
+ if clenStr, ok := w.header["Content-Length"]; ok {
+ var err os.Error
+ contentLength, err = strconv.Atoi64(clenStr)
+ if err == nil {
+ hasCL = true
+ } else {
+ log.Printf("http: invalid Content-Length of %q sent", clenStr)
+ w.SetHeader("Content-Length", "")
+ }
+ }
+
+ te, hasTE := w.header["Transfer-Encoding"]
+ if hasCL && hasTE && te != "identity" {
+ // TODO: return an error if WriteHeader gets a return parameter
+ // For now just ignore the Content-Length.
+ log.Printf("http: WriteHeader called with both Transfer-Encoding of %q and a Content-Length of %d",
+ te, contentLength)
+ w.SetHeader("Content-Length", "")
+ hasCL = false
+ }
+
+ if w.req.Method == "HEAD" {
+ // do nothing
+ } else if hasCL {
w.chunking = false
+ w.contentLength = contentLength
+ w.SetHeader("Transfer-Encoding", "")
+ } else if w.req.ProtoAtLeast(1, 1) {
+ // HTTP/1.1 or greater: use chunked transfer encoding
+ // to avoid closing the connection at EOF.
+ // TODO: this blows away any custom or stacked Transfer-Encoding they
+ // might have set. Deal with that as need arises once we have a valid
+ // use case.
+ w.chunking = true
+ w.SetHeader("Transfer-Encoding", "chunked")
+ } else {
+ // HTTP version < 1.1: cannot do chunked transfer
+ // encoding and we don't know the Content-Length so
+ // signal EOF by closing connection.
+ w.closeAfterReply = true
+ w.chunking = false // redundant
+ w.SetHeader("Transfer-Encoding", "") // in case already set
+ }
+
+ if w.req.wantsHttp10KeepAlive() && (w.req.Method == "HEAD" || hasCL) {
+ _, connectionHeaderSet := w.header["Connection"]
+ if !connectionHeaderSet {
+ w.SetHeader("Connection", "keep-alive")
+ }
}
+
// Cannot use Content-Length with non-identity Transfer-Encoding.
if w.chunking {
- w.header["Content-Length"] = "", false
+ w.SetHeader("Content-Length", "")
}
if !w.req.ProtoAtLeast(1, 0) {
return
return 0, ErrHijacked
}
if !w.wroteHeader {
- if w.req.wantsHttp10KeepAlive() {
- _, hasLength := w.header["Content-Length"]
- if hasLength {
- _, connectionHeaderSet := w.header["Connection"]
- if !connectionHeaderSet {
- w.header["Connection"] = "keep-alive"
- }
- }
- }
w.WriteHeader(StatusOK)
}
if len(data) == 0 {
}
w.written += int64(len(data)) // ignoring errors, for errorKludge
+ if w.contentLength != -1 && w.written > w.contentLength {
+ return 0, ErrContentLength
+ }
// TODO(rsc): if chunking happened after the buffering,
// then there would be fewer chunk headers.
}
w.conn.buf.Flush()
w.req.Body.Close()
+
+ if w.contentLength != -1 && w.contentLength != w.written {
+ // Did not write enough. Avoid getting out of sync.
+ w.closeAfterReply = true
+ }
}
// Flush implements the ResponseWriter.Flush method.
// patterns and calls the handler for the pattern that
// most closely matches the URL.
//
-// Patterns named fixed paths, like "/favicon.ico",
-// or subtrees, like "/images/" (note the trailing slash).
-// Patterns must begin with /.
+// Patterns named fixed, rooted paths, like "/favicon.ico",
+// or rooted subtrees, like "/images/" (note the trailing slash).
// Longer patterns take precedence over shorter ones, so that
// if there are handlers registered for both "/images/"
// and "/images/thumbnails/", the latter handler will be
// former will receiver requests for any other paths in the
// "/images/" subtree.
//
-// In the future, the pattern syntax may be relaxed to allow
-// an optional host-name at the beginning of the pattern,
-// so that a handler might register for the two patterns
-// "/codesearch" and "codesearch.google.com/"
-// without taking over requests for http://www.google.com/.
+// Patterns may optionally begin with a host name, restricting matches to
+// URLs on that host only. Host-specific patterns take precedence over
+// general patterns, so that a handler might register for the two patterns
+// "/codesearch" and "codesearch.google.com/" without also taking over
+// requests for "http://www.google.com/".
//
// ServeMux also takes care of sanitizing the URL request path,
// redirecting any request containing . or .. elements to an
return np
}
-// ServeHTTP dispatches the request to the handler whose
-// pattern most closely matches the request URL.
-func (mux *ServeMux) ServeHTTP(w ResponseWriter, r *Request) {
- // Clean path to canonical form and redirect.
- if p := cleanPath(r.URL.Path); p != r.URL.Path {
- w.SetHeader("Location", p)
- w.WriteHeader(StatusMovedPermanently)
- return
- }
-
- // Most-specific (longest) pattern wins.
+// Find a handler on a handler map given a path string
+// Most-specific (longest) pattern wins
+func (mux *ServeMux) match(path string) Handler {
var h Handler
var n = 0
for k, v := range mux.m {
- if !pathMatch(k, r.URL.Path) {
+ if !pathMatch(k, path) {
continue
}
if h == nil || len(k) > n {
h = v
}
}
+ return h
+}
+
+// ServeHTTP dispatches the request to the handler whose
+// pattern most closely matches the request URL.
+func (mux *ServeMux) ServeHTTP(w ResponseWriter, r *Request) {
+ // Clean path to canonical form and redirect.
+ if p := cleanPath(r.URL.Path); p != r.URL.Path {
+ w.SetHeader("Location", p)
+ w.WriteHeader(StatusMovedPermanently)
+ return
+ }
+ // Host-specific pattern takes precedence over generic ones
+ h := mux.match(r.Host + r.URL.Path)
+ if h == nil {
+ h = mux.match(r.URL.Path)
+ }
if h == nil {
h = NotFoundHandler()
}
// Handle registers the handler for the given pattern.
func (mux *ServeMux) Handle(pattern string, handler Handler) {
- if pattern == "" || pattern[0] != '/' {
+ if pattern == "" {
panic("http: invalid pattern " + pattern)
}
// Handle registers the handler for the given pattern
// in the DefaultServeMux.
+// The documentation for ServeMux explains how patterns are matched.
func Handle(pattern string, handler Handler) { DefaultServeMux.Handle(pattern, handler) }
// HandleFunc registers the handler function for the given pattern
// in the DefaultServeMux.
+// The documentation for ServeMux explains how patterns are matched.
func HandleFunc(pattern string, handler func(ResponseWriter, *Request)) {
DefaultServeMux.HandleFunc(pattern, handler)
}
// read requests and then call handler to reply to them.
// Handler is typically nil, in which case the DefaultServeMux is used.
func Serve(l net.Listener, handler Handler) os.Error {
+ srv := &Server{Handler: handler}
+ return srv.Serve(l)
+}
+
+// A Server defines parameters for running an HTTP server.
+type Server struct {
+ Addr string // TCP address to listen on, ":http" if empty
+ Handler Handler // handler to invoke, http.DefaultServeMux if nil
+ ReadTimeout int64 // the net.Conn.SetReadTimeout value for new connections
+ WriteTimeout int64 // the net.Conn.SetWriteTimeout value for new connections
+}
+
+// ListenAndServe listens on the TCP network address srv.Addr and then
+// calls Serve to handle requests on incoming connections. If
+// srv.Addr is blank, ":http" is used.
+func (srv *Server) ListenAndServe() os.Error {
+ addr := srv.Addr
+ if addr == "" {
+ addr = ":http"
+ }
+ l, e := net.Listen("tcp", addr)
+ if e != nil {
+ return e
+ }
+ return srv.Serve(l)
+}
+
+// Serve accepts incoming connections on the Listener l, creating a
+// new service thread for each. The service threads read requests and
+// then call srv.Handler to reply to them.
+func (srv *Server) Serve(l net.Listener) os.Error {
+ defer l.Close()
+ handler := srv.Handler
if handler == nil {
handler = DefaultServeMux
}
if e != nil {
return e
}
+ if srv.ReadTimeout != 0 {
+ rw.SetReadTimeout(srv.ReadTimeout)
+ }
+ if srv.WriteTimeout != 0 {
+ rw.SetWriteTimeout(srv.WriteTimeout)
+ }
c, err := newConn(rw, handler)
if err != nil {
continue
// http.HandleFunc("/hello", HelloServer)
// err := http.ListenAndServe(":12345", nil)
// if err != nil {
-// log.Exit("ListenAndServe: ", err.String())
+// log.Fatal("ListenAndServe: ", err.String())
// }
// }
func ListenAndServe(addr string, handler Handler) os.Error {
- l, e := net.Listen("tcp", addr)
- if e != nil {
- return e
- }
- e = Serve(l, handler)
- l.Close()
- return e
+ server := &Server{Addr: addr, Handler: handler}
+ return server.ListenAndServe()
}
// ListenAndServeTLS acts identically to ListenAndServe, except that it
// log.Printf("About to listen on 10443. Go to https://127.0.0.1:10443/")
// err := http.ListenAndServeTLS(":10443", "cert.pem", "key.pem", nil)
// if err != nil {
-// log.Exit(err)
+// log.Fatal(err)
// }
// }
//
ContentLength int64
Close bool
TransferEncoding []string
- Trailer map[string]string
+ Trailer Header
}
func newTransferWriter(r interface{}) (t *transferWriter, err os.Error) {
type transferReader struct {
// Input
- Header map[string]string
+ Header Header
StatusCode int
RequestMethod string
ProtoMajor int
ContentLength int64
TransferEncoding []string
Close bool
- Trailer map[string]string
+ Trailer Header
+}
+
+// bodyAllowedForStatus returns whether a given response status code
+// permits a body. See RFC2616, section 4.4.
+func bodyAllowedForStatus(status int) bool {
+ switch {
+ case status >= 100 && status <= 199:
+ return false
+ case status == 204:
+ return false
+ case status == 304:
+ return false
+ }
+ return true
}
// msg is *Request or *Response.
return err
}
+ // If there is no Content-Length or chunked Transfer-Encoding on a *Response
+ // and the status is not 1xx, 204 or 304, then the body is unbounded.
+ // See RFC2616, section 4.4.
+ switch msg.(type) {
+ case *Response:
+ if t.ContentLength == -1 &&
+ !chunked(t.TransferEncoding) &&
+ bodyAllowedForStatus(t.StatusCode) {
+ // Unbounded body.
+ t.Close = true
+ }
+ }
+
// Prepare body reader. ContentLength < 0 means chunked encoding
// or close connection when finished, since multipart is not supported yet
switch {
func chunked(te []string) bool { return len(te) > 0 && te[0] == "chunked" }
// Sanitize transfer encoding
-func fixTransferEncoding(header map[string]string) ([]string, os.Error) {
+func fixTransferEncoding(header Header) ([]string, os.Error) {
raw, present := header["Transfer-Encoding"]
if !present {
return nil, nil
}
- header["Transfer-Encoding"] = "", false
- encodings := strings.Split(raw, ",", -1)
+ header["Transfer-Encoding"] = nil, false
+ encodings := strings.Split(raw[0], ",", -1)
te := make([]string, 0, len(encodings))
// TODO: Even though we only support "identity" and "chunked"
// encodings, the loop below is designed with foresight. One
// Chunked encoding trumps Content-Length. See RFC 2616
// Section 4.4. Currently len(te) > 0 implies chunked
// encoding.
- header["Content-Length"] = "", false
+ header["Content-Length"] = nil, false
return te, nil
}
// Determine the expected body length, using RFC 2616 Section 4.4. This
// function is not a method, because ultimately it should be shared by
// ReadResponse and ReadRequest.
-func fixLength(status int, requestMethod string, header map[string]string, te []string) (int64, os.Error) {
+func fixLength(status int, requestMethod string, header Header, te []string) (int64, os.Error) {
// Logic based on response type or status
if noBodyExpected(requestMethod) {
}
// Logic based on Content-Length
- if cl, present := header["Content-Length"]; present {
- cl = strings.TrimSpace(cl)
- if cl != "" {
- n, err := strconv.Atoi64(cl)
- if err != nil || n < 0 {
- return -1, &badStringError{"bad Content-Length", cl}
- }
- return n, nil
- } else {
- header["Content-Length"] = "", false
+ cl := strings.TrimSpace(header.Get("Content-Length"))
+ if cl != "" {
+ n, err := strconv.Atoi64(cl)
+ if err != nil || n < 0 {
+ return -1, &badStringError{"bad Content-Length", cl}
}
+ return n, nil
+ } else {
+ header.Del("Content-Length")
}
// Logic based on media type. The purpose of the following code is just
// to detect whether the unsupported "multipart/byteranges" is being
// used. A proper Content-Type parser is needed in the future.
- if strings.Contains(strings.ToLower(header["Content-Type"]), "multipart/byteranges") {
+ if strings.Contains(strings.ToLower(header.Get("Content-Type")), "multipart/byteranges") {
return -1, ErrNotSupported
}
// Determine whether to hang up after sending a request and body, or
// receiving a response and body
// 'header' is the request headers
-func shouldClose(major, minor int, header map[string]string) bool {
+func shouldClose(major, minor int, header Header) bool {
if major < 1 {
return true
} else if major == 1 && minor == 0 {
- v, present := header["Connection"]
- if !present {
- return true
- }
- v = strings.ToLower(v)
- if !strings.Contains(v, "keep-alive") {
+ if !strings.Contains(strings.ToLower(header.Get("Connection")), "keep-alive") {
return true
}
return false
- } else if v, present := header["Connection"]; present {
+ } else {
// TODO: Should split on commas, toss surrounding white space,
// and check each field.
- if v == "close" {
- header["Connection"] = "", false
+ if strings.ToLower(header.Get("Connection")) == "close" {
+ header.Del("Connection")
return true
}
}
}
// Parse the trailer header
-func fixTrailer(header map[string]string, te []string) (map[string]string, os.Error) {
- raw, present := header["Trailer"]
- if !present {
+func fixTrailer(header Header, te []string) (Header, os.Error) {
+ raw := header.Get("Trailer")
+ if raw == "" {
return nil, nil
}
- header["Trailer"] = "", false
- trailer := make(map[string]string)
+ header.Del("Trailer")
+ trailer := make(Header)
keys := strings.Split(raw, ",", -1)
for _, key := range keys {
key = CanonicalHeaderKey(strings.TrimSpace(key))
case "Transfer-Encoding", "Trailer", "Content-Length":
return nil, &badStringError{"bad trailer key", key}
}
- trailer[key] = ""
+ trailer.Del(key)
}
if len(trailer) == 0 {
return nil, nil
--- /dev/null
+// Copyright 2011 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package http
+
+import (
+ "bufio"
+ "crypto/tls"
+ "encoding/base64"
+ "fmt"
+ "net"
+ "os"
+ "strings"
+ "sync"
+)
+
+// DefaultTransport is the default implementation of ClientTransport
+// and is used by DefaultClient. It establishes a new network connection for
+// each call to Do and uses HTTP proxies as directed by the $HTTP_PROXY and
+// $NO_PROXY (or $http_proxy and $no_proxy) environment variables.
+var DefaultTransport ClientTransport = &transport{}
+
+// transport implements http.ClientTranport for the default case,
+// using TCP connections to either the host or a proxy, serving
+// http or https schemes. In the future this may become public
+// and support options on keep-alive connection duration, pipelining
+// controls, etc. For now this is simply a port of the old Go code
+// client code to the http.ClientTransport interface.
+type transport struct {
+ // TODO: keep-alives, pipelining, etc using a map from
+ // scheme/host to a connection. Something like:
+ l sync.Mutex
+ hostConn map[string]*ClientConn
+}
+
+func (ct *transport) Do(req *Request) (resp *Response, err os.Error) {
+ if req.URL.Scheme != "http" && req.URL.Scheme != "https" {
+ return nil, &badStringError{"unsupported protocol scheme", req.URL.Scheme}
+ }
+
+ addr := req.URL.Host
+ if !hasPort(addr) {
+ addr += ":" + req.URL.Scheme
+ }
+
+ var proxyURL *URL
+ proxyAuth := ""
+ proxy := ""
+ if !matchNoProxy(addr) {
+ proxy = os.Getenv("HTTP_PROXY")
+ if proxy == "" {
+ proxy = os.Getenv("http_proxy")
+ }
+ }
+
+ if proxy != "" {
+ proxyURL, err = ParseRequestURL(proxy)
+ if err != nil {
+ return nil, os.ErrorString("invalid proxy address")
+ }
+ if proxyURL.Host == "" {
+ proxyURL, err = ParseRequestURL("http://" + proxy)
+ if err != nil {
+ return nil, os.ErrorString("invalid proxy address")
+ }
+ }
+ addr = proxyURL.Host
+ proxyInfo := proxyURL.RawUserinfo
+ if proxyInfo != "" {
+ enc := base64.URLEncoding
+ encoded := make([]byte, enc.EncodedLen(len(proxyInfo)))
+ enc.Encode(encoded, []byte(proxyInfo))
+ proxyAuth = "Basic " + string(encoded)
+ }
+ }
+
+ // Connect to server or proxy
+ conn, err := net.Dial("tcp", "", addr)
+ if err != nil {
+ return nil, err
+ }
+
+ if req.URL.Scheme == "http" {
+ // Include proxy http header if needed.
+ if proxyAuth != "" {
+ req.Header.Set("Proxy-Authorization", proxyAuth)
+ }
+ } else { // https
+ if proxyURL != nil {
+ // Ask proxy for direct connection to server.
+ // addr defaults above to ":https" but we need to use numbers
+ addr = req.URL.Host
+ if !hasPort(addr) {
+ addr += ":443"
+ }
+ fmt.Fprintf(conn, "CONNECT %s HTTP/1.1\r\n", addr)
+ fmt.Fprintf(conn, "Host: %s\r\n", addr)
+ if proxyAuth != "" {
+ fmt.Fprintf(conn, "Proxy-Authorization: %s\r\n", proxyAuth)
+ }
+ fmt.Fprintf(conn, "\r\n")
+
+ // Read response.
+ // Okay to use and discard buffered reader here, because
+ // TLS server will not speak until spoken to.
+ br := bufio.NewReader(conn)
+ resp, err := ReadResponse(br, "CONNECT")
+ if err != nil {
+ return nil, err
+ }
+ if resp.StatusCode != 200 {
+ f := strings.Split(resp.Status, " ", 2)
+ return nil, os.ErrorString(f[1])
+ }
+ }
+
+ // Initiate TLS and check remote host name against certificate.
+ conn = tls.Client(conn, nil)
+ if err = conn.(*tls.Conn).Handshake(); err != nil {
+ return nil, err
+ }
+ h := req.URL.Host
+ if hasPort(h) {
+ h = h[:strings.LastIndex(h, ":")]
+ }
+ if err = conn.(*tls.Conn).VerifyHostname(h); err != nil {
+ return nil, err
+ }
+ }
+
+ err = req.Write(conn)
+ if err != nil {
+ conn.Close()
+ return nil, err
+ }
+
+ reader := bufio.NewReader(conn)
+ resp, err = ReadResponse(reader, req.Method)
+ if err != nil {
+ conn.Close()
+ return nil, err
+ }
+
+ resp.Body = readClose{resp.Body, conn}
+ return
+}
--- /dev/null
+// Copyright 2011 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package image_test
+
+import (
+ "bufio"
+ "image"
+ "os"
+ "testing"
+
+ // TODO(nigeltao): implement bmp, gif and tiff decoders.
+ _ "image/jpeg"
+ _ "image/png"
+)
+
+const goldenFile = "testdata/video-001.png"
+
+type imageTest struct {
+ filename string
+ tolerance int
+}
+
+var imageTests = []imageTest{
+ //{"testdata/video-001.bmp", 0},
+ // GIF images are restricted to a 256-color palette and the conversion
+ // to GIF loses significant image quality.
+ //{"testdata/video-001.gif", 64<<8},
+ // JPEG is a lossy format and hence needs a non-zero tolerance.
+ {"testdata/video-001.jpeg", 8 << 8},
+ {"testdata/video-001.png", 0},
+ //{"testdata/video-001.tiff", 0},
+}
+
+func decode(filename string) (image.Image, string, os.Error) {
+ f, err := os.Open(filename, os.O_RDONLY, 0400)
+ if err != nil {
+ return nil, "", err
+ }
+ defer f.Close()
+ return image.Decode(bufio.NewReader(f))
+}
+
+func delta(u0, u1 uint32) int {
+ d := int(u0) - int(u1)
+ if d < 0 {
+ return -d
+ }
+ return d
+}
+
+func withinTolerance(c0, c1 image.Color, tolerance int) bool {
+ r0, g0, b0, a0 := c0.RGBA()
+ r1, g1, b1, a1 := c1.RGBA()
+ r := delta(r0, r1)
+ g := delta(g0, g1)
+ b := delta(b0, b1)
+ a := delta(a0, a1)
+ return r <= tolerance && g <= tolerance && b <= tolerance && a <= tolerance
+}
+
+func TestDecode(t *testing.T) {
+ golden, _, err := decode(goldenFile)
+ if err != nil {
+ t.Errorf("%s: %v", goldenFile, err)
+ }
+loop:
+ for _, it := range imageTests {
+ m, _, err := decode(it.filename)
+ if err != nil {
+ t.Errorf("%s: %v", it.filename, err)
+ continue loop
+ }
+ b := golden.Bounds()
+ if !b.Eq(m.Bounds()) {
+ t.Errorf("%s: want bounds %v got %v", it.filename, b, m.Bounds())
+ continue loop
+ }
+ for y := b.Min.Y; y < b.Max.Y; y++ {
+ for x := b.Min.X; x < b.Max.X; x++ {
+ if !withinTolerance(golden.At(x, y), m.At(x, y), it.tolerance) {
+ t.Errorf("%s: at (%d, %d), want %v got %v", it.filename, x, y, golden.At(x, y), m.At(x, y))
+ continue loop
+ }
+ }
+ }
+ }
+}
// A cb is a combination of color type and bit depth.
const (
cbInvalid = iota
+ cbG1
+ cbG2
+ cbG4
cbG8
+ cbGA8
cbTC8
+ cbP1
+ cbP2
+ cbP4
cbP8
cbTCA8
cbG16
+ cbGA16
cbTC16
cbTCA16
)
type decoder struct {
width, height int
+ depth int
palette image.PalettedColorModel
cb int
stage int
return UnsupportedError("dimension overflow")
}
d.cb = cbInvalid
- switch d.tmp[8] {
+ d.depth = int(d.tmp[8])
+ switch d.depth {
+ case 1:
+ switch d.tmp[9] {
+ case ctGrayscale:
+ d.cb = cbG1
+ case ctPaletted:
+ d.cb = cbP1
+ }
+ case 2:
+ switch d.tmp[9] {
+ case ctGrayscale:
+ d.cb = cbG2
+ case ctPaletted:
+ d.cb = cbP2
+ }
+ case 4:
+ switch d.tmp[9] {
+ case ctGrayscale:
+ d.cb = cbG4
+ case ctPaletted:
+ d.cb = cbP4
+ }
case 8:
switch d.tmp[9] {
case ctGrayscale:
d.cb = cbTC8
case ctPaletted:
d.cb = cbP8
+ case ctGrayscaleAlpha:
+ d.cb = cbGA8
case ctTrueColorAlpha:
d.cb = cbTCA8
}
d.cb = cbG16
case ctTrueColor:
d.cb = cbTC16
+ case ctGrayscaleAlpha:
+ d.cb = cbGA16
case ctTrueColorAlpha:
d.cb = cbTCA16
}
func (d *decoder) parsePLTE(r io.Reader, crc hash.Hash32, length uint32) os.Error {
np := int(length / 3) // The number of palette entries.
- if length%3 != 0 || np <= 0 || np > 256 {
+ if length%3 != 0 || np <= 0 || np > 256 || np > 1<<uint(d.depth) {
return FormatError("bad PLTE length")
}
n, err := io.ReadFull(r, d.tmp[0:3*np])
}
crc.Write(d.tmp[0:n])
switch d.cb {
- case cbP8:
+ case cbP1, cbP2, cbP4, cbP8:
d.palette = image.PalettedColorModel(make([]image.Color, np))
for i := 0; i < np; i++ {
d.palette[i] = image.RGBAColor{d.tmp[3*i+0], d.tmp[3*i+1], d.tmp[3*i+2], 0xff}
return UnsupportedError("grayscale transparency")
case cbTC8, cbTC16:
return UnsupportedError("truecolor transparency")
- case cbP8:
+ case cbP1, cbP2, cbP4, cbP8:
if n > len(d.palette) {
return FormatError("bad tRNS length")
}
rgba := d.palette[i].(image.RGBAColor)
d.palette[i] = image.RGBAColor{rgba.R, rgba.G, rgba.B, d.tmp[i]}
}
- case cbTCA8, cbTCA16:
+ case cbGA8, cbGA16, cbTCA8, cbTCA16:
return FormatError("tRNS, color type mismatch")
}
return nil
return nil, err
}
defer r.Close()
- bpp := 0 // Bytes per pixel.
+ bitsPerPixel := 0
maxPalette := uint8(0)
var (
gray *image.Gray
img image.Image
)
switch d.cb {
- case cbG8:
- bpp = 1
+ case cbG1, cbG2, cbG4, cbG8:
+ bitsPerPixel = d.depth
gray = image.NewGray(d.width, d.height)
img = gray
+ case cbGA8:
+ bitsPerPixel = 16
+ nrgba = image.NewNRGBA(d.width, d.height)
+ img = nrgba
case cbTC8:
- bpp = 3
+ bitsPerPixel = 24
rgba = image.NewRGBA(d.width, d.height)
img = rgba
- case cbP8:
- bpp = 1
+ case cbP1, cbP2, cbP4, cbP8:
+ bitsPerPixel = d.depth
paletted = image.NewPaletted(d.width, d.height, d.palette)
img = paletted
maxPalette = uint8(len(d.palette) - 1)
case cbTCA8:
- bpp = 4
+ bitsPerPixel = 32
nrgba = image.NewNRGBA(d.width, d.height)
img = nrgba
case cbG16:
- bpp = 2
+ bitsPerPixel = 16
gray16 = image.NewGray16(d.width, d.height)
img = gray16
+ case cbGA16:
+ bitsPerPixel = 32
+ nrgba64 = image.NewNRGBA64(d.width, d.height)
+ img = nrgba64
case cbTC16:
- bpp = 6
+ bitsPerPixel = 48
rgba64 = image.NewRGBA64(d.width, d.height)
img = rgba64
case cbTCA16:
- bpp = 8
+ bitsPerPixel = 64
nrgba64 = image.NewNRGBA64(d.width, d.height)
img = nrgba64
}
+ bytesPerPixel := (bitsPerPixel + 7) / 8
+
// cr and pr are the bytes for the current and previous row.
// The +1 is for the per-row filter type, which is at cr[0].
- cr := make([]uint8, 1+bpp*d.width)
- pr := make([]uint8, 1+bpp*d.width)
+ cr := make([]uint8, 1+(bitsPerPixel*d.width+7)/8)
+ pr := make([]uint8, 1+(bitsPerPixel*d.width+7)/8)
for y := 0; y < d.height; y++ {
// Read the decompressed bytes.
case ftNone:
// No-op.
case ftSub:
- for i := bpp; i < len(cdat); i++ {
- cdat[i] += cdat[i-bpp]
+ for i := bytesPerPixel; i < len(cdat); i++ {
+ cdat[i] += cdat[i-bytesPerPixel]
}
case ftUp:
for i := 0; i < len(cdat); i++ {
cdat[i] += pdat[i]
}
case ftAverage:
- for i := 0; i < bpp; i++ {
+ for i := 0; i < bytesPerPixel; i++ {
cdat[i] += pdat[i] / 2
}
- for i := bpp; i < len(cdat); i++ {
- cdat[i] += uint8((int(cdat[i-bpp]) + int(pdat[i])) / 2)
+ for i := bytesPerPixel; i < len(cdat); i++ {
+ cdat[i] += uint8((int(cdat[i-bytesPerPixel]) + int(pdat[i])) / 2)
}
case ftPaeth:
- for i := 0; i < bpp; i++ {
+ for i := 0; i < bytesPerPixel; i++ {
cdat[i] += paeth(0, pdat[i], 0)
}
- for i := bpp; i < len(cdat); i++ {
- cdat[i] += paeth(cdat[i-bpp], pdat[i], pdat[i-bpp])
+ for i := bytesPerPixel; i < len(cdat); i++ {
+ cdat[i] += paeth(cdat[i-bytesPerPixel], pdat[i], pdat[i-bytesPerPixel])
}
default:
return nil, FormatError("bad filter type")
// Convert from bytes to colors.
switch d.cb {
+ case cbG1:
+ for x := 0; x < d.width; x += 8 {
+ b := cdat[x/8]
+ for x2 := 0; x2 < 8 && x+x2 < d.width; x2++ {
+ gray.Set(x+x2, y, image.GrayColor{(b >> 7) * 0xff})
+ b <<= 1
+ }
+ }
+ case cbG2:
+ for x := 0; x < d.width; x += 4 {
+ b := cdat[x/4]
+ for x2 := 0; x2 < 4 && x+x2 < d.width; x2++ {
+ gray.Set(x+x2, y, image.GrayColor{(b >> 6) * 0x55})
+ b <<= 2
+ }
+ }
+ case cbG4:
+ for x := 0; x < d.width; x += 2 {
+ b := cdat[x/2]
+ for x2 := 0; x2 < 2 && x+x2 < d.width; x2++ {
+ gray.Set(x+x2, y, image.GrayColor{(b >> 4) * 0x11})
+ b <<= 4
+ }
+ }
case cbG8:
for x := 0; x < d.width; x++ {
gray.Set(x, y, image.GrayColor{cdat[x]})
}
+ case cbGA8:
+ for x := 0; x < d.width; x++ {
+ ycol := cdat[2*x+0]
+ nrgba.Set(x, y, image.NRGBAColor{ycol, ycol, ycol, cdat[2*x+1]})
+ }
case cbTC8:
for x := 0; x < d.width; x++ {
rgba.Set(x, y, image.RGBAColor{cdat[3*x+0], cdat[3*x+1], cdat[3*x+2], 0xff})
}
+ case cbP1:
+ for x := 0; x < d.width; x += 8 {
+ b := cdat[x/8]
+ for x2 := 0; x2 < 8 && x+x2 < d.width; x2++ {
+ idx := b >> 7
+ if idx > maxPalette {
+ return nil, FormatError("palette index out of range")
+ }
+ paletted.SetColorIndex(x+x2, y, idx)
+ b <<= 1
+ }
+ }
+ case cbP2:
+ for x := 0; x < d.width; x += 4 {
+ b := cdat[x/4]
+ for x2 := 0; x2 < 4 && x+x2 < d.width; x2++ {
+ idx := b >> 6
+ if idx > maxPalette {
+ return nil, FormatError("palette index out of range")
+ }
+ paletted.SetColorIndex(x+x2, y, idx)
+ b <<= 2
+ }
+ }
+ case cbP4:
+ for x := 0; x < d.width; x += 2 {
+ b := cdat[x/2]
+ for x2 := 0; x2 < 2 && x+x2 < d.width; x2++ {
+ idx := b >> 4
+ if idx > maxPalette {
+ return nil, FormatError("palette index out of range")
+ }
+ paletted.SetColorIndex(x+x2, y, idx)
+ b <<= 4
+ }
+ }
case cbP8:
for x := 0; x < d.width; x++ {
if cdat[x] > maxPalette {
ycol := uint16(cdat[2*x+0])<<8 | uint16(cdat[2*x+1])
gray16.Set(x, y, image.Gray16Color{ycol})
}
+ case cbGA16:
+ for x := 0; x < d.width; x++ {
+ ycol := uint16(cdat[4*x+0])<<8 | uint16(cdat[4*x+1])
+ acol := uint16(cdat[4*x+2])<<8 | uint16(cdat[4*x+3])
+ nrgba64.Set(x, y, image.NRGBA64Color{ycol, ycol, ycol, acol})
+ }
case cbTC16:
for x := 0; x < d.width; x++ {
rcol := uint16(cdat[6*x+0])<<8 | uint16(cdat[6*x+1])
}
var cm image.ColorModel
switch d.cb {
- case cbG8:
+ case cbG1, cbG2, cbG4, cbG8:
cm = image.GrayColorModel
+ case cbGA8:
+ cm = image.NRGBAColorModel
case cbTC8:
cm = image.RGBAColorModel
- case cbP8:
+ case cbP1, cbP2, cbP4, cbP8:
cm = d.palette
case cbTCA8:
cm = image.NRGBAColorModel
case cbG16:
cm = image.Gray16ColorModel
+ case cbGA16:
+ cm = image.NRGBA64ColorModel
case cbTC16:
cm = image.RGBA64ColorModel
case cbTCA16:
"testing"
)
-// The go PNG library currently supports only a subset of the full PNG specification.
-// In particular, bit depths other than 8 or 16 are not supported, nor are grayscale-
-// alpha images.
var filenames = []string{
- //"basn0g01", // bit depth is not 8 or 16
- //"basn0g02", // bit depth is not 8 or 16
- //"basn0g04", // bit depth is not 8 or 16
+ "basn0g01",
+ "basn0g01-30",
+ "basn0g02",
+ "basn0g02-29",
+ "basn0g04",
+ "basn0g04-31",
"basn0g08",
"basn0g16",
"basn2c08",
"basn2c16",
- //"basn3p01", // bit depth is not 8 or 16
- //"basn3p02", // bit depth is not 8 or 16
- //"basn3p04", // bit depth is not 8 or 16
+ "basn3p01",
+ "basn3p02",
+ "basn3p04",
"basn3p08",
- //"basn4a08", // grayscale-alpha color model
- //"basn4a16", // grayscale-alpha color model
+ "basn4a08",
+ "basn4a16",
"basn6a08",
"basn6a16",
}
cpm, _ := cm.(image.PalettedColorModel)
var paletted *image.Paletted
if cpm != nil {
- bitdepth = 8
+ switch {
+ case len(cpm) <= 2:
+ bitdepth = 1
+ case len(cpm) <= 4:
+ bitdepth = 2
+ case len(cpm) <= 16:
+ bitdepth = 4
+ default:
+ bitdepth = 8
+ }
paletted = png.(*image.Paletted)
}
fmt.Fprintf(w, "%04x%04x%04x%04x ", nrgba64.R, nrgba64.G, nrgba64.B, nrgba64.A)
}
case cpm != nil:
+ var b, c int
for x := bounds.Min.X; x < bounds.Max.X; x++ {
- fmt.Fprintf(w, "%02x", paletted.ColorIndexAt(x, y))
+ b = b<<uint(bitdepth) | int(paletted.ColorIndexAt(x, y))
+ c++
+ if c == 8/bitdepth {
+ fmt.Fprintf(w, "%02x", b)
+ b = 0
+ c = 0
+ }
}
}
io.WriteString(w, "\n")
func TestReader(t *testing.T) {
for _, fn := range filenames {
// Read the .png file.
- image, err := readPng("testdata/pngsuite/" + fn + ".png")
+ img, err := readPng("testdata/pngsuite/" + fn + ".png")
if err != nil {
t.Error(fn, err)
continue
}
+
+ if fn == "basn4a16" {
+ // basn4a16.sng is gray + alpha but sng() will produce true color + alpha
+ // so we just check a single random pixel.
+ c := img.At(2, 1).(image.NRGBA64Color)
+ if c.R != 0x11a7 || c.G != 0x11a7 || c.B != 0x11a7 || c.A != 0x1085 {
+ t.Error(fn, fmt.Errorf("wrong pixel value at (2, 1): %x", c))
+ }
+ continue
+ }
+
piper, pipew := io.Pipe()
pb := bufio.NewReader(piper)
- go sng(pipew, fn, image)
+ go sng(pipew, fn, img)
defer piper.Close()
// Read the .sng file.
Permission to use, copy, and distribute these images for any purpose
and without fee is hereby granted.
+
+The files basn0g01-30.png, basn0g02-29.png and basn0g04-31.png are in fact
+not part of pngsuite but were created from files in pngsuite. Their non-power-
+of-two sizes makes them useful for testing bit-depths smaller than a byte.
+
The *.sng files in this directory were generated from the *.png files
-by the sng command-line tool.
+by the sng command-line tool and some hand editing. The files
+basn0g0{1,2,4}.sng were actually generated by first converting the PNG
+to a bitdepth of 8 and then running sng on them. basn4a08.sng was generated
+by from a 16-bit rgba version of basn4a08.png rather than the original
+gray + alpha.
--- /dev/null
+#SNG: from basn0g01-30.png
+IHDR {
+ width: 30; height: 30; bitdepth: 8;
+ using grayscale;
+}
+gAMA {1.0000}
+IMAGE {
+ pixels hex
+ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff
+ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff
+ffffffffffffffffffffffffffffffffffffffffffffffffffffffffff00
+ffffffffffffffffffffffffffffffffffffffffffffffffffffffff0000
+ffffffff0000ffffffffffff0000ffffffffffffffffffffffffff000000
+ffffffff0000ffffffffffff0000ffffffffffffffffffffffff00000000
+ffffffff0000ffffffffffff0000ffffffffffffffffffffff0000000000
+ffffffff0000ffff0000ffff0000ffffffffffffffffffff000000000000
+ffffffff0000ffff0000ffff0000ffffffffffffffffff00000000000000
+ffffffff0000ffff0000ffff0000ffffffffffffffff0000000000000000
+ffffffffff0000000000000000ffffffffffffffff000000000000000000
+ffffffffff0000000000000000ffffffffffffff00000000000000000000
+ffffffffffff0000ffff0000ffffffffffffff0000000000000000000000
+ffffffffffff0000ffff0000ffffffffffff000000000000000000000000
+ffffffffffffffffffffffffffffffffff00000000000000000000000000
+ffffffffffffffffffffffffffffffff0000000000000000000000000000
+ffffffffffffffffffffffffffffff000000000000000000000000000000
+ffffffffffffffffffffffffffff00000000000000000000000000000000
+ffffffffffffffffffffffffff00000000000000ffffffffffffff000000
+ffffffffffffffffffffffff0000000000000000ffffffffffffff000000
+ffffffffffffffffffffff000000000000000000ffff00000000ffff0000
+ffffffffffffffffffff00000000000000000000ffff00000000ffff0000
+ffffffffffffffffff0000000000000000000000ffffffffffffff000000
+ffffffffffffffff000000000000000000000000ffffffffffffff000000
+ffffffffffffff00000000000000000000000000ffff00000000ffff0000
+ffffffffffff0000000000000000000000000000ffff00000000ffff0000
+ffffffffff000000000000000000000000000000ffffffffffffff000000
+ffffffff00000000000000000000000000000000ffffffffffffff000000
+ffffff000000000000000000000000000000000000000000000000000000
+ffff00000000000000000000000000000000000000000000000000000000
+}
#SNG: from basn0g01.png
IHDR {
- width: 32; height: 32; bitdepth: 1;
+ width: 32; height: 32; bitdepth: 8;
using grayscale;
}
gAMA {1.0000}
IMAGE {
pixels hex
-fffffffe
-fffffffc
-fffffff8
-fffffff0
-f3f3ffe0
-f3f3ffc0
-f3f3ff80
-f333ff00
-f333fe00
-f333fc00
-f807f800
-f807f000
-fccfe000
-fccfc000
-ffff8000
-ffff0000
-fffe0000
-fffc0000
-fff80fe0
-fff00fe0
-ffe00c30
-ffc00c30
-ff800fe0
-ff000fe0
-fe000c30
-fc000c30
-f8000fe0
-f0000fe0
-e0000000
-c0000000
-80000000
-00000000
+ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff00
+ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff0000
+ffffffffffffffffffffffffffffffffffffffffffffffffffffffffff000000
+ffffffffffffffffffffffffffffffffffffffffffffffffffffffff00000000
+ffffffff0000ffffffffffff0000ffffffffffffffffffffffffff0000000000
+ffffffff0000ffffffffffff0000ffffffffffffffffffffffff000000000000
+ffffffff0000ffffffffffff0000ffffffffffffffffffffff00000000000000
+ffffffff0000ffff0000ffff0000ffffffffffffffffffff0000000000000000
+ffffffff0000ffff0000ffff0000ffffffffffffffffff000000000000000000
+ffffffff0000ffff0000ffff0000ffffffffffffffff00000000000000000000
+ffffffffff0000000000000000ffffffffffffffff0000000000000000000000
+ffffffffff0000000000000000ffffffffffffff000000000000000000000000
+ffffffffffff0000ffff0000ffffffffffffff00000000000000000000000000
+ffffffffffff0000ffff0000ffffffffffff0000000000000000000000000000
+ffffffffffffffffffffffffffffffffff000000000000000000000000000000
+ffffffffffffffffffffffffffffffff00000000000000000000000000000000
+ffffffffffffffffffffffffffffff0000000000000000000000000000000000
+ffffffffffffffffffffffffffff000000000000000000000000000000000000
+ffffffffffffffffffffffffff00000000000000ffffffffffffff0000000000
+ffffffffffffffffffffffff0000000000000000ffffffffffffff0000000000
+ffffffffffffffffffffff000000000000000000ffff00000000ffff00000000
+ffffffffffffffffffff00000000000000000000ffff00000000ffff00000000
+ffffffffffffffffff0000000000000000000000ffffffffffffff0000000000
+ffffffffffffffff000000000000000000000000ffffffffffffff0000000000
+ffffffffffffff00000000000000000000000000ffff00000000ffff00000000
+ffffffffffff0000000000000000000000000000ffff00000000ffff00000000
+ffffffffff000000000000000000000000000000ffffffffffffff0000000000
+ffffffff00000000000000000000000000000000ffffffffffffff0000000000
+ffffff0000000000000000000000000000000000000000000000000000000000
+ffff000000000000000000000000000000000000000000000000000000000000
+ff00000000000000000000000000000000000000000000000000000000000000
+0000000000000000000000000000000000000000000000000000000000000000
}
--- /dev/null
+#SNG: from basn0g02-29.png
+IHDR {
+ width: 29; height: 29; bitdepth: 8;
+ using grayscale;
+}
+gAMA {1.0000}
+IMAGE {
+ pixels hex
+0000000055555555aaaaaaaaffffffff0000000055555555aaaaaaaaff
+0000000055555555aaaaaaaaffffffff0000000055555555aaaaaaaaff
+0000000055555555aaaaaaaaffffffff0000000055555555aaaaaaaaff
+0000000055555555aaaaaaaaffffffff0000000055555555aaaaaaaaff
+55555555aaaaaaaaffffffff0000000055555555aaaaaaaaffffffff00
+55555555aaaaaaaaffffffff0000000055555555aaaaaaaaffffffff00
+55555555aaaaaaaaffffffff0000000055555555aaaaaaaaffffffff00
+55555555aaaaaaaaffffffff0000000055555555aaaaaaaaffffffff00
+aaaaaaaaffffffff0000000055555555aaaaaaaaffffffff0000000055
+aaaaaaaaffffffff0000000055555555aaaaaaaaffffffff0000000055
+aaaaaaaaffffffff0000000055555555aaaaaaaaffffffff0000000055
+aaaaaaaaffffffff0000000055555555aaaaaaaaffffffff0000000055
+ffffffff0000000055555555aaaaaaaaffffffff0000000055555555aa
+ffffffff0000000055555555aaaaaaaaffffffff0000000055555555aa
+ffffffff0000000055555555aaaaaaaaffffffff0000000055555555aa
+ffffffff0000000055555555aaaaaaaaffffffff0000000055555555aa
+0000000055555555aaaaaaaaffffffff0000000055555555aaaaaaaaff
+0000000055555555aaaaaaaaffffffff0000000055555555aaaaaaaaff
+0000000055555555aaaaaaaaffffffff0000000055555555aaaaaaaaff
+0000000055555555aaaaaaaaffffffff0000000055555555aaaaaaaaff
+55555555aaaaaaaaffffffff0000000055555555aaaaaaaaffffffff00
+55555555aaaaaaaaffffffff0000000055555555aaaaaaaaffffffff00
+55555555aaaaaaaaffffffff0000000055555555aaaaaaaaffffffff00
+55555555aaaaaaaaffffffff0000000055555555aaaaaaaaffffffff00
+aaaaaaaaffffffff0000000055555555aaaaaaaaffffffff0000000055
+aaaaaaaaffffffff0000000055555555aaaaaaaaffffffff0000000055
+aaaaaaaaffffffff0000000055555555aaaaaaaaffffffff0000000055
+aaaaaaaaffffffff0000000055555555aaaaaaaaffffffff0000000055
+ffffffff0000000055555555aaaaaaaaffffffff0000000055555555aa
+}
#SNG: from basn0g02.png
IHDR {
- width: 32; height: 32; bitdepth: 2;
+ width: 32; height: 32; bitdepth: 8;
using grayscale;
}
gAMA {1.0000}
IMAGE {
pixels hex
-0055aaff0055aaff
-0055aaff0055aaff
-0055aaff0055aaff
-0055aaff0055aaff
-55aaff0055aaff00
-55aaff0055aaff00
-55aaff0055aaff00
-55aaff0055aaff00
-aaff0055aaff0055
-aaff0055aaff0055
-aaff0055aaff0055
-aaff0055aaff0055
-ff0055aaff0055aa
-ff0055aaff0055aa
-ff0055aaff0055aa
-ff0055aaff0055aa
-0055aaff0055aaff
-0055aaff0055aaff
-0055aaff0055aaff
-0055aaff0055aaff
-55aaff0055aaff00
-55aaff0055aaff00
-55aaff0055aaff00
-55aaff0055aaff00
-aaff0055aaff0055
-aaff0055aaff0055
-aaff0055aaff0055
-aaff0055aaff0055
-ff0055aaff0055aa
-ff0055aaff0055aa
-ff0055aaff0055aa
-ff0055aaff0055aa
+0000000055555555aaaaaaaaffffffff0000000055555555aaaaaaaaffffffff
+0000000055555555aaaaaaaaffffffff0000000055555555aaaaaaaaffffffff
+0000000055555555aaaaaaaaffffffff0000000055555555aaaaaaaaffffffff
+0000000055555555aaaaaaaaffffffff0000000055555555aaaaaaaaffffffff
+55555555aaaaaaaaffffffff0000000055555555aaaaaaaaffffffff00000000
+55555555aaaaaaaaffffffff0000000055555555aaaaaaaaffffffff00000000
+55555555aaaaaaaaffffffff0000000055555555aaaaaaaaffffffff00000000
+55555555aaaaaaaaffffffff0000000055555555aaaaaaaaffffffff00000000
+aaaaaaaaffffffff0000000055555555aaaaaaaaffffffff0000000055555555
+aaaaaaaaffffffff0000000055555555aaaaaaaaffffffff0000000055555555
+aaaaaaaaffffffff0000000055555555aaaaaaaaffffffff0000000055555555
+aaaaaaaaffffffff0000000055555555aaaaaaaaffffffff0000000055555555
+ffffffff0000000055555555aaaaaaaaffffffff0000000055555555aaaaaaaa
+ffffffff0000000055555555aaaaaaaaffffffff0000000055555555aaaaaaaa
+ffffffff0000000055555555aaaaaaaaffffffff0000000055555555aaaaaaaa
+ffffffff0000000055555555aaaaaaaaffffffff0000000055555555aaaaaaaa
+0000000055555555aaaaaaaaffffffff0000000055555555aaaaaaaaffffffff
+0000000055555555aaaaaaaaffffffff0000000055555555aaaaaaaaffffffff
+0000000055555555aaaaaaaaffffffff0000000055555555aaaaaaaaffffffff
+0000000055555555aaaaaaaaffffffff0000000055555555aaaaaaaaffffffff
+55555555aaaaaaaaffffffff0000000055555555aaaaaaaaffffffff00000000
+55555555aaaaaaaaffffffff0000000055555555aaaaaaaaffffffff00000000
+55555555aaaaaaaaffffffff0000000055555555aaaaaaaaffffffff00000000
+55555555aaaaaaaaffffffff0000000055555555aaaaaaaaffffffff00000000
+aaaaaaaaffffffff0000000055555555aaaaaaaaffffffff0000000055555555
+aaaaaaaaffffffff0000000055555555aaaaaaaaffffffff0000000055555555
+aaaaaaaaffffffff0000000055555555aaaaaaaaffffffff0000000055555555
+aaaaaaaaffffffff0000000055555555aaaaaaaaffffffff0000000055555555
+ffffffff0000000055555555aaaaaaaaffffffff0000000055555555aaaaaaaa
+ffffffff0000000055555555aaaaaaaaffffffff0000000055555555aaaaaaaa
+ffffffff0000000055555555aaaaaaaaffffffff0000000055555555aaaaaaaa
+ffffffff0000000055555555aaaaaaaaffffffff0000000055555555aaaaaaaa
}
--- /dev/null
+#SNG: from basn0g04-31.png
+IHDR {
+ width: 31; height: 31; bitdepth: 8;
+ using grayscale;
+}
+gAMA {1.0000}
+IMAGE {
+ pixels hex
+00000000111111112222222233333333444444445555555566666666777777
+00000000111111112222222233333333444444445555555566666666777777
+00000000111111112222222233333333444444445555555566666666777777
+00000000111111112222222233333333444444445555555566666666777777
+11111111222222223333333344444444555555556666666677777777888888
+11111111222222223333333344444444555555556666666677777777888888
+11111111222222223333333344444444555555556666666677777777888888
+11111111222222223333333344444444555555556666666677777777888888
+22222222333333334444444455555555666666667777777788888888999999
+22222222333333334444444455555555666666667777777788888888999999
+22222222333333334444444455555555666666667777777788888888999999
+22222222333333334444444455555555666666667777777788888888999999
+33333333444444445555555566666666777777778888888899999999aaaaaa
+33333333444444445555555566666666777777778888888899999999aaaaaa
+33333333444444445555555566666666777777778888888899999999aaaaaa
+33333333444444445555555566666666777777778888888899999999aaaaaa
+444444445555555566666666777777778888888899999999aaaaaaaabbbbbb
+444444445555555566666666777777778888888899999999aaaaaaaabbbbbb
+444444445555555566666666777777778888888899999999aaaaaaaabbbbbb
+444444445555555566666666777777778888888899999999aaaaaaaabbbbbb
+5555555566666666777777778888888899999999aaaaaaaabbbbbbbbcccccc
+5555555566666666777777778888888899999999aaaaaaaabbbbbbbbcccccc
+5555555566666666777777778888888899999999aaaaaaaabbbbbbbbcccccc
+5555555566666666777777778888888899999999aaaaaaaabbbbbbbbcccccc
+66666666777777778888888899999999aaaaaaaabbbbbbbbccccccccdddddd
+66666666777777778888888899999999aaaaaaaabbbbbbbbccccccccdddddd
+66666666777777778888888899999999aaaaaaaabbbbbbbbccccccccdddddd
+66666666777777778888888899999999aaaaaaaabbbbbbbbccccccccdddddd
+777777778888888899999999aaaaaaaabbbbbbbbccccccccddddddddeeeeee
+777777778888888899999999aaaaaaaabbbbbbbbccccccccddddddddeeeeee
+777777778888888899999999aaaaaaaabbbbbbbbccccccccddddddddeeeeee
+}
#SNG: from basn0g04.png
IHDR {
- width: 32; height: 32; bitdepth: 4;
+ width: 32; height: 32; bitdepth: 8;
using grayscale;
}
gAMA {1.0000}
IMAGE {
pixels hex
-00001111222233334444555566667777
-00001111222233334444555566667777
-00001111222233334444555566667777
-00001111222233334444555566667777
-11112222333344445555666677778888
-11112222333344445555666677778888
-11112222333344445555666677778888
-11112222333344445555666677778888
-22223333444455556666777788889999
-22223333444455556666777788889999
-22223333444455556666777788889999
-22223333444455556666777788889999
-3333444455556666777788889999aaaa
-3333444455556666777788889999aaaa
-3333444455556666777788889999aaaa
-3333444455556666777788889999aaaa
-444455556666777788889999aaaabbbb
-444455556666777788889999aaaabbbb
-444455556666777788889999aaaabbbb
-444455556666777788889999aaaabbbb
-55556666777788889999aaaabbbbcccc
-55556666777788889999aaaabbbbcccc
-55556666777788889999aaaabbbbcccc
-55556666777788889999aaaabbbbcccc
-6666777788889999aaaabbbbccccdddd
-6666777788889999aaaabbbbccccdddd
-6666777788889999aaaabbbbccccdddd
-6666777788889999aaaabbbbccccdddd
-777788889999aaaabbbbccccddddeeee
-777788889999aaaabbbbccccddddeeee
-777788889999aaaabbbbccccddddeeee
-777788889999aaaabbbbccccddddeeee
+0000000011111111222222223333333344444444555555556666666677777777
+0000000011111111222222223333333344444444555555556666666677777777
+0000000011111111222222223333333344444444555555556666666677777777
+0000000011111111222222223333333344444444555555556666666677777777
+1111111122222222333333334444444455555555666666667777777788888888
+1111111122222222333333334444444455555555666666667777777788888888
+1111111122222222333333334444444455555555666666667777777788888888
+1111111122222222333333334444444455555555666666667777777788888888
+2222222233333333444444445555555566666666777777778888888899999999
+2222222233333333444444445555555566666666777777778888888899999999
+2222222233333333444444445555555566666666777777778888888899999999
+2222222233333333444444445555555566666666777777778888888899999999
+33333333444444445555555566666666777777778888888899999999aaaaaaaa
+33333333444444445555555566666666777777778888888899999999aaaaaaaa
+33333333444444445555555566666666777777778888888899999999aaaaaaaa
+33333333444444445555555566666666777777778888888899999999aaaaaaaa
+444444445555555566666666777777778888888899999999aaaaaaaabbbbbbbb
+444444445555555566666666777777778888888899999999aaaaaaaabbbbbbbb
+444444445555555566666666777777778888888899999999aaaaaaaabbbbbbbb
+444444445555555566666666777777778888888899999999aaaaaaaabbbbbbbb
+5555555566666666777777778888888899999999aaaaaaaabbbbbbbbcccccccc
+5555555566666666777777778888888899999999aaaaaaaabbbbbbbbcccccccc
+5555555566666666777777778888888899999999aaaaaaaabbbbbbbbcccccccc
+5555555566666666777777778888888899999999aaaaaaaabbbbbbbbcccccccc
+66666666777777778888888899999999aaaaaaaabbbbbbbbccccccccdddddddd
+66666666777777778888888899999999aaaaaaaabbbbbbbbccccccccdddddddd
+66666666777777778888888899999999aaaaaaaabbbbbbbbccccccccdddddddd
+66666666777777778888888899999999aaaaaaaabbbbbbbbccccccccdddddddd
+777777778888888899999999aaaaaaaabbbbbbbbccccccccddddddddeeeeeeee
+777777778888888899999999aaaaaaaabbbbbbbbccccccccddddddddeeeeeeee
+777777778888888899999999aaaaaaaabbbbbbbbccccccccddddddddeeeeeeee
+777777778888888899999999aaaaaaaabbbbbbbbccccccccddddddddeeeeeeee
}
using color palette;
}
gAMA {1.0000}
-sBIT {
- red: 1; green: 1; blue: 1;
-}
PLTE {
- ( 0,255, 0) # rgb = (0x00,0xff,0x00) green1
- (255, 0, 0) # rgb = (0xff,0x00,0x00) red1
- (255,255, 0) # rgb = (0xff,0xff,0x00) yellow1
- ( 0, 0,255) # rgb = (0x00,0x00,0xff) blue1
+ ( 0,255, 0) # rgb = (0x00,0xff,0x00)
+ (255, 0, 0) # rgb = (0xff,0x00,0x00)
+ (255,255, 0) # rgb = (0xff,0xff,0x00)
+ ( 0, 0,255) # rgb = (0x00,0x00,0xff)
}
IMAGE {
pixels hex
using color palette;
}
gAMA {1.0000}
-sBIT {
- red: 4; green: 4; blue: 4;
-}
PLTE {
( 34, 0,255) # rgb = (0x22,0x00,0xff)
- ( 0,255,255) # rgb = (0x00,0xff,0xff) cyan1
+ ( 0,255,255) # rgb = (0x00,0xff,0xff)
(136, 0,255) # rgb = (0x88,0x00,0xff)
( 34,255, 0) # rgb = (0x22,0xff,0x00)
( 0,153,255) # rgb = (0x00,0x99,0xff)
(255,102, 0) # rgb = (0xff,0x66,0x00)
(221, 0,255) # rgb = (0xdd,0x00,0xff)
(119,255, 0) # rgb = (0x77,0xff,0x00)
- (255, 0, 0) # rgb = (0xff,0x00,0x00) red1
+ (255, 0, 0) # rgb = (0xff,0x00,0x00)
( 0,255,153) # rgb = (0x00,0xff,0x99)
(221,255, 0) # rgb = (0xdd,0xff,0x00)
(255, 0,187) # rgb = (0xff,0x00,0xbb)
#SNG: from basn4a08.png
IHDR {
width: 32; height: 32; bitdepth: 8;
- using grayscale alpha;
+ using color alpha;
}
gAMA {1.0000}
IMAGE {
pixels hex
-ff00 ff08 ff10 ff18 ff20 ff29 ff31 ff39 ff41 ff4a ff52 ff5a ff62 ff6a ff73 ff7b ff83 ff8b ff94 ff9c ffa4 ffac ffb4 ffbd ffc5 ffcd ffd5 ffde ffe6 ffee fff6 ffff
-f600 f608 f610 f618 f620 f629 f631 f639 f641 f64a f652 f65a f662 f66a f673 f67b f683 f68b f694 f69c f6a4 f6ac f6b4 f6bd f6c5 f6cd f6d5 f6de f6e6 f6ee f6f6 f6ff
-ee00 ee08 ee10 ee18 ee20 ee29 ee31 ee39 ee41 ee4a ee52 ee5a ee62 ee6a ee73 ee7b ee83 ee8b ee94 ee9c eea4 eeac eeb4 eebd eec5 eecd eed5 eede eee6 eeee eef6 eeff
-e600 e608 e610 e618 e620 e629 e631 e639 e641 e64a e652 e65a e662 e66a e673 e67b e683 e68b e694 e69c e6a4 e6ac e6b4 e6bd e6c5 e6cd e6d5 e6de e6e6 e6ee e6f6 e6ff
-de00 de08 de10 de18 de20 de29 de31 de39 de41 de4a de52 de5a de62 de6a de73 de7b de83 de8b de94 de9c dea4 deac deb4 debd dec5 decd ded5 dede dee6 deee def6 deff
-d500 d508 d510 d518 d520 d529 d531 d539 d541 d54a d552 d55a d562 d56a d573 d57b d583 d58b d594 d59c d5a4 d5ac d5b4 d5bd d5c5 d5cd d5d5 d5de d5e6 d5ee d5f6 d5ff
-cd00 cd08 cd10 cd18 cd20 cd29 cd31 cd39 cd41 cd4a cd52 cd5a cd62 cd6a cd73 cd7b cd83 cd8b cd94 cd9c cda4 cdac cdb4 cdbd cdc5 cdcd cdd5 cdde cde6 cdee cdf6 cdff
-c500 c508 c510 c518 c520 c529 c531 c539 c541 c54a c552 c55a c562 c56a c573 c57b c583 c58b c594 c59c c5a4 c5ac c5b4 c5bd c5c5 c5cd c5d5 c5de c5e6 c5ee c5f6 c5ff
-bd00 bd08 bd10 bd18 bd20 bd29 bd31 bd39 bd41 bd4a bd52 bd5a bd62 bd6a bd73 bd7b bd83 bd8b bd94 bd9c bda4 bdac bdb4 bdbd bdc5 bdcd bdd5 bdde bde6 bdee bdf6 bdff
-b400 b408 b410 b418 b420 b429 b431 b439 b441 b44a b452 b45a b462 b46a b473 b47b b483 b48b b494 b49c b4a4 b4ac b4b4 b4bd b4c5 b4cd b4d5 b4de b4e6 b4ee b4f6 b4ff
-ac00 ac08 ac10 ac18 ac20 ac29 ac31 ac39 ac41 ac4a ac52 ac5a ac62 ac6a ac73 ac7b ac83 ac8b ac94 ac9c aca4 acac acb4 acbd acc5 accd acd5 acde ace6 acee acf6 acff
-a400 a408 a410 a418 a420 a429 a431 a439 a441 a44a a452 a45a a462 a46a a473 a47b a483 a48b a494 a49c a4a4 a4ac a4b4 a4bd a4c5 a4cd a4d5 a4de a4e6 a4ee a4f6 a4ff
-9c00 9c08 9c10 9c18 9c20 9c29 9c31 9c39 9c41 9c4a 9c52 9c5a 9c62 9c6a 9c73 9c7b 9c83 9c8b 9c94 9c9c 9ca4 9cac 9cb4 9cbd 9cc5 9ccd 9cd5 9cde 9ce6 9cee 9cf6 9cff
-9400 9408 9410 9418 9420 9429 9431 9439 9441 944a 9452 945a 9462 946a 9473 947b 9483 948b 9494 949c 94a4 94ac 94b4 94bd 94c5 94cd 94d5 94de 94e6 94ee 94f6 94ff
-8b00 8b08 8b10 8b18 8b20 8b29 8b31 8b39 8b41 8b4a 8b52 8b5a 8b62 8b6a 8b73 8b7b 8b83 8b8b 8b94 8b9c 8ba4 8bac 8bb4 8bbd 8bc5 8bcd 8bd5 8bde 8be6 8bee 8bf6 8bff
-8300 8308 8310 8318 8320 8329 8331 8339 8341 834a 8352 835a 8362 836a 8373 837b 8383 838b 8394 839c 83a4 83ac 83b4 83bd 83c5 83cd 83d5 83de 83e6 83ee 83f6 83ff
-7b00 7b08 7b10 7b18 7b20 7b29 7b31 7b39 7b41 7b4a 7b52 7b5a 7b62 7b6a 7b73 7b7b 7b83 7b8b 7b94 7b9c 7ba4 7bac 7bb4 7bbd 7bc5 7bcd 7bd5 7bde 7be6 7bee 7bf6 7bff
-7300 7308 7310 7318 7320 7329 7331 7339 7341 734a 7352 735a 7362 736a 7373 737b 7383 738b 7394 739c 73a4 73ac 73b4 73bd 73c5 73cd 73d5 73de 73e6 73ee 73f6 73ff
-6a00 6a08 6a10 6a18 6a20 6a29 6a31 6a39 6a41 6a4a 6a52 6a5a 6a62 6a6a 6a73 6a7b 6a83 6a8b 6a94 6a9c 6aa4 6aac 6ab4 6abd 6ac5 6acd 6ad5 6ade 6ae6 6aee 6af6 6aff
-6200 6208 6210 6218 6220 6229 6231 6239 6241 624a 6252 625a 6262 626a 6273 627b 6283 628b 6294 629c 62a4 62ac 62b4 62bd 62c5 62cd 62d5 62de 62e6 62ee 62f6 62ff
-5a00 5a08 5a10 5a18 5a20 5a29 5a31 5a39 5a41 5a4a 5a52 5a5a 5a62 5a6a 5a73 5a7b 5a83 5a8b 5a94 5a9c 5aa4 5aac 5ab4 5abd 5ac5 5acd 5ad5 5ade 5ae6 5aee 5af6 5aff
-5200 5208 5210 5218 5220 5229 5231 5239 5241 524a 5252 525a 5262 526a 5273 527b 5283 528b 5294 529c 52a4 52ac 52b4 52bd 52c5 52cd 52d5 52de 52e6 52ee 52f6 52ff
-4a00 4a08 4a10 4a18 4a20 4a29 4a31 4a39 4a41 4a4a 4a52 4a5a 4a62 4a6a 4a73 4a7b 4a83 4a8b 4a94 4a9c 4aa4 4aac 4ab4 4abd 4ac5 4acd 4ad5 4ade 4ae6 4aee 4af6 4aff
-4100 4108 4110 4118 4120 4129 4131 4139 4141 414a 4152 415a 4162 416a 4173 417b 4183 418b 4194 419c 41a4 41ac 41b4 41bd 41c5 41cd 41d5 41de 41e6 41ee 41f6 41ff
-3900 3908 3910 3918 3920 3929 3931 3939 3941 394a 3952 395a 3962 396a 3973 397b 3983 398b 3994 399c 39a4 39ac 39b4 39bd 39c5 39cd 39d5 39de 39e6 39ee 39f6 39ff
-3100 3108 3110 3118 3120 3129 3131 3139 3141 314a 3152 315a 3162 316a 3173 317b 3183 318b 3194 319c 31a4 31ac 31b4 31bd 31c5 31cd 31d5 31de 31e6 31ee 31f6 31ff
-2900 2908 2910 2918 2920 2929 2931 2939 2941 294a 2952 295a 2962 296a 2973 297b 2983 298b 2994 299c 29a4 29ac 29b4 29bd 29c5 29cd 29d5 29de 29e6 29ee 29f6 29ff
-2000 2008 2010 2018 2020 2029 2031 2039 2041 204a 2052 205a 2062 206a 2073 207b 2083 208b 2094 209c 20a4 20ac 20b4 20bd 20c5 20cd 20d5 20de 20e6 20ee 20f6 20ff
-1800 1808 1810 1818 1820 1829 1831 1839 1841 184a 1852 185a 1862 186a 1873 187b 1883 188b 1894 189c 18a4 18ac 18b4 18bd 18c5 18cd 18d5 18de 18e6 18ee 18f6 18ff
-1000 1008 1010 1018 1020 1029 1031 1039 1041 104a 1052 105a 1062 106a 1073 107b 1083 108b 1094 109c 10a4 10ac 10b4 10bd 10c5 10cd 10d5 10de 10e6 10ee 10f6 10ff
-0800 0808 0810 0818 0820 0829 0831 0839 0841 084a 0852 085a 0862 086a 0873 087b 0883 088b 0894 089c 08a4 08ac 08b4 08bd 08c5 08cd 08d5 08de 08e6 08ee 08f6 08ff
-0000 0008 0010 0018 0020 0029 0031 0039 0041 004a 0052 005a 0062 006a 0073 007b 0083 008b 0094 009c 00a4 00ac 00b4 00bd 00c5 00cd 00d5 00de 00e6 00ee 00f6 00ff
+ffffff00 ffffff08 ffffff10 ffffff18 ffffff20 ffffff29 ffffff31 ffffff39 ffffff41 ffffff4a ffffff52 ffffff5a ffffff62 ffffff6a ffffff73 ffffff7b ffffff83 ffffff8b ffffff94 ffffff9c ffffffa4 ffffffac ffffffb4 ffffffbd ffffffc5 ffffffcd ffffffd5 ffffffde ffffffe6 ffffffee fffffff6 ffffffff
+f6f6f600 f6f6f608 f6f6f610 f6f6f618 f6f6f620 f6f6f629 f6f6f631 f6f6f639 f6f6f641 f6f6f64a f6f6f652 f6f6f65a f6f6f662 f6f6f66a f6f6f673 f6f6f67b f6f6f683 f6f6f68b f6f6f694 f6f6f69c f6f6f6a4 f6f6f6ac f6f6f6b4 f6f6f6bd f6f6f6c5 f6f6f6cd f6f6f6d5 f6f6f6de f6f6f6e6 f6f6f6ee f6f6f6f6 f6f6f6ff
+eeeeee00 eeeeee08 eeeeee10 eeeeee18 eeeeee20 eeeeee29 eeeeee31 eeeeee39 eeeeee41 eeeeee4a eeeeee52 eeeeee5a eeeeee62 eeeeee6a eeeeee73 eeeeee7b eeeeee83 eeeeee8b eeeeee94 eeeeee9c eeeeeea4 eeeeeeac eeeeeeb4 eeeeeebd eeeeeec5 eeeeeecd eeeeeed5 eeeeeede eeeeeee6 eeeeeeee eeeeeef6 eeeeeeff
+e6e6e600 e6e6e608 e6e6e610 e6e6e618 e6e6e620 e6e6e629 e6e6e631 e6e6e639 e6e6e641 e6e6e64a e6e6e652 e6e6e65a e6e6e662 e6e6e66a e6e6e673 e6e6e67b e6e6e683 e6e6e68b e6e6e694 e6e6e69c e6e6e6a4 e6e6e6ac e6e6e6b4 e6e6e6bd e6e6e6c5 e6e6e6cd e6e6e6d5 e6e6e6de e6e6e6e6 e6e6e6ee e6e6e6f6 e6e6e6ff
+dedede00 dedede08 dedede10 dedede18 dedede20 dedede29 dedede31 dedede39 dedede41 dedede4a dedede52 dedede5a dedede62 dedede6a dedede73 dedede7b dedede83 dedede8b dedede94 dedede9c dededea4 dededeac dededeb4 dededebd dededec5 dededecd dededed5 dededede dededee6 dededeee dededef6 dededeff
+d5d5d500 d5d5d508 d5d5d510 d5d5d518 d5d5d520 d5d5d529 d5d5d531 d5d5d539 d5d5d541 d5d5d54a d5d5d552 d5d5d55a d5d5d562 d5d5d56a d5d5d573 d5d5d57b d5d5d583 d5d5d58b d5d5d594 d5d5d59c d5d5d5a4 d5d5d5ac d5d5d5b4 d5d5d5bd d5d5d5c5 d5d5d5cd d5d5d5d5 d5d5d5de d5d5d5e6 d5d5d5ee d5d5d5f6 d5d5d5ff
+cdcdcd00 cdcdcd08 cdcdcd10 cdcdcd18 cdcdcd20 cdcdcd29 cdcdcd31 cdcdcd39 cdcdcd41 cdcdcd4a cdcdcd52 cdcdcd5a cdcdcd62 cdcdcd6a cdcdcd73 cdcdcd7b cdcdcd83 cdcdcd8b cdcdcd94 cdcdcd9c cdcdcda4 cdcdcdac cdcdcdb4 cdcdcdbd cdcdcdc5 cdcdcdcd cdcdcdd5 cdcdcdde cdcdcde6 cdcdcdee cdcdcdf6 cdcdcdff
+c5c5c500 c5c5c508 c5c5c510 c5c5c518 c5c5c520 c5c5c529 c5c5c531 c5c5c539 c5c5c541 c5c5c54a c5c5c552 c5c5c55a c5c5c562 c5c5c56a c5c5c573 c5c5c57b c5c5c583 c5c5c58b c5c5c594 c5c5c59c c5c5c5a4 c5c5c5ac c5c5c5b4 c5c5c5bd c5c5c5c5 c5c5c5cd c5c5c5d5 c5c5c5de c5c5c5e6 c5c5c5ee c5c5c5f6 c5c5c5ff
+bdbdbd00 bdbdbd08 bdbdbd10 bdbdbd18 bdbdbd20 bdbdbd29 bdbdbd31 bdbdbd39 bdbdbd41 bdbdbd4a bdbdbd52 bdbdbd5a bdbdbd62 bdbdbd6a bdbdbd73 bdbdbd7b bdbdbd83 bdbdbd8b bdbdbd94 bdbdbd9c bdbdbda4 bdbdbdac bdbdbdb4 bdbdbdbd bdbdbdc5 bdbdbdcd bdbdbdd5 bdbdbdde bdbdbde6 bdbdbdee bdbdbdf6 bdbdbdff
+b4b4b400 b4b4b408 b4b4b410 b4b4b418 b4b4b420 b4b4b429 b4b4b431 b4b4b439 b4b4b441 b4b4b44a b4b4b452 b4b4b45a b4b4b462 b4b4b46a b4b4b473 b4b4b47b b4b4b483 b4b4b48b b4b4b494 b4b4b49c b4b4b4a4 b4b4b4ac b4b4b4b4 b4b4b4bd b4b4b4c5 b4b4b4cd b4b4b4d5 b4b4b4de b4b4b4e6 b4b4b4ee b4b4b4f6 b4b4b4ff
+acacac00 acacac08 acacac10 acacac18 acacac20 acacac29 acacac31 acacac39 acacac41 acacac4a acacac52 acacac5a acacac62 acacac6a acacac73 acacac7b acacac83 acacac8b acacac94 acacac9c acacaca4 acacacac acacacb4 acacacbd acacacc5 acacaccd acacacd5 acacacde acacace6 acacacee acacacf6 acacacff
+a4a4a400 a4a4a408 a4a4a410 a4a4a418 a4a4a420 a4a4a429 a4a4a431 a4a4a439 a4a4a441 a4a4a44a a4a4a452 a4a4a45a a4a4a462 a4a4a46a a4a4a473 a4a4a47b a4a4a483 a4a4a48b a4a4a494 a4a4a49c a4a4a4a4 a4a4a4ac a4a4a4b4 a4a4a4bd a4a4a4c5 a4a4a4cd a4a4a4d5 a4a4a4de a4a4a4e6 a4a4a4ee a4a4a4f6 a4a4a4ff
+9c9c9c00 9c9c9c08 9c9c9c10 9c9c9c18 9c9c9c20 9c9c9c29 9c9c9c31 9c9c9c39 9c9c9c41 9c9c9c4a 9c9c9c52 9c9c9c5a 9c9c9c62 9c9c9c6a 9c9c9c73 9c9c9c7b 9c9c9c83 9c9c9c8b 9c9c9c94 9c9c9c9c 9c9c9ca4 9c9c9cac 9c9c9cb4 9c9c9cbd 9c9c9cc5 9c9c9ccd 9c9c9cd5 9c9c9cde 9c9c9ce6 9c9c9cee 9c9c9cf6 9c9c9cff
+94949400 94949408 94949410 94949418 94949420 94949429 94949431 94949439 94949441 9494944a 94949452 9494945a 94949462 9494946a 94949473 9494947b 94949483 9494948b 94949494 9494949c 949494a4 949494ac 949494b4 949494bd 949494c5 949494cd 949494d5 949494de 949494e6 949494ee 949494f6 949494ff
+8b8b8b00 8b8b8b08 8b8b8b10 8b8b8b18 8b8b8b20 8b8b8b29 8b8b8b31 8b8b8b39 8b8b8b41 8b8b8b4a 8b8b8b52 8b8b8b5a 8b8b8b62 8b8b8b6a 8b8b8b73 8b8b8b7b 8b8b8b83 8b8b8b8b 8b8b8b94 8b8b8b9c 8b8b8ba4 8b8b8bac 8b8b8bb4 8b8b8bbd 8b8b8bc5 8b8b8bcd 8b8b8bd5 8b8b8bde 8b8b8be6 8b8b8bee 8b8b8bf6 8b8b8bff
+83838300 83838308 83838310 83838318 83838320 83838329 83838331 83838339 83838341 8383834a 83838352 8383835a 83838362 8383836a 83838373 8383837b 83838383 8383838b 83838394 8383839c 838383a4 838383ac 838383b4 838383bd 838383c5 838383cd 838383d5 838383de 838383e6 838383ee 838383f6 838383ff
+7b7b7b00 7b7b7b08 7b7b7b10 7b7b7b18 7b7b7b20 7b7b7b29 7b7b7b31 7b7b7b39 7b7b7b41 7b7b7b4a 7b7b7b52 7b7b7b5a 7b7b7b62 7b7b7b6a 7b7b7b73 7b7b7b7b 7b7b7b83 7b7b7b8b 7b7b7b94 7b7b7b9c 7b7b7ba4 7b7b7bac 7b7b7bb4 7b7b7bbd 7b7b7bc5 7b7b7bcd 7b7b7bd5 7b7b7bde 7b7b7be6 7b7b7bee 7b7b7bf6 7b7b7bff
+73737300 73737308 73737310 73737318 73737320 73737329 73737331 73737339 73737341 7373734a 73737352 7373735a 73737362 7373736a 73737373 7373737b 73737383 7373738b 73737394 7373739c 737373a4 737373ac 737373b4 737373bd 737373c5 737373cd 737373d5 737373de 737373e6 737373ee 737373f6 737373ff
+6a6a6a00 6a6a6a08 6a6a6a10 6a6a6a18 6a6a6a20 6a6a6a29 6a6a6a31 6a6a6a39 6a6a6a41 6a6a6a4a 6a6a6a52 6a6a6a5a 6a6a6a62 6a6a6a6a 6a6a6a73 6a6a6a7b 6a6a6a83 6a6a6a8b 6a6a6a94 6a6a6a9c 6a6a6aa4 6a6a6aac 6a6a6ab4 6a6a6abd 6a6a6ac5 6a6a6acd 6a6a6ad5 6a6a6ade 6a6a6ae6 6a6a6aee 6a6a6af6 6a6a6aff
+62626200 62626208 62626210 62626218 62626220 62626229 62626231 62626239 62626241 6262624a 62626252 6262625a 62626262 6262626a 62626273 6262627b 62626283 6262628b 62626294 6262629c 626262a4 626262ac 626262b4 626262bd 626262c5 626262cd 626262d5 626262de 626262e6 626262ee 626262f6 626262ff
+5a5a5a00 5a5a5a08 5a5a5a10 5a5a5a18 5a5a5a20 5a5a5a29 5a5a5a31 5a5a5a39 5a5a5a41 5a5a5a4a 5a5a5a52 5a5a5a5a 5a5a5a62 5a5a5a6a 5a5a5a73 5a5a5a7b 5a5a5a83 5a5a5a8b 5a5a5a94 5a5a5a9c 5a5a5aa4 5a5a5aac 5a5a5ab4 5a5a5abd 5a5a5ac5 5a5a5acd 5a5a5ad5 5a5a5ade 5a5a5ae6 5a5a5aee 5a5a5af6 5a5a5aff
+52525200 52525208 52525210 52525218 52525220 52525229 52525231 52525239 52525241 5252524a 52525252 5252525a 52525262 5252526a 52525273 5252527b 52525283 5252528b 52525294 5252529c 525252a4 525252ac 525252b4 525252bd 525252c5 525252cd 525252d5 525252de 525252e6 525252ee 525252f6 525252ff
+4a4a4a00 4a4a4a08 4a4a4a10 4a4a4a18 4a4a4a20 4a4a4a29 4a4a4a31 4a4a4a39 4a4a4a41 4a4a4a4a 4a4a4a52 4a4a4a5a 4a4a4a62 4a4a4a6a 4a4a4a73 4a4a4a7b 4a4a4a83 4a4a4a8b 4a4a4a94 4a4a4a9c 4a4a4aa4 4a4a4aac 4a4a4ab4 4a4a4abd 4a4a4ac5 4a4a4acd 4a4a4ad5 4a4a4ade 4a4a4ae6 4a4a4aee 4a4a4af6 4a4a4aff
+41414100 41414108 41414110 41414118 41414120 41414129 41414131 41414139 41414141 4141414a 41414152 4141415a 41414162 4141416a 41414173 4141417b 41414183 4141418b 41414194 4141419c 414141a4 414141ac 414141b4 414141bd 414141c5 414141cd 414141d5 414141de 414141e6 414141ee 414141f6 414141ff
+39393900 39393908 39393910 39393918 39393920 39393929 39393931 39393939 39393941 3939394a 39393952 3939395a 39393962 3939396a 39393973 3939397b 39393983 3939398b 39393994 3939399c 393939a4 393939ac 393939b4 393939bd 393939c5 393939cd 393939d5 393939de 393939e6 393939ee 393939f6 393939ff
+31313100 31313108 31313110 31313118 31313120 31313129 31313131 31313139 31313141 3131314a 31313152 3131315a 31313162 3131316a 31313173 3131317b 31313183 3131318b 31313194 3131319c 313131a4 313131ac 313131b4 313131bd 313131c5 313131cd 313131d5 313131de 313131e6 313131ee 313131f6 313131ff
+29292900 29292908 29292910 29292918 29292920 29292929 29292931 29292939 29292941 2929294a 29292952 2929295a 29292962 2929296a 29292973 2929297b 29292983 2929298b 29292994 2929299c 292929a4 292929ac 292929b4 292929bd 292929c5 292929cd 292929d5 292929de 292929e6 292929ee 292929f6 292929ff
+20202000 20202008 20202010 20202018 20202020 20202029 20202031 20202039 20202041 2020204a 20202052 2020205a 20202062 2020206a 20202073 2020207b 20202083 2020208b 20202094 2020209c 202020a4 202020ac 202020b4 202020bd 202020c5 202020cd 202020d5 202020de 202020e6 202020ee 202020f6 202020ff
+18181800 18181808 18181810 18181818 18181820 18181829 18181831 18181839 18181841 1818184a 18181852 1818185a 18181862 1818186a 18181873 1818187b 18181883 1818188b 18181894 1818189c 181818a4 181818ac 181818b4 181818bd 181818c5 181818cd 181818d5 181818de 181818e6 181818ee 181818f6 181818ff
+10101000 10101008 10101010 10101018 10101020 10101029 10101031 10101039 10101041 1010104a 10101052 1010105a 10101062 1010106a 10101073 1010107b 10101083 1010108b 10101094 1010109c 101010a4 101010ac 101010b4 101010bd 101010c5 101010cd 101010d5 101010de 101010e6 101010ee 101010f6 101010ff
+08080800 08080808 08080810 08080818 08080820 08080829 08080831 08080839 08080841 0808084a 08080852 0808085a 08080862 0808086a 08080873 0808087b 08080883 0808088b 08080894 0808089c 080808a4 080808ac 080808b4 080808bd 080808c5 080808cd 080808d5 080808de 080808e6 080808ee 080808f6 080808ff
+00000000 00000008 00000010 00000018 00000020 00000029 00000031 00000039 00000041 0000004a 00000052 0000005a 00000062 0000006a 00000073 0000007b 00000083 0000008b 00000094 0000009c 000000a4 000000ac 000000b4 000000bd 000000c5 000000cd 000000d5 000000de 000000e6 000000ee 000000f6 000000ff
}
func (x *suffixSortable) updateGroups(offset int) {
- prev := len(x.sa) - 1
- group := x.inv[x.sa[prev]+x.h]
- for i := prev; i >= 0; i-- {
- if g := x.inv[x.sa[i]+x.h]; g < group {
- if prev == i+1 { // previous group had size 1 and is thus sorted
- x.sa[i+1] = -1
- }
+ bounds := make([]int, 0, 4)
+ group := x.inv[x.sa[0]+x.h]
+ for i := 1; i < len(x.sa); i++ {
+ if g := x.inv[x.sa[i]+x.h]; g > group {
+ bounds = append(bounds, i)
group = g
- prev = i
}
- x.inv[x.sa[i]] = prev + offset
- if prev == 0 { // first group has size 1 and is thus sorted
- x.sa[0] = -1
+ }
+ bounds = append(bounds, len(x.sa))
+
+ // update the group numberings after all new groups are determined
+ prev := 0
+ for _, b := range bounds {
+ for i := prev; i < b; i++ {
+ x.inv[x.sa[i]] = offset + b - 1
+ }
+ if b-prev == 1 {
+ x.sa[prev] = -1
}
+ prev = b
}
}
}
-func (x *Index) search(s []byte) int {
- return sort.Search(len(x.sa), func(i int) bool { return bytes.Compare(x.at(i), s) >= 0 })
+// lookupAll returns a slice into the matching region of the index.
+// The runtime is O(log(N)*len(s)).
+func (x *Index) lookupAll(s []byte) []int {
+ // find matching suffix index range [i:j]
+ // find the first index where s would be the prefix
+ i := sort.Search(len(x.sa), func(i int) bool { return bytes.Compare(x.at(i), s) >= 0 })
+ // starting at i, find the first index at which s is not a prefix
+ j := i + sort.Search(len(x.sa)-i, func(j int) bool { return !bytes.HasPrefix(x.at(j+i), s) })
+ return x.sa[i:j]
}
// Lookup returns an unsorted list of at most n indices where the byte string s
// occurs in the indexed data. If n < 0, all occurrences are returned.
// The result is nil if s is empty, s is not found, or n == 0.
-// Lookup time is O((log(N) + len(result))*len(s)) where N is the
+// Lookup time is O(log(N)*len(s) + len(result)) where N is the
// size of the indexed data.
//
func (x *Index) Lookup(s []byte, n int) (result []int) {
if len(s) > 0 && n != 0 {
- // find matching suffix index i
- i := x.search(s)
- // x.at(i-1) < s <= x.at(i)
-
- // collect the following suffixes with matching prefixes
- for (n < 0 || len(result) < n) && i < len(x.sa) && bytes.HasPrefix(x.at(i), s) {
- result = append(result, x.sa[i])
- i++
+ matches := x.lookupAll(s)
+ if len(matches) < n || n < 0 {
+ n = len(matches)
+ }
+ if n > 0 {
+ result = make([]int, n)
+ copy(result, matches)
}
}
return
"to (come|the)?",
},
},
+
+ {
+ "godoc simulation",
+ "package main\n\nimport(\n \"rand\"\n ",
+ []string{},
+ },
}
WriteAt(p []byte, off int64) (n int, err os.Error)
}
-// ReadByter is the interface that wraps the ReadByte method.
+// ByteReader is the interface that wraps the ReadByte method.
//
// ReadByte reads and returns the next byte from the input.
// If no byte is available, err will be set.
-type ReadByter interface {
+type ByteReader interface {
ReadByte() (c byte, err os.Error)
}
+// RuneReader is the interface that wraps the ReadRune method.
+//
+// ReadRune reads a single UTF-8 encoded Unicode character
+// and returns the rune and its size in bytes. If no character is
+// available, err will be set.
+type RuneReader interface {
+ ReadRune() (rune int, size int, err os.Error)
+}
+
// WriteString writes the contents of the string s to w, which accepts an array of bytes.
func WriteString(w Writer, s string) (n int, err os.Error) {
return w.Write([]byte(s))
nconflict := 0
for i := 0; i < 10000; i++ {
+ // TODO(rsc): use filepath.Join
name := dir + "/" + prefix + nextSuffix()
f, err = os.Open(name, os.O_RDWR|os.O_CREATE|os.O_EXCL, 0600)
if pe, ok := err.(*os.PathError); ok && pe.Error == os.EEXIST {
}
return
}
+
+// TempDir creates a new temporary directory in the directory dir
+// with a name beginning with prefix and returns the path of the
+// new directory. If dir is the empty string, TempDir uses the
+// default directory for temporary files (see os.TempDir).
+// Multiple programs calling TempDir simultaneously
+// will not choose the same directory. It is the caller's responsibility
+// to remove the directory when no longer needed.
+func TempDir(dir, prefix string) (name string, err os.Error) {
+ if dir == "" {
+ dir = os.TempDir()
+ }
+
+ nconflict := 0
+ for i := 0; i < 10000; i++ {
+ // TODO(rsc): use filepath.Join
+ try := dir + "/" + prefix + nextSuffix()
+ err = os.Mkdir(try, 0700)
+ if pe, ok := err.(*os.PathError); ok && pe.Error == os.EEXIST {
+ if nconflict++; nconflict > 10 {
+ rand = reseed()
+ }
+ continue
+ }
+ if err == nil {
+ name = try
+ }
+ break
+ }
+ return
+}
t.Errorf("TempFile(dir, `ioutil_test`) = %v, %v", f, err)
}
if f != nil {
+ f.Close()
+ os.Remove(f.Name())
re := regexp.MustCompile("^" + regexp.QuoteMeta(dir) + "/ioutil_test[0-9]+$")
if !re.MatchString(f.Name()) {
t.Errorf("TempFile(`"+dir+"`, `ioutil_test`) created bad name %s", f.Name())
}
- os.Remove(f.Name())
}
- f.Close()
+}
+
+func TestTempDir(t *testing.T) {
+ name, err := TempDir("/_not_exists_", "foo")
+ if name != "" || err == nil {
+ t.Errorf("TempDir(`/_not_exists_`, `foo`) = %v, %v", name, err)
+ }
+
+ dir := os.TempDir()
+ name, err = TempDir(dir, "ioutil_test")
+ if name == "" || err != nil {
+ t.Errorf("TempDir(dir, `ioutil_test`) = %v, %v", name, err)
+ }
+ if name != "" {
+ os.Remove(name)
+ re := regexp.MustCompile("^" + regexp.QuoteMeta(dir) + "/ioutil_test[0-9]+$")
+ if !re.MatchString(name) {
+ t.Errorf("TempDir(`"+dir+"`, `ioutil_test`) created bad name %s", name)
+ }
+ }
}
import (
"container/vector"
+ "encoding/base64"
"os"
"reflect"
"runtime"
} else {
var f reflect.StructField
var ok bool
- // First try for field with that tag.
st := sv.Type().(*reflect.StructType)
- for i := 0; i < sv.NumField(); i++ {
- f = st.Field(i)
- if f.Tag == key {
- ok = true
- break
+ // First try for field with that tag.
+ if isValidTag(key) {
+ for i := 0; i < sv.NumField(); i++ {
+ f = st.Field(i)
+ if f.Tag == key {
+ ok = true
+ break
+ }
}
}
if !ok {
}
case '"': // string
- s, ok := unquote(item)
+ s, ok := unquoteBytes(item)
if !ok {
d.error(errPhase)
}
switch v := v.(type) {
default:
d.saveError(&UnmarshalTypeError{"string", v.Type()})
+ case *reflect.SliceValue:
+ if v.Type() != byteSliceType {
+ d.saveError(&UnmarshalTypeError{"string", v.Type()})
+ break
+ }
+ b := make([]byte, base64.StdEncoding.DecodedLen(len(s)))
+ n, err := base64.StdEncoding.Decode(b, s)
+ if err != nil {
+ d.saveError(err)
+ break
+ }
+ v.Set(reflect.NewValue(b[0:n]).(*reflect.SliceValue))
case *reflect.StringValue:
- v.Set(s)
+ v.Set(string(s))
case *reflect.InterfaceValue:
- v.Set(reflect.NewValue(s))
+ v.Set(reflect.NewValue(string(s)))
}
default: // number
// unquote converts a quoted JSON string literal s into an actual string t.
// The rules are different than for Go, so cannot use strconv.Unquote.
func unquote(s []byte) (t string, ok bool) {
+ s, ok = unquoteBytes(s)
+ t = string(s)
+ return
+}
+
+func unquoteBytes(s []byte) (t []byte, ok bool) {
if len(s) < 2 || s[0] != '"' || s[len(s)-1] != '"' {
return
}
+ s = s[1 : len(s)-1]
+
+ // Check for unusual characters. If there are none,
+ // then no unquoting is needed, so return a slice of the
+ // original bytes.
+ r := 0
+ for r < len(s) {
+ c := s[r]
+ if c == '\\' || c == '"' || c < ' ' {
+ break
+ }
+ if c < utf8.RuneSelf {
+ r++
+ continue
+ }
+ rune, size := utf8.DecodeRune(s[r:])
+ if rune == utf8.RuneError && size == 1 {
+ break
+ }
+ r += size
+ }
+ if r == len(s) {
+ return s, true
+ }
+
b := make([]byte, len(s)+2*utf8.UTFMax)
- w := 0
- for r := 1; r < len(s)-1; {
+ w := copy(b, s[0:r])
+ for r < len(s) {
// Out of room? Can only happen if s is full of
// malformed UTF-8 and we're replacing each
// byte with RuneError.
switch c := s[r]; {
case c == '\\':
r++
- if r >= len(s)-1 {
+ if r >= len(s) {
return
}
switch s[r] {
w += utf8.EncodeRune(b[w:], rune)
}
}
- return string(b[0:w]), true
+ return b[0:w], true
}
umtrue = unmarshaler{true}
)
+type badTag struct {
+ X string
+ Y string "y"
+ Z string "@#*%(#@"
+}
type unmarshalTest struct {
in string
{`{"X": [1,2,3], "Y": 4}`, new(T), T{Y: 4}, &UnmarshalTypeError{"array", reflect.Typeof("")}},
{`{"x": 1}`, new(tx), tx{}, &UnmarshalFieldError{"x", txType, txType.Field(0)}},
+ // skip invalid tags
+ {`{"X":"a", "y":"b", "Z":"c"}`, new(badTag), badTag{"a", "b", "c"}, nil},
+
// syntax errors
{`{"X": "foo", "Y"}`, nil, nil, SyntaxError("invalid character '}' after object key")},
}
}
+func TestLargeByteSlice(t *testing.T) {
+ s0 := make([]byte, 2000)
+ for i := range s0 {
+ s0[i] = byte(i)
+ }
+ b, err := Marshal(s0)
+ if err != nil {
+ t.Fatalf("Marshal: %v", err)
+ }
+ var s1 []byte
+ if err := Unmarshal(b, &s1); err != nil {
+ t.Fatalf("Unmarshal: %v", err)
+ }
+ if bytes.Compare(s0, s1) != 0 {
+ t.Errorf("Marshal large byte slice")
+ diff(t, s0, s1)
+ }
+}
+
type Xint struct {
X int
}
"str25",
"str26"
],
- "ByteSlice": [
- 27,
- 28,
- 29
- ],
+ "ByteSlice": "Gxwd",
"Small": {
"Tag": "tag30"
},
"EmptySlice": [],
"NilSlice": [],
"StringSlice": [],
- "ByteSlice": [],
+ "ByteSlice": "",
"Small": {
"Tag": ""
},
package json
import (
- "os"
"bytes"
+ "encoding/base64"
+ "os"
"reflect"
"runtime"
"sort"
"strconv"
+ "unicode"
"utf8"
)
// String values encode as JSON strings, with each invalid UTF-8 sequence
// replaced by the encoding of the Unicode replacement character U+FFFD.
//
-// Array and slice values encode as JSON arrays.
+// Array and slice values encode as JSON arrays, except that
+// []byte encodes as a base64-encoded string.
//
// Struct values encode as JSON objects. Each struct field becomes
// a member of the object. By default the object's key name is the
-// struct field name converted to lower case. If the struct field
-// has a tag, that tag will be used as the name instead.
-// Only exported fields will be encoded.
+// struct field name. If the struct field has a non-empty tag consisting
+// of only Unicode letters, digits, and underscores, that tag will be used
+// as the name instead. Only exported fields will be encoded.
//
// Map values encode as JSON objects.
// The map's key type must be string; the object keys are used directly
panic(err)
}
+var byteSliceType = reflect.Typeof([]byte(nil))
+
func (e *encodeState) reflectValue(v reflect.Value) {
if v == nil {
e.WriteString("null")
} else {
e.WriteByte(',')
}
- if f.Tag != "" {
+ if isValidTag(f.Tag) {
e.string(f.Tag)
} else {
e.string(f.Name)
e.WriteByte('}')
case reflect.ArrayOrSliceValue:
+ if v.Type() == byteSliceType {
+ e.WriteByte('"')
+ s := v.Interface().([]byte)
+ if len(s) < 1024 {
+ // for small buffers, using Encode directly is much faster.
+ dst := make([]byte, base64.StdEncoding.EncodedLen(len(s)))
+ base64.StdEncoding.Encode(dst, s)
+ e.Write(dst)
+ } else {
+ // for large buffers, avoid unnecessary extra temporary
+ // buffer space.
+ enc := base64.NewEncoder(base64.StdEncoding, e)
+ enc.Write(s)
+ enc.Close()
+ }
+ e.WriteByte('"')
+ break
+ }
e.WriteByte('[')
n := v.Len()
for i := 0; i < n; i++ {
return
}
+func isValidTag(s string) bool {
+ if s == "" {
+ return false
+ }
+ for _, c := range s {
+ if c != '_' && !unicode.IsLetter(c) && !unicode.IsDigit(c) {
+ return false
+ }
+ }
+ return true
+}
+
// stringValues is a slice of reflect.Value holding *reflect.StringValue.
// It implements the methods to sort by string.
type stringValues []reflect.Value
s.step = stateDot
return scanContinue
}
- if c == 'e' {
+ if c == 'e' || c == 'E' {
s.step = stateE
return scanContinue
}
s.step = stateDot0
return scanContinue
}
- if c == 'e' {
+ if c == 'e' || c == 'E' {
s.step = stateE
return scanContinue
}
// Simple logging package. It defines a type, Logger, with methods
// for formatting output. It also has a predefined 'standard' Logger
-// accessible through helper functions Print[f|ln], Exit[f|ln], and
+// accessible through helper functions Print[f|ln], Fatal[f|ln], and
// Panic[f|ln], which are easier to use than creating a Logger manually.
// That logger writes to standard error and prints the date and time
// of each logged message.
-// The Exit functions call os.Exit(1) after writing the log message.
+// The Fatal functions call os.Exit(1) after writing the log message.
// The Panic functions call panic after writing the log message.
package log
// Arguments are handled in the manner of fmt.Println.
func (l *Logger) Println(v ...interface{}) { l.Output(2, fmt.Sprintln(v...)) }
-// Exit is equivalent to l.Print() followed by a call to os.Exit(1).
-func (l *Logger) Exit(v ...interface{}) {
+// Fatal is equivalent to l.Print() followed by a call to os.Exit(1).
+func (l *Logger) Fatal(v ...interface{}) {
l.Output(2, fmt.Sprint(v...))
os.Exit(1)
}
-// Exitf is equivalent to l.Printf() followed by a call to os.Exit(1).
-func (l *Logger) Exitf(format string, v ...interface{}) {
+// Fatalf is equivalent to l.Printf() followed by a call to os.Exit(1).
+func (l *Logger) Fatalf(format string, v ...interface{}) {
l.Output(2, fmt.Sprintf(format, v...))
os.Exit(1)
}
-// Exitln is equivalent to l.Println() followed by a call to os.Exit(1).
-func (l *Logger) Exitln(v ...interface{}) {
+// Fatalln is equivalent to l.Println() followed by a call to os.Exit(1).
+func (l *Logger) Fatalln(v ...interface{}) {
l.Output(2, fmt.Sprintln(v...))
os.Exit(1)
}
std.Output(2, fmt.Sprintln(v...))
}
-// Exit is equivalent to Print() followed by a call to os.Exit(1).
-func Exit(v ...interface{}) {
+// Fatal is equivalent to Print() followed by a call to os.Exit(1).
+func Fatal(v ...interface{}) {
std.Output(2, fmt.Sprint(v...))
os.Exit(1)
}
-// Exitf is equivalent to Printf() followed by a call to os.Exit(1).
-func Exitf(format string, v ...interface{}) {
+// Fatalf is equivalent to Printf() followed by a call to os.Exit(1).
+func Fatalf(format string, v ...interface{}) {
std.Output(2, fmt.Sprintf(format, v...))
os.Exit(1)
}
-// Exitln is equivalent to Println() followed by a call to os.Exit(1).
-func Exitln(v ...interface{}) {
+// Fatalln is equivalent to Println() followed by a call to os.Exit(1).
+func Fatalln(v ...interface{}) {
std.Output(2, fmt.Sprintln(v...))
os.Exit(1)
}
// Dial("tcp", "127.0.0.1:123", "127.0.0.1:88")
//
func Dial(net, laddr, raddr string) (c Conn, err os.Error) {
- switch prefixBefore(net, ':') {
+ switch net {
case "tcp", "tcp4", "tcp6":
var la, ra *TCPAddr
if laddr != "" {
// The network string net must be a packet-oriented network:
// "udp", "udp4", "udp6", or "unixgram".
func ListenPacket(net, laddr string) (c PacketConn, err os.Error) {
- switch prefixBefore(net, ':') {
+ switch net {
case "udp", "udp4", "udp6":
var la *UDPAddr
if laddr != "" {
return nil, err
}
return c, nil
- case "ip", "ip4", "ip6":
- var la *IPAddr
- if laddr != "" {
- if la, err = ResolveIPAddr(laddr); err != nil {
+ }
+
+ if i := last(net, ':'); i > 0 {
+ switch net[0:i] {
+ case "ip", "ip4", "ip6":
+ var la *IPAddr
+ if laddr != "" {
+ if la, err = ResolveIPAddr(laddr); err != nil {
+ return nil, err
+ }
+ }
+ c, err := ListenIP(net, la)
+ if err != nil {
return nil, err
}
+ return c, nil
}
- c, err := ListenIP(net, la)
- if err != nil {
- return nil, err
- }
- return c, nil
}
+
return nil, UnknownNetworkError(net)
}
// Find answer for name in dns message.
// On return, if err == nil, addrs != nil.
-func answer(name, server string, dns *dnsMsg, qtype uint16) (addrs []dnsRR, err os.Error) {
+func answer(name, server string, dns *dnsMsg, qtype uint16) (cname string, addrs []dnsRR, err os.Error) {
addrs = make([]dnsRR, 0, len(dns.answer))
if dns.rcode == dnsRcodeNameError && dns.recursion_available {
- return nil, &DNSError{Error: noSuchHost, Name: name}
+ return "", nil, &DNSError{Error: noSuchHost, Name: name}
}
if dns.rcode != dnsRcodeSuccess {
// None of the error codes make sense
// for the query we sent. If we didn't get
// a name error and we didn't get success,
// the server is behaving incorrectly.
- return nil, &DNSError{Error: "server misbehaving", Name: name, Server: server}
+ return "", nil, &DNSError{Error: "server misbehaving", Name: name, Server: server}
}
// Look for the name.
}
}
if len(addrs) == 0 {
- return nil, &DNSError{Error: noSuchHost, Name: name, Server: server}
+ return "", nil, &DNSError{Error: noSuchHost, Name: name, Server: server}
}
- return addrs, nil
+ return name, addrs, nil
}
- return nil, &DNSError{Error: "too many redirects", Name: name, Server: server}
+ return "", nil, &DNSError{Error: "too many redirects", Name: name, Server: server}
}
// Do a lookup for a single name, which must be rooted
// (otherwise answer will not find the answers).
-func tryOneName(cfg *dnsConfig, name string, qtype uint16) (addrs []dnsRR, err os.Error) {
+func tryOneName(cfg *dnsConfig, name string, qtype uint16) (cname string, addrs []dnsRR, err os.Error) {
if len(cfg.servers) == 0 {
- return nil, &DNSError{Error: "no DNS servers", Name: name}
+ return "", nil, &DNSError{Error: "no DNS servers", Name: name}
}
for i := 0; i < len(cfg.servers); i++ {
// Calling Dial here is scary -- we have to be sure
err = merr
continue
}
- addrs, err = answer(name, server, msg, qtype)
+ cname, addrs, err = answer(name, server, msg, qtype)
if err == nil || err.(*DNSError).Error == noSuchHost {
break
}
rname += "."
}
// Can try as ordinary name.
- addrs, err = tryOneName(cfg, rname, qtype)
+ cname, addrs, err = tryOneName(cfg, rname, qtype)
if err == nil {
- cname = rname
return
}
}
if rname[len(rname)-1] != '.' {
rname += "."
}
- addrs, err = tryOneName(cfg, rname, qtype)
+ cname, addrs, err = tryOneName(cfg, rname, qtype)
if err == nil {
- cname = rname
return
}
}
if !rooted {
rname += "."
}
- addrs, err = tryOneName(cfg, rname, qtype)
+ cname, addrs, err = tryOneName(cfg, rname, qtype)
if err == nil {
- cname = rname
return
}
return
nn, _ = s.pr.Read(scratch[0:])
}
// Read from channels
- for fd, ok := <-s.cr; ok; fd, ok = <-s.cr {
- s.AddFD(fd, 'r')
- }
- for fd, ok := <-s.cw; ok; fd, ok = <-s.cw {
- s.AddFD(fd, 'w')
+ Update:
+ for {
+ select {
+ case fd := <-s.cr:
+ s.AddFD(fd, 'r')
+ case fd := <-s.cw:
+ s.AddFD(fd, 'w')
+ default:
+ break Update
+ }
}
} else {
netfd := s.LookupFD(fd, mode)
"unsafe"
)
+type InvalidConnError struct{}
+
+func (e *InvalidConnError) String() string { return "invalid net.Conn" }
+func (e *InvalidConnError) Temporary() bool { return false }
+func (e *InvalidConnError) Timeout() bool { return false }
+
+var initErr os.Error
+
+func init() {
+ var d syscall.WSAData
+ e := syscall.WSAStartup(uint32(0x101), &d)
+ if e != 0 {
+ initErr = os.NewSyscallError("WSAStartup", e)
+ }
+}
+
+func closesocket(s int) (errno int) {
+ return syscall.Closesocket(int32(s))
+}
+
+// Interface for all io operations.
+type anOpIface interface {
+ Op() *anOp
+ Name() string
+ Submit() (errno int)
+}
+
// IO completion result parameters.
type ioResult struct {
- key uint32
- qty uint32
- errno int
+ qty uint32
+ err int
}
-// Network file descriptor.
-type netFD struct {
- // locking/lifetime of sysfd
- sysmu sync.Mutex
- sysref int
- closing bool
+// anOp implements functionality common to all io operations.
+type anOp struct {
+ // Used by IOCP interface, it must be first field
+ // of the struct, as our code rely on it.
+ o syscall.Overlapped
- // immutable until Close
- sysfd int
- family int
- proto int
- cr chan *ioResult
- cw chan *ioResult
- net string
- laddr Addr
- raddr Addr
+ resultc chan ioResult // io completion results
+ errnoc chan int // io submit / cancel operation errors
+ fd *netFD
+}
- // owned by client
- rdeadline_delta int64
- rdeadline int64
- rio sync.Mutex
- wdeadline_delta int64
- wdeadline int64
- wio sync.Mutex
+func (o *anOp) Init(fd *netFD) {
+ o.fd = fd
+ o.resultc = make(chan ioResult, 1)
+ o.errnoc = make(chan int)
}
-type InvalidConnError struct{}
+func (o *anOp) Op() *anOp {
+ return o
+}
-func (e *InvalidConnError) String() string { return "invalid net.Conn" }
-func (e *InvalidConnError) Temporary() bool { return false }
-func (e *InvalidConnError) Timeout() bool { return false }
+// bufOp is used by io operations that read / write
+// data from / to client buffer.
+type bufOp struct {
+ anOp
+ buf syscall.WSABuf
+}
-// pollServer will run around waiting for io completion request
-// to arrive. Every request received will contain channel to signal
-// io owner about the completion.
+func (o *bufOp) Init(fd *netFD, buf []byte) {
+ o.anOp.Init(fd)
+ o.buf.Len = uint32(len(buf))
+ if len(buf) == 0 {
+ o.buf.Buf = nil
+ } else {
+ o.buf.Buf = (*byte)(unsafe.Pointer(&buf[0]))
+ }
+}
-type pollServer struct {
+// resultSrv will retreive all io completion results from
+// iocp and send them to the correspondent waiting client
+// goroutine via channel supplied in the request.
+type resultSrv struct {
iocp int32
}
-func newPollServer() (s *pollServer, err os.Error) {
- s = new(pollServer)
- var e int
- if s.iocp, e = syscall.CreateIoCompletionPort(-1, 0, 0, 1); e != 0 {
- return nil, os.NewSyscallError("CreateIoCompletionPort", e)
+func (s *resultSrv) Run() {
+ var o *syscall.Overlapped
+ var key uint32
+ var r ioResult
+ for {
+ r.err = syscall.GetQueuedCompletionStatus(s.iocp, &(r.qty), &key, &o, syscall.INFINITE)
+ switch {
+ case r.err == 0:
+ // Dequeued successfully completed io packet.
+ case r.err == syscall.WAIT_TIMEOUT && o == nil:
+ // Wait has timed out (should not happen now, but might be used in the future).
+ panic("GetQueuedCompletionStatus timed out")
+ case o == nil:
+ // Failed to dequeue anything -> report the error.
+ panic("GetQueuedCompletionStatus failed " + syscall.Errstr(r.err))
+ default:
+ // Dequeued failed io packet.
+ }
+ (*anOp)(unsafe.Pointer(o)).resultc <- r
}
- go s.Run()
- return s, nil
}
-type ioPacket struct {
- // Used by IOCP interface,
- // it must be first field of the struct,
- // as our code rely on it.
- o syscall.Overlapped
- // Link to the io owner.
- c chan *ioResult
-
- w *syscall.WSABuf
+// ioSrv executes net io requests.
+type ioSrv struct {
+ submchan chan anOpIface // submit io requests
+ canchan chan anOpIface // cancel io requests
}
-func (s *pollServer) getCompletedIO() (ov *syscall.Overlapped, result *ioResult, err os.Error) {
- var r ioResult
- var o *syscall.Overlapped
- _, e := syscall.GetQueuedCompletionStatus(s.iocp, &r.qty, &r.key, &o, syscall.INFINITE)
- switch {
- case e == 0:
- // Dequeued successfully completed io packet.
- return o, &r, nil
- case e == syscall.WAIT_TIMEOUT && o == nil:
- // Wait has timed out (should not happen now, but might be used in the future).
- return nil, &r, os.NewSyscallError("GetQueuedCompletionStatus", e)
- case o == nil:
- // Failed to dequeue anything -> report the error.
- return nil, &r, os.NewSyscallError("GetQueuedCompletionStatus", e)
- default:
- // Dequeued failed io packet.
- r.errno = e
- return o, &r, nil
+// ProcessRemoteIO will execute submit io requests on behalf
+// of other goroutines, all on a single os thread, so it can
+// cancel them later. Results of all operations will be sent
+// back to their requesters via channel supplied in request.
+func (s *ioSrv) ProcessRemoteIO() {
+ runtime.LockOSThread()
+ defer runtime.UnlockOSThread()
+ for {
+ select {
+ case o := <-s.submchan:
+ o.Op().errnoc <- o.Submit()
+ case o := <-s.canchan:
+ o.Op().errnoc <- syscall.CancelIo(uint32(o.Op().fd.sysfd))
+ }
}
- return
}
-func (s *pollServer) Run() {
- for {
- o, r, err := s.getCompletedIO()
- if err != nil {
- panic("Run pollServer: " + err.String() + "\n")
+// ExecIO executes a single io operation. It either executes it
+// inline, or, if timeouts are employed, passes the request onto
+// a special goroutine and waits for completion or cancels request.
+func (s *ioSrv) ExecIO(oi anOpIface, deadline_delta int64) (n int, err os.Error) {
+ var e int
+ o := oi.Op()
+ if deadline_delta > 0 {
+ // Send request to a special dedicated thread,
+ // so it can stop the io with CancelIO later.
+ s.submchan <- oi
+ e = <-o.errnoc
+ } else {
+ e = oi.Submit()
+ }
+ switch e {
+ case 0:
+ // IO completed immediately, but we need to get our completion message anyway.
+ case syscall.ERROR_IO_PENDING:
+ // IO started, and we have to wait for it's completion.
+ default:
+ return 0, &OpError{oi.Name(), o.fd.net, o.fd.laddr, os.Errno(e)}
+ }
+ // Wait for our request to complete.
+ var r ioResult
+ if deadline_delta > 0 {
+ select {
+ case r = <-o.resultc:
+ case <-time.After(deadline_delta):
+ s.canchan <- oi
+ <-o.errnoc
+ r = <-o.resultc
+ if r.err == syscall.ERROR_OPERATION_ABORTED { // IO Canceled
+ r.err = syscall.EWOULDBLOCK
+ }
}
- p := (*ioPacket)(unsafe.Pointer(o))
- p.c <- r
+ } else {
+ r = <-o.resultc
}
+ if r.err != 0 {
+ err = &OpError{oi.Name(), o.fd.net, o.fd.laddr, os.Errno(r.err)}
+ }
+ return int(r.qty), err
}
-// Network FD methods.
-// All the network FDs use a single pollServer.
-
-var pollserver *pollServer
+// Start helper goroutines.
+var resultsrv *resultSrv
+var iosrv *ioSrv
var onceStartServer sync.Once
func startServer() {
- p, err := newPollServer()
- if err != nil {
- panic("Start pollServer: " + err.String() + "\n")
- }
- pollserver = p
-
- go timeoutIO()
+ resultsrv = new(resultSrv)
+ var errno int
+ resultsrv.iocp, errno = syscall.CreateIoCompletionPort(-1, 0, 0, 1)
+ if errno != 0 {
+ panic("CreateIoCompletionPort failed " + syscall.Errstr(errno))
+ }
+ go resultsrv.Run()
+
+ iosrv = new(ioSrv)
+ iosrv.submchan = make(chan anOpIface)
+ iosrv.canchan = make(chan anOpIface)
+ go iosrv.ProcessRemoteIO()
}
-var initErr os.Error
+// Network file descriptor.
+type netFD struct {
+ // locking/lifetime of sysfd
+ sysmu sync.Mutex
+ sysref int
+ closing bool
-func newFD(fd, family, proto int, net string, laddr, raddr Addr) (f *netFD, err os.Error) {
- if initErr != nil {
- return nil, initErr
- }
- onceStartServer.Do(startServer)
- // Associate our socket with pollserver.iocp.
- if _, e := syscall.CreateIoCompletionPort(int32(fd), pollserver.iocp, 0, 0); e != 0 {
- return nil, &OpError{"CreateIoCompletionPort", net, laddr, os.Errno(e)}
- }
+ // immutable until Close
+ sysfd int
+ family int
+ proto int
+ net string
+ laddr Addr
+ raddr Addr
+
+ // owned by client
+ rdeadline_delta int64
+ rdeadline int64
+ rio sync.Mutex
+ wdeadline_delta int64
+ wdeadline int64
+ wio sync.Mutex
+}
+
+func allocFD(fd, family, proto int, net string, laddr, raddr Addr) (f *netFD) {
f = &netFD{
sysfd: fd,
family: family,
proto: proto,
- cr: make(chan *ioResult, 1),
- cw: make(chan *ioResult, 1),
net: net,
laddr: laddr,
raddr: raddr,
}
runtime.SetFinalizer(f, (*netFD).Close)
- return f, nil
+ return f
+}
+
+func newFD(fd, family, proto int, net string, laddr, raddr Addr) (f *netFD, err os.Error) {
+ if initErr != nil {
+ return nil, initErr
+ }
+ onceStartServer.Do(startServer)
+ // Associate our socket with resultsrv.iocp.
+ if _, e := syscall.CreateIoCompletionPort(int32(fd), resultsrv.iocp, 0, 0); e != 0 {
+ return nil, &OpError{"CreateIoCompletionPort", net, laddr, os.Errno(e)}
+ }
+ return allocFD(fd, family, proto, net, laddr, raddr), nil
}
// Add a reference to this fd.
// In case the user has set linger, switch to blocking mode so
// the close blocks. As long as this doesn't happen often, we
// can handle the extra OS processes. Otherwise we'll need to
- // use the pollserver for Close too. Sigh.
+ // use the resultsrv for Close too. Sigh.
syscall.SetNonblock(fd.sysfd, false)
closesocket(fd.sysfd)
fd.sysfd = -1
return nil
}
-func newWSABuf(p []byte) *syscall.WSABuf {
- var p0 *byte
- if len(p) > 0 {
- p0 = (*byte)(unsafe.Pointer(&p[0]))
- }
- return &syscall.WSABuf{uint32(len(p)), p0}
-}
-
-func waitPacket(fd *netFD, pckt *ioPacket, mode int) (r *ioResult) {
- var delta int64
- if mode == 'r' {
- delta = fd.rdeadline_delta
- }
- if mode == 'w' {
- delta = fd.wdeadline_delta
- }
- if delta <= 0 {
- return <-pckt.c
- }
+// Read from network.
- select {
- case r = <-pckt.c:
- case <-time.After(delta):
- a := &arg{f: cancel, fd: fd, pckt: pckt, c: make(chan int)}
- ioChan <- a
- <-a.c
- r = <-pckt.c
- if r.errno == 995 { // IO Canceled
- r.errno = syscall.EWOULDBLOCK
- }
- }
- return r
+type readOp struct {
+ bufOp
}
-const (
- read = iota
- readfrom
- write
- writeto
- cancel
-)
+func (o *readOp) Submit() (errno int) {
+ var d, f uint32
+ return syscall.WSARecv(uint32(o.fd.sysfd), &o.buf, 1, &d, &f, &o.o, nil)
+}
-type arg struct {
- f int
- fd *netFD
- pckt *ioPacket
- done *uint32
- flags *uint32
- rsa *syscall.RawSockaddrAny
- size *int32
- sa *syscall.Sockaddr
- c chan int
-}
-
-var ioChan chan *arg = make(chan *arg)
-
-func timeoutIO() {
- // CancelIO only cancels all pending input and output (I/O) operations that are
- // issued by the calling thread for the specified file, does not cancel I/O
- // operations that other threads issue for a file handle. So we need do all timeout
- // I/O in single OS thread.
- runtime.LockOSThread()
- defer runtime.UnlockOSThread()
- for {
- o := <-ioChan
- var e int
- switch o.f {
- case read:
- e = syscall.WSARecv(uint32(o.fd.sysfd), o.pckt.w, 1, o.done, o.flags, &o.pckt.o, nil)
- case readfrom:
- e = syscall.WSARecvFrom(uint32(o.fd.sysfd), o.pckt.w, 1, o.done, o.flags, o.rsa, o.size, &o.pckt.o, nil)
- case write:
- e = syscall.WSASend(uint32(o.fd.sysfd), o.pckt.w, 1, o.done, uint32(0), &o.pckt.o, nil)
- case writeto:
- e = syscall.WSASendto(uint32(o.fd.sysfd), o.pckt.w, 1, o.done, 0, *o.sa, &o.pckt.o, nil)
- case cancel:
- _, e = syscall.CancelIo(uint32(o.fd.sysfd))
- }
- o.c <- e
- }
+func (o *readOp) Name() string {
+ return "WSARecv"
}
-func (fd *netFD) Read(p []byte) (n int, err os.Error) {
+func (fd *netFD) Read(buf []byte) (n int, err os.Error) {
if fd == nil {
return 0, os.EINVAL
}
if fd.sysfd == -1 {
return 0, os.EINVAL
}
- // Submit receive request.
- var pckt ioPacket
- pckt.c = fd.cr
- pckt.w = newWSABuf(p)
- var done uint32
- flags := uint32(0)
- var e int
- if fd.rdeadline_delta > 0 {
- a := &arg{f: read, fd: fd, pckt: &pckt, done: &done, flags: &flags, c: make(chan int)}
- ioChan <- a
- e = <-a.c
- } else {
- e = syscall.WSARecv(uint32(fd.sysfd), pckt.w, 1, &done, &flags, &pckt.o, nil)
- }
- switch e {
- case 0:
- // IO completed immediately, but we need to get our completion message anyway.
- case syscall.ERROR_IO_PENDING:
- // IO started, and we have to wait for it's completion.
- default:
- return 0, &OpError{"WSARecv", fd.net, fd.laddr, os.Errno(e)}
- }
- // Wait for our request to complete.
- r := waitPacket(fd, &pckt, 'r')
- if r.errno != 0 {
- err = &OpError{"WSARecv", fd.net, fd.laddr, os.Errno(r.errno)}
- }
- n = int(r.qty)
+ var o readOp
+ o.Init(fd, buf)
+ n, err = iosrv.ExecIO(&o, fd.rdeadline_delta)
if err == nil && n == 0 {
err = os.EOF
}
return
}
-func (fd *netFD) ReadFrom(p []byte) (n int, sa syscall.Sockaddr, err os.Error) {
+// ReadFrom from network.
+
+type readFromOp struct {
+ bufOp
+ rsa syscall.RawSockaddrAny
+}
+
+func (o *readFromOp) Submit() (errno int) {
+ var d, f uint32
+ l := int32(unsafe.Sizeof(o.rsa))
+ return syscall.WSARecvFrom(uint32(o.fd.sysfd), &o.buf, 1, &d, &f, &o.rsa, &l, &o.o, nil)
+}
+
+func (o *readFromOp) Name() string {
+ return "WSARecvFrom"
+}
+
+func (fd *netFD) ReadFrom(buf []byte) (n int, sa syscall.Sockaddr, err os.Error) {
if fd == nil {
return 0, nil, os.EINVAL
}
- if len(p) == 0 {
+ if len(buf) == 0 {
return 0, nil, nil
}
fd.rio.Lock()
if fd.sysfd == -1 {
return 0, nil, os.EINVAL
}
- // Submit receive request.
- var pckt ioPacket
- pckt.c = fd.cr
- pckt.w = newWSABuf(p)
- var done uint32
- flags := uint32(0)
- var rsa syscall.RawSockaddrAny
- l := int32(unsafe.Sizeof(rsa))
- var e int
- if fd.rdeadline_delta > 0 {
- a := &arg{f: readfrom, fd: fd, pckt: &pckt, done: &done, flags: &flags, rsa: &rsa, size: &l, c: make(chan int)}
- ioChan <- a
- e = <-a.c
- } else {
- e = syscall.WSARecvFrom(uint32(fd.sysfd), pckt.w, 1, &done, &flags, &rsa, &l, &pckt.o, nil)
- }
- switch e {
- case 0:
- // IO completed immediately, but we need to get our completion message anyway.
- case syscall.ERROR_IO_PENDING:
- // IO started, and we have to wait for it's completion.
- default:
- return 0, nil, &OpError{"WSARecvFrom", fd.net, fd.laddr, os.Errno(e)}
- }
- // Wait for our request to complete.
- r := waitPacket(fd, &pckt, 'r')
- if r.errno != 0 {
- err = &OpError{"WSARecvFrom", fd.net, fd.laddr, os.Errno(r.errno)}
- }
- n = int(r.qty)
- sa, _ = rsa.Sockaddr()
+ var o readFromOp
+ o.Init(fd, buf)
+ n, err = iosrv.ExecIO(&o, fd.rdeadline_delta)
+ sa, _ = o.rsa.Sockaddr()
return
}
-func (fd *netFD) Write(p []byte) (n int, err os.Error) {
+// Write to network.
+
+type writeOp struct {
+ bufOp
+}
+
+func (o *writeOp) Submit() (errno int) {
+ var d uint32
+ return syscall.WSASend(uint32(o.fd.sysfd), &o.buf, 1, &d, 0, &o.o, nil)
+}
+
+func (o *writeOp) Name() string {
+ return "WSASend"
+}
+
+func (fd *netFD) Write(buf []byte) (n int, err os.Error) {
if fd == nil {
return 0, os.EINVAL
}
if fd.sysfd == -1 {
return 0, os.EINVAL
}
- // Submit send request.
- var pckt ioPacket
- pckt.c = fd.cw
- pckt.w = newWSABuf(p)
- var done uint32
- var e int
- if fd.wdeadline_delta > 0 {
- a := &arg{f: write, fd: fd, pckt: &pckt, done: &done, c: make(chan int)}
- ioChan <- a
- e = <-a.c
- } else {
- e = syscall.WSASend(uint32(fd.sysfd), pckt.w, 1, &done, uint32(0), &pckt.o, nil)
- }
- switch e {
- case 0:
- // IO completed immediately, but we need to get our completion message anyway.
- case syscall.ERROR_IO_PENDING:
- // IO started, and we have to wait for it's completion.
- default:
- return 0, &OpError{"WSASend", fd.net, fd.laddr, os.Errno(e)}
- }
- // Wait for our request to complete.
- r := waitPacket(fd, &pckt, 'w')
- if r.errno != 0 {
- err = &OpError{"WSASend", fd.net, fd.laddr, os.Errno(r.errno)}
- }
- n = int(r.qty)
- return
+ var o writeOp
+ o.Init(fd, buf)
+ return iosrv.ExecIO(&o, fd.wdeadline_delta)
+}
+
+// WriteTo to network.
+
+type writeToOp struct {
+ bufOp
+ sa syscall.Sockaddr
+}
+
+func (o *writeToOp) Submit() (errno int) {
+ var d uint32
+ return syscall.WSASendto(uint32(o.fd.sysfd), &o.buf, 1, &d, 0, o.sa, &o.o, nil)
+}
+
+func (o *writeToOp) Name() string {
+ return "WSASendto"
}
-func (fd *netFD) WriteTo(p []byte, sa syscall.Sockaddr) (n int, err os.Error) {
+func (fd *netFD) WriteTo(buf []byte, sa syscall.Sockaddr) (n int, err os.Error) {
if fd == nil {
return 0, os.EINVAL
}
- if len(p) == 0 {
+ if len(buf) == 0 {
return 0, nil
}
fd.wio.Lock()
if fd.sysfd == -1 {
return 0, os.EINVAL
}
- // Submit send request.
- var pckt ioPacket
- pckt.c = fd.cw
- pckt.w = newWSABuf(p)
- var done uint32
- var e int
- if fd.wdeadline_delta > 0 {
- a := &arg{f: writeto, fd: fd, pckt: &pckt, done: &done, sa: &sa, c: make(chan int)}
- ioChan <- a
- e = <-a.c
- } else {
- e = syscall.WSASendto(uint32(fd.sysfd), pckt.w, 1, &done, 0, sa, &pckt.o, nil)
- }
- switch e {
- case 0:
- // IO completed immediately, but we need to get our completion message anyway.
- case syscall.ERROR_IO_PENDING:
- // IO started, and we have to wait for it's completion.
- default:
- return 0, &OpError{"WSASendTo", fd.net, fd.laddr, os.Errno(e)}
- }
- // Wait for our request to complete.
- r := waitPacket(fd, &pckt, 'w')
- if r.errno != 0 {
- err = &OpError{"WSASendTo", fd.net, fd.laddr, os.Errno(r.errno)}
- }
- n = int(r.qty)
- return
+ var o writeToOp
+ o.Init(fd, buf)
+ o.sa = sa
+ return iosrv.ExecIO(&o, fd.wdeadline_delta)
+}
+
+// Accept new network connections.
+
+type acceptOp struct {
+ anOp
+ newsock int
+ attrs [2]syscall.RawSockaddrAny // space for local and remote address only
+}
+
+func (o *acceptOp) Submit() (errno int) {
+ var d uint32
+ l := uint32(unsafe.Sizeof(o.attrs[0]))
+ return syscall.AcceptEx(uint32(o.fd.sysfd), uint32(o.newsock),
+ (*byte)(unsafe.Pointer(&o.attrs[0])), 0, l, l, &d, &o.o)
+}
+
+func (o *acceptOp) Name() string {
+ return "AcceptEx"
}
func (fd *netFD) accept(toAddr func(syscall.Sockaddr) Addr) (nfd *netFD, err os.Error) {
// Associate our new socket with IOCP.
onceStartServer.Do(startServer)
- if _, e = syscall.CreateIoCompletionPort(int32(s), pollserver.iocp, 0, 0); e != 0 {
+ if _, e = syscall.CreateIoCompletionPort(int32(s), resultsrv.iocp, 0, 0); e != 0 {
return nil, &OpError{"CreateIoCompletionPort", fd.net, fd.laddr, os.Errno(e)}
}
// Submit accept request.
- // Will use new unique channel here, because, unlike Read or Write,
- // Accept is expected to be executed by many goroutines simultaniously.
- var pckt ioPacket
- pckt.c = make(chan *ioResult)
- attrs, e := syscall.AcceptIOCP(fd.sysfd, s, &pckt.o)
- switch e {
- case 0:
- // IO completed immediately, but we need to get our completion message anyway.
- case syscall.ERROR_IO_PENDING:
- // IO started, and we have to wait for it's completion.
- default:
- closesocket(s)
- return nil, &OpError{"AcceptEx", fd.net, fd.laddr, os.Errno(e)}
- }
-
- // Wait for peer connection.
- r := <-pckt.c
- if r.errno != 0 {
+ var o acceptOp
+ o.Init(fd)
+ o.newsock = s
+ _, err = iosrv.ExecIO(&o, 0)
+ if err != nil {
closesocket(s)
- return nil, &OpError{"AcceptEx", fd.net, fd.laddr, os.Errno(r.errno)}
+ return nil, err
}
// Inherit properties of the listening socket.
e = syscall.SetsockoptInt(s, syscall.SOL_SOCKET, syscall.SO_UPDATE_ACCEPT_CONTEXT, fd.sysfd)
if e != 0 {
closesocket(s)
- return nil, &OpError{"Setsockopt", fd.net, fd.laddr, os.Errno(r.errno)}
+ return nil, err
}
// Get local and peer addr out of AcceptEx buffer.
- lsa, rsa := syscall.GetAcceptIOCPSockaddrs(attrs)
-
- // Create our netFD and return it for further use.
- laddr := toAddr(lsa)
- raddr := toAddr(rsa)
-
- f := &netFD{
- sysfd: s,
- family: fd.family,
- proto: fd.proto,
- cr: make(chan *ioResult, 1),
- cw: make(chan *ioResult, 1),
- net: fd.net,
- laddr: laddr,
- raddr: raddr,
- }
- runtime.SetFinalizer(f, (*netFD).Close)
- return f, nil
-}
-
-func closesocket(s int) (errno int) {
- return syscall.Closesocket(int32(s))
+ var lrsa, rrsa *syscall.RawSockaddrAny
+ var llen, rlen int32
+ l := uint32(unsafe.Sizeof(*lrsa))
+ syscall.GetAcceptExSockaddrs((*byte)(unsafe.Pointer(&o.attrs[0])),
+ 0, l, l, &lrsa, &llen, &rrsa, &rlen)
+ lsa, _ := lrsa.Sockaddr()
+ rsa, _ := rrsa.Sockaddr()
+
+ return allocFD(s, fd.family, fd.proto, fd.net, toAddr(lsa), toAddr(rsa)), nil
}
-func init() {
- var d syscall.WSAData
- e := syscall.WSAStartup(uint32(0x101), &d)
- if e != 0 {
- initErr = os.NewSyscallError("WSAStartup", e)
- }
-}
+// Not implemeted functions.
func (fd *netFD) dup() (f *os.File, err os.Error) {
// TODO: Implement this
err = err1
goto Error
}
- addr = ParseIP(addrs[0])
+ addr = firstSupportedAddr(addrs)
if addr == nil {
// should not happen
err = &AddrError{"LookupHost returned invalid address", addrs[0]}
if err != nil {
return
}
- switch prefixBefore(net, ':') {
+ switch net {
case "ip", "ip4", "ip6":
default:
return nil, UnknownNetworkError(net)
if err != nil {
return
}
- switch prefixBefore(net, ':') {
+ switch net {
case "ip", "ip4", "ip6":
default:
return nil, UnknownNetworkError(net)
// Unfortunately, we need to run on kernels built without IPv6 support too.
// So probe the kernel to figure it out.
func kernelSupportsIPv6() bool {
- // FreeBSD does not support this sort of interface.
- if syscall.OS == "freebsd" {
+ s, err := syscall.Socket(syscall.AF_INET6, syscall.SOCK_STREAM, syscall.IPPROTO_TCP)
+ if err != 0 {
return false
}
- fd, e := syscall.Socket(syscall.AF_INET6, syscall.SOCK_STREAM, syscall.IPPROTO_TCP)
- if fd >= 0 {
- closesocket(fd)
+ defer closesocket(s)
+
+ la := &TCPAddr{IP: IPv4(127, 0, 0, 1)}
+ sa, oserr := la.toAddr().sockaddr(syscall.AF_INET6)
+ if oserr != nil {
+ return false
}
- return e == 0
+
+ return syscall.Bind(s, sa) == 0
}
var preferIPv4 = !kernelSupportsIPv6()
+func firstSupportedAddr(addrs []string) (addr IP) {
+ for _, s := range addrs {
+ addr = ParseIP(s)
+ if !preferIPv4 || addr.To4() != nil {
+ break
+ }
+ addr = nil
+ }
+ return addr
+}
+
// TODO(rsc): if syscall.OS == "linux", we're supposd to read
// /proc/sys/net/core/somaxconn,
// to take advantage of kernels that have raised the limit.
err = err1
goto Error
}
- addr = ParseIP(addrs[0])
+ addr = firstSupportedAddr(addrs)
if addr == nil {
// should not happen
err = &AddrError{"LookupHost returned invalid address", addrs[0]}
--- /dev/null
+// Copyright 2011 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package net
+
+import (
+ "runtime"
+ "testing"
+)
+
+func TestMulticastJoinAndLeave(t *testing.T) {
+ if runtime.GOOS == "windows" {
+ return
+ }
+
+ addr := &UDPAddr{
+ IP: IPv4zero,
+ Port: 0,
+ }
+ // open a UDPConn
+ conn, err := ListenUDP("udp4", addr)
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer conn.Close()
+
+ // try to join group
+ mcast := IPv4(224, 0, 0, 254)
+ err = conn.JoinGroup(mcast)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ // try to leave group
+ err = conn.LeaveGroup(mcast)
+ if err != nil {
+ t.Fatal(err)
+ }
+}
+
+func TestJoinFailureWithIPv6Address(t *testing.T) {
+ addr := &UDPAddr{
+ IP: IPv4zero,
+ Port: 0,
+ }
+
+ // open a UDPConn
+ conn, err := ListenUDP("udp4", addr)
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer conn.Close()
+
+ // try to join group
+ mcast := ParseIP("ff02::1")
+ err = conn.JoinGroup(mcast)
+ if err == nil {
+ t.Fatal("JoinGroup succeeded, should fail")
+ }
+ t.Logf("%s", err)
+}
Write(b []byte) (n int, err os.Error)
// Close closes the connection.
- // The error returned is an os.Error to satisfy io.Closer;
Close() os.Error
// LocalAddr returns the local network address.
WriteTo(b []byte, addr Addr) (n int, err os.Error)
// Close closes the connection.
- // The error returned is an os.Error to satisfy io.Closer;
Close() os.Error
// LocalAddr returns the local network address.
Accept() (c Conn, err os.Error)
// Close closes the listener.
- // The error returned is an os.Error to satisfy io.Closer;
Close() os.Error
// Addr returns the listener's network address.
return n
}
-// Returns the prefix of s up to but not including the character c
-func prefixBefore(s string, c byte) string {
- for i, v := range s {
- if v == int(c) {
- return s[0:i]
- }
- }
- return s
-}
-
// Index of rightmost occurrence of b in s.
func last(s string, b byte) int {
i := len(s)
for {
n, err := fd.Read(buf[0:])
- if err != nil || n == 0 {
+ if err != nil || n == 0 || string(buf[:n]) == "END" {
break
}
fd.Write(buf[0:n])
if n != len(b) || err1 != nil {
t.Fatalf("fd.Read() = %d, %v (want %d, nil)", n, err1, len(b))
}
+
+ // Send explicit ending for unixpacket.
+ // Older Linux kernels do stop reads on close.
+ if network == "unixpacket" {
+ fd.Write([]byte("END"))
+ }
+
fd.Close()
}
listening <- c.LocalAddr().String()
c.SetReadTimeout(10e6) // 10ms
var buf [1000]byte
+Run:
for {
n, addr, err := c.ReadFrom(buf[0:])
if e, ok := err.(Error); ok && e.Timeout() {
- if done <- 1 {
- break
+ select {
+ case done <- 1:
+ break Run
+ default:
+ continue Run
}
- continue
}
if err != nil {
break
--- /dev/null
+// Copyright 2010 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package textproto
+
+// A MIMEHeader represents a MIME-style header mapping
+// keys to sets of values.
+type MIMEHeader map[string][]string
+
+// Add adds the key, value pair to the header.
+// It appends to any existing values associated with key.
+func (h MIMEHeader) Add(key, value string) {
+ key = CanonicalMIMEHeaderKey(key)
+ h[key] = append(h[key], value)
+}
+
+// Set sets the header entries associated with key to
+// the single element value. It replaces any existing
+// values associated with key.
+func (h MIMEHeader) Set(key, value string) {
+ h[CanonicalMIMEHeaderKey(key)] = []string{value}
+}
+
+// Get gets the first value associated with the given key.
+// If there are no values associated with the key, Get returns "".
+// Get is a convenience method. For more complex queries,
+// access the map directly.
+func (h MIMEHeader) Get(key string) string {
+ if h == nil {
+ return ""
+ }
+ v := h[CanonicalMIMEHeaderKey(key)]
+ if len(v) == 0 {
+ return ""
+ }
+ return v[0]
+}
+
+// Del deletes the values associated with key.
+func (h MIMEHeader) Del(key string) {
+ h[CanonicalMIMEHeaderKey(key)] = nil, false
+}
// ReadMIMEHeader reads a MIME-style header from r.
// The header is a sequence of possibly continued Key: Value lines
// ending in a blank line.
-// The returned map m maps CanonicalHeaderKey(key) to a
+// The returned map m maps CanonicalMIMEHeaderKey(key) to a
// sequence of values in the same order encountered in the input.
//
// For example, consider this input:
// Given that input, ReadMIMEHeader returns the map:
//
// map[string][]string{
-// "My-Key": []string{"Value 1", "Value 2"},
-// "Long-Key": []string{"Even Longer Value"},
+// "My-Key": {"Value 1", "Value 2"},
+// "Long-Key": {"Even Longer Value"},
// }
//
-func (r *Reader) ReadMIMEHeader() (map[string][]string, os.Error) {
- m := make(map[string][]string)
+func (r *Reader) ReadMIMEHeader() (MIMEHeader, os.Error) {
+ m := make(MIMEHeader)
for {
kv, err := r.ReadContinuedLineBytes()
if len(kv) == 0 {
if i < 0 || bytes.IndexByte(kv[0:i], ' ') >= 0 {
return m, ProtocolError("malformed MIME header line: " + string(kv))
}
- key := CanonicalHeaderKey(string(kv[0:i]))
+ key := CanonicalMIMEHeaderKey(string(kv[0:i]))
// Skip initial spaces in value.
i++ // skip colon
panic("unreachable")
}
-// CanonicalHeaderKey returns the canonical format of the
+// CanonicalMIMEHeaderKey returns the canonical format of the
// MIME header key s. The canonicalization converts the first
// letter and any letter following a hyphen to upper case;
// the rest are converted to lowercase. For example, the
// canonical key for "accept-encoding" is "Accept-Encoding".
-func CanonicalHeaderKey(s string) string {
+func CanonicalMIMEHeaderKey(s string) string {
// Quick check for canonical encoding.
needUpper := true
for i := 0; i < len(s); i++ {
{"USER-AGENT", "User-Agent"},
}
-func TestCanonicalHeaderKey(t *testing.T) {
+func TestCanonicalMIMEHeaderKey(t *testing.T) {
for _, tt := range canonicalHeaderKeyTests {
- if s := CanonicalHeaderKey(tt.in); s != tt.out {
- t.Errorf("CanonicalHeaderKey(%q) = %q, want %q", tt.in, s, tt.out)
+ if s := CanonicalMIMEHeaderKey(tt.in); s != tt.out {
+ t.Errorf("CanonicalMIMEHeaderKey(%q) = %q, want %q", tt.in, s, tt.out)
}
}
}
func TestReadMIMEHeader(t *testing.T) {
r := reader("my-key: Value 1 \r\nLong-key: Even \n Longer Value\r\nmy-Key: Value 2\r\n\n")
m, err := r.ReadMIMEHeader()
- want := map[string][]string{
+ want := MIMEHeader{
"My-Key": {"Value 1", "Value 2"},
"Long-Key": {"Even Longer Value"},
}
// It is the caller's responsibility to close f when finished.
// Closing c does not affect f, and closing f does not affect c.
func (c *UDPConn) File() (f *os.File, err os.Error) { return c.fd.dup() }
+
+var errInvalidMulticast = os.ErrorString("invalid IPv4 multicast address")
+
+// JoinGroup joins the IPv4 multicast group named by addr.
+// The UDPConn must use the "udp4" network.
+func (c *UDPConn) JoinGroup(addr IP) os.Error {
+ if !c.ok() {
+ return os.EINVAL
+ }
+ ip := addr.To4()
+ if ip == nil {
+ return &OpError{"joingroup", "udp", &IPAddr{ip}, errInvalidMulticast}
+ }
+ mreq := &syscall.IpMreq{
+ Multiaddr: [4]byte{ip[0], ip[1], ip[2], ip[3]},
+ }
+ err := os.NewSyscallError("setsockopt", syscall.SetsockoptIpMreq(c.fd.sysfd, syscall.IPPROTO_IP, syscall.IP_ADD_MEMBERSHIP, mreq))
+ if err != nil {
+ return &OpError{"joingroup", "udp", &IPAddr{ip}, err}
+ }
+ return nil
+}
+
+// LeaveGroup exits the IPv4 multicast group named by addr.
+func (c *UDPConn) LeaveGroup(addr IP) os.Error {
+ if !c.ok() {
+ return os.EINVAL
+ }
+ ip := addr.To4()
+ if ip == nil {
+ return &OpError{"leavegroup", "udp", &IPAddr{ip}, errInvalidMulticast}
+ }
+ mreq := &syscall.IpMreq{
+ Multiaddr: [4]byte{ip[0], ip[1], ip[2], ip[3]},
+ }
+ err := os.NewSyscallError("setsockopt", syscall.SetsockoptIpMreq(c.fd.sysfd, syscall.IPPROTO_IP, syscall.IP_DROP_MEMBERSHIP, mreq))
+ if err != nil {
+ return &OpError{"leavegroup", "udp", &IPAddr{ip}, err}
+ }
+ return nil
+}
import (
"gob"
- "net"
+ "io"
"os"
"reflect"
"sync"
enc *gob.Encoder
}
-func newEncDec(conn net.Conn) *encDec {
+func newEncDec(conn io.ReadWriter) *encDec {
return &encDec{
dec: gob.NewDecoder(conn),
enc: gob.NewEncoder(conn),
// are delivered into the local channel.
type netChan struct {
*chanDir
- name string
- id int
- size int // buffer size of channel.
+ name string
+ id int
+ size int // buffer size of channel.
+ closed bool
// sender-specific state
ackCh chan bool // buffered with space for all the acks we need
// Close the channel.
func (nch *netChan) close() {
+ if nch.closed {
+ return
+ }
if nch.dir == Recv {
if nch.sendCh != nil {
// If the sender goroutine is active, close the channel to it.
nch.ch.Close()
close(nch.ackCh)
}
+ nch.closed = true
}
// Send message from remote side to local receiver.
nch.sendCh = make(chan reflect.Value, nch.size)
go nch.sender()
}
- if ok := nch.sendCh <- val; !ok {
+ select {
+ case nch.sendCh <- val:
+ // ok
+ default:
// TODO: should this be more resilient?
panic("netchan: remote sender sent more values than allowed")
}
if nch.dir != Send {
panic("recv on wrong direction of channel")
}
- if ok := nch.ackCh <- true; !ok {
- panic("netchan: remote receiver sent too many acks")
+ select {
+ case nch.ackCh <- true:
+ // ok
+ default:
// TODO: should this be more resilient?
+ panic("netchan: remote receiver sent too many acks")
}
}
import (
"log"
+ "io"
"net"
"os"
"reflect"
// but they must use different ports.
type Exporter struct {
*clientSet
- listener net.Listener
}
type expClient struct {
seqLock sync.Mutex // guarantees messages are in sequence, only locked under mu
}
-func newClient(exp *Exporter, conn net.Conn) *expClient {
+func newClient(exp *Exporter, conn io.ReadWriter) *expClient {
client := new(expClient)
client.exp = exp
client.encDec = newEncDec(conn)
for {
*hdr = header{}
if err := client.decode(hdrValue); err != nil {
- expLog("error decoding client header:", err)
+ if err != os.EOF {
+ expLog("error decoding client header:", err)
+ }
break
}
switch hdr.PayloadType {
nch.acked()
}
default:
- log.Exit("netchan export: unknown payload type", hdr.PayloadType)
+ log.Fatal("netchan export: unknown payload type", hdr.PayloadType)
}
}
client.exp.delClient(client)
return n
}
-// Wait for incoming connections, start a new runner for each
-func (exp *Exporter) listen() {
+// Serve waits for incoming connections on the listener
+// and serves the Exporter's channels on each.
+// It blocks until the listener is closed.
+func (exp *Exporter) Serve(listener net.Listener) {
for {
- conn, err := exp.listener.Accept()
+ conn, err := listener.Accept()
if err != nil {
expLog("listen:", err)
break
}
- client := exp.addClient(conn)
- go client.run()
+ go exp.ServeConn(conn)
}
}
-// NewExporter creates a new Exporter to export channels
-// on the network and local address defined as in net.Listen.
-func NewExporter(network, localaddr string) (*Exporter, os.Error) {
- listener, err := net.Listen(network, localaddr)
- if err != nil {
- return nil, err
- }
+// ServeConn exports the Exporter's channels on conn.
+// It blocks until the connection is terminated.
+func (exp *Exporter) ServeConn(conn io.ReadWriter) {
+ exp.addClient(conn).run()
+}
+
+// NewExporter creates a new Exporter that exports a set of channels.
+func NewExporter() *Exporter {
e := &Exporter{
- listener: listener,
clientSet: &clientSet{
names: make(map[string]*chanDir),
clients: make(map[unackedCounter]bool),
},
}
- go e.listen()
- return e, nil
+ return e
+}
+
+// ListenAndServe exports the exporter's channels through the
+// given network and local address defined as in net.Listen.
+func (exp *Exporter) ListenAndServe(network, localaddr string) os.Error {
+ listener, err := net.Listen(network, localaddr)
+ if err != nil {
+ return err
+ }
+ go exp.Serve(listener)
+ return nil
}
// addClient creates a new expClient and records its existence
-func (exp *Exporter) addClient(conn net.Conn) *expClient {
+func (exp *Exporter) addClient(conn io.ReadWriter) *expClient {
client := newClient(exp, conn)
exp.mu.Lock()
exp.clients[client] = true
return exp.clientSet.sync(timeout)
}
-// Addr returns the Exporter's local network address.
-func (exp *Exporter) Addr() net.Addr { return exp.listener.Addr() }
-
func checkChan(chT interface{}, dir Dir) (*reflect.ChanValue, os.Error) {
chanType, ok := reflect.Typeof(chT).(*reflect.ChanType)
if !ok {
package netchan
import (
+ "io"
"log"
"net"
"os"
// importers, even from the same machine/network port.
type Importer struct {
*encDec
- conn net.Conn
chanLock sync.Mutex // protects access to channel map
names map[string]*netChan
chans map[int]*netChan
maxId int
}
-// NewImporter creates a new Importer object to import channels
-// from an Exporter at the network and remote address as defined in net.Dial.
-// The Exporter must be available and serving when the Importer is
-// created.
-func NewImporter(network, remoteaddr string) (*Importer, os.Error) {
- conn, err := net.Dial(network, "", remoteaddr)
- if err != nil {
- return nil, err
- }
+// NewImporter creates a new Importer object to import a set of channels
+// from the given connection. The Exporter must be available and serving when
+// the Importer is created.
+func NewImporter(conn io.ReadWriter) *Importer {
imp := new(Importer)
imp.encDec = newEncDec(conn)
- imp.conn = conn
imp.chans = make(map[int]*netChan)
imp.names = make(map[string]*netChan)
imp.errors = make(chan os.Error, 10)
go imp.run()
- return imp, nil
+ return imp
+}
+
+// Import imports a set of channels from the given network and address.
+func Import(network, remoteaddr string) (*Importer, os.Error) {
+ conn, err := net.Dial(network, "", remoteaddr)
+ if err != nil {
+ return nil, err
+ }
+ return NewImporter(conn), nil
}
// shutdown closes all channels for which we are receiving data from the remote side.
}
if err.Error != "" {
impLog("response error:", err.Error)
- if sent := imp.errors <- os.ErrorString(err.Error); !sent {
+ select {
+ case imp.errors <- os.ErrorString(err.Error):
+ continue // errors are not acknowledged
+ default:
imp.shutdown()
return
}
- continue // errors are not acknowledged.
}
case payClosed:
nch := imp.getChan(hdr.Id, false)
// The channel to be bound to the remote site's channel is provided
// in the call and may be of arbitrary channel type.
// Despite the literal signature, the effective signature is
-// ImportNValues(name string, chT chan T, dir Dir, n int) os.Error
+// ImportNValues(name string, chT chan T, dir Dir, size, n int) os.Error
// Example usage:
// imp, err := NewImporter("tcp", "netchanserver.mydomain.com:1234")
-// if err != nil { log.Exit(err) }
+// if err != nil { log.Fatal(err) }
// ch := make(chan myType)
-// err = imp.ImportNValues("name", ch, Recv, 1)
-// if err != nil { log.Exit(err) }
+// err = imp.ImportNValues("name", ch, Recv, 1, 1)
+// if err != nil { log.Fatal(err) }
// fmt.Printf("%+v\n", <-ch)
func (imp *Importer) ImportNValues(name string, chT interface{}, dir Dir, size, n int) os.Error {
ch, err := checkChan(chT, dir)
// the channel. Messages in flight for the channel may be dropped.
func (imp *Importer) Hangup(name string) os.Error {
imp.chanLock.Lock()
- nc, ok := imp.names[name]
- if ok {
- imp.names[name] = nil, false
- imp.chans[nc.id] = nil, false
- }
- imp.chanLock.Unlock()
- if !ok {
+ defer imp.chanLock.Unlock()
+ nc := imp.names[name]
+ if nc == nil {
return os.ErrorString("netchan import: hangup: no such channel: " + name)
}
+ imp.names[name] = nil, false
+ imp.chans[nc.id] = nil, false
nc.close()
return nil
}
package netchan
import (
+ "net"
"strings"
"testing"
"time"
}
go func() {
for i := 0; i < n; i++ {
- ch <- base+i
+ ch <- base + i
}
close(ch)
if done != nil {
}
go func() {
for i := 0; i < n; i++ {
- ch <- base+i
+ ch <- base + i
}
close(ch)
if done != nil {
}
func TestExportSendImportReceive(t *testing.T) {
- exp, err := NewExporter("tcp", "127.0.0.1:0")
- if err != nil {
- t.Fatal("new exporter:", err)
- }
- imp, err := NewImporter("tcp", exp.Addr().String())
- if err != nil {
- t.Fatal("new importer:", err)
- }
+ exp, imp := pair(t)
exportSend(exp, count, t, nil)
importReceive(imp, t, nil)
}
func TestExportReceiveImportSend(t *testing.T) {
- exp, err := NewExporter("tcp", "127.0.0.1:0")
- if err != nil {
- t.Fatal("new exporter:", err)
- }
- imp, err := NewImporter("tcp", exp.Addr().String())
- if err != nil {
- t.Fatal("new importer:", err)
- }
+ exp, imp := pair(t)
expDone := make(chan bool)
done := make(chan bool)
go func() {
}
func TestClosingExportSendImportReceive(t *testing.T) {
- exp, err := NewExporter("tcp", "127.0.0.1:0")
- if err != nil {
- t.Fatal("new exporter:", err)
- }
- imp, err := NewImporter("tcp", exp.Addr().String())
- if err != nil {
- t.Fatal("new importer:", err)
- }
+ exp, imp := pair(t)
exportSend(exp, closeCount, t, nil)
importReceive(imp, t, nil)
}
func TestClosingImportSendExportReceive(t *testing.T) {
- exp, err := NewExporter("tcp", "127.0.0.1:0")
- if err != nil {
- t.Fatal("new exporter:", err)
- }
- imp, err := NewImporter("tcp", exp.Addr().String())
- if err != nil {
- t.Fatal("new importer:", err)
- }
+ exp, imp := pair(t)
expDone := make(chan bool)
done := make(chan bool)
go func() {
}
func TestErrorForIllegalChannel(t *testing.T) {
- exp, err := NewExporter("tcp", "127.0.0.1:0")
- if err != nil {
- t.Fatal("new exporter:", err)
- }
- imp, err := NewImporter("tcp", exp.Addr().String())
- if err != nil {
- t.Fatal("new importer:", err)
- }
+ exp, imp := pair(t)
// Now export a channel.
ch := make(chan int, 1)
- err = exp.Export("aChannel", ch, Send)
+ err := exp.Export("aChannel", ch, Send)
if err != nil {
t.Fatal("export:", err)
}
// Not a great test but it does at least invoke Drain.
func TestExportDrain(t *testing.T) {
- exp, err := NewExporter("tcp", "127.0.0.1:0")
- if err != nil {
- t.Fatal("new exporter:", err)
- }
- imp, err := NewImporter("tcp", exp.Addr().String())
- if err != nil {
- t.Fatal("new importer:", err)
- }
+ exp, imp := pair(t)
done := make(chan bool)
go func() {
exportSend(exp, closeCount, t, nil)
// Not a great test but it does at least invoke Sync.
func TestExportSync(t *testing.T) {
- exp, err := NewExporter("tcp", "127.0.0.1:0")
- if err != nil {
- t.Fatal("new exporter:", err)
- }
- imp, err := NewImporter("tcp", exp.Addr().String())
- if err != nil {
- t.Fatal("new importer:", err)
- }
+ exp, imp := pair(t)
done := make(chan bool)
exportSend(exp, closeCount, t, nil)
go importReceive(imp, t, done)
// Test hanging up the send side of an export.
// TODO: test hanging up the receive side of an export.
func TestExportHangup(t *testing.T) {
- exp, err := NewExporter("tcp", "127.0.0.1:0")
- if err != nil {
- t.Fatal("new exporter:", err)
- }
- imp, err := NewImporter("tcp", exp.Addr().String())
- if err != nil {
- t.Fatal("new importer:", err)
- }
+ exp, imp := pair(t)
ech := make(chan int)
- err = exp.Export("exportedSend", ech, Send)
+ err := exp.Export("exportedSend", ech, Send)
if err != nil {
t.Fatal("export:", err)
}
// Test hanging up the send side of an import.
// TODO: test hanging up the receive side of an import.
func TestImportHangup(t *testing.T) {
- exp, err := NewExporter("tcp", "127.0.0.1:0")
- if err != nil {
- t.Fatal("new exporter:", err)
- }
- imp, err := NewImporter("tcp", exp.Addr().String())
- if err != nil {
- t.Fatal("new importer:", err)
- }
+ exp, imp := pair(t)
ech := make(chan int)
- err = exp.Export("exportedRecv", ech, Recv)
+ err := exp.Export("exportedRecv", ech, Recv)
if err != nil {
t.Fatal("export:", err)
}
// This test checks that channel operations can proceed
// even when other concurrent operations are blocked.
func TestIndependentSends(t *testing.T) {
- exp, err := NewExporter("tcp", "127.0.0.1:0")
- if err != nil {
- t.Fatal("new exporter:", err)
- }
- imp, err := NewImporter("tcp", exp.Addr().String())
- if err != nil {
- t.Fatal("new importer:", err)
- }
+ exp, imp := pair(t)
exportLoopback(exp, t)
}
func TestCrossConnect(t *testing.T) {
- e1, err := NewExporter("tcp", "127.0.0.1:0")
- if err != nil {
- t.Fatal("new exporter:", err)
- }
- i1, err := NewImporter("tcp", e1.Addr().String())
- if err != nil {
- t.Fatal("new importer:", err)
- }
-
- e2, err := NewExporter("tcp", "127.0.0.1:0")
- if err != nil {
- t.Fatal("new exporter:", err)
- }
- i2, err := NewImporter("tcp", e2.Addr().String())
- if err != nil {
- t.Fatal("new importer:", err)
- }
+ e1, i1 := pair(t)
+ e2, i2 := pair(t)
crossExport(e1, e2, t)
crossImport(i1, i2, t)
// test flow control from exporter to importer.
func TestExportFlowControl(t *testing.T) {
- exp, err := NewExporter("tcp", "127.0.0.1:0")
- if err != nil {
- t.Fatal("new exporter:", err)
- }
- imp, err := NewImporter("tcp", exp.Addr().String())
- if err != nil {
- t.Fatal("new importer:", err)
- }
+ exp, imp := pair(t)
sendDone := make(chan bool, 1)
exportSend(exp, flowCount, t, sendDone)
ch := make(chan int)
- err = imp.ImportNValues("exportedSend", ch, Recv, 20, -1)
+ err := imp.ImportNValues("exportedSend", ch, Recv, 20, -1)
if err != nil {
t.Fatal("importReceive:", err)
}
// test flow control from importer to exporter.
func TestImportFlowControl(t *testing.T) {
- exp, err := NewExporter("tcp", "127.0.0.1:0")
- if err != nil {
- t.Fatal("new exporter:", err)
- }
- imp, err := NewImporter("tcp", exp.Addr().String())
- if err != nil {
- t.Fatal("new importer:", err)
- }
+ exp, imp := pair(t)
ch := make(chan int)
- err = exp.Export("exportedRecv", ch, Recv)
+ err := exp.Export("exportedRecv", ch, Recv)
if err != nil {
t.Fatal("importReceive:", err)
}
t.Fatalf("expected %d values; got %d", N, n)
}
}
+
+func pair(t *testing.T) (*Exporter, *Importer) {
+ c0, c1 := net.Pipe()
+ exp := NewExporter()
+ go exp.ServeConn(c0)
+ imp := NewImporter(c1)
+ return exp, imp
+}
if len(value) > 0 {
v = syscall.StringToUTF16Ptr(value)
}
- ok, e := syscall.SetEnvironmentVariable(syscall.StringToUTF16Ptr(key), v)
- if !ok {
+ e := syscall.SetEnvironmentVariable(syscall.StringToUTF16Ptr(key), v)
+ if e != 0 {
return NewSyscallError("SetEnvironmentVariable", e)
}
return nil
}
func (e Errno) Timeout() bool {
- return e == Errno(syscall.EAGAIN) || e == Errno(syscall.EWOULDBLOCK)
+ return e == Errno(syscall.EAGAIN) || e == Errno(syscall.EWOULDBLOCK) || e == Errno(syscall.ETIMEDOUT)
}
// Commonly known Unix errors.
ECONNREFUSED Error = Errno(syscall.ECONNREFUSED)
ENAMETOOLONG Error = Errno(syscall.ENAMETOOLONG)
EAFNOSUPPORT Error = Errno(syscall.EAFNOSUPPORT)
+ ETIMEDOUT Error = Errno(syscall.ETIMEDOUT)
+ ENOTCONN Error = Errno(syscall.ENOTCONN)
)
// PathError records an error and the operation and file path that caused it.
package os
import (
+ "runtime"
"syscall"
)
-// ForkExec forks the current process and invokes Exec with the program, arguments,
-// and environment specified by name, argv, and envv. It returns the process
-// id of the forked process and an Error, if any. The fd array specifies the
+// Process stores the information about a process created by StartProcess.
+type Process struct {
+ Pid int
+ handle int
+}
+
+func newProcess(pid, handle int) *Process {
+ p := &Process{pid, handle}
+ runtime.SetFinalizer(p, (*Process).Release)
+ return p
+}
+
+// StartProcess starts a new process with the program, arguments,
+// and environment specified by name, argv, and envv. The fd array specifies the
// file descriptors to be set up in the new process: fd[0] will be Unix file
// descriptor 0 (standard input), fd[1] descriptor 1, and so on. A nil entry
// will cause the child to have no open file descriptor with that index.
// If dir is not empty, the child chdirs into the directory before execing the program.
-func ForkExec(name string, argv []string, envv []string, dir string, fd []*File) (pid int, err Error) {
+func StartProcess(name string, argv []string, envv []string, dir string, fd []*File) (p *Process, err Error) {
if envv == nil {
envv = Environ()
}
}
}
- p, e := syscall.ForkExec(name, argv, envv, dir, intfd)
+ pid, h, e := syscall.StartProcess(name, argv, envv, dir, intfd)
if e != 0 {
- return 0, &PathError{"fork/exec", name, Errno(e)}
+ return nil, &PathError{"fork/exec", name, Errno(e)}
}
- return p, nil
+ return newProcess(pid, h), nil
}
// Exec replaces the current process with an execution of the
// named binary, with arguments argv and environment envv.
// If successful, Exec never returns. If it fails, it returns an Error.
-// ForkExec is almost always a better way to execute a program.
+// StartProcess is almost always a better way to execute a program.
func Exec(name string, argv []string, envv []string) Error {
if envv == nil {
envv = Environ()
Rusage *syscall.Rusage // System-dependent resource usage info.
}
-// Options for Wait.
-const (
- WNOHANG = syscall.WNOHANG // Don't wait if no process has exited.
- WSTOPPED = syscall.WSTOPPED // If set, status of stopped subprocesses is also reported.
- WUNTRACED = syscall.WUNTRACED // Usually an alias for WSTOPPED.
- WRUSAGE = 1 << 20 // Record resource usage.
-)
-
-// WRUSAGE must not be too high a bit, to avoid clashing with Linux's
-// WCLONE, WALL, and WNOTHREAD flags, which sit in the top few bits of
-// the options
-
// Wait waits for process pid to exit or stop, and then returns a
// Waitmsg describing its status and an Error, if any. The options
// (WNOHANG etc.) affect the behavior of the Wait call.
+// Wait is equivalent to calling FindProcess and then Wait
+// and Release on the result.
func Wait(pid int, options int) (w *Waitmsg, err Error) {
- var status syscall.WaitStatus
- var rusage *syscall.Rusage
- if options&WRUSAGE != 0 {
- rusage = new(syscall.Rusage)
- options ^= WRUSAGE
- }
- pid1, e := syscall.Wait4(pid, &status, options, rusage)
- if e != 0 {
- return nil, NewSyscallError("wait", e)
+ p, e := FindProcess(pid)
+ if e != nil {
+ return nil, e
}
- w = new(Waitmsg)
- w.Pid = pid1
- w.WaitStatus = status
- w.Rusage = rusage
- return w, nil
+ defer p.Release()
+ return p.Wait(options)
}
// Convert i to decimal string.
--- /dev/null
+// Copyright 2009 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package os
+
+import (
+ "runtime"
+ "syscall"
+)
+
+// Options for Wait.
+const (
+ WNOHANG = syscall.WNOHANG // Don't wait if no process has exited.
+ WSTOPPED = syscall.WSTOPPED // If set, status of stopped subprocesses is also reported.
+ WUNTRACED = syscall.WUNTRACED // Usually an alias for WSTOPPED.
+ WRUSAGE = 1 << 20 // Record resource usage.
+)
+
+// WRUSAGE must not be too high a bit, to avoid clashing with Linux's
+// WCLONE, WALL, and WNOTHREAD flags, which sit in the top few bits of
+// the options
+
+// Wait waits for the Process to exit or stop, and then returns a
+// Waitmsg describing its status and an Error, if any. The options
+// (WNOHANG etc.) affect the behavior of the Wait call.
+func (p *Process) Wait(options int) (w *Waitmsg, err Error) {
+ if p.Pid == -1 {
+ return nil, EINVAL
+ }
+ var status syscall.WaitStatus
+ var rusage *syscall.Rusage
+ if options&WRUSAGE != 0 {
+ rusage = new(syscall.Rusage)
+ options ^= WRUSAGE
+ }
+ pid1, e := syscall.Wait4(p.Pid, &status, options, rusage)
+ if e != 0 {
+ return nil, NewSyscallError("wait", e)
+ }
+ w = new(Waitmsg)
+ w.Pid = pid1
+ w.WaitStatus = status
+ w.Rusage = rusage
+ return w, nil
+}
+
+// Release releases any resources associated with the Process.
+func (p *Process) Release() Error {
+ // NOOP for unix.
+ p.Pid = -1
+ // no need for a finalizer anymore
+ runtime.SetFinalizer(p, nil)
+ return nil
+}
+
+// FindProcess looks for a running process by its pid.
+// The Process it returns can be used to obtain information
+// about the underlying operating system process.
+func FindProcess(pid int) (p *Process, err Error) {
+ // NOOP for unix.
+ return newProcess(pid, 0), nil
+}
--- /dev/null
+// Copyright 2009 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package os
+
+import (
+ "runtime"
+ "syscall"
+)
+
+func (p *Process) Wait(options int) (w *Waitmsg, err Error) {
+ s, e := syscall.WaitForSingleObject(int32(p.handle), syscall.INFINITE)
+ switch s {
+ case syscall.WAIT_OBJECT_0:
+ break
+ case syscall.WAIT_FAILED:
+ return nil, NewSyscallError("WaitForSingleObject", e)
+ default:
+ return nil, ErrorString("os: unexpected result from WaitForSingleObject")
+ }
+ var ec uint32
+ e = syscall.GetExitCodeProcess(uint32(p.handle), &ec)
+ if e != 0 {
+ return nil, NewSyscallError("GetExitCodeProcess", e)
+ }
+ return &Waitmsg{p.Pid, syscall.WaitStatus{s, ec}, new(syscall.Rusage)}, nil
+}
+
+func (p *Process) Release() Error {
+ if p.handle == -1 {
+ return EINVAL
+ }
+ e := syscall.CloseHandle(int32(p.handle))
+ if e != 0 {
+ return NewSyscallError("CloseHandle", e)
+ }
+ p.handle = -1
+ // no need for a finalizer anymore
+ runtime.SetFinalizer(p, nil)
+ return nil
+}
+
+func FindProcess(pid int) (p *Process, err Error) {
+ const da = syscall.STANDARD_RIGHTS_READ |
+ syscall.PROCESS_QUERY_INFORMATION | syscall.SYNCHRONIZE
+ h, e := syscall.OpenProcess(da, false, uint32(pid))
+ if e != 0 {
+ return nil, NewSyscallError("OpenProcess", e)
+ }
+ return newProcess(pid, int(h)), nil
+}
Example:
watcher, err := inotify.NewWatcher()
if err != nil {
- log.Exit(err)
+ log.Fatal(err)
}
err = watcher.Watch("/tmp")
if err != nil {
- log.Exit(err)
+ log.Fatal(err)
}
for {
select {
for {
n, errno = syscall.Read(w.fd, buf[0:])
// See if there is a message on the "done" channel
- _, done := <-w.done
+ var done bool
+ select {
+ case done = <-w.done:
+ default:
+ }
// If EOF or a "done" message is received
if n == 0 || done {
adir = "/"
expect = "/\n"
}
- pid, err := ForkExec(cmd, args, nil, adir, []*File{nil, w, Stderr})
+ p, err := StartProcess(cmd, args, nil, adir, []*File{nil, w, Stderr})
if err != nil {
- t.Fatalf("ForkExec: %v", err)
+ t.Fatalf("StartProcess: %v", err)
}
+ defer p.Release()
w.Close()
var b bytes.Buffer
args[0] = cmd
t.Errorf("exec %q returned %q wanted %q", strings.Join(args, " "), output, expect)
}
- Wait(pid, 0)
+ p.Wait(0)
}
func checkMode(t *testing.T, path string, mode uint32) {
if err != nil {
t.Fatal(err)
}
- pid, err := ForkExec("/bin/hostname", []string{"hostname"}, nil, "/", []*File{nil, w, Stderr})
+ p, err := StartProcess("/bin/hostname", []string{"hostname"}, nil, "/", []*File{nil, w, Stderr})
if err != nil {
t.Fatal(err)
}
+ defer p.Release()
w.Close()
var b bytes.Buffer
io.Copy(&b, r)
- Wait(pid, 0)
+ p.Wait(0)
output := b.String()
if n := len(output); n > 0 && output[n-1] == '\n' {
output = output[0 : n-1]
// 2) handle errors, expect none
errors := make(chan os.Error, 64)
Walk(tree.name, v, errors)
- if err, ok := <-errors; ok {
+ select {
+ case err := <-errors:
t.Errorf("no error expected, found: %s", err)
+ default:
+ // ok
}
checkMarks(t)
errors = make(chan os.Error, 64)
os.Chmod(Join(tree.name, tree.entries[1].name), 0)
Walk(tree.name, v, errors)
+ Loop:
for i := 1; i <= 2; i++ {
- if _, ok := <-errors; !ok {
+ select {
+ case <-errors:
+ // ok
+ default:
t.Errorf("%d. error expected, none found", i)
- break
+ break Loop
}
}
- if err, ok := <-errors; ok {
+ select {
+ case err := <-errors:
t.Errorf("only two errors expected, found 3rd: %v", err)
+ default:
+ // ok
}
// the inaccessible subtrees were marked manually
checkMarks(t)
t.Errorf("Value Method returned %d; want 250", i)
}
+ // Curried method of pointer.
+ i = NewValue(&p).Method(0).Call([]Value{NewValue(10)})[0].(*IntValue).Get()
+ if i != 250 {
+ t.Errorf("Value Method returned %d; want 250", i)
+ }
+
+ // Curried method of pointer to value.
+ i = NewValue(p).Addr().Method(0).Call([]Value{NewValue(10)})[0].(*IntValue).Get()
+ if i != 250 {
+ t.Errorf("Value Method returned %d; want 250", i)
+ }
+
// Curried method of interface value.
// Have to wrap interface value in a struct to get at it.
// Passing it to NewValue directly would
t.Errorf("f(o) = %d, want 2", v)
}
}
+
+func TestPtrTo(t *testing.T) {
+ var i int
+
+ typ := Typeof(i)
+ for i = 0; i < 100; i++ {
+ typ = PtrTo(typ)
+ }
+ for i = 0; i < 100; i++ {
+ typ = typ.(*PtrType).Elem()
+ }
+ if typ != Typeof(i) {
+ t.Errorf("after 100 PtrTo and Elem, have %s, want %s", typ, Typeof(i))
+ }
+}
+
+func TestAddr(t *testing.T) {
+ var p struct {
+ X, Y int
+ }
+
+ v := NewValue(&p)
+ v = v.(*PtrValue).Elem()
+ v = v.Addr()
+ v = v.(*PtrValue).Elem()
+ v = v.(*StructValue).Field(0)
+ v.(*IntValue).Set(2)
+ if p.X != 2 {
+ t.Errorf("Addr.Elem.Set failed to set value")
+ }
+
+ // Again but take address of the NewValue value.
+ // Exercises generation of PtrTypes not present in the binary.
+ v = NewValue(&p)
+ v = v.Addr()
+ v = v.(*PtrValue).Elem()
+ v = v.(*PtrValue).Elem()
+ v = v.Addr()
+ v = v.(*PtrValue).Elem()
+ v = v.(*StructValue).Field(0)
+ v.(*IntValue).Set(3)
+ if p.X != 3 {
+ t.Errorf("Addr.Elem.Set failed to set value")
+ }
+
+ // Starting without pointer we should get changed value
+ // in interface.
+ v = NewValue(p)
+ v0 := v
+ v = v.Addr()
+ v = v.(*PtrValue).Elem()
+ v = v.(*StructValue).Field(0)
+ v.(*IntValue).Set(4)
+ if p.X != 3 { // should be unchanged from last time
+ t.Errorf("somehow value Set changed original p")
+ }
+ p = v0.Interface().(struct {
+ X, Y int
+ })
+ if p.X != 4 {
+ t.Errorf("Addr.Elem.Set valued to set value in top value")
+ }
+}
// if depth > 10 { panic("deepValueEqual") } // for debugging
- addr1 := v1.Addr()
- addr2 := v2.Addr()
+ addr1 := v1.UnsafeAddr()
+ addr2 := v2.UnsafeAddr()
if addr1 > addr2 {
// Canonicalize order to reduce number of entries in visited.
addr1, addr2 = addr2, addr1
equalfn func(unsafe.Pointer, unsafe.Pointer, uintptr)
string *string
*uncommonType
+ ptrToThis *runtime.Type
}
type method struct {
elem *runtime.Type
}
+// arrayOrSliceType is an unexported method that guarantees only
+// arrays and slices implement ArrayOrSliceType.
+func (*SliceType) arrayOrSliceType() {}
+
// Struct field
type structField struct {
name *string
// Kind returns the specific kind of this type.
Kind() Kind
- // For non-interface types, Method returns the i'th method with receiver T.
- // For interface types, Method returns the i'th method in the interface.
- // NumMethod returns the number of such methods.
+ // Method returns the i'th method in the type's method set.
+ //
+ // For a non-interface type T or *T, the returned Method's Type and Func
+ // fields describe a function whose first argument is the receiver.
+ //
+ // For an interface type, the returned Method's Type field gives the
+ // method signature, without a receiver, and the Func field is nil.
Method(int) Method
+
+ // NumMethods returns the number of methods in the type's method set.
NumMethod() int
+
+ common() *commonType
uncommon() *uncommonType
}
func (t *commonType) Kind() Kind { return Kind(t.kind & kindMask) }
+func (t *commonType) common() *commonType { return t }
+
func (t *uncommonType) Method(i int) (m Method) {
if t == nil || i < 0 || i >= len(t.methods) {
return
}
m.Type = runtimeToType(p.typ).(*FuncType)
fn := p.tfn
- m.Func = &FuncValue{value: value{m.Type, addr(&fn), true}}
+ m.Func = &FuncValue{value: value{m.Type, addr(&fn), canSet}}
return
}
// Elem returns the type of the array's elements.
func (t *ArrayType) Elem() Type { return runtimeToType(t.elem) }
+// arrayOrSliceType is an unexported method that guarantees only
+// arrays and slices implement ArrayOrSliceType.
+func (*ArrayType) arrayOrSliceType() {}
+
// Dir returns the channel direction.
func (t *ChanType) Dir() ChanDir { return ChanDir(t.dir) }
// NumOut returns the number of function output parameters.
func (t *FuncType) NumOut() int { return len(t.out) }
-// Method returns the i'th interface method.
+// Method returns the i'th method in the type's method set.
func (t *InterfaceType) Method(i int) (m Method) {
if i < 0 || i >= len(t.methods) {
return
return
}
-// NumMethod returns the number of interface methods.
+// NumMethod returns the number of interface methods in the type's method set.
func (t *InterfaceType) NumMethod() int { return len(t.methods) }
// Key returns the map key type.
type ArrayOrSliceType interface {
Type
Elem() Type
+ arrayOrSliceType() // Guarantees only Array and Slice implement this interface.
}
// Typeof returns the reflection Type of the value in the interface{}.
func Typeof(i interface{}) Type { return canonicalize(toType(unsafe.Typeof(i))) }
+
+// ptrMap is the cache for PtrTo.
+var ptrMap struct {
+ sync.RWMutex
+ m map[Type]*PtrType
+}
+
+// runtimePtrType is the runtime layout for a *PtrType.
+// The memory immediately before the *PtrType is always
+// the canonical runtime.Type to be used for a *runtime.Type
+// describing this PtrType.
+type runtimePtrType struct {
+ runtime.Type
+ runtime.PtrType
+}
+
+// PtrTo returns the pointer type with element t.
+// For example, if t represents type Foo, PtrTo(t) represents *Foo.
+func PtrTo(t Type) *PtrType {
+ // If t records its pointer-to type, use it.
+ ct := t.common()
+ if p := ct.ptrToThis; p != nil {
+ return runtimeToType(p).(*PtrType)
+ }
+
+ // Otherwise, synthesize one.
+ // This only happens for pointers with no methods.
+ // We keep the mapping in a map on the side, because
+ // this operation is rare and a separate map lets us keep
+ // the type structures in read-only memory.
+ ptrMap.RLock()
+ if m := ptrMap.m; m != nil {
+ if p := m[t]; p != nil {
+ ptrMap.RUnlock()
+ return p
+ }
+ }
+ ptrMap.RUnlock()
+ ptrMap.Lock()
+ if ptrMap.m == nil {
+ ptrMap.m = make(map[Type]*PtrType)
+ }
+ p := ptrMap.m[t]
+ if p != nil {
+ // some other goroutine won the race and created it
+ ptrMap.Unlock()
+ return p
+ }
+
+ rt := (*runtime.Type)(unsafe.Pointer(ct))
+
+ rp := new(runtime.PtrType)
+
+ // initialize rp used *byte's PtrType as a prototype.
+ // have to do assignment as PtrType, not runtime.PtrType,
+ // in order to write to unexported fields.
+ p = (*PtrType)(unsafe.Pointer(rp))
+ bp := (*PtrType)(unsafe.Pointer(unsafe.Typeof((*byte)(nil)).(*runtime.PtrType)))
+ *p = *bp
+
+ s := "*" + *ct.string
+ p.string = &s
+
+ // For the type structures linked into the binary, the
+ // compiler provides a good hash of the string.
+ // Create a good hash for the new string by using
+ // the FNV-1 hash's mixing function to combine the
+ // old hash and the new "*".
+ p.hash = ct.hash*16777619 ^ '*'
+
+ p.uncommonType = nil
+ p.ptrToThis = nil
+ p.elem = rt
+
+ ptrMap.m[t] = (*PtrType)(unsafe.Pointer(rp))
+ ptrMap.Unlock()
+ return p
+}
)
const ptrSize = uintptr(unsafe.Sizeof((*byte)(nil)))
-const cannotSet = "cannot set value obtained via unexported struct field"
+const cannotSet = "cannot set value obtained from unexported struct field"
type addr unsafe.Pointer
// Interface returns the value as an interface{}.
Interface() interface{}
- // CanSet returns whether the value can be changed.
+ // CanSet returns true if the value can be changed.
// Values obtained by the use of non-exported struct fields
// can be used in Get but not Set.
- // If CanSet() returns false, calling the type-specific Set
- // will cause a crash.
+ // If CanSet returns false, calling the type-specific Set will panic.
CanSet() bool
// SetValue assigns v to the value; v must have the same type as the value.
SetValue(v Value)
- // Addr returns a pointer to the underlying data.
- // It is for advanced clients that also
- // import the "unsafe" package.
- Addr() uintptr
+ // CanAddr returns true if the value's address can be obtained with Addr.
+ // Such values are called addressable. A value is addressable if it is
+ // an element of a slice, an element of an addressable array,
+ // a field of an addressable struct, the result of dereferencing a pointer,
+ // or the result of a call to NewValue, MakeChan, MakeMap, or MakeZero.
+ // If CanAddr returns false, calling Addr will panic.
+ CanAddr() bool
+
+ // Addr returns the address of the value.
+ // If the value is not addressable, Addr panics.
+ // Addr is typically used to obtain a pointer to a struct field or slice element
+ // in order to call a method that requires a pointer receiver.
+ Addr() *PtrValue
+
+ // UnsafeAddr returns a pointer to the underlying data.
+ // It is for advanced clients that also import the "unsafe" package.
+ UnsafeAddr() uintptr
// Method returns a FuncValue corresponding to the value's i'th method.
// The arguments to a Call on the returned FuncValue
getAddr() addr
}
+// flags for value
+const (
+ canSet uint32 = 1 << iota // can set value (write to *v.addr)
+ canAddr // can take address of value
+ canStore // can store through value (write to **v.addr)
+)
+
// value is the common implementation of most values.
// It is embedded in other, public struct types, but always
// with a unique tag like "uint" or "float" so that the client cannot
// convert from, say, *UintValue to *FloatValue.
type value struct {
- typ Type
- addr addr
- canSet bool
+ typ Type
+ addr addr
+ flag uint32
}
func (v *value) Type() Type { return v.typ }
-func (v *value) Addr() uintptr { return uintptr(v.addr) }
+func (v *value) Addr() *PtrValue {
+ if !v.CanAddr() {
+ panic("reflect: cannot take address of value")
+ }
+ a := v.addr
+ flag := canSet
+ if v.CanSet() {
+ flag |= canStore
+ }
+ // We could safely set canAddr here too -
+ // the caller would get the address of a -
+ // but it doesn't match the Go model.
+ // The language doesn't let you say &&v.
+ return newValue(PtrTo(v.typ), addr(&a), flag).(*PtrValue)
+}
+
+func (v *value) UnsafeAddr() uintptr { return uintptr(v.addr) }
func (v *value) getAddr() addr { return v.addr }
return unsafe.Unreflect(v.typ, unsafe.Pointer(v.addr))
}
-func (v *value) CanSet() bool { return v.canSet }
+func (v *value) CanSet() bool { return v.flag&canSet != 0 }
+
+func (v *value) CanAddr() bool { return v.flag&canAddr != 0 }
+
/*
* basic types
// Set sets v to the value x.
func (v *BoolValue) Set(x bool) {
- if !v.canSet {
+ if !v.CanSet() {
panic(cannotSet)
}
*(*bool)(v.addr) = x
// Set sets v to the value x.
func (v *FloatValue) Set(x float64) {
- if !v.canSet {
+ if !v.CanSet() {
panic(cannotSet)
}
switch v.typ.Kind() {
// Set sets v to the value x.
func (v *ComplexValue) Set(x complex128) {
- if !v.canSet {
+ if !v.CanSet() {
panic(cannotSet)
}
switch v.typ.Kind() {
// Set sets v to the value x.
func (v *IntValue) Set(x int64) {
- if !v.canSet {
+ if !v.CanSet() {
panic(cannotSet)
}
switch v.typ.Kind() {
// Set sets v to the value x.
func (v *StringValue) Set(x string) {
- if !v.canSet {
+ if !v.CanSet() {
panic(cannotSet)
}
*(*string)(v.addr) = x
// Set sets v to the value x.
func (v *UintValue) Set(x uint64) {
- if !v.canSet {
+ if !v.CanSet() {
panic(cannotSet)
}
switch v.typ.Kind() {
// Set sets v to the value x.
func (v *UnsafePointerValue) Set(x unsafe.Pointer) {
- if !v.canSet {
+ if !v.CanSet() {
panic(cannotSet)
}
*(*unsafe.Pointer)(v.addr) = x
// Set assigns x to v.
// The new value x must have the same type as v.
func (v *ArrayValue) Set(x *ArrayValue) {
- if !v.canSet {
+ if !v.CanSet() {
panic(cannotSet)
}
typesMustMatch(v.typ, x.typ)
panic("array index out of bounds")
}
p := addr(uintptr(v.addr()) + uintptr(i)*typ.Size())
- return newValue(typ, p, v.canSet)
+ return newValue(typ, p, v.flag)
}
/*
// Set assigns x to v.
// The new value x must have the same type as v.
func (v *SliceValue) Set(x *SliceValue) {
- if !v.canSet {
+ if !v.CanSet() {
panic(cannotSet)
}
typesMustMatch(v.typ, x.typ)
s.Data = uintptr(v.addr()) + uintptr(beg)*typ.Elem().Size()
s.Len = end - beg
s.Cap = cap - beg
- return newValue(typ, addr(s), v.canSet).(*SliceValue)
+
+ // Like the result of Addr, we treat Slice as an
+ // unaddressable temporary, so don't set canAddr.
+ flag := canSet
+ if v.flag&canStore != 0 {
+ flag |= canStore
+ }
+ return newValue(typ, addr(s), flag).(*SliceValue)
}
// Elem returns the i'th element of v.
panic("reflect: slice index out of range")
}
p := addr(uintptr(v.addr()) + uintptr(i)*typ.Size())
- return newValue(typ, p, v.canSet)
+ flag := canAddr
+ if v.flag&canStore != 0 {
+ flag |= canSet | canStore
+ }
+ return newValue(typ, p, flag)
}
// MakeSlice creates a new zero-initialized slice value
Len: len,
Cap: cap,
}
- return newValue(typ, addr(s), true).(*SliceValue)
+ return newValue(typ, addr(s), canAddr|canSet|canStore).(*SliceValue)
}
/*
// Set assigns x to v.
// The new value x must have the same type as v.
func (v *ChanValue) Set(x *ChanValue) {
- if !v.canSet {
+ if !v.CanSet() {
panic(cannotSet)
}
typesMustMatch(v.typ, x.typ)
// Set assigns x to v.
// The new value x must have the same type as v.
func (v *FuncValue) Set(x *FuncValue) {
- if !v.canSet {
+ if !v.CanSet() {
panic(cannotSet)
}
typesMustMatch(v.typ, x.typ)
}
p := &t.methods[i]
fn := p.tfn
- fv := &FuncValue{value: value{runtimeToType(p.typ), addr(&fn), true}, first: v, isInterface: false}
+ fv := &FuncValue{value: value{runtimeToType(p.typ), addr(&fn), 0}, first: v, isInterface: false}
return fv
}
// implemented in ../pkg/runtime/*/asm.s
func call(typ *FuncType, fnaddr *byte, isInterface bool, params *addr, results *addr)
+// Interface returns the fv as an interface value.
+// If fv is a method obtained by invoking Value.Method
+// (as opposed to Type.Method), Interface cannot return an
+// interface value, so it panics.
+func (fv *FuncValue) Interface() interface{} {
+ if fv.first != nil {
+ panic("FuncValue: cannot create interface value for method with bound receiver")
+ }
+ return fv.value.Interface()
+}
+
// Call calls the function fv with input parameters in.
// It returns the function's output parameters as Values.
func (fv *FuncValue) Call(in []Value) []Value {
if x != nil {
i = x.Interface()
}
- if !v.canSet {
+ if !v.CanSet() {
panic(cannotSet)
}
// Two different representations; see comment in Get.
// Interface is two words: itable, data.
tab := *(**[10000]addr)(v.addr)
- data := &value{Typeof((*byte)(nil)), addr(uintptr(v.addr) + ptrSize), true}
+ data := &value{Typeof((*byte)(nil)), addr(uintptr(v.addr) + ptrSize), 0}
fn := tab[i+1]
- fv := &FuncValue{value: value{runtimeToType(p.typ), addr(&fn), true}, first: data, isInterface: true}
+ fv := &FuncValue{value: value{runtimeToType(p.typ), addr(&fn), 0}, first: data, isInterface: true}
return fv
}
// Set assigns x to v.
// The new value x must have the same type as v.
func (v *MapValue) Set(x *MapValue) {
- if !v.canSet {
+ if !v.CanSet() {
panic(cannotSet)
}
if x == nil {
func (v *PtrValue) Get() uintptr { return *(*uintptr)(v.addr) }
// Set assigns x to v.
-// The new value x must have the same type as v.
+// The new value x must have the same type as v, and x.Elem().CanSet() must be true.
func (v *PtrValue) Set(x *PtrValue) {
if x == nil {
*(**uintptr)(v.addr) = nil
return
}
- if !v.canSet {
+ if !v.CanSet() {
panic(cannotSet)
}
+ if x.flag&canStore == 0 {
+ panic("cannot copy pointer obtained from unexported struct field")
+ }
typesMustMatch(v.typ, x.typ)
// TODO: This will have to move into the runtime
// once the new gc goes in
typesMustMatch(v.typ.(*PtrType).Elem(), x.Type())
// TODO: This will have to move into the runtime
// once the new gc goes in.
- *(*uintptr)(v.addr) = x.Addr()
+ *(*uintptr)(v.addr) = x.UnsafeAddr()
}
// Elem returns the value that v points to.
if v.IsNil() {
return nil
}
- return newValue(v.typ.(*PtrType).Elem(), *(*addr)(v.addr), v.canSet)
+ flag := canAddr
+ if v.flag&canStore != 0 {
+ flag |= canSet | canStore
+ }
+ return newValue(v.typ.(*PtrType).Elem(), *(*addr)(v.addr), flag)
}
// Indirect returns the value that v points to.
func (v *StructValue) Set(x *StructValue) {
// TODO: This will have to move into the runtime
// once the gc goes in.
- if !v.canSet {
+ if !v.CanSet() {
panic(cannotSet)
}
typesMustMatch(v.typ, x.typ)
return nil
}
f := t.Field(i)
- return newValue(f.Type, addr(uintptr(v.addr)+f.Offset), v.canSet && f.PkgPath == "")
+ flag := v.flag
+ if f.PkgPath != "" {
+ // unexported field
+ flag &^= canSet | canStore
+ }
+ return newValue(f.Type, addr(uintptr(v.addr)+f.Offset), flag)
}
// FieldByIndex returns the nested field corresponding to index.
return nil
}
t, a := unsafe.Reflect(i)
- return newValue(canonicalize(toType(t)), addr(a), true)
+ return newValue(canonicalize(toType(t)), addr(a), canSet|canAddr|canStore)
}
-func newValue(typ Type, addr addr, canSet bool) Value {
- v := value{typ, addr, canSet}
+func newValue(typ Type, addr addr, flag uint32) Value {
+ v := value{typ, addr, flag}
switch typ.(type) {
case *ArrayType:
return &ArrayValue{v}
if typ == nil {
return nil
}
- return newValue(typ, addr(unsafe.New(typ)), true)
+ return newValue(typ, addr(unsafe.New(typ)), canSet|canAddr|canStore)
}
}
func BenchmarkLiteral(b *testing.B) {
- x := strings.Repeat("x", 50)
+ x := strings.Repeat("x", 50) + "y"
b.StopTimer()
- re := MustCompile(x)
+ re := MustCompile("y")
b.StartTimer()
for i := 0; i < b.N; i++ {
if !re.MatchString(x) {
}
func BenchmarkNotLiteral(b *testing.B) {
- x := strings.Repeat("x", 49)
+ x := strings.Repeat("x", 50) + "y"
b.StopTimer()
- re := MustCompile("^" + x)
+ re := MustCompile(".y")
b.StartTimer()
for i := 0; i < b.N; i++ {
if !re.MatchString(x) {
import (
"fmt"
+ "strings"
"testing"
)
}
}
+func TestFindReaderIndex(t *testing.T) {
+ for _, test := range findTests {
+ testFindIndex(&test, MustCompile(test.pat).FindReaderIndex(strings.NewReader(test.text)), t)
+ }
+}
+
// Now come the simple All cases.
func TestFindAll(t *testing.T) {
}
}
-func TestFindStringSubmatchndex(t *testing.T) {
+func TestFindStringSubmatchIndex(t *testing.T) {
for _, test := range findTests {
testFindSubmatchIndex(&test, MustCompile(test.pat).FindStringSubmatchIndex(test.text), t)
}
}
+func TestFindReaderSubmatchIndex(t *testing.T) {
+ for _, test := range findTests {
+ testFindSubmatchIndex(&test, MustCompile(test.pat).FindReaderSubmatchIndex(strings.NewReader(test.text)), t)
+ }
+}
+
// Now come the monster AllSubmatch cases.
func TestFindAllSubmatch(t *testing.T) {
}
}
-func TestFindAllStringSubmatchndex(t *testing.T) {
+func TestFindAllStringSubmatchIndex(t *testing.T) {
for _, test := range findTests {
testFindAllSubmatchIndex(&test, MustCompile(test.pat).FindAllStringSubmatchIndex(test.text, -1), t)
}
// text of the match/submatch. If an index is negative, it means that
// subexpression did not match any string in the input.
//
+// There is also a subset of the methods that can be applied to text read
+// from a RuneReader:
+//
+// MatchReader, FindReaderIndex, FindReaderSubmatchIndex
+//
+// This set may grow. Note that regular expression matches may need to
+// examine text beyond the text returned by a match, so the methods that
+// match text from a RuneReader may read arbitrarily far into the input
+// before returning.
+//
// (There are a few other methods that do not match this pattern.)
//
package regexp
panic(err)
}
-const endOfFile = -1
+const endOfText = -1
func (p *parser) c() int { return p.ch }
func (p *parser) nextc() int {
if p.pos >= len(p.re.expr) {
- p.ch = endOfFile
+ p.ch = endOfText
} else {
c, w := utf8.DecodeRuneInString(p.re.expr[p.pos:])
p.ch = c
if c == '\\' {
c = p.nextc()
switch {
- case c == endOfFile:
+ case c == endOfText:
p.error(ErrExtraneousBackslash)
case ispunct(c):
// c is as delivered
left := -1
for {
switch c := p.c(); c {
- case ']', endOfFile:
+ case ']', endOfText:
if left >= 0 {
p.error(ErrBadRange)
}
func (p *parser) term() (start, end *instr) {
switch c := p.c(); c {
- case '|', endOfFile:
+ case '|', endOfText:
return nil, nil
case '*', '+', '?':
p.error(ErrBareClosure)
// match vectors away as we execute. Matches are ref counted and returned
// to a free list when no longer active. Increases a simple benchmark by 22X.
type matchArena struct {
- head *matchVec
- len int // length of match vector
+ head *matchVec
+ len int // length of match vector
+ pos int
+ atBOT bool // whether we're at beginning of text
+ atEOT bool // whether we're at end of text
}
type matchVec struct {
// Append new state to to-do list. Leftmost-longest wins so avoid
// adding a state that's already active. The matchVec will be inc-ref'ed
// if it is assigned to a state.
-func (a *matchArena) addState(s []state, inst *instr, prefixed bool, match *matchVec, pos, end int) []state {
+func (a *matchArena) addState(s []state, inst *instr, prefixed bool, match *matchVec) []state {
switch inst.kind {
case iBOT:
- if pos == 0 {
- s = a.addState(s, inst.next, prefixed, match, pos, end)
+ if a.atBOT {
+ s = a.addState(s, inst.next, prefixed, match)
}
return s
case iEOT:
- if pos == end {
- s = a.addState(s, inst.next, prefixed, match, pos, end)
+ if a.atEOT {
+ s = a.addState(s, inst.next, prefixed, match)
}
return s
case iBra:
- match.m[inst.braNum] = pos
- s = a.addState(s, inst.next, prefixed, match, pos, end)
+ match.m[inst.braNum] = a.pos
+ s = a.addState(s, inst.next, prefixed, match)
return s
}
l := len(s)
s = append(s, state{inst, prefixed, match})
match.ref++
if inst.kind == iAlt {
- s = a.addState(s, inst.left, prefixed, a.copy(match), pos, end)
+ s = a.addState(s, inst.left, prefixed, a.copy(match))
// give other branch a copy of this match vector
- s = a.addState(s, inst.next, prefixed, a.copy(match), pos, end)
+ s = a.addState(s, inst.next, prefixed, a.copy(match))
}
return s
}
-// Accepts either string or bytes - the logic is identical either way.
-// If bytes == nil, scan str.
-func (re *Regexp) doExecute(str string, bytestr []byte, pos int) []int {
+// input abstracts different representations of the input text. It provides
+// one-character lookahead.
+type input interface {
+ step(pos int) (rune int, width int) // advance one rune
+ canCheckPrefix() bool // can we look ahead without losing info?
+ hasPrefix(re *Regexp) bool
+ index(re *Regexp, pos int) int
+}
+
+// inputString scans a string.
+type inputString struct {
+ str string
+}
+
+func newInputString(str string) *inputString {
+ return &inputString{str: str}
+}
+
+func (i *inputString) step(pos int) (int, int) {
+ if pos < len(i.str) {
+ return utf8.DecodeRuneInString(i.str[pos:len(i.str)])
+ }
+ return endOfText, 0
+}
+
+func (i *inputString) canCheckPrefix() bool {
+ return true
+}
+
+func (i *inputString) hasPrefix(re *Regexp) bool {
+ return strings.HasPrefix(i.str, re.prefix)
+}
+
+func (i *inputString) index(re *Regexp, pos int) int {
+ return strings.Index(i.str[pos:], re.prefix)
+}
+
+// inputBytes scans a byte slice.
+type inputBytes struct {
+ str []byte
+}
+
+func newInputBytes(str []byte) *inputBytes {
+ return &inputBytes{str: str}
+}
+
+func (i *inputBytes) step(pos int) (int, int) {
+ if pos < len(i.str) {
+ return utf8.DecodeRune(i.str[pos:len(i.str)])
+ }
+ return endOfText, 0
+}
+
+func (i *inputBytes) canCheckPrefix() bool {
+ return true
+}
+
+func (i *inputBytes) hasPrefix(re *Regexp) bool {
+ return bytes.HasPrefix(i.str, re.prefixBytes)
+}
+
+func (i *inputBytes) index(re *Regexp, pos int) int {
+ return bytes.Index(i.str[pos:], re.prefixBytes)
+}
+
+// inputReader scans a RuneReader.
+type inputReader struct {
+ r io.RuneReader
+ atEOT bool
+ pos int
+}
+
+func newInputReader(r io.RuneReader) *inputReader {
+ return &inputReader{r: r}
+}
+
+func (i *inputReader) step(pos int) (int, int) {
+ if !i.atEOT && pos != i.pos {
+ return endOfText, 0
+
+ }
+ r, w, err := i.r.ReadRune()
+ if err != nil {
+ i.atEOT = true
+ return endOfText, 0
+ }
+ i.pos += w
+ return r, w
+}
+
+func (i *inputReader) canCheckPrefix() bool {
+ return false
+}
+
+func (i *inputReader) hasPrefix(re *Regexp) bool {
+ return false
+}
+
+func (i *inputReader) index(re *Regexp, pos int) int {
+ return -1
+}
+
+// Search match starting from pos bytes into the input.
+func (re *Regexp) doExecute(i input, pos int) []int {
var s [2][]state
s[0] = make([]state, 0, 10)
s[1] = make([]state, 0, 10)
in, out := 0, 1
var final state
found := false
- end := len(str)
- if bytestr != nil {
- end = len(bytestr)
- }
anchored := re.inst[0].next.kind == iBOT
if anchored && pos > 0 {
return nil
}
// fast check for initial plain substring
- if re.prefix != "" {
+ if i.canCheckPrefix() && re.prefix != "" {
advance := 0
if anchored {
- if bytestr == nil {
- if !strings.HasPrefix(str, re.prefix) {
- return nil
- }
- } else {
- if !bytes.HasPrefix(bytestr, re.prefixBytes) {
- return nil
- }
+ if !i.hasPrefix(re) {
+ return nil
}
} else {
- if bytestr == nil {
- advance = strings.Index(str[pos:], re.prefix)
- } else {
- advance = bytes.Index(bytestr[pos:], re.prefixBytes)
+ advance = i.index(re, pos)
+ if advance == -1 {
+ return nil
}
}
- if advance == -1 {
- return nil
- }
pos += advance
}
- arena := &matchArena{nil, 2 * (re.nbra + 1)}
- for startPos := pos; pos <= end; {
+ // We look one character ahead so we can match $, which checks whether
+ // we are at EOT.
+ nextChar, nextWidth := i.step(pos)
+ arena := &matchArena{
+ len: 2 * (re.nbra + 1),
+ pos: pos,
+ atBOT: pos == 0,
+ atEOT: nextChar == endOfText,
+ }
+ for c, startPos := 0, pos; c != endOfText; {
if !found && (pos == startPos || !anchored) {
// prime the pump if we haven't seen a match yet
match := arena.noMatch()
match.m[0] = pos
- s[out] = arena.addState(s[out], re.start.next, false, match, pos, end)
+ s[out] = arena.addState(s[out], re.start.next, false, match)
arena.free(match) // if addState saved it, ref was incremented
} else if len(s[out]) == 0 {
// machine has completed
arena.free(state.match)
}
s[out] = old[0:0] // truncate state vector
- charwidth := 1
- c := endOfFile
- if pos < end {
- if bytestr == nil {
- c, charwidth = utf8.DecodeRuneInString(str[pos:end])
- } else {
- c, charwidth = utf8.DecodeRune(bytestr[pos:end])
- }
- }
- pos += charwidth
+ c = nextChar
+ thisPos := pos
+ pos += nextWidth
+ nextChar, nextWidth = i.step(pos)
+ arena.atEOT = nextChar == endOfText
+ arena.atBOT = false
+ arena.pos = pos
for _, st := range s[in] {
switch st.inst.kind {
case iBOT:
case iEOT:
case iChar:
if c == st.inst.char {
- s[out] = arena.addState(s[out], st.inst.next, st.prefixed, st.match, pos, end)
+ s[out] = arena.addState(s[out], st.inst.next, st.prefixed, st.match)
}
case iCharClass:
if st.inst.cclass.matches(c) {
- s[out] = arena.addState(s[out], st.inst.next, st.prefixed, st.match, pos, end)
+ s[out] = arena.addState(s[out], st.inst.next, st.prefixed, st.match)
}
case iAny:
- if c != endOfFile {
- s[out] = arena.addState(s[out], st.inst.next, st.prefixed, st.match, pos, end)
+ if c != endOfText {
+ s[out] = arena.addState(s[out], st.inst.next, st.prefixed, st.match)
}
case iNotNL:
- if c != endOfFile && c != '\n' {
- s[out] = arena.addState(s[out], st.inst.next, st.prefixed, st.match, pos, end)
+ if c != endOfText && c != '\n' {
+ s[out] = arena.addState(s[out], st.inst.next, st.prefixed, st.match)
}
case iBra:
case iAlt:
// choose leftmost longest
if !found || // first
st.match.m[0] < final.match.m[0] || // leftmost
- (st.match.m[0] == final.match.m[0] && pos-charwidth > final.match.m[1]) { // longest
+ (st.match.m[0] == final.match.m[0] && thisPos > final.match.m[1]) { // longest
if final.match != nil {
arena.free(final.match)
}
final = st
final.match.ref++
- final.match.m[1] = pos - charwidth
+ final.match.m[1] = thisPos
}
found = true
default:
return string(c[:i]), true
}
+// MatchReader returns whether the Regexp matches the text read by the
+// RuneReader. The return value is a boolean: true for match, false for no
+// match.
+func (re *Regexp) MatchReader(r io.RuneReader) bool {
+ return len(re.doExecute(newInputReader(r), 0)) > 0
+}
+
// MatchString returns whether the Regexp matches the string s.
// The return value is a boolean: true for match, false for no match.
-func (re *Regexp) MatchString(s string) bool { return len(re.doExecute(s, nil, 0)) > 0 }
+func (re *Regexp) MatchString(s string) bool { return len(re.doExecute(newInputString(s), 0)) > 0 }
// Match returns whether the Regexp matches the byte slice b.
// The return value is a boolean: true for match, false for no match.
-func (re *Regexp) Match(b []byte) bool { return len(re.doExecute("", b, 0)) > 0 }
+func (re *Regexp) Match(b []byte) bool { return len(re.doExecute(newInputBytes(b), 0)) > 0 }
+// MatchReader checks whether a textual regular expression matches the text
+// read by the RuneReader. More complicated queries need to use Compile and
+// the full Regexp interface.
+func MatchReader(pattern string, r io.RuneReader) (matched bool, error os.Error) {
+ re, err := Compile(pattern)
+ if err != nil {
+ return false, err
+ }
+ return re.MatchReader(r), nil
+}
// MatchString checks whether a textual regular expression
// matches a string. More complicated queries need
searchPos := 0 // position where we next look for a match
buf := new(bytes.Buffer)
for searchPos <= len(src) {
- a := re.doExecute(src, nil, searchPos)
+ a := re.doExecute(newInputString(src), searchPos)
if len(a) == 0 {
break // no more matches
}
searchPos := 0 // position where we next look for a match
buf := new(bytes.Buffer)
for searchPos <= len(src) {
- a := re.doExecute("", src, searchPos)
+ a := re.doExecute(newInputBytes(src), searchPos)
if len(a) == 0 {
break // no more matches
}
}
for pos, i, prevMatchEnd := 0, 0, -1; i < n && pos <= end; {
- matches := re.doExecute(s, b, pos)
+ var in input
+ if b == nil {
+ in = newInputString(s)
+ } else {
+ in = newInputBytes(b)
+ }
+ matches := re.doExecute(in, pos)
if len(matches) == 0 {
break
}
accept = false
}
var width int
+ // TODO: use step()
if b == nil {
_, width = utf8.DecodeRuneInString(s[pos:end])
} else {
// Find returns a slice holding the text of the leftmost match in b of the regular expression.
// A return value of nil indicates no match.
func (re *Regexp) Find(b []byte) []byte {
- a := re.doExecute("", b, 0)
+ a := re.doExecute(newInputBytes(b), 0)
if a == nil {
return nil
}
// b[loc[0]:loc[1]].
// A return value of nil indicates no match.
func (re *Regexp) FindIndex(b []byte) (loc []int) {
- a := re.doExecute("", b, 0)
+ a := re.doExecute(newInputBytes(b), 0)
if a == nil {
return nil
}
// an empty string. Use FindStringIndex or FindStringSubmatch if it is
// necessary to distinguish these cases.
func (re *Regexp) FindString(s string) string {
- a := re.doExecute(s, nil, 0)
+ a := re.doExecute(newInputString(s), 0)
if a == nil {
return ""
}
// itself is at s[loc[0]:loc[1]].
// A return value of nil indicates no match.
func (re *Regexp) FindStringIndex(s string) []int {
- a := re.doExecute(s, nil, 0)
+ a := re.doExecute(newInputString(s), 0)
+ if a == nil {
+ return nil
+ }
+ return a[0:2]
+}
+
+// FindReaderIndex returns a two-element slice of integers defining the
+// location of the leftmost match of the regular expression in text read from
+// the RuneReader. The match itself is at s[loc[0]:loc[1]]. A return
+// value of nil indicates no match.
+func (re *Regexp) FindReaderIndex(r io.RuneReader) []int {
+ a := re.doExecute(newInputReader(r), 0)
if a == nil {
return nil
}
// comment.
// A return value of nil indicates no match.
func (re *Regexp) FindSubmatch(b []byte) [][]byte {
- a := re.doExecute("", b, 0)
+ a := re.doExecute(newInputBytes(b), 0)
if a == nil {
return nil
}
// in the package comment.
// A return value of nil indicates no match.
func (re *Regexp) FindSubmatchIndex(b []byte) []int {
- return re.doExecute("", b, 0)
+ return re.doExecute(newInputBytes(b), 0)
}
// FindStringSubmatch returns a slice of strings holding the text of the
// package comment.
// A return value of nil indicates no match.
func (re *Regexp) FindStringSubmatch(s string) []string {
- a := re.doExecute(s, nil, 0)
+ a := re.doExecute(newInputString(s), 0)
if a == nil {
return nil
}
// 'Index' descriptions in the package comment.
// A return value of nil indicates no match.
func (re *Regexp) FindStringSubmatchIndex(s string) []int {
- return re.doExecute(s, nil, 0)
+ return re.doExecute(newInputString(s), 0)
+}
+
+// FindReaderSubmatchIndex returns a slice holding the index pairs
+// identifying the leftmost match of the regular expression of text read by
+// the RuneReader, and the matches, if any, of its subexpressions, as defined
+// by the 'Submatch' and 'Index' descriptions in the package comment. A
+// return value of nil indicates no match.
+func (re *Regexp) FindReaderSubmatchIndex(r io.RuneReader) []int {
+ return re.doExecute(newInputReader(r), 0)
}
const startSize = 10 // The size at which to start a slice in the 'All' routines.
"sync"
)
+// ServerError represents an error that has been returned from
+// the remote side of the RPC connection.
+type ServerError string
+
+func (e ServerError) String() string {
+ return string(e)
+}
+
+const ErrShutdown = os.ErrorString("connection is shut down")
+
// Call represents an active RPC.
type Call struct {
ServiceMethod string // The name of the service and method to call.
// with a single Client.
type Client struct {
mutex sync.Mutex // protects pending, seq
- shutdown os.Error // non-nil if the client is shut down
sending sync.Mutex
seq uint64
codec ClientCodec
pending map[uint64]*Call
closing bool
+ shutdown bool
}
// A ClientCodec implements writing of RPC requests and
// The client calls WriteRequest to write a request to the connection
// and calls ReadResponseHeader and ReadResponseBody in pairs
// to read responses. The client calls Close when finished with the
-// connection.
+// connection. ReadResponseBody may be called with a nil
+// argument to force the body of the response to be read and then
+// discarded.
type ClientCodec interface {
WriteRequest(*Request, interface{}) os.Error
ReadResponseHeader(*Response) os.Error
func (client *Client) send(c *Call) {
// Register this call.
client.mutex.Lock()
- if client.shutdown != nil {
- c.Error = client.shutdown
+ if client.shutdown {
+ c.Error = ErrShutdown
client.mutex.Unlock()
- _ = c.Done <- c // do not block
+ c.done()
return
}
c.seq = client.seq
c := client.pending[seq]
client.pending[seq] = c, false
client.mutex.Unlock()
- err = client.codec.ReadResponseBody(c.Reply)
- if response.Error != "" {
- c.Error = os.ErrorString(response.Error)
- } else if err != nil {
- c.Error = err
+
+ if response.Error == "" {
+ err = client.codec.ReadResponseBody(c.Reply)
+ if err != nil {
+ c.Error = os.ErrorString("reading body " + err.String())
+ }
} else {
- // Empty strings should turn into nil os.Errors
- c.Error = nil
+ // We've got an error response. Give this to the request;
+ // any subsequent requests will get the ReadResponseBody
+ // error if there is one.
+ c.Error = ServerError(response.Error)
+ err = client.codec.ReadResponseBody(nil)
+ if err != nil {
+ err = os.ErrorString("reading error body: " + err.String())
+ }
}
- // We don't want to block here. It is the caller's responsibility to make
- // sure the channel has enough buffer space. See comment in Go().
- _ = c.Done <- c // do not block
+ c.done()
}
// Terminate pending calls.
client.mutex.Lock()
- client.shutdown = err
+ client.shutdown = true
for _, call := range client.pending {
call.Error = err
- _ = call.Done <- call // do not block
+ call.done()
}
client.mutex.Unlock()
if err != os.EOF || !client.closing {
}
}
+func (call *Call) done() {
+ select {
+ case call.Done <- call:
+ // ok
+ default:
+ // We don't want to block here. It is the caller's responsibility to make
+ // sure the channel has enough buffer space. See comment in Go().
+ }
+}
+
// NewClient returns a new Client to handle requests to the
// set of services at the other end of the connection.
func NewClient(conn io.ReadWriteCloser) *Client {
}
func (client *Client) Close() os.Error {
- if client.shutdown != nil || client.closing {
- return os.ErrorString("rpc: already closed")
- }
client.mutex.Lock()
+ if client.shutdown || client.closing {
+ client.mutex.Unlock()
+ return ErrShutdown
+ }
client.closing = true
client.mutex.Unlock()
return client.codec.Close()
}
}
c.Done = done
- if client.shutdown != nil {
- c.Error = client.shutdown
- _ = c.Done <- c // do not block
+ if client.shutdown {
+ c.Error = ErrShutdown
+ c.done()
return c
}
client.send(c)
// Call invokes the named function, waits for it to complete, and returns its error status.
func (client *Client) Call(serviceMethod string, args interface{}, reply interface{}) os.Error {
- if client.shutdown != nil {
- return client.shutdown
+ if client.shutdown {
+ return ErrShutdown
}
call := <-client.Go(serviceMethod, args, reply, nil).Done
return call.Error
}
server.Unlock()
sort.Sort(services)
- err := debug.Execute(services, w)
+ err := debug.Execute(w, services)
if err != nil {
fmt.Fprintln(w, "rpc: error executing template:", err.String())
}
}
func (c *clientCodec) ReadResponseBody(x interface{}) os.Error {
+ if x == nil {
+ return nil
+ }
return json.Unmarshal(*c.resp.Result, x)
}
}
func (c *serverCodec) ReadRequestBody(x interface{}) os.Error {
+ if x == nil {
+ return nil
+ }
// JSON params is array value.
// RPC params is struct.
// Unmarshal into array containing struct for now.
rpc.HandleHTTP()
l, e := net.Listen("tcp", ":1234")
if e != nil {
- log.Exit("listen error:", e)
+ log.Fatal("listen error:", e)
}
go http.Serve(l, nil)
client, err := rpc.DialHTTP("tcp", serverAddress + ":1234")
if err != nil {
- log.Exit("dialing:", err)
+ log.Fatal("dialing:", err)
}
Then it can make a remote call:
var reply int
err = client.Call("Arith.Multiply", args, &reply)
if err != nil {
- log.Exit("arith error:", err)
+ log.Fatal("arith error:", err)
}
fmt.Printf("Arith: %d*%d=%d", args.A, args.B, *reply)
sname = name
}
if sname == "" {
- log.Exit("rpc: no service name for type", s.typ.String())
+ log.Fatal("rpc: no service name for type", s.typ.String())
}
if s.typ.PkgPath() != "" && !isExported(sname) && !useName {
s := "rpc Register: type " + sname + " is not exported"
// A value sent as a placeholder for the response when the server receives an invalid request.
type InvalidRequest struct {
- marker int
+ Marker int
}
-var invalidRequest = InvalidRequest{1}
+var invalidRequest = InvalidRequest{}
func _new(t *reflect.PtrType) *reflect.PtrValue {
v := reflect.MakeZero(t).(*reflect.PtrValue)
resp.ServiceMethod = req.ServiceMethod
if errmsg != "" {
resp.Error = errmsg
+ reply = invalidRequest
}
resp.Seq = req.Seq
sending.Lock()
func (server *Server) ServeCodec(codec ServerCodec) {
sending := new(sync.Mutex)
for {
- // Grab the request header.
- req := new(Request)
- err := codec.ReadRequestHeader(req)
+ req, service, mtype, err := server.readRequest(codec)
if err != nil {
+ if err != os.EOF {
+ log.Println("rpc:", err)
+ }
if err == os.EOF || err == io.ErrUnexpectedEOF {
- if err == io.ErrUnexpectedEOF {
- log.Println("rpc:", err)
- }
break
}
- s := "rpc: server cannot decode request: " + err.String()
- sendResponse(sending, req, invalidRequest, codec, s)
- break
- }
- serviceMethod := strings.Split(req.ServiceMethod, ".", -1)
- if len(serviceMethod) != 2 {
- s := "rpc: service/method request ill-formed: " + req.ServiceMethod
- sendResponse(sending, req, invalidRequest, codec, s)
- continue
- }
- // Look up the request.
- server.Lock()
- service, ok := server.serviceMap[serviceMethod[0]]
- server.Unlock()
- if !ok {
- s := "rpc: can't find service " + req.ServiceMethod
- sendResponse(sending, req, invalidRequest, codec, s)
- continue
- }
- mtype, ok := service.method[serviceMethod[1]]
- if !ok {
- s := "rpc: can't find method " + req.ServiceMethod
- sendResponse(sending, req, invalidRequest, codec, s)
+ // discard body
+ codec.ReadRequestBody(nil)
+
+ // send a response if we actually managed to read a header.
+ if req != nil {
+ sendResponse(sending, req, invalidRequest, codec, err.String())
+ }
continue
}
+
// Decode the argument value.
argv := _new(mtype.ArgType)
replyv := _new(mtype.ReplyType)
err = codec.ReadRequestBody(argv.Interface())
if err != nil {
- log.Println("rpc: tearing down", serviceMethod[0], "connection:", err)
+ if err == os.EOF || err == io.ErrUnexpectedEOF {
+ if err == io.ErrUnexpectedEOF {
+ log.Println("rpc:", err)
+ }
+ break
+ }
sendResponse(sending, req, replyv.Interface(), codec, err.String())
- break
+ continue
}
go service.call(sending, mtype, req, argv, replyv, codec)
}
codec.Close()
}
+func (server *Server) readRequest(codec ServerCodec) (req *Request, service *service, mtype *methodType, err os.Error) {
+ // Grab the request header.
+ req = new(Request)
+ err = codec.ReadRequestHeader(req)
+ if err != nil {
+ req = nil
+ if err == os.EOF || err == io.ErrUnexpectedEOF {
+ return
+ }
+ err = os.ErrorString("rpc: server cannot decode request: " + err.String())
+ return
+ }
+
+ serviceMethod := strings.Split(req.ServiceMethod, ".", -1)
+ if len(serviceMethod) != 2 {
+ err = os.ErrorString("rpc: service/method request ill-formed: " + req.ServiceMethod)
+ return
+ }
+ // Look up the request.
+ server.Lock()
+ service = server.serviceMap[serviceMethod[0]]
+ server.Unlock()
+ if service == nil {
+ err = os.ErrorString("rpc: can't find service " + req.ServiceMethod)
+ return
+ }
+ mtype = service.method[serviceMethod[1]]
+ if mtype == nil {
+ err = os.ErrorString("rpc: can't find method " + req.ServiceMethod)
+ }
+ return
+}
// Accept accepts connections on the listener and serves requests
// for each incoming connection. Accept blocks; the caller typically
for {
conn, err := lis.Accept()
if err != nil {
- log.Exit("rpc.Serve: accept:", err.String()) // TODO(r): exit?
+ log.Fatal("rpc.Serve: accept:", err.String()) // TODO(r): exit?
}
go server.ServeConn(conn)
}
// The server calls ReadRequestHeader and ReadRequestBody in pairs
// to read requests from the connection, and it calls WriteResponse to
// write a response back. The server calls Close when finished with the
-// connection.
+// connection. ReadRequestBody may be called with a nil
+// argument to force the body of the request to be read and discarded.
type ServerCodec interface {
ReadRequestHeader(*Request) os.Error
ReadRequestBody(interface{}) os.Error
func listenTCP() (net.Listener, string) {
l, e := net.Listen("tcp", "127.0.0.1:0") // any available address
if e != nil {
- log.Exitf("net.Listen tcp :0: %v", e)
+ log.Fatalf("net.Listen tcp :0: %v", e)
}
return l, l.Addr().String()
}
t.Errorf("Add: expected %d got %d", reply.C, args.A+args.B)
}
- args = &Args{7, 8}
+ // Nonexistent method
+ args = &Args{7, 0}
reply = new(Reply)
- err = client.Call("Arith.Mul", args, reply)
- if err != nil {
- t.Errorf("Mul: expected no error but got string %q", err.String())
+ err = client.Call("Arith.BadOperation", args, reply)
+ // expect an error
+ if err == nil {
+ t.Error("BadOperation: expected error")
+ } else if !strings.HasPrefix(err.String(), "rpc: can't find method ") {
+ t.Errorf("BadOperation: expected can't find method error; got %q", err)
}
- if reply.C != args.A*args.B {
- t.Errorf("Mul: expected %d got %d", reply.C, args.A*args.B)
+
+ // Unknown service
+ args = &Args{7, 8}
+ reply = new(Reply)
+ err = client.Call("Arith.Unknown", args, reply)
+ if err == nil {
+ t.Error("expected error calling unknown service")
+ } else if strings.Index(err.String(), "method") < 0 {
+ t.Error("expected error about method; got", err)
}
// Out of order.
t.Error("Div: expected divide by zero error; got", err)
}
+ // Bad type.
+ reply = new(Reply)
+ err = client.Call("Arith.Add", reply, reply) // args, reply would be the correct thing to use
+ if err == nil {
+ t.Error("expected error calling Arith.Add with wrong arg type")
+ } else if strings.Index(err.String(), "type") < 0 {
+ t.Error("expected error about type; got", err)
+ }
+
// Non-struct argument
const Val = 12345
str := fmt.Sprint(Val)
if str != expect {
t.Errorf("String: expected %s got %s", expect, str)
}
+
+ args = &Args{7, 8}
+ reply = new(Reply)
+ err = client.Call("Arith.Mul", args, reply)
+ if err != nil {
+ t.Errorf("Mul: expected no error but got string %q", err.String())
+ }
+ if reply.C != args.A*args.B {
+ t.Errorf("Mul: expected %d got %d", reply.C, args.A*args.B)
+ }
}
-func TestHTTPRPC(t *testing.T) {
+func TestHTTP(t *testing.T) {
once.Do(startServer)
testHTTPRPC(t, "")
newOnce.Do(startNewServer)
}
}
-func TestCheckUnknownService(t *testing.T) {
- once.Do(startServer)
-
- conn, err := net.Dial("tcp", "", serverAddr)
- if err != nil {
- t.Fatal("dialing:", err)
- }
-
- client := NewClient(conn)
-
- args := &Args{7, 8}
- reply := new(Reply)
- err = client.Call("Unknown.Add", args, reply)
- if err == nil {
- t.Error("expected error calling unknown service")
- } else if strings.Index(err.String(), "service") < 0 {
- t.Error("expected error about service; got", err)
- }
-}
-
-func TestCheckUnknownMethod(t *testing.T) {
- once.Do(startServer)
-
- conn, err := net.Dial("tcp", "", serverAddr)
- if err != nil {
- t.Fatal("dialing:", err)
- }
-
- client := NewClient(conn)
-
- args := &Args{7, 8}
- reply := new(Reply)
- err = client.Call("Arith.Unknown", args, reply)
- if err == nil {
- t.Error("expected error calling unknown service")
- } else if strings.Index(err.String(), "method") < 0 {
- t.Error("expected error about method; got", err)
- }
-}
-
-func TestCheckBadType(t *testing.T) {
- once.Do(startServer)
-
- conn, err := net.Dial("tcp", "", serverAddr)
- if err != nil {
- t.Fatal("dialing:", err)
- }
-
- client := NewClient(conn)
-
- reply := new(Reply)
- err = client.Call("Arith.Add", reply, reply) // args, reply would be the correct thing to use
- if err == nil {
- t.Error("expected error calling Arith.Add with wrong arg type")
- } else if strings.Index(err.String(), "type") < 0 {
- t.Error("expected error about type; got", err)
- }
-}
-
type ArgNotPointer int
type ReplyNotPointer int
type ArgNotPublic int
testSendDeadlock(client)
done <- true
}()
- for i := 0; i < 50; i++ {
- time.Sleep(100 * 1e6)
- _, ok := <-done
- if ok {
- return
- }
+ select {
+ case <-done:
+ return
+ case <-time.After(5e9):
+ t.Fatal("deadlock")
}
- t.Fatal("deadlock")
}
func testSendDeadlock(client *Client) {
+++ /dev/null
-// Copyright 2010 The Go Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-// Go definitions of internal structures. Master is chan.c
-
-package runtime
-
-type sudoG struct {
- g *g_
- selgen uint32
- offset int16
- isfree int8
- link *sudoG
- elem [8]byte
-}
-
-type waitQ struct {
- first *sudoG
- last *sudoG
-}
-
-type hChan struct {
- qcount uint32
- dataqsiz uint32
- elemsize uint16
- closed uint16
- elemalign uint8
- elemalg *alg
- senddataq *link
- recvdataq *link
- recvq waitQ
- sendq waitQ
- free sudoG
- lock
-}
-
-type link struct {
- link *link
- elem [8]byte
-}
-
-type scase struct {
- chan_ *hChan
- pc *byte
- send uint16
- so uint16
- elemp *byte // union elem [8]byte
-}
-
-type select_ struct {
- tcase uint16
- ncase uint16
- link *select_
- scase [1]*scase
-}
MSpanSys uint64
MCacheInuse uint64 // mcache structures
MCacheSys uint64
- MHeapMapSys uint64 // heap map
BuckHashSys uint64 // profiling bucket hash table
// Garbage collector statistics.
// Per-size allocation statistics.
// Not locked during update; approximate.
- BySize [67]struct {
+ // 61 is NumSizeClasses in the C code.
+ BySize [61]struct {
Size uint32
Mallocs uint64
Frees uint64
// It returns the number of entries written to pc.
func Callers(skip int, pc []uintptr) int
+type Func struct { // Keep in sync with runtime.h:struct Func
+ name string
+ typ string // go type string
+ src string // src file name
+ pcln []byte // pc/ln tab for this func
+ entry uintptr // entry pc
+ pc0 uintptr // starting pc, ln for table
+ ln0 int32
+ frame int32 // stack frame size
+ args int32 // number of 32-bit in/out args
+ locals int32 // number of 32-bit locals
+}
+
// FuncForPC returns a *Func describing the function that contains the
// given program counter address, or else nil.
func FuncForPC(pc uintptr) *Func
// counter within f.
func (f *Func) FileLine(pc uintptr) (file string, line int) {
// NOTE(rsc): If you edit this function, also edit
- // symtab.c:/^funcline.
+ // symtab.c:/^funcline. That function also has the
+ // comments explaining the logic.
+ targetpc := pc
+
var pcQuant uintptr = 1
if GOARCH == "arm" {
pcQuant = 4
}
- targetpc := pc
p := f.pcln
pc = f.pc0
line = int(f.ln0)
- file = f.src
- for i := 0; i < len(p) && pc <= targetpc; i++ {
- switch {
- case p[i] == 0:
+ i := 0
+ //print("FileLine start pc=", pc, " targetpc=", targetpc, " line=", line,
+ // " tab=", p, " ", p[0], " quant=", pcQuant, " GOARCH=", GOARCH, "\n")
+ for {
+ for i < len(p) && p[i] > 128 {
+ pc += pcQuant * uintptr(p[i]-128)
+ i++
+ }
+ //print("pc<", pc, " targetpc=", targetpc, " line=", line, "\n")
+ if pc > targetpc || i >= len(p) {
+ break
+ }
+ if p[i] == 0 {
+ if i+5 > len(p) {
+ break
+ }
line += int(p[i+1]<<24) | int(p[i+2]<<16) | int(p[i+3]<<8) | int(p[i+4])
- i += 4
- case p[i] <= 64:
+ i += 5
+ } else if p[i] <= 64 {
line += int(p[i])
- case p[i] <= 128:
+ i++
+ } else {
line -= int(p[i] - 64)
- default:
- pc += pcQuant * uintptr(p[i]-129)
+ i++
}
+ //print("pc=", pc, " targetpc=", targetpc, " line=", line, "\n")
pc += pcQuant
}
+ file = f.src
return
}
+++ /dev/null
-// Copyright 2010 The Go Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-// Go definitions of internal structures. Master is hashmap.[c,h]
-
-package runtime
-
-type hash_hash uintptr
-
-type hash_entry struct {
- hash hash_hash
- key byte // dwarf.c substitutes the real type
- val byte // for key and val
-}
-
-type hash_subtable struct {
- power uint8
- used uint8
- datasize uint8
- max_probes uint8
- limit_bytes int16
- end *hash_entry
- entry hash_entry // TODO: [0]hash_entry
-}
-
-type hash struct {
- count uint32
- datasize uint8
- max_power uint8
- max_probes uint8
- indirectval uint8
- changes int32
- data_hash func(uint32, uintptr) hash_hash
- data_eq func(uint32, uintptr, uintptr) uint32
- data_del func(uint32, uintptr, uintptr)
- st *hash_subtable
- keysize uint32
- valsize uint32
- datavo uint32
- ko0 uint32
- vo0 uint32
- ko1 uint32
- vo1 uint32
- po1 uint32
- ko2 uint32
- vo2 uint32
- po2 uint32
- keyalg *alg
- valalg *alg
-}
+++ /dev/null
-// Copyright 2010 The Go Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-package runtime
-
-/*
- * Must match iface.c:/Itable and compilers.
- * NOTE: type.go has an Itable, that is the version of Itab used by the reflection code.
- */
-type itab struct {
- Itype *Type
- Type *Type
- link *itab
- bad int32
- unused int32
- Fn func() // TODO: [0]func()
-}
+++ /dev/null
-// Copyright 2010 The Go Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-// Go definitions of internal structures. Master is malloc.h
-
-package runtime
-
-import "unsafe"
-
-const (
- pageShift = 12
- pageSize = 1 << pageShift
- pageMask = pageSize - 1
-)
-
-type pageID uintptr
-
-const (
- numSizeClasses = 67
- maxSmallSize = 32 << 10
- fixAllocChunk = 128 << 10
- maxMCacheListLen = 256
- maxMCacheSize = 2 << 20
- maxMHeapList = 1 << 8 // 1 << (20 - pageShift)
- heapAllocChunk = 1 << 20
-)
-
-type mLink struct {
- next *mLink
-}
-
-type fixAlloc struct {
- size uintptr
- alloc func(uintptr)
- first func(unsafe.Pointer, *byte)
- arg unsafe.Pointer
- list *mLink
- chunk *byte
- nchunk uint32
- inuse uintptr
- sys uintptr
-}
-
-
-// MStats? used to be in extern.go
-
-type mCacheList struct {
- list *mLink
- nlist uint32
- nlistmin uint32
-}
-
-type mCache struct {
- list [numSizeClasses]mCacheList
- size uint64
- local_alloc int64
- local_objects int64
- next_sample int32
-}
-
-type mSpan struct {
- next *mSpan
- prev *mSpan
- allnext *mSpan
- start pageID
- npages uintptr
- freelist *mLink
- ref uint32
- sizeclass uint32
- state uint32
- // union {
- gcref *uint32 // sizeclass > 0
- // gcref0 uint32; // sizeclass == 0
- // }
-}
-
-type mCentral struct {
- // lock
- sizeclass int32
- nonempty mSpan
- empty mSpan
- nfree int32
-}
-
-type mHeap struct {
- // lock
- free [maxMHeapList]mSpan
- large mSpan
- allspans *mSpan
- // map_ mHeapMap
- min *byte
- max *byte
- closure_min *byte
- closure_max *byte
-
- central [numSizeClasses]struct {
- pad [64]byte
- // union: mCentral
- }
-
- spanalloc fixAlloc
- cachealloc fixAlloc
-}
-
-const (
- refFree = iota
- refStack
- refNone
- refSome
- refcountOverhead = 4
- refNoPointers = 0x80000000
- refHasFinalizer = 0x40000000
- refProfiled = 0x20000000
- refNoProfiling = 0x10000000
- refFlags = 0xFFFF0000
-)
-
-const (
- mProf_None = iota
- mProf_Sample
- mProf_All
-)
-
-type finalizer struct {
- next *finalizer
- fn func(unsafe.Pointer)
- arg unsafe.Pointer
- nret int32
-}
+++ /dev/null
-// Copyright 2010 The Go Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-package runtime
-
-const (
- mHeapMap_Level1Bits = 10
- mHeapMap_Level2Bits = 10
- mHeapMap_TotalBits = mHeapMap_Level1Bits + mHeapMap_Level2Bits
-
- mHeapMap_Level1Mask = (1 << mHeapMap_Level1Bits) - 1
- mHeapMap_Level2Mask = (1 << mHeapMap_Level2Bits) - 1
-)
-
-type mHeapMap struct {
- allocator func(uintptr)
- p [1 << mHeapMap_Level1Bits]*mHeapMapNode2
-}
-
-type mHeapMapNode2 struct {
- s [1 << mHeapMap_Level2Bits]*mSpan
-}
+++ /dev/null
-// Copyright 2010 The Go Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-package runtime
-
-const (
- mHeapMap_Level1Bits = 18
- mHeapMap_Level2Bits = 18
- mHeapMap_Level3Bits = 16
- mHeapMap_TotalBits = mHeapMap_Level1Bits + mHeapMap_Level2Bits + mHeapMap_Level3Bits
-
- mHeapMap_Level1Mask = (1 << mHeapMap_Level1Bits) - 1
- mHeapMap_Level2Mask = (1 << mHeapMap_Level2Bits) - 1
- mHeapMap_Level3Mask = (1 << mHeapMap_Level3Bits) - 1
-)
-
-type mHeapMap struct {
- allocator func(uintptr)
- p [1 << mHeapMap_Level1Bits]*mHeapMapNode2
-}
-
-
-type mHeapMapNode2 struct {
- p [1 << mHeapMap_Level2Bits]*mHeapMapNode3
-}
-
-
-type mHeapMapNode3 struct {
- s [1 << mHeapMap_Level3Bits]*mSpan
-}
fmt.Fprintf(b, "# Stack = %d / %d\n", s.StackInuse, s.StackSys)
fmt.Fprintf(b, "# MSpan = %d / %d\n", s.MSpanInuse, s.MSpanSys)
fmt.Fprintf(b, "# MCache = %d / %d\n", s.MCacheInuse, s.MCacheSys)
- fmt.Fprintf(b, "# MHeapMapSys = %d\n", s.MHeapMapSys)
fmt.Fprintf(b, "# BuckHashSys = %d\n", s.BuckHashSys)
fmt.Fprintf(b, "# NextGC = %d\n", s.NextGC)
+++ /dev/null
-// Copyright 2010 The Go Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-// Go definitions of internal structures. Master is runtime.h
-
-// TODO(lvd): automate conversion to all the _defs.go files
-
-package runtime
-
-import "unsafe"
-
-const (
- gidle = iota
- grunnable
- grunning
- gsyscall
- gwaiting
- gmoribund
- gdead
- grecovery
-)
-
-// const ( Structrnd = sizeof(uintptr) )
-
-type string_ struct {
- str *byte
- len int32
-}
-
-type iface struct {
- // tab *itab
- data unsafe.Pointer
-}
-
-type eface struct {
- type_ *Type
- data unsafe.Pointer
-}
-
-type complex64 struct {
- real float32
- imag float32
-}
-
-type complex128 struct {
- real float64
- imag float64
-}
-
-type slice struct {
- array *byte
- len uint32
- cap uint32
-}
-
-type gobuf struct {
- sp unsafe.Pointer
- pc unsafe.Pointer
- g *g_
-}
-
-type g_ struct {
- stackguard unsafe.Pointer
- stackbase unsafe.Pointer
- defer_ *defer_
- panic_ *panic_
- sched gobuf
- stack0 unsafe.Pointer
- entry unsafe.Pointer
- alllink *g_
- param unsafe.Pointer
- status int16
- goid int32
- selgen uint32
- schedlink *g_
- readyonstop bool
- ispanic bool
- m *m_
- lockedm *m_
- sig int32
- sigcode0 uintptr
- sigcode1 uintptr
-}
-
-type m_ struct {
- g0 *g_
- morepc unsafe.Pointer
- moreargp unsafe.Pointer
- morebuf gobuf
- moreframesize uint32
- moreargsize uint32
- cret uintptr
- procid uint64
- gsignal *g_
- tls [8]uint32
- sched gobuf
- curg *g_
- id int32
- mallocing int32
- gcing int32
- locks int32
- nomemprof int32
- waitnextg int32
- // havenextg note
- nextg *g_
- alllink *m_
- schedlink *m_
- machport uint32
- mcache *mCache
- lockedg *g_
- freg [8]uint64
- // gostack unsafe.Pointer // __WINDOWS__
-}
-
-type stktop struct {
- stackguard *uint8
- stackbase *uint8
- gobuf gobuf
- args uint32
- fp *uint8
- free bool
- panic_ bool
-}
-
-type alg struct {
- hash func(uint32, unsafe.Pointer) uintptr
- equal func(uint32, unsafe.Pointer, unsafe.Pointer) uint32
- print func(uint32, unsafe.Pointer)
- copy func(uint32, unsafe.Pointer, unsafe.Pointer)
-}
-
-type sigtab struct {
- flags int32
- name *int8
-}
-
-const (
- sigCatch = (1 << iota)
- sigIgnore
- sigRestart
- sigQueue
- sigPanic
-)
-
-type Func struct {
- name string
- typ string
- src string
- pcln []byte
- entry uintptr
- pc0 uintptr
- ln0 int32
- frame int32
- args int32
- locals int32
-}
-
-const (
- aMEM = iota
- aNOEQ
- aSTRING
- aINTER
- aNILINTER
- aMEMWORD
- amax
-)
-
-type defer_ struct {
- siz int32
- sp unsafe.Pointer
- pc unsafe.Pointer
- fn unsafe.Pointer
- link *defer_
- args [8]byte // padded to actual size
-}
-
-type panic_ struct {
- arg eface
- stackbase unsafe.Pointer
- link *panic_
- recovered bool
-}
-
-/*
- * External data.
- */
-
-var (
- algarray [amax]alg
- emptystring string
- allg *g_
- allm *m_
- goidgen int32
- gomaxprocs int32
- panicking int32
- fd int32
- gcwaiting int32
- goos *int8
-)
* data structures and must be kept in sync with this file:
*
* ../../cmd/gc/reflect.c
- * ../../cmd/ld/dwarf.c
+ * ../../cmd/ld/dwarf.c decodetype_*
* ../reflect/type.go
* type.h
*/
string *string // string form; unnecessary but undeniably useful
*uncommonType // (relatively) uncommon fields
+ ptrToThis *Type // pointer to this type, if used in binary or has methods
}
// Values for commonType.kind.
type Type commonType
// Method on non-interface type
-type method struct {
+type _method struct { // underscore is to avoid collision with C
name *string // name of method
pkgPath *string // nil for exported Names; otherwise import path
mtyp *Type // method type (without receiver)
// Using a pointer to this struct reduces the overall size required
// to describe an unnamed type with no methods.
type uncommonType struct {
- name *string // name of type
- pkgPath *string // import path; nil for built-in types like int, string
- methods []method // methods associated with type
+ name *string // name of type
+ pkgPath *string // import path; nil for built-in types like int, string
+ methods []_method // methods associated with type
}
// BoolType represents a boolean type.
}
// Method on interface type
-type imethod struct {
+type _imethod struct { // underscore is to avoid collision with C
name *string // name of method
pkgPath *string // nil for exported Names; otherwise import path
typ *Type // .(*FuncType) underneath
// InterfaceType represents an interface type.
type InterfaceType struct {
commonType
- methods []imethod // sorted by hash
+ methods []_imethod // sorted by hash
}
// MapType represents a map type.
)
+// TODO(gri): Consider changing this to use the new (token) Position package.
+
// A source position is represented by a Position value.
// A position is valid if Line > 0.
type Position struct {
Filename string // filename, if any
Offset int // byte offset, starting at 0
Line int // line number, starting at 1
- Column int // column number, starting at 0 (character count per line)
+ Column int // column number, starting at 1 (character count per line)
}
// Source position
srcBufOffset int // byte offset of srcBuf[0] in source
- line int // newline count + 1
- column int // character count on line
+ line int // line count
+ column int // character count
+ lastLineLen int // length of last line in characters (for correct column reporting)
+ lastCharLen int // length of last character in bytes
// Token text buffer
// Typically, token text is stored completely in srcBuf, but in general
// the token text's head may be buffered in tokBuf while the token text's
// tail is stored in srcBuf.
tokBuf bytes.Buffer // token text head that is not in srcBuf anymore
- tokPos int // token text tail position (srcBuf index)
+ tokPos int // token text tail position (srcBuf index); valid if >= 0
tokEnd int // token text tail end (srcBuf index)
// One character look-ahead
}
-// Init initializes a Scanner with a new source and returns itself.
+// Init initializes a Scanner with a new source and returns s.
// Error is set to nil, ErrorCount is set to 0, Mode is set to GoTokens,
// and Whitespace is set to GoWhitespace.
func (s *Scanner) Init(src io.Reader) *Scanner {
s.src = src
// initialize source buffer
+ // (the first call to next() will fill it by calling src.Read)
s.srcBuf[0] = utf8.RuneSelf // sentinel
s.srcPos = 0
s.srcEnd = 0
s.srcBufOffset = 0
s.line = 1
s.column = 0
+ s.lastLineLen = 0
+ s.lastCharLen = 0
// initialize token text buffer
+ // (required for first call to next()).
s.tokPos = -1
// initialize one character look-ahead
- s.ch = s.next()
+ s.ch = -1 // no char read yet
// initialize public fields
s.Error = nil
}
+// TODO(gri): The code for next() and the internal scanner state could benefit
+// from a rethink. While next() is optimized for the common ASCII
+// case, the "corrections" needed for proper position tracking undo
+// some of the attempts for fast-path optimization.
+
// next reads and returns the next Unicode character. It is designed such
// that only a minimal amount of work needs to be done in the common ASCII
// case (one test to check for both ASCII and end-of-buffer, and one test
// to check for newlines).
func (s *Scanner) next() int {
- ch := int(s.srcBuf[s.srcPos])
+ ch, width := int(s.srcBuf[s.srcPos]), 1
if ch >= utf8.RuneSelf {
// uncommon case: not ASCII or not enough bytes
if s.tokPos >= 0 {
s.tokBuf.Write(s.srcBuf[s.tokPos:s.srcPos])
s.tokPos = 0
+ // s.tokEnd is set by Scan()
}
// move unread bytes to beginning of buffer
copy(s.srcBuf[0:], s.srcBuf[s.srcPos:s.srcEnd])
s.srcBufOffset += s.srcPos
// read more bytes
+ // (an io.Reader must return os.EOF when it reaches
+ // the end of what it is reading - simply returning
+ // n == 0 will make this loop retry forever; but the
+ // error is in the reader implementation in that case)
i := s.srcEnd - s.srcPos
n, err := s.src.Read(s.srcBuf[i:bufLen])
- s.srcEnd = i + n
s.srcPos = 0
+ s.srcEnd = i + n
s.srcBuf[s.srcEnd] = utf8.RuneSelf // sentinel
if err != nil {
if s.srcEnd == 0 {
+ if s.lastCharLen > 0 {
+ // previous character was not EOF
+ s.column++
+ }
+ s.lastCharLen = 0
return EOF
}
if err != os.EOF {
s.error(err.String())
- break
}
+ // If err == EOF, we won't be getting more
+ // bytes; break to avoid infinite loop. If
+ // err is something else, we don't know if
+ // we can get more bytes; thus also break.
+ break
}
}
// at least one byte
ch = int(s.srcBuf[s.srcPos])
if ch >= utf8.RuneSelf {
// uncommon case: not ASCII
- var width int
ch, width = utf8.DecodeRune(s.srcBuf[s.srcPos:s.srcEnd])
if ch == utf8.RuneError && width == 1 {
s.error("illegal UTF-8 encoding")
}
- s.srcPos += width - 1
}
}
- s.srcPos++
+ // advance
+ s.srcPos += width
+ s.lastCharLen = width
s.column++
+
+ // special situations
switch ch {
case 0:
// implementation restriction for compatibility with other tools
s.error("illegal character NUL")
case '\n':
s.line++
+ s.lastLineLen = s.column
s.column = 0
}
// Next reads and returns the next Unicode character.
// It returns EOF at the end of the source. It reports
-// a read error by calling s.Error, if set, or else
-// prints an error message to os.Stderr. Next does not
+// a read error by calling s.Error, if not nil; otherwise
+// it prints an error message to os.Stderr. Next does not
// update the Scanner's Position field; use Pos() to
// get the current position.
func (s *Scanner) Next() int {
s.tokPos = -1 // don't collect token text
- ch := s.ch
+ ch := s.Peek()
s.ch = s.next()
return ch
}
// the scanner. It returns EOF if the scanner's position is at the last
// character of the source.
func (s *Scanner) Peek() int {
+ if s.ch < 0 {
+ s.ch = s.next()
+ }
return s.ch
}
// Scan reads the next token or Unicode character from source and returns it.
// It only recognizes tokens t for which the respective Mode bit (1<<-t) is set.
// It returns EOF at the end of the source. It reports scanner errors (read and
-// token errors) by calling s.Error, if set; otherwise it prints an error message
-// to os.Stderr.
+// token errors) by calling s.Error, if not nil; otherwise it prints an error
+// message to os.Stderr.
func (s *Scanner) Scan() int {
- ch := s.ch
+ ch := s.Peek()
// reset token text position
s.tokPos = -1
// start collecting token text
s.tokBuf.Reset()
- s.tokPos = s.srcPos - 1
+ s.tokPos = s.srcPos - s.lastCharLen
// set token position
+ // (this is a slightly optimized version of the code in Pos())
s.Offset = s.srcBufOffset + s.tokPos
- s.Line = s.line
- s.Column = s.column
+ if s.column > 0 {
+ // common case: last character was not a '\n'
+ s.Line = s.line
+ s.Column = s.column
+ } else {
+ // last character was a '\n'
+ // (we cannot be at the beginning of the source
+ // since we have called next() at least once)
+ s.Line = s.line - 1
+ s.Column = s.lastLineLen
+ }
// determine token value
tok := ch
}
// end of token text
- s.tokEnd = s.srcPos - 1
+ s.tokEnd = s.srcPos - s.lastCharLen
s.ch = ch
return tok
}
-// Position returns the current source position. If called before Next()
-// or Scan(), it returns the position of the next Unicode character or token
-// returned by these functions. If called afterwards, it returns the position
-// immediately after the last character of the most recent token or character
-// scanned.
-func (s *Scanner) Pos() Position {
- return Position{
- s.Filename,
- s.srcBufOffset + s.srcPos - 1,
- s.line,
- s.column,
+// Pos returns the position of the character immediately after
+// the character or token returned by the last call to Next or Scan.
+func (s *Scanner) Pos() (pos Position) {
+ pos.Filename = s.Filename
+ pos.Offset = s.srcBufOffset + s.srcPos - s.lastCharLen
+ switch {
+ case s.column > 0:
+ // common case: last character was not a '\n'
+ pos.Line = s.line
+ pos.Column = s.column
+ case s.lastLineLen > 0:
+ // last character was a '\n'
+ pos.Line = s.line - 1
+ pos.Column = s.lastLineLen
+ default:
+ // at the beginning of the source
+ pos.Line = 1
+ pos.Column = 1
}
+ return
}
"os"
"strings"
"testing"
+ "utf8"
)
func testError(t *testing.T, src, msg string, tok int) {
s := new(Scanner).Init(bytes.NewBufferString(src))
errorCalled := false
- s.Error = func(s *Scanner, m string) {
+ s.Error = func(_ *Scanner, m string) {
if !errorCalled {
// only look at first error
if m != msg {
func TestError(t *testing.T) {
+ testError(t, "\x00", "illegal character NUL", 0)
+ testError(t, "\xff", "illegal UTF-8 encoding", utf8.RuneError)
testError(t, `01238`, "illegal octal number", Int)
testError(t, `'\"'`, "illegal char escape", Char)
testError(t, `'aa'`, "illegal char literal", Char)
}
-func checkPos(t *testing.T, s *Scanner, offset, line, column, char int) {
- pos := s.Pos()
- if pos.Offset != offset {
- t.Errorf("offset = %d, want %d", pos.Offset, offset)
+func checkPos(t *testing.T, got, want Position) {
+ if got.Offset != want.Offset || got.Line != want.Line || got.Column != want.Column {
+ t.Errorf("got offset, line, column = %d, %d, %d; want %d, %d, %d",
+ got.Offset, got.Line, got.Column, want.Offset, want.Line, want.Column)
}
- if pos.Line != line {
- t.Errorf("line = %d, want %d", pos.Line, line)
- }
- if pos.Column != column {
- t.Errorf("column = %d, want %d", pos.Column, column)
+}
+
+
+func checkNextPos(t *testing.T, s *Scanner, offset, line, column, char int) {
+ if ch := s.Next(); ch != char {
+ t.Errorf("ch = %s, want %s", TokenString(ch), TokenString(char))
}
- ch := s.Scan()
- if ch != char {
+ want := Position{Offset: offset, Line: line, Column: column}
+ checkPos(t, s.Pos(), want)
+}
+
+
+func checkScanPos(t *testing.T, s *Scanner, offset, line, column, char int) {
+ want := Position{Offset: offset, Line: line, Column: column}
+ checkPos(t, s.Pos(), want)
+ if ch := s.Scan(); ch != char {
t.Errorf("ch = %s, want %s", TokenString(ch), TokenString(char))
+ if string(ch) != s.TokenText() {
+ t.Errorf("tok = %q, want %q", s.TokenText(), string(ch))
+ }
}
+ checkPos(t, s.Position, want)
}
func TestPos(t *testing.T) {
- s := new(Scanner).Init(bytes.NewBufferString("abc\n012\n\nx"))
+ // corner case: empty source
+ s := new(Scanner).Init(bytes.NewBufferString(""))
+ checkPos(t, s.Pos(), Position{Offset: 0, Line: 1, Column: 1})
+ s.Peek() // peek doesn't affect the position
+ checkPos(t, s.Pos(), Position{Offset: 0, Line: 1, Column: 1})
+
+ // corner case: source with only a newline
+ s = new(Scanner).Init(bytes.NewBufferString("\n"))
+ checkPos(t, s.Pos(), Position{Offset: 0, Line: 1, Column: 1})
+ checkNextPos(t, s, 1, 2, 1, '\n')
+ // after EOF position doesn't change
+ for i := 10; i > 0; i-- {
+ checkScanPos(t, s, 1, 2, 1, EOF)
+ }
+
+ // corner case: source with only a single character
+ s = new(Scanner).Init(bytes.NewBufferString("本"))
+ checkPos(t, s.Pos(), Position{Offset: 0, Line: 1, Column: 1})
+ checkNextPos(t, s, 3, 1, 2, '本')
+ // after EOF position doesn't change
+ for i := 10; i > 0; i-- {
+ checkScanPos(t, s, 3, 1, 2, EOF)
+ }
+
+ // positions after calling Next
+ s = new(Scanner).Init(bytes.NewBufferString(" foo६४ \n\n本語\n"))
+ checkNextPos(t, s, 1, 1, 2, ' ')
+ s.Peek() // peek doesn't affect the position
+ checkNextPos(t, s, 2, 1, 3, ' ')
+ checkNextPos(t, s, 3, 1, 4, 'f')
+ checkNextPos(t, s, 4, 1, 5, 'o')
+ checkNextPos(t, s, 5, 1, 6, 'o')
+ checkNextPos(t, s, 8, 1, 7, '६')
+ checkNextPos(t, s, 11, 1, 8, '४')
+ checkNextPos(t, s, 12, 1, 9, ' ')
+ checkNextPos(t, s, 13, 1, 10, ' ')
+ checkNextPos(t, s, 14, 2, 1, '\n')
+ checkNextPos(t, s, 15, 3, 1, '\n')
+ checkNextPos(t, s, 18, 3, 2, '本')
+ checkNextPos(t, s, 21, 3, 3, '語')
+ checkNextPos(t, s, 22, 4, 1, '\n')
+ // after EOF position doesn't change
+ for i := 10; i > 0; i-- {
+ checkScanPos(t, s, 22, 4, 1, EOF)
+ }
+
+ // positions after calling Scan
+ s = new(Scanner).Init(bytes.NewBufferString("abc\n本語\n\nx"))
s.Mode = 0
s.Whitespace = 0
- checkPos(t, s, 0, 1, 1, 'a')
- checkPos(t, s, 1, 1, 2, 'b')
- checkPos(t, s, 2, 1, 3, 'c')
- checkPos(t, s, 3, 2, 0, '\n')
- checkPos(t, s, 4, 2, 1, '0')
- checkPos(t, s, 5, 2, 2, '1')
- checkPos(t, s, 6, 2, 3, '2')
- checkPos(t, s, 7, 3, 0, '\n')
- checkPos(t, s, 8, 4, 0, '\n')
- checkPos(t, s, 9, 4, 1, 'x')
- checkPos(t, s, 9, 4, 1, EOF)
- checkPos(t, s, 9, 4, 1, EOF) // after EOF, position doesn't change
+ checkScanPos(t, s, 0, 1, 1, 'a')
+ s.Peek() // peek doesn't affect the position
+ checkScanPos(t, s, 1, 1, 2, 'b')
+ checkScanPos(t, s, 2, 1, 3, 'c')
+ checkScanPos(t, s, 3, 1, 4, '\n')
+ checkScanPos(t, s, 4, 2, 1, '本')
+ checkScanPos(t, s, 7, 2, 2, '語')
+ checkScanPos(t, s, 10, 2, 3, '\n')
+ checkScanPos(t, s, 11, 3, 1, '\n')
+ checkScanPos(t, s, 12, 4, 1, 'x')
+ // after EOF position doesn't change
+ for i := 10; i > 0; i-- {
+ checkScanPos(t, s, 13, 4, 2, EOF)
+ }
}
{"1e", "0", os.EINVAL},
{"1e-", "0", os.EINVAL},
{".e-1", "0", os.EINVAL},
+
+ // http://www.exploringbinary.com/java-hangs-when-converting-2-2250738585072012e-308/
+ {"2.2250738585072012e-308", "2.2250738585072014e-308", nil},
+ // http://www.exploringbinary.com/php-hangs-on-numeric-value-2-2250738585072011e-308/
+ {"2.2250738585072011e-308", "2.225073858507201e-308", nil},
}
func init() {
}
func genericFtoa(bits uint64, fmt byte, prec int, flt *floatInfo) string {
- neg := bits>>flt.expbits>>flt.mantbits != 0
+ neg := bits>>(flt.expbits+flt.mantbits) != 0
exp := int(bits>>flt.mantbits) & (1<<flt.expbits - 1)
mant := bits & (uint64(1)<<flt.mantbits - 1)
{0.5, 'f', 1, "0.5"},
{0.5, 'f', 0, "0"},
{1.5, 'f', 0, "2"},
+
+ // http://www.exploringbinary.com/java-hangs-when-converting-2-2250738585072012e-308/
+ {2.2250738585072012e-308, 'g', -1, "2.2250738585072014e-308"},
+ // http://www.exploringbinary.com/php-hangs-on-numeric-value-2-2250738585072011e-308/
+ {2.2250738585072011e-308, 'g', -1, "2.225073858507201e-308"},
}
func TestFtoa(t *testing.T) {
--- /dev/null
+/* atomic.c -- implement atomic routines for Go.
+
+ Copyright 2011 The Go Authors. All rights reserved.
+ Use of this source code is governed by a BSD-style
+ license that can be found in the LICENSE file. */
+
+#include <stdint.h>
+
+_Bool CompareAndSwapInt32 (int32_t *, int32_t, int32_t)
+ asm ("libgo_sync.atomic.CompareAndSwapInt32");
+
+_Bool
+CompareAndSwapInt32 (int32_t *val, int32_t old, int32_t new)
+{
+ return __sync_bool_compare_and_swap (val, old, new);
+}
+
+_Bool CompareAndSwapInt64 (int64_t *, int64_t, int64_t)
+ asm ("libgo_sync.atomic.CompareAndSwapInt64");
+
+_Bool
+CompareAndSwapInt64 (int64_t *val, int64_t old, int64_t new)
+{
+ return __sync_bool_compare_and_swap (val, old, new);
+}
+
+_Bool CompareAndSwapUint32 (uint32_t *, uint32_t, uint32_t)
+ asm ("libgo_sync.atomic.CompareAndSwapUint32");
+
+_Bool
+CompareAndSwapUint32 (uint32_t *val, uint32_t old, uint32_t new)
+{
+ return __sync_bool_compare_and_swap (val, old, new);
+}
+
+_Bool CompareAndSwapUint64 (uint64_t *, uint64_t, uint64_t)
+ asm ("libgo_sync.atomic.CompareAndSwapUint64");
+
+_Bool
+CompareAndSwapUint64 (uint64_t *val, uint64_t old, uint64_t new)
+{
+ return __sync_bool_compare_and_swap (val, old, new);
+}
+
+_Bool CompareAndSwapUintptr (uintptr_t *, uintptr_t, uintptr_t)
+ asm ("libgo_sync.atomic.CompareAndSwapUintptr");
+
+_Bool
+CompareAndSwapUintptr (uintptr_t *val, uintptr_t old, uintptr_t new)
+{
+ return __sync_bool_compare_and_swap (val, old, new);
+}
+
+int32_t AddInt32 (int32_t *, int32_t)
+ asm ("libgo_sync.atomic.AddInt32");
+
+int32_t
+AddInt32 (int32_t *val, int32_t delta)
+{
+ return __sync_add_and_fetch (val, delta);
+}
+
+uint32_t AddUint32 (uint32_t *, uint32_t)
+ asm ("libgo_sync.atomic.AddUint32");
+
+uint32_t
+AddUint32 (uint32_t *val, uint32_t delta)
+{
+ return __sync_add_and_fetch (val, delta);
+}
+
+int64_t AddInt64 (int64_t *, int64_t)
+ asm ("libgo_sync.atomic.AddInt64");
+
+int64_t
+AddInt64 (int64_t *val, int64_t delta)
+{
+ return __sync_add_and_fetch (val, delta);
+}
+
+uint64_t AddUint64 (uint64_t *, uint64_t)
+ asm ("libgo_sync.atomic.AddUint64");
+
+uint64_t
+AddUint64 (uint64_t *val, uint64_t delta)
+{
+ return __sync_add_and_fetch (val, delta);
+}
+
+uintptr_t AddUintptr (uintptr_t *, uintptr_t)
+ asm ("libgo_sync.atomic.AddUintptr");
+
+uintptr_t
+AddUintptr (uintptr_t *val, uintptr_t delta)
+{
+ return __sync_add_and_fetch (val, delta);
+}
--- /dev/null
+// Copyright 2011 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package atomic
+
+import (
+ "runtime"
+ "testing"
+ "unsafe"
+)
+
+// Tests of correct behavior, without contention.
+// (Does the function work as advertised?)
+//
+// Test that the Add functions add correctly.
+// Test that the CompareAndSwap functions actually
+// do the comparison and the swap correctly.
+//
+// The loop over power-of-two values is meant to
+// ensure that the operations apply to the full word size.
+// The struct fields x.before and x.after check that the
+// operations do not extend past the full word size.
+
+const (
+ magic32 = 0xdedbeef
+ magic64 = 0xdeddeadbeefbeef
+)
+
+func TestAddInt32(t *testing.T) {
+ var x struct {
+ before int32
+ i int32
+ after int32
+ }
+ x.before = magic32
+ x.after = magic32
+ var j int32
+ for delta := int32(1); delta+delta > delta; delta += delta {
+ k := AddInt32(&x.i, delta)
+ j += delta
+ if x.i != j || k != j {
+ t.Fatalf("delta=%d i=%d j=%d k=%d", delta, x.i, j, k)
+ }
+ }
+ if x.before != magic32 || x.after != magic32 {
+ t.Fatalf("wrong magic: %#x _ %#x != %#x _ %#x", x.before, x.after, magic32, magic32)
+ }
+}
+
+func TestAddUint32(t *testing.T) {
+ var x struct {
+ before uint32
+ i uint32
+ after uint32
+ }
+ x.before = magic32
+ x.after = magic32
+ var j uint32
+ for delta := uint32(1); delta+delta > delta; delta += delta {
+ k := AddUint32(&x.i, delta)
+ j += delta
+ if x.i != j || k != j {
+ t.Fatalf("delta=%d i=%d j=%d k=%d", delta, x.i, j, k)
+ }
+ }
+ if x.before != magic32 || x.after != magic32 {
+ t.Fatalf("wrong magic: %#x _ %#x != %#x _ %#x", x.before, x.after, magic32, magic32)
+ }
+}
+
+func TestAddInt64(t *testing.T) {
+ var x struct {
+ before int64
+ i int64
+ after int64
+ }
+ x.before = magic64
+ x.after = magic64
+ var j int64
+ for delta := int64(1); delta+delta > delta; delta += delta {
+ k := AddInt64(&x.i, delta)
+ j += delta
+ if x.i != j || k != j {
+ t.Fatalf("delta=%d i=%d j=%d k=%d", delta, x.i, j, k)
+ }
+ }
+ if x.before != magic64 || x.after != magic64 {
+ t.Fatalf("wrong magic: %#x _ %#x != %#x _ %#x", x.before, x.after, int64(magic64), int64(magic64))
+ }
+}
+
+func TestAddUint64(t *testing.T) {
+ var x struct {
+ before uint64
+ i uint64
+ after uint64
+ }
+ x.before = magic64
+ x.after = magic64
+ var j uint64
+ for delta := uint64(1); delta+delta > delta; delta += delta {
+ k := AddUint64(&x.i, delta)
+ j += delta
+ if x.i != j || k != j {
+ t.Fatalf("delta=%d i=%d j=%d k=%d", delta, x.i, j, k)
+ }
+ }
+ if x.before != magic64 || x.after != magic64 {
+ t.Fatalf("wrong magic: %#x _ %#x != %#x _ %#x", x.before, x.after, uint64(magic64), uint64(magic64))
+ }
+}
+
+func TestAddUintptr(t *testing.T) {
+ var x struct {
+ before uintptr
+ i uintptr
+ after uintptr
+ }
+ var m uint64 = magic64
+ magicptr := uintptr(m)
+ x.before = magicptr
+ x.after = magicptr
+ var j uintptr
+ for delta := uintptr(1); delta+delta > delta; delta += delta {
+ k := AddUintptr(&x.i, delta)
+ j += delta
+ if x.i != j || k != j {
+ t.Fatalf("delta=%d i=%d j=%d k=%d", delta, x.i, j, k)
+ }
+ }
+ if x.before != magicptr || x.after != magicptr {
+ t.Fatalf("wrong magic: %#x _ %#x != %#x _ %#x", x.before, x.after, magicptr, magicptr)
+ }
+}
+
+func TestCompareAndSwapInt32(t *testing.T) {
+ var x struct {
+ before int32
+ i int32
+ after int32
+ }
+ x.before = magic32
+ x.after = magic32
+ for val := int32(1); val+val > val; val += val {
+ x.i = val
+ if !CompareAndSwapInt32(&x.i, val, val+1) {
+ t.Errorf("should have swapped %#x %#x", val, val+1)
+ }
+ if x.i != val+1 {
+ t.Errorf("wrong x.i after swap: x.i=%#x val+1=%#x", x.i, val+1)
+ }
+ x.i = val + 1
+ if CompareAndSwapInt32(&x.i, val, val+2) {
+ t.Errorf("should not have swapped %#x %#x", val, val+2)
+ }
+ if x.i != val+1 {
+ t.Errorf("wrong x.i after swap: x.i=%#x val+1=%#x", x.i, val+1)
+ }
+ }
+ if x.before != magic32 || x.after != magic32 {
+ t.Fatalf("wrong magic: %#x _ %#x != %#x _ %#x", x.before, x.after, magic32, magic32)
+ }
+}
+
+func TestCompareAndSwapUint32(t *testing.T) {
+ var x struct {
+ before uint32
+ i uint32
+ after uint32
+ }
+ x.before = magic32
+ x.after = magic32
+ for val := uint32(1); val+val > val; val += val {
+ x.i = val
+ if !CompareAndSwapUint32(&x.i, val, val+1) {
+ t.Errorf("should have swapped %#x %#x", val, val+1)
+ }
+ if x.i != val+1 {
+ t.Errorf("wrong x.i after swap: x.i=%#x val+1=%#x", x.i, val+1)
+ }
+ x.i = val + 1
+ if CompareAndSwapUint32(&x.i, val, val+2) {
+ t.Errorf("should not have swapped %#x %#x", val, val+2)
+ }
+ if x.i != val+1 {
+ t.Errorf("wrong x.i after swap: x.i=%#x val+1=%#x", x.i, val+1)
+ }
+ }
+ if x.before != magic32 || x.after != magic32 {
+ t.Fatalf("wrong magic: %#x _ %#x != %#x _ %#x", x.before, x.after, magic32, magic32)
+ }
+}
+
+func TestCompareAndSwapInt64(t *testing.T) {
+ var x struct {
+ before int64
+ i int64
+ after int64
+ }
+ x.before = magic64
+ x.after = magic64
+ for val := int64(1); val+val > val; val += val {
+ x.i = val
+ if !CompareAndSwapInt64(&x.i, val, val+1) {
+ t.Errorf("should have swapped %#x %#x", val, val+1)
+ }
+ if x.i != val+1 {
+ t.Errorf("wrong x.i after swap: x.i=%#x val+1=%#x", x.i, val+1)
+ }
+ x.i = val + 1
+ if CompareAndSwapInt64(&x.i, val, val+2) {
+ t.Errorf("should not have swapped %#x %#x", val, val+2)
+ }
+ if x.i != val+1 {
+ t.Errorf("wrong x.i after swap: x.i=%#x val+1=%#x", x.i, val+1)
+ }
+ }
+ if x.before != magic64 || x.after != magic64 {
+ t.Fatalf("wrong magic: %#x _ %#x != %#x _ %#x", x.before, x.after, uint64(magic64), uint64(magic64))
+ }
+}
+
+func TestCompareAndSwapUint64(t *testing.T) {
+ var x struct {
+ before uint64
+ i uint64
+ after uint64
+ }
+ x.before = magic64
+ x.after = magic64
+ for val := uint64(1); val+val > val; val += val {
+ x.i = val
+ if !CompareAndSwapUint64(&x.i, val, val+1) {
+ t.Errorf("should have swapped %#x %#x", val, val+1)
+ }
+ if x.i != val+1 {
+ t.Errorf("wrong x.i after swap: x.i=%#x val+1=%#x", x.i, val+1)
+ }
+ x.i = val + 1
+ if CompareAndSwapUint64(&x.i, val, val+2) {
+ t.Errorf("should not have swapped %#x %#x", val, val+2)
+ }
+ if x.i != val+1 {
+ t.Errorf("wrong x.i after swap: x.i=%#x val+1=%#x", x.i, val+1)
+ }
+ }
+ if x.before != magic64 || x.after != magic64 {
+ t.Fatalf("wrong magic: %#x _ %#x != %#x _ %#x", x.before, x.after, uint64(magic64), uint64(magic64))
+ }
+}
+
+func TestCompareAndSwapUintptr(t *testing.T) {
+ var x struct {
+ before uintptr
+ i uintptr
+ after uintptr
+ }
+ var m uint64 = magic64
+ magicptr := uintptr(m)
+ x.before = magicptr
+ x.after = magicptr
+ for val := uintptr(1); val+val > val; val += val {
+ x.i = val
+ if !CompareAndSwapUintptr(&x.i, val, val+1) {
+ t.Errorf("should have swapped %#x %#x", val, val+1)
+ }
+ if x.i != val+1 {
+ t.Errorf("wrong x.i after swap: x.i=%#x val+1=%#x", x.i, val+1)
+ }
+ x.i = val + 1
+ if CompareAndSwapUintptr(&x.i, val, val+2) {
+ t.Errorf("should not have swapped %#x %#x", val, val+2)
+ }
+ if x.i != val+1 {
+ t.Errorf("wrong x.i after swap: x.i=%#x val+1=%#x", x.i, val+1)
+ }
+ }
+ if x.before != magicptr || x.after != magicptr {
+ t.Fatalf("wrong magic: %#x _ %#x != %#x _ %#x", x.before, x.after, magicptr, magicptr)
+ }
+}
+
+// Tests of correct behavior, with contention.
+// (Is the function atomic?)
+//
+// For each function, we write a "hammer" function that repeatedly
+// uses the atomic operation to add 1 to a value. After running
+// multiple hammers in parallel, check that we end with the correct
+// total.
+
+var hammer32 = []struct {
+ name string
+ f func(*uint32, int)
+}{
+ {"AddInt32", hammerAddInt32},
+ {"AddUint32", hammerAddUint32},
+ {"AddUintptr", hammerAddUintptr32},
+ {"CompareAndSwapInt32", hammerCompareAndSwapInt32},
+ {"CompareAndSwapUint32", hammerCompareAndSwapUint32},
+ {"CompareAndSwapUintptr", hammerCompareAndSwapUintptr32},
+}
+
+func init() {
+ var v uint64 = 1 << 50
+ if uintptr(v) != 0 {
+ // 64-bit system; clear uintptr tests
+ hammer32[2].f = nil
+ hammer32[5].f = nil
+ }
+}
+
+func hammerAddInt32(uval *uint32, count int) {
+ val := (*int32)(unsafe.Pointer(uval))
+ for i := 0; i < count; i++ {
+ AddInt32(val, 1)
+ }
+}
+
+func hammerAddUint32(val *uint32, count int) {
+ for i := 0; i < count; i++ {
+ AddUint32(val, 1)
+ }
+}
+
+func hammerAddUintptr32(uval *uint32, count int) {
+ // only safe when uintptr is 32-bit.
+ // not called on 64-bit systems.
+ val := (*uintptr)(unsafe.Pointer(uval))
+ for i := 0; i < count; i++ {
+ AddUintptr(val, 1)
+ }
+}
+
+func hammerCompareAndSwapInt32(uval *uint32, count int) {
+ val := (*int32)(unsafe.Pointer(uval))
+ for i := 0; i < count; i++ {
+ for {
+ v := *val
+ if CompareAndSwapInt32(val, v, v+1) {
+ break
+ }
+ }
+ }
+}
+
+func hammerCompareAndSwapUint32(val *uint32, count int) {
+ for i := 0; i < count; i++ {
+ for {
+ v := *val
+ if CompareAndSwapUint32(val, v, v+1) {
+ break
+ }
+ }
+ }
+}
+
+func hammerCompareAndSwapUintptr32(uval *uint32, count int) {
+ // only safe when uintptr is 32-bit.
+ // not called on 64-bit systems.
+ val := (*uintptr)(unsafe.Pointer(uval))
+ for i := 0; i < count; i++ {
+ for {
+ v := *val
+ if CompareAndSwapUintptr(val, v, v+1) {
+ break
+ }
+ }
+ }
+}
+
+func TestHammer32(t *testing.T) {
+ const (
+ n = 100000
+ p = 4
+ )
+ defer runtime.GOMAXPROCS(runtime.GOMAXPROCS(p))
+
+ for _, tt := range hammer32 {
+ if tt.f == nil {
+ continue
+ }
+ c := make(chan int)
+ var val uint32
+ for i := 0; i < p; i++ {
+ go func() {
+ tt.f(&val, n)
+ c <- 1
+ }()
+ }
+ for i := 0; i < p; i++ {
+ <-c
+ }
+ if val != n*p {
+ t.Errorf("%s: val=%d want %d", tt.name, val, n*p)
+ }
+ }
+}
+
+var hammer64 = []struct {
+ name string
+ f func(*uint64, int)
+}{
+ {"AddInt64", hammerAddInt64},
+ {"AddUint64", hammerAddUint64},
+ {"AddUintptr", hammerAddUintptr64},
+ {"CompareAndSwapInt64", hammerCompareAndSwapInt64},
+ {"CompareAndSwapUint64", hammerCompareAndSwapUint64},
+ {"CompareAndSwapUintptr", hammerCompareAndSwapUintptr64},
+}
+
+func init() {
+ var v uint64 = 1 << 50
+ if uintptr(v) == 0 {
+ // 32-bit system; clear uintptr tests
+ hammer64[2].f = nil
+ hammer64[5].f = nil
+ }
+}
+
+func hammerAddInt64(uval *uint64, count int) {
+ val := (*int64)(unsafe.Pointer(uval))
+ for i := 0; i < count; i++ {
+ AddInt64(val, 1)
+ }
+}
+
+func hammerAddUint64(val *uint64, count int) {
+ for i := 0; i < count; i++ {
+ AddUint64(val, 1)
+ }
+}
+
+func hammerAddUintptr64(uval *uint64, count int) {
+ // only safe when uintptr is 64-bit.
+ // not called on 32-bit systems.
+ val := (*uintptr)(unsafe.Pointer(uval))
+ for i := 0; i < count; i++ {
+ AddUintptr(val, 1)
+ }
+}
+
+func hammerCompareAndSwapInt64(uval *uint64, count int) {
+ val := (*int64)(unsafe.Pointer(uval))
+ for i := 0; i < count; i++ {
+ for {
+ v := *val
+ if CompareAndSwapInt64(val, v, v+1) {
+ break
+ }
+ }
+ }
+}
+
+func hammerCompareAndSwapUint64(val *uint64, count int) {
+ for i := 0; i < count; i++ {
+ for {
+ v := *val
+ if CompareAndSwapUint64(val, v, v+1) {
+ break
+ }
+ }
+ }
+}
+
+func hammerCompareAndSwapUintptr64(uval *uint64, count int) {
+ // only safe when uintptr is 64-bit.
+ // not called on 32-bit systems.
+ val := (*uintptr)(unsafe.Pointer(uval))
+ for i := 0; i < count; i++ {
+ for {
+ v := *val
+ if CompareAndSwapUintptr(val, v, v+1) {
+ break
+ }
+ }
+ }
+}
+
+func TestHammer64(t *testing.T) {
+ const (
+ n = 100000
+ p = 4
+ )
+ defer runtime.GOMAXPROCS(runtime.GOMAXPROCS(p))
+
+ for _, tt := range hammer64 {
+ if tt.f == nil {
+ continue
+ }
+ c := make(chan int)
+ var val uint64
+ for i := 0; i < p; i++ {
+ go func() {
+ tt.f(&val, n)
+ c <- 1
+ }()
+ }
+ for i := 0; i < p; i++ {
+ <-c
+ }
+ if val != n*p {
+ t.Errorf("%s: val=%d want %d", tt.name, val, n*p)
+ }
+ }
+}
--- /dev/null
+// Copyright 2011 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Package atomic provides low-level atomic memory primitives
+// useful for implementing synchronization algorithms.
+//
+// These functions require great care to be used correctly.
+// Except for special, low-level applications, synchronization is better
+// done with channels or the facilities of the sync package.
+// Share memory by communicating;
+// don't communicate by sharing memory.
+//
+// The compare-and-swap operation, implemented by the CompareAndSwapT
+// functions, is the atomic equivalent of:
+//
+// if *val == old {
+// *val = new
+// return true
+// }
+// return false
+//
+package atomic
+
+// BUG(rsc): On ARM, the 64-bit functions use instructions unavailable before ARM 11.
+//
+// On x86-32, the 64-bit functions use instructions unavailable before the Pentium.
+
+// CompareAndSwapInt32 executes the compare-and-swap operation for an int32 value.
+func CompareAndSwapInt32(val *int32, old, new int32) (swapped bool)
+
+// CompareAndSwapInt64 executes the compare-and-swap operation for an int64 value.
+func CompareAndSwapInt64(val *int64, old, new int64) (swapped bool)
+
+// CompareAndSwapUint32 executes the compare-and-swap operation for a uint32 value.
+func CompareAndSwapUint32(val *uint32, old, new uint32) (swapped bool)
+
+// CompareAndSwapUint64 executes the compare-and-swap operation for a uint64 value.
+func CompareAndSwapUint64(val *uint64, old, new uint64) (swapped bool)
+
+// CompareAndSwapUintptr executes the compare-and-swap operation for a uintptr value.
+func CompareAndSwapUintptr(val *uintptr, old, new uintptr) (swapped bool)
+
+// AddInt32 atomically adds delta to *val and returns the new value.
+func AddInt32(val *int32, delta int32) (new int32)
+
+// AddUint32 atomically adds delta to *val and returns the new value.
+func AddUint32(val *uint32, delta uint32) (new uint32)
+
+// AddInt64 atomically adds delta to *val and returns the new value.
+func AddInt64(val *int64, delta int64) (new int64)
+
+// AddUint64 atomically adds delta to *val and returns the new value.
+func AddUint64(val *uint64, delta uint64) (new uint64)
+
+// AddUintptr atomically adds delta to *val and returns the new value.
+func AddUintptr(val *uintptr, delta uintptr) (new uintptr)
--- /dev/null
+// Copyright 2011 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package sync
+
+import "runtime"
+
+// Cond implements a condition variable, a rendezvous point
+// for goroutines waiting for or announcing the occurrence
+// of an event.
+//
+// Each Cond has an associated Locker L (often a *Mutex or *RWMutex),
+// which must be held when changing the condition and
+// when calling the Wait method.
+type Cond struct {
+ L Locker // held while observing or changing the condition
+ m Mutex // held to avoid internal races
+ waiters int // number of goroutines blocked on Wait
+ sema *uint32
+}
+
+// NewCond returns a new Cond with Locker l.
+func NewCond(l Locker) *Cond {
+ return &Cond{L: l}
+}
+
+// Wait atomically unlocks c.L and suspends execution
+// of the calling goroutine. After later resuming execution,
+// Wait locks c.L before returning.
+//
+// Because L is not locked when Wait first resumes, the caller
+// typically cannot assume that the condition is true when
+// Wait returns. Instead, the caller should Wait in a loop:
+//
+// c.L.Lock()
+// for !condition() {
+// c.Wait()
+// }
+// ... make use of condition ...
+// c.L.Unlock()
+//
+func (c *Cond) Wait() {
+ c.m.Lock()
+ if c.sema == nil {
+ c.sema = new(uint32)
+ }
+ s := c.sema
+ c.waiters++
+ c.m.Unlock()
+ c.L.Unlock()
+ runtime.Semacquire(s)
+ c.L.Lock()
+}
+
+// Signal wakes one goroutine waiting on c, if there is any.
+//
+// It is allowed but not required for the caller to hold c.L
+// during the call.
+func (c *Cond) Signal() {
+ c.m.Lock()
+ if c.waiters > 0 {
+ c.waiters--
+ runtime.Semrelease(c.sema)
+ }
+ c.m.Unlock()
+}
+
+// Broadcast wakes all goroutines waiting on c.
+//
+// It is allowed but not required for the caller to hold c.L
+// during the call.
+func (c *Cond) Broadcast() {
+ c.m.Lock()
+ if c.waiters > 0 {
+ s := c.sema
+ n := c.waiters
+ for i := 0; i < n; i++ {
+ runtime.Semrelease(s)
+ }
+ // We just issued n wakeups via the semaphore s.
+ // To ensure that they wake up the existing waiters
+ // and not waiters that arrive after Broadcast returns,
+ // clear c.sema. The next operation will allocate
+ // a new one.
+ c.sema = nil
+ c.waiters = 0
+ }
+ c.m.Unlock()
+}
--- /dev/null
+// Copyright 2011 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+package sync_test
+
+import (
+ . "sync"
+ "testing"
+)
+
+func TestCondSignal(t *testing.T) {
+ var m Mutex
+ c := NewCond(&m)
+ n := 2
+ running := make(chan bool, n)
+ awake := make(chan bool, n)
+ for i := 0; i < n; i++ {
+ go func() {
+ m.Lock()
+ running <- true
+ c.Wait()
+ awake <- true
+ m.Unlock()
+ }()
+ }
+ for i := 0; i < n; i++ {
+ <-running // Wait for everyone to run.
+ }
+ for n > 0 {
+ select {
+ case <-awake:
+ t.Fatal("goroutine not asleep")
+ default:
+ }
+ m.Lock()
+ c.Signal()
+ m.Unlock()
+ <-awake // Will deadlock if no goroutine wakes up
+ select {
+ case <-awake:
+ t.Fatal("too many goroutines awake")
+ default:
+ }
+ n--
+ }
+ c.Signal()
+}
+
+func TestCondBroadcast(t *testing.T) {
+ var m Mutex
+ c := NewCond(&m)
+ n := 200
+ running := make(chan int, n)
+ awake := make(chan int, n)
+ exit := false
+ for i := 0; i < n; i++ {
+ go func(g int) {
+ m.Lock()
+ for !exit {
+ running <- g
+ c.Wait()
+ awake <- g
+ }
+ m.Unlock()
+ }(i)
+ }
+ for i := 0; i < n; i++ {
+ for i := 0; i < n; i++ {
+ <-running // Will deadlock unless n are running.
+ }
+ if i == n-1 {
+ m.Lock()
+ exit = true
+ m.Unlock()
+ }
+ select {
+ case <-awake:
+ t.Fatal("goroutine not asleep")
+ default:
+ }
+ m.Lock()
+ c.Broadcast()
+ m.Unlock()
+ seen := make([]bool, n)
+ for i := 0; i < n; i++ {
+ g := <-awake
+ if seen[g] {
+ t.Fatal("goroutine woke up twice")
+ }
+ seen[g] = true
+ }
+ }
+ select {
+ case <-running:
+ t.Fatal("goroutine did not exit")
+ default:
+ }
+ c.Broadcast()
+}
// license that can be found in the LICENSE file.
// The sync package provides basic synchronization primitives
-// such as mutual exclusion locks. Other than the Once type,
-// most are intended for use by low-level library routines.
-// Higher-level synchronization is better done via channels
-// and communication.
+// such as mutual exclusion locks. Other than the Once and
+// WaitGroup types, most are intended for use by low-level
+// library routines. Higher-level synchronization is better
+// done via channels and communication.
package sync
-import "runtime"
-
-func cas(val *uint32, old, new uint32) bool
+import (
+ "runtime"
+ "sync/atomic"
+)
// A Mutex is a mutual exclusion lock.
// Mutexes can be created as part of other structures;
// the zero value for a Mutex is an unlocked mutex.
type Mutex struct {
- key uint32
+ key int32
sema uint32
}
-// Add delta to *val, and return the new *val in a thread-safe way. If multiple
-// goroutines call xadd on the same val concurrently, the changes will be
-// serialized, and all the deltas will be added in an undefined order.
-func xadd(val *uint32, delta int32) (new uint32) {
- for {
- v := *val
- nv := v + uint32(delta)
- if cas(val, v, nv) {
- return nv
- }
- }
- panic("unreached")
+// A Locker represents an object that can be locked and unlocked.
+type Locker interface {
+ Lock()
+ Unlock()
}
// Lock locks m.
// If the lock is already in use, the calling goroutine
// blocks until the mutex is available.
func (m *Mutex) Lock() {
- if xadd(&m.key, 1) == 1 {
+ if atomic.AddInt32(&m.key, 1) == 1 {
// changed from 0 to 1; we hold lock
return
}
// It is allowed for one goroutine to lock a Mutex and then
// arrange for another goroutine to unlock it.
func (m *Mutex) Unlock() {
- if xadd(&m.key, -1) == 0 {
+ switch v := atomic.AddInt32(&m.key, -1); {
+ case v == 0:
// changed from 1 to 0; no contention
return
+ case v == -1:
+ // changed from 0 to -1: wasn't locked
+ // (or there are 4 billion goroutines waiting)
+ panic("sync: unlock of unlocked mutex")
}
runtime.Semrelease(&m.sema)
}
<-c
<-c
}
+
+func TestMutexPanic(t *testing.T) {
+ defer func() {
+ if recover() == nil {
+ t.Fatalf("unlock of unlocked mutex did not panic")
+ }
+ }()
+
+ var mu Mutex
+ mu.Lock()
+ mu.Unlock()
+ mu.Unlock()
+}
// Do calls the function f if and only if the method is being called for the
// first time with this receiver. In other words, given
// var once Once
-// if Do(f) is called multiple times, only the first call will invoke f,
+// if once.Do(f) is called multiple times, only the first call will invoke f,
// even if f has a different value in each invocation. A new instance of
// Once is required for each function to execute.
//
package sync
+import "sync/atomic"
+
// An RWMutex is a reader/writer mutual exclusion lock.
// The lock can be held by an arbitrary number of readers
// or a single writer.
// Writers take priority over Readers: no new RLocks
// are granted while a blocked Lock call is waiting.
type RWMutex struct {
- w Mutex // held if there are pending readers or writers
- r Mutex // held if the w is being rd
- readerCount uint32 // number of pending readers
+ w Mutex // held if there are pending readers or writers
+ r Mutex // held if the w is being rd
+ readerCount int32 // number of pending readers
}
// RLock locks rw for reading.
// B: rw.RUnlock()
// ... (new readers come and go indefinitely, W is starving)
rw.r.Lock()
- if xadd(&rw.readerCount, 1) == 1 {
+ if atomic.AddInt32(&rw.readerCount, 1) == 1 {
// The first reader locks rw.w, so writers will be blocked
// while the readers have the RLock.
rw.w.Lock()
// It is a run-time error if rw is not locked for reading
// on entry to RUnlock.
func (rw *RWMutex) RUnlock() {
- if xadd(&rw.readerCount, -1) == 0 {
+ if atomic.AddInt32(&rw.readerCount, -1) == 0 {
// last reader finished, enable writers
rw.w.Unlock()
}
rw.r.Unlock()
}
-// Unlock unlocks rw for writing.
-// It is a run-time error if rw is not locked for writing
-// on entry to Unlock.
+// Unlock unlocks rw for writing. It is a run-time error if rw is
+// not locked for writing on entry to Unlock.
//
-// Like for Mutexes,
-// a locked RWMutex is not associated with a particular goroutine.
-// It is allowed for one goroutine to RLock (Lock) an RWMutex and then
+// As with Mutexes, a locked RWMutex is not associated with a particular
+// goroutine. One goroutine may RLock (Lock) an RWMutex and then
// arrange for another goroutine to RUnlock (Unlock) it.
func (rw *RWMutex) Unlock() { rw.w.Unlock() }
+
+// RLocker returns a Locker interface that implements
+// the Lock and Unlock methods by calling rw.RLock and rw.RUnlock.
+func (rw *RWMutex) RLocker() Locker {
+ return (*rlocker)(rw)
+}
+
+type rlocker RWMutex
+
+func (r *rlocker) Lock() { (*RWMutex)(r).RLock() }
+func (r *rlocker) Unlock() { (*RWMutex)(r).RUnlock() }
"fmt"
"runtime"
. "sync"
+ "sync/atomic"
"testing"
)
doTestParallelReaders(4, 2)
}
-func reader(rwm *RWMutex, num_iterations int, activity *uint32, cdone chan bool) {
+func reader(rwm *RWMutex, num_iterations int, activity *int32, cdone chan bool) {
for i := 0; i < num_iterations; i++ {
rwm.RLock()
- n := Xadd(activity, 1)
+ n := atomic.AddInt32(activity, 1)
if n < 1 || n >= 10000 {
panic(fmt.Sprintf("wlock(%d)\n", n))
}
for i := 0; i < 100; i++ {
}
- Xadd(activity, -1)
+ atomic.AddInt32(activity, -1)
rwm.RUnlock()
}
cdone <- true
}
-func writer(rwm *RWMutex, num_iterations int, activity *uint32, cdone chan bool) {
+func writer(rwm *RWMutex, num_iterations int, activity *int32, cdone chan bool) {
for i := 0; i < num_iterations; i++ {
rwm.Lock()
- n := Xadd(activity, 10000)
+ n := atomic.AddInt32(activity, 10000)
if n != 10000 {
panic(fmt.Sprintf("wlock(%d)\n", n))
}
for i := 0; i < 100; i++ {
}
- Xadd(activity, -10000)
+ atomic.AddInt32(activity, -10000)
rwm.Unlock()
}
cdone <- true
func HammerRWMutex(gomaxprocs, numReaders, num_iterations int) {
runtime.GOMAXPROCS(gomaxprocs)
// Number of active readers + 10000 * number of active writers.
- var activity uint32
+ var activity int32
var rwm RWMutex
cdone := make(chan bool)
go writer(&rwm, num_iterations, &activity, cdone)
HammerRWMutex(10, 10, 1000)
HammerRWMutex(10, 5, 10000)
}
+
+func TestRLocker(t *testing.T) {
+ var wl RWMutex
+ var rl Locker
+ wlocked := make(chan bool, 1)
+ rlocked := make(chan bool, 1)
+ rl = wl.RLocker()
+ n := 10
+ go func() {
+ for i := 0; i < n; i++ {
+ rl.Lock()
+ rl.Lock()
+ rlocked <- true
+ wl.Lock()
+ wlocked <- true
+ }
+ }()
+ for i := 0; i < n; i++ {
+ <-rlocked
+ rl.Unlock()
+ select {
+ case <-wlocked:
+ t.Fatal("RLocker() didn't read-lock it")
+ default:
+ }
+ rl.Unlock()
+ <-wlocked
+ select {
+ case <-rlocked:
+ t.Fatal("RLocker() didn't respect the write lock")
+ default:
+ }
+ wl.Unlock()
+ }
+}
--- /dev/null
+// Copyright 2011 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package sync
+
+import "runtime"
+
+// A WaitGroup waits for a collection of goroutines to finish.
+// The main goroutine calls Add to set the number of
+// goroutines to wait for. Then each of the goroutines
+// runs and calls Done when finished. At the same time,
+// Wait can be used to block until all goroutines have finished.
+//
+// For example:
+//
+// for i := 0; i < n; i++ {
+// if !condition(i) {
+// continue
+// }
+// wg.Add(1)
+// go func() {
+// // Do something.
+// wg.Done()
+// }
+// }
+// wg.Wait()
+//
+type WaitGroup struct {
+ m Mutex
+ counter int
+ waiters int
+ sema *uint32
+}
+
+// WaitGroup creates a new semaphore each time the old semaphore
+// is released. This is to avoid the following race:
+//
+// G1: Add(1)
+// G1: go G2()
+// G1: Wait() // Context switch after Unlock() and before Semacquire().
+// G2: Done() // Release semaphore: sema == 1, waiters == 0. G1 doesn't run yet.
+// G3: Wait() // Finds counter == 0, waiters == 0, doesn't block.
+// G3: Add(1) // Makes counter == 1, waiters == 0.
+// G3: go G4()
+// G3: Wait() // G1 still hasn't run, G3 finds sema == 1, unblocked! Bug.
+
+// Add adds delta, which may be negative, to the WaitGroup counter.
+// If the counter becomes zero, all goroutines blocked on Wait() are released.
+func (wg *WaitGroup) Add(delta int) {
+ wg.m.Lock()
+ if delta < -wg.counter {
+ wg.m.Unlock()
+ panic("sync: negative WaitGroup count")
+ }
+ wg.counter += delta
+ if wg.counter == 0 && wg.waiters > 0 {
+ for i := 0; i < wg.waiters; i++ {
+ runtime.Semrelease(wg.sema)
+ }
+ wg.waiters = 0
+ wg.sema = nil
+ }
+ wg.m.Unlock()
+}
+
+// Done decrements the WaitGroup counter.
+func (wg *WaitGroup) Done() {
+ wg.Add(-1)
+}
+
+// Wait blocks until the WaitGroup counter is zero.
+func (wg *WaitGroup) Wait() {
+ wg.m.Lock()
+ if wg.counter == 0 {
+ wg.m.Unlock()
+ return
+ }
+ wg.waiters++
+ if wg.sema == nil {
+ wg.sema = new(uint32)
+ }
+ s := wg.sema
+ wg.m.Unlock()
+ runtime.Semacquire(s)
+}
--- /dev/null
+// Copyright 2011 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package sync_test
+
+import (
+ . "sync"
+ "testing"
+)
+
+func testWaitGroup(t *testing.T, wg1 *WaitGroup, wg2 *WaitGroup) {
+ n := 16
+ wg1.Add(n)
+ wg2.Add(n)
+ exited := make(chan bool, n)
+ for i := 0; i != n; i++ {
+ go func(i int) {
+ wg1.Done()
+ wg2.Wait()
+ exited <- true
+ }(i)
+ }
+ wg1.Wait()
+ for i := 0; i != n; i++ {
+ select {
+ case <-exited:
+ t.Fatal("WaitGroup released group too soon")
+ default:
+ }
+ wg2.Done()
+ }
+ for i := 0; i != n; i++ {
+ <-exited // Will block if barrier fails to unlock someone.
+ }
+}
+
+func TestWaitGroup(t *testing.T) {
+ wg1 := &WaitGroup{}
+ wg2 := &WaitGroup{}
+
+ // Run the same test a few times to ensure barrier is in a proper state.
+ for i := 0; i != 8; i++ {
+ testWaitGroup(t, wg1, wg2)
+ }
+}
+
+func TestWaitGroupMisuse(t *testing.T) {
+ defer func() {
+ err := recover()
+ if err != "sync: negative WaitGroup count" {
+ t.Fatalf("Unexpected panic: %#v", err)
+ }
+ }()
+ wg := &WaitGroup{}
+ wg.Add(1)
+ wg.Done()
+ wg.Done()
+ t.Fatal("Should panic")
+}
+++ /dev/null
-// Copyright 2009 The Go Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-package sync
-
-func Xadd(val *uint32, delta int32) (new uint32) {
- return xadd(val, delta)
-}
func startServer(done chan<- string) {
c, e := net.ListenPacket("udp", "127.0.0.1:0")
if e != nil {
- log.Exitf("net.ListenPacket failed udp :0 %v", e)
+ log.Fatalf("net.ListenPacket failed udp :0 %v", e)
}
serverAddr = c.LocalAddr().String()
c.SetReadTimeout(100e6) // 100ms
- The result of invoking a niladic single-valued method with that name
(result = data.field())
- Major constructs ({} are metacharacters; [] marks optional elements):
+ Major constructs ({} are the default delimiters for template actions;
+ [] are the notation in this comment for optional elements):
{# comment }
{field1 field2 ...}
{field|formatter}
{field1 field2...|formatter}
+ {field|formatter1|formatter2}
Insert the value of the fields into the output. Each field is
first looked for in the cursor, as in .section and .repeated.
If it is not found, the search continues in outer sections
until the top level is reached.
+ If the field value is a pointer, leading asterisks indicate
+ that the value to be inserted should be evaluated through the
+ pointer. For example, if x.p is of type *int, {x.p} will
+ insert the value of the pointer but {*x.p} will insert the
+ value of the underlying integer. If the value is nil or not a
+ pointer, asterisks have no effect.
+
If a formatter is specified, it must be named in the formatter
map passed to the template set up routines or in the default
set ("html","str","") and is used to process the data for
values at the instantiation, and formatter is its name at
the invocation site. The default formatter just concatenates
the string representations of the fields.
+
+ Multiple formatters separated by the pipeline character | are
+ executed sequentially, with each formatter receiving the bytes
+ emitted by the one to its left.
+
+ The delimiter strings get their default value, "{" and "}", from
+ JSON-template. They may be set to any non-empty, space-free
+ string using the SetDelims method. Their value can be printed
+ in the output using {.meta-left} and {.meta-right}.
*/
package template
import (
+ "bytes"
"container/vector"
"fmt"
"io"
// A variable invocation to be evaluated
type variableElement struct {
- linenum int
- word []string // The fields in the invocation.
- formatter string // TODO(r): implement pipelines
+ linenum int
+ word []string // The fields in the invocation.
+ fmts []string // Names of formatters to apply. len(fmts) > 0
}
// A .section block, possibly with a .or
// the data item descends into the fields associated with sections, etc.
// Parent is used to walk upwards to find variables higher in the tree.
type state struct {
- parent *state // parent in hierarchy
- data reflect.Value // the driver data for this section etc.
- wr io.Writer // where to send output
+ parent *state // parent in hierarchy
+ data reflect.Value // the driver data for this section etc.
+ wr io.Writer // where to send output
+ buf [2]bytes.Buffer // alternating buffers used when chaining formatters
}
func (parent *state) clone(data reflect.Value) *state {
- return &state{parent, data, parent.wr}
+ return &state{parent: parent, data: data, wr: parent.wr}
}
// New creates a new template with the specified formatter map (which
return
}
+// formatter returns the Formatter with the given name in the Template, or nil if none exists.
+func (t *Template) formatter(name string) func(io.Writer, string, ...interface{}) {
+ if t.fmap != nil {
+ if fn := t.fmap[name]; fn != nil {
+ return fn
+ }
+ }
+ return builtins[name]
+}
+
// -- Parsing
// Allocate a new variable-evaluation element.
-func (t *Template) newVariable(words []string) (v *variableElement) {
- // The words are tokenized elements from the {item}. The last one may be of
- // the form "|fmt". For example: {a b c|d}
- formatter := ""
+func (t *Template) newVariable(words []string) *variableElement {
+ // After the final space-separated argument, formatters may be specified separated
+ // by pipe symbols, for example: {a b c|d|e}
+
+ // Until we learn otherwise, formatters contains a single name: "", the default formatter.
+ formatters := []string{""}
lastWord := words[len(words)-1]
- bar := strings.Index(lastWord, "|")
+ bar := strings.IndexRune(lastWord, '|')
if bar >= 0 {
words[len(words)-1] = lastWord[0:bar]
- formatter = lastWord[bar+1:]
+ formatters = strings.Split(lastWord[bar+1:], "|", -1)
}
- // Probably ok, so let's build it.
- v = &variableElement{t.linenum, words, formatter}
// We could remember the function address here and avoid the lookup later,
// but it's more dynamic to let the user change the map contents underfoot.
// We do require the name to be present, though.
// Is it in user-supplied map?
- if t.fmap != nil {
- if _, ok := t.fmap[formatter]; ok {
- return
+ for _, f := range formatters {
+ if t.formatter(f) == nil {
+ t.parseError("unknown formatter: %q", f)
}
}
- // Is it in builtin map?
- if _, ok := builtins[formatter]; ok {
- return
- }
- t.parseError("unknown formatter: %s", formatter)
- return
+ return &variableElement{t.linenum, words, formatters}
}
// Grab the next item. If it's simple, just append it to the template.
}
return av.FieldByName(name)
case *reflect.MapValue:
- return av.Elem(reflect.NewValue(name))
+ if v := av.Elem(reflect.NewValue(name)); v != nil {
+ return v
+ }
+ return reflect.MakeZero(typ.(*reflect.MapType).Elem())
default:
return nil
}
return v
}
+// indirectPtr returns the item numLevels levels of indirection below the value.
+// It is forgiving: if the value is not a pointer, it returns it rather than giving
+// an error. If the pointer is nil, it is returned as is.
+func indirectPtr(v reflect.Value, numLevels int) reflect.Value {
+ for i := numLevels; v != nil && i > 0; i++ {
+ if p, ok := v.(*reflect.PtrValue); ok {
+ if p.IsNil() {
+ return v
+ }
+ v = p.Elem()
+ } else {
+ break
+ }
+ }
+ return v
+}
+
// Walk v through pointers and interfaces, extracting the elements within.
func indirect(v reflect.Value) reflect.Value {
loop:
// The special name "@" (the "cursor") denotes the current data.
// The value coming in (st.data) might need indirecting to reach
// a struct while the return value is not indirected - that is,
-// it represents the actual named field.
+// it represents the actual named field. Leading stars indicate
+// levels of indirection to be applied to the value.
func (t *Template) findVar(st *state, s string) reflect.Value {
+ data := st.data
+ flattenedName := strings.TrimLeft(s, "*")
+ numStars := len(s) - len(flattenedName)
+ s = flattenedName
if s == "@" {
- return st.data
+ return indirectPtr(data, numStars)
}
- data := st.data
for _, elem := range strings.Split(s, ".", -1) {
// Look up field; data must be a struct or map.
data = t.lookup(st, data, elem)
return nil
}
}
- return data
+ return indirectPtr(data, numStars)
}
// Is there no data to look at?
return field
}
+func (t *Template) format(wr io.Writer, fmt string, val []interface{}, v *variableElement, st *state) {
+ fn := t.formatter(fmt)
+ if fn == nil {
+ t.execError(st, v.linenum, "missing formatter %s for variable %s", fmt, v.word[0])
+ }
+ fn(wr, fmt, val...)
+}
+
// Evaluate a variable, looking up through the parent if necessary.
// If it has a formatter attached ({var|formatter}) run that too.
func (t *Template) writeVariable(v *variableElement, st *state) {
- formatter := v.formatter
// Turn the words of the invocation into values.
val := make([]interface{}, len(v.word))
for i, word := range v.word {
val[i] = t.varValue(word, st).Interface()
}
- // is it in user-supplied map?
- if t.fmap != nil {
- if fn, ok := t.fmap[formatter]; ok {
- fn(st.wr, formatter, val...)
- return
- }
- }
- // is it in builtin map?
- if fn, ok := builtins[formatter]; ok {
- fn(st.wr, formatter, val...)
- return
+
+ for i, fmt := range v.fmts[:len(v.fmts)-1] {
+ b := &st.buf[i&1]
+ b.Reset()
+ t.format(b, fmt, val, v, st)
+ val = val[0:1]
+ val[0] = b.Bytes()
}
- t.execError(st, v.linenum, "missing formatter %s for variable %s", formatter, v.word[0])
+ t.format(st.wr, v.fmts[len(v.fmts)-1], val, v, st)
}
// Execute element i. Return next index to execute.
// Execute applies a parsed template to the specified data object,
// generating output to wr.
-func (t *Template) Execute(data interface{}, wr io.Writer) (err os.Error) {
+func (t *Template) Execute(wr io.Writer, data interface{}) (err os.Error) {
// Extract the driver data.
val := reflect.NewValue(data)
defer checkError(&err)
t.p = 0
- t.execute(0, t.elems.Len(), &state{nil, val, wr})
+ t.execute(0, t.elems.Len(), &state{parent: nil, data: val, wr: wr})
return nil
}
type S struct {
Header string
+ HeaderPtr *string
Integer int
- Raw string
+ IntegerPtr *int
+ NilPtr *int
InnerT T
InnerPointerT *T
Data []T
JSON interface{}
Innermap U
Stringmap map[string]string
- Bytes []byte
+ Ptrmap map[string]*string
Iface interface{}
Ifaceptr interface{}
}
out: "Header=77\n",
},
+ &Test{
+ in: "Pointers: {*HeaderPtr}={*IntegerPtr}\n",
+
+ out: "Pointers: Header=77\n",
+ },
+
+ &Test{
+ in: "Stars but not pointers: {*Header}={*Integer}\n",
+
+ out: "Stars but not pointers: Header=77\n",
+ },
+
+ &Test{
+ in: "nil pointer: {*NilPtr}={*Integer}\n",
+
+ out: "nil pointer: <nil>=77\n",
+ },
+
// Method at top level
&Test{
in: "ptrmethod={PointerMethod}\n",
out: "ItemNumber1=ValueNumber1\n",
},
-
- // Formatters
- &Test{
- in: "{.section Pdata }\n" +
- "{Header|uppercase}={Integer|+1}\n" +
- "{Header|html}={Integer|str}\n" +
- "{.end}\n",
-
- out: "HEADER=78\n" +
- "Header=77\n",
- },
-
- &Test{
- in: "{.section Pdata }\n" +
- "{Header|uppercase}={Integer Header|multiword}\n" +
- "{Header|html}={Header Integer|multiword}\n" +
- "{Header|html}={Header Integer}\n" +
- "{.end}\n",
-
- out: "HEADER=<77><Header>\n" +
- "Header=<Header><77>\n" +
- "Header=Header77\n",
- },
-
- &Test{
- in: "{Raw}\n" +
- "{Raw|html}\n",
-
- out: "&<>!@ #$%^\n" +
- "&<>!@ #$%^\n",
- },
-
&Test{
in: "{.section Emptystring}emptystring{.end}\n" +
"{.section Header}header{.end}\n",
out: "1\n4\n",
},
- &Test{
- in: "{Bytes}",
-
- out: "hello",
- },
-
// Maps
&Test{
out: "\tstringresult\n" +
"\tstringresult\n",
},
+ &Test{
+ in: "{*Ptrmap.stringkey1}\n",
+
+ out: "pointedToString\n",
+ },
+ &Test{
+ in: "{.repeated section Ptrmap}\n" +
+ "{*@}\n" +
+ "{.end}",
+
+ out: "pointedToString\n" +
+ "pointedToString\n",
+ },
+
// Interface values
s := new(S)
// initialized by hand for clarity.
s.Header = "Header"
+ s.HeaderPtr = &s.Header
s.Integer = 77
- s.Raw = "&<>!@ #$%^"
+ s.IntegerPtr = &s.Integer
s.InnerT = t1
s.Data = []T{t1, t2}
s.Pdata = []*T{&t1, &t2}
s.Stringmap = make(map[string]string)
s.Stringmap["stringkey1"] = "stringresult" // the same value so repeated section is order-independent
s.Stringmap["stringkey2"] = "stringresult"
- s.Bytes = []byte("hello")
+ s.Ptrmap = make(map[string]*string)
+ x := "pointedToString"
+ s.Ptrmap["stringkey1"] = &x // the same value so repeated section is order-independent
+ s.Ptrmap["stringkey2"] = &x
s.Iface = []int{1, 2, 3}
s.Ifaceptr = &T{"Item", "Value"}
t.Error("unexpected parse error: ", err)
continue
}
- err = tmpl.Execute(s, &buf)
+ err = tmpl.Execute(&buf, s)
if test.err == "" {
if err != nil {
t.Error("unexpected execute error:", err)
t.Error("unexpected parse error:", err)
}
var b bytes.Buffer
- err = tmpl.Execute(mp, &b)
+ err = tmpl.Execute(&b, mp)
if err != nil {
t.Error("unexpected execute error:", err)
}
s := b.String()
- expected := "template: Ahoy!"
- if s != expected {
- t.Errorf("failed passing string as data: expected %q got %q", "template: Ahoy!", s)
+ expect := "template: Ahoy!"
+ if s != expect {
+ t.Errorf("failed passing string as data: expected %q got %q", expect, s)
+ }
+}
+
+func TestMapNoEntry(t *testing.T) {
+ mp := make(map[string]int)
+ tmpl, err := Parse("template: {notthere}!", nil)
+ if err != nil {
+ t.Error("unexpected parse error:", err)
+ }
+ var b bytes.Buffer
+ err = tmpl.Execute(&b, mp)
+ if err != nil {
+ t.Error("unexpected execute error:", err)
+ }
+ s := b.String()
+ expect := "template: 0!"
+ if s != expect {
+ t.Errorf("failed passing string as data: expected %q got %q", expect, s)
}
}
t.Error("unexpected parse error:", err)
}
var b bytes.Buffer
- err = tmpl.Execute("hello", &b)
+ err = tmpl.Execute(&b, "hello")
if err != nil {
t.Error("unexpected execute error:", err)
}
s := b.String()
- if s != "template: hello" {
- t.Errorf("failed passing string as data: expected %q got %q", "template: hello", s)
+ expect := "template: hello"
+ if s != expect {
+ t.Errorf("failed passing string as data: expected %q got %q", expect, s)
}
}
t.Error("unexpected parse error:", err)
}
var b bytes.Buffer
- err = tmpl.Execute("hello", &b)
+ err = tmpl.Execute(&b, "hello")
if err != nil {
t.Error("unexpected parse error:", err)
}
s := b.String()
- text := "template: hello"
- if s != text {
- t.Errorf("failed passing string as data: expected %q got %q", text, s)
+ expect := "template: hello"
+ if s != expect {
+ t.Errorf("failed passing string as data: expected %q got %q", expect, s)
}
- err = tmpl.Execute("hello", &b)
+ err = tmpl.Execute(&b, "hello")
if err != nil {
t.Error("unexpected parse error:", err)
}
s = b.String()
- text += text
- if s != text {
- t.Errorf("failed passing string as data: expected %q got %q", text, s)
+ expect += expect
+ if s != expect {
+ t.Errorf("failed passing string as data: expected %q got %q", expect, s)
}
}
continue
}
var b bytes.Buffer
- err = tmpl.Execute("hello", &b)
+ err = tmpl.Execute(&b, "hello")
s := b.String()
if s != "template: hello"+ldelim+rdelim {
t.Errorf("failed delim check(%q %q) %q got %q", ldelim, rdelim, text, s)
if err != nil {
t.Fatal("unexpected parse error:", err)
}
- err = tmpl.Execute(s, &buf)
+ err = tmpl.Execute(&buf, s)
if err != nil {
t.Fatal("unexpected execute error:", err)
}
if err != nil {
t.Fatal("unexpected parse error:", err)
}
- err = tmpl.Execute(u, &buf)
+ err = tmpl.Execute(&buf, u)
if err == nil {
t.Fatal("expected execute error, got none")
}
t.Fatal("expected unexported error; got", err)
}
}
+
+var formatterTests = []Test{
+ {
+ in: "{Header|uppercase}={Integer|+1}\n" +
+ "{Header|html}={Integer|str}\n",
+
+ out: "HEADER=78\n" +
+ "Header=77\n",
+ },
+
+ {
+ in: "{Header|uppercase}={Integer Header|multiword}\n" +
+ "{Header|html}={Header Integer|multiword}\n" +
+ "{Header|html}={Header Integer}\n",
+
+ out: "HEADER=<77><Header>\n" +
+ "Header=<Header><77>\n" +
+ "Header=Header77\n",
+ },
+ {
+ in: "{Raw}\n" +
+ "{Raw|html}\n",
+
+ out: "a <&> b\n" +
+ "a <&> b\n",
+ },
+ {
+ in: "{Bytes}",
+ out: "hello",
+ },
+ {
+ in: "{Raw|uppercase|html|html}",
+ out: "A &lt;&amp;&gt; B",
+ },
+ {
+ in: "{Header Integer|multiword|html}",
+ out: "<Header><77>",
+ },
+ {
+ in: "{Integer|no_formatter|html}",
+ err: `unknown formatter: "no_formatter"`,
+ },
+ {
+ in: "{Integer|||||}", // empty string is a valid formatter
+ out: "77",
+ },
+}
+
+func TestFormatters(t *testing.T) {
+ data := map[string]interface{}{
+ "Header": "Header",
+ "Integer": 77,
+ "Raw": "a <&> b",
+ "Bytes": []byte("hello"),
+ }
+ for _, c := range formatterTests {
+ tmpl, err := Parse(c.in, formatters)
+ if err != nil {
+ if c.err == "" {
+ t.Error("unexpected parse error:", err)
+ continue
+ }
+ if strings.Index(err.String(), c.err) < 0 {
+ t.Errorf("unexpected error: expected %q, got %q", c.err, err.String())
+ continue
+ }
+ } else {
+ if c.err != "" {
+ t.Errorf("For %q, expected error, got none.", c.in)
+ continue
+ }
+ buf := bytes.NewBuffer(nil)
+ err = tmpl.Execute(buf, data)
+ if err != nil {
+ t.Error("unexpected Execute error: ", err)
+ continue
+ }
+ actual := buf.String()
+ if actual != c.out {
+ t.Errorf("for %q: expected %q but got %q.", c.in, c.out, actual)
+ }
+ }
+ }
+}
"flag"
"fmt"
"os"
+ "runtime"
"time"
)
-var matchBenchmarks = flag.String("benchmarks", "", "regular expression to select benchmarks to run")
+var matchBenchmarks = flag.String("test.bench", "", "regular expression to select benchmarks to run")
// An internal type but exported because it is cross-package; part of the implementation
// of gotest.
// runN runs a single benchmark for the specified number of iterations.
func (b *B) runN(n int) {
+ // Try to get a comparable environment for each run
+ // by clearing garbage from previous runs.
+ runtime.GC()
b.N = n
b.ResetTimer()
b.StartTimer()
for _, Benchmark := range benchmarks {
matched, err := matchString(*matchBenchmarks, Benchmark.Name)
if err != nil {
- println("invalid regexp for -benchmarks:", err.String())
+ println("invalid regexp for -test.bench:", err.String())
os.Exit(1)
}
if !matched {
//
// Functions of the form
// func BenchmarkXxx(*testing.B)
-// are considered benchmarks, and are executed by gotest when the -benchmarks
+// are considered benchmarks, and are executed by gotest when the -test.bench
// flag is provided.
//
// A sample benchmark function looks like this:
"fmt"
"os"
"runtime"
+ "time"
)
// Report as tests are run; default is silent for success.
-var chatty = flag.Bool("v", false, "verbose: print additional output")
-var match = flag.String("match", "", "regular expression to select tests to run")
+var chatty = flag.Bool("test.v", false, "verbose: print additional output")
+var match = flag.String("test.run", "", "regular expression to select tests to run")
// Insert final newline if needed and tabs after internal newlines.
// and records the text in the error log.
func (t *T) Log(args ...interface{}) { t.errors += "\t" + tabify(fmt.Sprintln(args...)) }
-// Log formats its arguments according to the format, analogous to Printf(),
+// Logf formats its arguments according to the format, analogous to Printf(),
// and records the text in the error log.
func (t *T) Logf(format string, args ...interface{}) {
t.errors += "\t" + tabify(fmt.Sprintf(format, args...))
for i := 0; i < len(tests); i++ {
matched, err := matchString(*match, tests[i].Name)
if err != nil {
- println("invalid regexp for -match:", err.String())
+ println("invalid regexp for -test.run:", err.String())
os.Exit(1)
}
if !matched {
if *chatty {
println("=== RUN ", tests[i].Name)
}
+ ns := -time.Nanoseconds()
t := new(T)
t.ch = make(chan *T)
go tRunner(t, &tests[i])
<-t.ch
+ ns += time.Nanoseconds()
+ tstr := fmt.Sprintf("(%.1f seconds)", float64(ns)/1e9)
if t.failed {
- println("--- FAIL:", tests[i].Name)
+ println("--- FAIL:", tests[i].Name, tstr)
print(t.errors)
ok = false
} else if *chatty {
- println("--- PASS:", tests[i].Name)
+ println("--- PASS:", tests[i].Name, tstr)
print(t.errors)
}
}
"container/heap"
)
-// The event type represents a single After or AfterFunc event.
-type event struct {
- t int64 // The absolute time that the event should fire.
- f func(int64) // The function to call when the event fires.
- sleeping bool // A sleeper is sleeping for this event.
+// The Timer type represents a single event.
+// When the Timer expires, the current time will be sent on C
+// unless the Timer represents an AfterFunc event.
+type Timer struct {
+ C <-chan int64
+ t int64 // The absolute time that the event should fire.
+ f func(int64) // The function to call when the event fires.
+ i int // The event's index inside eventHeap.
}
-type eventHeap []*event
+type timerHeap []*Timer
-var events eventHeap
-var eventMutex sync.Mutex
+// forever is the absolute time (in ns) of an event that is forever away.
+const forever = 1 << 62
+
+// maxSleepTime is the maximum length of time that a sleeper
+// sleeps for before checking if it is defunct.
+const maxSleepTime = 1e9
+
+var (
+ // timerMutex guards the variables inside this var group.
+ timerMutex sync.Mutex
+
+ // timers holds a binary heap of pending events, terminated with a sentinel.
+ timers timerHeap
+
+ // currentSleeper is an ever-incrementing counter which represents
+ // the current sleeper. It allows older sleepers to detect that they are
+ // defunct and exit.
+ currentSleeper int64
+)
func init() {
- events.Push(&event{1 << 62, nil, true}) // sentinel
+ timers.Push(&Timer{t: forever}) // sentinel
}
// Sleep pauses the current goroutine for at least ns nanoseconds.
return t, nil
}
+// NewTimer creates a new Timer that will send
+// the current time on its channel after at least ns nanoseconds.
+func NewTimer(ns int64) *Timer {
+ c := make(chan int64, 1)
+ e := after(ns, func(t int64) { c <- t })
+ e.C = c
+ return e
+}
+
// After waits at least ns nanoseconds before sending the current time
// on the returned channel.
+// It is equivalent to NewTimer(ns).C.
func After(ns int64) <-chan int64 {
- c := make(chan int64, 1)
- after(ns, func(t int64) { c <- t })
- return c
+ return NewTimer(ns).C
}
// AfterFunc waits at least ns nanoseconds before calling f
-// in its own goroutine.
-func AfterFunc(ns int64, f func()) {
- after(ns, func(_ int64) {
+// in its own goroutine. It returns a Timer that can
+// be used to cancel the call using its Stop method.
+func AfterFunc(ns int64, f func()) *Timer {
+ return after(ns, func(_ int64) {
go f()
})
}
+// Stop prevents the Timer from firing.
+// It returns true if the call stops the timer, false if the timer has already
+// expired or stopped.
+func (e *Timer) Stop() (ok bool) {
+ timerMutex.Lock()
+ // Avoid removing the first event in the queue so that
+ // we don't start a new sleeper unnecessarily.
+ if e.i > 0 {
+ heap.Remove(timers, e.i)
+ }
+ ok = e.f != nil
+ e.f = nil
+ timerMutex.Unlock()
+ return
+}
+
// after is the implementation of After and AfterFunc.
// When the current time is after ns, it calls f with the current time.
// It assumes that f will not block.
-func after(ns int64, f func(int64)) {
+func after(ns int64, f func(int64)) (e *Timer) {
+ now := Nanoseconds()
t := Nanoseconds() + ns
- eventMutex.Lock()
- t0 := events[0].t
- heap.Push(events, &event{t, f, false})
- if t < t0 {
- go sleeper()
+ if ns > 0 && t < now {
+ panic("time: time overflow")
}
- eventMutex.Unlock()
+ timerMutex.Lock()
+ t0 := timers[0].t
+ e = &Timer{nil, t, f, -1}
+ heap.Push(timers, e)
+ // Start a new sleeper if the new event is before
+ // the first event in the queue. If the length of time
+ // until the new event is at least maxSleepTime,
+ // then we're guaranteed that the sleeper will wake up
+ // in time to service it, so no new sleeper is needed.
+ if t0 > t && (t0 == forever || ns < maxSleepTime) {
+ currentSleeper++
+ go sleeper(currentSleeper)
+ }
+ timerMutex.Unlock()
+ return
}
-// sleeper continually looks at the earliest event in the queue, marks it
-// as sleeping, waits until it happens, then removes any events
-// in the queue that are due. It stops when it finds an event that is
-// already marked as sleeping. When an event is inserted before the first item,
-// a new sleeper is started.
-//
-// Scheduling vagaries mean that sleepers may not wake up in
-// exactly the order of the events that they are waiting for,
-// but this does not matter as long as there are at least as
-// many sleepers as events marked sleeping (invariant). This ensures that
-// there is always a sleeper to service the remaining events.
-//
-// A sleeper will remove at least the event it has been waiting for
-// unless the event has already been removed by another sleeper. Both
-// cases preserve the invariant described above.
-func sleeper() {
- eventMutex.Lock()
- e := events[0]
- for !e.sleeping {
- t := Nanoseconds()
+// sleeper continually looks at the earliest event in the queue, waits until it happens,
+// then removes any events in the queue that are due. It stops when the queue
+// is empty or when another sleeper has been started.
+func sleeper(sleeperId int64) {
+ timerMutex.Lock()
+ e := timers[0]
+ t := Nanoseconds()
+ for e.t != forever {
if dt := e.t - t; dt > 0 {
- e.sleeping = true
- eventMutex.Unlock()
- if nt, err := sleep(t, dt); err != nil {
- // If sleep has encountered an error,
- // there's not much we can do. We pretend
- // that time really has advanced by the required
- // amount and lie to the rest of the system.
- t = e.t
- } else {
- t = nt
+ if dt > maxSleepTime {
+ dt = maxSleepTime
+ }
+ timerMutex.Unlock()
+ syscall.Sleep(dt)
+ timerMutex.Lock()
+ if currentSleeper != sleeperId {
+ // Another sleeper has been started, making this one redundant.
+ break
}
- eventMutex.Lock()
- e = events[0]
}
+ e = timers[0]
+ t = Nanoseconds()
for t >= e.t {
- e.f(t)
- heap.Pop(events)
- e = events[0]
+ if e.f != nil {
+ e.f(t)
+ e.f = nil
+ }
+ heap.Pop(timers)
+ e = timers[0]
}
}
- eventMutex.Unlock()
+ timerMutex.Unlock()
}
-func (eventHeap) Len() int {
- return len(events)
+func (timerHeap) Len() int {
+ return len(timers)
}
-func (eventHeap) Less(i, j int) bool {
- return events[i].t < events[j].t
+func (timerHeap) Less(i, j int) bool {
+ return timers[i].t < timers[j].t
}
-func (eventHeap) Swap(i, j int) {
- events[i], events[j] = events[j], events[i]
+func (timerHeap) Swap(i, j int) {
+ timers[i], timers[j] = timers[j], timers[i]
+ timers[i].i = i
+ timers[j].i = j
}
-func (eventHeap) Push(x interface{}) {
- events = append(events, x.(*event))
+func (timerHeap) Push(x interface{}) {
+ e := x.(*Timer)
+ e.i = len(timers)
+ timers = append(timers, e)
}
-func (eventHeap) Pop() interface{} {
+func (timerHeap) Pop() interface{} {
// TODO: possibly shrink array.
- n := len(events) - 1
- e := events[n]
- events[n] = nil
- events = events[0:n]
+ n := len(timers) - 1
+ e := timers[n]
+ timers[n] = nil
+ timers = timers[0:n]
+ e.i = -1
return e
}
<-c
}
+func BenchmarkAfter(b *testing.B) {
+ for i := 0; i < b.N; i++ {
+ <-After(1)
+ }
+}
+
+func BenchmarkStop(b *testing.B) {
+ for i := 0; i < b.N; i++ {
+ NewTimer(1e9).Stop()
+ }
+}
+
func TestAfter(t *testing.T) {
const delay = int64(100e6)
start := Nanoseconds()
}
}
+func TestAfterStop(t *testing.T) {
+ const msec = 1e6
+ AfterFunc(100*msec, func() {})
+ t0 := NewTimer(50 * msec)
+ c1 := make(chan bool, 1)
+ t1 := AfterFunc(150*msec, func() { c1 <- true })
+ c2 := After(200 * msec)
+ if !t0.Stop() {
+ t.Fatalf("failed to stop event 0")
+ }
+ if !t1.Stop() {
+ t.Fatalf("failed to stop event 1")
+ }
+ <-c2
+ select {
+ case <-t0.C:
+ t.Fatalf("event 0 was not stopped")
+ case <-c1:
+ t.Fatalf("event 1 was not stopped")
+ default:
+ }
+ if t1.Stop() {
+ t.Fatalf("Stop returned true twice")
+ }
+}
+
var slots = []int{5, 3, 6, 6, 6, 1, 1, 2, 7, 9, 4, 8, 0}
type afterResult struct {
// Stop turns off a ticker. After Stop, no more ticks will be sent.
func (t *Ticker) Stop() {
- // Make it non-blocking so multiple Stops don't block.
- _ = t.shutdown <- true
+ select {
+ case t.shutdown <- true:
+ // ok
+ default:
+ // Stop in progress already
+ }
}
// Tick is a convenience wrapper for NewTicker providing access to the ticking
// that need it and determining the next wake time.
// TODO(r): list should be sorted in time order.
for t := tickers; t != nil; t = t.next {
- if _, ok := <-t.shutdown; ok {
+ select {
+ case <-t.shutdown:
// Ticker is done; remove it from list.
if prev == nil {
tickers = t.next
prev.next = t.next
}
continue
+ default:
}
if t.nextTick <= now {
if len(t.c) == 0 {
}
// Now test that the ticker stopped
Sleep(2 * Delta)
- _, received := <-ticker.C
- if received {
+ select {
+ case <-ticker.C:
t.Fatal("Ticker did not shut down")
+ default:
+ // ok
}
}
ticker.Stop()
}
}
+
+func BenchmarkTicker(b *testing.B) {
+ ticker := NewTicker(1)
+ b.ResetTimer()
+ b.StartTimer()
+ for i := 0; i < b.N; i++ {
+ <-ticker.C
+ }
+ b.StopTimer()
+ ticker.Stop()
+}
0xf9,
0x2ec,
0x535,
+ 0x620,
0x6e6,
0x93d,
0xa15,
0x20,
0x35,
0x375,
- 0x620,
+ 0x619,
0x700,
0xfffe,
0x1ffff,
script string
}
-// Hand-chosen tests from Unicode 5.1.0, mostly to discover when new
+// Hand-chosen tests from Unicode 5.1.0 & 6.0..0, mostly to discover when new
// scripts and categories arise.
var inTest = []T{
{0x06e2, "Arabic"},
{0x10b20, "Avestan"},
{0x1b37, "Balinese"},
{0xa6af, "Bamum"},
+ {0x1be1, "Batak"},
{0x09c2, "Bengali"},
{0x3115, "Bopomofo"},
{0x282d, "Braille"},
{0x1a1a, "Buginese"},
{0x1747, "Buhid"},
+ {0x11011, "Brahmi"},
{0x156d, "Canadian_Aboriginal"},
{0x102a9, "Carian"},
{0xaa4d, "Cham"},
{0x10290, "Lycian"},
{0x10930, "Lydian"},
{0x0d42, "Malayalam"},
+ {0x0843, "Mandaic"},
{0xabd0, "Meetei_Mayek"},
{0x1822, "Mongolian"},
{0x104c, "Myanmar"},
}
}
for k := range notTested {
- t.Error("not tested:", k)
+ t.Error("script not tested:", k)
}
}
notTested[test.script] = false, false
}
for k := range notTested {
- t.Error("not tested:", k)
+ t.Error("category not tested:", k)
}
}
notTested[test.script] = false, false
}
for k := range notTested {
- t.Error("not tested:", k)
+ t.Error("property not tested:", k)
}
}
// Generated by running
-// maketables --tables=all --data=http://www.unicode.org/Public/5.2.0/ucd/UnicodeData.txt
+// maketables --tables=all --data=http://www.unicode.org/Public/6.0.0/ucd/UnicodeData.txt
// DO NOT EDIT
package unicode
// Version is the Unicode edition from which the tables are derived.
-const Version = "5.2.0"
+const Version = "6.0.0"
// Categories is the set of Unicode data tables.
var Categories = map[string][]Range{
{0x1d78, 0x1d9b, 35},
{0x1d9c, 0x1dbf, 1},
{0x2071, 0x207f, 14},
- {0x2090, 0x2094, 1},
+ {0x2090, 0x209c, 1},
{0x2c7d, 0x2d6f, 242},
{0x2e2f, 0x3005, 470},
{0x3031, 0x3035, 1},
{0x0461, 0x0481, 2},
{0x048b, 0x04bf, 2},
{0x04c2, 0x04ce, 2},
- {0x04cf, 0x0525, 2},
+ {0x04cf, 0x0527, 2},
{0x0561, 0x0587, 1},
{0x1d00, 0x1d2b, 1},
{0x1d62, 0x1d77, 1},
{0x2ce4, 0x2cec, 8},
{0x2cee, 0x2d00, 18},
{0x2d01, 0x2d25, 1},
- {0xa641, 0xa65f, 2},
- {0xa663, 0xa66d, 2},
+ {0xa641, 0xa66d, 2},
{0xa681, 0xa697, 2},
{0xa723, 0xa72f, 2},
{0xa730, 0xa731, 1},
{0xa772, 0xa778, 1},
{0xa77a, 0xa77c, 2},
{0xa77f, 0xa787, 2},
- {0xa78c, 0xfb00, 21364},
+ {0xa78c, 0xa78e, 2},
+ {0xa791, 0xa7a1, 16},
+ {0xa7a3, 0xa7a9, 2},
+ {0xa7fa, 0xfb00, 21254},
{0xfb01, 0xfb06, 1},
{0xfb13, 0xfb17, 1},
{0xff41, 0xff5a, 1},
var _Me = []Range{
{0x0488, 0x0489, 1},
- {0x06de, 0x20dd, 6655},
- {0x20de, 0x20e0, 1},
+ {0x20dd, 0x20e0, 1},
{0x20e2, 0x20e4, 1},
{0xa670, 0xa672, 1},
}
var _Mc = []Range{
- {0x0903, 0x093e, 59},
- {0x093f, 0x0940, 1},
+ {0x0903, 0x093b, 56},
+ {0x093e, 0x0940, 1},
{0x0949, 0x094c, 1},
- {0x094e, 0x0982, 52},
- {0x0983, 0x09be, 59},
- {0x09bf, 0x09c0, 1},
+ {0x094e, 0x094f, 1},
+ {0x0982, 0x0983, 1},
+ {0x09be, 0x09c0, 1},
{0x09c7, 0x09c8, 1},
{0x09cb, 0x09cc, 1},
{0x09d7, 0x0a03, 44},
{0x1b43, 0x1b44, 1},
{0x1b82, 0x1ba1, 31},
{0x1ba6, 0x1ba7, 1},
- {0x1baa, 0x1c24, 122},
+ {0x1baa, 0x1be7, 61},
+ {0x1bea, 0x1bec, 1},
+ {0x1bee, 0x1bf2, 4},
+ {0x1bf3, 0x1c24, 49},
{0x1c25, 0x1c2b, 1},
{0x1c34, 0x1c35, 1},
{0x1ce1, 0x1cf2, 17},
{0xabe3, 0xabe4, 1},
{0xabe6, 0xabe7, 1},
{0xabe9, 0xabea, 1},
- {0xabec, 0x11082, 25750},
+ {0xabec, 0x11000, 25620},
+ {0x11002, 0x11082, 128},
{0x110b0, 0x110b2, 1},
{0x110b7, 0x110b8, 1},
{0x1d165, 0x1d166, 1},
{0x05c2, 0x05c4, 2},
{0x05c5, 0x05c7, 2},
{0x0610, 0x061a, 1},
- {0x064b, 0x065e, 1},
+ {0x064b, 0x065f, 1},
{0x0670, 0x06d6, 102},
{0x06d7, 0x06dc, 1},
{0x06df, 0x06e4, 1},
{0x081b, 0x0823, 1},
{0x0825, 0x0827, 1},
{0x0829, 0x082d, 1},
+ {0x0859, 0x085b, 1},
{0x0900, 0x0902, 1},
- {0x093c, 0x0941, 5},
- {0x0942, 0x0948, 1},
+ {0x093a, 0x093c, 2},
+ {0x0941, 0x0948, 1},
{0x094d, 0x0951, 4},
- {0x0952, 0x0955, 1},
+ {0x0952, 0x0957, 1},
{0x0962, 0x0963, 1},
{0x0981, 0x09bc, 59},
{0x09c1, 0x09c4, 1},
{0x0f71, 0x0f7e, 1},
{0x0f80, 0x0f84, 1},
{0x0f86, 0x0f87, 1},
- {0x0f90, 0x0f97, 1},
+ {0x0f8d, 0x0f97, 1},
{0x0f99, 0x0fbc, 1},
{0x0fc6, 0x102d, 103},
{0x102e, 0x1030, 1},
{0x1071, 0x1074, 1},
{0x1082, 0x1085, 3},
{0x1086, 0x108d, 7},
- {0x109d, 0x135f, 706},
+ {0x109d, 0x135d, 704},
+ {0x135e, 0x135f, 1},
{0x1712, 0x1714, 1},
{0x1732, 0x1734, 1},
{0x1752, 0x1753, 1},
{0x1b80, 0x1b81, 1},
{0x1ba2, 0x1ba5, 1},
{0x1ba8, 0x1ba9, 1},
+ {0x1be6, 0x1be8, 2},
+ {0x1be9, 0x1bed, 4},
+ {0x1bef, 0x1bf1, 1},
{0x1c2c, 0x1c33, 1},
{0x1c36, 0x1c37, 1},
{0x1cd0, 0x1cd2, 1},
{0x1ce2, 0x1ce8, 1},
{0x1ced, 0x1dc0, 211},
{0x1dc1, 0x1de6, 1},
- {0x1dfd, 0x1dff, 1},
+ {0x1dfc, 0x1dff, 1},
{0x20d0, 0x20dc, 1},
{0x20e1, 0x20e5, 4},
{0x20e6, 0x20f0, 1},
{0x2cef, 0x2cf1, 1},
- {0x2de0, 0x2dff, 1},
+ {0x2d7f, 0x2de0, 97},
+ {0x2de1, 0x2dff, 1},
{0x302a, 0x302f, 1},
{0x3099, 0x309a, 1},
{0xa66f, 0xa67c, 13},
{0x10a05, 0x10a06, 1},
{0x10a0c, 0x10a0f, 1},
{0x10a38, 0x10a3a, 1},
- {0x10a3f, 0x11080, 1601},
- {0x11081, 0x110b3, 50},
- {0x110b4, 0x110b6, 1},
+ {0x10a3f, 0x11001, 1474},
+ {0x11038, 0x11046, 1},
+ {0x11080, 0x11081, 1},
+ {0x110b3, 0x110b6, 1},
{0x110b9, 0x110ba, 1},
{0x1d167, 0x1d169, 1},
{0x1d17b, 0x1d182, 1},
{0x038f, 0x03a1, 1},
{0x03a3, 0x03f5, 1},
{0x03f7, 0x0481, 1},
- {0x048a, 0x0525, 1},
+ {0x048a, 0x0527, 1},
{0x0531, 0x0556, 1},
{0x0559, 0x0561, 8},
{0x0562, 0x0587, 1},
{0x05d0, 0x05ea, 1},
{0x05f0, 0x05f2, 1},
- {0x0621, 0x064a, 1},
+ {0x0620, 0x064a, 1},
{0x066e, 0x066f, 1},
{0x0671, 0x06d3, 1},
{0x06d5, 0x06e5, 16},
{0x07fa, 0x0800, 6},
{0x0801, 0x0815, 1},
{0x081a, 0x0824, 10},
- {0x0828, 0x0904, 220},
- {0x0905, 0x0939, 1},
+ {0x0828, 0x0840, 24},
+ {0x0841, 0x0858, 1},
+ {0x0904, 0x0939, 1},
{0x093d, 0x0950, 19},
{0x0958, 0x0961, 1},
- {0x0971, 0x0972, 1},
+ {0x0971, 0x0977, 1},
{0x0979, 0x097f, 1},
{0x0985, 0x098c, 1},
{0x098f, 0x0990, 1},
{0x0cb5, 0x0cb9, 1},
{0x0cbd, 0x0cde, 33},
{0x0ce0, 0x0ce1, 1},
+ {0x0cf1, 0x0cf2, 1},
{0x0d05, 0x0d0c, 1},
{0x0d0e, 0x0d10, 1},
- {0x0d12, 0x0d28, 1},
- {0x0d2a, 0x0d39, 1},
- {0x0d3d, 0x0d60, 35},
- {0x0d61, 0x0d7a, 25},
- {0x0d7b, 0x0d7f, 1},
+ {0x0d12, 0x0d3a, 1},
+ {0x0d3d, 0x0d4e, 17},
+ {0x0d60, 0x0d61, 1},
+ {0x0d7a, 0x0d7f, 1},
{0x0d85, 0x0d96, 1},
{0x0d9a, 0x0db1, 1},
{0x0db3, 0x0dbb, 1},
{0x0edd, 0x0f00, 35},
{0x0f40, 0x0f47, 1},
{0x0f49, 0x0f6c, 1},
- {0x0f88, 0x0f8b, 1},
+ {0x0f88, 0x0f8c, 1},
{0x1000, 0x102a, 1},
{0x103f, 0x1050, 17},
{0x1051, 0x1055, 1},
{0x1b45, 0x1b4b, 1},
{0x1b83, 0x1ba0, 1},
{0x1bae, 0x1baf, 1},
+ {0x1bc0, 0x1be5, 1},
{0x1c00, 0x1c23, 1},
{0x1c4d, 0x1c4f, 1},
{0x1c5a, 0x1c7d, 1},
{0x1ff2, 0x1ff4, 1},
{0x1ff6, 0x1ffc, 1},
{0x2071, 0x207f, 14},
- {0x2090, 0x2094, 1},
+ {0x2090, 0x209c, 1},
{0x2102, 0x2107, 5},
{0x210a, 0x2113, 1},
{0x2115, 0x2119, 4},
{0x30fc, 0x30ff, 1},
{0x3105, 0x312d, 1},
{0x3131, 0x318e, 1},
- {0x31a0, 0x31b7, 1},
+ {0x31a0, 0x31ba, 1},
{0x31f0, 0x31ff, 1},
{0x3400, 0x4db5, 1},
{0x4e00, 0x9fcb, 1},
{0xa500, 0xa60c, 1},
{0xa610, 0xa61f, 1},
{0xa62a, 0xa62b, 1},
- {0xa640, 0xa65f, 1},
- {0xa662, 0xa66e, 1},
+ {0xa640, 0xa66e, 1},
{0xa67f, 0xa697, 1},
{0xa6a0, 0xa6e5, 1},
{0xa717, 0xa71f, 1},
{0xa722, 0xa788, 1},
- {0xa78b, 0xa78c, 1},
- {0xa7fb, 0xa801, 1},
+ {0xa78b, 0xa78e, 1},
+ {0xa790, 0xa791, 1},
+ {0xa7a0, 0xa7a9, 1},
+ {0xa7fa, 0xa801, 1},
{0xa803, 0xa805, 1},
{0xa807, 0xa80a, 1},
{0xa80c, 0xa822, 1},
{0xaaba, 0xaabd, 1},
{0xaac0, 0xaac2, 2},
{0xaadb, 0xaadd, 1},
+ {0xab01, 0xab06, 1},
+ {0xab09, 0xab0e, 1},
+ {0xab11, 0xab16, 1},
+ {0xab20, 0xab26, 1},
+ {0xab28, 0xab2e, 1},
{0xabc0, 0xabe2, 1},
{0xac00, 0xd7a3, 1},
{0xd7b0, 0xd7c6, 1},
{0x10b40, 0x10b55, 1},
{0x10b60, 0x10b72, 1},
{0x10c00, 0x10c48, 1},
+ {0x11003, 0x11037, 1},
{0x11083, 0x110af, 1},
{0x12000, 0x1236e, 1},
{0x13000, 0x1342e, 1},
+ {0x16800, 0x16a38, 1},
+ {0x1b000, 0x1b001, 1},
{0x1d400, 0x1d454, 1},
{0x1d456, 0x1d49c, 1},
{0x1d49e, 0x1d49f, 1},
{0x1d7c4, 0x1d7cb, 1},
{0x20000, 0x2a6d6, 1},
{0x2a700, 0x2b734, 1},
+ {0x2b740, 0x2b81d, 1},
{0x2f800, 0x2fa1d, 1},
}
{0x0701, 0x070d, 1},
{0x07f7, 0x07f9, 1},
{0x0830, 0x083e, 1},
- {0x0964, 0x0965, 1},
- {0x0970, 0x0df4, 1156},
- {0x0e4f, 0x0e5a, 11},
- {0x0e5b, 0x0f04, 169},
- {0x0f05, 0x0f12, 1},
+ {0x085e, 0x0964, 262},
+ {0x0965, 0x0970, 11},
+ {0x0df4, 0x0e4f, 91},
+ {0x0e5a, 0x0e5b, 1},
+ {0x0f04, 0x0f12, 1},
{0x0f85, 0x0fd0, 75},
{0x0fd1, 0x0fd4, 1},
+ {0x0fd9, 0x0fda, 1},
{0x104a, 0x104f, 1},
{0x10fb, 0x1361, 614},
{0x1362, 0x1368, 1},
{0x1800, 0x1805, 1},
{0x1807, 0x180a, 1},
{0x1944, 0x1945, 1},
- {0x19de, 0x19df, 1},
{0x1a1e, 0x1a1f, 1},
{0x1aa0, 0x1aa6, 1},
{0x1aa8, 0x1aad, 1},
{0x1b5a, 0x1b60, 1},
+ {0x1bfc, 0x1bff, 1},
{0x1c3b, 0x1c3f, 1},
{0x1c7e, 0x1c7f, 1},
{0x1cd3, 0x2016, 835},
{0x2056, 0x205e, 1},
{0x2cf9, 0x2cfc, 1},
{0x2cfe, 0x2cff, 1},
- {0x2e00, 0x2e01, 1},
- {0x2e06, 0x2e08, 1},
+ {0x2d70, 0x2e00, 144},
+ {0x2e01, 0x2e06, 5},
+ {0x2e07, 0x2e08, 1},
{0x2e0b, 0x2e0e, 3},
{0x2e0f, 0x2e16, 1},
{0x2e18, 0x2e19, 1},
{0x10a50, 0x10a58, 1},
{0x10a7f, 0x10b39, 186},
{0x10b3a, 0x10b3f, 1},
+ {0x11047, 0x1104d, 1},
{0x110bb, 0x110bc, 1},
{0x110be, 0x110c1, 1},
{0x12470, 0x12473, 1},
{0x17e0, 0x17e9, 1},
{0x1810, 0x1819, 1},
{0x1946, 0x194f, 1},
- {0x19d0, 0x19da, 1},
+ {0x19d0, 0x19d9, 1},
{0x1a80, 0x1a89, 1},
{0x1a90, 0x1a99, 1},
{0x1b50, 0x1b59, 1},
{0xabf0, 0xabf9, 1},
{0xff10, 0xff19, 1},
{0x104a0, 0x104a9, 1},
+ {0x11066, 0x1106f, 1},
{0x1d7ce, 0x1d7ff, 1},
}
{0x00b9, 0x00bc, 3},
{0x00bd, 0x00be, 1},
{0x09f4, 0x09f9, 1},
+ {0x0b72, 0x0b77, 1},
{0x0bf0, 0x0bf2, 1},
{0x0c78, 0x0c7e, 1},
{0x0d70, 0x0d75, 1},
{0x0f2a, 0x0f33, 1},
{0x1369, 0x137c, 1},
{0x17f0, 0x17f9, 1},
- {0x2070, 0x2074, 4},
- {0x2075, 0x2079, 1},
+ {0x19da, 0x2070, 1686},
+ {0x2074, 0x2079, 1},
{0x2080, 0x2089, 1},
{0x2150, 0x215f, 1},
{0x2189, 0x2460, 727},
{0x10b58, 0x10b5f, 1},
{0x10b78, 0x10b7f, 1},
{0x10e60, 0x10e7e, 1},
+ {0x11052, 0x11065, 1},
{0x1d360, 0x1d371, 1},
{0x1f100, 0x1f10a, 1},
}
{0x00a9, 0x00ae, 5},
{0x00b0, 0x00b6, 6},
{0x0482, 0x060e, 396},
- {0x060f, 0x06e9, 218},
- {0x06fd, 0x06fe, 1},
- {0x07f6, 0x09fa, 516},
- {0x0b70, 0x0bf3, 131},
- {0x0bf4, 0x0bf8, 1},
+ {0x060f, 0x06de, 207},
+ {0x06e9, 0x06fd, 20},
+ {0x06fe, 0x07f6, 248},
+ {0x09fa, 0x0b70, 374},
+ {0x0bf3, 0x0bf8, 1},
{0x0bfa, 0x0c7f, 133},
- {0x0cf1, 0x0cf2, 1},
{0x0d79, 0x0f01, 392},
{0x0f02, 0x0f03, 1},
{0x0f13, 0x0f17, 1},
{0x109e, 0x109f, 1},
{0x1360, 0x1390, 48},
{0x1391, 0x1399, 1},
- {0x1940, 0x19e0, 160},
- {0x19e1, 0x19ff, 1},
+ {0x1940, 0x19de, 158},
+ {0x19df, 0x19ff, 1},
{0x1b61, 0x1b6a, 1},
{0x1b74, 0x1b7c, 1},
{0x2100, 0x2101, 1},
{0x2103, 0x2106, 1},
{0x2108, 0x2109, 1},
{0x2114, 0x2116, 2},
- {0x2117, 0x2118, 1},
- {0x211e, 0x2123, 1},
+ {0x2117, 0x211e, 7},
+ {0x211f, 0x2123, 1},
{0x2125, 0x2129, 2},
{0x212e, 0x213a, 12},
{0x213b, 0x214a, 15},
{0x232b, 0x237b, 1},
{0x237d, 0x239a, 1},
{0x23b4, 0x23db, 1},
- {0x23e2, 0x23e8, 1},
+ {0x23e2, 0x23f3, 1},
{0x2400, 0x2426, 1},
{0x2440, 0x244a, 1},
{0x249c, 0x24e9, 1},
{0x25b8, 0x25c0, 1},
{0x25c2, 0x25f7, 1},
{0x2600, 0x266e, 1},
- {0x2670, 0x26cd, 1},
- {0x26cf, 0x26e1, 1},
- {0x26e3, 0x26e8, 5},
- {0x26e9, 0x26ff, 1},
- {0x2701, 0x2704, 1},
- {0x2706, 0x2709, 1},
- {0x270c, 0x2727, 1},
- {0x2729, 0x274b, 1},
- {0x274d, 0x274f, 2},
- {0x2750, 0x2752, 1},
- {0x2756, 0x275e, 1},
- {0x2761, 0x2767, 1},
- {0x2794, 0x2798, 4},
- {0x2799, 0x27af, 1},
- {0x27b1, 0x27be, 1},
+ {0x2670, 0x26ff, 1},
+ {0x2701, 0x2767, 1},
+ {0x2794, 0x27bf, 1},
{0x2800, 0x28ff, 1},
{0x2b00, 0x2b2f, 1},
{0x2b45, 0x2b46, 1},
{0x1d301, 0x1d356, 1},
{0x1f000, 0x1f02b, 1},
{0x1f030, 0x1f093, 1},
+ {0x1f0a0, 0x1f0ae, 1},
+ {0x1f0b1, 0x1f0be, 1},
+ {0x1f0c1, 0x1f0cf, 1},
+ {0x1f0d1, 0x1f0df, 1},
{0x1f110, 0x1f12e, 1},
- {0x1f131, 0x1f13d, 12},
- {0x1f13f, 0x1f142, 3},
- {0x1f146, 0x1f14a, 4},
- {0x1f14b, 0x1f14e, 1},
- {0x1f157, 0x1f15f, 8},
- {0x1f179, 0x1f17b, 2},
- {0x1f17c, 0x1f17f, 3},
- {0x1f18a, 0x1f18d, 1},
- {0x1f190, 0x1f200, 112},
- {0x1f210, 0x1f231, 1},
+ {0x1f130, 0x1f169, 1},
+ {0x1f170, 0x1f19a, 1},
+ {0x1f1e6, 0x1f202, 1},
+ {0x1f210, 0x1f23a, 1},
{0x1f240, 0x1f248, 1},
+ {0x1f250, 0x1f251, 1},
+ {0x1f300, 0x1f320, 1},
+ {0x1f330, 0x1f335, 1},
+ {0x1f337, 0x1f37c, 1},
+ {0x1f380, 0x1f393, 1},
+ {0x1f3a0, 0x1f3c4, 1},
+ {0x1f3c6, 0x1f3ca, 1},
+ {0x1f3e0, 0x1f3f0, 1},
+ {0x1f400, 0x1f43e, 1},
+ {0x1f440, 0x1f442, 2},
+ {0x1f443, 0x1f4f7, 1},
+ {0x1f4f9, 0x1f4fc, 1},
+ {0x1f500, 0x1f53d, 1},
+ {0x1f550, 0x1f567, 1},
+ {0x1f5fb, 0x1f5ff, 1},
+ {0x1f601, 0x1f610, 1},
+ {0x1f612, 0x1f614, 1},
+ {0x1f616, 0x1f61c, 2},
+ {0x1f61d, 0x1f61e, 1},
+ {0x1f620, 0x1f625, 1},
+ {0x1f628, 0x1f62b, 1},
+ {0x1f62d, 0x1f630, 3},
+ {0x1f631, 0x1f633, 1},
+ {0x1f635, 0x1f640, 1},
+ {0x1f645, 0x1f64f, 1},
+ {0x1f680, 0x1f6c5, 1},
+ {0x1f700, 0x1f773, 1},
}
var _Sm = []Range{
{0x2044, 0x2052, 14},
{0x207a, 0x207c, 1},
{0x208a, 0x208c, 1},
- {0x2140, 0x2144, 1},
+ {0x2118, 0x2140, 40},
+ {0x2141, 0x2144, 1},
{0x214b, 0x2190, 69},
{0x2191, 0x2194, 1},
{0x219a, 0x219b, 1},
{0x266f, 0x27c0, 337},
{0x27c1, 0x27c4, 1},
{0x27c7, 0x27ca, 1},
- {0x27cc, 0x27d0, 4},
- {0x27d1, 0x27e5, 1},
+ {0x27cc, 0x27ce, 2},
+ {0x27cf, 0x27e5, 1},
{0x27f0, 0x27ff, 1},
{0x2900, 0x2982, 1},
{0x2999, 0x29d7, 1},
{0xa700, 0xa716, 1},
{0xa720, 0xa721, 1},
{0xa789, 0xa78a, 1},
+ {0xfbb2, 0xfbc1, 1},
{0xff3e, 0xff40, 2},
{0xffe3, 0xffe3, 1},
}
{0x09f3, 0x09fb, 8},
{0x0af1, 0x0bf9, 264},
{0x0e3f, 0x17db, 2460},
- {0x20a0, 0x20b8, 1},
+ {0x20a0, 0x20b9, 1},
{0xa838, 0xfdfc, 21956},
{0xfe69, 0xff04, 155},
{0xffe0, 0xffe1, 1},
{0x0460, 0x0480, 2},
{0x048a, 0x04c0, 2},
{0x04c1, 0x04cd, 2},
- {0x04d0, 0x0524, 2},
+ {0x04d0, 0x0526, 2},
{0x0531, 0x0556, 1},
{0x10a0, 0x10c5, 1},
{0x1e00, 0x1e94, 2},
{0x2c7e, 0x2c80, 1},
{0x2c82, 0x2ce2, 2},
{0x2ceb, 0x2ced, 2},
- {0xa640, 0xa65e, 2},
- {0xa662, 0xa66c, 2},
+ {0xa640, 0xa66c, 2},
{0xa680, 0xa696, 2},
{0xa722, 0xa72e, 2},
{0xa732, 0xa76e, 2},
{0xa779, 0xa77d, 2},
{0xa77e, 0xa786, 2},
- {0xa78b, 0xff21, 22422},
- {0xff22, 0xff3a, 1},
+ {0xa78b, 0xa78d, 2},
+ {0xa790, 0xa7a0, 16},
+ {0xa7a2, 0xa7a8, 2},
+ {0xff21, 0xff3a, 1},
{0x10400, 0x10427, 1},
{0x1d400, 0x1d419, 1},
{0x1d434, 0x1d44d, 1},
{0x0294, 0x05d0, 828},
{0x05d1, 0x05ea, 1},
{0x05f0, 0x05f2, 1},
- {0x0621, 0x063f, 1},
+ {0x0620, 0x063f, 1},
{0x0641, 0x064a, 1},
{0x066e, 0x066f, 1},
{0x0671, 0x06d3, 1},
{0x07b1, 0x07ca, 25},
{0x07cb, 0x07ea, 1},
{0x0800, 0x0815, 1},
+ {0x0840, 0x0858, 1},
{0x0904, 0x0939, 1},
{0x093d, 0x0950, 19},
{0x0958, 0x0961, 1},
- {0x0972, 0x0979, 7},
- {0x097a, 0x097f, 1},
+ {0x0972, 0x0977, 1},
+ {0x0979, 0x097f, 1},
{0x0985, 0x098c, 1},
{0x098f, 0x0990, 1},
{0x0993, 0x09a8, 1},
{0x0cb5, 0x0cb9, 1},
{0x0cbd, 0x0cde, 33},
{0x0ce0, 0x0ce1, 1},
+ {0x0cf1, 0x0cf2, 1},
{0x0d05, 0x0d0c, 1},
{0x0d0e, 0x0d10, 1},
- {0x0d12, 0x0d28, 1},
- {0x0d2a, 0x0d39, 1},
- {0x0d3d, 0x0d60, 35},
- {0x0d61, 0x0d7a, 25},
- {0x0d7b, 0x0d7f, 1},
+ {0x0d12, 0x0d3a, 1},
+ {0x0d3d, 0x0d4e, 17},
+ {0x0d60, 0x0d61, 1},
+ {0x0d7a, 0x0d7f, 1},
{0x0d85, 0x0d96, 1},
{0x0d9a, 0x0db1, 1},
{0x0db3, 0x0dbb, 1},
{0x0f00, 0x0f40, 64},
{0x0f41, 0x0f47, 1},
{0x0f49, 0x0f6c, 1},
- {0x0f88, 0x0f8b, 1},
+ {0x0f88, 0x0f8c, 1},
{0x1000, 0x102a, 1},
{0x103f, 0x1050, 17},
{0x1051, 0x1055, 1},
{0x1b45, 0x1b4b, 1},
{0x1b83, 0x1ba0, 1},
{0x1bae, 0x1baf, 1},
+ {0x1bc0, 0x1be5, 1},
{0x1c00, 0x1c23, 1},
{0x1c4d, 0x1c4f, 1},
{0x1c5a, 0x1c77, 1},
{0x30ff, 0x3105, 6},
{0x3106, 0x312d, 1},
{0x3131, 0x318e, 1},
- {0x31a0, 0x31b7, 1},
+ {0x31a0, 0x31ba, 1},
{0x31f0, 0x31ff, 1},
{0x3400, 0x4db5, 1},
{0x4e00, 0x9fcb, 1},
{0xaaba, 0xaabd, 1},
{0xaac0, 0xaac2, 2},
{0xaadb, 0xaadc, 1},
+ {0xab01, 0xab06, 1},
+ {0xab09, 0xab0e, 1},
+ {0xab11, 0xab16, 1},
+ {0xab20, 0xab26, 1},
+ {0xab28, 0xab2e, 1},
{0xabc0, 0xabe2, 1},
{0xac00, 0xd7a3, 1},
{0xd7b0, 0xd7c6, 1},
{0x10b40, 0x10b55, 1},
{0x10b60, 0x10b72, 1},
{0x10c00, 0x10c48, 1},
+ {0x11003, 0x11037, 1},
{0x11083, 0x110af, 1},
{0x12000, 0x1236e, 1},
{0x13000, 0x1342e, 1},
+ {0x16800, 0x16a38, 1},
+ {0x1b000, 0x1b001, 1},
{0x20000, 0x2a6d6, 1},
{0x2a700, 0x2b734, 1},
+ {0x2b740, 0x2b81d, 1},
{0x2f800, 0x2fa1d, 1},
}
)
// Generated by running
-// maketables --scripts=all --url=http://www.unicode.org/Public/5.2.0/ucd/
+// maketables --scripts=all --url=http://www.unicode.org/Public/6.0.0/ucd/
// DO NOT EDIT
// Scripts is the set of Unicode script tables.
"Bopomofo": Bopomofo,
"Imperial_Aramaic": Imperial_Aramaic,
"Kaithi": Kaithi,
+ "Mandaic": Mandaic,
"Old_South_Arabian": Old_South_Arabian,
"Kayah_Li": Kayah_Li,
"New_Tai_Lue": New_Tai_Lue,
"Old_Turkic": Old_Turkic,
"Tamil": Tamil,
"Tagalog": Tagalog,
+ "Brahmi": Brahmi,
"Arabic": Arabic,
"Tagbanwa": Tagbanwa,
"Canadian_Aboriginal": Canadian_Aboriginal,
"Vai": Vai,
"Cherokee": Cherokee,
"Ogham": Ogham,
+ "Batak": Batak,
"Syriac": Syriac,
"Gurmukhi": Gurmukhi,
"Tai_Tham": Tai_Tham,
{0x3300, 0x3357, 1},
{0xff66, 0xff6f, 1},
{0xff71, 0xff9d, 1},
+ {0x1b000, 0x1b000, 1},
}
var _Malayalam = []Range{
{0x0d02, 0x0d03, 1},
{0x0d05, 0x0d0c, 1},
{0x0d0e, 0x0d10, 1},
- {0x0d12, 0x0d28, 1},
- {0x0d2a, 0x0d39, 1},
+ {0x0d12, 0x0d3a, 1},
{0x0d3d, 0x0d44, 1},
{0x0d46, 0x0d48, 1},
- {0x0d4a, 0x0d4d, 1},
+ {0x0d4a, 0x0d4e, 1},
{0x0d57, 0x0d57, 1},
{0x0d60, 0x0d63, 1},
{0x0d66, 0x0d75, 1},
{0x1e00, 0x1eff, 1},
{0x2071, 0x2071, 1},
{0x207f, 0x207f, 1},
- {0x2090, 0x2094, 1},
+ {0x2090, 0x209c, 1},
{0x212a, 0x212b, 1},
{0x2132, 0x2132, 1},
{0x214e, 0x214e, 1},
{0x2160, 0x2188, 1},
{0x2c60, 0x2c7f, 1},
{0xa722, 0xa787, 1},
- {0xa78b, 0xa78c, 1},
- {0xa7fb, 0xa7ff, 1},
+ {0xa78b, 0xa78e, 1},
+ {0xa790, 0xa791, 1},
+ {0xa7a0, 0xa7a9, 1},
+ {0xa7fa, 0xa7ff, 1},
{0xfb00, 0xfb06, 1},
{0xff21, 0xff3a, 1},
{0xff41, 0xff5a, 1},
{0x0300, 0x036f, 1},
{0x0485, 0x0486, 1},
{0x064b, 0x0655, 1},
+ {0x065f, 0x065f, 1},
{0x0670, 0x0670, 1},
{0x0951, 0x0952, 1},
{0x1cd0, 0x1cd2, 1},
{0x1ce2, 0x1ce8, 1},
{0x1ced, 0x1ced, 1},
{0x1dc0, 0x1de6, 1},
- {0x1dfd, 0x1dff, 1},
+ {0x1dfc, 0x1dff, 1},
{0x200c, 0x200d, 1},
{0x20d0, 0x20f0, 1},
- {0x302a, 0x302f, 1},
+ {0x302a, 0x302d, 1},
{0x3099, 0x309a, 1},
{0xfe00, 0xfe0f, 1},
{0xfe20, 0xfe26, 1},
}
var _Bopomofo = []Range{
+ {0x02ea, 0x02eb, 1},
{0x3105, 0x312d, 1},
- {0x31a0, 0x31b7, 1},
+ {0x31a0, 0x31ba, 1},
}
var _Imperial_Aramaic = []Range{
{0x11080, 0x110c1, 1},
}
+var _Mandaic = []Range{
+ {0x0840, 0x085b, 1},
+ {0x085e, 0x085e, 1},
+}
+
var _Old_South_Arabian = []Range{
{0x10a60, 0x10a7f, 1},
}
{0x00d7, 0x00d7, 1},
{0x00f7, 0x00f7, 1},
{0x02b9, 0x02df, 1},
- {0x02e5, 0x02ff, 1},
+ {0x02e5, 0x02e9, 1},
+ {0x02ec, 0x02ff, 1},
{0x0374, 0x0374, 1},
{0x037e, 0x037e, 1},
{0x0385, 0x0385, 1},
{0x0387, 0x0387, 1},
{0x0589, 0x0589, 1},
- {0x0600, 0x0603, 1},
{0x060c, 0x060c, 1},
{0x061b, 0x061b, 1},
{0x061f, 0x061f, 1},
{0x06dd, 0x06dd, 1},
{0x0964, 0x0965, 1},
{0x0970, 0x0970, 1},
- {0x0cf1, 0x0cf2, 1},
{0x0e3f, 0x0e3f, 1},
{0x0fd5, 0x0fd8, 1},
{0x10fb, 0x10fb, 1},
{0x206a, 0x2070, 1},
{0x2074, 0x207e, 1},
{0x2080, 0x208e, 1},
- {0x20a0, 0x20b8, 1},
+ {0x20a0, 0x20b9, 1},
{0x2100, 0x2125, 1},
{0x2127, 0x2129, 1},
{0x212c, 0x2131, 1},
{0x2133, 0x214d, 1},
{0x214f, 0x215f, 1},
{0x2189, 0x2189, 1},
- {0x2190, 0x23e8, 1},
+ {0x2190, 0x23f3, 1},
{0x2400, 0x2426, 1},
{0x2440, 0x244a, 1},
- {0x2460, 0x26cd, 1},
- {0x26cf, 0x26e1, 1},
- {0x26e3, 0x26e3, 1},
- {0x26e8, 0x26ff, 1},
- {0x2701, 0x2704, 1},
- {0x2706, 0x2709, 1},
- {0x270c, 0x2727, 1},
- {0x2729, 0x274b, 1},
- {0x274d, 0x274d, 1},
- {0x274f, 0x2752, 1},
- {0x2756, 0x275e, 1},
- {0x2761, 0x2794, 1},
- {0x2798, 0x27af, 1},
- {0x27b1, 0x27be, 1},
- {0x27c0, 0x27ca, 1},
+ {0x2460, 0x26ff, 1},
+ {0x2701, 0x27ca, 1},
{0x27cc, 0x27cc, 1},
- {0x27d0, 0x27ff, 1},
+ {0x27ce, 0x27ff, 1},
{0x2900, 0x2b4c, 1},
{0x2b50, 0x2b59, 1},
{0x2e00, 0x2e31, 1},
{0x1d7ce, 0x1d7ff, 1},
{0x1f000, 0x1f02b, 1},
{0x1f030, 0x1f093, 1},
+ {0x1f0a0, 0x1f0ae, 1},
+ {0x1f0b1, 0x1f0be, 1},
+ {0x1f0c1, 0x1f0cf, 1},
+ {0x1f0d1, 0x1f0df, 1},
{0x1f100, 0x1f10a, 1},
{0x1f110, 0x1f12e, 1},
- {0x1f131, 0x1f131, 1},
- {0x1f13d, 0x1f13d, 1},
- {0x1f13f, 0x1f13f, 1},
- {0x1f142, 0x1f142, 1},
- {0x1f146, 0x1f146, 1},
- {0x1f14a, 0x1f14e, 1},
- {0x1f157, 0x1f157, 1},
- {0x1f15f, 0x1f15f, 1},
- {0x1f179, 0x1f179, 1},
- {0x1f17b, 0x1f17c, 1},
- {0x1f17f, 0x1f17f, 1},
- {0x1f18a, 0x1f18d, 1},
- {0x1f190, 0x1f190, 1},
- {0x1f210, 0x1f231, 1},
+ {0x1f130, 0x1f169, 1},
+ {0x1f170, 0x1f19a, 1},
+ {0x1f1e6, 0x1f1ff, 1},
+ {0x1f201, 0x1f202, 1},
+ {0x1f210, 0x1f23a, 1},
{0x1f240, 0x1f248, 1},
+ {0x1f250, 0x1f251, 1},
+ {0x1f300, 0x1f320, 1},
+ {0x1f330, 0x1f335, 1},
+ {0x1f337, 0x1f37c, 1},
+ {0x1f380, 0x1f393, 1},
+ {0x1f3a0, 0x1f3c4, 1},
+ {0x1f3c6, 0x1f3ca, 1},
+ {0x1f3e0, 0x1f3f0, 1},
+ {0x1f400, 0x1f43e, 1},
+ {0x1f440, 0x1f440, 1},
+ {0x1f442, 0x1f4f7, 1},
+ {0x1f4f9, 0x1f4fc, 1},
+ {0x1f500, 0x1f53d, 1},
+ {0x1f550, 0x1f567, 1},
+ {0x1f5fb, 0x1f5ff, 1},
+ {0x1f601, 0x1f610, 1},
+ {0x1f612, 0x1f614, 1},
+ {0x1f616, 0x1f616, 1},
+ {0x1f618, 0x1f618, 1},
+ {0x1f61a, 0x1f61a, 1},
+ {0x1f61c, 0x1f61e, 1},
+ {0x1f620, 0x1f625, 1},
+ {0x1f628, 0x1f62b, 1},
+ {0x1f62d, 0x1f62d, 1},
+ {0x1f630, 0x1f633, 1},
+ {0x1f635, 0x1f640, 1},
+ {0x1f645, 0x1f64f, 1},
+ {0x1f680, 0x1f6c5, 1},
+ {0x1f700, 0x1f773, 1},
{0xe0001, 0xe0001, 1},
{0xe0020, 0xe007f, 1},
}
{0x0cde, 0x0cde, 1},
{0x0ce0, 0x0ce3, 1},
{0x0ce6, 0x0cef, 1},
+ {0x0cf1, 0x0cf2, 1},
}
var _Old_Turkic = []Range{
{0x170e, 0x1714, 1},
}
+var _Brahmi = []Range{
+ {0x11000, 0x1104d, 1},
+ {0x11052, 0x1106f, 1},
+}
+
var _Arabic = []Range{
+ {0x0600, 0x0603, 1},
{0x0606, 0x060b, 1},
{0x060d, 0x061a, 1},
{0x061e, 0x061e, 1},
- {0x0621, 0x063f, 1},
+ {0x0620, 0x063f, 1},
{0x0641, 0x064a, 1},
{0x0656, 0x065e, 1},
{0x066a, 0x066f, 1},
{0x0671, 0x06dc, 1},
{0x06de, 0x06ff, 1},
{0x0750, 0x077f, 1},
- {0xfb50, 0xfbb1, 1},
+ {0xfb50, 0xfbc1, 1},
{0xfbd3, 0xfd3d, 1},
{0xfd50, 0xfd8f, 1},
{0xfd92, 0xfdc7, 1},
var _Tibetan = []Range{
{0x0f00, 0x0f47, 1},
{0x0f49, 0x0f6c, 1},
- {0x0f71, 0x0f8b, 1},
- {0x0f90, 0x0f97, 1},
+ {0x0f71, 0x0f97, 1},
{0x0f99, 0x0fbc, 1},
{0x0fbe, 0x0fcc, 1},
{0x0fce, 0x0fd4, 1},
+ {0x0fd9, 0x0fda, 1},
}
var _Coptic = []Range{
var _Hiragana = []Range{
{0x3041, 0x3096, 1},
{0x309d, 0x309f, 1},
+ {0x1b001, 0x1b001, 1},
{0x1f200, 0x1f200, 1},
}
var _Hangul = []Range{
{0x1100, 0x11ff, 1},
+ {0x302e, 0x302f, 1},
{0x3131, 0x318e, 1},
{0x3200, 0x321e, 1},
{0x3260, 0x327e, 1},
{0x0b56, 0x0b57, 1},
{0x0b5c, 0x0b5d, 1},
{0x0b5f, 0x0b63, 1},
- {0x0b66, 0x0b71, 1},
+ {0x0b66, 0x0b77, 1},
}
var _Buhid = []Range{
{0x12d8, 0x1310, 1},
{0x1312, 0x1315, 1},
{0x1318, 0x135a, 1},
- {0x135f, 0x137c, 1},
+ {0x135d, 0x137c, 1},
{0x1380, 0x1399, 1},
{0x2d80, 0x2d96, 1},
{0x2da0, 0x2da6, 1},
{0x2dc8, 0x2dce, 1},
{0x2dd0, 0x2dd6, 1},
{0x2dd8, 0x2dde, 1},
+ {0xab01, 0xab06, 1},
+ {0xab09, 0xab0e, 1},
+ {0xab11, 0xab16, 1},
+ {0xab20, 0xab26, 1},
+ {0xab28, 0xab2e, 1},
}
var _Javanese = []Range{
{0x1680, 0x169c, 1},
}
+var _Batak = []Range{
+ {0x1bc0, 0x1bf3, 1},
+ {0x1bfc, 0x1bff, 1},
+}
+
var _Syriac = []Range{
{0x0700, 0x070d, 1},
{0x070f, 0x074a, 1},
var _Bamum = []Range{
{0xa6a0, 0xa6f7, 1},
+ {0x16800, 0x16a38, 1},
}
var _Lepcha = []Range{
}
var _Devanagari = []Range{
- {0x0900, 0x0939, 1},
- {0x093c, 0x094e, 1},
- {0x0950, 0x0950, 1},
- {0x0953, 0x0955, 1},
- {0x0958, 0x0963, 1},
+ {0x0900, 0x0950, 1},
+ {0x0953, 0x0963, 1},
{0x0966, 0x096f, 1},
- {0x0971, 0x0972, 1},
+ {0x0971, 0x0977, 1},
{0x0979, 0x097f, 1},
{0xa8e0, 0xa8fb, 1},
}
var _Tifinagh = []Range{
{0x2d30, 0x2d65, 1},
- {0x2d6f, 0x2d6f, 1},
+ {0x2d6f, 0x2d70, 1},
+ {0x2d7f, 0x2d7f, 1},
}
var _Ugaritic = []Range{
var _Cyrillic = []Range{
{0x0400, 0x0484, 1},
- {0x0487, 0x0525, 1},
+ {0x0487, 0x0527, 1},
{0x1d2b, 0x1d2b, 1},
{0x1d78, 0x1d78, 1},
{0x2de0, 0x2dff, 1},
- {0xa640, 0xa65f, 1},
- {0xa662, 0xa673, 1},
+ {0xa640, 0xa673, 1},
{0xa67c, 0xa697, 1},
}
{0xfa70, 0xfad9, 1},
{0x20000, 0x2a6d6, 1},
{0x2a700, 0x2b734, 1},
+ {0x2b740, 0x2b81d, 1},
{0x2f800, 0x2fa1d, 1},
}
Avestan = _Avestan // Avestan is the set of Unicode characters in script Avestan.
Balinese = _Balinese // Balinese is the set of Unicode characters in script Balinese.
Bamum = _Bamum // Bamum is the set of Unicode characters in script Bamum.
+ Batak = _Batak // Batak is the set of Unicode characters in script Batak.
Bengali = _Bengali // Bengali is the set of Unicode characters in script Bengali.
Bopomofo = _Bopomofo // Bopomofo is the set of Unicode characters in script Bopomofo.
+ Brahmi = _Brahmi // Brahmi is the set of Unicode characters in script Brahmi.
Braille = _Braille // Braille is the set of Unicode characters in script Braille.
Buginese = _Buginese // Buginese is the set of Unicode characters in script Buginese.
Buhid = _Buhid // Buhid is the set of Unicode characters in script Buhid.
Lycian = _Lycian // Lycian is the set of Unicode characters in script Lycian.
Lydian = _Lydian // Lydian is the set of Unicode characters in script Lydian.
Malayalam = _Malayalam // Malayalam is the set of Unicode characters in script Malayalam.
+ Mandaic = _Mandaic // Mandaic is the set of Unicode characters in script Mandaic.
Meetei_Mayek = _Meetei_Mayek // Meetei_Mayek is the set of Unicode characters in script Meetei_Mayek.
Mongolian = _Mongolian // Mongolian is the set of Unicode characters in script Mongolian.
Myanmar = _Myanmar // Myanmar is the set of Unicode characters in script Myanmar.
)
// Generated by running
-// maketables --props=all --url=http://www.unicode.org/Public/5.2.0/ucd/
+// maketables --props=all --url=http://www.unicode.org/Public/6.0.0/ucd/
// DO NOT EDIT
// Properties is the set of Unicode property tables.
var _Deprecated = []Range{
{0x0149, 0x0149, 1},
+ {0x0673, 0x0673, 1},
{0x0f77, 0x0f77, 1},
{0x0f79, 0x0f79, 1},
{0x17a3, 0x17a4, 1},
{0x070c, 0x070c, 1},
{0x07f8, 0x07f9, 1},
{0x0830, 0x083e, 1},
+ {0x085e, 0x085e, 1},
{0x0964, 0x0965, 1},
{0x0e5a, 0x0e5b, 1},
{0x0f08, 0x0f08, 1},
{0x10857, 0x10857, 1},
{0x1091f, 0x1091f, 1},
{0x10b3a, 0x10b3f, 1},
+ {0x11047, 0x1104d, 1},
{0x110be, 0x110c1, 1},
{0x12470, 0x12473, 1},
}
{0x00b7, 0x00b7, 1},
{0x0387, 0x0387, 1},
{0x1369, 0x1371, 1},
+ {0x19da, 0x19da, 1},
}
var _Bidi_Control = []Range{
{0x20e5, 0x20e6, 1},
{0x20eb, 0x20ef, 1},
{0x2102, 0x2102, 1},
+ {0x2107, 0x2107, 1},
{0x210a, 0x2113, 1},
{0x2115, 0x2115, 1},
{0x2119, 0x211d, 1},
{0xfa27, 0xfa29, 1},
{0x20000, 0x2a6d6, 1},
{0x2a700, 0x2b734, 1},
+ {0x2b740, 0x2b81d, 1},
}
var _Hyphen = []Range{
{0x1362, 0x1362, 1},
{0x1367, 0x1368, 1},
{0x166e, 0x166e, 1},
+ {0x1735, 0x1736, 1},
{0x1803, 0x1803, 1},
{0x1809, 0x1809, 1},
{0x1944, 0x1945, 1},
+ {0x1aa8, 0x1aab, 1},
{0x1b5a, 0x1b5b, 1},
{0x1b5e, 0x1b5f, 1},
{0x1c3b, 0x1c3c, 1},
{0xff0e, 0xff0e, 1},
{0xff1f, 0xff1f, 1},
{0xff61, 0xff61, 1},
+ {0x10a56, 0x10a57, 1},
+ {0x11047, 0x11048, 1},
{0x110be, 0x110c1, 1},
}
{0x05c7, 0x05c7, 1},
{0x0610, 0x061a, 1},
{0x064b, 0x0657, 1},
- {0x0659, 0x065e, 1},
+ {0x0659, 0x065f, 1},
{0x0670, 0x0670, 1},
{0x06d6, 0x06dc, 1},
{0x06e1, 0x06e4, 1},
{0x0825, 0x0827, 1},
{0x0829, 0x082c, 1},
{0x0900, 0x0903, 1},
+ {0x093a, 0x093b, 1},
{0x093e, 0x094c, 1},
- {0x094e, 0x094e, 1},
- {0x0955, 0x0955, 1},
+ {0x094e, 0x094f, 1},
+ {0x0955, 0x0957, 1},
{0x0962, 0x0963, 1},
{0x0981, 0x0983, 1},
{0x09be, 0x09c4, 1},
{0x0ebb, 0x0ebc, 1},
{0x0ecd, 0x0ecd, 1},
{0x0f71, 0x0f81, 1},
- {0x0f90, 0x0f97, 1},
+ {0x0f8d, 0x0f97, 1},
{0x0f99, 0x0fbc, 1},
{0x102b, 0x1036, 1},
{0x1038, 0x1038, 1},
{0x1b35, 0x1b43, 1},
{0x1b80, 0x1b82, 1},
{0x1ba1, 0x1ba9, 1},
+ {0x1be7, 0x1bf1, 1},
{0x1c24, 0x1c35, 1},
{0x1cf2, 0x1cf2, 1},
{0x24b6, 0x24e9, 1},
{0xa926, 0xa92a, 1},
{0xa947, 0xa952, 1},
{0xa980, 0xa983, 1},
- {0xa9b3, 0xa9bf, 1},
+ {0xa9b4, 0xa9bf, 1},
{0xaa29, 0xaa36, 1},
{0xaa43, 0xaa43, 1},
{0xaa4c, 0xaa4d, 1},
{0x10a01, 0x10a03, 1},
{0x10a05, 0x10a06, 1},
{0x10a0c, 0x10a0f, 1},
+ {0x11000, 0x11002, 1},
+ {0x11038, 0x11045, 1},
{0x11082, 0x11082, 1},
{0x110b0, 0x110b8, 1},
}
{0xfa70, 0xfad9, 1},
{0x20000, 0x2a6d6, 1},
{0x2a700, 0x2b734, 1},
+ {0x2b740, 0x2b81d, 1},
{0x2f800, 0x2fa1d, 1},
}
)
// Generated by running
-// maketables --data=http://www.unicode.org/Public/5.2.0/ucd/UnicodeData.txt
+// maketables --data=http://www.unicode.org/Public/6.0.0/ucd/UnicodeData.txt
// DO NOT EDIT
// CaseRanges is the table describing case mappings for all letters with
{0x025B, 0x025B, d{-203, 0, -203}},
{0x0260, 0x0260, d{-205, 0, -205}},
{0x0263, 0x0263, d{-207, 0, -207}},
+ {0x0265, 0x0265, d{42280, 0, 42280}},
{0x0268, 0x0268, d{-209, 0, -209}},
{0x0269, 0x0269, d{-211, 0, -211}},
{0x026B, 0x026B, d{10743, 0, 10743}},
{0x04C0, 0x04C0, d{0, 15, 0}},
{0x04C1, 0x04CE, d{UpperLower, UpperLower, UpperLower}},
{0x04CF, 0x04CF, d{-15, 0, -15}},
- {0x04D0, 0x0525, d{UpperLower, UpperLower, UpperLower}},
+ {0x04D0, 0x0527, d{UpperLower, UpperLower, UpperLower}},
{0x0531, 0x0556, d{0, 48, 0}},
{0x0561, 0x0586, d{-48, 0, -48}},
{0x10A0, 0x10C5, d{0, 7264, 0}},
{0x2C80, 0x2CE3, d{UpperLower, UpperLower, UpperLower}},
{0x2CEB, 0x2CEE, d{UpperLower, UpperLower, UpperLower}},
{0x2D00, 0x2D25, d{-7264, 0, -7264}},
- {0xA640, 0xA65F, d{UpperLower, UpperLower, UpperLower}},
- {0xA662, 0xA66D, d{UpperLower, UpperLower, UpperLower}},
+ {0xA640, 0xA66D, d{UpperLower, UpperLower, UpperLower}},
{0xA680, 0xA697, d{UpperLower, UpperLower, UpperLower}},
{0xA722, 0xA72F, d{UpperLower, UpperLower, UpperLower}},
{0xA732, 0xA76F, d{UpperLower, UpperLower, UpperLower}},
{0xA77D, 0xA77D, d{0, -35332, 0}},
{0xA77E, 0xA787, d{UpperLower, UpperLower, UpperLower}},
{0xA78B, 0xA78C, d{UpperLower, UpperLower, UpperLower}},
+ {0xA78D, 0xA78D, d{0, -42280, 0}},
+ {0xA790, 0xA791, d{UpperLower, UpperLower, UpperLower}},
+ {0xA7A0, 0xA7A9, d{UpperLower, UpperLower, UpperLower}},
{0xFF21, 0xFF3A, d{0, 32, 0}},
{0xFF41, 0xFF5A, d{-32, 0, -32}},
{0x10400, 0x10427, d{0, 40, 0}},
}
// Step 41. check websocket headers.
- if resp.Header["Upgrade"] != "WebSocket" ||
- strings.ToLower(resp.Header["Connection"]) != "upgrade" {
+ if resp.Header.Get("Upgrade") != "WebSocket" ||
+ strings.ToLower(resp.Header.Get("Connection")) != "upgrade" {
return ErrBadUpgrade
}
- if resp.Header["Sec-Websocket-Origin"] != origin {
+ if resp.Header.Get("Sec-Websocket-Origin") != origin {
return ErrBadWebSocketOrigin
}
- if resp.Header["Sec-Websocket-Location"] != location {
+ if resp.Header.Get("Sec-Websocket-Location") != location {
return ErrBadWebSocketLocation
}
- if protocol != "" && resp.Header["Sec-Websocket-Protocol"] != protocol {
+ if protocol != "" && resp.Header.Get("Sec-Websocket-Protocol") != protocol {
return ErrBadWebSocketProtocol
}
if resp.Status != "101 Web Socket Protocol Handshake" {
return ErrBadStatus
}
- if resp.Header["Upgrade"] != "WebSocket" ||
- resp.Header["Connection"] != "Upgrade" {
+ if resp.Header.Get("Upgrade") != "WebSocket" ||
+ resp.Header.Get("Connection") != "Upgrade" {
return ErrBadUpgrade
}
- if resp.Header["Websocket-Origin"] != origin {
+ if resp.Header.Get("Websocket-Origin") != origin {
return ErrBadWebSocketOrigin
}
- if resp.Header["Websocket-Location"] != location {
+ if resp.Header.Get("Websocket-Location") != location {
return ErrBadWebSocketLocation
}
- if protocol != "" && resp.Header["Websocket-Protocol"] != protocol {
+ if protocol != "" && resp.Header.Get("Websocket-Protocol") != protocol {
return ErrBadWebSocketProtocol
}
return
}
// HTTP version can be safely ignored.
- if strings.ToLower(req.Header["Upgrade"]) != "websocket" ||
- strings.ToLower(req.Header["Connection"]) != "upgrade" {
+ if strings.ToLower(req.Header.Get("Upgrade")) != "websocket" ||
+ strings.ToLower(req.Header.Get("Connection")) != "upgrade" {
return
}
// TODO(ukai): check Host
- origin, found := req.Header["Origin"]
- if !found {
+ origin := req.Header.Get("Origin")
+ if origin == "" {
return
}
- key1, found := req.Header["Sec-Websocket-Key1"]
- if !found {
+ key1 := req.Header.Get("Sec-Websocket-Key1")
+ if key1 == "" {
return
}
- key2, found := req.Header["Sec-Websocket-Key2"]
- if !found {
+ key2 := req.Header.Get("Sec-Websocket-Key2")
+ if key2 == "" {
return
}
key3 := make([]byte, 8)
buf.WriteString("Connection: Upgrade\r\n")
buf.WriteString("Sec-WebSocket-Location: " + location + "\r\n")
buf.WriteString("Sec-WebSocket-Origin: " + origin + "\r\n")
- protocol, found := req.Header["Sec-Websocket-Protocol"]
- if found {
+ protocol := strings.TrimSpace(req.Header.Get("Sec-Websocket-Protocol"))
+ if protocol != "" {
buf.WriteString("Sec-WebSocket-Protocol: " + protocol + "\r\n")
}
// Step 12. send CRLF.
io.WriteString(w, "Unexpected request")
return
}
- if req.Header["Upgrade"] != "WebSocket" {
+ if req.Header.Get("Upgrade") != "WebSocket" {
w.WriteHeader(http.StatusBadRequest)
io.WriteString(w, "missing Upgrade: WebSocket header")
return
}
- if req.Header["Connection"] != "Upgrade" {
+ if req.Header.Get("Connection") != "Upgrade" {
w.WriteHeader(http.StatusBadRequest)
io.WriteString(w, "missing Connection: Upgrade header")
return
}
- origin, found := req.Header["Origin"]
- if !found {
+ origin := strings.TrimSpace(req.Header.Get("Origin"))
+ if origin == "" {
w.WriteHeader(http.StatusBadRequest)
io.WriteString(w, "missing Origin header")
return
buf.WriteString("Connection: Upgrade\r\n")
buf.WriteString("WebSocket-Origin: " + origin + "\r\n")
buf.WriteString("WebSocket-Location: " + location + "\r\n")
- protocol, found := req.Header["Websocket-Protocol"]
+ protocol := strings.TrimSpace(req.Header.Get("Websocket-Protocol"))
// canonical header key of WebSocket-Protocol.
- if found {
+ if protocol != "" {
buf.WriteString("WebSocket-Protocol: " + protocol + "\r\n")
}
buf.WriteString("\r\n")
func startServer() {
l, e := net.Listen("tcp", "127.0.0.1:0") // any available address
if e != nil {
- log.Exitf("net.Listen tcp :0 %v", e)
+ log.Fatalf("net.Listen tcp :0 %v", e)
}
serverAddr = l.Addr().String()
log.Print("Test WebSocket server listening on ", serverAddr)
func TestUnmarshalFeed(t *testing.T) {
var f Feed
- if err := Unmarshal(StringReader(rssFeedString), &f); err != nil {
+ if err := Unmarshal(StringReader(atomFeedString), &f); err != nil {
t.Fatalf("Unmarshal: %s", err)
}
- if !reflect.DeepEqual(f, rssFeed) {
- t.Fatalf("have %#v\nwant %#v", f, rssFeed)
+ if !reflect.DeepEqual(f, atomFeed) {
+ t.Fatalf("have %#v\nwant %#v", f, atomFeed)
}
}
// hget http://codereview.appspot.com/rss/mine/rsc
-const rssFeedString = `
+const atomFeedString = `
<?xml version="1.0" encoding="utf-8"?>
<feed xmlns="http://www.w3.org/2005/Atom" xml:lang="en-us"><title>Code Review - My issues</title><link href="http://codereview.appspot.com/" rel="alternate"></link><li-nk href="http://codereview.appspot.com/rss/mine/rsc" rel="self"></li-nk><id>http://codereview.appspot.com/</id><updated>2009-10-04T01:35:58+00:00</updated><author><name>rietveld<></name></author><entry><title>rietveld: an attempt at pubsubhubbub
</title><link hre-f="http://codereview.appspot.com/126085" rel="alternate"></link><updated>2009-10-04T01:35:58+00:00</updated><author><name>email-address-removed</name></author><id>urn:md5:134d9179c41f806be79b3a5f7877d19a</id><summary type="html">
type Time string
-var rssFeed = Feed{
+var atomFeed = Feed{
XMLName: Name{"http://www.w3.org/2005/Atom", "feed"},
Title: "Code Review - My issues",
Link: []Link{
// "quot": `"`,
Entity map[string]string
- r io.ReadByter
+ r io.ByteReader
buf bytes.Buffer
saved *bytes.Buffer
stk *stack
// Assume that if reader has its own
// ReadByte, it's efficient enough.
// Otherwise, use bufio.
- if rb, ok := r.(io.ReadByter); ok {
+ if rb, ok := r.(io.ByteReader); ok {
p.r = rb
} else {
p.r = bufio.NewReader(r)
}
// Probably a directive: <!DOCTYPE ...>, <!ENTITY ...>, etc.
- // We don't care, but accumulate for caller.
+ // We don't care, but accumulate for caller. Quoted angle
+ // brackets do not count for nesting.
p.buf.Reset()
p.buf.WriteByte(b)
+ inquote := uint8(0)
+ depth := 0
for {
if b, ok = p.mustgetc(); !ok {
return nil, p.err
}
- if b == '>' {
+ if inquote == 0 && b == '>' && depth == 0 {
break
}
p.buf.WriteByte(b)
+ switch {
+ case b == inquote:
+ inquote = 0
+
+ case inquote != 0:
+ // in quotes, no special action
+
+ case b == '\'' || b == '"':
+ inquote = b
+
+ case b == '>' && inquote == 0:
+ depth--
+
+ case b == '<' && inquote == 0:
+ depth++
+ }
}
return Directive(p.buf.Bytes()), nil
}
}
}
+// Ensure that directives (specifically !DOCTYPE) include the complete
+// text of any nested directives, noting that < and > do not change
+// nesting depth if they are in single or double quotes.
+
+var nestedDirectivesInput = `
+<!DOCTYPE [<!ENTITY rdf "http://www.w3.org/1999/02/22-rdf-syntax-ns#">]>
+<!DOCTYPE [<!ENTITY xlt ">">]>
+<!DOCTYPE [<!ENTITY xlt "<">]>
+<!DOCTYPE [<!ENTITY xlt '>'>]>
+<!DOCTYPE [<!ENTITY xlt '<'>]>
+<!DOCTYPE [<!ENTITY xlt '">'>]>
+<!DOCTYPE [<!ENTITY xlt "'<">]>
+`
+
+var nestedDirectivesTokens = []Token{
+ CharData([]byte("\n")),
+ Directive([]byte(`DOCTYPE [<!ENTITY rdf "http://www.w3.org/1999/02/22-rdf-syntax-ns#">]`)),
+ CharData([]byte("\n")),
+ Directive([]byte(`DOCTYPE [<!ENTITY xlt ">">]`)),
+ CharData([]byte("\n")),
+ Directive([]byte(`DOCTYPE [<!ENTITY xlt "<">]`)),
+ CharData([]byte("\n")),
+ Directive([]byte(`DOCTYPE [<!ENTITY xlt '>'>]`)),
+ CharData([]byte("\n")),
+ Directive([]byte(`DOCTYPE [<!ENTITY xlt '<'>]`)),
+ CharData([]byte("\n")),
+ Directive([]byte(`DOCTYPE [<!ENTITY xlt '">'>]`)),
+ CharData([]byte("\n")),
+ Directive([]byte(`DOCTYPE [<!ENTITY xlt "'<">]`)),
+ CharData([]byte("\n")),
+}
+
+func TestNestedDirectives(t *testing.T) {
+ p := NewParser(StringReader(nestedDirectivesInput))
+
+ for i, want := range nestedDirectivesTokens {
+ have, err := p.Token()
+ if err != nil {
+ t.Fatalf("token %d: unexpected error: %s", i, err)
+ }
+ if !reflect.DeepEqual(have, want) {
+ t.Errorf("token %d = %#v want %#v", i, have, want)
+ }
+ }
+}
+
func TestToken(t *testing.T) {
p := NewParser(StringReader(testInput))
-e 's/msg_flags/Flags/' \
>> ${OUT}
+# The ip_mreq struct
+grep '^type _ip_mreq ' gen-sysinfo.go | \
+ sed -e 's/_ip_mreq/IpMreq/' \
+ -e 's/imr_multiaddr/Multiaddr/' \
+ -e 's/imr_interface/Interface/' \
+ -e 's/_in_addr/[4]byte/g' \
+ >> ${OUT}
+
exit $?
struct __go_string ret;
bytes = (const unsigned char *) p;
- retdata = runtime_mallocgc (len, RefNoPointers, 1, 0);
+ retdata = runtime_mallocgc (len, FlagNoPointers, 1, 0);
__builtin_memcpy (retdata, bytes, len);
ret.__data = retdata;
ret.__length = len;
slen += 4;
}
- retdata = runtime_mallocgc (slen, RefNoPointers, 1, 0);
+ retdata = runtime_mallocgc (slen, FlagNoPointers, 1, 0);
ret.__data = retdata;
ret.__length = slen;
}
}
- retdata = runtime_mallocgc (len, RefNoPointers, 1, 0);
+ retdata = runtime_mallocgc (len, FlagNoPointers, 1, 0);
__builtin_memcpy (retdata, buf, len);
ret.__data = retdata;
ret.__length = len;
void *
__go_new_nopointers (size_t size)
{
- return runtime_mallocgc (size, RefNoPointers, 1, 1);
+ return runtime_mallocgc (size, FlagNoPointers, 1, 1);
}
struct __go_empty_interface arg;
len = __builtin_strlen (msg);
- sdata = runtime_mallocgc (len, RefNoPointers, 0, 0);
+ sdata = runtime_mallocgc (len, FlagNoPointers, 0, 0);
__builtin_memcpy (sdata, msg, len);
s.__data = sdata;
s.__length = len;
unsigned char *data;
struct __go_open_array ret;
- data = (unsigned char *) runtime_mallocgc (str.__length, RefNoPointers, 1, 0);
+ data = (unsigned char *) runtime_mallocgc (str.__length, FlagNoPointers, 1, 0);
__builtin_memcpy (data, str.__data, str.__length);
ret.__values = (void *) data;
ret.__count = str.__length;
p += __go_get_rune (p, pend - p, &rune);
}
- data = (uint32_t *) runtime_mallocgc (c * sizeof (uint32_t), RefNoPointers,
+ data = (uint32_t *) runtime_mallocgc (c * sizeof (uint32_t), FlagNoPointers,
1, 0);
p = str.__data;
pd = data;
return s1;
len = s1.__length + s2.__length;
- retdata = runtime_mallocgc (len, RefNoPointers, 1, 0);
+ retdata = runtime_mallocgc (len, FlagNoPointers, 1, 0);
__builtin_memcpy (retdata, s1.__data, s1.__length);
__builtin_memcpy (retdata + s1.__length, s2.__data, s2.__length);
ret.__data = retdata;
/* A pointer to fields which are only used for some types. */
const struct __go_uncommon_type *__uncommon;
+
+ /* The descriptor for the type which is a pointer to this type.
+ This may be NULL. */
+ const struct __go_type_descriptor *__pointer_to_this;
};
/* The information we store for each method of a type. */
/* __reflection */
&reflection_string,
/* __uncommon */
+ NULL,
+ /* __pointer_to_this */
NULL
};
/* __reflection */
&preflection_string,
/* __uncommon */
+ NULL,
+ /* __pointer_to_this */
NULL
},
/* __element_type */
// Small objects are allocated from the per-thread cache's free lists.
// Large objects (> 32 kB) are allocated straight from the heap.
void*
-runtime_mallocgc(uintptr size, uint32 refflag, int32 dogc, int32 zeroed)
+runtime_mallocgc(uintptr size, uint32 flag, int32 dogc, int32 zeroed)
{
int32 sizeclass, rate;
MCache *c;
uintptr npages;
MSpan *s;
void *v;
- uint32 *ref;
if(!__sync_bool_compare_and_swap(&m->mallocing, 0, 1))
runtime_throw("malloc/free - deadlock");
mstats.alloc += size;
mstats.total_alloc += size;
mstats.by_size[sizeclass].nmalloc++;
-
- if(!runtime_mlookup(v, nil, nil, nil, &ref)) {
- // runtime_printf("malloc %D; runtime_mlookup failed\n", (uint64)size);
- runtime_throw("malloc runtime_mlookup");
- }
- *ref = RefNone | refflag;
} else {
// TODO(rsc): Report tracebacks for very large allocations.
v = (void*)(s->start << PageShift);
// setup for mark sweep
- s->gcref0 = RefNone | refflag;
- ref = &s->gcref0;
+ runtime_markspan(v, 0, 0, true);
}
+ if(!(flag & FlagNoGC))
+ runtime_markallocated(v, size, (flag&FlagNoPointers) != 0);
__sync_bool_compare_and_swap(&m->mallocing, 1, 0);
if(__sync_bool_compare_and_swap(&m->gcing, 1, 0)) {
- if(!(refflag & RefNoProfiling))
+ if(!(flag & FlagNoProfiling))
__go_run_goroutine_gc(0);
else {
// We are being called from the profiler. Tell it
}
}
- if(!(refflag & RefNoProfiling) && (rate = runtime_MemProfileRate) > 0) {
+ if(!(flag & FlagNoProfiling) && (rate = runtime_MemProfileRate) > 0) {
if(size >= (uint32) rate)
goto profile;
if((uint32) m->mcache->next_sample > size)
rate = 0x3fffffff;
m->mcache->next_sample = fastrand1() % (2*rate);
profile:
- *ref |= RefProfiled;
+ runtime_setblockspecial(v);
runtime_MProf_Malloc(v, size);
}
}
void
__go_free(void *v)
{
- int32 sizeclass, size;
+ int32 sizeclass;
MSpan *s;
MCache *c;
- uint32 prof, *ref;
+ uint32 prof;
+ uintptr size;
if(v == nil)
return;
+
+ // If you change this also change mgc0.c:/^sweepspan,
+ // which has a copy of the guts of free.
if(!__sync_bool_compare_and_swap(&m->mallocing, 0, 1))
runtime_throw("malloc/free - deadlock");
- if(!runtime_mlookup(v, nil, nil, &s, &ref)) {
+ if(!runtime_mlookup(v, nil, nil, &s)) {
// runtime_printf("free %p: not an allocated block\n", v);
runtime_throw("free runtime_mlookup");
}
- prof = *ref & RefProfiled;
- *ref = RefFree;
+ prof = runtime_blockspecial(v);
// Find size class for v.
sizeclass = s->sizeclass;
if(sizeclass == 0) {
// Large object.
- if(prof)
- runtime_MProf_Free(v, s->npages<<PageShift);
- mstats.alloc -= s->npages<<PageShift;
- runtime_memclr(v, s->npages<<PageShift);
+ size = s->npages<<PageShift;
+ *(uintptr*)(s->start<<PageShift) = 1; // mark as "needs to be zeroed"
+ // Must mark v freed before calling unmarkspan and MHeap_Free:
+ // they might coalesce v into other spans and change the bitmap further.
+ runtime_markfreed(v, size);
+ runtime_unmarkspan(v, 1<<PageShift);
runtime_MHeap_Free(&runtime_mheap, s, 1);
} else {
// Small object.
size = runtime_class_to_size[sizeclass];
if(size > (int32)sizeof(uintptr))
((uintptr*)v)[1] = 1; // mark as "needs to be zeroed"
- if(prof)
- runtime_MProf_Free(v, size);
- mstats.alloc -= size;
+ // Must mark v freed before calling MCache_Free:
+ // it might coalesce v and other blocks into a bigger span
+ // and change the bitmap further.
+ runtime_markfreed(v, size);
mstats.by_size[sizeclass].nfree++;
runtime_MCache_Free(c, v, sizeclass, size);
}
+ mstats.alloc -= size;
+ if(prof)
+ runtime_MProf_Free(v, size);
+
__sync_bool_compare_and_swap(&m->mallocing, 1, 0);
if(__sync_bool_compare_and_swap(&m->gcing, 1, 0))
}
int32
-runtime_mlookup(void *v, byte **base, uintptr *size, MSpan **sp, uint32 **ref)
+runtime_mlookup(void *v, byte **base, uintptr *size, MSpan **sp)
{
- uintptr n, nobj, i;
+ uintptr n, i;
byte *p;
MSpan *s;
mstats.nlookup++;
- s = runtime_MHeap_LookupMaybe(&runtime_mheap, (uintptr)v>>PageShift);
+ s = runtime_MHeap_LookupMaybe(&runtime_mheap, v);
if(sp)
*sp = s;
if(s == nil) {
+ runtime_checkfreed(v, 1);
if(base)
*base = nil;
if(size)
*size = 0;
- if(ref)
- *ref = 0;
return 0;
}
*base = p;
if(size)
*size = s->npages<<PageShift;
- if(ref)
- *ref = &s->gcref0;
return 1;
}
- if((byte*)v >= (byte*)s->gcref) {
- // pointers into the gc ref counts
- // do not count as pointers.
+ if((byte*)v >= (byte*)s->limit) {
+ // pointers past the last block do not count as pointers.
return 0;
}
if(size)
*size = n;
- // good for error checking, but expensive
- if(0) {
- nobj = (s->npages << PageShift) / (n + RefcountOverhead);
- if((byte*)s->gcref < p || (byte*)(s->gcref+nobj) > p+(s->npages<<PageShift)) {
- // runtime_printf("odd span state=%d span=%p base=%p sizeclass=%d n=%D size=%D npages=%D\n",
- // s->state, s, p, s->sizeclass, (uint64)nobj, (uint64)n, (uint64)s->npages);
- // runtime_printf("s->base sizeclass %d v=%p base=%p gcref=%p blocksize=%D nobj=%D size=%D end=%p end=%p\n",
- // s->sizeclass, v, p, s->gcref, (uint64)s->npages<<PageShift,
- // (uint64)nobj, (uint64)n, s->gcref + nobj, p+(s->npages<<PageShift));
- runtime_throw("bad gcref");
- }
- }
- if(ref)
- *ref = &s->gcref[i];
-
return 1;
}
extern int32 runtime_sizeof_C_MStats
__asm__ ("libgo_runtime.runtime.Sizeof_C_MStats");
+#define MaxArena32 (2U<<30)
+
void
runtime_mallocinit(void)
{
- runtime_sizeof_C_MStats = sizeof(MStats);
+ byte *p;
+ uintptr arena_size, bitmap_size;
+ extern byte end[];
- runtime_initfintab();
- runtime_Mprof_Init();
+ runtime_sizeof_C_MStats = sizeof(MStats);
- runtime_SysMemInit();
runtime_InitSizes();
+
+ // Set up the allocation arena, a contiguous area of memory where
+ // allocated data will be found. The arena begins with a bitmap large
+ // enough to hold 4 bits per allocated word.
+ if(sizeof(void*) == 8) {
+ // On a 64-bit machine, allocate from a single contiguous reservation.
+ // 16 GB should be big enough for now.
+ //
+ // The code will work with the reservation at any address, but ask
+ // SysReserve to use 0x000000f800000000 if possible.
+ // Allocating a 16 GB region takes away 36 bits, and the amd64
+ // doesn't let us choose the top 17 bits, so that leaves the 11 bits
+ // in the middle of 0x00f8 for us to choose. Choosing 0x00f8 means
+ // that the valid memory addresses will begin 0x00f8, 0x00f9, 0x00fa, 0x00fb.
+ // None of the bytes f8 f9 fa fb can appear in valid UTF-8, and
+ // they are otherwise as far from ff (likely a common byte) as possible.
+ // Choosing 0x00 for the leading 6 bits was more arbitrary, but it
+ // is not a common ASCII code point either. Using 0x11f8 instead
+ // caused out of memory errors on OS X during thread allocations.
+ // These choices are both for debuggability and to reduce the
+ // odds of the conservative garbage collector not collecting memory
+ // because some non-pointer block of memory had a bit pattern
+ // that matched a memory address.
+ //
+ // Actually we reserve 17 GB (because the bitmap ends up being 1 GB)
+ // but it hardly matters: fc is not valid UTF-8 either, and we have to
+ // allocate 15 GB before we get that far.
+ arena_size = (uintptr)(16LL<<30);
+ bitmap_size = arena_size / (sizeof(void*)*8/4);
+ p = runtime_SysReserve((void*)(0x00f8ULL<<32), bitmap_size + arena_size);
+ if(p == nil)
+ runtime_throw("runtime: cannot reserve arena virtual address space");
+ } else {
+ // On a 32-bit machine, we can't typically get away
+ // with a giant virtual address space reservation.
+ // Instead we map the memory information bitmap
+ // immediately after the data segment, large enough
+ // to handle another 2GB of mappings (256 MB),
+ // along with a reservation for another 512 MB of memory.
+ // When that gets used up, we'll start asking the kernel
+ // for any memory anywhere and hope it's in the 2GB
+ // following the bitmap (presumably the executable begins
+ // near the bottom of memory, so we'll have to use up
+ // most of memory before the kernel resorts to giving out
+ // memory before the beginning of the text segment).
+ //
+ // Alternatively we could reserve 512 MB bitmap, enough
+ // for 4GB of mappings, and then accept any memory the
+ // kernel threw at us, but normally that's a waste of 512 MB
+ // of address space, which is probably too much in a 32-bit world.
+ bitmap_size = MaxArena32 / (sizeof(void*)*8/4);
+ arena_size = 512<<20;
+
+ // SysReserve treats the address we ask for, end, as a hint,
+ // not as an absolute requirement. If we ask for the end
+ // of the data segment but the operating system requires
+ // a little more space before we can start allocating, it will
+ // give out a slightly higher pointer. That's fine.
+ // Run with what we get back.
+ p = runtime_SysReserve(end, bitmap_size + arena_size);
+ if(p == nil)
+ runtime_throw("runtime: cannot reserve arena virtual address space");
+ }
+ if((uintptr)p & (((uintptr)1<<PageShift)-1))
+ runtime_throw("runtime: SysReserve returned unaligned address");
+
+ runtime_mheap.bitmap = p;
+ runtime_mheap.arena_start = p + bitmap_size;
+ runtime_mheap.arena_used = runtime_mheap.arena_start;
+ runtime_mheap.arena_end = runtime_mheap.arena_start + arena_size;
+
+ // Initialize the rest of the allocator.
runtime_MHeap_Init(&runtime_mheap, runtime_SysAlloc);
m->mcache = runtime_allocmcache();
runtime_free(runtime_malloc(1));
}
+void*
+runtime_MHeap_SysAlloc(MHeap *h, uintptr n)
+{
+ byte *p;
+
+ if(n <= (uintptr)(h->arena_end - h->arena_used)) {
+ // Keep taking from our reservation.
+ p = h->arena_used;
+ runtime_SysMap(p, n);
+ h->arena_used += n;
+ runtime_MHeap_MapBits(h);
+ return p;
+ }
+
+ // On 64-bit, our reservation is all we have.
+ if(sizeof(void*) == 8)
+ return nil;
+
+ // On 32-bit, once the reservation is gone we can
+ // try to get memory at a location chosen by the OS
+ // and hope that it is in the range we allocated bitmap for.
+ p = runtime_SysAlloc(n);
+ if(p == nil)
+ return nil;
+
+ if(p < h->arena_start || (uintptr)(p+n - h->arena_start) >= MaxArena32) {
+ runtime_printf("runtime: memory allocated by OS not in usable range");
+ runtime_SysFree(p, n);
+ return nil;
+ }
+
+ if(p+n > h->arena_used) {
+ h->arena_used = p+n;
+ if(h->arena_used > h->arena_end)
+ h->arena_end = h->arena_used;
+ runtime_MHeap_MapBits(h);
+ }
+
+ return p;
+}
+
// Runtime stubs.
void*
return runtime_mallocgc(n, 0, 1, 1);
}
+func new(n uint32) (ret *uint8) {
+ ret = runtime_mal(n);
+}
+
func Alloc(n uintptr) (p *byte) {
p = runtime_malloc(n);
}
}
func Lookup(p *byte) (base *byte, size uintptr) {
- runtime_mlookup(p, &base, &size, nil, nil);
+ runtime_mlookup(p, &base, &size, nil);
}
func GC() {
// runtime_printf("runtime.SetFinalizer: first argument is %S, not pointer\n", *obj.type->string);
goto throw;
}
- if(!runtime_mlookup(obj.__object, &base, &size, nil, nil) || obj.__object != base) {
+ if(!runtime_mlookup(obj.__object, &base, &size, nil) || obj.__object != base) {
// runtime_printf("runtime.SetFinalizer: pointer not at beginning of allocated block\n");
goto throw;
}
// used to manage storage used by the allocator.
// MHeap: the malloc heap, managed at page (4096-byte) granularity.
// MSpan: a run of pages managed by the MHeap.
-// MHeapMap: a mapping from page IDs to MSpans.
// MCentral: a shared free list for a given size class.
// MCache: a per-thread (in Go, per-M) cache for small objects.
// MStats: allocation statistics.
typedef struct FixAlloc FixAlloc;
typedef struct MCentral MCentral;
typedef struct MHeap MHeap;
-typedef struct MHeapMap MHeapMap;
typedef struct MSpan MSpan;
typedef struct MStats MStats;
typedef struct MLink MLink;
enum
{
+ // Computed constant. The definition of MaxSmallSize and the
+ // algorithm in msize.c produce some number of different allocation
+ // size classes. NumSizeClasses is that number. It's needed here
+ // because there are static arrays of this length; when msize runs its
+ // size choosing algorithm it double-checks that NumSizeClasses agrees.
+ NumSizeClasses = 61,
+
// Tunable constants.
- NumSizeClasses = 67, // Number of size classes (must match msize.c)
MaxSmallSize = 32<<10,
FixAllocChunk = 128<<10, // Chunk size for FixAlloc
MaxMCacheSize = 2<<20, // Maximum bytes in one MCache
MaxMHeapList = 1<<(20 - PageShift), // Maximum page length for fixed-size list in MHeap.
HeapAllocChunk = 1<<20, // Chunk size for heap growth
-};
+ // Number of bits in page to span calculations (4k pages).
+ // On 64-bit, we limit the arena to 16G, so 22 bits suffices.
+ // On 32-bit, we don't bother limiting anything: 20 bits for 4G.
#if __SIZEOF_POINTER__ == 8
-#include "mheapmap64.h"
+ MHeapMap_Bits = 22,
#else
-#include "mheapmap32.h"
+ MHeapMap_Bits = 20,
#endif
+};
// A generic linked list of blocks. (Typically the block is bigger than sizeof(MLink).)
struct MLink
// SysAlloc obtains a large chunk of zeroed memory from the
// operating system, typically on the order of a hundred kilobytes
-// or a megabyte.
+// or a megabyte. If the pointer argument is non-nil, the caller
+// wants a mapping there or nowhere.
//
// SysUnused notifies the operating system that the contents
// of the memory region are no longer needed and can be reused
// SysFree returns it unconditionally; this is only used if
// an out-of-memory error has been detected midway through
// an allocation. It is okay if SysFree is a no-op.
+//
+// SysReserve reserves address space without allocating memory.
+// If the pointer passed to it is non-nil, the caller wants the
+// reservation there, but SysReserve can still choose another
+// location if that one is unavailable.
+//
+// SysMap maps previously reserved address space for use.
void* runtime_SysAlloc(uintptr nbytes);
void runtime_SysFree(void *v, uintptr nbytes);
void runtime_SysUnused(void *v, uintptr nbytes);
-void runtime_SysMemInit(void);
+void runtime_SysMap(void *v, uintptr nbytes);
+void* runtime_SysReserve(void *v, uintptr nbytes);
// FixAlloc is a simple free-list allocator for fixed size objects.
// Malloc uses a FixAlloc wrapped around SysAlloc to manages its
uint64 mspan_sys;
uint64 mcache_inuse; // MCache structures
uint64 mcache_sys;
- uint64 heapmap_sys; // heap map
uint64 buckhash_sys; // profiling bucket hash table
// Statistics about garbage collector.
uint32 ref; // number of allocated objects in this span
uint32 sizeclass; // size class
uint32 state; // MSpanInUse etc
- union {
- uint32 *gcref; // sizeclass > 0
- uint32 gcref0; // sizeclass == 0
- };
+ byte *limit; // end of data in span
};
void runtime_MSpan_Init(MSpan *span, PageID start, uintptr npages);
MSpan *allspans;
// span lookup
- MHeapMap map;
+ MSpan *map[1<<MHeapMap_Bits];
// range of addresses we might see in the heap
- byte *min;
- byte *max;
+ byte *bitmap;
+ uintptr bitmap_mapped;
+ byte *arena_start;
+ byte *arena_used;
+ byte *arena_end;
// central free lists for small size classes.
// the union makes sure that the MCentrals are
void runtime_MHeap_Init(MHeap *h, void *(*allocator)(uintptr));
MSpan* runtime_MHeap_Alloc(MHeap *h, uintptr npage, int32 sizeclass, int32 acct);
void runtime_MHeap_Free(MHeap *h, MSpan *s, int32 acct);
-MSpan* runtime_MHeap_Lookup(MHeap *h, PageID p);
-MSpan* runtime_MHeap_LookupMaybe(MHeap *h, PageID p);
-void runtime_MGetSizeClassInfo(int32 sizeclass, int32 *size, int32 *npages, int32 *nobj);
+MSpan* runtime_MHeap_Lookup(MHeap *h, void *v);
+MSpan* runtime_MHeap_LookupMaybe(MHeap *h, void *v);
+void runtime_MGetSizeClassInfo(int32 sizeclass, uintptr *size, int32 *npages, int32 *nobj);
+void* runtime_MHeap_SysAlloc(MHeap *h, uintptr n);
+void runtime_MHeap_MapBits(MHeap *h);
void* runtime_mallocgc(uintptr size, uint32 flag, int32 dogc, int32 zeroed);
-int32 runtime_mlookup(void *v, byte **base, uintptr *size, MSpan **s, uint32 **ref);
+int32 runtime_mlookup(void *v, byte **base, uintptr *size, MSpan **s);
void runtime_gc(int32 force);
-
-void* runtime_SysAlloc(uintptr);
-void runtime_SysUnused(void*, uintptr);
-void runtime_SysFree(void*, uintptr);
+void runtime_markallocated(void *v, uintptr n, bool noptr);
+void runtime_checkallocated(void *v, uintptr n);
+void runtime_markfreed(void *v, uintptr n);
+void runtime_checkfreed(void *v, uintptr n);
+int32 runtime_checking;
+void runtime_markspan(void *v, uintptr size, uintptr n, bool leftover);
+void runtime_unmarkspan(void *v, uintptr size);
+bool runtime_blockspecial(void*);
+void runtime_setblockspecial(void*);
enum
{
- RefcountOverhead = 4, // one uint32 per object
-
- RefFree = 0, // must be zero
- RefStack, // stack segment - don't free and don't scan for pointers
- RefNone, // no references
- RefSome, // some references
- RefNoPointers = 0x80000000U, // flag - no pointers here
- RefHasFinalizer = 0x40000000U, // flag - has finalizer
- RefProfiled = 0x20000000U, // flag - is in profiling table
- RefNoProfiling = 0x10000000U, // flag - must not profile
- RefFlags = 0xFFFF0000U,
+ // flags to malloc
+ FlagNoPointers = 1<<0, // no pointers here
+ FlagNoProfiling = 1<<1, // must not profile
+ FlagNoGC = 1<<2, // must not free or scan for pointers
};
void runtime_Mprof_Init(void);
MCentral_Free(MCentral *c, void *v)
{
MSpan *s;
- PageID page;
- MLink *p, *next;
+ MLink *p;
int32 size;
// Find span for v.
- page = (uintptr)v >> PageShift;
- s = runtime_MHeap_Lookup(&runtime_mheap, page);
+ s = runtime_MHeap_Lookup(&runtime_mheap, v);
if(s == nil || s->ref == 0)
runtime_throw("invalid free");
if(--s->ref == 0) {
size = runtime_class_to_size[c->sizeclass];
runtime_MSpanList_Remove(s);
- // The second word of each freed block indicates
- // whether it needs to be zeroed. The first word
- // is the link pointer and must always be cleared.
- for(p=s->freelist; p; p=next) {
- next = p->next;
- if(size > (int32)sizeof(uintptr) && ((uintptr*)p)[1] != 0)
- runtime_memclr((byte*)p, size);
- else
- p->next = nil;
- }
+ runtime_unmarkspan((byte*)(s->start<<PageShift), s->npages<<PageShift);
+ *(uintptr*)(s->start<<PageShift) = 1; // needs zeroing
s->freelist = nil;
c->nfree -= (s->npages << PageShift) / size;
runtime_unlock(c);
}
void
-runtime_MGetSizeClassInfo(int32 sizeclass, int32 *sizep, int32 *npagesp, int32 *nobj)
+runtime_MGetSizeClassInfo(int32 sizeclass, uintptr *sizep, int32 *npagesp, int32 *nobj)
{
int32 size;
int32 npages;
size = runtime_class_to_size[sizeclass];
*npagesp = npages;
*sizep = size;
- *nobj = (npages << PageShift) / (size + RefcountOverhead);
+ *nobj = (npages << PageShift) / size;
}
// Fetch a new span from the heap and
static bool
MCentral_Grow(MCentral *c)
{
- int32 i, n, npages, size;
+ int32 i, n, npages;
+ uintptr size;
MLink **tailp, *v;
byte *p;
MSpan *s;
// Carve span into sequence of blocks.
tailp = &s->freelist;
p = (byte*)(s->start << PageShift);
- s->gcref = (uint32*)(p + size*n);
+ s->limit = p + size*n;
for(i=0; i<n; i++) {
v = (MLink*)p;
*tailp = v;
p += size;
}
*tailp = nil;
+ runtime_markspan((byte*)(s->start<<PageShift), size, n, size*n < (s->npages<<PageShift));
runtime_lock(c);
c->nfree += n;
p = runtime_mmap(nil, n, PROT_READ|PROT_WRITE|PROT_EXEC, MAP_ANON|MAP_PRIVATE, fd, 0);
if (p == MAP_FAILED) {
if(errno == EACCES) {
- printf("mmap: access denied\n");
- printf("If you're running SELinux, enable execmem for this process.\n");
- } else {
- printf("mmap: errno=%d\n", errno);
+ printf("runtime: mmap: access denied\n");
+ printf("if you're running SELinux, enable execmem for this process.\n");
+ exit(2);
}
- exit(2);
+ return nil;
}
return p;
}
runtime_munmap(v, n);
}
+void*
+runtime_SysReserve(void *v, uintptr n)
+{
+ int fd = -1;
+
+ // On 64-bit, people with ulimit -v set complain if we reserve too
+ // much address space. Instead, assume that the reservation is okay
+ // and check the assumption in SysMap.
+ if(sizeof(void*) == 8)
+ return v;
+
+#ifdef USE_DEV_ZERO
+ if (dev_zero == -1) {
+ dev_zero = open("/dev/zero", O_RDONLY);
+ if (dev_zero < 0) {
+ printf("open /dev/zero: errno=%d\n", errno);
+ exit(2);
+ }
+ }
+ fd = dev_zero;
+#endif
+
+ return runtime_mmap(v, n, PROT_NONE, MAP_ANON|MAP_PRIVATE, fd, 0);
+}
+
void
-runtime_SysMemInit(void)
+runtime_SysMap(void *v, uintptr n)
{
- // Code generators assume that references to addresses
- // on the first page will fault. Map the page explicitly with
- // no permissions, to head off possible bugs like the system
- // allocating that page as the virtual address space fills.
- // Ignore any error, since other systems might be smart
- // enough to never allow anything there.
- runtime_mmap(nil, 4096, PROT_NONE, MAP_FIXED|MAP_ANON|MAP_PRIVATE, -1, 0);
+ void *p;
+ int fd = -1;
+
+ mstats.sys += n;
+
+#ifdef USE_DEV_ZERO
+ if (dev_zero == -1) {
+ dev_zero = open("/dev/zero", O_RDONLY);
+ if (dev_zero < 0) {
+ printf("open /dev/zero: errno=%d\n", errno);
+ exit(2);
+ }
+ }
+ fd = dev_zero;
+#endif
+
+ // On 64-bit, we don't actually have v reserved, so tread carefully.
+ if(sizeof(void*) == 8) {
+ p = runtime_mmap(v, n, PROT_READ|PROT_WRITE|PROT_EXEC, MAP_ANON|MAP_PRIVATE, fd, 0);
+ if(p != v) {
+ runtime_printf("runtime: address space conflict: map(%p) = %p\n", v, p);
+ runtime_throw("runtime: address space conflict");
+ }
+ return;
+ }
+
+ p = runtime_mmap(v, n, PROT_READ|PROT_WRITE|PROT_EXEC, MAP_ANON|MAP_FIXED|MAP_PRIVATE, fd, 0);
+ if(p != v)
+ runtime_throw("runtime: cannot map pages in arena address space");
}
free(v);
}
+void*
+runtime_SysReserve(void *v, uintptr n)
+{
+ return runtime_SysAlloc(n);
+}
+
void
-runtime_SysMemInit(void)
+runtime_SysMap(void *v, uintptr n)
{
}
#include "runtime.h"
#include "malloc.h"
+// Lock to protect finalizer data structures.
+// Cannot reuse mheap.Lock because the finalizer
+// maintenance requires allocation.
static Lock finlock;
void
{
Fintab newtab;
int32 i;
- uint32 *ref;
byte *base;
Finalizer *e;
runtime_throw("finalizer deadlock");
runtime_lock(&finlock);
- if(!runtime_mlookup(p, &base, nil, nil, &ref) || p != base) {
+ if(!runtime_mlookup(p, &base, nil, nil) || p != base) {
runtime_unlock(&finlock);
__sync_bool_compare_and_swap(&m->holds_finlock, 1, 0);
runtime_throw("addfinalizer on invalid pointer");
}
if(f == nil) {
- if(*ref & RefHasFinalizer) {
- lookfintab(&fintab, p, 1);
- *ref &= ~RefHasFinalizer;
- }
+ lookfintab(&fintab, p, 1);
goto unlock;
}
- if(*ref & RefHasFinalizer) {
+ if(lookfintab(&fintab, p, 0)) {
runtime_unlock(&finlock);
__sync_bool_compare_and_swap(&m->holds_finlock, 1, 0);
runtime_throw("double finalizer");
}
- *ref |= RefHasFinalizer;
+ runtime_setblockspecial(p);
if(fintab.nkey >= fintab.max/2+fintab.max/4) {
// keep table at most 3/4 full:
newtab.max *= 3;
}
- newtab.key = runtime_mallocgc(newtab.max*sizeof newtab.key[0], RefNoPointers, 0, 1);
+ newtab.key = runtime_mallocgc(newtab.max*sizeof newtab.key[0], FlagNoPointers, 0, 1);
newtab.val = runtime_mallocgc(newtab.max*sizeof newtab.val[0], 0, 0, 1);
for(i=0; i<fintab.max; i++) {
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
-// Garbage collector -- step 0.
-//
-// Stop the world, mark and sweep garbage collector.
-// NOT INTENDED FOR PRODUCTION USE.
-//
-// A mark and sweep collector provides a way to exercise
-// and test the memory allocator and the stack walking machinery
-// without also needing to get reference counting
-// exactly right.
+// Garbage collector.
#include "runtime.h"
#include "malloc.h"
enum {
- Debug = 0
+ Debug = 0,
+ UseCas = 1,
+ PtrSize = sizeof(void*),
+
+ // Four bits per word (see #defines below).
+ wordsPerBitmapWord = sizeof(void*)*8/4,
+ bitShift = sizeof(void*)*8/4,
};
-typedef struct BlockList BlockList;
-struct BlockList
+// Bits in per-word bitmap.
+// #defines because enum might not be able to hold the values.
+//
+// Each word in the bitmap describes wordsPerBitmapWord words
+// of heap memory. There are 4 bitmap bits dedicated to each heap word,
+// so on a 64-bit system there is one bitmap word per 16 heap words.
+// The bits in the word are packed together by type first, then by
+// heap location, so each 64-bit bitmap word consists of, from top to bottom,
+// the 16 bitSpecial bits for the corresponding heap words, then the 16 bitMarked bits,
+// then the 16 bitNoPointers/bitBlockBoundary bits, then the 16 bitAllocated bits.
+// This layout makes it easier to iterate over the bits of a given type.
+//
+// The bitmap starts at mheap.arena_start and extends *backward* from
+// there. On a 64-bit system the off'th word in the arena is tracked by
+// the off/16+1'th word before mheap.arena_start. (On a 32-bit system,
+// the only difference is that the divisor is 8.)
+//
+// To pull out the bits corresponding to a given pointer p, we use:
+//
+// off = p - (uintptr*)mheap.arena_start; // word offset
+// b = (uintptr*)mheap.arena_start - off/wordsPerBitmapWord - 1;
+// shift = off % wordsPerBitmapWord
+// bits = *b >> shift;
+// /* then test bits & bitAllocated, bits & bitMarked, etc. */
+//
+#define bitAllocated ((uintptr)1<<(bitShift*0))
+#define bitNoPointers ((uintptr)1<<(bitShift*1)) /* when bitAllocated is set */
+#define bitMarked ((uintptr)1<<(bitShift*2)) /* when bitAllocated is set */
+#define bitSpecial ((uintptr)1<<(bitShift*3)) /* when bitAllocated is set - has finalizer or being profiled */
+#define bitBlockBoundary ((uintptr)1<<(bitShift*1)) /* when bitAllocated is NOT set */
+
+#define bitMask (bitBlockBoundary | bitAllocated | bitMarked | bitSpecial)
+
+static uint64 nlookup;
+static uint64 nsizelookup;
+static uint64 naddrlookup;
+static int32 gctrace;
+
+typedef struct Workbuf Workbuf;
+struct Workbuf
{
- byte *obj;
- uintptr size;
+ Workbuf *next;
+ uintptr nw;
+ byte *w[2048-2];
};
static bool finstarted;
static pthread_cond_t finqcond = PTHREAD_COND_INITIALIZER;
static Finalizer *finq;
static int32 fingwait;
-static BlockList *bl, *ebl;
static void runfinq(void*);
-
-enum {
- PtrSize = sizeof(void*)
-};
-
+static Workbuf* getempty(Workbuf*);
+static Workbuf* getfull(Workbuf*);
+
+// scanblock scans a block of n bytes starting at pointer b for references
+// to other objects, scanning any it finds recursively until there are no
+// unscanned objects left. Instead of using an explicit recursion, it keeps
+// a work list in the Workbuf* structures and loops in the main function
+// body. Keeping an explicit work list is easier on the stack allocator and
+// more efficient.
static void
scanblock(byte *b, int64 n)
{
- int32 off;
- void *obj;
- uintptr size;
- uint32 *refp, ref;
+ byte *obj, *arena_start, *p;
void **vp;
- int64 i;
- BlockList *w;
-
- w = bl;
- w->obj = b;
- w->size = n;
- w++;
+ uintptr size, *bitp, bits, shift, i, j, x, xbits, off;
+ MSpan *s;
+ PageID k;
+ void **bw, **w, **ew;
+ Workbuf *wbuf;
- while(w > bl) {
- w--;
- b = w->obj;
- n = w->size;
+ // Memory arena parameters.
+ arena_start = runtime_mheap.arena_start;
+
+ wbuf = nil; // current work buffer
+ ew = nil; // end of work buffer
+ bw = nil; // beginning of work buffer
+ w = nil; // current pointer into work buffer
+
+ // Align b to a word boundary.
+ off = (uintptr)b & (PtrSize-1);
+ if(off != 0) {
+ b += PtrSize - off;
+ n -= PtrSize - off;
+ }
+ for(;;) {
+ // Each iteration scans the block b of length n, queueing pointers in
+ // the work buffer.
if(Debug > 1)
runtime_printf("scanblock %p %lld\n", b, (long long) n);
- off = (uint32)(uintptr)b & (PtrSize-1);
- if(off) {
- b += PtrSize - off;
- n -= PtrSize - off;
- }
-
+
vp = (void**)b;
n /= PtrSize;
- for(i=0; i<n; i++) {
- obj = vp[i];
- if(obj == nil)
+ for(i=0; i<(uintptr)n; i++) {
+ obj = (byte*)vp[i];
+
+ // Words outside the arena cannot be pointers.
+ if((byte*)obj < arena_start || (byte*)obj >= runtime_mheap.arena_used)
continue;
- if(runtime_mheap.min <= (byte*)obj && (byte*)obj < runtime_mheap.max) {
- if(runtime_mlookup(obj, (byte**)&obj, &size, nil, &refp)) {
- ref = *refp;
- switch(ref & ~RefFlags) {
- case RefNone:
- if(Debug > 1)
- runtime_printf("found at %p: ", &vp[i]);
- *refp = RefSome | (ref & RefFlags);
- if(!(ref & RefNoPointers)) {
- if(w >= ebl)
- runtime_throw("scanblock: garbage collection stack overflow");
- w->obj = obj;
- w->size = size;
- w++;
- }
- break;
- }
+
+ // obj may be a pointer to a live object.
+ // Try to find the beginning of the object.
+
+ // Round down to word boundary.
+ obj = (void*)((uintptr)obj & ~((uintptr)PtrSize-1));
+
+ // Find bits for this word.
+ off = (uintptr*)obj - (uintptr*)arena_start;
+ bitp = (uintptr*)arena_start - off/wordsPerBitmapWord - 1;
+ shift = off % wordsPerBitmapWord;
+ xbits = *bitp;
+ bits = xbits >> shift;
+
+ // Pointing at the beginning of a block?
+ if((bits & (bitAllocated|bitBlockBoundary)) != 0)
+ goto found;
+
+ // Pointing just past the beginning?
+ // Scan backward a little to find a block boundary.
+ for(j=shift; j-->0; ) {
+ if(((xbits>>j) & (bitAllocated|bitBlockBoundary)) != 0) {
+ obj = (byte*)obj - (shift-j)*PtrSize;
+ shift = j;
+ bits = xbits>>shift;
+ goto found;
}
}
+
+ // Otherwise consult span table to find beginning.
+ // (Manually inlined copy of MHeap_LookupMaybe.)
+ nlookup++;
+ naddrlookup++;
+ k = (uintptr)obj>>PageShift;
+ x = k;
+ if(sizeof(void*) == 8)
+ x -= (uintptr)arena_start>>PageShift;
+ s = runtime_mheap.map[x];
+ if(s == nil || k < s->start || k - s->start >= s->npages || s->state != MSpanInUse)
+ continue;
+ p = (byte*)((uintptr)s->start<<PageShift);
+ if(s->sizeclass == 0) {
+ obj = p;
+ } else {
+ if((byte*)obj >= (byte*)s->limit)
+ continue;
+ size = runtime_class_to_size[s->sizeclass];
+ int32 i = ((byte*)obj - p)/size;
+ obj = p+i*size;
+ }
+
+ // Now that we know the object header, reload bits.
+ off = (uintptr*)obj - (uintptr*)arena_start;
+ bitp = (uintptr*)arena_start - off/wordsPerBitmapWord - 1;
+ shift = off % wordsPerBitmapWord;
+ xbits = *bitp;
+ bits = xbits >> shift;
+
+ found:
+ // Now we have bits, bitp, and shift correct for
+ // obj pointing at the base of the object.
+ // If not allocated or already marked, done.
+ if((bits & bitAllocated) == 0 || (bits & bitMarked) != 0)
+ continue;
+ *bitp |= bitMarked<<shift;
+
+ // If object has no pointers, don't need to scan further.
+ if((bits & bitNoPointers) != 0)
+ continue;
+
+ // If buffer is full, get a new one.
+ if(w >= ew) {
+ wbuf = getempty(wbuf);
+ bw = (void**)wbuf->w;
+ w = bw;
+ ew = bw + nelem(wbuf->w);
+ }
+ *w++ = obj;
}
+
+ // Done scanning [b, b+n). Prepare for the next iteration of
+ // the loop by setting b and n to the parameters for the next block.
+
+ // Fetch b from the work buffers.
+ if(w <= bw) {
+ // Emptied our buffer: refill.
+ wbuf = getfull(wbuf);
+ if(wbuf == nil)
+ break;
+ bw = (void**)wbuf->w;
+ ew = (void**)(wbuf->w + nelem(wbuf->w));
+ w = bw+wbuf->nw;
+ }
+ b = *--w;
+
+ // Figure out n = size of b. Start by loading bits for b.
+ off = (uintptr*)b - (uintptr*)arena_start;
+ bitp = (uintptr*)arena_start - off/wordsPerBitmapWord - 1;
+ shift = off % wordsPerBitmapWord;
+ xbits = *bitp;
+ bits = xbits >> shift;
+
+ // Might be small; look for nearby block boundary.
+ // A block boundary is marked by either bitBlockBoundary
+ // or bitAllocated being set (see notes near their definition).
+ enum {
+ boundary = bitBlockBoundary|bitAllocated
+ };
+ // Look for a block boundary both after and before b
+ // in the same bitmap word.
+ //
+ // A block boundary j words after b is indicated by
+ // bits>>j & boundary
+ // assuming shift+j < bitShift. (If shift+j >= bitShift then
+ // we'll be bleeding other bit types like bitMarked into our test.)
+ // Instead of inserting the conditional shift+j < bitShift into the loop,
+ // we can let j range from 1 to bitShift as long as we first
+ // apply a mask to keep only the bits corresponding
+ // to shift+j < bitShift aka j < bitShift-shift.
+ bits &= (boundary<<(bitShift-shift)) - boundary;
+
+ // A block boundary j words before b is indicated by
+ // xbits>>(shift-j) & boundary
+ // (assuming shift >= j). There is no cleverness here
+ // avoid the test, because when j gets too large the shift
+ // turns negative, which is undefined in C.
+
+ for(j=1; j<bitShift; j++) {
+ if(((bits>>j)&boundary) != 0 || (shift>=j && ((xbits>>(shift-j))&boundary) != 0)) {
+ n = j*PtrSize;
+ goto scan;
+ }
+ }
+
+ // Fall back to asking span about size class.
+ // (Manually inlined copy of MHeap_Lookup.)
+ nlookup++;
+ nsizelookup++;
+ x = (uintptr)b>>PageShift;
+ if(sizeof(void*) == 8)
+ x -= (uintptr)arena_start>>PageShift;
+ s = runtime_mheap.map[x];
+ if(s->sizeclass == 0)
+ n = s->npages<<PageShift;
+ else
+ n = runtime_class_to_size[s->sizeclass];
+ scan:;
+ }
+}
+
+static struct {
+ Workbuf *full;
+ Workbuf *empty;
+ byte *chunk;
+ uintptr nchunk;
+} work;
+
+// Get an empty work buffer off the work.empty list,
+// allocating new buffers as needed.
+static Workbuf*
+getempty(Workbuf *b)
+{
+ if(b != nil) {
+ b->nw = nelem(b->w);
+ b->next = work.full;
+ work.full = b;
+ }
+ b = work.empty;
+ if(b != nil) {
+ work.empty = b->next;
+ return b;
+ }
+
+ if(work.nchunk < sizeof *b) {
+ work.nchunk = 1<<20;
+ work.chunk = runtime_SysAlloc(work.nchunk);
}
+ b = (Workbuf*)work.chunk;
+ work.chunk += sizeof *b;
+ work.nchunk -= sizeof *b;
+ return b;
}
+// Get a full work buffer off the work.full list, or return nil.
+static Workbuf*
+getfull(Workbuf *b)
+{
+ if(b != nil) {
+ b->nw = 0;
+ b->next = work.empty;
+ work.empty = b;
+ }
+ b = work.full;
+ if(b != nil)
+ work.full = b->next;
+ return b;
+}
+
+// Scanstack calls scanblock on each of gp's stack segments.
static void
markfin(void *v)
{
uintptr size;
- uint32 *refp;
size = 0;
- refp = nil;
- if(!runtime_mlookup(v, (byte**)&v, &size, nil, &refp) || !(*refp & RefHasFinalizer))
+ if(!runtime_mlookup(v, (byte**)&v, &size, nil) || !runtime_blockspecial(v))
runtime_throw("mark - finalizer inconsistency");
-
+
// do not mark the finalizer block itself. just mark the things it points at.
scanblock(v, size);
}
roots = r;
}
+// Mark
static void
mark(void)
{
- uintptr blsize, nobj;
struct root_list *pl;
- // Figure out how big an object stack we need.
- // Get a new one if we need more than we have
- // or we need significantly less than we have.
- nobj = mstats.heap_objects;
- if(nobj > (uintptr)(ebl - bl) || nobj < (uintptr)(ebl-bl)/4) {
- if(bl != nil)
- runtime_SysFree(bl, (byte*)ebl - (byte*)bl);
-
- // While we're allocated a new object stack,
- // add 20% headroom and also round up to
- // the nearest page boundary, since mmap
- // will anyway.
- nobj = nobj * 12/10;
- blsize = nobj * sizeof *bl;
- blsize = (blsize + 4095) & ~4095;
- nobj = blsize / sizeof *bl;
- bl = runtime_SysAlloc(blsize);
- ebl = bl + nobj;
- }
-
for(pl = roots; pl != nil; pl = pl->next) {
struct root* pr = &pl->roots[0];
while(1) {
runtime_walkfintab(markfin, scanblock);
}
-// free RefNone, free & queue finalizers for RefNone|RefHasFinalizer, reset RefSome
+// Sweep frees or calls finalizers for blocks not marked in the mark phase.
+// It clears the mark bits in preparation for the next GC round.
static void
-sweepspan(MSpan *s)
+sweep(void)
{
- int32 n, npages, size;
+ MSpan *s;
+ int32 cl, n, npages;
+ uintptr size;
byte *p;
- uint32 ref, *gcrefp, *gcrefep;
MCache *c;
Finalizer *f;
- p = (byte*)(s->start << PageShift);
- if(s->sizeclass == 0) {
- // Large block.
- ref = s->gcref0;
- switch(ref & ~(RefFlags^RefHasFinalizer)) {
- case RefNone:
- // Free large object.
- mstats.alloc -= s->npages<<PageShift;
- mstats.nfree++;
- runtime_memclr(p, s->npages<<PageShift);
- if(ref & RefProfiled)
- runtime_MProf_Free(p, s->npages<<PageShift);
- s->gcref0 = RefFree;
- runtime_MHeap_Free(&runtime_mheap, s, 1);
- break;
- case RefNone|RefHasFinalizer:
- f = runtime_getfinalizer(p, 1);
- if(f == nil)
- runtime_throw("finalizer inconsistency");
- f->arg = p;
- f->next = finq;
- finq = f;
- ref &= ~RefHasFinalizer;
- // fall through
- case RefSome:
- case RefSome|RefHasFinalizer:
- s->gcref0 = RefNone | (ref&RefFlags);
- break;
+ for(s = runtime_mheap.allspans; s != nil; s = s->allnext) {
+ if(s->state != MSpanInUse)
+ continue;
+
+ p = (byte*)(s->start << PageShift);
+ cl = s->sizeclass;
+ if(cl == 0) {
+ size = s->npages<<PageShift;
+ n = 1;
+ } else {
+ // Chunk full of small blocks.
+ size = runtime_class_to_size[cl];
+ npages = runtime_class_to_allocnpages[cl];
+ n = (npages << PageShift) / size;
}
- return;
- }
+
+ // sweep through n objects of given size starting at p.
+ for(; n > 0; n--, p += size) {
+ uintptr off, *bitp, shift, bits;
- // Chunk full of small blocks.
- runtime_MGetSizeClassInfo(s->sizeclass, &size, &npages, &n);
- gcrefp = s->gcref;
- gcrefep = s->gcref + n;
- for(; gcrefp < gcrefep; gcrefp++, p += size) {
- ref = *gcrefp;
- if(ref < RefNone) // RefFree or RefStack
- continue;
- switch(ref & ~(RefFlags^RefHasFinalizer)) {
- case RefNone:
- // Free small object.
- if(ref & RefProfiled)
+ off = (uintptr*)p - (uintptr*)runtime_mheap.arena_start;
+ bitp = (uintptr*)runtime_mheap.arena_start - off/wordsPerBitmapWord - 1;
+ shift = off % wordsPerBitmapWord;
+ bits = *bitp>>shift;
+
+ if((bits & bitAllocated) == 0)
+ continue;
+
+ if((bits & bitMarked) != 0) {
+ *bitp &= ~(bitMarked<<shift);
+ continue;
+ }
+
+ if((bits & bitSpecial) != 0) {
+ // Special means it has a finalizer or is being profiled.
+ f = runtime_getfinalizer(p, 1);
+ if(f != nil) {
+ f->arg = p;
+ f->next = finq;
+ finq = f;
+ continue;
+ }
runtime_MProf_Free(p, size);
- *gcrefp = RefFree;
- c = m->mcache;
- if(size > (int32)sizeof(uintptr))
- ((uintptr*)p)[1] = 1; // mark as "needs to be zeroed"
+ }
+
+ // Mark freed; restore block boundary bit.
+ *bitp = (*bitp & ~(bitMask<<shift)) | (bitBlockBoundary<<shift);
+
+ if(s->sizeclass == 0) {
+ // Free large span.
+ runtime_unmarkspan(p, 1<<PageShift);
+ *(uintptr*)p = 1; // needs zeroing
+ runtime_MHeap_Free(&runtime_mheap, s, 1);
+ } else {
+ // Free small object.
+ c = m->mcache;
+ if(size > sizeof(uintptr))
+ ((uintptr*)p)[1] = 1; // mark as "needs to be zeroed"
+ mstats.by_size[s->sizeclass].nfree++;
+ runtime_MCache_Free(c, p, s->sizeclass, size);
+ }
mstats.alloc -= size;
mstats.nfree++;
- mstats.by_size[s->sizeclass].nfree++;
- runtime_MCache_Free(c, p, s->sizeclass, size);
- break;
- case RefNone|RefHasFinalizer:
- f = runtime_getfinalizer(p, 1);
- if(f == nil)
- runtime_throw("finalizer inconsistency");
- f->arg = p;
- f->next = finq;
- finq = f;
- ref &= ~RefHasFinalizer;
- // fall through
- case RefSome:
- case RefSome|RefHasFinalizer:
- *gcrefp = RefNone | (ref&RefFlags);
- break;
}
}
}
-static void
-sweep(void)
-{
- MSpan *s;
-
- for(s = runtime_mheap.allspans; s != nil; s = s->allnext)
- if(s->state == MSpanInUse)
- sweepspan(s);
-}
-
static pthread_mutex_t gcsema = PTHREAD_MUTEX_INITIALIZER;
// Initialized from $GOGC. GOGC=off means no gc.
void
runtime_gc(int32 force __attribute__ ((unused)))
{
- int64 t0, t1;
+ int64 t0, t1, t2, t3;
+ uint64 heap0, heap1, obj0, obj1;
char *p;
Finalizer *fp;
gcpercent = -1;
else
gcpercent = runtime_atoi(p);
+
+ p = runtime_getenv("GOGCTRACE");
+ if(p != nil)
+ gctrace = runtime_atoi(p);
}
if(gcpercent < 0)
return;
pthread_mutex_lock(&finqlock);
pthread_mutex_lock(&gcsema);
- m->locks++; // disable gc during the mallocs in newproc
+ if(!force && mstats.heap_alloc < mstats.next_gc) {
+ pthread_mutex_unlock(&gcsema);
+ pthread_mutex_unlock(&finqlock);
+ return;
+ }
+
t0 = runtime_nanotime();
+ nlookup = 0;
+ nsizelookup = 0;
+ naddrlookup = 0;
+
+ m->gcing = 1;
runtime_stoptheworld();
- if(force || mstats.heap_alloc >= mstats.next_gc) {
- __go_cachestats();
- mark();
- sweep();
- __go_stealcache();
- mstats.next_gc = mstats.heap_alloc+mstats.heap_alloc*gcpercent/100;
- }
+ if(runtime_mheap.Lock.key != 0)
+ runtime_throw("runtime_mheap locked during gc");
+ __go_cachestats();
+ heap0 = mstats.heap_alloc;
+ obj0 = mstats.nmalloc - mstats.nfree;
+
+ mark();
t1 = runtime_nanotime();
+ sweep();
+ t2 = runtime_nanotime();
+ __go_stealcache();
+
+ mstats.next_gc = mstats.heap_alloc+mstats.heap_alloc*gcpercent/100;
+ m->gcing = 0;
+
+ m->locks++; // disable gc during the mallocs in newproc
+
+ heap1 = mstats.heap_alloc;
+ obj1 = mstats.nmalloc - mstats.nfree;
+
+ t3 = runtime_nanotime();
+ mstats.pause_ns[mstats.numgc%nelem(mstats.pause_ns)] = t3 - t0;
+ mstats.pause_total_ns += t3 - t0;
mstats.numgc++;
- mstats.pause_ns[mstats.numgc%nelem(mstats.pause_ns)] = t1 - t0;
- mstats.pause_total_ns += t1 - t0;
if(mstats.debuggc)
- runtime_printf("pause %llu\n", (unsigned long long)t1-t0);
+ runtime_printf("pause %llu\n", (unsigned long long)t3-t0);
+
+ if(gctrace) {
+ runtime_printf("gc%d: %llu+%llu+%llu ms %llu -> %llu MB %llu -> %llu (%llu-%llu) objects %llu pointer lookups (%llu size, %llu addr)\n",
+ mstats.numgc, (unsigned long long)(t1-t0)/1000000, (unsigned long long)(t2-t1)/1000000, (unsigned long long)(t3-t2)/1000000,
+ (unsigned long long)heap0>>20, (unsigned long long)heap1>>20, (unsigned long long)obj0, (unsigned long long)obj1,
+ (unsigned long long)mstats.nmalloc, (unsigned long long)mstats.nfree,
+ (unsigned long long)nlookup, (unsigned long long)nsizelookup, (unsigned long long)naddrlookup);
+ }
+
pthread_mutex_unlock(&gcsema);
runtime_starttheworld();
}
m->locks--;
pthread_mutex_unlock(&finqlock);
+
+ if(gctrace > 1 && !force)
+ runtime_gc(1);
}
static void
}
}
+#define runtime_gomaxprocs 2
+
+// mark the block at v of size n as allocated.
+// If noptr is true, mark it as having no pointers.
+void
+runtime_markallocated(void *v, uintptr n, bool noptr)
+{
+ uintptr *b, obits, bits, off, shift;
+
+ // if(0)
+ // runtime_printf("markallocated %p+%p\n", v, n);
+
+ if((byte*)v+n > (byte*)runtime_mheap.arena_used || (byte*)v < runtime_mheap.arena_start)
+ runtime_throw("markallocated: bad pointer");
+
+ off = (uintptr*)v - (uintptr*)runtime_mheap.arena_start; // word offset
+ b = (uintptr*)runtime_mheap.arena_start - off/wordsPerBitmapWord - 1;
+ shift = off % wordsPerBitmapWord;
+
+ for(;;) {
+ obits = *b;
+ bits = (obits & ~(bitMask<<shift)) | (bitAllocated<<shift);
+ if(noptr)
+ bits |= bitNoPointers<<shift;
+ if(runtime_gomaxprocs == 1) {
+ *b = bits;
+ break;
+ } else {
+ // gomaxprocs > 1: use atomic op
+ if(runtime_casp((void**)b, (void*)obits, (void*)bits))
+ break;
+ }
+ }
+}
+
+// mark the block at v of size n as freed.
+void
+runtime_markfreed(void *v, uintptr n)
+{
+ uintptr *b, obits, bits, off, shift;
+
+ // if(0)
+ // runtime_printf("markallocated %p+%p\n", v, n);
+
+ if((byte*)v+n > (byte*)runtime_mheap.arena_used || (byte*)v < runtime_mheap.arena_start)
+ runtime_throw("markallocated: bad pointer");
+
+ off = (uintptr*)v - (uintptr*)runtime_mheap.arena_start; // word offset
+ b = (uintptr*)runtime_mheap.arena_start - off/wordsPerBitmapWord - 1;
+ shift = off % wordsPerBitmapWord;
+
+ for(;;) {
+ obits = *b;
+ bits = (obits & ~(bitMask<<shift)) | (bitBlockBoundary<<shift);
+ if(runtime_gomaxprocs == 1) {
+ *b = bits;
+ break;
+ } else {
+ // gomaxprocs > 1: use atomic op
+ if(runtime_casp((void**)b, (void*)obits, (void*)bits))
+ break;
+ }
+ }
+}
+
+// check that the block at v of size n is marked freed.
+void
+runtime_checkfreed(void *v, uintptr n)
+{
+ uintptr *b, bits, off, shift;
+
+ if(!runtime_checking)
+ return;
+
+ if((byte*)v+n > (byte*)runtime_mheap.arena_used || (byte*)v < runtime_mheap.arena_start)
+ return; // not allocated, so okay
+
+ off = (uintptr*)v - (uintptr*)runtime_mheap.arena_start; // word offset
+ b = (uintptr*)runtime_mheap.arena_start - off/wordsPerBitmapWord - 1;
+ shift = off % wordsPerBitmapWord;
+
+ bits = *b>>shift;
+ if((bits & bitAllocated) != 0) {
+ runtime_printf("checkfreed %p+%p: off=%p have=%p\n",
+ v, (void*)n, (void*)off, (void*)(bits & bitMask));
+ runtime_throw("checkfreed: not freed");
+ }
+}
+
+// mark the span of memory at v as having n blocks of the given size.
+// if leftover is true, there is left over space at the end of the span.
+void
+runtime_markspan(void *v, uintptr size, uintptr n, bool leftover)
+{
+ uintptr *b, off, shift;
+ byte *p;
+
+ if((byte*)v+size*n > (byte*)runtime_mheap.arena_used || (byte*)v < runtime_mheap.arena_start)
+ runtime_throw("markspan: bad pointer");
+
+ p = v;
+ if(leftover) // mark a boundary just past end of last block too
+ n++;
+ for(; n-- > 0; p += size) {
+ // Okay to use non-atomic ops here, because we control
+ // the entire span, and each bitmap word has bits for only
+ // one span, so no other goroutines are changing these
+ // bitmap words.
+ off = (uintptr*)p - (uintptr*)runtime_mheap.arena_start; // word offset
+ b = (uintptr*)runtime_mheap.arena_start - off/wordsPerBitmapWord - 1;
+ shift = off % wordsPerBitmapWord;
+ *b = (*b & ~(bitMask<<shift)) | (bitBlockBoundary<<shift);
+ }
+}
+
+// unmark the span of memory at v of length n bytes.
+void
+runtime_unmarkspan(void *v, uintptr n)
+{
+ uintptr *p, *b, off;
+
+ if((byte*)v+n > (byte*)runtime_mheap.arena_used || (byte*)v < runtime_mheap.arena_start)
+ runtime_throw("markspan: bad pointer");
+
+ p = v;
+ off = p - (uintptr*)runtime_mheap.arena_start; // word offset
+ if(off % wordsPerBitmapWord != 0)
+ runtime_throw("markspan: unaligned pointer");
+ b = (uintptr*)runtime_mheap.arena_start - off/wordsPerBitmapWord - 1;
+ n /= PtrSize;
+ if(n%wordsPerBitmapWord != 0)
+ runtime_throw("unmarkspan: unaligned length");
+ // Okay to use non-atomic ops here, because we control
+ // the entire span, and each bitmap word has bits for only
+ // one span, so no other goroutines are changing these
+ // bitmap words.
+ n /= wordsPerBitmapWord;
+ while(n-- > 0)
+ *b-- = 0;
+}
+
+bool
+runtime_blockspecial(void *v)
+{
+ uintptr *b, off, shift;
+
+ off = (uintptr*)v - (uintptr*)runtime_mheap.arena_start;
+ b = (uintptr*)runtime_mheap.arena_start - off/wordsPerBitmapWord - 1;
+ shift = off % wordsPerBitmapWord;
+
+ return (*b & (bitSpecial<<shift)) != 0;
+}
+
+void
+runtime_setblockspecial(void *v)
+{
+ uintptr *b, off, shift, bits, obits;
+
+ off = (uintptr*)v - (uintptr*)runtime_mheap.arena_start;
+ b = (uintptr*)runtime_mheap.arena_start - off/wordsPerBitmapWord - 1;
+ shift = off % wordsPerBitmapWord;
+
+ for(;;) {
+ obits = *b;
+ bits = obits | (bitSpecial<<shift);
+ if(runtime_gomaxprocs == 1) {
+ *b = bits;
+ break;
+ } else {
+ // gomaxprocs > 1: use atomic op
+ if(runtime_casp((void**)b, (void*)obits, (void*)bits))
+ break;
+ }
+ }
+}
+
+void
+runtime_MHeap_MapBits(MHeap *h)
+{
+ // Caller has added extra mappings to the arena.
+ // Add extra mappings of bitmap words as needed.
+ // We allocate extra bitmap pieces in chunks of bitmapChunk.
+ enum {
+ bitmapChunk = 8192
+ };
+ uintptr n;
+
+ n = (h->arena_used - h->arena_start) / wordsPerBitmapWord;
+ n = (n+bitmapChunk-1) & ~(bitmapChunk-1);
+ if(h->bitmap_mapped >= n)
+ return;
+
+ runtime_SysMap(h->arena_start - n, n - h->bitmap_mapped);
+ h->bitmap_mapped = n;
+}
+
void
__go_enable_gc()
{
runtime_initlock(h);
runtime_FixAlloc_Init(&h->spanalloc, sizeof(MSpan), alloc, RecordSpan, h);
runtime_FixAlloc_Init(&h->cachealloc, sizeof(MCache), alloc, nil, nil);
- runtime_MHeapMap_Init(&h->map, alloc);
// h->mapcache needs no init
for(i=0; i<nelem(h->free); i++)
runtime_MSpanList_Init(&h->free[i]);
{
uintptr n;
MSpan *s, *t;
+ PageID p;
// Try in fixed-size lists up to max.
for(n=npage; n < nelem(h->free); n++) {
mstats.mspan_sys = h->spanalloc.sys;
runtime_MSpan_Init(t, s->start + npage, s->npages - npage);
s->npages = npage;
- runtime_MHeapMap_Set(&h->map, t->start - 1, s);
- runtime_MHeapMap_Set(&h->map, t->start, t);
- runtime_MHeapMap_Set(&h->map, t->start + t->npages - 1, t);
+ p = t->start;
+ if(sizeof(void*) == 8)
+ p -= ((uintptr)h->arena_start>>PageShift);
+ if(p > 0)
+ h->map[p-1] = s;
+ h->map[p] = t;
+ h->map[p+t->npages-1] = t;
+ *(uintptr*)(t->start<<PageShift) = *(uintptr*)(s->start<<PageShift); // copy "needs zeroing" mark
t->state = MSpanInUse;
MHeap_FreeLocked(h, t);
}
+ if(*(uintptr*)(s->start<<PageShift) != 0)
+ runtime_memclr((byte*)(s->start<<PageShift), s->npages<<PageShift);
+
// Record span info, because gc needs to be
// able to map interior pointer to containing span.
s->sizeclass = sizeclass;
+ p = s->start;
+ if(sizeof(void*) == 8)
+ p -= ((uintptr)h->arena_start>>PageShift);
for(n=0; n<npage; n++)
- runtime_MHeapMap_Set(&h->map, s->start+n, s);
+ h->map[p+n] = s;
return s;
}
uintptr ask;
void *v;
MSpan *s;
+ PageID p;
// Ask for a big chunk, to reduce the number of mappings
// the operating system needs to track; also amortizes
// Allocate a multiple of 64kB (16 pages).
npage = (npage+15)&~15;
ask = npage<<PageShift;
- if(ask < HeapAllocChunk)
+ if(ask > (uintptr)(h->arena_end - h->arena_used))
+ return false;
+ if(ask < HeapAllocChunk && HeapAllocChunk <= h->arena_end - h->arena_used)
ask = HeapAllocChunk;
- v = runtime_SysAlloc(ask);
+ v = runtime_MHeap_SysAlloc(h, ask);
if(v == nil) {
if(ask > (npage<<PageShift)) {
ask = npage<<PageShift;
- v = runtime_SysAlloc(ask);
+ v = runtime_MHeap_SysAlloc(h, ask);
}
if(v == nil)
return false;
}
mstats.heap_sys += ask;
- if((byte*)v < h->min || h->min == nil)
- h->min = v;
- if((byte*)v+ask > h->max)
- h->max = (byte*)v+ask;
-
- // NOTE(rsc): In tcmalloc, if we've accumulated enough
- // system allocations, the heap map gets entirely allocated
- // in 32-bit mode. (In 64-bit mode that's not practical.)
- if(!runtime_MHeapMap_Preallocate(&h->map, ((uintptr)v>>PageShift) - 1, (ask>>PageShift) + 2)) {
- runtime_SysFree(v, ask);
- return false;
- }
-
// Create a fake "in use" span and free it, so that the
// right coalescing happens.
s = runtime_FixAlloc_Alloc(&h->spanalloc);
mstats.mspan_inuse = h->spanalloc.inuse;
mstats.mspan_sys = h->spanalloc.sys;
runtime_MSpan_Init(s, (uintptr)v>>PageShift, ask>>PageShift);
- runtime_MHeapMap_Set(&h->map, s->start, s);
- runtime_MHeapMap_Set(&h->map, s->start + s->npages - 1, s);
+ p = s->start;
+ if(sizeof(void*) == 8)
+ p -= ((uintptr)h->arena_start>>PageShift);
+ h->map[p] = s;
+ h->map[p + s->npages - 1] = s;
s->state = MSpanInUse;
MHeap_FreeLocked(h, s);
return true;
}
-// Look up the span at the given page number.
-// Page number is guaranteed to be in map
+// Look up the span at the given address.
+// Address is guaranteed to be in map
// and is guaranteed to be start or end of span.
MSpan*
-runtime_MHeap_Lookup(MHeap *h, PageID p)
+runtime_MHeap_Lookup(MHeap *h, void *v)
{
- return runtime_MHeapMap_Get(&h->map, p);
+ uintptr p;
+
+ p = (uintptr)v;
+ if(sizeof(void*) == 8)
+ p -= (uintptr)h->arena_start;
+ return h->map[p >> PageShift];
}
-// Look up the span at the given page number.
-// Page number is *not* guaranteed to be in map
+// Look up the span at the given address.
+// Address is *not* guaranteed to be in map
// and may be anywhere in the span.
// Map entries for the middle of a span are only
// valid for allocated spans. Free spans may have
// other garbage in their middles, so we have to
// check for that.
MSpan*
-runtime_MHeap_LookupMaybe(MHeap *h, PageID p)
+runtime_MHeap_LookupMaybe(MHeap *h, void *v)
{
MSpan *s;
+ PageID p, q;
- s = runtime_MHeapMap_GetMaybe(&h->map, p);
+ if((byte*)v < h->arena_start || (byte*)v >= h->arena_used)
+ return nil;
+ p = (uintptr)v>>PageShift;
+ q = p;
+ if(sizeof(void*) == 8)
+ q -= (uintptr)h->arena_start >> PageShift;
+ s = h->map[q];
if(s == nil || p < s->start || p - s->start >= s->npages)
return nil;
if(s->state != MSpanInUse)
static void
MHeap_FreeLocked(MHeap *h, MSpan *s)
{
+ uintptr *sp, *tp;
MSpan *t;
+ PageID p;
if(s->state != MSpanInUse || s->ref != 0) {
// runtime_printf("MHeap_FreeLocked - span %p ptr %p state %d ref %d\n", s, s->start<<PageShift, s->state, s->ref);
}
s->state = MSpanFree;
runtime_MSpanList_Remove(s);
+ sp = (uintptr*)(s->start<<PageShift);
// Coalesce with earlier, later spans.
- if((t = runtime_MHeapMap_Get(&h->map, s->start - 1)) != nil && t->state != MSpanInUse) {
+ p = s->start;
+ if(sizeof(void*) == 8)
+ p -= (uintptr)h->arena_start >> PageShift;
+ if(p > 0 && (t = h->map[p-1]) != nil && t->state != MSpanInUse) {
+ tp = (uintptr*)(t->start<<PageShift);
+ *tp |= *sp; // propagate "needs zeroing" mark
s->start = t->start;
s->npages += t->npages;
- runtime_MHeapMap_Set(&h->map, s->start, s);
+ p -= t->npages;
+ h->map[p] = s;
runtime_MSpanList_Remove(t);
t->state = MSpanDead;
runtime_FixAlloc_Free(&h->spanalloc, t);
mstats.mspan_inuse = h->spanalloc.inuse;
mstats.mspan_sys = h->spanalloc.sys;
}
- if((t = runtime_MHeapMap_Get(&h->map, s->start + s->npages)) != nil && t->state != MSpanInUse) {
+ if(p+s->npages < nelem(h->map) && (t = h->map[p+s->npages]) != nil && t->state != MSpanInUse) {
+ tp = (uintptr*)(t->start<<PageShift);
+ *sp |= *tp; // propagate "needs zeroing" mark
s->npages += t->npages;
- runtime_MHeapMap_Set(&h->map, s->start + s->npages - 1, s);
+ h->map[p + s->npages - 1] = s;
runtime_MSpanList_Remove(t);
t->state = MSpanDead;
runtime_FixAlloc_Free(&h->spanalloc, t);
void
runtime_MSpanList_Insert(MSpan *list, MSpan *span)
{
- if(span->next != nil || span->prev != nil)
+ if(span->next != nil || span->prev != nil) {
+ // runtime_printf("failed MSpanList_Insert %p %p %p\n", span, span->next, span->prev);
runtime_throw("MSpanList_Insert");
+ }
span->next = list->next;
span->prev = list;
span->next->prev = span;
span->prev->next = span;
}
+
+
+++ /dev/null
-// Copyright 2009 The Go Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-// Heap map, 32-bit version
-// See malloc.h and mheap.c for overview.
-
-#include "runtime.h"
-#include "malloc.h"
-
-#if __SIZEOF_POINTER__ == 4
-
-// 3-level radix tree mapping page ids to Span*.
-void
-runtime_MHeapMap_Init(MHeapMap *m, void *(*allocator)(uintptr))
-{
- m->allocator = allocator;
-}
-
-MSpan*
-runtime_MHeapMap_Get(MHeapMap *m, PageID k)
-{
- int32 i1, i2;
-
- i2 = k & MHeapMap_Level2Mask;
- k >>= MHeapMap_Level2Bits;
- i1 = k & MHeapMap_Level1Mask;
- k >>= MHeapMap_Level1Bits;
- if(k != 0)
- runtime_throw("MHeapMap_Get");
-
- return m->p[i1]->s[i2];
-}
-
-MSpan*
-runtime_MHeapMap_GetMaybe(MHeapMap *m, PageID k)
-{
- int32 i1, i2;
- MHeapMapNode2 *p2;
-
- i2 = k & MHeapMap_Level2Mask;
- k >>= MHeapMap_Level2Bits;
- i1 = k & MHeapMap_Level1Mask;
- k >>= MHeapMap_Level1Bits;
- if(k != 0)
- runtime_throw("MHeapMap_Get");
-
- p2 = m->p[i1];
- if(p2 == nil)
- return nil;
- return p2->s[i2];
-}
-
-void
-runtime_MHeapMap_Set(MHeapMap *m, PageID k, MSpan *s)
-{
- int32 i1, i2;
-
- i2 = k & MHeapMap_Level2Mask;
- k >>= MHeapMap_Level2Bits;
- i1 = k & MHeapMap_Level1Mask;
- k >>= MHeapMap_Level1Bits;
- if(k != 0)
- runtime_throw("MHeapMap_Set");
-
- m->p[i1]->s[i2] = s;
-}
-
-// Allocate the storage required for entries [k, k+1, ..., k+len-1]
-// so that Get and Set calls need not check for nil pointers.
-bool
-runtime_MHeapMap_Preallocate(MHeapMap *m, PageID k, uintptr len)
-{
- uintptr end;
- int32 i1;
- MHeapMapNode2 *p2;
-
- end = k+len;
- while(k < end) {
- if((k >> MHeapMap_TotalBits) != 0)
- return false;
- i1 = (k >> MHeapMap_Level2Bits) & MHeapMap_Level1Mask;
-
- // first-level pointer
- if(m->p[i1] == nil) {
- p2 = m->allocator(sizeof *p2);
- if(p2 == nil)
- return false;
- mstats.heapmap_sys += sizeof *p2;
- m->p[i1] = p2;
- }
-
- // advance key past this leaf node
- k = ((k >> MHeapMap_Level2Bits) + 1) << MHeapMap_Level2Bits;
- }
- return true;
-}
-
-#endif /* __SIZEOF_POINTER__ == 4 */
+++ /dev/null
-// Copyright 2009 The Go Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-// Free(v) must be able to determine the MSpan containing v.
-// The MHeapMap is a 2-level radix tree mapping page numbers to MSpans.
-
-typedef struct MHeapMapNode2 MHeapMapNode2;
-
-enum
-{
- // 32 bit address - 12 bit page size = 20 bits to map
- MHeapMap_Level1Bits = 10,
- MHeapMap_Level2Bits = 10,
-
- MHeapMap_TotalBits =
- MHeapMap_Level1Bits +
- MHeapMap_Level2Bits,
-
- MHeapMap_Level1Mask = (1<<MHeapMap_Level1Bits) - 1,
- MHeapMap_Level2Mask = (1<<MHeapMap_Level2Bits) - 1,
-};
-
-struct MHeapMap
-{
- void *(*allocator)(uintptr);
- MHeapMapNode2 *p[1<<MHeapMap_Level1Bits];
-};
-
-struct MHeapMapNode2
-{
- MSpan *s[1<<MHeapMap_Level2Bits];
-};
-
-void runtime_MHeapMap_Init(MHeapMap *m, void *(*allocator)(uintptr));
-bool runtime_MHeapMap_Preallocate(MHeapMap *m, PageID k, uintptr npages);
-MSpan* runtime_MHeapMap_Get(MHeapMap *m, PageID k);
-MSpan* runtime_MHeapMap_GetMaybe(MHeapMap *m, PageID k);
-void runtime_MHeapMap_Set(MHeapMap *m, PageID k, MSpan *v);
-
-
+++ /dev/null
-// Copyright 2009 The Go Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-// Heap map, 64-bit version
-// See malloc.h and mheap.c for overview.
-
-#include "runtime.h"
-#include "malloc.h"
-
-#if __SIZEOF_POINTER__ == 8
-
-// 3-level radix tree mapping page ids to Span*.
-void
-runtime_MHeapMap_Init(MHeapMap *m, void *(*allocator)(uintptr))
-{
- m->allocator = allocator;
-}
-
-MSpan*
-runtime_MHeapMap_Get(MHeapMap *m, PageID k)
-{
- int32 i1, i2, i3;
-
- i3 = k & MHeapMap_Level3Mask;
- k >>= MHeapMap_Level3Bits;
- i2 = k & MHeapMap_Level2Mask;
- k >>= MHeapMap_Level2Bits;
- i1 = k & MHeapMap_Level1Mask;
- k >>= MHeapMap_Level1Bits;
- if(k != 0)
- runtime_throw("MHeapMap_Get");
-
- return m->p[i1]->p[i2]->s[i3];
-}
-
-MSpan*
-runtime_MHeapMap_GetMaybe(MHeapMap *m, PageID k)
-{
- int32 i1, i2, i3;
- MHeapMapNode2 *p2;
- MHeapMapNode3 *p3;
-
- i3 = k & MHeapMap_Level3Mask;
- k >>= MHeapMap_Level3Bits;
- i2 = k & MHeapMap_Level2Mask;
- k >>= MHeapMap_Level2Bits;
- i1 = k & MHeapMap_Level1Mask;
- k >>= MHeapMap_Level1Bits;
- if(k != 0)
- runtime_throw("MHeapMap_Get");
-
- p2 = m->p[i1];
- if(p2 == nil)
- return nil;
- p3 = p2->p[i2];
- if(p3 == nil)
- return nil;
- return p3->s[i3];
-}
-
-void
-runtime_MHeapMap_Set(MHeapMap *m, PageID k, MSpan *s)
-{
- int32 i1, i2, i3;
-
- i3 = k & MHeapMap_Level3Mask;
- k >>= MHeapMap_Level3Bits;
- i2 = k & MHeapMap_Level2Mask;
- k >>= MHeapMap_Level2Bits;
- i1 = k & MHeapMap_Level1Mask;
- k >>= MHeapMap_Level1Bits;
- if(k != 0)
- runtime_throw("MHeapMap_Set");
-
- m->p[i1]->p[i2]->s[i3] = s;
-}
-
-// Allocate the storage required for entries [k, k+1, ..., k+len-1]
-// so that Get and Set calls need not check for nil pointers.
-bool
-runtime_MHeapMap_Preallocate(MHeapMap *m, PageID k, uintptr len)
-{
- uintptr end;
- int32 i1, i2;
- MHeapMapNode2 *p2;
- MHeapMapNode3 *p3;
-
- end = k+len;
- while(k < end) {
- if((k >> MHeapMap_TotalBits) != 0)
- return false;
- i2 = (k >> MHeapMap_Level3Bits) & MHeapMap_Level2Mask;
- i1 = (k >> (MHeapMap_Level3Bits + MHeapMap_Level2Bits)) & MHeapMap_Level1Mask;
-
- // first-level pointer
- if((p2 = m->p[i1]) == nil) {
- p2 = m->allocator(sizeof *p2);
- if(p2 == nil)
- return false;
- mstats.heapmap_sys += sizeof *p2;
- m->p[i1] = p2;
- }
-
- // second-level pointer
- if(p2->p[i2] == nil) {
- p3 = m->allocator(sizeof *p3);
- if(p3 == nil)
- return false;
- mstats.heapmap_sys += sizeof *p3;
- p2->p[i2] = p3;
- }
-
- // advance key past this leaf node
- k = ((k >> MHeapMap_Level3Bits) + 1) << MHeapMap_Level3Bits;
- }
- return true;
-}
-
-#endif /* __SIZEOF_POINTER__ == 8 */
+++ /dev/null
-// Copyright 2009 The Go Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-// Free(v) must be able to determine the MSpan containing v.
-// The MHeapMap is a 3-level radix tree mapping page numbers to MSpans.
-//
-// NOTE(rsc): On a 32-bit platform (= 20-bit page numbers),
-// we can swap in a 2-level radix tree.
-//
-// NOTE(rsc): We use a 3-level tree because tcmalloc does, but
-// having only three levels requires approximately 1 MB per node
-// in the tree, making the minimum map footprint 3 MB.
-// Using a 4-level tree would cut the minimum footprint to 256 kB.
-// On the other hand, it's just virtual address space: most of
-// the memory is never going to be touched, thus never paged in.
-
-typedef struct MHeapMapNode2 MHeapMapNode2;
-typedef struct MHeapMapNode3 MHeapMapNode3;
-
-enum
-{
- // 64 bit address - 12 bit page size = 52 bits to map
- MHeapMap_Level1Bits = 18,
- MHeapMap_Level2Bits = 18,
- MHeapMap_Level3Bits = 16,
-
- MHeapMap_TotalBits =
- MHeapMap_Level1Bits +
- MHeapMap_Level2Bits +
- MHeapMap_Level3Bits,
-
- MHeapMap_Level1Mask = (1<<MHeapMap_Level1Bits) - 1,
- MHeapMap_Level2Mask = (1<<MHeapMap_Level2Bits) - 1,
- MHeapMap_Level3Mask = (1<<MHeapMap_Level3Bits) - 1,
-};
-
-struct MHeapMap
-{
- void *(*allocator)(uintptr);
- MHeapMapNode2 *p[1<<MHeapMap_Level1Bits];
-};
-
-struct MHeapMapNode2
-{
- MHeapMapNode3 *p[1<<MHeapMap_Level2Bits];
-};
-
-struct MHeapMapNode3
-{
- MSpan *s[1<<MHeapMap_Level3Bits];
-};
-
-void runtime_MHeapMap_Init(MHeapMap *m, void *(*allocator)(uintptr));
-bool runtime_MHeapMap_Preallocate(MHeapMap *m, PageID k, uintptr npages);
-MSpan* runtime_MHeapMap_Get(MHeapMap *m, PageID k);
-MSpan* runtime_MHeapMap_GetMaybe(MHeapMap *m, PageID k);
-void runtime_MHeapMap_Set(MHeapMap *m, PageID k, MSpan *v);
-
-
runtime_mcmp((byte*)b->stk, (byte*)stk, nstk*sizeof stk[0]) == 0)
return b;
- b = runtime_mallocgc(sizeof *b + nstk*sizeof stk[0], RefNoProfiling, 0, 1);
+ b = runtime_mallocgc(sizeof *b + nstk*sizeof stk[0], FlagNoProfiling, 0, 1);
bucketmem += sizeof *b + nstk*sizeof stk[0];
runtime_memmove(b->stk, stk, nstk*sizeof stk[0]);
b->hash = h;
if(ah->addr == (addr>>20))
goto found;
- ah = runtime_mallocgc(sizeof *ah, RefNoProfiling, 0, 1);
+ ah = runtime_mallocgc(sizeof *ah, FlagNoProfiling, 0, 1);
addrmem += sizeof *ah;
ah->next = addrhash[h];
ah->addr = addr>>20;
found:
if((e = addrfree) == nil) {
- e = runtime_mallocgc(64*sizeof *e, RefNoProfiling, 0, 0);
+ e = runtime_mallocgc(64*sizeof *e, FlagNoProfiling, 0, 0);
addrmem += 64*sizeof *e;
for(i=0; i+1<64; i++)
e[i].next = &e[i+1];
void
runtime_InitSizes(void)
{
- int32 align, sizeclass, size, osize, nextsize, n;
+ int32 align, sizeclass, size, nextsize, n;
uint32 i;
uintptr allocsize, npages;
// the leftover is less than 1/8 of the total,
// so wasted space is at most 12.5%.
allocsize = PageSize;
- osize = size + RefcountOverhead;
- while(allocsize%osize > (allocsize/8))
+ while(allocsize%size > allocsize/8)
allocsize += PageSize;
npages = allocsize >> PageShift;
// different sizes.
if(sizeclass > 1
&& (int32)npages == runtime_class_to_allocnpages[sizeclass-1]
- && allocsize/osize == allocsize/(runtime_class_to_size[sizeclass-1]+RefcountOverhead)) {
+ && allocsize/size == allocsize/runtime_class_to_size[sizeclass-1]) {
runtime_class_to_size[sizeclass-1] = size;
continue;
}
#define runtime_mmap mmap
#define runtime_munmap(p, s) munmap((p), (s))
#define runtime_cas(pval, old, new) __sync_bool_compare_and_swap (pval, old, new)
+#define runtime_casp(pval, old, new) __sync_bool_compare_and_swap (pval, old, new)
struct __go_func_type;
void reflect_call(const struct __go_func_type *, const void *, _Bool, void **,
s = buf;
}
int32 len = __builtin_strlen(s);
- unsigned char *data = runtime_mallocgc(len, RefNoPointers, 0, 0);
+ unsigned char *data = runtime_mallocgc(len, FlagNoPointers, 0, 0);
__builtin_memcpy(data, s, len);
name.__data = data;
name.__length = len;
func libc_fcntl(fd int, cmd int, arg int) int __asm__ ("fcntl")
func libc_fork() Pid_t __asm__ ("fork")
-func libc_chdir(name *byte) int __asm__ ("chdir");
+func libc_chdir(name *byte) int __asm__ ("chdir")
func libc_dup2(int, int) int __asm__ ("dup2")
func libc_execve(*byte, **byte, **byte) int __asm__ ("execve")
func libc_sysexit(int) __asm__ ("_exit")
func forkAndExecInChild(argv0 *byte, argv []*byte, envv []*byte, traceme bool, dir *byte, fd []int, pipe int) (pid int, err int) {
// Declare all variables at top in case any
// declarations require heap allocation (e.g., err1).
- var r1, r2, err1 uintptr;
- var nextfd int;
- var i int;
+ var r1, r2, err1 uintptr
+ var nextfd int
+ var i int
- darwin := OS == "darwin";
+ darwin := OS == "darwin"
// About to call fork.
// No more allocation or calls of non-assembly functions.
- child := libc_fork();
+ child := libc_fork()
if child == -1 {
- return 0, GetErrno();
+ return 0, GetErrno()
}
if child != 0 {
// Enable tracing if requested.
if traceme {
if libc_ptrace(_PTRACE_TRACEME, 0, 0, nil) < 0 {
- goto childerror;
+ goto childerror
}
}
// Chdir
if dir != nil {
- r := libc_chdir(dir);
+ r := libc_chdir(dir)
if r < 0 {
- goto childerror;
+ goto childerror
}
}
// Pass 1: look for fd[i] < i and move those up above len(fd)
// so that pass 2 won't stomp on an fd it needs later.
- nextfd = int(len(fd));
+ nextfd = int(len(fd))
if pipe < nextfd {
- r := libc_dup2(pipe, nextfd);
+ r := libc_dup2(pipe, nextfd)
if r == -1 {
- goto childerror;
+ goto childerror
}
- libc_fcntl(nextfd, F_SETFD, FD_CLOEXEC);
- pipe = nextfd;
- nextfd++;
+ libc_fcntl(nextfd, F_SETFD, FD_CLOEXEC)
+ pipe = nextfd
+ nextfd++
}
for i = 0; i < len(fd); i++ {
if fd[i] >= 0 && fd[i] < int(i) {
- r := libc_dup2(fd[i], nextfd);
+ r := libc_dup2(fd[i], nextfd)
if r == -1 {
- goto childerror;
+ goto childerror
}
- libc_fcntl(nextfd, F_SETFD, FD_CLOEXEC);
- fd[i] = nextfd;
- nextfd++;
- if nextfd == pipe { // don't stomp on pipe
- nextfd++;
+ libc_fcntl(nextfd, F_SETFD, FD_CLOEXEC)
+ fd[i] = nextfd
+ nextfd++
+ if nextfd == pipe { // don't stomp on pipe
+ nextfd++
}
}
}
// Pass 2: dup fd[i] down onto i.
for i = 0; i < len(fd); i++ {
if fd[i] == -1 {
- libc_close(i);
- continue;
+ libc_close(i)
+ continue
}
if fd[i] == int(i) {
// dup2(i, i) won't clear close-on-exec flag on Linux,
// probably not elsewhere either.
- r := libc_fcntl(fd[i], F_SETFD, 0);
+ r := libc_fcntl(fd[i], F_SETFD, 0)
if r != 0 {
- goto childerror;
+ goto childerror
}
- continue;
+ continue
}
// The new fd is created NOT close-on-exec,
// which is exactly what we want.
- r := libc_dup2(fd[i], i);
+ r := libc_dup2(fd[i], i)
if r == -1 {
- goto childerror;
+ goto childerror
}
}
// Programs that know they inherit fds >= 3 will need
// to set them close-on-exec.
for i = len(fd); i < 3; i++ {
- libc_close(i);
+ libc_close(i)
}
// Time to exec.
- libc_execve(argv0, &argv[0], &envv[0]);
+ libc_execve(argv0, &argv[0], &envv[0])
childerror:
// send error code on pipe
- var e uintptr = uintptr(GetErrno());
+ var e uintptr = uintptr(GetErrno())
libc_write(pipe, (*byte)(unsafe.Pointer(&e)),
- Size_t(unsafe.Sizeof(err1)));
+ Size_t(unsafe.Sizeof(err1)))
for {
libc_sysexit(253)
}
// Calling panic is not actually safe,
// but the for loop above won't break
// and this shuts up the compiler.
- panic("unreached");
+ panic("unreached")
}
func forkExec(argv0 string, argv []string, envv []string, traceme bool, dir string, fd []int) (pid int, err int) {
- var p [2]int;
- var r1 int;
- var err1 uintptr;
- var wstatus WaitStatus;
+ var p [2]int
+ var r1 int
+ var err1 uintptr
+ var wstatus WaitStatus
- p[0] = -1;
- p[1] = -1;
+ p[0] = -1
+ p[1] = -1
// Convert args to C form.
- argv0p := StringBytePtr(argv0);
- argvp := StringArrayPtr(argv);
- envvp := StringArrayPtr(envv);
- var dirp *byte;
+ argv0p := StringBytePtr(argv0)
+ argvp := StringArrayPtr(argv)
+ envvp := StringArrayPtr(envv)
+ var dirp *byte
if len(dir) > 0 {
- dirp = StringBytePtr(dir);
+ dirp = StringBytePtr(dir)
}
// Acquire the fork lock so that no other threads
// create new fds that are not yet close-on-exec
// before we fork.
- ForkLock.Lock();
+ ForkLock.Lock()
// Allocate child status pipe close on exec.
if err = Pipe(p[0:]); err != 0 {
- goto error;
+ goto error
}
- var val int;
- if val, err = fcntl(p[0], F_SETFD, FD_CLOEXEC); err != 0 {
- goto error;
+ if _, err = fcntl(p[0], F_SETFD, FD_CLOEXEC); err != 0 {
+ goto error
}
- if val, err = fcntl(p[1], F_SETFD, FD_CLOEXEC); err != 0 {
- goto error;
+ if _, err = fcntl(p[1], F_SETFD, FD_CLOEXEC); err != 0 {
+ goto error
}
// Kick off child.
- pid, err = forkAndExecInChild(argv0p, argvp, envvp, traceme, dirp, fd, p[1]);
+ pid, err = forkAndExecInChild(argv0p, argvp, envvp, traceme, dirp, fd, p[1])
if err != 0 {
error:
if p[0] >= 0 {
- Close(p[0]);
- Close(p[1]);
+ Close(p[0])
+ Close(p[1])
}
- ForkLock.Unlock();
+ ForkLock.Unlock()
return 0, err
}
- ForkLock.Unlock();
+ ForkLock.Unlock()
// Read child error status from pipe.
- Close(p[1]);
+ Close(p[1])
n := libc_read(p[0], (*byte)(unsafe.Pointer(&err1)),
- Size_t(unsafe.Sizeof(err1)));
- err = 0;
+ Size_t(unsafe.Sizeof(err1)))
+ err = 0
if n < 0 {
- err = GetErrno();
+ err = GetErrno()
}
- Close(p[0]);
+ Close(p[0])
if err != 0 || n != 0 {
if int(n) == unsafe.Sizeof(err1) {
- err = int(err1);
+ err = int(err1)
}
if err == 0 {
- err = EPIPE;
+ err = EPIPE
}
// Child failed; wait for it to exit, to make sure
// the zombies don't accumulate.
- pid1, err1 := Wait4(pid, &wstatus, 0, nil);
+ _, err1 := Wait4(pid, &wstatus, 0, nil)
for err1 == EINTR {
- pid1, err1 = Wait4(pid, &wstatus, 0, nil);
+ _, err1 = Wait4(pid, &wstatus, 0, nil)
}
return 0, err
}
// Combination of fork and exec, careful to be thread safe.
func ForkExec(argv0 string, argv []string, envv []string, dir string, fd []int) (pid int, err int) {
- return forkExec(argv0, argv, envv, false, dir, fd);
+ return forkExec(argv0, argv, envv, false, dir, fd)
}
// PtraceForkExec is like ForkExec, but starts the child in a traced state.
func PtraceForkExec(argv0 string, argv []string, envv []string, dir string, fd []int) (pid int, err int) {
- return forkExec(argv0, argv, envv, true, dir, fd);
+ return forkExec(argv0, argv, envv, true, dir, fd)
}
// Ordinary exec.
func Exec(argv0 string, argv []string, envv []string) (err int) {
- argv_arg := StringArrayPtr(argv);
- envv_arg := StringArrayPtr(envv);
- libc_execve(StringBytePtr(argv0), &argv_arg[0], &envv_arg[0]);
- return GetErrno();
+ argv_arg := StringArrayPtr(argv)
+ envv_arg := StringArrayPtr(envv)
+ libc_execve(StringBytePtr(argv0), &argv_arg[0], &envv_arg[0])
+ return GetErrno()
+}
+
+// StartProcess wraps ForkExec for package os.
+func StartProcess(argv0 string, argv []string, envv []string, dir string, fd []int) (pid, handle int, err int) {
+ pid, err = forkExec(argv0, argv, envv, false, dir, fd)
+ return pid, 0, err
}
func Wait4(pid int, wstatus *WaitStatus, options int, rusage *Rusage) (wpid int, errno int) {
- var status int;
- r := libc_wait4(Pid_t(pid), &status, options, rusage);
- wpid = int(r);
+ var status int
+ r := libc_wait4(Pid_t(pid), &status, options, rusage)
+ wpid = int(r)
if r < 0 {
- errno = GetErrno();
+ errno = GetErrno()
}
if wstatus != nil {
- *wstatus = WaitStatus(status);
+ *wstatus = WaitStatus(status)
}
- return;
+ return
}
return setsockopt(fd, level, opt, uintptr(unsafe.Pointer(&[]byte(s)[0])), Socklen_t(len(s)))
}
+func SetsockoptIpMreq(fd, level, opt int, mreq *IpMreq) (errno int) {
+ return setsockopt(fd, level, opt, uintptr(unsafe.Pointer(mreq)), unsafe.Sizeof(*mreq))
+}
+
func Getsockname(fd int) (sa Sockaddr, errno int) {
var rsa RawSockaddrAny;
var len Socklen_t = SizeofSockaddrAny;
# so that the tests do not have to refer to srcdir to find test data.
ln -s $srcdir/* .
+# Some tests refer to a ../testdata directory.
+if test -e $srcdir/../testdata; then
+ rm -f ../testdata
+ abssrcdir=`cd $srcdir && pwd`
+ ln -s $abssrcdir/../testdata ../testdata
+fi
+
# Copy the .go files because io/utils_test.go expects a regular file.
case "x$gofiles" in
x)
projects / gcc.git / commitdiff