From: Ian Lance Taylor Date: Thu, 14 Sep 2017 17:11:35 +0000 (+0000) Subject: libgo: update to go1.9 X-Git-Url: https://git.libre-soc.org/?a=commitdiff_plain;h=bc998d034f45d1828a8663b2eed928faf22a7d01;p=gcc.git libgo: update to go1.9 Reviewed-on: https://go-review.googlesource.com/63753 From-SVN: r252767 --- diff --git a/gcc/go/gofrontend/MERGE b/gcc/go/gofrontend/MERGE index b07bce85187..a1ccb2bc75e 100644 --- a/gcc/go/gofrontend/MERGE +++ b/gcc/go/gofrontend/MERGE @@ -1,4 +1,4 @@ -199f175f4239d1ca6d7e80d08639955d41c3b09f +4e063a8eee636cce17aea48c7183e78431174de3 The first line of this file holds the git revision number of the last merge done from the gofrontend repository. diff --git a/gcc/testsuite/go.test/test/fixedbugs/bug273.go b/gcc/testsuite/go.test/test/fixedbugs/bug273.go index aabb912b913..c04f2116c5f 100644 --- a/gcc/testsuite/go.test/test/fixedbugs/bug273.go +++ b/gcc/testsuite/go.test/test/fixedbugs/bug273.go @@ -1,10 +1,10 @@ // run -// Copyright 2010 The Go Authors. All rights reserved. +// Copyright 2010 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -// http://code.google.com/p/go/issues/detail?id=589 +// https://golang.org/issue/589 package main @@ -48,15 +48,6 @@ func bigcap() { g1 = make([]block, 10, big) } -var g3 map[block]block -func badmapcap() { - g3 = make(map[block]block, minus1) -} - -func bigmapcap() { - g3 = make(map[block]block, big) -} - type cblock [1<<16-1]byte var g4 chan cblock @@ -78,8 +69,6 @@ func main() { shouldfail(badcap, "badcap") shouldfail(badcap1, "badcap1") shouldfail(bigcap, "bigcap") - shouldfail(badmapcap, "badmapcap") - shouldfail(bigmapcap, "bigmapcap") shouldfail(badchancap, "badchancap") shouldfail(bigchancap, "bigchancap") shouldfail(overflowchan, "overflowchan") diff --git a/gotools/ChangeLog b/gotools/ChangeLog index 0a84eefba66..8fac16e3712 100644 --- a/gotools/ChangeLog +++ b/gotools/ChangeLog @@ -1,3 +1,15 @@ +2017-09-14 Ian Lance Taylor + + * Makefile.am (LIBGOTOOL): Define. + (go_cmd_go_files): Update for Go 1.9 release. + (go$(EXEEXT)): Depend on and link against $(LIBGOTOOL). + (CHECK_ENV): Add definition of shell variable fl. + (check-go-tool): Update for rearrangement of cmd/go sources in Go + 1.9 release. Echo failure message if test fails. + (check-runtime): Echo failure message if test fails. + (check-cgo-test, check-carchive-test): Likewise. + * Makefile.in: Rebuild. + 2017-08-30 Ian Lance Taylor * configure.ac: Substitute GOC_FOR_TARGET and GCC_FOR_TARGET. diff --git a/gotools/Makefile.am b/gotools/Makefile.am index 41a2c72dc5b..4f13ffe31c3 100644 --- a/gotools/Makefile.am +++ b/gotools/Makefile.am @@ -28,6 +28,8 @@ STAMP = echo timestamp > libgodir = ../$(target_noncanonical)/libgo LIBGODEP = $(libgodir)/libgo.la +LIBGOTOOL = $(libgodir)/libgotool.a + if NATIVE # Use the compiler we just built. GOCOMPILER = $(GOC_FOR_TARGET) $(XGCC_FLAGS_FOR_TARGET) @@ -48,34 +50,8 @@ libgomiscdir = $(srcdir)/../libgo/misc go_cmd_go_files = \ $(cmdsrcdir)/go/alldocs.go \ - $(cmdsrcdir)/go/bug.go \ - $(cmdsrcdir)/go/build.go \ - $(cmdsrcdir)/go/clean.go \ - $(cmdsrcdir)/go/context.go \ - $(cmdsrcdir)/go/discovery.go \ - $(cmdsrcdir)/go/doc.go \ - $(cmdsrcdir)/go/env.go \ - $(cmdsrcdir)/go/fix.go \ - $(cmdsrcdir)/go/fmt.go \ - $(cmdsrcdir)/go/generate.go \ - $(cmdsrcdir)/go/get.go \ $(cmdsrcdir)/go/go11.go \ - $(cmdsrcdir)/go/help.go \ - $(cmdsrcdir)/go/http.go \ - $(cmdsrcdir)/go/list.go \ - $(cmdsrcdir)/go/main.go \ - $(cmdsrcdir)/go/note.go \ - $(cmdsrcdir)/go/pkg.go \ - $(cmdsrcdir)/go/run.go \ - $(cmdsrcdir)/go/signal.go \ - $(cmdsrcdir)/go/signal_unix.go \ - $(cmdsrcdir)/go/test.go \ - $(cmdsrcdir)/go/testflag.go \ - $(cmdsrcdir)/go/tool.go \ - $(cmdsrcdir)/go/vcs.go \ - $(cmdsrcdir)/go/version.go \ - $(cmdsrcdir)/go/vet.go \ - $(libgodir)/zstdpkglist.go + $(cmdsrcdir)/go/main.go go_cmd_gofmt_files = \ $(cmdsrcdir)/gofmt/doc.go \ @@ -124,8 +100,8 @@ bin_PROGRAMS = go$(EXEEXT) gofmt$(EXEEXT) noinst_PROGRAMS = cgo$(EXEEXT) man_MANS = go.1 gofmt.1 -go$(EXEEXT): $(go_cmd_go_files) zdefaultcc.go $(LIBGODEP) - $(GOLINK) $(go_cmd_go_files) zdefaultcc.go $(LIBS) $(NET_LIBS) +go$(EXEEXT): $(go_cmd_go_files) $(LIBGOTOOL) $(LIBGODEP) + $(GOLINK) $(go_cmd_go_files) $(LIBGOTOOL) $(LIBS) $(NET_LIBS) gofmt$(EXEEXT): $(go_cmd_gofmt_files) $(LIBGODEP) $(GOLINK) $(go_cmd_gofmt_files) $(LIBS) $(NET_LIBS) cgo$(EXEEXT): $(go_cmd_cgo_files) zdefaultcc.go $(LIBGODEP) @@ -175,6 +151,8 @@ check-gcc: Makefile # CHECK_ENV sets up the environment to run the newly built go tool. # If you change this, change ECHO_ENV, below. +# The fl shell variable is used to avoid having FAIL appear +# in the log unnecessarily. CHECK_ENV = \ PATH=`echo $(abs_builddir):$${PATH} | sed 's,::*,:,g;s,^:*,,;s,:*$$,,'`; \ export PATH; \ @@ -190,7 +168,8 @@ CHECK_ENV = \ LD_LIBRARY_PATH=`echo $${abs_libgodir}/.libs:$${LD_LIBRARY_PATH} | sed 's,::*,:,g;s,^:*,,;s,:*$$,,'`; \ export LD_LIBRARY_PATH; \ GOROOT=$${abs_libgodir}; \ - export GOROOT; + export GOROOT; \ + fl1="FA"; fl2="IL"; fl="$${fl1}$${fl2}"; # ECHO_ENV is a variant of CHECK_ENV to put into a testlog file. # It assumes that abs_libgodir is set. @@ -201,16 +180,18 @@ check-go-tool: go$(EXEEXT) cgo$(EXEEXT) check-head check-gccgo check-gcc rm -rf check-go-dir cmd_go-testlog $(MKDIR_P) check-go-dir/src/cmd/go cp $(cmdsrcdir)/go/*.go check-go-dir/src/cmd/go/ - cp $(libgodir)/zstdpkglist.go check-go-dir/src/cmd/go/ - cp zdefaultcc.go check-go-dir/src/cmd/go/ + cp -r $(cmdsrcdir)/go/internal check-go-dir/src/cmd/go/ + cp $(libgodir)/zstdpkglist.go check-go-dir/src/cmd/go/internal/load/ + cp $(libgodir)/zdefaultcc.go check-go-dir/src/cmd/go/internal/cfg/ cp -r $(cmdsrcdir)/go/testdata check-go-dir/src/cmd/go/ + cp -r $(cmdsrcdir)/internal check-go-dir/src/cmd/ @abs_libgodir=`cd $(libgodir) && $(PWD_COMMAND)`; \ abs_checkdir=`cd check-go-dir && $(PWD_COMMAND)`; \ echo "cd check-go-dir/src/cmd/go && $(ECHO_ENV) GOPATH=$${abs_checkdir} $(abs_builddir)/go$(EXEEXT) test -test.short -test.v" > cmd_go-testlog $(CHECK_ENV) \ GOPATH=`cd check-go-dir && $(PWD_COMMAND)`; \ export GOPATH; \ - (cd check-go-dir/src/cmd/go && $(abs_builddir)/go$(EXEEXT) test -test.short -test.v) > cmd_go-testlog 2>&1 || true + (cd check-go-dir/src/cmd/go && $(abs_builddir)/go$(EXEEXT) test -test.short -test.v) >> cmd_go-testlog 2>&1 || echo "--- $${fl} go test cmd/go (0.00s)" >> cmd_go-testlog grep '^--- ' cmd_go-testlog | sed -e 's/^--- \(.*\) ([^)]*)$$/\1/' # check-runtime runs `go test runtime` in our environment. @@ -232,7 +213,7 @@ check-runtime: go$(EXEEXT) cgo$(EXEEXT) check-head check-gccgo check-gcc GOARCH=`$(abs_builddir)/go$(EXEEXT) env GOARCH`; \ GOOS=`$(abs_builddir)/go$(EXEEXT) env GOOS`; \ files=`$(SHELL) $(libgosrcdir)/../match.sh --goarch=$${GOARCH} --goos=$${GOOS} --srcdir=$(libgosrcdir)/runtime --extrafiles="$(libgodir)/runtime_sysinfo.go $(libgodir)/sigtab.go" --tag=libffi`; \ - $(SHELL) $(libgosrcdir)/../testsuite/gotest --goarch=$${GOARCH} --goos=$${GOOS} --basedir=$(libgosrcdir)/.. --srcdir=$(libgosrcdir)/runtime --pkgpath=runtime --pkgfiles="$${files}" -test.v >> runtime-testlog 2>&1 || true + $(SHELL) $(libgosrcdir)/../testsuite/gotest --goarch=$${GOARCH} --goos=$${GOOS} --basedir=$(libgosrcdir)/.. --srcdir=$(libgosrcdir)/runtime --pkgpath=runtime --pkgfiles="$${files}" -test.v >> runtime-testlog 2>&1 || echo "--- $${fl} go test runtime (0.00s)" >> runtime-testlog grep '^--- ' runtime-testlog | sed -e 's/^--- \(.*\) ([^)]*)$$/\1/' # check-cgo-test runs `go test misc/cgo/test` in our environment. @@ -245,7 +226,7 @@ check-cgo-test: go$(EXEEXT) cgo$(EXEEXT) check-head check-gccgo check-gcc $(CHECK_ENV) \ GOTRACEBACK=2; \ export GOTRACEBACK; \ - (cd cgo-test-dir/misc/cgo/test && $(abs_builddir)/go$(EXEEXT) test -test.short -test.v) >> cgo-testlog 2>&1 || true + (cd cgo-test-dir/misc/cgo/test && $(abs_builddir)/go$(EXEEXT) test -test.short -test.v) >> cgo-testlog 2>&1 || echo "--- $${fl} go test misc/cgo/test (0.00s)" >> cgo-testlog grep '^--- ' cgo-testlog | sed -e 's/^--- \(.*\) ([^)]*)$$/\1/' # check-carchive-test runs `go test misc/cgo/testcarchive/carchive_test.go` @@ -259,7 +240,7 @@ check-carchive-test: go$(EXEEXT) cgo$(EXEEXT) check-head check-gccgo check-gcc $(CHECK_ENV) \ LIBRARY_PATH=`echo $${abs_libgodir}/.libs:$${LIBRARY_PATH} | sed 's,::*,:,g;s,^:*,,;s,:*$$,,'`; \ export LIBRARY_PATH; \ - (cd carchive-test-dir/misc/cgo/testcarchive && $(abs_builddir)/go$(EXEEXT) test -test.v carchive_test.go) >> carchive-testlog 2>&1 || true + (cd carchive-test-dir/misc/cgo/testcarchive && $(abs_builddir)/go$(EXEEXT) test -test.v carchive_test.go) >> carchive-testlog 2>&1 || echo "--- $${fl} go test misc/cgo/testcarchive (0.00s)" >> carchive-testlog grep '^--- ' carchive-testlog | sed -e 's/^--- \(.*\) ([^)]*)$$/\1/' # The check targets runs the tests and assembles the output files. diff --git a/gotools/Makefile.in b/gotools/Makefile.in index 148665a46e9..b9734f151cc 100644 --- a/gotools/Makefile.in +++ b/gotools/Makefile.in @@ -255,6 +255,7 @@ PWD_COMMAND = $${PWDCMD-pwd} STAMP = echo timestamp > libgodir = ../$(target_noncanonical)/libgo LIBGODEP = $(libgodir)/libgo.la +LIBGOTOOL = $(libgodir)/libgotool.a @NATIVE_FALSE@GOCOMPILER = $(GOC) # Use the compiler we just built. @@ -268,34 +269,8 @@ cmdsrcdir = $(libgosrcdir)/cmd libgomiscdir = $(srcdir)/../libgo/misc go_cmd_go_files = \ $(cmdsrcdir)/go/alldocs.go \ - $(cmdsrcdir)/go/bug.go \ - $(cmdsrcdir)/go/build.go \ - $(cmdsrcdir)/go/clean.go \ - $(cmdsrcdir)/go/context.go \ - $(cmdsrcdir)/go/discovery.go \ - $(cmdsrcdir)/go/doc.go \ - $(cmdsrcdir)/go/env.go \ - $(cmdsrcdir)/go/fix.go \ - $(cmdsrcdir)/go/fmt.go \ - $(cmdsrcdir)/go/generate.go \ - $(cmdsrcdir)/go/get.go \ $(cmdsrcdir)/go/go11.go \ - $(cmdsrcdir)/go/help.go \ - $(cmdsrcdir)/go/http.go \ - $(cmdsrcdir)/go/list.go \ - $(cmdsrcdir)/go/main.go \ - $(cmdsrcdir)/go/note.go \ - $(cmdsrcdir)/go/pkg.go \ - $(cmdsrcdir)/go/run.go \ - $(cmdsrcdir)/go/signal.go \ - $(cmdsrcdir)/go/signal_unix.go \ - $(cmdsrcdir)/go/test.go \ - $(cmdsrcdir)/go/testflag.go \ - $(cmdsrcdir)/go/tool.go \ - $(cmdsrcdir)/go/vcs.go \ - $(cmdsrcdir)/go/version.go \ - $(cmdsrcdir)/go/vet.go \ - $(libgodir)/zstdpkglist.go + $(cmdsrcdir)/go/main.go go_cmd_gofmt_files = \ $(cmdsrcdir)/gofmt/doc.go \ @@ -330,6 +305,8 @@ MOSTLYCLEANFILES = \ # CHECK_ENV sets up the environment to run the newly built go tool. # If you change this, change ECHO_ENV, below. +# The fl shell variable is used to avoid having FAIL appear +# in the log unnecessarily. @NATIVE_TRUE@CHECK_ENV = \ @NATIVE_TRUE@ PATH=`echo $(abs_builddir):$${PATH} | sed 's,::*,:,g;s,^:*,,;s,:*$$,,'`; \ @NATIVE_TRUE@ export PATH; \ @@ -345,7 +322,8 @@ MOSTLYCLEANFILES = \ @NATIVE_TRUE@ LD_LIBRARY_PATH=`echo $${abs_libgodir}/.libs:$${LD_LIBRARY_PATH} | sed 's,::*,:,g;s,^:*,,;s,:*$$,,'`; \ @NATIVE_TRUE@ export LD_LIBRARY_PATH; \ @NATIVE_TRUE@ GOROOT=$${abs_libgodir}; \ -@NATIVE_TRUE@ export GOROOT; +@NATIVE_TRUE@ export GOROOT; \ +@NATIVE_TRUE@ fl1="FA"; fl2="IL"; fl="$${fl1}$${fl2}"; # ECHO_ENV is a variant of CHECK_ENV to put into a testlog file. @@ -598,8 +576,8 @@ distclean-generic: maintainer-clean-generic: @echo "This command is intended for maintainers to use" @echo "it deletes files that may require special tools to rebuild." -@NATIVE_FALSE@uninstall-local: @NATIVE_FALSE@install-exec-local: +@NATIVE_FALSE@uninstall-local: clean: clean-am clean-am: clean-binPROGRAMS clean-generic clean-noinstPROGRAMS \ @@ -707,8 +685,8 @@ s-zdefaultcc: Makefile mostlyclean-local: rm -rf check-go-dir check-runtime-dir cgo-test-dir carchive-test-dir -@NATIVE_TRUE@go$(EXEEXT): $(go_cmd_go_files) zdefaultcc.go $(LIBGODEP) -@NATIVE_TRUE@ $(GOLINK) $(go_cmd_go_files) zdefaultcc.go $(LIBS) $(NET_LIBS) +@NATIVE_TRUE@go$(EXEEXT): $(go_cmd_go_files) $(LIBGOTOOL) $(LIBGODEP) +@NATIVE_TRUE@ $(GOLINK) $(go_cmd_go_files) $(LIBGOTOOL) $(LIBS) $(NET_LIBS) @NATIVE_TRUE@gofmt$(EXEEXT): $(go_cmd_gofmt_files) $(LIBGODEP) @NATIVE_TRUE@ $(GOLINK) $(go_cmd_gofmt_files) $(LIBS) $(NET_LIBS) @NATIVE_TRUE@cgo$(EXEEXT): $(go_cmd_cgo_files) zdefaultcc.go $(LIBGODEP) @@ -761,16 +739,18 @@ mostlyclean-local: @NATIVE_TRUE@ rm -rf check-go-dir cmd_go-testlog @NATIVE_TRUE@ $(MKDIR_P) check-go-dir/src/cmd/go @NATIVE_TRUE@ cp $(cmdsrcdir)/go/*.go check-go-dir/src/cmd/go/ -@NATIVE_TRUE@ cp $(libgodir)/zstdpkglist.go check-go-dir/src/cmd/go/ -@NATIVE_TRUE@ cp zdefaultcc.go check-go-dir/src/cmd/go/ +@NATIVE_TRUE@ cp -r $(cmdsrcdir)/go/internal check-go-dir/src/cmd/go/ +@NATIVE_TRUE@ cp $(libgodir)/zstdpkglist.go check-go-dir/src/cmd/go/internal/load/ +@NATIVE_TRUE@ cp $(libgodir)/zdefaultcc.go check-go-dir/src/cmd/go/internal/cfg/ @NATIVE_TRUE@ cp -r $(cmdsrcdir)/go/testdata check-go-dir/src/cmd/go/ +@NATIVE_TRUE@ cp -r $(cmdsrcdir)/internal check-go-dir/src/cmd/ @NATIVE_TRUE@ @abs_libgodir=`cd $(libgodir) && $(PWD_COMMAND)`; \ @NATIVE_TRUE@ abs_checkdir=`cd check-go-dir && $(PWD_COMMAND)`; \ @NATIVE_TRUE@ echo "cd check-go-dir/src/cmd/go && $(ECHO_ENV) GOPATH=$${abs_checkdir} $(abs_builddir)/go$(EXEEXT) test -test.short -test.v" > cmd_go-testlog @NATIVE_TRUE@ $(CHECK_ENV) \ @NATIVE_TRUE@ GOPATH=`cd check-go-dir && $(PWD_COMMAND)`; \ @NATIVE_TRUE@ export GOPATH; \ -@NATIVE_TRUE@ (cd check-go-dir/src/cmd/go && $(abs_builddir)/go$(EXEEXT) test -test.short -test.v) > cmd_go-testlog 2>&1 || true +@NATIVE_TRUE@ (cd check-go-dir/src/cmd/go && $(abs_builddir)/go$(EXEEXT) test -test.short -test.v) >> cmd_go-testlog 2>&1 || echo "--- $${fl} go test cmd/go (0.00s)" >> cmd_go-testlog @NATIVE_TRUE@ grep '^--- ' cmd_go-testlog | sed -e 's/^--- \(.*\) ([^)]*)$$/\1/' # check-runtime runs `go test runtime` in our environment. @@ -792,7 +772,7 @@ mostlyclean-local: @NATIVE_TRUE@ GOARCH=`$(abs_builddir)/go$(EXEEXT) env GOARCH`; \ @NATIVE_TRUE@ GOOS=`$(abs_builddir)/go$(EXEEXT) env GOOS`; \ @NATIVE_TRUE@ files=`$(SHELL) $(libgosrcdir)/../match.sh --goarch=$${GOARCH} --goos=$${GOOS} --srcdir=$(libgosrcdir)/runtime --extrafiles="$(libgodir)/runtime_sysinfo.go $(libgodir)/sigtab.go" --tag=libffi`; \ -@NATIVE_TRUE@ $(SHELL) $(libgosrcdir)/../testsuite/gotest --goarch=$${GOARCH} --goos=$${GOOS} --basedir=$(libgosrcdir)/.. --srcdir=$(libgosrcdir)/runtime --pkgpath=runtime --pkgfiles="$${files}" -test.v >> runtime-testlog 2>&1 || true +@NATIVE_TRUE@ $(SHELL) $(libgosrcdir)/../testsuite/gotest --goarch=$${GOARCH} --goos=$${GOOS} --basedir=$(libgosrcdir)/.. --srcdir=$(libgosrcdir)/runtime --pkgpath=runtime --pkgfiles="$${files}" -test.v >> runtime-testlog 2>&1 || echo "--- $${fl} go test runtime (0.00s)" >> runtime-testlog @NATIVE_TRUE@ grep '^--- ' runtime-testlog | sed -e 's/^--- \(.*\) ([^)]*)$$/\1/' # check-cgo-test runs `go test misc/cgo/test` in our environment. @@ -805,7 +785,7 @@ mostlyclean-local: @NATIVE_TRUE@ $(CHECK_ENV) \ @NATIVE_TRUE@ GOTRACEBACK=2; \ @NATIVE_TRUE@ export GOTRACEBACK; \ -@NATIVE_TRUE@ (cd cgo-test-dir/misc/cgo/test && $(abs_builddir)/go$(EXEEXT) test -test.short -test.v) >> cgo-testlog 2>&1 || true +@NATIVE_TRUE@ (cd cgo-test-dir/misc/cgo/test && $(abs_builddir)/go$(EXEEXT) test -test.short -test.v) >> cgo-testlog 2>&1 || echo "--- $${fl} go test misc/cgo/test (0.00s)" >> cgo-testlog @NATIVE_TRUE@ grep '^--- ' cgo-testlog | sed -e 's/^--- \(.*\) ([^)]*)$$/\1/' # check-carchive-test runs `go test misc/cgo/testcarchive/carchive_test.go` @@ -819,7 +799,7 @@ mostlyclean-local: @NATIVE_TRUE@ $(CHECK_ENV) \ @NATIVE_TRUE@ LIBRARY_PATH=`echo $${abs_libgodir}/.libs:$${LIBRARY_PATH} | sed 's,::*,:,g;s,^:*,,;s,:*$$,,'`; \ @NATIVE_TRUE@ export LIBRARY_PATH; \ -@NATIVE_TRUE@ (cd carchive-test-dir/misc/cgo/testcarchive && $(abs_builddir)/go$(EXEEXT) test -test.v carchive_test.go) >> carchive-testlog 2>&1 || true +@NATIVE_TRUE@ (cd carchive-test-dir/misc/cgo/testcarchive && $(abs_builddir)/go$(EXEEXT) test -test.v carchive_test.go) >> carchive-testlog 2>&1 || echo "--- $${fl} go test misc/cgo/testcarchive (0.00s)" >> carchive-testlog @NATIVE_TRUE@ grep '^--- ' carchive-testlog | sed -e 's/^--- \(.*\) ([^)]*)$$/\1/' # The check targets runs the tests and assembles the output files. diff --git a/libgo/MERGE b/libgo/MERGE index ddfb006437b..85703d6ec4f 100644 --- a/libgo/MERGE +++ b/libgo/MERGE @@ -1,4 +1,4 @@ -352996a381701cfa0c16e8de29cbde8f3922182f +c8aec4095e089ff6ac50d18e97c3f46561f14f48 The first line of this file holds the git revision number of the last merge done from the master library sources. diff --git a/libgo/Makefile.am b/libgo/Makefile.am index 9800d20bd9b..23761302f47 100644 --- a/libgo/Makefile.am +++ b/libgo/Makefile.am @@ -117,6 +117,8 @@ toolexeclib_LTLIBRARIES = libgo.la toolexeclib_LIBRARIES = libgobegin.a libgolibbegin.a endif +noinst_LIBRARIES = libgotool.a + toolexeclibgo_DATA = \ bufio.gox \ bytes.gox \ @@ -299,6 +301,7 @@ toolexeclibgomathdir = $(toolexeclibgodir)/math toolexeclibgomath_DATA = \ math/big.gox \ + math/bits.gox \ math/cmplx.gox \ math/rand.gox @@ -534,6 +537,21 @@ s-sigtab: $(srcdir)/mksigtab.sh gen-sysinfo.go $(SHELL) $(srcdir)/mvifdiff.sh tmp-sigtab.go sigtab.go $(STAMP) $@ +GCCGO_INSTALL_NAME := $(shell echo gccgo|sed '$(program_transform_name)') +GCC_INSTALL_NAME := $(shell echo gcc|sed '$(program_transform_name)') +GXX_INSTALL_NAME := $(shell echo g++|sed '$(program_transform_name)') + +zdefaultcc.go: s-zdefaultcc; @true +s-zdefaultcc: Makefile + echo 'package cfg' > zdefaultcc.go.tmp + echo >> zdefaultcc.go.tmp + echo 'const DefaultGCCGO = "$(bindir)/$(GCCGO_INSTALL_NAME)"' >> zdefaultcc.go.tmp + echo 'const DefaultCC = "$(GCC_INSTALL_NAME)"' >> zdefaultcc.go.tmp + echo 'const DefaultCXX = "$(GXX_INSTALL_NAME)"' >> zdefaultcc.go.tmp + echo 'const DefaultPkgConfig = "pkg-config"' >> zdefaultcc.go.tmp + $(SHELL) $(srcdir)/../move-if-change zdefaultcc.go.tmp zdefaultcc.go + $(STAMP) $@ + # _Complex_lock and _Reader_lock are Go translations of some AIX system # types and should not be exported back to C # semt is a Go translation of the C type sem_t; it fails to convert on @@ -555,13 +573,13 @@ s-runtime-inc: runtime.lo Makefile rm -f runtime.inc.tmp2 runtime.inc.tmp3 $(STAMP) $@ -noinst_DATA = zstdpkglist.go +noinst_DATA = zstdpkglist.go zdefaultcc.go # Generate the list of go std packages that were included in libgo zstdpkglist.go: s-zstdpkglist; @true s-zstdpkglist: Makefile rm -f zstdpkglist.go.tmp - echo 'package main' > zstdpkglist.go.tmp + echo 'package load' > zstdpkglist.go.tmp echo "" >> zstdpkglist.go.tmp echo 'var stdpkg = map[string]bool{' >> zstdpkglist.go.tmp echo $(libgo_go_objs) 'unsafe.lo' 'runtime/cgo.lo' | sed 's|[a-z0-9_/]*_c\.lo||g' | sed 's|\([a-z0-9_/]*\)\.lo|"\1": true,|g' >> zstdpkglist.go.tmp @@ -660,7 +678,6 @@ PACKAGES = \ archive/zip \ bufio \ bytes \ - cmd/internal/browser \ compress/bzip2 \ compress/flate \ compress/gzip \ @@ -724,6 +741,7 @@ PACKAGES = \ go/importer \ go/internal/gccgoimporter \ go/internal/gcimporter \ + go/internal/srcimporter \ go/parser \ go/printer \ go/scanner \ @@ -736,7 +754,10 @@ PACKAGES = \ golang_org/x/net/http2/hpack \ golang_org/x/net/idna \ golang_org/x/net/lex/httplex \ + golang_org/x/net/proxy \ + golang_org/x/text/secure/bidirule \ golang_org/x/text/transform \ + golang_org/x/text/unicode/bidi \ golang_org/x/text/unicode/norm \ golang_org/x/text/width \ hash \ @@ -756,7 +777,7 @@ PACKAGES = \ image/png \ index/suffixarray \ internal/nettrace \ - internal/pprof/profile \ + internal/poll \ internal/race \ internal/singleflight \ internal/syscall/unix \ @@ -768,6 +789,7 @@ PACKAGES = \ log/syslog \ math \ math/big \ + math/bits \ math/cmplx \ math/rand \ mime \ @@ -804,7 +826,7 @@ PACKAGES = \ runtime/internal/atomic \ runtime/internal/sys \ runtime/pprof \ - runtime/pprof/internal/protopprof \ + runtime/pprof/internal/profile \ runtime/trace \ sort \ strconv \ @@ -871,6 +893,37 @@ libgolibbegin_a_SOURCES = \ libgolibbegin_a_CFLAGS = $(AM_CFLAGS) -fPIC +GOTOOL_PACKAGES = \ + cmd/go/internal/base \ + cmd/go/internal/bug \ + cmd/go/internal/buildid \ + cmd/go/internal/cfg \ + cmd/go/internal/clean \ + cmd/go/internal/cmdflag \ + cmd/go/internal/doc \ + cmd/go/internal/envcmd \ + cmd/go/internal/fix \ + cmd/go/internal/fmtcmd \ + cmd/go/internal/generate \ + cmd/go/internal/get \ + cmd/go/internal/help \ + cmd/go/internal/list \ + cmd/go/internal/load \ + cmd/go/internal/run \ + cmd/go/internal/str \ + cmd/go/internal/test \ + cmd/go/internal/tool \ + cmd/go/internal/version \ + cmd/go/internal/vet \ + cmd/go/internal/web \ + cmd/go/internal/work \ + cmd/internal/browser \ + cmd/internal/objabi + +libgotool_a_SOURCES = +libgotool_a_DEPENDENCIES = $(addsuffix .lo,$(GOTOOL_PACKAGES)) +libgotool_a_LIBADD = $(addsuffix .o,$(GOTOOL_PACKAGES)) + # Make sure runtime.inc is built before compiling any .c file. $(libgo_la_OBJECTS): runtime.inc $(libgo_llgo_la_OBJECTS): runtime.inc @@ -925,7 +978,7 @@ GOBENCH = CHECK = \ GC="$(GOC) $(GOCFLAGS) $($(subst /,_,$@)_GOCFLAGS) -L `${PWD_COMMAND}` -L `${PWD_COMMAND}`/.libs"; \ export GC; \ - GOLIBS="$(MATH_LIBS) $(NET_LIBS) $(LIBS)"; \ + GOLIBS="$(extra_check_libs_$(subst /,_,$(@D))) $(MATH_LIBS) $(NET_LIBS) $(LIBS)"; \ export GOLIBS; \ RUNTESTFLAGS="$(RUNTESTFLAGS)"; \ export RUNTESTFLAGS; \ @@ -985,7 +1038,8 @@ CHECK_DEPS = \ $(toolexeclibgotesting_DATA) \ $(toolexeclibgotext_DATA) \ $(toolexeclibgotexttemplate_DATA) \ - $(toolexeclibgounicode_DATA) + $(toolexeclibgounicode_DATA) \ + $(noinst_LIBRARIES) if GOC_IS_LLGO CHECK_DEPS += libgo-llgo.la libgobegin-llgo.a @@ -1025,6 +1079,7 @@ endef # This line expands PACKAGE_template once for each package name listed # in $(PACKAGES). $(foreach package,$(PACKAGES),$(eval $(call PACKAGE_template,$(package)))) +$(foreach package,$(GOTOOL_PACKAGES),$(eval $(call PACKAGE_template,$(package)))) # Pass -ffp-contract=off, or 386-specific options, when building the # math package. MATH_FLAG is defined in configure.ac. @@ -1066,6 +1121,17 @@ runtime_pprof_check_GOCFLAGS = -static-libgo -fno-inline extra_go_files_runtime_internal_sys = version.go runtime/internal/sys.lo.dep: $(extra_go_files_runtime_internal_sys) +extra_go_files_cmd_go_internal_cfg = zdefaultcc.go +cmd/go/internal/cfg.lo.dep: $(extra_go_files_cmd_go_internal_cfg) + +extra_go_files_cmd_go_internal_load = zstdpkglist.go +cmd/go/internal/load.lo.dep: $(extra_go_files_cmd_go_internal_load) + +extra_check_libs_cmd_go_internal_generate = $(abs_builddir)/libgotool.a +extra_check_libs_cmd_go_internal_get = $(abs_builddir)/libgotool.a +extra_check_libs_cmd_go_internal_load = $(abs_builddir)/libgotool.a +extra_check_libs_cmd_go_internal_work = $(abs_builddir)/libgotool.a + # FIXME: The following C files may as well move to the runtime # directory and be treated like other C files. @@ -1177,6 +1243,11 @@ TEST_PACKAGES = \ unicode/check \ archive/tar/check \ archive/zip/check \ + cmd/go/internal/generate/check \ + cmd/go/internal/get/check \ + cmd/go/internal/load/check \ + cmd/go/internal/work/check \ + cmd/internal/objabi/check \ compress/bzip2/check \ compress/flate/check \ compress/gzip/check \ @@ -1230,6 +1301,7 @@ TEST_PACKAGES = \ go/format/check \ go/internal/gcimporter/check \ go/internal/gccgoimporter/check \ + go/internal/srcimporter/check \ go/parser/check \ go/printer/check \ go/scanner/check \ @@ -1243,6 +1315,7 @@ TEST_PACKAGES = \ golang_org/x/net/idna/check \ golang_org/x/net/lex/httplex/check \ $(golang_org_x_net_lif_check) \ + golang_org/x/net/proxy/check \ $(golang_org_x_net_route_check) \ hash/adler32/check \ hash/crc32/check \ @@ -1253,12 +1326,13 @@ TEST_PACKAGES = \ image/jpeg/check \ image/png/check \ index/suffixarray/check \ - internal/pprof/profile/check \ + internal/poll/check \ internal/singleflight/check \ internal/trace/check \ io/ioutil/check \ log/syslog/check \ math/big/check \ + math/bits/check \ math/cmplx/check \ math/rand/check \ mime/multipart/check \ @@ -1287,7 +1361,7 @@ TEST_PACKAGES = \ runtime/internal/atomic/check \ runtime/internal/sys/check \ runtime/pprof/check \ - runtime/pprof/internal/protopprof/check \ + runtime/pprof/internal/profile/check \ runtime/trace/check \ sync/atomic/check \ text/scanner/check \ @@ -1413,7 +1487,7 @@ mostlyclean-local: find . -name '*-testsum' -print | xargs rm -f find . -name '*-testlog' -print | xargs rm -f -CLEANFILES = *.go *.c s-version libgo.sum libgo.log runtime.inc +CLEANFILES = *.go *.c s-* libgo.sum libgo.log runtime.inc clean-local: find . -name '*.la' -print | xargs $(LIBTOOL) --mode=clean rm -f diff --git a/libgo/Makefile.in b/libgo/Makefile.in index 9a2bd47ff7b..5f7adb21dd2 100644 --- a/libgo/Makefile.in +++ b/libgo/Makefile.in @@ -155,7 +155,7 @@ am__installdirs = "$(DESTDIR)$(toolexeclibdir)" \ "$(DESTDIR)$(toolexeclibgotextdir)" \ "$(DESTDIR)$(toolexeclibgotexttemplatedir)" \ "$(DESTDIR)$(toolexeclibgounicodedir)" -LIBRARIES = $(toolexeclib_LIBRARIES) +LIBRARIES = $(noinst_LIBRARIES) $(toolexeclib_LIBRARIES) ARFLAGS = cru libgobegin_llgo_a_AR = $(AR) $(ARFLAGS) libgobegin_llgo_a_LIBADD = @@ -169,6 +169,9 @@ libgolibbegin_a_AR = $(AR) $(ARFLAGS) libgolibbegin_a_LIBADD = am_libgolibbegin_a_OBJECTS = libgolibbegin_a-go-libmain.$(OBJEXT) libgolibbegin_a_OBJECTS = $(am_libgolibbegin_a_OBJECTS) +libgotool_a_AR = $(AR) $(ARFLAGS) +am_libgotool_a_OBJECTS = +libgotool_a_OBJECTS = $(am_libgotool_a_OBJECTS) LTLIBRARIES = $(toolexeclib_LTLIBRARIES) @LIBGO_IS_LINUX_TRUE@am__DEPENDENCIES_1 = syscall/clone_linux.lo am__DEPENDENCIES_2 = $(addsuffix .lo,$(PACKAGES)) bytes/index.lo \ @@ -229,8 +232,8 @@ LINK = $(LIBTOOL) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) \ --mode=link $(CCLD) $(AM_CFLAGS) $(CFLAGS) $(AM_LDFLAGS) \ $(LDFLAGS) -o $@ SOURCES = $(libgobegin_llgo_a_SOURCES) $(libgobegin_a_SOURCES) \ - $(libgolibbegin_a_SOURCES) $(libgo_llgo_la_SOURCES) \ - $(libgo_la_SOURCES) + $(libgolibbegin_a_SOURCES) $(libgotool_a_SOURCES) \ + $(libgo_llgo_la_SOURCES) $(libgo_la_SOURCES) MULTISRCTOP = MULTIBUILDTOP = MULTIDIRS = @@ -518,6 +521,7 @@ FLAGS_TO_PASS = $(AM_MAKEFLAGS) @GOC_IS_LLGO_TRUE@toolexeclib_LTLIBRARIES = libgo-llgo.la @GOC_IS_LLGO_FALSE@toolexeclib_LIBRARIES = libgobegin.a libgolibbegin.a @GOC_IS_LLGO_TRUE@toolexeclib_LIBRARIES = libgobegin-llgo.a +noinst_LIBRARIES = libgotool.a toolexeclibgo_DATA = \ bufio.gox \ bytes.gox \ @@ -681,6 +685,7 @@ toolexeclibgolog_DATA = \ toolexeclibgomathdir = $(toolexeclibgodir)/math toolexeclibgomath_DATA = \ math/big.gox \ + math/bits.gox \ math/cmplx.gox \ math/rand.gox @@ -809,7 +814,10 @@ runtime_files = \ $(rtems_task_variable_add_file) \ $(runtime_getncpu_file) -noinst_DATA = zstdpkglist.go +GCCGO_INSTALL_NAME := $(shell echo gccgo|sed '$(program_transform_name)') +GCC_INSTALL_NAME := $(shell echo gcc|sed '$(program_transform_name)') +GXX_INSTALL_NAME := $(shell echo g++|sed '$(program_transform_name)') +noinst_DATA = zstdpkglist.go zdefaultcc.go @LIBGO_IS_LINUX_FALSE@syscall_epoll_file = @LIBGO_IS_LINUX_TRUE@syscall_epoll_file = epoll.go SYSINFO_FLAGS = \ @@ -823,7 +831,6 @@ PACKAGES = \ archive/zip \ bufio \ bytes \ - cmd/internal/browser \ compress/bzip2 \ compress/flate \ compress/gzip \ @@ -887,6 +894,7 @@ PACKAGES = \ go/importer \ go/internal/gccgoimporter \ go/internal/gcimporter \ + go/internal/srcimporter \ go/parser \ go/printer \ go/scanner \ @@ -899,7 +907,10 @@ PACKAGES = \ golang_org/x/net/http2/hpack \ golang_org/x/net/idna \ golang_org/x/net/lex/httplex \ + golang_org/x/net/proxy \ + golang_org/x/text/secure/bidirule \ golang_org/x/text/transform \ + golang_org/x/text/unicode/bidi \ golang_org/x/text/unicode/norm \ golang_org/x/text/width \ hash \ @@ -919,7 +930,7 @@ PACKAGES = \ image/png \ index/suffixarray \ internal/nettrace \ - internal/pprof/profile \ + internal/poll \ internal/race \ internal/singleflight \ internal/syscall/unix \ @@ -931,6 +942,7 @@ PACKAGES = \ log/syslog \ math \ math/big \ + math/bits \ math/cmplx \ math/rand \ mime \ @@ -967,7 +979,7 @@ PACKAGES = \ runtime/internal/atomic \ runtime/internal/sys \ runtime/pprof \ - runtime/pprof/internal/protopprof \ + runtime/pprof/internal/profile \ runtime/trace \ sort \ strconv \ @@ -1031,6 +1043,36 @@ libgolibbegin_a_SOURCES = \ runtime/go-libmain.c libgolibbegin_a_CFLAGS = $(AM_CFLAGS) -fPIC +GOTOOL_PACKAGES = \ + cmd/go/internal/base \ + cmd/go/internal/bug \ + cmd/go/internal/buildid \ + cmd/go/internal/cfg \ + cmd/go/internal/clean \ + cmd/go/internal/cmdflag \ + cmd/go/internal/doc \ + cmd/go/internal/envcmd \ + cmd/go/internal/fix \ + cmd/go/internal/fmtcmd \ + cmd/go/internal/generate \ + cmd/go/internal/get \ + cmd/go/internal/help \ + cmd/go/internal/list \ + cmd/go/internal/load \ + cmd/go/internal/run \ + cmd/go/internal/str \ + cmd/go/internal/test \ + cmd/go/internal/tool \ + cmd/go/internal/version \ + cmd/go/internal/vet \ + cmd/go/internal/web \ + cmd/go/internal/work \ + cmd/internal/browser \ + cmd/internal/objabi + +libgotool_a_SOURCES = +libgotool_a_DEPENDENCIES = $(addsuffix .lo,$(GOTOOL_PACKAGES)) +libgotool_a_LIBADD = $(addsuffix .o,$(GOTOOL_PACKAGES)) LTLDFLAGS = $(shell $(SHELL) $(top_srcdir)/../libtool-ldflags $(LDFLAGS)) AM_GOCFLAGS = $(STRINGOPS_FLAG) $(GO_SPLIT_STACK) GOCOMPILE = $(GOC) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_GOCFLAGS) $(GOCFLAGS) @@ -1078,7 +1120,7 @@ GOBENCH = CHECK = \ GC="$(GOC) $(GOCFLAGS) $($(subst /,_,$@)_GOCFLAGS) -L `${PWD_COMMAND}` -L `${PWD_COMMAND}`/.libs"; \ export GC; \ - GOLIBS="$(MATH_LIBS) $(NET_LIBS) $(LIBS)"; \ + GOLIBS="$(extra_check_libs_$(subst /,_,$(@D))) $(MATH_LIBS) $(NET_LIBS) $(LIBS)"; \ export GOLIBS; \ RUNTESTFLAGS="$(RUNTESTFLAGS)"; \ export RUNTESTFLAGS; \ @@ -1125,7 +1167,8 @@ CHECK_DEPS = $(toolexeclibgo_DATA) $(toolexeclibgoarchive_DATA) \ $(toolexeclibgorpc_DATA) $(toolexeclibgoruntime_DATA) \ $(toolexeclibgosync_DATA) $(toolexeclibgotesting_DATA) \ $(toolexeclibgotext_DATA) $(toolexeclibgotexttemplate_DATA) \ - $(toolexeclibgounicode_DATA) $(am__append_3) $(am__append_4) + $(toolexeclibgounicode_DATA) $(noinst_LIBRARIES) \ + $(am__append_3) $(am__append_4) # Pass -ffp-contract=off, or 386-specific options, when building the # math package. MATH_FLAG is defined in configure.ac. @@ -1160,6 +1203,12 @@ runtime_internal_sys_lo_check_GOCFLAGS = -fgo-compiling-runtime # Also use -fno-inline to get better results from the memory profiler. runtime_pprof_check_GOCFLAGS = -static-libgo -fno-inline extra_go_files_runtime_internal_sys = version.go +extra_go_files_cmd_go_internal_cfg = zdefaultcc.go +extra_go_files_cmd_go_internal_load = zstdpkglist.go +extra_check_libs_cmd_go_internal_generate = $(abs_builddir)/libgotool.a +extra_check_libs_cmd_go_internal_get = $(abs_builddir)/libgotool.a +extra_check_libs_cmd_go_internal_load = $(abs_builddir)/libgotool.a +extra_check_libs_cmd_go_internal_work = $(abs_builddir)/libgotool.a @HAVE_STAT_TIMESPEC_FALSE@@LIBGO_IS_SOLARIS_TRUE@matchargs_os = # Solaris 12 changed the type of fields in struct stat. @@ -1207,6 +1256,11 @@ TEST_PACKAGES = \ unicode/check \ archive/tar/check \ archive/zip/check \ + cmd/go/internal/generate/check \ + cmd/go/internal/get/check \ + cmd/go/internal/load/check \ + cmd/go/internal/work/check \ + cmd/internal/objabi/check \ compress/bzip2/check \ compress/flate/check \ compress/gzip/check \ @@ -1260,6 +1314,7 @@ TEST_PACKAGES = \ go/format/check \ go/internal/gcimporter/check \ go/internal/gccgoimporter/check \ + go/internal/srcimporter/check \ go/parser/check \ go/printer/check \ go/scanner/check \ @@ -1273,6 +1328,7 @@ TEST_PACKAGES = \ golang_org/x/net/idna/check \ golang_org/x/net/lex/httplex/check \ $(golang_org_x_net_lif_check) \ + golang_org/x/net/proxy/check \ $(golang_org_x_net_route_check) \ hash/adler32/check \ hash/crc32/check \ @@ -1283,12 +1339,13 @@ TEST_PACKAGES = \ image/jpeg/check \ image/png/check \ index/suffixarray/check \ - internal/pprof/profile/check \ + internal/poll/check \ internal/singleflight/check \ internal/trace/check \ io/ioutil/check \ log/syslog/check \ math/big/check \ + math/bits/check \ math/cmplx/check \ math/rand/check \ mime/multipart/check \ @@ -1317,7 +1374,7 @@ TEST_PACKAGES = \ runtime/internal/atomic/check \ runtime/internal/sys/check \ runtime/pprof/check \ - runtime/pprof/internal/protopprof/check \ + runtime/pprof/internal/profile/check \ runtime/trace/check \ sync/atomic/check \ text/scanner/check \ @@ -1335,7 +1392,7 @@ MOSTLYCLEANFILES = \ libgo.head libgo.sum.sep libgo.log.sep libgo.var \ libcalls-list runtime.inc runtime.inc.tmp2 runtime.inc.tmp3 -CLEANFILES = *.go *.c s-version libgo.sum libgo.log runtime.inc +CLEANFILES = *.go *.c s-* libgo.sum libgo.log runtime.inc all: config.h $(MAKE) $(AM_MAKEFLAGS) all-recursive @@ -1390,6 +1447,9 @@ $(srcdir)/config.h.in: @MAINTAINER_MODE_TRUE@ $(am__configure_deps) distclean-hdr: -rm -f config.h stamp-h1 + +clean-noinstLIBRARIES: + -test -z "$(noinst_LIBRARIES)" || rm -f $(noinst_LIBRARIES) install-toolexeclibLIBRARIES: $(toolexeclib_LIBRARIES) @$(NORMAL_INSTALL) @list='$(toolexeclib_LIBRARIES)'; test -n "$(toolexeclibdir)" || list=; \ @@ -1433,6 +1493,10 @@ libgolibbegin.a: $(libgolibbegin_a_OBJECTS) $(libgolibbegin_a_DEPENDENCIES) $(EX -rm -f libgolibbegin.a $(libgolibbegin_a_AR) libgolibbegin.a $(libgolibbegin_a_OBJECTS) $(libgolibbegin_a_LIBADD) $(RANLIB) libgolibbegin.a +libgotool.a: $(libgotool_a_OBJECTS) $(libgotool_a_DEPENDENCIES) $(EXTRA_libgotool_a_DEPENDENCIES) + -rm -f libgotool.a + $(libgotool_a_AR) libgotool.a $(libgotool_a_OBJECTS) $(libgotool_a_LIBADD) + $(RANLIB) libgotool.a install-toolexeclibLTLIBRARIES: $(toolexeclib_LTLIBRARIES) @$(NORMAL_INSTALL) @list='$(toolexeclib_LTLIBRARIES)'; test -n "$(toolexeclibdir)" || list=; \ @@ -2797,8 +2861,8 @@ maintainer-clean-generic: clean: clean-multi clean-recursive clean-am: clean-generic clean-libtool clean-local \ - clean-toolexeclibLIBRARIES clean-toolexeclibLTLIBRARIES \ - mostlyclean-am + clean-noinstLIBRARIES clean-toolexeclibLIBRARIES \ + clean-toolexeclibLTLIBRARIES mostlyclean-am distclean: distclean-multi distclean-recursive -rm -f $(am__CONFIG_DISTCLEAN_FILES) @@ -2931,17 +2995,18 @@ uninstall-am: uninstall-toolexeclibLIBRARIES \ .PHONY: $(RECURSIVE_CLEAN_TARGETS) $(RECURSIVE_TARGETS) CTAGS GTAGS \ all all-am all-multi am--refresh check check-am clean \ clean-generic clean-libtool clean-local clean-multi \ - clean-toolexeclibLIBRARIES clean-toolexeclibLTLIBRARIES ctags \ - ctags-recursive distclean distclean-compile distclean-generic \ - distclean-hdr distclean-libtool distclean-local \ - distclean-multi distclean-tags dvi dvi-am html html-am info \ - info-am install install-am install-data install-data-am \ - install-dvi install-dvi-am install-exec install-exec-am \ - install-html install-html-am install-info install-info-am \ - install-man install-multi install-pdf install-pdf-am \ - install-ps install-ps-am install-strip \ - install-toolexeclibLIBRARIES install-toolexeclibLTLIBRARIES \ - install-toolexeclibgoDATA install-toolexeclibgoarchiveDATA \ + clean-noinstLIBRARIES clean-toolexeclibLIBRARIES \ + clean-toolexeclibLTLIBRARIES ctags ctags-recursive distclean \ + distclean-compile distclean-generic distclean-hdr \ + distclean-libtool distclean-local distclean-multi \ + distclean-tags dvi dvi-am html html-am info info-am install \ + install-am install-data install-data-am install-dvi \ + install-dvi-am install-exec install-exec-am install-html \ + install-html-am install-info install-info-am install-man \ + install-multi install-pdf install-pdf-am install-ps \ + install-ps-am install-strip install-toolexeclibLIBRARIES \ + install-toolexeclibLTLIBRARIES install-toolexeclibgoDATA \ + install-toolexeclibgoarchiveDATA \ install-toolexeclibgocompressDATA \ install-toolexeclibgocontainerDATA \ install-toolexeclibgocryptoDATA \ @@ -3069,6 +3134,17 @@ s-sigtab: $(srcdir)/mksigtab.sh gen-sysinfo.go $(SHELL) $(srcdir)/mvifdiff.sh tmp-sigtab.go sigtab.go $(STAMP) $@ +zdefaultcc.go: s-zdefaultcc; @true +s-zdefaultcc: Makefile + echo 'package cfg' > zdefaultcc.go.tmp + echo >> zdefaultcc.go.tmp + echo 'const DefaultGCCGO = "$(bindir)/$(GCCGO_INSTALL_NAME)"' >> zdefaultcc.go.tmp + echo 'const DefaultCC = "$(GCC_INSTALL_NAME)"' >> zdefaultcc.go.tmp + echo 'const DefaultCXX = "$(GXX_INSTALL_NAME)"' >> zdefaultcc.go.tmp + echo 'const DefaultPkgConfig = "pkg-config"' >> zdefaultcc.go.tmp + $(SHELL) $(srcdir)/../move-if-change zdefaultcc.go.tmp zdefaultcc.go + $(STAMP) $@ + # _Complex_lock and _Reader_lock are Go translations of some AIX system # types and should not be exported back to C # semt is a Go translation of the C type sem_t; it fails to convert on @@ -3094,7 +3170,7 @@ s-runtime-inc: runtime.lo Makefile zstdpkglist.go: s-zstdpkglist; @true s-zstdpkglist: Makefile rm -f zstdpkglist.go.tmp - echo 'package main' > zstdpkglist.go.tmp + echo 'package load' > zstdpkglist.go.tmp echo "" >> zstdpkglist.go.tmp echo 'var stdpkg = map[string]bool{' >> zstdpkglist.go.tmp echo $(libgo_go_objs) 'unsafe.lo' 'runtime/cgo.lo' | sed 's|[a-z0-9_/]*_c\.lo||g' | sed 's|\([a-z0-9_/]*\)\.lo|"\1": true,|g' >> zstdpkglist.go.tmp @@ -3211,9 +3287,12 @@ endef # This line expands PACKAGE_template once for each package name listed # in $(PACKAGES). $(foreach package,$(PACKAGES),$(eval $(call PACKAGE_template,$(package)))) +$(foreach package,$(GOTOOL_PACKAGES),$(eval $(call PACKAGE_template,$(package)))) runtime.lo.dep: $(extra_go_files_runtime) syscall.lo.dep: $(extra_go_files_syscall) runtime/internal/sys.lo.dep: $(extra_go_files_runtime_internal_sys) +cmd/go/internal/cfg.lo.dep: $(extra_go_files_cmd_go_internal_cfg) +cmd/go/internal/load.lo.dep: $(extra_go_files_cmd_go_internal_load) # FIXME: The following C files may as well move to the runtime # directory and be treated like other C files. diff --git a/libgo/VERSION b/libgo/VERSION index b38ce7712f8..6d1d72fa431 100644 --- a/libgo/VERSION +++ b/libgo/VERSION @@ -1 +1 @@ -go1.8.3 +go1.9 diff --git a/libgo/configure b/libgo/configure index 3b0d0d2f7ca..1e5e41bed21 100755 --- a/libgo/configure +++ b/libgo/configure @@ -2494,7 +2494,7 @@ ac_compiler_gnu=$ac_cv_c_compiler_gnu ac_config_headers="$ac_config_headers config.h" -libtool_VERSION=11:0:0 +libtool_VERSION=12:0:0 # Default to --enable-multilib diff --git a/libgo/configure.ac b/libgo/configure.ac index 80ad3dfa431..4bfe1698d32 100644 --- a/libgo/configure.ac +++ b/libgo/configure.ac @@ -11,7 +11,7 @@ AC_INIT(package-unused, version-unused,, libgo) AC_CONFIG_SRCDIR(Makefile.am) AC_CONFIG_HEADER(config.h) -libtool_VERSION=11:0:0 +libtool_VERSION=12:0:0 AC_SUBST(libtool_VERSION) AM_ENABLE_MULTILIB(, ..) diff --git a/libgo/go/archive/tar/common.go b/libgo/go/archive/tar/common.go index d2ae66d554d..d49c5c3fd9e 100644 --- a/libgo/go/archive/tar/common.go +++ b/libgo/go/archive/tar/common.go @@ -158,11 +158,15 @@ func (fi headerFileInfo) Mode() (mode os.FileMode) { // sysStat, if non-nil, populates h from system-dependent fields of fi. var sysStat func(fi os.FileInfo, h *Header) error -// Mode constants from the tar spec. const ( - c_ISUID = 04000 // Set uid - c_ISGID = 02000 // Set gid - c_ISVTX = 01000 // Save text (sticky bit) + // Mode constants from the USTAR spec: + // See http://pubs.opengroup.org/onlinepubs/9699919799/utilities/pax.html#tag_20_92_13_06 + c_ISUID = 04000 // Set uid + c_ISGID = 02000 // Set gid + c_ISVTX = 01000 // Save text (sticky bit) + + // Common Unix mode constants; these are not defined in any common tar standard. + // Header.FileInfo understands these, but FileInfoHeader will never produce these. c_ISDIR = 040000 // Directory c_ISFIFO = 010000 // FIFO c_ISREG = 0100000 // Regular file @@ -208,30 +212,24 @@ func FileInfoHeader(fi os.FileInfo, link string) (*Header, error) { } switch { case fm.IsRegular(): - h.Mode |= c_ISREG h.Typeflag = TypeReg h.Size = fi.Size() case fi.IsDir(): h.Typeflag = TypeDir - h.Mode |= c_ISDIR h.Name += "/" case fm&os.ModeSymlink != 0: h.Typeflag = TypeSymlink - h.Mode |= c_ISLNK h.Linkname = link case fm&os.ModeDevice != 0: if fm&os.ModeCharDevice != 0 { - h.Mode |= c_ISCHR h.Typeflag = TypeChar } else { - h.Mode |= c_ISBLK h.Typeflag = TypeBlock } case fm&os.ModeNamedPipe != 0: h.Typeflag = TypeFifo - h.Mode |= c_ISFIFO case fm&os.ModeSocket != 0: - h.Mode |= c_ISSOCK + return nil, fmt.Errorf("archive/tar: sockets not supported") default: return nil, fmt.Errorf("archive/tar: unknown file mode %v", fm) } diff --git a/libgo/go/archive/tar/tar_test.go b/libgo/go/archive/tar/tar_test.go index cf8337c2ad9..fb7a9dcfc47 100644 --- a/libgo/go/archive/tar/tar_test.go +++ b/libgo/go/archive/tar/tar_test.go @@ -6,9 +6,11 @@ package tar import ( "bytes" + "internal/testenv" "io/ioutil" "os" "path" + "path/filepath" "reflect" "strings" "testing" @@ -27,7 +29,7 @@ func TestFileInfoHeader(t *testing.T) { if g, e := h.Name, "small.txt"; g != e { t.Errorf("Name = %q; want %q", g, e) } - if g, e := h.Mode, int64(fi.Mode().Perm())|c_ISREG; g != e { + if g, e := h.Mode, int64(fi.Mode().Perm()); g != e { t.Errorf("Mode = %#o; want %#o", g, e) } if g, e := h.Size, int64(5); g != e { @@ -55,7 +57,7 @@ func TestFileInfoHeaderDir(t *testing.T) { t.Errorf("Name = %q; want %q", g, e) } // Ignoring c_ISGID for golang.org/issue/4867 - if g, e := h.Mode&^c_ISGID, int64(fi.Mode().Perm())|c_ISDIR; g != e { + if g, e := h.Mode&^c_ISGID, int64(fi.Mode().Perm()); g != e { t.Errorf("Mode = %#o; want %#o", g, e) } if g, e := h.Size, int64(0); g != e { @@ -67,40 +69,56 @@ func TestFileInfoHeaderDir(t *testing.T) { } func TestFileInfoHeaderSymlink(t *testing.T) { - h, err := FileInfoHeader(symlink{}, "some-target") + testenv.MustHaveSymlink(t) + + tmpdir, err := ioutil.TempDir("", "TestFileInfoHeaderSymlink") + if err != nil { + t.Fatal(err) + } + defer os.RemoveAll(tmpdir) + + link := filepath.Join(tmpdir, "link") + target := tmpdir + err = os.Symlink(target, link) + if err != nil { + t.Fatal(err) + } + fi, err := os.Lstat(link) if err != nil { t.Fatal(err) } - if g, e := h.Name, "some-symlink"; g != e { + + h, err := FileInfoHeader(fi, target) + if err != nil { + t.Fatal(err) + } + if g, e := h.Name, fi.Name(); g != e { t.Errorf("Name = %q; want %q", g, e) } - if g, e := h.Linkname, "some-target"; g != e { + if g, e := h.Linkname, target; g != e { t.Errorf("Linkname = %q; want %q", g, e) } + if g, e := h.Typeflag, byte(TypeSymlink); g != e { + t.Errorf("Typeflag = %v; want %v", g, e) + } } -type symlink struct{} - -func (symlink) Name() string { return "some-symlink" } -func (symlink) Size() int64 { return 0 } -func (symlink) Mode() os.FileMode { return os.ModeSymlink } -func (symlink) ModTime() time.Time { return time.Time{} } -func (symlink) IsDir() bool { return false } -func (symlink) Sys() interface{} { return nil } - func TestRoundTrip(t *testing.T) { data := []byte("some file contents") var b bytes.Buffer tw := NewWriter(&b) hdr := &Header{ - Name: "file.txt", - Uid: 1 << 21, // too big for 8 octal digits - Size: int64(len(data)), - ModTime: time.Now(), + Name: "file.txt", + Uid: 1 << 21, // too big for 8 octal digits + Size: int64(len(data)), + // AddDate to strip monotonic clock reading, + // and Round to discard sub-second precision, + // both of which are not included in the tar header + // and would otherwise break the round-trip check + // below. + ModTime: time.Now().AddDate(0, 0, 0).Round(1 * time.Second), } - // tar only supports second precision. - hdr.ModTime = hdr.ModTime.Add(-time.Duration(hdr.ModTime.Nanosecond()) * time.Nanosecond) if err := tw.WriteHeader(hdr); err != nil { t.Fatalf("tw.WriteHeader: %v", err) } @@ -139,7 +157,7 @@ func TestHeaderRoundTrip(t *testing.T) { // regular file. h: &Header{ Name: "test.txt", - Mode: 0644 | c_ISREG, + Mode: 0644, Size: 12, ModTime: time.Unix(1360600916, 0), Typeflag: TypeReg, @@ -149,7 +167,7 @@ func TestHeaderRoundTrip(t *testing.T) { // symbolic link. h: &Header{ Name: "link.txt", - Mode: 0777 | c_ISLNK, + Mode: 0777, Size: 0, ModTime: time.Unix(1360600852, 0), Typeflag: TypeSymlink, @@ -159,7 +177,7 @@ func TestHeaderRoundTrip(t *testing.T) { // character device node. h: &Header{ Name: "dev/null", - Mode: 0666 | c_ISCHR, + Mode: 0666, Size: 0, ModTime: time.Unix(1360578951, 0), Typeflag: TypeChar, @@ -169,7 +187,7 @@ func TestHeaderRoundTrip(t *testing.T) { // block device node. h: &Header{ Name: "dev/sda", - Mode: 0660 | c_ISBLK, + Mode: 0660, Size: 0, ModTime: time.Unix(1360578954, 0), Typeflag: TypeBlock, @@ -179,7 +197,7 @@ func TestHeaderRoundTrip(t *testing.T) { // directory. h: &Header{ Name: "dir/", - Mode: 0755 | c_ISDIR, + Mode: 0755, Size: 0, ModTime: time.Unix(1360601116, 0), Typeflag: TypeDir, @@ -189,7 +207,7 @@ func TestHeaderRoundTrip(t *testing.T) { // fifo node. h: &Header{ Name: "dev/initctl", - Mode: 0600 | c_ISFIFO, + Mode: 0600, Size: 0, ModTime: time.Unix(1360578949, 0), Typeflag: TypeFifo, @@ -199,7 +217,7 @@ func TestHeaderRoundTrip(t *testing.T) { // setuid. h: &Header{ Name: "bin/su", - Mode: 0755 | c_ISREG | c_ISUID, + Mode: 0755 | c_ISUID, Size: 23232, ModTime: time.Unix(1355405093, 0), Typeflag: TypeReg, @@ -209,7 +227,7 @@ func TestHeaderRoundTrip(t *testing.T) { // setguid. h: &Header{ Name: "group.txt", - Mode: 0750 | c_ISREG | c_ISGID, + Mode: 0750 | c_ISGID, Size: 0, ModTime: time.Unix(1360602346, 0), Typeflag: TypeReg, @@ -219,7 +237,7 @@ func TestHeaderRoundTrip(t *testing.T) { // sticky. h: &Header{ Name: "sticky.txt", - Mode: 0600 | c_ISREG | c_ISVTX, + Mode: 0600 | c_ISVTX, Size: 7, ModTime: time.Unix(1360602540, 0), Typeflag: TypeReg, @@ -229,7 +247,7 @@ func TestHeaderRoundTrip(t *testing.T) { // hard link. h: &Header{ Name: "hard.txt", - Mode: 0644 | c_ISREG, + Mode: 0644, Size: 0, Linkname: "file.txt", ModTime: time.Unix(1360600916, 0), @@ -240,7 +258,7 @@ func TestHeaderRoundTrip(t *testing.T) { // More information. h: &Header{ Name: "info.txt", - Mode: 0600 | c_ISREG, + Mode: 0600, Size: 0, Uid: 1000, Gid: 1000, diff --git a/libgo/go/archive/tar/writer.go b/libgo/go/archive/tar/writer.go index 596fb8b9e17..c51c243a8b8 100644 --- a/libgo/go/archive/tar/writer.go +++ b/libgo/go/archive/tar/writer.go @@ -121,9 +121,15 @@ func (tw *Writer) writeHeader(hdr *Header, allowPax bool) error { needsPaxHeader := paxKeyword != paxNone && len(s) > len(b) || !isASCII(s) if needsPaxHeader { paxHeaders[paxKeyword] = s - return } - f.formatString(b, s) + + // Write string in a best-effort manner to satisfy readers that expect + // the field to be non-empty. + s = toASCII(s) + if len(s) > len(b) { + s = s[:len(b)] + } + f.formatString(b, s) // Should never error } var formatNumeric = func(b []byte, x int64, paxKeyword string) { // Try octal first. diff --git a/libgo/go/archive/zip/register.go b/libgo/go/archive/zip/register.go index 2e76386b1fd..51e9c3e4d4b 100644 --- a/libgo/go/archive/zip/register.go +++ b/libgo/go/archive/zip/register.go @@ -103,51 +103,46 @@ func (r *pooledFlateReader) Close() error { } var ( - mu sync.RWMutex // guards compressor and decompressor maps + compressors sync.Map // map[uint16]Compressor + decompressors sync.Map // map[uint16]Decompressor +) - compressors = map[uint16]Compressor{ - Store: func(w io.Writer) (io.WriteCloser, error) { return &nopCloser{w}, nil }, - Deflate: func(w io.Writer) (io.WriteCloser, error) { return newFlateWriter(w), nil }, - } +func init() { + compressors.Store(Store, Compressor(func(w io.Writer) (io.WriteCloser, error) { return &nopCloser{w}, nil })) + compressors.Store(Deflate, Compressor(func(w io.Writer) (io.WriteCloser, error) { return newFlateWriter(w), nil })) - decompressors = map[uint16]Decompressor{ - Store: ioutil.NopCloser, - Deflate: newFlateReader, - } -) + decompressors.Store(Store, Decompressor(ioutil.NopCloser)) + decompressors.Store(Deflate, Decompressor(newFlateReader)) +} // RegisterDecompressor allows custom decompressors for a specified method ID. // The common methods Store and Deflate are built in. func RegisterDecompressor(method uint16, dcomp Decompressor) { - mu.Lock() - defer mu.Unlock() - - if _, ok := decompressors[method]; ok { + if _, dup := decompressors.LoadOrStore(method, dcomp); dup { panic("decompressor already registered") } - decompressors[method] = dcomp } // RegisterCompressor registers custom compressors for a specified method ID. // The common methods Store and Deflate are built in. func RegisterCompressor(method uint16, comp Compressor) { - mu.Lock() - defer mu.Unlock() - - if _, ok := compressors[method]; ok { + if _, dup := compressors.LoadOrStore(method, comp); dup { panic("compressor already registered") } - compressors[method] = comp } func compressor(method uint16) Compressor { - mu.RLock() - defer mu.RUnlock() - return compressors[method] + ci, ok := compressors.Load(method) + if !ok { + return nil + } + return ci.(Compressor) } func decompressor(method uint16) Decompressor { - mu.RLock() - defer mu.RUnlock() - return decompressors[method] + di, ok := decompressors.Load(method) + if !ok { + return nil + } + return di.(Decompressor) } diff --git a/libgo/go/archive/zip/struct.go b/libgo/go/archive/zip/struct.go index e92d02f8a2e..0be210e8e73 100644 --- a/libgo/go/archive/zip/struct.go +++ b/libgo/go/archive/zip/struct.go @@ -5,7 +5,7 @@ /* Package zip provides support for reading and writing ZIP archives. -See: https://www.pkware.com/documents/casestudies/APPNOTE.TXT +See: https://www.pkware.com/appnote This package does not support disk spanning. diff --git a/libgo/go/archive/zip/writer.go b/libgo/go/archive/zip/writer.go index 8940e25560e..9f4fceee844 100644 --- a/libgo/go/archive/zip/writer.go +++ b/libgo/go/archive/zip/writer.go @@ -11,10 +11,9 @@ import ( "hash" "hash/crc32" "io" + "unicode/utf8" ) -// TODO(adg): support zip file comments - // Writer implements a zip file writer. type Writer struct { cw *countWriter @@ -201,6 +200,20 @@ func (w *Writer) Create(name string) (io.Writer, error) { return w.CreateHeader(header) } +func hasValidUTF8(s string) bool { + n := 0 + for _, r := range s { + // By default, ZIP uses CP437, which is only identical to ASCII for the printable characters. + if r < 0x20 || r >= 0x7f { + if !utf8.ValidRune(r) { + return false + } + n++ + } + } + return n > 0 +} + // CreateHeader adds a file to the zip file using the provided FileHeader // for the file metadata. // It returns a Writer to which the file contents should be written. @@ -221,6 +234,10 @@ func (w *Writer) CreateHeader(fh *FileHeader) (io.Writer, error) { fh.Flags |= 0x8 // we will write a data descriptor + if hasValidUTF8(fh.Name) || hasValidUTF8(fh.Comment) { + fh.Flags |= 0x800 // filename or comment have valid utf-8 string + } + fh.CreatorVersion = fh.CreatorVersion&0xff00 | zipVersion20 // preserve compatibility byte fh.ReaderVersion = zipVersion20 diff --git a/libgo/go/archive/zip/writer_test.go b/libgo/go/archive/zip/writer_test.go index 86841c755f1..92fb6ecf0ed 100644 --- a/libgo/go/archive/zip/writer_test.go +++ b/libgo/go/archive/zip/writer_test.go @@ -87,6 +87,69 @@ func TestWriter(t *testing.T) { } } +func TestWriterUTF8(t *testing.T) { + var utf8Tests = []struct { + name string + comment string + expect uint16 + }{ + { + name: "hi, hello", + comment: "in the world", + expect: 0x8, + }, + { + name: "hi, こんにちわ", + comment: "in the world", + expect: 0x808, + }, + { + name: "hi, hello", + comment: "in the 世界", + expect: 0x808, + }, + { + name: "hi, こんにちわ", + comment: "in the 世界", + expect: 0x808, + }, + } + + // write a zip file + buf := new(bytes.Buffer) + w := NewWriter(buf) + + for _, test := range utf8Tests { + h := &FileHeader{ + Name: test.name, + Comment: test.comment, + Method: Deflate, + } + w, err := w.CreateHeader(h) + if err != nil { + t.Fatal(err) + } + w.Write([]byte{}) + } + + if err := w.Close(); err != nil { + t.Fatal(err) + } + + // read it back + r, err := NewReader(bytes.NewReader(buf.Bytes()), int64(buf.Len())) + if err != nil { + t.Fatal(err) + } + for i, test := range utf8Tests { + got := r.File[i].Flags + t.Logf("name %v, comment %v", test.name, test.comment) + if got != test.expect { + t.Fatalf("Flags: got %v, want %v", got, test.expect) + } + } +} + func TestWriterOffset(t *testing.T) { largeData := make([]byte, 1<<17) for i := range largeData { @@ -181,12 +244,11 @@ func testReadFile(t *testing.T, f *File, wt *WriteTest) { } func BenchmarkCompressedZipGarbage(b *testing.B) { - b.ReportAllocs() - var buf bytes.Buffer bigBuf := bytes.Repeat([]byte("a"), 1<<20) - for i := 0; i <= b.N; i++ { + + runOnce := func(buf *bytes.Buffer) { buf.Reset() - zw := NewWriter(&buf) + zw := NewWriter(buf) for j := 0; j < 3; j++ { w, _ := zw.CreateHeader(&FileHeader{ Name: "foo", @@ -195,11 +257,19 @@ func BenchmarkCompressedZipGarbage(b *testing.B) { w.Write(bigBuf) } zw.Close() - if i == 0 { - // Reset the timer after the first time through. - // This effectively discards the very large initial flate setup cost, - // as well as the initialization of bigBuf. - b.ResetTimer() - } } + + b.ReportAllocs() + // Run once and then reset the timer. + // This effectively discards the very large initial flate setup cost, + // as well as the initialization of bigBuf. + runOnce(&bytes.Buffer{}) + b.ResetTimer() + + b.RunParallel(func(pb *testing.PB) { + var buf bytes.Buffer + for pb.Next() { + runOnce(&buf) + } + }) } diff --git a/libgo/go/archive/zip/zip_test.go b/libgo/go/archive/zip/zip_test.go index 57edb2cabf4..18c2171ba6c 100644 --- a/libgo/go/archive/zip/zip_test.go +++ b/libgo/go/archive/zip/zip_test.go @@ -255,7 +255,7 @@ func TestZip64EdgeCase(t *testing.T) { testZip64DirectoryRecordLength(buf, t) } -// Tests that we generate a zip64 file if the the directory at offset +// Tests that we generate a zip64 file if the directory at offset // 0xFFFFFFFF, but not before. func TestZip64DirectoryOffset(t *testing.T) { if testing.Short() && race.Enabled { @@ -681,6 +681,18 @@ func BenchmarkZip64Test(b *testing.B) { } } +func BenchmarkZip64TestSizes(b *testing.B) { + for _, size := range []int64{1 << 12, 1 << 20, 1 << 26} { + b.Run(fmt.Sprint(size), func(b *testing.B) { + b.RunParallel(func(pb *testing.PB) { + for pb.Next() { + testZip64(b, size) + } + }) + }) + } +} + func TestSuffixSaver(t *testing.T) { const keep = 10 ss := &suffixSaver{keep: keep} diff --git a/libgo/go/bufio/bufio.go b/libgo/go/bufio/bufio.go index e1e8fb22720..da94a2503f0 100644 --- a/libgo/go/bufio/bufio.go +++ b/libgo/go/bufio/bufio.go @@ -2,7 +2,7 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -// Package bufio implements buffered I/O. It wraps an io.Reader or io.Writer +// Package bufio implements buffered I/O. It wraps an io.Reader or io.Writer // object, creating another object (Reader or Writer) that also implements // the interface but provides buffering and some help for textual I/O. package bufio @@ -458,6 +458,7 @@ func (b *Reader) ReadString(delim byte) (string, error) { } // WriteTo implements io.WriterTo. +// This may make multiple calls to the Read method of the underlying Reader. func (b *Reader) WriteTo(w io.Writer) (n int64, err error) { n, err = b.writeBuf(w) if err != nil { @@ -513,7 +514,7 @@ func (b *Reader) writeBuf(w io.Writer) (int64, error) { // Writer implements buffering for an io.Writer object. // If an error occurs writing to a Writer, no more data will be -// accepted and all subsequent writes will return the error. +// accepted and all subsequent writes, and Flush, will return the error. // After all data has been written, the client should call the // Flush method to guarantee all data has been forwarded to // the underlying io.Writer. diff --git a/libgo/go/bufio/scan_test.go b/libgo/go/bufio/scan_test.go index 1bb1e883932..25682256887 100644 --- a/libgo/go/bufio/scan_test.go +++ b/libgo/go/bufio/scan_test.go @@ -169,7 +169,6 @@ func genLine(buf *bytes.Buffer, lineNum, n int, addNewline bool) { } buf.WriteByte('\n') } - return } // Test the line splitter, including some carriage returns but no long lines. diff --git a/libgo/go/builtin/builtin.go b/libgo/go/builtin/builtin.go index 281de0b4368..1c7c041d680 100644 --- a/libgo/go/builtin/builtin.go +++ b/libgo/go/builtin/builtin.go @@ -85,11 +85,11 @@ type uintptr uintptr // byte is an alias for uint8 and is equivalent to uint8 in all ways. It is // used, by convention, to distinguish byte values from 8-bit unsigned // integer values. -type byte byte +type byte = uint8 // rune is an alias for int32 and is equivalent to int32 in all ways. It is // used, by convention, to distinguish character values from integer values. -type rune rune +type rune = int32 // iota is a predeclared identifier representing the untyped integer ordinal // number of the current const specification in a (usually parenthesized) @@ -179,7 +179,7 @@ func cap(v Type) int // Channel: The channel's buffer is initialized with the specified // buffer capacity. If zero, or the size is omitted, the channel is // unbuffered. -func make(Type, size IntegerType) Type +func make(t Type, size ...IntegerType) Type // The new built-in function allocates memory. The first argument is a type, // not a value, and the value returned is a pointer to a newly diff --git a/libgo/go/bytes/buffer.go b/libgo/go/bytes/buffer.go index 196419dc3da..20e42bbbbca 100644 --- a/libgo/go/bytes/buffer.go +++ b/libgo/go/bytes/buffer.go @@ -15,10 +15,15 @@ import ( // A Buffer is a variable-sized buffer of bytes with Read and Write methods. // The zero value for Buffer is an empty buffer ready to use. type Buffer struct { - buf []byte // contents are the bytes buf[off : len(buf)] - off int // read at &buf[off], write at &buf[len(buf)] - bootstrap [64]byte // memory to hold first slice; helps small buffers avoid allocation. - lastRead readOp // last read operation, so that Unread* can work correctly. + buf []byte // contents are the bytes buf[off : len(buf)] + off int // read at &buf[off], write at &buf[len(buf)] + lastRead readOp // last read operation, so that Unread* can work correctly. + // FIXME: lastRead can fit in a single byte + + // memory to hold first slice; helps small buffers avoid allocation. + // FIXME: it would be advisable to align Buffer to cachelines to avoid false + // sharing. + bootstrap [64]byte } // The readOp constants describe the last action performed on @@ -68,13 +73,13 @@ func (b *Buffer) Cap() int { return cap(b.buf) } // but continues to use the same allocated storage. // It panics if n is negative or greater than the length of the buffer. func (b *Buffer) Truncate(n int) { + if n == 0 { + b.Reset() + return + } b.lastRead = opInvalid - switch { - case n < 0 || n > b.Len(): + if n < 0 || n > b.Len() { panic("bytes.Buffer: truncation out of range") - case n == 0: - // Reuse buffer space. - b.off = 0 } b.buf = b.buf[0 : b.off+n] } @@ -82,7 +87,22 @@ func (b *Buffer) Truncate(n int) { // Reset resets the buffer to be empty, // but it retains the underlying storage for use by future writes. // Reset is the same as Truncate(0). -func (b *Buffer) Reset() { b.Truncate(0) } +func (b *Buffer) Reset() { + b.buf = b.buf[:0] + b.off = 0 + b.lastRead = opInvalid +} + +// tryGrowByReslice is a inlineable version of grow for the fast-case where the +// internal buffer only needs to be resliced. +// It returns the index where bytes should be written and whether it succeeded. +func (b *Buffer) tryGrowByReslice(n int) (int, bool) { + if l := len(b.buf); l+n <= cap(b.buf) { + b.buf = b.buf[:l+n] + return l, true + } + return 0, false +} // grow grows the buffer to guarantee space for n more bytes. // It returns the index where bytes should be written. @@ -91,29 +111,33 @@ func (b *Buffer) grow(n int) int { m := b.Len() // If buffer is empty, reset to recover space. if m == 0 && b.off != 0 { - b.Truncate(0) + b.Reset() } - if len(b.buf)+n > cap(b.buf) { - var buf []byte - if b.buf == nil && n <= len(b.bootstrap) { - buf = b.bootstrap[0:] - } else if m+n <= cap(b.buf)/2 { - // We can slide things down instead of allocating a new - // slice. We only need m+n <= cap(b.buf) to slide, but - // we instead let capacity get twice as large so we - // don't spend all our time copying. - copy(b.buf[:], b.buf[b.off:]) - buf = b.buf[:m] - } else { - // not enough space anywhere - buf = makeSlice(2*cap(b.buf) + n) - copy(buf, b.buf[b.off:]) - } + // Try to grow by means of a reslice. + if i, ok := b.tryGrowByReslice(n); ok { + return i + } + // Check if we can make use of bootstrap array. + if b.buf == nil && n <= len(b.bootstrap) { + b.buf = b.bootstrap[:n] + return 0 + } + if m+n <= cap(b.buf)/2 { + // We can slide things down instead of allocating a new + // slice. We only need m+n <= cap(b.buf) to slide, but + // we instead let capacity get twice as large so we + // don't spend all our time copying. + copy(b.buf[:], b.buf[b.off:]) + } else { + // Not enough space anywhere, we need to allocate. + buf := makeSlice(2*cap(b.buf) + n) + copy(buf, b.buf[b.off:]) b.buf = buf - b.off = 0 } - b.buf = b.buf[0 : b.off+m+n] - return b.off + m + // Restore b.off and len(b.buf). + b.off = 0 + b.buf = b.buf[:m+n] + return m } // Grow grows the buffer's capacity, if necessary, to guarantee space for @@ -134,7 +158,10 @@ func (b *Buffer) Grow(n int) { // buffer becomes too large, Write will panic with ErrTooLarge. func (b *Buffer) Write(p []byte) (n int, err error) { b.lastRead = opInvalid - m := b.grow(len(p)) + m, ok := b.tryGrowByReslice(len(p)) + if !ok { + m = b.grow(len(p)) + } return copy(b.buf[m:], p), nil } @@ -143,7 +170,10 @@ func (b *Buffer) Write(p []byte) (n int, err error) { // buffer becomes too large, WriteString will panic with ErrTooLarge. func (b *Buffer) WriteString(s string) (n int, err error) { b.lastRead = opInvalid - m := b.grow(len(s)) + m, ok := b.tryGrowByReslice(len(s)) + if !ok { + m = b.grow(len(s)) + } return copy(b.buf[m:], s), nil } @@ -161,7 +191,7 @@ func (b *Buffer) ReadFrom(r io.Reader) (n int64, err error) { b.lastRead = opInvalid // If buffer is empty, reset to recover space. if b.off >= len(b.buf) { - b.Truncate(0) + b.Reset() } for { if free := cap(b.buf) - len(b.buf); free < MinRead { @@ -225,7 +255,7 @@ func (b *Buffer) WriteTo(w io.Writer) (n int64, err error) { } } // Buffer is now empty; reset. - b.Truncate(0) + b.Reset() return } @@ -235,7 +265,10 @@ func (b *Buffer) WriteTo(w io.Writer) (n int64, err error) { // ErrTooLarge. func (b *Buffer) WriteByte(c byte) error { b.lastRead = opInvalid - m := b.grow(1) + m, ok := b.tryGrowByReslice(1) + if !ok { + m = b.grow(1) + } b.buf[m] = c return nil } @@ -250,7 +283,10 @@ func (b *Buffer) WriteRune(r rune) (n int, err error) { return 1, nil } b.lastRead = opInvalid - m := b.grow(utf8.UTFMax) + m, ok := b.tryGrowByReslice(utf8.UTFMax) + if !ok { + m = b.grow(utf8.UTFMax) + } n = utf8.EncodeRune(b.buf[m:m+utf8.UTFMax], r) b.buf = b.buf[:m+n] return n, nil @@ -264,7 +300,7 @@ func (b *Buffer) Read(p []byte) (n int, err error) { b.lastRead = opInvalid if b.off >= len(b.buf) { // Buffer is empty, reset to recover space. - b.Truncate(0) + b.Reset() if len(p) == 0 { return } @@ -302,7 +338,7 @@ func (b *Buffer) ReadByte() (byte, error) { b.lastRead = opInvalid if b.off >= len(b.buf) { // Buffer is empty, reset to recover space. - b.Truncate(0) + b.Reset() return 0, io.EOF } c := b.buf[b.off] @@ -320,7 +356,7 @@ func (b *Buffer) ReadRune() (r rune, size int, err error) { b.lastRead = opInvalid if b.off >= len(b.buf) { // Buffer is empty, reset to recover space. - b.Truncate(0) + b.Reset() return 0, 0, io.EOF } c := b.buf[b.off] @@ -337,12 +373,12 @@ func (b *Buffer) ReadRune() (r rune, size int, err error) { // UnreadRune unreads the last rune returned by ReadRune. // If the most recent read or write operation on the buffer was -// not a ReadRune, UnreadRune returns an error. (In this regard +// not a successful ReadRune, UnreadRune returns an error. (In this regard // it is stricter than UnreadByte, which will unread the last byte // from any read operation.) func (b *Buffer) UnreadRune() error { if b.lastRead <= opInvalid { - return errors.New("bytes.Buffer: UnreadRune: previous operation was not ReadRune") + return errors.New("bytes.Buffer: UnreadRune: previous operation was not a successful ReadRune") } if b.off >= int(b.lastRead) { b.off -= int(b.lastRead) @@ -351,12 +387,13 @@ func (b *Buffer) UnreadRune() error { return nil } -// UnreadByte unreads the last byte returned by the most recent -// read operation. If write has happened since the last read, UnreadByte -// returns an error. +// UnreadByte unreads the last byte returned by the most recent successful +// read operation that read at least one byte. If a write has happened since +// the last read, if the last read returned an error, or if the read read zero +// bytes, UnreadByte returns an error. func (b *Buffer) UnreadByte() error { if b.lastRead == opInvalid { - return errors.New("bytes.Buffer: UnreadByte: previous operation was not a read") + return errors.New("bytes.Buffer: UnreadByte: previous operation was not a successful read") } b.lastRead = opInvalid if b.off > 0 { @@ -404,10 +441,12 @@ func (b *Buffer) ReadString(delim byte) (line string, err error) { return string(slice), err } -// NewBuffer creates and initializes a new Buffer using buf as its initial -// contents. It is intended to prepare a Buffer to read existing data. It -// can also be used to size the internal buffer for writing. To do that, -// buf should have the desired capacity but a length of zero. +// NewBuffer creates and initializes a new Buffer using buf as its +// initial contents. The new Buffer takes ownership of buf, and the +// caller should not use buf after this call. NewBuffer is intended to +// prepare a Buffer to read existing data. It can also be used to size +// the internal buffer for writing. To do that, buf should have the +// desired capacity but a length of zero. // // In most cases, new(Buffer) (or just declaring a Buffer variable) is // sufficient to initialize a Buffer. diff --git a/libgo/go/bytes/buffer_test.go b/libgo/go/bytes/buffer_test.go index b1b85f979af..ce2f01a0ad3 100644 --- a/libgo/go/bytes/buffer_test.go +++ b/libgo/go/bytes/buffer_test.go @@ -6,8 +6,10 @@ package bytes_test import ( . "bytes" + "internal/testenv" "io" "math/rand" + "os/exec" "runtime" "testing" "unicode/utf8" @@ -311,6 +313,19 @@ func TestRuneIO(t *testing.T) { // Check that UnreadRune works buf.Reset() + + // check at EOF + if err := buf.UnreadRune(); err == nil { + t.Fatal("UnreadRune at EOF: got no error") + } + if _, _, err := buf.ReadRune(); err == nil { + t.Fatal("ReadRune at EOF: got no error") + } + if err := buf.UnreadRune(); err == nil { + t.Fatal("UnreadRune after ReadRune at EOF: got no error") + } + + // check not at EOF buf.Write(b) for r := rune(0); r < NRune; r++ { r1, size, _ := buf.ReadRune() @@ -473,15 +488,34 @@ func TestReadEmptyAtEOF(t *testing.T) { func TestUnreadByte(t *testing.T) { b := new(Buffer) + + // check at EOF + if err := b.UnreadByte(); err == nil { + t.Fatal("UnreadByte at EOF: got no error") + } + if _, err := b.ReadByte(); err == nil { + t.Fatal("ReadByte at EOF: got no error") + } + if err := b.UnreadByte(); err == nil { + t.Fatal("UnreadByte after ReadByte at EOF: got no error") + } + + // check not at EOF b.WriteString("abcdefghijklmnopqrstuvwxyz") - _, err := b.ReadBytes('m') - if err != nil { - t.Fatalf("ReadBytes: %v", err) + // after unsuccessful read + if n, err := b.Read(nil); n != 0 || err != nil { + t.Fatalf("Read(nil) = %d,%v; want 0,nil", n, err) + } + if err := b.UnreadByte(); err == nil { + t.Fatal("UnreadByte after Read(nil): got no error") } - err = b.UnreadByte() - if err != nil { + // after successful read + if _, err := b.ReadBytes('m'); err != nil { + t.Fatalf("ReadBytes: %v", err) + } + if err := b.UnreadByte(); err != nil { t.Fatalf("UnreadByte: %v", err) } c, err := b.ReadByte() @@ -514,6 +548,38 @@ func TestBufferGrowth(t *testing.T) { } } +// Test that tryGrowByReslice is inlined. +// Only execute on "linux-amd64" builder in order to avoid breakage. +func TestTryGrowByResliceInlined(t *testing.T) { + targetBuilder := "linux-amd64" + if testenv.Builder() != targetBuilder { + t.Skipf("%q gets executed on %q builder only", t.Name(), targetBuilder) + } + t.Parallel() + goBin := testenv.GoToolPath(t) + out, err := exec.Command(goBin, "tool", "nm", goBin).CombinedOutput() + if err != nil { + t.Fatalf("go tool nm: %v: %s", err, out) + } + // Verify this doesn't exist: + sym := "bytes.(*Buffer).tryGrowByReslice" + if Contains(out, []byte(sym)) { + t.Errorf("found symbol %q in cmd/go, but should be inlined", sym) + } +} + +func BenchmarkWriteByte(b *testing.B) { + const n = 4 << 10 + b.SetBytes(n) + buf := NewBuffer(make([]byte, n)) + for i := 0; i < b.N; i++ { + buf.Reset() + for i := 0; i < n; i++ { + buf.WriteByte('x') + } + } +} + func BenchmarkWriteRune(b *testing.B) { const n = 4 << 10 const r = '☺' diff --git a/libgo/go/bytes/bytes.go b/libgo/go/bytes/bytes.go index 406a38257a7..7c878af688c 100644 --- a/libgo/go/bytes/bytes.go +++ b/libgo/go/bytes/bytes.go @@ -46,36 +46,21 @@ func explode(s []byte, n int) [][]byte { return a[0:na] } -// Count counts the number of non-overlapping instances of sep in s. -// If sep is an empty slice, Count returns 1 + the number of Unicode code points in s. -func Count(s, sep []byte) int { - n := len(sep) - if n == 0 { +// countGeneric actually implements Count +func countGeneric(s, sep []byte) int { + // special case + if len(sep) == 0 { return utf8.RuneCount(s) + 1 } - if n > len(s) { - return 0 - } - count := 0 - c := sep[0] - i := 0 - t := s[:len(s)-n+1] - for i < len(t) { - if t[i] != c { - o := IndexByte(t[i:], c) - if o < 0 { - break - } - i += o - } - if n == 1 || Equal(s[i:i+n], sep) { - count++ - i += n - continue + n := 0 + for { + i := Index(s, sep) + if i == -1 { + return n } - i++ + n++ + s = s[i+len(sep):] } - return count } // Contains reports whether subslice is within b. @@ -229,20 +214,21 @@ func genSplit(s, sep []byte, sepSave, n int) [][]byte { if n < 0 { n = Count(s, sep) + 1 } - c := sep[0] - start := 0 + a := make([][]byte, n) - na := 0 - for i := 0; i+len(sep) <= len(s) && na+1 < n; i++ { - if s[i] == c && (len(sep) == 1 || Equal(s[i:i+len(sep)], sep)) { - a[na] = s[start : i+sepSave] - na++ - start = i + len(sep) - i += len(sep) - 1 + n-- + i := 0 + for i < n { + m := Index(s, sep) + if m < 0 { + break } + a[i] = s[:m+sepSave] + s = s[m+len(sep):] + i++ } - a[na] = s[start:] - return a[0 : na+1] + a[i] = s + return a[:i+1] } // SplitN slices s into subslices separated by sep and returns a slice of diff --git a/libgo/go/bytes/bytes_amd64.go b/libgo/go/bytes/bytes_amd64.go index 58a07efc58f..d40c7441e94 100644 --- a/libgo/go/bytes/bytes_amd64.go +++ b/libgo/go/bytes/bytes_amd64.go @@ -6,17 +6,19 @@ package bytes +import "internal/cpu" + //go:noescape // indexShortStr returns the index of the first instance of c in s, or -1 if c is not present in s. // indexShortStr requires 2 <= len(c) <= shortStringLen -func indexShortStr(s, c []byte) int // ../runtime/asm_$GOARCH.s -func supportAVX2() bool // ../runtime/asm_$GOARCH.s +func indexShortStr(s, c []byte) int // ../runtime/asm_amd64.s +func countByte(s []byte, c byte) int // ../runtime/asm_amd64.s var shortStringLen int func init() { - if supportAVX2() { + if cpu.X86.HasAVX2 { shortStringLen = 63 } else { shortStringLen = 31 @@ -96,6 +98,15 @@ func Index(s, sep []byte) int { return -1 } +// Count counts the number of non-overlapping instances of sep in s. +// If sep is an empty slice, Count returns 1 + the number of Unicode code points in s. +func Count(s, sep []byte) int { + if len(sep) == 1 && cpu.X86.HasPOPCNT { + return countByte(s, sep[0]) + } + return countGeneric(s, sep) +} + // primeRK is the prime base used in Rabin-Karp algorithm. const primeRK = 16777619 diff --git a/libgo/go/bytes/bytes_generic.go b/libgo/go/bytes/bytes_generic.go index 91baa22aa23..75a9c36dc42 100644 --- a/libgo/go/bytes/bytes_generic.go +++ b/libgo/go/bytes/bytes_generic.go @@ -39,3 +39,9 @@ func Index(s, sep []byte) int { } return -1 } + +// Count counts the number of non-overlapping instances of sep in s. +// If sep is an empty slice, Count returns 1 + the number of Unicode code points in s. +func Count(s, sep []byte) int { + return countGeneric(s, sep) +} diff --git a/libgo/go/bytes/bytes_s390x.go b/libgo/go/bytes/bytes_s390x.go index a05ca47aa16..54e013e9ada 100644 --- a/libgo/go/bytes/bytes_s390x.go +++ b/libgo/go/bytes/bytes_s390x.go @@ -99,6 +99,12 @@ func Index(s, sep []byte) int { return -1 } +// Count counts the number of non-overlapping instances of sep in s. +// If sep is an empty slice, Count returns 1 + the number of Unicode code points in s. +func Count(s, sep []byte) int { + return countGeneric(s, sep) +} + // primeRK is the prime base used in Rabin-Karp algorithm. const primeRK = 16777619 diff --git a/libgo/go/bytes/bytes_test.go b/libgo/go/bytes/bytes_test.go index ad01952bb73..d571eb3ab49 100644 --- a/libgo/go/bytes/bytes_test.go +++ b/libgo/go/bytes/bytes_test.go @@ -401,6 +401,79 @@ func TestIndexRune(t *testing.T) { } } +// test count of a single byte across page offsets +func TestCountByte(t *testing.T) { + b := make([]byte, 5015) // bigger than a page + windows := []int{1, 2, 3, 4, 15, 16, 17, 31, 32, 33, 63, 64, 65, 128} + testCountWindow := func(i, window int) { + for j := 0; j < window; j++ { + b[i+j] = byte(100) + p := Count(b[i:i+window], []byte{100}) + if p != j+1 { + t.Errorf("TestCountByte.Count(%q, 100) = %d", b[i:i+window], p) + } + pGeneric := CountGeneric(b[i:i+window], []byte{100}) + if pGeneric != j+1 { + t.Errorf("TestCountByte.CountGeneric(%q, 100) = %d", b[i:i+window], p) + } + } + } + + maxWnd := windows[len(windows)-1] + + for i := 0; i <= 2*maxWnd; i++ { + for _, window := range windows { + if window > len(b[i:]) { + window = len(b[i:]) + } + testCountWindow(i, window) + for j := 0; j < window; j++ { + b[i+j] = byte(0) + } + } + } + for i := 4096 - (maxWnd + 1); i < len(b); i++ { + for _, window := range windows { + if window > len(b[i:]) { + window = len(b[i:]) + } + testCountWindow(i, window) + for j := 0; j < window; j++ { + b[i+j] = byte(0) + } + } + } +} + +// Make sure we don't count bytes outside our window +func TestCountByteNoMatch(t *testing.T) { + b := make([]byte, 5015) + windows := []int{1, 2, 3, 4, 15, 16, 17, 31, 32, 33, 63, 64, 65, 128} + for i := 0; i <= len(b); i++ { + for _, window := range windows { + if window > len(b[i:]) { + window = len(b[i:]) + } + // Fill the window with non-match + for j := 0; j < window; j++ { + b[i+j] = byte(100) + } + // Try to find something that doesn't exist + p := Count(b[i:i+window], []byte{0}) + if p != 0 { + t.Errorf("TestCountByteNoMatch(%q, 0) = %d", b[i:i+window], p) + } + pGeneric := CountGeneric(b[i:i+window], []byte{0}) + if pGeneric != 0 { + t.Errorf("TestCountByteNoMatch.CountGeneric(%q, 100) = %d", b[i:i+window], p) + } + for j := 0; j < window; j++ { + b[i+j] = byte(0) + } + } + } +} + var bmbuf []byte func valName(x int) string { @@ -594,6 +667,26 @@ func BenchmarkCountEasy(b *testing.B) { }) } +func BenchmarkCountSingle(b *testing.B) { + benchBytes(b, indexSizes, func(b *testing.B, n int) { + buf := bmbuf[0:n] + step := 8 + for i := 0; i < len(buf); i += step { + buf[i] = 1 + } + expect := (len(buf) + (step - 1)) / step + for i := 0; i < b.N; i++ { + j := Count(buf, []byte{1}) + if j != expect { + b.Fatal("bad count", j, expect) + } + } + for i := 0; i < len(buf); i++ { + buf[i] = 0 + } + }) +} + type ExplodeTest struct { s string n int @@ -1437,6 +1530,59 @@ func BenchmarkTrimSpace(b *testing.B) { } } +func makeBenchInputHard() []byte { + tokens := [...]string{ + "", "

", "", "", + "", "

", "", "", + "hello", "world", + } + x := make([]byte, 0, 1<<20) + for { + i := rand.Intn(len(tokens)) + if len(x)+len(tokens[i]) >= 1<<20 { + break + } + x = append(x, tokens[i]...) + } + return x +} + +var benchInputHard = makeBenchInputHard() + +func BenchmarkSplitEmptySeparator(b *testing.B) { + for i := 0; i < b.N; i++ { + Split(benchInputHard, nil) + } +} + +func BenchmarkSplitSingleByteSeparator(b *testing.B) { + sep := []byte("/") + for i := 0; i < b.N; i++ { + Split(benchInputHard, sep) + } +} + +func BenchmarkSplitMultiByteSeparator(b *testing.B) { + sep := []byte("hello") + for i := 0; i < b.N; i++ { + Split(benchInputHard, sep) + } +} + +func BenchmarkSplitNSingleByteSeparator(b *testing.B) { + sep := []byte("/") + for i := 0; i < b.N; i++ { + SplitN(benchInputHard, sep, 10) + } +} + +func BenchmarkSplitNMultiByteSeparator(b *testing.B) { + sep := []byte("hello") + for i := 0; i < b.N; i++ { + SplitN(benchInputHard, sep, 10) + } +} + func BenchmarkRepeat(b *testing.B) { for i := 0; i < b.N; i++ { Repeat([]byte("-"), 80) diff --git a/libgo/go/bytes/example_test.go b/libgo/go/bytes/example_test.go index 0d35a0dc9c5..93972770ab2 100644 --- a/libgo/go/bytes/example_test.go +++ b/libgo/go/bytes/example_test.go @@ -30,6 +30,15 @@ func ExampleBuffer_reader() { // Output: Gophers rule! } +func ExampleBuffer_Grow() { + var b bytes.Buffer + b.Grow(64) + bb := b.Bytes() + b.Write([]byte("64 bytes or fewer")) + fmt.Printf("%q", bb[:b.Len()]) + // Output: "64 bytes or fewer" +} + func ExampleCompare() { // Interpret Compare's result by comparing it to zero. var a, b []byte diff --git a/libgo/go/bytes/export_test.go b/libgo/go/bytes/export_test.go index f61523e60bb..823c8b09eef 100644 --- a/libgo/go/bytes/export_test.go +++ b/libgo/go/bytes/export_test.go @@ -7,3 +7,4 @@ package bytes // Export func for testing var IndexBytePortable = indexBytePortable var EqualPortable = equalPortable +var CountGeneric = countGeneric diff --git a/libgo/go/cmd/cgo/ast.go b/libgo/go/cmd/cgo/ast.go index 8ce824196d9..7122a9dbbeb 100644 --- a/libgo/go/cmd/cgo/ast.go +++ b/libgo/go/cmd/cgo/ast.go @@ -17,8 +17,8 @@ import ( "strings" ) -func parse(name string, flags parser.Mode) *ast.File { - ast1, err := parser.ParseFile(fset, name, nil, flags) +func parse(name string, src []byte, flags parser.Mode) *ast.File { + ast1, err := parser.ParseFile(fset, name, src, flags) if err != nil { if list, ok := err.(scanner.ErrorList); ok { // If err is a scanner.ErrorList, its String will print just @@ -39,12 +39,12 @@ func sourceLine(n ast.Node) int { return fset.Position(n.Pos()).Line } -// ReadGo populates f with information learned from reading the -// Go source file with the given file name. It gathers the C preamble +// ParseGo populates f with information learned from the Go source code +// which was read from the named file. It gathers the C preamble // attached to the import "C" comment, a list of references to C.xxx, // a list of exported functions, and the actual AST, to be rewritten and // printed. -func (f *File) ReadGo(name string) { +func (f *File) ParseGo(name string, src []byte) { // Create absolute path for file, so that it will be used in error // messages and recorded in debug line number information. // This matches the rest of the toolchain. See golang.org/issue/5122. @@ -58,8 +58,8 @@ func (f *File) ReadGo(name string) { // so we use ast1 to look for the doc comments on import "C" // and on exported functions, and we use ast2 for translating // and reprinting. - ast1 := parse(name, parser.ParseComments) - ast2 := parse(name, 0) + ast1 := parse(name, src, parser.ParseComments) + ast2 := parse(name, src, 0) f.Package = ast1.Name.Name f.Name = make(map[string]*Name) diff --git a/libgo/go/cmd/cgo/doc.go b/libgo/go/cmd/cgo/doc.go index 85441e61c04..b2388829a87 100644 --- a/libgo/go/cmd/cgo/doc.go +++ b/libgo/go/cmd/cgo/doc.go @@ -14,27 +14,27 @@ as C.stdout, or functions such as C.putchar. If the import of "C" is immediately preceded by a comment, that comment, called the preamble, is used as a header when compiling -the C parts of the package. For example: +the C parts of the package. For example: // #include // #include import "C" The preamble may contain any C code, including function and variable -declarations and definitions. These may then be referred to from Go -code as though they were defined in the package "C". All names +declarations and definitions. These may then be referred to from Go +code as though they were defined in the package "C". All names declared in the preamble may be used, even if they start with a -lower-case letter. Exception: static variables in the preamble may +lower-case letter. Exception: static variables in the preamble may not be referenced from Go code; static functions are permitted. -See $GOROOT/misc/cgo/stdio and $GOROOT/misc/cgo/gmp for examples. See +See $GOROOT/misc/cgo/stdio and $GOROOT/misc/cgo/gmp for examples. See "C? Go? Cgo!" for an introduction to using cgo: https://golang.org/doc/articles/c_go_cgo.html. CFLAGS, CPPFLAGS, CXXFLAGS, FFLAGS and LDFLAGS may be defined with pseudo #cgo directives within these comments to tweak the behavior of the C, C++ -or Fortran compiler. Values defined in multiple directives are concatenated -together. The directive can include a list of build constraints limiting its +or Fortran compiler. Values defined in multiple directives are concatenated +together. The directive can include a list of build constraints limiting its effect to systems satisfying one of the constraints (see https://golang.org/pkg/go/build/#hdr-Build_Constraints for details about the constraint syntax). For example: @@ -57,16 +57,16 @@ The default pkg-config tool may be changed by setting the PKG_CONFIG environment When building, the CGO_CFLAGS, CGO_CPPFLAGS, CGO_CXXFLAGS, CGO_FFLAGS and CGO_LDFLAGS environment variables are added to the flags derived from -these directives. Package-specific flags should be set using the +these directives. Package-specific flags should be set using the directives, not the environment variables, so that builds work in unmodified environments. All the cgo CPPFLAGS and CFLAGS directives in a package are concatenated and -used to compile C files in that package. All the CPPFLAGS and CXXFLAGS +used to compile C files in that package. All the CPPFLAGS and CXXFLAGS directives in a package are concatenated and used to compile C++ files in that -package. All the CPPFLAGS and FFLAGS directives in a package are concatenated -and used to compile Fortran files in that package. All the LDFLAGS directives -in any package in the program are concatenated and used at link time. All the +package. All the CPPFLAGS and FFLAGS directives in a package are concatenated +and used to compile Fortran files in that package. All the LDFLAGS directives +in any package in the program are concatenated and used at link time. All the pkg-config directives are concatenated and sent to pkg-config simultaneously to add to each appropriate set of command-line flags. @@ -84,27 +84,27 @@ Will be expanded to: When the Go tool sees that one or more Go files use the special import "C", it will look for other non-Go files in the directory and compile -them as part of the Go package. Any .c, .s, or .S files will be -compiled with the C compiler. Any .cc, .cpp, or .cxx files will be -compiled with the C++ compiler. Any .f, .F, .for or .f90 files will be +them as part of the Go package. Any .c, .s, or .S files will be +compiled with the C compiler. Any .cc, .cpp, or .cxx files will be +compiled with the C++ compiler. Any .f, .F, .for or .f90 files will be compiled with the fortran compiler. Any .h, .hh, .hpp, or .hxx files will not be compiled separately, but, if these header files are changed, -the C and C++ files will be recompiled. The default C and C++ +the C and C++ files will be recompiled. The default C and C++ compilers may be changed by the CC and CXX environment variables, respectively; those environment variables may include command line options. The cgo tool is enabled by default for native builds on systems where -it is expected to work. It is disabled by default when -cross-compiling. You can control this by setting the CGO_ENABLED +it is expected to work. It is disabled by default when +cross-compiling. You can control this by setting the CGO_ENABLED environment variable when running the go tool: set it to 1 to enable -the use of cgo, and to 0 to disable it. The go tool will set the +the use of cgo, and to 0 to disable it. The go tool will set the build constraint "cgo" if cgo is enabled. When cross-compiling, you must specify a C cross-compiler for cgo to -use. You can do this by setting the CC_FOR_TARGET environment +use. You can do this by setting the CC_FOR_TARGET environment variable when building the toolchain using make.bash, or by setting -the CC environment variable any time you run the go tool. The +the CC environment variable any time you run the go tool. The CXX_FOR_TARGET and CXX environment variables work in a similar way for C++ code. @@ -138,7 +138,7 @@ C's union types are represented as a Go byte array with the same length. Go structs cannot embed fields with C types. Go code cannot refer to zero-sized fields that occur at the end of -non-empty C structs. To get the address of such a field (which is the +non-empty C structs. To get the address of such a field (which is the only operation you can do with a zero-sized field) you must take the address of the struct and add the size of the struct. @@ -150,7 +150,7 @@ is different from the same C type used in another. Any C function (even void functions) may be called in a multiple assignment context to retrieve both the return value (if any) and the C errno variable as an error (use _ to skip the result value if the -function returns void). For example: +function returns void). For example: n, err = C.sqrt(-1) _, err := C.voidFunc() @@ -187,11 +187,11 @@ received from Go. For example: In C, a function argument written as a fixed size array actually requires a pointer to the first element of the array. C compilers are aware of this calling convention and adjust -the call accordingly, but Go cannot. In Go, you must pass +the call accordingly, but Go cannot. In Go, you must pass the pointer to the first element explicitly: C.f(&C.x[0]). A few special functions convert between Go and C types -by making copies of the data. In pseudo-Go definitions: +by making copies of the data. In pseudo-Go definitions: // Go string to C string // The C string is allocated in the C heap using malloc. @@ -253,50 +253,50 @@ must be placed in preambles in other files, or in C source files. Passing pointers Go is a garbage collected language, and the garbage collector needs to -know the location of every pointer to Go memory. Because of this, +know the location of every pointer to Go memory. Because of this, there are restrictions on passing pointers between Go and C. In this section the term Go pointer means a pointer to memory allocated by Go (such as by using the & operator or calling the predefined new function) and the term C pointer means a pointer to -memory allocated by C (such as by a call to C.malloc). Whether a +memory allocated by C (such as by a call to C.malloc). Whether a pointer is a Go pointer or a C pointer is a dynamic property determined by how the memory was allocated; it has nothing to do with the type of the pointer. Go code may pass a Go pointer to C provided the Go memory to which it -points does not contain any Go pointers. The C code must preserve +points does not contain any Go pointers. The C code must preserve this property: it must not store any Go pointers in Go memory, even -temporarily. When passing a pointer to a field in a struct, the Go +temporarily. When passing a pointer to a field in a struct, the Go memory in question is the memory occupied by the field, not the entire -struct. When passing a pointer to an element in an array or slice, +struct. When passing a pointer to an element in an array or slice, the Go memory in question is the entire array or the entire backing array of the slice. C code may not keep a copy of a Go pointer after the call returns. -A Go function called by C code may not return a Go pointer. A Go +A Go function called by C code may not return a Go pointer. A Go function called by C code may take C pointers as arguments, and it may store non-pointer or C pointer data through those pointers, but it may -not store a Go pointer in memory pointed to by a C pointer. A Go +not store a Go pointer in memory pointed to by a C pointer. A Go function called by C code may take a Go pointer as an argument, but it must preserve the property that the Go memory to which it points does not contain any Go pointers. -Go code may not store a Go pointer in C memory. C code may store Go +Go code may not store a Go pointer in C memory. C code may store Go pointers in C memory, subject to the rule above: it must stop storing the Go pointer when the C function returns. -These rules are checked dynamically at runtime. The checking is +These rules are checked dynamically at runtime. The checking is controlled by the cgocheck setting of the GODEBUG environment -variable. The default setting is GODEBUG=cgocheck=1, which implements -reasonably cheap dynamic checks. These checks may be disabled -entirely using GODEBUG=cgocheck=0. Complete checking of pointer +variable. The default setting is GODEBUG=cgocheck=1, which implements +reasonably cheap dynamic checks. These checks may be disabled +entirely using GODEBUG=cgocheck=0. Complete checking of pointer handling, at some cost in run time, is available via GODEBUG=cgocheck=2. It is possible to defeat this enforcement by using the unsafe package, and of course there is nothing stopping the C code from doing anything -it likes. However, programs that break these rules are likely to fail +it likes. However, programs that break these rules are likely to fail in unexpected and unpredictable ways. Using cgo directly @@ -499,7 +499,7 @@ Here is a _cgo_gotypes.go containing definitions for needed C types: type _Ctype_void [0]byte The _cgo_gotypes.go file also contains the definitions of the -functions. They all have similar bodies that invoke runtime·cgocall +functions. They all have similar bodies that invoke runtime·cgocall to make a switch from the Go runtime world to the system C (GCC-based) world. @@ -835,7 +835,7 @@ to avoid conflicts), write the go.o file to that directory, and invoke the host linker. The default value for the host linker is $CC, split into fields, or else "gcc". The specific host linker command line can be overridden using command line flags: cmd/link -extld=clang --extldflags='-ggdb -O3'. If any package in a build includes a .cc or +-extldflags='-ggdb -O3'. If any package in a build includes a .cc or other file compiled by the C++ compiler, the go tool will use the -extld option to set the host linker to the C++ compiler. diff --git a/libgo/go/cmd/cgo/gcc.go b/libgo/go/cmd/cgo/gcc.go index 028804e4ab1..03239a0fc0d 100644 --- a/libgo/go/cmd/cgo/gcc.go +++ b/libgo/go/cmd/cgo/gcc.go @@ -20,6 +20,7 @@ import ( "go/ast" "go/parser" "go/token" + "math" "os" "strconv" "strings" @@ -259,26 +260,26 @@ func (p *Package) guessKinds(f *File) []*Name { // If we've already found this name as a #define // and we can translate it as a constant value, do so. if n.Define != "" { - isConst := false - if _, err := strconv.Atoi(n.Define); err == nil { - isConst = true - } else if n.Define[0] == '"' || n.Define[0] == '\'' { - if _, err := parser.ParseExpr(n.Define); err == nil { - isConst = true - } - } - if isConst { - n.Kind = "const" + if i, err := strconv.ParseInt(n.Define, 0, 64); err == nil { + n.Kind = "iconst" // Turn decimal into hex, just for consistency // with enum-derived constants. Otherwise // in the cgo -godefs output half the constants // are in hex and half are in whatever the #define used. - i, err := strconv.ParseInt(n.Define, 0, 64) - if err == nil { - n.Const = fmt.Sprintf("%#x", i) - } else { + n.Const = fmt.Sprintf("%#x", i) + } else if n.Define[0] == '\'' { + if _, err := parser.ParseExpr(n.Define); err == nil { + n.Kind = "iconst" n.Const = n.Define } + } else if n.Define[0] == '"' { + if _, err := parser.ParseExpr(n.Define); err == nil { + n.Kind = "sconst" + n.Const = n.Define + } + } + + if n.IsConst() { continue } @@ -287,11 +288,10 @@ func (p *Package) guessKinds(f *File) []*Name { } } - needType = append(needType, n) - // If this is a struct, union, or enum type name, no need to guess the kind. if strings.HasPrefix(n.C, "struct ") || strings.HasPrefix(n.C, "union ") || strings.HasPrefix(n.C, "enum ") { n.Kind = "type" + needType = append(needType, n) continue } @@ -317,14 +317,24 @@ func (p *Package) guessKinds(f *File) []*Name { // void __cgo_f_xxx_1(void) { __typeof__(name) *__cgo_undefined__; } // #line xxx "not-type" // void __cgo_f_xxx_2(void) { name *__cgo_undefined__; } - // #line xxx "not-const" + // #line xxx "not-int-const" // void __cgo_f_xxx_3(void) { enum { __cgo_undefined__ = (name)*1 }; } + // #line xxx "not-num-const" + // void __cgo_f_xxx_4(void) { static const double x = (name); } + // #line xxx "not-str-lit" + // void __cgo_f_xxx_5(void) { static const char x[] = (name); } + // #line xxx "not-signed-int-const" + // #if 0 < -(name) + // #line xxx "not-signed-int-const" + // #error found unsigned int + // #endif // // If we see an error at not-declared:xxx, the corresponding name is not declared. // If we see an error at not-type:xxx, the corresponding name is a type. - // If we see an error at not-const:xxx, the corresponding name is not an integer constant. - // If we see no errors, we assume the name is an expression but not a constant - // (so a variable or a function). + // If we see an error at not-int-const:xxx, the corresponding name is not an integer constant. + // If we see an error at not-num-const:xxx, the corresponding name is not a number constant. + // If we see an error at not-str-lit:xxx, the corresponding name is not a string literal. + // If we see an error at not-signed-int-const:xxx, the corresponding name is not a signed integer literal. // // The specific input forms are chosen so that they are valid C syntax regardless of // whether name denotes a type or an expression. @@ -338,11 +348,24 @@ func (p *Package) guessKinds(f *File) []*Name { "void __cgo_f_%d_1(void) { __typeof__(%s) *__cgo_undefined__; }\n"+ "#line %d \"not-type\"\n"+ "void __cgo_f_%d_2(void) { %s *__cgo_undefined__; }\n"+ - "#line %d \"not-const\"\n"+ - "void __cgo_f_%d_3(void) { enum { __cgo__undefined__ = (%s)*1 }; }\n", + "#line %d \"not-int-const\"\n"+ + "void __cgo_f_%d_3(void) { enum { __cgo_undefined__ = (%s)*1 }; }\n"+ + "#line %d \"not-num-const\"\n"+ + "void __cgo_f_%d_4(void) { static const double x = (%s); }\n"+ + "#line %d \"not-str-lit\"\n"+ + "void __cgo_f_%d_5(void) { static const char s[] = (%s); }\n"+ + "#line %d \"not-signed-int-const\"\n"+ + "#if 0 < (%s)\n"+ + "#line %d \"not-signed-int-const\"\n"+ + "#error found unsigned int\n"+ + "#endif\n", + i+1, i+1, n.C, + i+1, i+1, n.C, i+1, i+1, n.C, i+1, i+1, n.C, - i+1, i+1, n.C) + i+1, i+1, n.C, + i+1, n.C, i+1, + ) } fmt.Fprintf(&b, "#line 1 \"completed\"\n"+ "int __cgo__1 = __cgo__2;\n") @@ -356,13 +379,23 @@ func (p *Package) guessKinds(f *File) []*Name { sniff := make([]int, len(names)) const ( notType = 1 << iota - notConst + notIntConst + notNumConst + notStrLiteral notDeclared + notSignedIntConst ) + sawUnmatchedErrors := false for _, line := range strings.Split(stderr, "\n") { - if !strings.Contains(line, ": error:") { - // we only care about errors. - // we tried to turn off warnings on the command line, but one never knows. + // Ignore warnings and random comments, with one + // exception: newer GCC versions will sometimes emit + // an error on a macro #define with a note referring + // to where the expansion occurs. We care about where + // the expansion occurs, so in that case treat the note + // as an error. + isError := strings.Contains(line, ": error:") + isErrorNote := strings.Contains(line, ": note:") && sawUnmatchedErrors + if !isError && !isErrorNote { continue } @@ -380,6 +413,9 @@ func (p *Package) guessKinds(f *File) []*Name { i, _ := strconv.Atoi(line[c1+1 : c2]) i-- if i < 0 || i >= len(names) { + if isError { + sawUnmatchedErrors = true + } continue } @@ -395,9 +431,22 @@ func (p *Package) guessKinds(f *File) []*Name { sniff[i] |= notDeclared case "not-type": sniff[i] |= notType - case "not-const": - sniff[i] |= notConst + case "not-int-const": + sniff[i] |= notIntConst + case "not-num-const": + sniff[i] |= notNumConst + case "not-str-lit": + sniff[i] |= notStrLiteral + case "not-signed-int-const": + sniff[i] |= notSignedIntConst + default: + if isError { + sawUnmatchedErrors = true + } + continue } + + sawUnmatchedErrors = false } if !completed { @@ -405,14 +454,29 @@ func (p *Package) guessKinds(f *File) []*Name { } for i, n := range names { - switch sniff[i] { + switch sniff[i] &^ notSignedIntConst { default: - error_(token.NoPos, "could not determine kind of name for C.%s", fixGo(n.Go)) - case notType: - n.Kind = "const" - case notConst: + var tpos token.Pos + for _, ref := range f.Ref { + if ref.Name == n { + tpos = ref.Pos() + break + } + } + error_(tpos, "could not determine kind of name for C.%s", fixGo(n.Go)) + case notStrLiteral | notType: + if sniff[i]¬SignedIntConst != 0 { + n.Kind = "uconst" + } else { + n.Kind = "iconst" + } + case notIntConst | notStrLiteral | notType: + n.Kind = "fconst" + case notIntConst | notNumConst | notType: + n.Kind = "sconst" + case notIntConst | notNumConst | notStrLiteral: n.Kind = "type" - case notConst | notType: + case notIntConst | notNumConst | notStrLiteral | notType: n.Kind = "not-type" } } @@ -450,18 +514,16 @@ func (p *Package) loadDWARF(f *File, names []*Name) { b.WriteString("#line 1 \"cgo-dwarf-inference\"\n") for i, n := range names { fmt.Fprintf(&b, "__typeof__(%s) *__cgo__%d;\n", n.C, i) - if n.Kind == "const" { + if n.Kind == "iconst" || n.Kind == "uconst" { fmt.Fprintf(&b, "enum { __cgo_enum__%d = %s };\n", i, n.C) } } - // Apple's LLVM-based gcc does not include the enumeration - // names and values in its DWARF debug output. In case we're - // using such a gcc, create a data block initialized with the values. - // We can read them out of the object file. - fmt.Fprintf(&b, "long long __cgodebug_data[] = {\n") + // We create a data block initialized with the values, + // so we can read them out of the object file. + fmt.Fprintf(&b, "long long __cgodebug_ints[] = {\n") for _, n := range names { - if n.Kind == "const" { + if n.Kind == "iconst" || n.Kind == "uconst" { fmt.Fprintf(&b, "\t%s,\n", n.C) } else { fmt.Fprintf(&b, "\t0,\n") @@ -475,15 +537,30 @@ func (p *Package) loadDWARF(f *File, names []*Name) { fmt.Fprintf(&b, "\t1\n") fmt.Fprintf(&b, "};\n") - d, bo, debugData := p.gccDebug(b.Bytes()) - enumVal := make([]int64, len(debugData)/8) - for i := range enumVal { - enumVal[i] = int64(bo.Uint64(debugData[i*8:])) + // do the same work for floats. + fmt.Fprintf(&b, "double __cgodebug_floats[] = {\n") + for _, n := range names { + if n.Kind == "fconst" { + fmt.Fprintf(&b, "\t%s,\n", n.C) + } else { + fmt.Fprintf(&b, "\t0,\n") + } + } + fmt.Fprintf(&b, "\t1\n") + fmt.Fprintf(&b, "};\n") + + // do the same work for strings. + for i, n := range names { + if n.Kind == "sconst" { + fmt.Fprintf(&b, "const char __cgodebug_str__%d[] = %s;\n", i, n.C) + fmt.Fprintf(&b, "const unsigned long long __cgodebug_strlen__%d = sizeof(%s)-1;\n", i, n.C) + } } + d, ints, floats, strs := p.gccDebug(b.Bytes(), len(names)) + // Scan DWARF info for top-level TagVariable entries with AttrName __cgo__i. types := make([]dwarf.Type, len(names)) - enums := make([]dwarf.Offset, len(names)) nameToIndex := make(map[*Name]int) for i, n := range names { nameToIndex[n] = i @@ -502,26 +579,6 @@ func (p *Package) loadDWARF(f *File, names []*Name) { break } switch e.Tag { - case dwarf.TagEnumerationType: - offset := e.Offset - for { - e, err := r.Next() - if err != nil { - fatalf("reading DWARF entry: %s", err) - } - if e.Tag == 0 { - break - } - if e.Tag == dwarf.TagEnumerator { - entryName := e.Val(dwarf.AttrName).(string) - if strings.HasPrefix(entryName, "__cgo_enum__") { - n, _ := strconv.Atoi(entryName[len("__cgo_enum__"):]) - if 0 <= n && n < len(names) { - enums[n] = offset - } - } - } - } case dwarf.TagVariable: name, _ := e.Val(dwarf.AttrName).(string) typOff, _ := e.Val(dwarf.AttrType).(dwarf.Offset) @@ -548,15 +605,7 @@ func (p *Package) loadDWARF(f *File, names []*Name) { if err != nil { fatalf("malformed __cgo__ name: %s", name) } - if enums[i] != 0 { - t, err := d.Type(enums[i]) - if err != nil { - fatalf("loading DWARF type: %s", err) - } - types[i] = t - } else { - types[i] = t.Type - } + types[i] = t.Type } if e.Tag != dwarf.TagCompileUnit { r.SkipChildren() @@ -580,17 +629,23 @@ func (p *Package) loadDWARF(f *File, names []*Name) { n.FuncType = conv.FuncType(f, pos) } else { n.Type = conv.Type(types[i], pos) - if enums[i] != 0 && n.Type.EnumValues != nil { - k := fmt.Sprintf("__cgo_enum__%d", i) - n.Kind = "const" - n.Const = fmt.Sprintf("%#x", n.Type.EnumValues[k]) - // Remove injected enum to ensure the value will deep-compare - // equally in future loads of the same constant. - delete(n.Type.EnumValues, k) - } - // Prefer debug data over DWARF debug output, if we have it. - if n.Kind == "const" && i < len(enumVal) { - n.Const = fmt.Sprintf("%#x", enumVal[i]) + switch n.Kind { + case "iconst": + if i < len(ints) { + n.Const = fmt.Sprintf("%#x", ints[i]) + } + case "uconst": + if i < len(ints) { + n.Const = fmt.Sprintf("%#x", uint64(ints[i])) + } + case "fconst": + if i < len(floats) { + n.Const = fmt.Sprintf("%f", floats[i]) + } + case "sconst": + if i < len(strs) { + n.Const = fmt.Sprintf("%q", strs[i]) + } } } conv.FinishType(pos) @@ -1069,7 +1124,7 @@ func (p *Package) rewriteRef(f *File) { // are trying to do a ,err call. Also check that // functions are only used in calls. for _, r := range f.Ref { - if r.Name.Kind == "const" && r.Name.Const == "" { + if r.Name.IsConst() && r.Name.Const == "" { error_(r.Pos(), "unable to find value of constant C.%s", fixGo(r.Name.Go)) } var expr ast.Expr = ast.NewIdent(r.Name.Mangle) // default @@ -1109,6 +1164,10 @@ func (p *Package) rewriteRef(f *File) { } case "expr": if r.Name.Kind == "func" { + if builtinDefs[r.Name.C] != "" { + error_(r.Pos(), "use of builtin '%s' not in function call", fixGo(r.Name.C)) + } + // Function is being used in an expression, to e.g. pass around a C function pointer. // Create a new Name for this Ref which causes the variable to be declared in Go land. fpName := "fp_" + r.Name.Go @@ -1277,12 +1336,55 @@ func (p *Package) gccCmd() []string { // gccDebug runs gcc -gdwarf-2 over the C program stdin and // returns the corresponding DWARF data and, if present, debug data block. -func (p *Package) gccDebug(stdin []byte) (*dwarf.Data, binary.ByteOrder, []byte) { +func (p *Package) gccDebug(stdin []byte, nnames int) (d *dwarf.Data, ints []int64, floats []float64, strs []string) { runGcc(stdin, p.gccCmd()) - isDebugData := func(s string) bool { + isDebugInts := func(s string) bool { + // Some systems use leading _ to denote non-assembly symbols. + return s == "__cgodebug_ints" || s == "___cgodebug_ints" + } + isDebugFloats := func(s string) bool { + // Some systems use leading _ to denote non-assembly symbols. + return s == "__cgodebug_floats" || s == "___cgodebug_floats" + } + indexOfDebugStr := func(s string) int { + // Some systems use leading _ to denote non-assembly symbols. + if strings.HasPrefix(s, "___") { + s = s[1:] + } + if strings.HasPrefix(s, "__cgodebug_str__") { + if n, err := strconv.Atoi(s[len("__cgodebug_str__"):]); err == nil { + return n + } + } + return -1 + } + indexOfDebugStrlen := func(s string) int { // Some systems use leading _ to denote non-assembly symbols. - return s == "__cgodebug_data" || s == "___cgodebug_data" + if strings.HasPrefix(s, "___") { + s = s[1:] + } + if strings.HasPrefix(s, "__cgodebug_strlen__") { + if n, err := strconv.Atoi(s[len("__cgodebug_strlen__"):]); err == nil { + return n + } + } + return -1 + } + + strs = make([]string, nnames) + + strdata := make(map[int]string, nnames) + strlens := make(map[int]int, nnames) + + buildStrings := func() { + for n, strlen := range strlens { + data := strdata[n] + if len(data) <= strlen { + fatalf("invalid string literal") + } + strs[n] = string(data[:strlen]) + } } if f, err := macho.Open(gccTmp()); err == nil { @@ -1291,24 +1393,76 @@ func (p *Package) gccDebug(stdin []byte) (*dwarf.Data, binary.ByteOrder, []byte) if err != nil { fatalf("cannot load DWARF output from %s: %v", gccTmp(), err) } - var data []byte + bo := f.ByteOrder if f.Symtab != nil { for i := range f.Symtab.Syms { s := &f.Symtab.Syms[i] - if isDebugData(s.Name) { + switch { + case isDebugInts(s.Name): // Found it. Now find data section. if i := int(s.Sect) - 1; 0 <= i && i < len(f.Sections) { sect := f.Sections[i] if sect.Addr <= s.Value && s.Value < sect.Addr+sect.Size { if sdat, err := sect.Data(); err == nil { - data = sdat[s.Value-sect.Addr:] + data := sdat[s.Value-sect.Addr:] + ints = make([]int64, len(data)/8) + for i := range ints { + ints[i] = int64(bo.Uint64(data[i*8:])) + } } } } + case isDebugFloats(s.Name): + // Found it. Now find data section. + if i := int(s.Sect) - 1; 0 <= i && i < len(f.Sections) { + sect := f.Sections[i] + if sect.Addr <= s.Value && s.Value < sect.Addr+sect.Size { + if sdat, err := sect.Data(); err == nil { + data := sdat[s.Value-sect.Addr:] + floats = make([]float64, len(data)/8) + for i := range floats { + floats[i] = math.Float64frombits(bo.Uint64(data[i*8:])) + } + } + } + } + default: + if n := indexOfDebugStr(s.Name); n != -1 { + // Found it. Now find data section. + if i := int(s.Sect) - 1; 0 <= i && i < len(f.Sections) { + sect := f.Sections[i] + if sect.Addr <= s.Value && s.Value < sect.Addr+sect.Size { + if sdat, err := sect.Data(); err == nil { + data := sdat[s.Value-sect.Addr:] + strdata[n] = string(data) + } + } + } + break + } + if n := indexOfDebugStrlen(s.Name); n != -1 { + // Found it. Now find data section. + if i := int(s.Sect) - 1; 0 <= i && i < len(f.Sections) { + sect := f.Sections[i] + if sect.Addr <= s.Value && s.Value < sect.Addr+sect.Size { + if sdat, err := sect.Data(); err == nil { + data := sdat[s.Value-sect.Addr:] + strlen := bo.Uint64(data[:8]) + if strlen > (1<<(uint(p.IntSize*8)-1) - 1) { // greater than MaxInt? + fatalf("string literal too big") + } + strlens[n] = int(strlen) + } + } + } + break + } } } + + buildStrings() } - return d, f.ByteOrder, data + return d, ints, floats, strs } if f, err := elf.Open(gccTmp()); err == nil { @@ -1317,25 +1471,77 @@ func (p *Package) gccDebug(stdin []byte) (*dwarf.Data, binary.ByteOrder, []byte) if err != nil { fatalf("cannot load DWARF output from %s: %v", gccTmp(), err) } - var data []byte + bo := f.ByteOrder symtab, err := f.Symbols() if err == nil { for i := range symtab { s := &symtab[i] - if isDebugData(s.Name) { + switch { + case isDebugInts(s.Name): + // Found it. Now find data section. + if i := int(s.Section); 0 <= i && i < len(f.Sections) { + sect := f.Sections[i] + if sect.Addr <= s.Value && s.Value < sect.Addr+sect.Size { + if sdat, err := sect.Data(); err == nil { + data := sdat[s.Value-sect.Addr:] + ints = make([]int64, len(data)/8) + for i := range ints { + ints[i] = int64(bo.Uint64(data[i*8:])) + } + } + } + } + case isDebugFloats(s.Name): // Found it. Now find data section. if i := int(s.Section); 0 <= i && i < len(f.Sections) { sect := f.Sections[i] if sect.Addr <= s.Value && s.Value < sect.Addr+sect.Size { if sdat, err := sect.Data(); err == nil { - data = sdat[s.Value-sect.Addr:] + data := sdat[s.Value-sect.Addr:] + floats = make([]float64, len(data)/8) + for i := range floats { + floats[i] = math.Float64frombits(bo.Uint64(data[i*8:])) + } + } + } + } + default: + if n := indexOfDebugStr(s.Name); n != -1 { + // Found it. Now find data section. + if i := int(s.Section); 0 <= i && i < len(f.Sections) { + sect := f.Sections[i] + if sect.Addr <= s.Value && s.Value < sect.Addr+sect.Size { + if sdat, err := sect.Data(); err == nil { + data := sdat[s.Value-sect.Addr:] + strdata[n] = string(data) + } } } + break + } + if n := indexOfDebugStrlen(s.Name); n != -1 { + // Found it. Now find data section. + if i := int(s.Section); 0 <= i && i < len(f.Sections) { + sect := f.Sections[i] + if sect.Addr <= s.Value && s.Value < sect.Addr+sect.Size { + if sdat, err := sect.Data(); err == nil { + data := sdat[s.Value-sect.Addr:] + strlen := bo.Uint64(data[:8]) + if strlen > (1<<(uint(p.IntSize*8)-1) - 1) { // greater than MaxInt? + fatalf("string literal too big") + } + strlens[n] = int(strlen) + } + } + } + break } } } + + buildStrings() } - return d, f.ByteOrder, data + return d, ints, floats, strs } if f, err := pe.Open(gccTmp()); err == nil { @@ -1344,20 +1550,70 @@ func (p *Package) gccDebug(stdin []byte) (*dwarf.Data, binary.ByteOrder, []byte) if err != nil { fatalf("cannot load DWARF output from %s: %v", gccTmp(), err) } - var data []byte + bo := binary.LittleEndian for _, s := range f.Symbols { - if isDebugData(s.Name) { + switch { + case isDebugInts(s.Name): if i := int(s.SectionNumber) - 1; 0 <= i && i < len(f.Sections) { sect := f.Sections[i] if s.Value < sect.Size { if sdat, err := sect.Data(); err == nil { - data = sdat[s.Value:] + data := sdat[s.Value:] + ints = make([]int64, len(data)/8) + for i := range ints { + ints[i] = int64(bo.Uint64(data[i*8:])) + } + } + } + } + case isDebugFloats(s.Name): + if i := int(s.SectionNumber) - 1; 0 <= i && i < len(f.Sections) { + sect := f.Sections[i] + if s.Value < sect.Size { + if sdat, err := sect.Data(); err == nil { + data := sdat[s.Value:] + floats = make([]float64, len(data)/8) + for i := range floats { + floats[i] = math.Float64frombits(bo.Uint64(data[i*8:])) + } } } } + default: + if n := indexOfDebugStr(s.Name); n != -1 { + if i := int(s.SectionNumber) - 1; 0 <= i && i < len(f.Sections) { + sect := f.Sections[i] + if s.Value < sect.Size { + if sdat, err := sect.Data(); err == nil { + data := sdat[s.Value:] + strdata[n] = string(data) + } + } + } + break + } + if n := indexOfDebugStrlen(s.Name); n != -1 { + if i := int(s.SectionNumber) - 1; 0 <= i && i < len(f.Sections) { + sect := f.Sections[i] + if s.Value < sect.Size { + if sdat, err := sect.Data(); err == nil { + data := sdat[s.Value:] + strlen := bo.Uint64(data[:8]) + if strlen > (1<<(uint(p.IntSize*8)-1) - 1) { // greater than MaxInt? + fatalf("string literal too big") + } + strlens[n] = int(strlen) + } + } + } + break + } } } - return d, binary.LittleEndian, data + + buildStrings() + + return d, ints, floats, strs } fatalf("cannot parse gcc output %s as ELF, Mach-O, PE object", gccTmp()) @@ -2048,7 +2304,7 @@ func (c *typeConv) FuncType(dtype *dwarf.FuncType, pos token.Pos) *FuncType { } var r *Type var gr []*ast.Field - if _, ok := dtype.ReturnType.(*dwarf.VoidType); ok { + if _, ok := base(dtype.ReturnType).(*dwarf.VoidType); ok { gr = []*ast.Field{{Type: c.goVoid}} } else if dtype.ReturnType != nil { r = c.Type(unqual(dtype.ReturnType), pos) diff --git a/libgo/go/cmd/cgo/main.go b/libgo/go/cmd/cgo/main.go index ac102050dad..c9a44fdd166 100644 --- a/libgo/go/cmd/cgo/main.go +++ b/libgo/go/cmd/cgo/main.go @@ -17,7 +17,7 @@ import ( "go/ast" "go/printer" "go/token" - "io" + "io/ioutil" "os" "path/filepath" "reflect" @@ -88,7 +88,7 @@ type Name struct { Mangle string // name used in generated Go C string // name used in C Define string // #define expansion - Kind string // "const", "type", "var", "fpvar", "func", "not-type" + Kind string // "iconst", "uconst", "fconst", "sconst", "type", "var", "fpvar", "func", "not-type" Type *Type // the type of xxx FuncType *FuncType AddError bool @@ -100,6 +100,11 @@ func (n *Name) IsVar() bool { return n.Kind == "var" || n.Kind == "fpvar" } +// IsConst reports whether Kind is either "iconst", "uconst", "fconst" or "sconst" +func (n *Name) IsConst() bool { + return strings.HasSuffix(n.Kind, "const") +} + // A ExpFunc is an exported function, callable from C. // Such functions are identified in the Go input file // by doc comments containing the line //export ExpName @@ -274,30 +279,28 @@ func main() { // concern is other cgo wrappers for the same functions. // Use the beginning of the md5 of the input to disambiguate. h := md5.New() - for _, input := range goFiles { + fs := make([]*File, len(goFiles)) + for i, input := range goFiles { if *srcDir != "" { input = filepath.Join(*srcDir, input) } - f, err := os.Open(input) + + b, err := ioutil.ReadFile(input) if err != nil { fatalf("%s", err) } - io.Copy(h, f) - f.Close() - } - cPrefix = fmt.Sprintf("_%x", h.Sum(nil)[0:6]) - - fs := make([]*File, len(goFiles)) - for i, input := range goFiles { - if *srcDir != "" { - input = filepath.Join(*srcDir, input) + if _, err = h.Write(b); err != nil { + fatalf("%s", err) } + f := new(File) - f.ReadGo(input) + f.ParseGo(input, b) f.DiscardCgoDirectives() fs[i] = f } + cPrefix = fmt.Sprintf("_%x", h.Sum(nil)[0:6]) + if *objDir == "" { // make sure that _obj directory exists, so that we can write // all the output files there. diff --git a/libgo/go/cmd/cgo/out.go b/libgo/go/cmd/cgo/out.go index a8292f22d3f..c4ff07d2668 100644 --- a/libgo/go/cmd/cgo/out.go +++ b/libgo/go/cmd/cgo/out.go @@ -190,7 +190,7 @@ func (p *Package) writeDefs() { for _, key := range nameKeys(p.Name) { n := p.Name[key] if n.Const != "" { - fmt.Fprintf(fgo2, "const _Cconst_%s = %s\n", n.Go, n.Const) + fmt.Fprintf(fgo2, "const %s = %s\n", n.Mangle, n.Const) } } fmt.Fprintf(fgo2, "\n") @@ -1303,7 +1303,7 @@ const gccProlog = ` */ #define __cgo_compile_assert_eq(x, y, name) typedef char name[(x-y)*(x-y)*-2+1]; -// Check at compile time that the sizes we use match our expectations. +/* Check at compile time that the sizes we use match our expectations. */ #define __cgo_size_assert(t, n) __cgo_compile_assert_eq(sizeof(t), n, _cgo_sizeof_##t##_is_not_##n) __cgo_size_assert(char, 1) @@ -1328,6 +1328,27 @@ const noTsanProlog = ` ` // This must match the TSAN code in runtime/cgo/libcgo.h. +// This is used when the code is built with the C/C++ Thread SANitizer, +// which is not the same as the Go race detector. +// __tsan_acquire tells TSAN that we are acquiring a lock on a variable, +// in this case _cgo_sync. __tsan_release releases the lock. +// (There is no actual lock, we are just telling TSAN that there is.) +// +// When we call from Go to C we call _cgo_tsan_acquire. +// When the C function returns we call _cgo_tsan_release. +// Similarly, when C calls back into Go we call _cgo_tsan_release +// and then call _cgo_tsan_acquire when we return to C. +// These calls tell TSAN that there is a serialization point at the C call. +// +// This is necessary because TSAN, which is a C/C++ tool, can not see +// the synchronization in the Go code. Without these calls, when +// multiple goroutines call into C code, TSAN does not understand +// that the calls are properly synchronized on the Go side. +// +// To be clear, if the calls are not properly synchronized on the Go side, +// we will be hiding races. But when using TSAN on mixed Go C/C++ code +// it is more important to avoid false positives, which reduce confidence +// in the tool, than to avoid false negatives. const yesTsanProlog = ` #line 1 "cgo-tsan-prolog" #define CGO_NO_SANITIZE_THREAD __attribute__ ((no_sanitize_thread)) diff --git a/libgo/go/cmd/go/alldocs.go b/libgo/go/cmd/go/alldocs.go index f946bf6105f..6f9dc326dc5 100644 --- a/libgo/go/cmd/go/alldocs.go +++ b/libgo/go/cmd/go/alldocs.go @@ -118,8 +118,8 @@ // a suffix to use in the name of the package installation directory, // in order to keep output separate from default builds. // If using the -race flag, the install suffix is automatically set to race -// or, if set explicitly, has _race appended to it. Likewise for the -msan -// flag. Using a -buildmode option that requires non-default compile flags +// or, if set explicitly, has _race appended to it. Likewise for the -msan +// flag. Using a -buildmode option that requires non-default compile flags // has a similar effect. // -ldflags 'flag list' // arguments to pass on each go tool link invocation. @@ -131,16 +131,17 @@ // For example, when building with a non-standard configuration, // use -pkgdir to keep generated packages in a separate location. // -tags 'tag list' -// a list of build tags to consider satisfied during the build. -// For more information about build tags, see the description of +// a space-separated list of build tags to consider satisfied during the +// build. For more information about build tags, see the description of // build constraints in the documentation for the go/build package. // -toolexec 'cmd args' // a program to use to invoke toolchain programs like vet and asm. // For example, instead of running asm, the go command will run // 'cmd args /path/to/asm '. // -// The list flags accept a space-separated list of strings. To embed spaces -// in an element in the list, surround it with either single or double quotes. +// All the flags that take a list of arguments accept a space-separated +// list of strings. To embed spaces in an element in the list, surround +// it with either single or double quotes. // // For more about specifying packages, see 'go help packages'. // For more about where packages and binaries are installed, @@ -208,12 +209,13 @@ // // Usage: // -// go doc [-u] [-c] [package|[package.]symbol[.method]] +// go doc [-u] [-c] [package|[package.]symbol[.methodOrField]] // // Doc prints the documentation comments associated with the item identified by its -// arguments (a package, const, func, type, var, or method) followed by a one-line -// summary of each of the first-level items "under" that item (package-level -// declarations for a package, methods for a type, etc.). +// arguments (a package, const, func, type, var, method, or struct field) +// followed by a one-line summary of each of the first-level items "under" +// that item (package-level declarations for a package, methods for a type, +// etc.). // // Doc accepts zero, one, or two arguments. // @@ -231,9 +233,9 @@ // which is schematically one of these: // // go doc -// go doc [.] -// go doc [.][.] -// go doc [.][.] +// go doc [.] +// go doc [.][.] +// go doc [.][.] // // The first item in this list matched by the argument is the one whose documentation // is printed. (See the examples below.) However, if the argument starts with a capital @@ -241,7 +243,7 @@ // // For packages, the order of scanning is determined lexically in breadth-first order. // That is, the package presented is the one that matches the search and is nearest -// the root and lexically first at its level of the hierarchy. The GOROOT tree is +// the root and lexically first at its level of the hierarchy. The GOROOT tree is // always scanned in its entirety before GOPATH. // // If there is no package specified or matched, the package in the current @@ -253,10 +255,10 @@ // elements like . and ... are not implemented by go doc. // // When run with two arguments, the first must be a full package path (not just a -// suffix), and the second is a symbol or symbol and method; this is similar to the -// syntax accepted by godoc: +// suffix), and the second is a symbol, or symbol with method or struct field. +// This is similar to the syntax accepted by godoc: // -// go doc [.] +// go doc [.] // // In all forms, when matching symbols, lower-case letters in the argument match // either case but upper-case letters match exactly. This means that there may be @@ -307,22 +309,25 @@ // when showing the package's top-level documentation. // -u // Show documentation for unexported as well as exported -// symbols and methods. +// symbols, methods, and fields. // // // Print Go environment information // // Usage: // -// go env [var ...] +// go env [-json] [var ...] // // Env prints Go environment information. // // By default env prints information as a shell script -// (on Windows, a batch file). If one or more variable -// names is given as arguments, env prints the value of +// (on Windows, a batch file). If one or more variable +// names is given as arguments, env prints the value of // each named variable on its own line. // +// The -json flag prints the environment in JSON format +// instead of as a shell script. +// // // Start a bug report // @@ -357,7 +362,7 @@ // go fmt [-n] [-x] [packages] // // Fmt runs the command 'gofmt -l -w' on the packages named -// by the import paths. It prints the names of the files that are modified. +// by the import paths. It prints the names of the files that are modified. // // For more about gofmt, see 'go doc cmd/gofmt'. // For more about specifying packages, see 'go help packages'. @@ -427,7 +432,7 @@ // As a last step before running the command, any invocations of any // environment variables with alphanumeric names, such as $GOFILE or // $HOME, are expanded throughout the command line. The syntax for -// variable expansion is $NAME on all operating systems. Due to the +// variable expansion is $NAME on all operating systems. Due to the // order of evaluation, variables are expanded even inside quoted // strings. If the variable NAME is not set, $NAME expands to the // empty string. @@ -504,7 +509,7 @@ // the tests for the specified packages. // // The -u flag instructs get to use the network to update the named packages -// and their dependencies. By default, get uses the network to check out +// and their dependencies. By default, get uses the network to check out // missing packages but does not use it to look for updates to existing packages. // // The -v flag enables verbose progress and debug output. @@ -518,8 +523,8 @@ // When checking out or updating a package, get looks for a branch or tag // that matches the locally installed version of Go. The most important // rule is that if the local installation is running version "go1", get -// searches for a branch or tag named "go1". If no such version exists it -// retrieves the most recent version of the package. +// searches for a branch or tag named "go1". If no such version exists +// it retrieves the default branch of the package. // // When go get checks out or updates a Git repository, // it also updates any git submodules referenced by the repository. @@ -565,7 +570,7 @@ // golang.org/x/net/html // // The -f flag specifies an alternate format for the list, using the -// syntax of package template. The default output is equivalent to -f +// syntax of package template. The default output is equivalent to -f // '{{.ImportPath}}'. The struct being passed to the template is: // // type Package struct { @@ -658,12 +663,12 @@ // instead of using the template format. // // The -e flag changes the handling of erroneous packages, those that -// cannot be found or are malformed. By default, the list command +// cannot be found or are malformed. By default, the list command // prints an error to standard error for each erroneous package and // omits the packages from consideration during the usual printing. // With the -e flag, the list command never prints errors to standard // error and instead processes the erroneous packages with the usual -// printing. Erroneous packages will have a non-empty ImportPath and +// printing. Erroneous packages will have a non-empty ImportPath and // a non-nil Error field; other information may or may not be missing // (zeroed). // @@ -716,7 +721,7 @@ // the file pattern "*_test.go". // Files whose names begin with "_" (including "_test.go") or "." are ignored. // These additional files can contain test functions, benchmark functions, and -// example functions. See 'go help testfunc' for more. +// example functions. See 'go help testfunc' for more. // Each listed package causes the execution of a separate test binary. // // Test files that declare a package with the suffix "_test" will be compiled as a @@ -725,7 +730,7 @@ // The go tool will ignore a directory named "testdata", making it available // to hold ancillary data needed by the tests. // -// By default, go test needs no arguments. It compiles and tests the package +// By default, go test needs no arguments. It compiles and tests the package // with source in the current directory, including tests, and runs the tests. // // The package is built in a temporary directory so it does not interfere with the @@ -793,15 +798,13 @@ // // Usage: // -// go vet [-n] [-x] [build flags] [packages] +// go vet [-n] [-x] [build flags] [vet flags] [packages] // // Vet runs the Go vet command on the packages named by the import paths. // -// For more about vet, see 'go doc cmd/vet'. +// For more about vet and its flags, see 'go doc cmd/vet'. // For more about specifying packages, see 'go help packages'. // -// To run the vet tool with specific options, run 'go tool vet'. -// // The -n flag prints commands that would be executed. // The -x flag prints commands as they are executed. // @@ -814,18 +817,18 @@ // // There are two different ways to call between Go and C/C++ code. // -// The first is the cgo tool, which is part of the Go distribution. For +// The first is the cgo tool, which is part of the Go distribution. For // information on how to use it see the cgo documentation (go doc cmd/cgo). // // The second is the SWIG program, which is a general tool for -// interfacing between languages. For information on SWIG see -// http://swig.org/. When running go build, any file with a .swig -// extension will be passed to SWIG. Any file with a .swigcxx extension +// interfacing between languages. For information on SWIG see +// http://swig.org/. When running go build, any file with a .swig +// extension will be passed to SWIG. Any file with a .swigcxx extension // will be passed to SWIG with the -c++ option. // // When either cgo or SWIG is used, go build will pass any .c, .m, .s, // or .S files to the C compiler, and any .cc, .cpp, .cxx files to the C++ -// compiler. The CC or CXX environment variables may be set to determine +// compiler. The CC or CXX environment variables may be set to determine // the C or C++ compiler, respectively, to use. // // @@ -846,10 +849,10 @@ // exactly one main package to be listed. // // -buildmode=c-shared -// Build the listed main packages, plus all packages that they -// import, into C shared libraries. The only callable symbols will +// Build the listed main package, plus all packages it imports, +// into a C shared library. The only callable symbols will // be those functions exported using a cgo //export comment. -// Non-main packages are ignored. +// Requires exactly one main package to be listed. // // -buildmode=default // Listed main packages are built into executables and listed @@ -938,7 +941,7 @@ // // Each directory listed in GOPATH must have a prescribed structure: // -// The src directory holds source code. The path below src +// The src directory holds source code. The path below src // determines the import path or executable name. // // The pkg directory holds installed package objects. @@ -952,11 +955,11 @@ // // The bin directory holds compiled commands. // Each command is named for its source directory, but only -// the final element, not the entire path. That is, the +// the final element, not the entire path. That is, the // command with source in DIR/src/foo/quux is installed into -// DIR/bin/quux, not DIR/bin/foo/quux. The "foo/" prefix is stripped +// DIR/bin/quux, not DIR/bin/foo/quux. The "foo/" prefix is stripped // so that you can add DIR/bin to your PATH to get at the -// installed commands. If the GOBIN environment variable is +// installed commands. If the GOBIN environment variable is // set, commands are installed to the directory it names instead // of DIR/bin. GOBIN must be an absolute path. // @@ -1099,7 +1102,7 @@ // CC // The command to use to compile C code. // CGO_ENABLED -// Whether the cgo command is supported. Either 0 or 1. +// Whether the cgo command is supported. Either 0 or 1. // CGO_CFLAGS // Flags that cgo will pass to the compiler when compiling // C code. @@ -1151,7 +1154,7 @@ // Import path syntax // // An import path (see 'go help packages') denotes a package stored in the local -// file system. In general, an import path denotes either a standard package (such +// file system. In general, an import path denotes either a standard package (such // as "unicode/utf8") or a package found in one of the work spaces (For more // details see: 'go help gopath'). // @@ -1222,7 +1225,7 @@ // // specifies the given repository, with or without the .vcs suffix, // using the named version control system, and then the path inside -// that repository. The supported version control systems are: +// that repository. The supported version control systems are: // // Bazaar .bzr // Git .git @@ -1242,7 +1245,7 @@ // example.org/repo or repo.git. // // When a version control system supports multiple protocols, -// each is tried in turn when downloading. For example, a Git +// each is tried in turn when downloading. For example, a Git // download tries https://, then git+ssh://. // // By default, downloads are restricted to known secure protocols @@ -1360,17 +1363,28 @@ // // An import path is a pattern if it includes one or more "..." wildcards, // each of which can match any string, including the empty string and -// strings containing slashes. Such a pattern expands to all package +// strings containing slashes. Such a pattern expands to all package // directories found in the GOPATH trees with names matching the -// patterns. As a special case, x/... matches x as well as x's subdirectories. -// For example, net/... expands to net and packages in its subdirectories. +// patterns. +// +// To make common patterns more convenient, there are two special cases. +// First, /... at the end of the pattern can match an empty string, +// so that net/... matches both net and packages in its subdirectories, like net/http. +// Second, any slash-separated pattern element containing a wildcard never +// participates in a match of the "vendor" element in the path of a vendored +// package, so that ./... does not match packages in subdirectories of +// ./vendor or ./mycode/vendor, but ./vendor/... and ./mycode/vendor/... do. +// Note, however, that a directory named vendor that itself contains code +// is not a vendored package: cmd/vendor would be a command named vendor, +// and the pattern cmd/... matches it. +// See golang.org/s/go15vendor for more about vendoring. // // An import path can also name a package to be downloaded from -// a remote repository. Run 'go help importpath' for details. +// a remote repository. Run 'go help importpath' for details. // // Every package in a program must have a unique import path. // By convention, this is arranged by starting each path with a -// unique prefix that belongs to you. For example, paths used +// unique prefix that belongs to you. For example, paths used // internally at Google all begin with 'google', and paths // denoting remote repositories begin with the path to the code, // such as 'github.com/user/repo'. @@ -1399,19 +1413,24 @@ // // Several of the flags control profiling and write an execution profile // suitable for "go tool pprof"; run "go tool pprof -h" for more -// information. The --alloc_space, --alloc_objects, and --show_bytes +// information. The --alloc_space, --alloc_objects, and --show_bytes // options of pprof control how the information is presented. // // The following flags are recognized by the 'go test' command and // control the execution of any test: // // -bench regexp -// Run (sub)benchmarks matching a regular expression. -// The given regular expression is split into smaller ones by -// top-level '/', where each must match the corresponding part of a -// benchmark's identifier. -// By default, no benchmarks run. To run all benchmarks, -// use '-bench .' or '-bench=.'. +// Run only those benchmarks matching a regular expression. +// By default, no benchmarks are run. +// To run all benchmarks, use '-bench .' or '-bench=.'. +// The regular expression is split by unbracketed slash (/) +// characters into a sequence of regular expressions, and each +// part of a benchmark's identifier must match the corresponding +// element in the sequence, if any. Possible parents of matches +// are run with b.N=1 to identify sub-benchmarks. For example, +// given -bench=X/Y, top-level benchmarks matching X are run +// with b.N=1 to find any sub-benchmarks matching Y, which are +// then run in full. // // -benchtime t // Run enough iterations of each benchmark to take t, specified @@ -1425,6 +1444,10 @@ // // -cover // Enable coverage analysis. +// Note that because coverage works by annotating the source +// code before compilation, compilation and test failures with +// coverage enabled may report line numbers that don't correspond +// to the original sources. // // -covermode set,count,atomic // Set the mode for coverage analysis for the package[s] @@ -1445,9 +1468,14 @@ // // -cpu 1,2,4 // Specify a list of GOMAXPROCS values for which the tests or -// benchmarks should be executed. The default is the current value +// benchmarks should be executed. The default is the current value // of GOMAXPROCS. // +// -list regexp +// List tests, benchmarks, or examples matching the regular expression. +// No tests, benchmarks or examples will be run. This will only +// list top-level tests. No subtest or subbenchmarks will be shown. +// // -parallel n // Allow parallel execution of test functions that call t.Parallel. // The value of this flag is the maximum number of tests to run @@ -1459,9 +1487,13 @@ // // -run regexp // Run only those tests and examples matching the regular expression. -// For tests the regular expression is split into smaller ones by -// top-level '/', where each must match the corresponding part of a -// test's identifier. +// For tests, the regular expression is split by unbracketed slash (/) +// characters into a sequence of regular expressions, and each part +// of a test's identifier must match the corresponding element in +// the sequence, if any. Note that possible parents of matches are +// run too, so that -run=X/Y matches and runs and reports the result +// of all tests matching X, even those without sub-tests matching Y, +// because it must run them to look for those sub-tests. // // -short // Tell long-running tests to shorten their run time. @@ -1469,8 +1501,8 @@ // the Go tree can run a sanity check but not spend time running // exhaustive tests. // -// -timeout t -// If a test runs longer than t, panic. +// -timeout d +// If a test binary runs longer than duration d, panic. // The default is 10 minutes (10m). // // -v @@ -1493,7 +1525,7 @@ // calling runtime.SetBlockProfileRate with n. // See 'go doc runtime.SetBlockProfileRate'. // The profiler aims to sample, on average, one blocking event every -// n nanoseconds the program spends blocked. By default, +// n nanoseconds the program spends blocked. By default, // if -test.blockprofile is set without this flag, all blocking events // are recorded, equivalent to -test.blockprofilerate=1. // @@ -1511,7 +1543,7 @@ // // -memprofilerate n // Enable more precise (and expensive) memory profiles by setting -// runtime.MemProfileRate. See 'go doc runtime.MemProfileRate'. +// runtime.MemProfileRate. See 'go doc runtime.MemProfileRate'. // To profile all memory allocations, use -test.memprofilerate=1 // and pass --alloc_space flag to the pprof tool. // @@ -1614,8 +1646,8 @@ // "Output:" is compiled, executed, and expected to produce no output. // // Godoc displays the body of ExampleXXX to demonstrate the use -// of the function, constant, or variable XXX. An example of a method M with -// receiver type T or *T is named ExampleT_M. There may be multiple examples +// of the function, constant, or variable XXX. An example of a method M with +// receiver type T or *T is named ExampleT_M. There may be multiple examples // for a given function, constant, or variable, distinguished by a trailing _xxx, // where xxx is a suffix not beginning with an upper case letter. // diff --git a/libgo/go/cmd/go/bootstrap.go b/libgo/go/cmd/go/bootstrap.go deleted file mode 100644 index 2148d12685a..00000000000 --- a/libgo/go/cmd/go/bootstrap.go +++ /dev/null @@ -1,41 +0,0 @@ -// Copyright 2012 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// +build cmd_go_bootstrap - -// This code is compiled only into the bootstrap 'go' binary. -// These stubs avoid importing packages with large dependency -// trees, like the use of "net/http" in vcs.go. - -package main - -import ( - "errors" - "io" -) - -var errHTTP = errors.New("no http in bootstrap go command") - -type httpError struct { - statusCode int -} - -func (e *httpError) Error() string { - panic("unreachable") -} - -func httpGET(url string) ([]byte, error) { - return nil, errHTTP -} - -func httpsOrHTTP(importPath string, security securityMode) (string, io.ReadCloser, error) { - return "", nil, errHTTP -} - -func parseMetaGoImports(r io.Reader) ([]metaImport, error) { - panic("unreachable") -} - -func queryEscape(s string) string { panic("unreachable") } -func openBrowser(url string) bool { panic("unreachable") } diff --git a/libgo/go/cmd/go/bug.go b/libgo/go/cmd/go/bug.go deleted file mode 100644 index 658f6dabd90..00000000000 --- a/libgo/go/cmd/go/bug.go +++ /dev/null @@ -1,212 +0,0 @@ -// Copyright 2016 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package main - -import ( - "bytes" - "fmt" - "io" - "io/ioutil" - "os" - "os/exec" - "path/filepath" - "regexp" - "runtime" - "strings" -) - -var cmdBug = &Command{ - Run: runBug, - UsageLine: "bug", - Short: "start a bug report", - Long: ` -Bug opens the default browser and starts a new bug report. -The report includes useful system information. - `, -} - -func init() { - cmdBug.Flag.BoolVar(&buildV, "v", false, "") -} - -func runBug(cmd *Command, args []string) { - var buf bytes.Buffer - buf.WriteString(bugHeader) - inspectGoVersion(&buf) - fmt.Fprint(&buf, "#### System details\n\n") - fmt.Fprintln(&buf, "```") - fmt.Fprintf(&buf, "go version %s %s/%s\n", runtime.Version(), runtime.GOOS, runtime.GOARCH) - env := newEnv - env = append(env, extraEnvVars()...) - for _, e := range env { - // Hide the TERM environment variable from "go bug". - // See issue #18128 - if e.name != "TERM" { - fmt.Fprintf(&buf, "%s=\"%s\"\n", e.name, e.value) - } - } - printGoDetails(&buf) - printOSDetails(&buf) - printCDetails(&buf) - fmt.Fprintln(&buf, "```") - - body := buf.String() - url := "https://github.com/golang/go/issues/new?body=" + queryEscape(body) - if !openBrowser(url) { - fmt.Print("Please file a new issue at golang.org/issue/new using this template:\n\n") - fmt.Print(body) - } -} - -const bugHeader = `Please answer these questions before submitting your issue. Thanks! - -#### What did you do? -If possible, provide a recipe for reproducing the error. -A complete runnable program is good. -A link on play.golang.org is best. - - -#### What did you expect to see? - - -#### What did you see instead? - - -` - -func printGoDetails(w io.Writer) { - printCmdOut(w, "GOROOT/bin/go version: ", filepath.Join(runtime.GOROOT(), "bin/go"), "version") - printCmdOut(w, "GOROOT/bin/go tool compile -V: ", filepath.Join(runtime.GOROOT(), "bin/go"), "tool", "compile", "-V") -} - -func printOSDetails(w io.Writer) { - switch runtime.GOOS { - case "darwin": - printCmdOut(w, "uname -v: ", "uname", "-v") - printCmdOut(w, "", "sw_vers") - case "linux": - printCmdOut(w, "uname -sr: ", "uname", "-sr") - printCmdOut(w, "", "lsb_release", "-a") - printGlibcVersion(w) - case "openbsd", "netbsd", "freebsd", "dragonfly": - printCmdOut(w, "uname -v: ", "uname", "-v") - case "solaris": - out, err := ioutil.ReadFile("/etc/release") - if err == nil { - fmt.Fprintf(w, "/etc/release: %s\n", out) - } else { - if buildV { - fmt.Printf("failed to read /etc/release: %v\n", err) - } - } - } -} - -func printCDetails(w io.Writer) { - printCmdOut(w, "lldb --version: ", "lldb", "--version") - cmd := exec.Command("gdb", "--version") - out, err := cmd.Output() - if err == nil { - // There's apparently no combination of command line flags - // to get gdb to spit out its version without the license and warranty. - // Print up to the first newline. - fmt.Fprintf(w, "gdb --version: %s\n", firstLine(out)) - } else { - if buildV { - fmt.Printf("failed to run gdb --version: %v\n", err) - } - } -} - -func inspectGoVersion(w io.Writer) { - data, err := httpGET("https://golang.org/VERSION?m=text") - if err != nil { - if buildV { - fmt.Printf("failed to read from golang.org/VERSION: %v\n", err) - } - return - } - - // golang.org/VERSION currently returns a whitespace-free string, - // but just in case, protect against that changing. - // Similarly so for runtime.Version. - release := string(bytes.TrimSpace(data)) - vers := strings.TrimSpace(runtime.Version()) - - if vers == release { - // Up to date - return - } - - // Devel version or outdated release. Either way, this request is apropos. - fmt.Fprintf(w, "#### Does this issue reproduce with the latest release (%s)?\n\n\n", release) -} - -// printCmdOut prints the output of running the given command. -// It ignores failures; 'go bug' is best effort. -func printCmdOut(w io.Writer, prefix, path string, args ...string) { - cmd := exec.Command(path, args...) - out, err := cmd.Output() - if err != nil { - if buildV { - fmt.Printf("%s %s: %v\n", path, strings.Join(args, " "), err) - } - return - } - fmt.Fprintf(w, "%s%s\n", prefix, bytes.TrimSpace(out)) -} - -// firstLine returns the first line of a given byte slice. -func firstLine(buf []byte) []byte { - idx := bytes.IndexByte(buf, '\n') - if idx > 0 { - buf = buf[:idx] - } - return bytes.TrimSpace(buf) -} - -// printGlibcVersion prints information about the glibc version. -// It ignores failures. -func printGlibcVersion(w io.Writer) { - tempdir := os.TempDir() - if tempdir == "" { - return - } - src := []byte(`int main() {}`) - srcfile := filepath.Join(tempdir, "go-bug.c") - outfile := filepath.Join(tempdir, "go-bug") - err := ioutil.WriteFile(srcfile, src, 0644) - if err != nil { - return - } - defer os.Remove(srcfile) - cmd := exec.Command("gcc", "-o", outfile, srcfile) - if _, err = cmd.CombinedOutput(); err != nil { - return - } - defer os.Remove(outfile) - - cmd = exec.Command("ldd", outfile) - out, err := cmd.CombinedOutput() - if err != nil { - return - } - re := regexp.MustCompile(`libc\.so[^ ]* => ([^ ]+)`) - m := re.FindStringSubmatch(string(out)) - if m == nil { - return - } - cmd = exec.Command(m[1]) - out, err = cmd.Output() - if err != nil { - return - } - fmt.Fprintf(w, "%s: %s\n", m[1], firstLine(out)) - - // print another line (the one containing version string) in case of musl libc - if idx := bytes.IndexByte(out, '\n'); bytes.Index(out, []byte("musl")) != -1 && idx > -1 { - fmt.Fprintf(w, "%s\n", firstLine(out[idx+1:])) - } -} diff --git a/libgo/go/cmd/go/build.go b/libgo/go/cmd/go/build.go deleted file mode 100644 index a62bc86c262..00000000000 --- a/libgo/go/cmd/go/build.go +++ /dev/null @@ -1,3919 +0,0 @@ -// Copyright 2011 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package main - -import ( - "bufio" - "bytes" - "container/heap" - "debug/elf" - "errors" - "flag" - "fmt" - "go/build" - "io" - "io/ioutil" - "log" - "os" - "os/exec" - "path" - "path/filepath" - "regexp" - "runtime" - "strconv" - "strings" - "sync" - "time" -) - -var cmdBuild = &Command{ - UsageLine: "build [-o output] [-i] [build flags] [packages]", - Short: "compile packages and dependencies", - Long: ` -Build compiles the packages named by the import paths, -along with their dependencies, but it does not install the results. - -If the arguments to build are a list of .go files, build treats -them as a list of source files specifying a single package. - -When compiling a single main package, build writes -the resulting executable to an output file named after -the first source file ('go build ed.go rx.go' writes 'ed' or 'ed.exe') -or the source code directory ('go build unix/sam' writes 'sam' or 'sam.exe'). -The '.exe' suffix is added when writing a Windows executable. - -When compiling multiple packages or a single non-main package, -build compiles the packages but discards the resulting object, -serving only as a check that the packages can be built. - -When compiling packages, build ignores files that end in '_test.go'. - -The -o flag, only allowed when compiling a single package, -forces build to write the resulting executable or object -to the named output file, instead of the default behavior described -in the last two paragraphs. - -The -i flag installs the packages that are dependencies of the target. - -The build flags are shared by the build, clean, get, install, list, run, -and test commands: - - -a - force rebuilding of packages that are already up-to-date. - -n - print the commands but do not run them. - -p n - the number of programs, such as build commands or - test binaries, that can be run in parallel. - The default is the number of CPUs available. - -race - enable data race detection. - Supported only on linux/amd64, freebsd/amd64, darwin/amd64 and windows/amd64. - -msan - enable interoperation with memory sanitizer. - Supported only on linux/amd64, - and only with Clang/LLVM as the host C compiler. - -v - print the names of packages as they are compiled. - -work - print the name of the temporary work directory and - do not delete it when exiting. - -x - print the commands. - - -asmflags 'flag list' - arguments to pass on each go tool asm invocation. - -buildmode mode - build mode to use. See 'go help buildmode' for more. - -compiler name - name of compiler to use, as in runtime.Compiler (gccgo or gc). - -gccgoflags 'arg list' - arguments to pass on each gccgo compiler/linker invocation. - -gcflags 'arg list' - arguments to pass on each go tool compile invocation. - -installsuffix suffix - a suffix to use in the name of the package installation directory, - in order to keep output separate from default builds. - If using the -race flag, the install suffix is automatically set to race - or, if set explicitly, has _race appended to it. Likewise for the -msan - flag. Using a -buildmode option that requires non-default compile flags - has a similar effect. - -ldflags 'flag list' - arguments to pass on each go tool link invocation. - -linkshared - link against shared libraries previously created with - -buildmode=shared. - -pkgdir dir - install and load all packages from dir instead of the usual locations. - For example, when building with a non-standard configuration, - use -pkgdir to keep generated packages in a separate location. - -tags 'tag list' - a list of build tags to consider satisfied during the build. - For more information about build tags, see the description of - build constraints in the documentation for the go/build package. - -toolexec 'cmd args' - a program to use to invoke toolchain programs like vet and asm. - For example, instead of running asm, the go command will run - 'cmd args /path/to/asm '. - -The list flags accept a space-separated list of strings. To embed spaces -in an element in the list, surround it with either single or double quotes. - -For more about specifying packages, see 'go help packages'. -For more about where packages and binaries are installed, -run 'go help gopath'. -For more about calling between Go and C/C++, run 'go help c'. - -Note: Build adheres to certain conventions such as those described -by 'go help gopath'. Not all projects can follow these conventions, -however. Installations that have their own conventions or that use -a separate software build system may choose to use lower-level -invocations such as 'go tool compile' and 'go tool link' to avoid -some of the overheads and design decisions of the build tool. - -See also: go install, go get, go clean. - `, -} - -func init() { - // break init cycle - cmdBuild.Run = runBuild - cmdInstall.Run = runInstall - - cmdBuild.Flag.BoolVar(&buildI, "i", false, "") - - addBuildFlags(cmdBuild) - addBuildFlags(cmdInstall) -} - -// Flags set by multiple commands. -var buildA bool // -a flag -var buildN bool // -n flag -var buildP = runtime.NumCPU() // -p flag -var buildV bool // -v flag -var buildX bool // -x flag -var buildI bool // -i flag -var buildO = cmdBuild.Flag.String("o", "", "output file") -var buildWork bool // -work flag -var buildAsmflags []string // -asmflags flag -var buildGcflags []string // -gcflags flag -var buildLdflags []string // -ldflags flag -var buildGccgoflags []string // -gccgoflags flag -var buildRace bool // -race flag -var buildMSan bool // -msan flag -var buildToolExec []string // -toolexec flag -var buildBuildmode string // -buildmode flag -var buildLinkshared bool // -linkshared flag -var buildPkgdir string // -pkgdir flag - -// Require the source for go std packages -var reqStdPkgSrc bool -var buildContext = build.Default -var buildToolchain toolchain = noToolchain{} -var ldBuildmode string - -// buildCompiler implements flag.Var. -// It implements Set by updating both -// buildToolchain and buildContext.Compiler. -type buildCompiler struct{} - -func (c buildCompiler) Set(value string) error { - switch value { - case "gc": - buildToolchain = gcToolchain{} - case "gccgo": - buildToolchain = gccgoToolchain{} - default: - return fmt.Errorf("unknown compiler %q", value) - } - buildContext.Compiler = value - return nil -} - -func (c buildCompiler) String() string { - return buildContext.Compiler -} - -func init() { - switch build.Default.Compiler { - case "gc": - buildToolchain = gcToolchain{} - case "gccgo": - buildToolchain = gccgoToolchain{} - } -} - -// addBuildFlags adds the flags common to the build, clean, get, -// install, list, run, and test commands. -func addBuildFlags(cmd *Command) { - cmd.Flag.BoolVar(&buildA, "a", false, "") - cmd.Flag.BoolVar(&buildN, "n", false, "") - cmd.Flag.IntVar(&buildP, "p", buildP, "") - cmd.Flag.BoolVar(&buildV, "v", false, "") - cmd.Flag.BoolVar(&buildX, "x", false, "") - - cmd.Flag.Var((*stringsFlag)(&buildAsmflags), "asmflags", "") - cmd.Flag.Var(buildCompiler{}, "compiler", "") - cmd.Flag.StringVar(&buildBuildmode, "buildmode", "default", "") - cmd.Flag.Var((*stringsFlag)(&buildGcflags), "gcflags", "") - cmd.Flag.Var((*stringsFlag)(&buildGccgoflags), "gccgoflags", "") - cmd.Flag.StringVar(&buildContext.InstallSuffix, "installsuffix", "", "") - cmd.Flag.Var((*stringsFlag)(&buildLdflags), "ldflags", "") - cmd.Flag.BoolVar(&buildLinkshared, "linkshared", false, "") - cmd.Flag.StringVar(&buildPkgdir, "pkgdir", "", "") - cmd.Flag.BoolVar(&buildRace, "race", false, "") - cmd.Flag.BoolVar(&buildMSan, "msan", false, "") - cmd.Flag.Var((*stringsFlag)(&buildContext.BuildTags), "tags", "") - cmd.Flag.Var((*stringsFlag)(&buildToolExec), "toolexec", "") - cmd.Flag.BoolVar(&buildWork, "work", false, "") - switch build.Default.Compiler { - case "gc": - reqStdPkgSrc = true - case "gccgo": - reqStdPkgSrc = false - } -} - -func addBuildFlagsNX(cmd *Command) { - cmd.Flag.BoolVar(&buildN, "n", false, "") - cmd.Flag.BoolVar(&buildX, "x", false, "") -} - -func isSpaceByte(c byte) bool { - return c == ' ' || c == '\t' || c == '\n' || c == '\r' -} - -// fileExtSplit expects a filename and returns the name -// and ext (without the dot). If the file has no -// extension, ext will be empty. -func fileExtSplit(file string) (name, ext string) { - dotExt := filepath.Ext(file) - name = file[:len(file)-len(dotExt)] - if dotExt != "" { - ext = dotExt[1:] - } - return -} - -type stringsFlag []string - -func (v *stringsFlag) Set(s string) error { - var err error - *v, err = splitQuotedFields(s) - if *v == nil { - *v = []string{} - } - return err -} - -func splitQuotedFields(s string) ([]string, error) { - // Split fields allowing '' or "" around elements. - // Quotes further inside the string do not count. - var f []string - for len(s) > 0 { - for len(s) > 0 && isSpaceByte(s[0]) { - s = s[1:] - } - if len(s) == 0 { - break - } - // Accepted quoted string. No unescaping inside. - if s[0] == '"' || s[0] == '\'' { - quote := s[0] - s = s[1:] - i := 0 - for i < len(s) && s[i] != quote { - i++ - } - if i >= len(s) { - return nil, fmt.Errorf("unterminated %c string", quote) - } - f = append(f, s[:i]) - s = s[i+1:] - continue - } - i := 0 - for i < len(s) && !isSpaceByte(s[i]) { - i++ - } - f = append(f, s[:i]) - s = s[i:] - } - return f, nil -} - -func (v *stringsFlag) String() string { - return "" -} - -func pkgsMain(pkgs []*Package) (res []*Package) { - for _, p := range pkgs { - if p.Name == "main" { - res = append(res, p) - } - } - return res -} - -func pkgsNotMain(pkgs []*Package) (res []*Package) { - for _, p := range pkgs { - if p.Name != "main" { - res = append(res, p) - } - } - return res -} - -var pkgsFilter = func(pkgs []*Package) []*Package { return pkgs } - -func buildModeInit() { - _, gccgo := buildToolchain.(gccgoToolchain) - var codegenArg string - platform := goos + "/" + goarch - switch buildBuildmode { - case "archive": - pkgsFilter = pkgsNotMain - case "c-archive": - pkgsFilter = func(p []*Package) []*Package { - if len(p) != 1 || p[0].Name != "main" { - fatalf("-buildmode=c-archive requires exactly one main package") - } - return p - } - if gccgo { - codegenArg = "-fPIC" - } else { - switch platform { - case "darwin/arm", "darwin/arm64": - codegenArg = "-shared" - default: - switch goos { - case "dragonfly", "freebsd", "linux", "netbsd", "openbsd", "solaris": - // Use -shared so that the result is - // suitable for inclusion in a PIE or - // shared library. - codegenArg = "-shared" - } - } - } - exeSuffix = ".a" - ldBuildmode = "c-archive" - case "c-shared": - pkgsFilter = pkgsMain - if gccgo { - codegenArg = "-fPIC" - } else { - switch platform { - case "linux/amd64", "linux/arm", "linux/arm64", "linux/386", - "android/amd64", "android/arm", "android/arm64", "android/386": - codegenArg = "-shared" - case "darwin/amd64", "darwin/386": - default: - fatalf("-buildmode=c-shared not supported on %s\n", platform) - } - } - ldBuildmode = "c-shared" - case "default": - switch platform { - case "android/arm", "android/arm64", "android/amd64", "android/386": - if !gccgo { - codegenArg = "-shared" - } - ldBuildmode = "pie" - case "darwin/arm", "darwin/arm64": - if !gccgo { - codegenArg = "-shared" - } - fallthrough - default: - ldBuildmode = "exe" - } - case "exe": - pkgsFilter = pkgsMain - ldBuildmode = "exe" - case "pie": - if gccgo { - codegenArg = "-fPIE" - } else { - switch platform { - case "linux/386", "linux/amd64", "linux/arm", "linux/arm64", "linux/ppc64le", "linux/s390x", - "android/amd64", "android/arm", "android/arm64", "android/386": - codegenArg = "-shared" - default: - fatalf("-buildmode=pie not supported on %s\n", platform) - } - } - ldBuildmode = "pie" - case "shared": - pkgsFilter = pkgsNotMain - if gccgo { - codegenArg = "-fPIC" - } else { - switch platform { - case "linux/386", "linux/amd64", "linux/arm", "linux/arm64", "linux/ppc64le", "linux/s390x": - default: - fatalf("-buildmode=shared not supported on %s\n", platform) - } - codegenArg = "-dynlink" - } - if *buildO != "" { - fatalf("-buildmode=shared and -o not supported together") - } - ldBuildmode = "shared" - case "plugin": - pkgsFilter = pkgsMain - if gccgo { - codegenArg = "-fPIC" - } else { - switch platform { - case "linux/amd64", "linux/arm", "linux/arm64", "linux/386", - "android/amd64", "android/arm", "android/arm64", "android/386": - default: - fatalf("-buildmode=plugin not supported on %s\n", platform) - } - codegenArg = "-dynlink" - } - exeSuffix = ".so" - ldBuildmode = "plugin" - default: - fatalf("buildmode=%s not supported", buildBuildmode) - } - if buildLinkshared { - if gccgo { - codegenArg = "-fPIC" - } else { - switch platform { - case "linux/386", "linux/amd64", "linux/arm", "linux/arm64", "linux/ppc64le", "linux/s390x": - buildAsmflags = append(buildAsmflags, "-D=GOBUILDMODE_shared=1") - default: - fatalf("-linkshared not supported on %s\n", platform) - } - codegenArg = "-dynlink" - // TODO(mwhudson): remove -w when that gets fixed in linker. - buildLdflags = append(buildLdflags, "-linkshared", "-w") - } - } - if codegenArg != "" { - if gccgo { - buildGccgoflags = append(buildGccgoflags, codegenArg) - } else { - buildAsmflags = append(buildAsmflags, codegenArg) - buildGcflags = append(buildGcflags, codegenArg) - } - // Don't alter InstallSuffix when modifying default codegen args. - if buildBuildmode != "default" || buildLinkshared { - if buildContext.InstallSuffix != "" { - buildContext.InstallSuffix += "_" - } - buildContext.InstallSuffix += codegenArg[1:] - } - } -} - -func runBuild(cmd *Command, args []string) { - instrumentInit() - buildModeInit() - var b builder - b.init() - - pkgs := packagesForBuild(args) - - if len(pkgs) == 1 && pkgs[0].Name == "main" && *buildO == "" { - _, *buildO = path.Split(pkgs[0].ImportPath) - *buildO += exeSuffix - } - - // Special case -o /dev/null by not writing at all. - if *buildO == os.DevNull { - *buildO = "" - } - - // sanity check some often mis-used options - switch buildContext.Compiler { - case "gccgo": - if len(buildGcflags) != 0 { - fmt.Println("go build: when using gccgo toolchain, please pass compiler flags using -gccgoflags, not -gcflags") - } - if len(buildLdflags) != 0 { - fmt.Println("go build: when using gccgo toolchain, please pass linker flags using -gccgoflags, not -ldflags") - } - case "gc": - if len(buildGccgoflags) != 0 { - fmt.Println("go build: when using gc toolchain, please pass compile flags using -gcflags, and linker flags using -ldflags") - } - } - - depMode := modeBuild - if buildI { - depMode = modeInstall - } - - if *buildO != "" { - if len(pkgs) > 1 { - fatalf("go build: cannot use -o with multiple packages") - } else if len(pkgs) == 0 { - fatalf("no packages to build") - } - p := pkgs[0] - p.target = *buildO - p.Stale = true // must build - not up to date - p.StaleReason = "build -o flag in use" - a := b.action(modeInstall, depMode, p) - b.do(a) - return - } - - var a *action - if buildBuildmode == "shared" { - pkgs := pkgsFilter(packages(args)) - if libName, err := libname(args, pkgs); err != nil { - fatalf("%s", err.Error()) - } else { - a = b.libaction(libName, pkgs, modeBuild, depMode) - } - } else { - a = &action{} - for _, p := range pkgsFilter(packages(args)) { - a.deps = append(a.deps, b.action(modeBuild, depMode, p)) - } - } - b.do(a) -} - -var cmdInstall = &Command{ - UsageLine: "install [build flags] [packages]", - Short: "compile and install packages and dependencies", - Long: ` -Install compiles and installs the packages named by the import paths, -along with their dependencies. - -For more about the build flags, see 'go help build'. -For more about specifying packages, see 'go help packages'. - -See also: go build, go get, go clean. - `, -} - -// isMetaPackage checks if name is a reserved package name that expands to multiple packages -func isMetaPackage(name string) bool { - return name == "std" || name == "cmd" || name == "all" -} - -// libname returns the filename to use for the shared library when using -// -buildmode=shared. The rules we use are: -// Use arguments for special 'meta' packages: -// std --> libstd.so -// std cmd --> libstd,cmd.so -// A single non-meta argument with trailing "/..." is special cased: -// foo/... --> libfoo.so -// (A relative path like "./..." expands the "." first) -// Use import paths for other cases, changing '/' to '-': -// somelib --> libsubdir-somelib.so -// ./ or ../ --> libsubdir-somelib.so -// gopkg.in/tomb.v2 -> libgopkg.in-tomb.v2.so -// a/... b/... ---> liba/c,b/d.so - all matching import paths -// Name parts are joined with ','. -func libname(args []string, pkgs []*Package) (string, error) { - var libname string - appendName := func(arg string) { - if libname == "" { - libname = arg - } else { - libname += "," + arg - } - } - var haveNonMeta bool - for _, arg := range args { - if isMetaPackage(arg) { - appendName(arg) - } else { - haveNonMeta = true - } - } - if len(libname) == 0 { // non-meta packages only. use import paths - if len(args) == 1 && strings.HasSuffix(args[0], "/...") { - // Special case of "foo/..." as mentioned above. - arg := strings.TrimSuffix(args[0], "/...") - if build.IsLocalImport(arg) { - cwd, _ := os.Getwd() - bp, _ := buildContext.ImportDir(filepath.Join(cwd, arg), build.FindOnly) - if bp.ImportPath != "" && bp.ImportPath != "." { - arg = bp.ImportPath - } - } - appendName(strings.Replace(arg, "/", "-", -1)) - } else { - for _, pkg := range pkgs { - appendName(strings.Replace(pkg.ImportPath, "/", "-", -1)) - } - } - } else if haveNonMeta { // have both meta package and a non-meta one - return "", errors.New("mixing of meta and non-meta packages is not allowed") - } - // TODO(mwhudson): Needs to change for platforms that use different naming - // conventions... - return "lib" + libname + ".so", nil -} - -func runInstall(cmd *Command, args []string) { - installPackages(args, false) -} - -func installPackages(args []string, forGet bool) { - if gobin != "" && !filepath.IsAbs(gobin) { - fatalf("cannot install, GOBIN must be an absolute path") - } - - instrumentInit() - buildModeInit() - pkgs := pkgsFilter(packagesForBuild(args)) - - for _, p := range pkgs { - if p.Target == "" && (!p.Standard || p.ImportPath != "unsafe") { - switch { - case p.gobinSubdir: - errorf("go install: cannot install cross-compiled binaries when GOBIN is set") - case p.cmdline: - errorf("go install: no install location for .go files listed on command line (GOBIN not set)") - case p.ConflictDir != "": - errorf("go install: no install location for %s: hidden by %s", p.Dir, p.ConflictDir) - default: - errorf("go install: no install location for directory %s outside GOPATH\n"+ - "\tFor more details see: 'go help gopath'", p.Dir) - } - } - } - exitIfErrors() - - var b builder - b.init() - // Set the behavior for `go get` to not error on packages with test files only. - b.testFilesOnlyOK = forGet - var a *action - if buildBuildmode == "shared" { - if libName, err := libname(args, pkgs); err != nil { - fatalf("%s", err.Error()) - } else { - a = b.libaction(libName, pkgs, modeInstall, modeInstall) - } - } else { - a = &action{} - var tools []*action - for _, p := range pkgs { - // If p is a tool, delay the installation until the end of the build. - // This avoids installing assemblers/compilers that are being executed - // by other steps in the build. - // cmd/cgo is handled specially in b.action, so that we can - // both build and use it in the same 'go install'. - action := b.action(modeInstall, modeInstall, p) - if goTools[p.ImportPath] == toTool && p.ImportPath != "cmd/cgo" { - a.deps = append(a.deps, action.deps...) - action.deps = append(action.deps, a) - tools = append(tools, action) - continue - } - a.deps = append(a.deps, action) - } - if len(tools) > 0 { - a = &action{ - deps: tools, - } - } - } - b.do(a) - exitIfErrors() - - // Success. If this command is 'go install' with no arguments - // and the current directory (the implicit argument) is a command, - // remove any leftover command binary from a previous 'go build'. - // The binary is installed; it's not needed here anymore. - // And worse it might be a stale copy, which you don't want to find - // instead of the installed one if $PATH contains dot. - // One way to view this behavior is that it is as if 'go install' first - // runs 'go build' and the moves the generated file to the install dir. - // See issue 9645. - if len(args) == 0 && len(pkgs) == 1 && pkgs[0].Name == "main" { - // Compute file 'go build' would have created. - // If it exists and is an executable file, remove it. - _, targ := filepath.Split(pkgs[0].ImportPath) - targ += exeSuffix - if filepath.Join(pkgs[0].Dir, targ) != pkgs[0].Target { // maybe $GOBIN is the current directory - fi, err := os.Stat(targ) - if err == nil { - m := fi.Mode() - if m.IsRegular() { - if m&0111 != 0 || goos == "windows" { // windows never sets executable bit - os.Remove(targ) - } - } - } - } - } -} - -// Global build parameters (used during package load) -var ( - goarch string - goos string - exeSuffix string - gopath []string -) - -func init() { - goarch = buildContext.GOARCH - goos = buildContext.GOOS - - if goos == "windows" { - exeSuffix = ".exe" - } - gopath = filepath.SplitList(buildContext.GOPATH) -} - -// A builder holds global state about a build. -// It does not hold per-package state, because we -// build packages in parallel, and the builder is shared. -type builder struct { - work string // the temporary work directory (ends in filepath.Separator) - actionCache map[cacheKey]*action // a cache of already-constructed actions - mkdirCache map[string]bool // a cache of created directories - flagCache map[string]bool // a cache of supported compiler flags - print func(args ...interface{}) (int, error) - - testFilesOnlyOK bool // do not error if the packages only have test files - - output sync.Mutex - scriptDir string // current directory in printed script - - exec sync.Mutex - readySema chan bool - ready actionQueue -} - -// An action represents a single action in the action graph. -type action struct { - p *Package // the package this action works on - deps []*action // actions that must happen before this one - triggers []*action // inverse of deps - cgo *action // action for cgo binary if needed - args []string // additional args for runProgram - testOutput *bytes.Buffer // test output buffer - - f func(*builder, *action) error // the action itself (nil = no-op) - ignoreFail bool // whether to run f even if dependencies fail - - // Generated files, directories. - link bool // target is executable, not just package - pkgdir string // the -I or -L argument to use when importing this package - objdir string // directory for intermediate objects - objpkg string // the intermediate package .a file created during the action - target string // goal of the action: the created package or executable - - // Execution state. - pending int // number of deps yet to complete - priority int // relative execution priority - failed bool // whether the action failed -} - -// cacheKey is the key for the action cache. -type cacheKey struct { - mode buildMode - p *Package - shlib string -} - -// buildMode specifies the build mode: -// are we just building things or also installing the results? -type buildMode int - -const ( - modeBuild buildMode = iota - modeInstall -) - -var ( - goroot = filepath.Clean(runtime.GOROOT()) - gobin = os.Getenv("GOBIN") - gorootBin = filepath.Join(goroot, "bin") - gorootPkg = filepath.Join(goroot, "pkg") - gorootSrc = filepath.Join(goroot, "src") -) - -func (b *builder) init() { - var err error - b.print = func(a ...interface{}) (int, error) { - return fmt.Fprint(os.Stderr, a...) - } - b.actionCache = make(map[cacheKey]*action) - b.mkdirCache = make(map[string]bool) - - if buildN { - b.work = "$WORK" - } else { - b.work, err = ioutil.TempDir("", "go-build") - if err != nil { - fatalf("%s", err) - } - if buildX || buildWork { - fmt.Fprintf(os.Stderr, "WORK=%s\n", b.work) - } - if !buildWork { - workdir := b.work - atexit(func() { os.RemoveAll(workdir) }) - } - } -} - -// goFilesPackage creates a package for building a collection of Go files -// (typically named on the command line). The target is named p.a for -// package p or named after the first Go file for package main. -func goFilesPackage(gofiles []string) *Package { - // TODO: Remove this restriction. - for _, f := range gofiles { - if !strings.HasSuffix(f, ".go") { - fatalf("named files must be .go files") - } - } - - var stk importStack - ctxt := buildContext - ctxt.UseAllFiles = true - - // Synthesize fake "directory" that only shows the named files, - // to make it look like this is a standard package or - // command directory. So that local imports resolve - // consistently, the files must all be in the same directory. - var dirent []os.FileInfo - var dir string - for _, file := range gofiles { - fi, err := os.Stat(file) - if err != nil { - fatalf("%s", err) - } - if fi.IsDir() { - fatalf("%s is a directory, should be a Go file", file) - } - dir1, _ := filepath.Split(file) - if dir1 == "" { - dir1 = "./" - } - if dir == "" { - dir = dir1 - } else if dir != dir1 { - fatalf("named files must all be in one directory; have %s and %s", dir, dir1) - } - dirent = append(dirent, fi) - } - ctxt.ReadDir = func(string) ([]os.FileInfo, error) { return dirent, nil } - - var err error - if dir == "" { - dir = cwd - } - dir, err = filepath.Abs(dir) - if err != nil { - fatalf("%s", err) - } - - bp, err := ctxt.ImportDir(dir, 0) - pkg := new(Package) - pkg.local = true - pkg.cmdline = true - stk.push("main") - pkg.load(&stk, bp, err) - stk.pop() - pkg.localPrefix = dirToImportPath(dir) - pkg.ImportPath = "command-line-arguments" - pkg.target = "" - - if pkg.Name == "main" { - _, elem := filepath.Split(gofiles[0]) - exe := elem[:len(elem)-len(".go")] + exeSuffix - if *buildO == "" { - *buildO = exe - } - if gobin != "" { - pkg.target = filepath.Join(gobin, exe) - } - } - - pkg.Target = pkg.target - pkg.Stale = true - pkg.StaleReason = "files named on command line" - - computeStale(pkg) - return pkg -} - -// readpkglist returns the list of packages that were built into the shared library -// at shlibpath. For the native toolchain this list is stored, newline separated, in -// an ELF note with name "Go\x00\x00" and type 1. For GCCGO it is extracted from the -// .go_export section. -func readpkglist(shlibpath string) (pkgs []*Package) { - var stk importStack - if _, gccgo := buildToolchain.(gccgoToolchain); gccgo { - f, _ := elf.Open(shlibpath) - sect := f.Section(".go_export") - data, _ := sect.Data() - scanner := bufio.NewScanner(bytes.NewBuffer(data)) - for scanner.Scan() { - t := scanner.Text() - if strings.HasPrefix(t, "pkgpath ") { - t = strings.TrimPrefix(t, "pkgpath ") - t = strings.TrimSuffix(t, ";") - pkgs = append(pkgs, loadPackage(t, &stk)) - } - } - } else { - pkglistbytes, err := readELFNote(shlibpath, "Go\x00\x00", 1) - if err != nil { - fatalf("readELFNote failed: %v", err) - } - scanner := bufio.NewScanner(bytes.NewBuffer(pkglistbytes)) - for scanner.Scan() { - t := scanner.Text() - pkgs = append(pkgs, loadPackage(t, &stk)) - } - } - return -} - -// action returns the action for applying the given operation (mode) to the package. -// depMode is the action to use when building dependencies. -// action never looks for p in a shared library, but may find p's dependencies in a -// shared library if buildLinkshared is true. -func (b *builder) action(mode buildMode, depMode buildMode, p *Package) *action { - return b.action1(mode, depMode, p, false, "") -} - -// action1 returns the action for applying the given operation (mode) to the package. -// depMode is the action to use when building dependencies. -// action1 will look for p in a shared library if lookshared is true. -// forShlib is the shared library that p will become part of, if any. -func (b *builder) action1(mode buildMode, depMode buildMode, p *Package, lookshared bool, forShlib string) *action { - shlib := "" - if lookshared { - shlib = p.Shlib - } - key := cacheKey{mode, p, shlib} - - a := b.actionCache[key] - if a != nil { - return a - } - if shlib != "" { - key2 := cacheKey{modeInstall, nil, shlib} - a = b.actionCache[key2] - if a != nil { - b.actionCache[key] = a - return a - } - pkgs := readpkglist(shlib) - a = b.libaction(filepath.Base(shlib), pkgs, modeInstall, depMode) - b.actionCache[key2] = a - b.actionCache[key] = a - return a - } - - a = &action{p: p, pkgdir: p.build.PkgRoot} - if p.pkgdir != "" { // overrides p.t - a.pkgdir = p.pkgdir - } - b.actionCache[key] = a - - for _, p1 := range p.imports { - if forShlib != "" { - // p is part of a shared library. - if p1.Shlib != "" && p1.Shlib != forShlib { - // p1 is explicitly part of a different shared library. - // Put the action for that shared library into a.deps. - a.deps = append(a.deps, b.action1(depMode, depMode, p1, true, p1.Shlib)) - } else { - // p1 is (implicitly or not) part of this shared library. - // Put the action for p1 into a.deps. - a.deps = append(a.deps, b.action1(depMode, depMode, p1, false, forShlib)) - } - } else { - // p is not part of a shared library. - // If p1 is in a shared library, put the action for that into - // a.deps, otherwise put the action for p1 into a.deps. - a.deps = append(a.deps, b.action1(depMode, depMode, p1, buildLinkshared, p1.Shlib)) - } - } - - // If we are not doing a cross-build, then record the binary we'll - // generate for cgo as a dependency of the build of any package - // using cgo, to make sure we do not overwrite the binary while - // a package is using it. If this is a cross-build, then the cgo we - // are writing is not the cgo we need to use. - if goos == runtime.GOOS && goarch == runtime.GOARCH && !buildRace && !buildMSan && reqStdPkgSrc { - if (len(p.CgoFiles) > 0 || p.Standard && p.ImportPath == "runtime/cgo") && !buildLinkshared && buildBuildmode != "shared" { - var stk importStack - p1 := loadPackage("cmd/cgo", &stk) - if p1.Error != nil { - fatalf("load cmd/cgo: %v", p1.Error) - } - a.cgo = b.action(depMode, depMode, p1) - a.deps = append(a.deps, a.cgo) - } - } - - if p.Standard { - switch p.ImportPath { - case "builtin", "unsafe": - // Fake packages - nothing to build. - return a - } - // gccgo standard library is "fake" too. - if _, ok := buildToolchain.(gccgoToolchain); ok { - // the target name is needed for cgo. - a.target = p.target - return a - } - } - - if !p.Stale && p.target != "" { - // p.Stale==false implies that p.target is up-to-date. - // Record target name for use by actions depending on this one. - a.target = p.target - return a - } - - if p.local && p.target == "" { - // Imported via local path. No permanent target. - mode = modeBuild - } - work := p.pkgdir - if work == "" { - work = b.work - } - a.objdir = filepath.Join(work, a.p.ImportPath, "_obj") + string(filepath.Separator) - a.objpkg = buildToolchain.pkgpath(work, a.p) - a.link = p.Name == "main" - - switch mode { - case modeInstall: - a.f = (*builder).install - a.deps = []*action{b.action1(modeBuild, depMode, p, lookshared, forShlib)} - a.target = a.p.target - - // Install header for cgo in c-archive and c-shared modes. - if p.usesCgo() && (buildBuildmode == "c-archive" || buildBuildmode == "c-shared") { - hdrTarget := a.target[:len(a.target)-len(filepath.Ext(a.target))] + ".h" - if buildContext.Compiler == "gccgo" && *buildO == "" { - // For the header file, remove the "lib" - // added by go/build, so we generate pkg.h - // rather than libpkg.h. - dir, file := filepath.Split(hdrTarget) - file = strings.TrimPrefix(file, "lib") - hdrTarget = filepath.Join(dir, file) - } - ah := &action{ - p: a.p, - deps: []*action{a.deps[0]}, - f: (*builder).installHeader, - pkgdir: a.pkgdir, - objdir: a.objdir, - target: hdrTarget, - } - a.deps = append(a.deps, ah) - } - - case modeBuild: - a.f = (*builder).build - a.target = a.objpkg - if a.link { - // An executable file. (This is the name of a temporary file.) - // Because we run the temporary file in 'go run' and 'go test', - // the name will show up in ps listings. If the caller has specified - // a name, use that instead of a.out. The binary is generated - // in an otherwise empty subdirectory named exe to avoid - // naming conflicts. The only possible conflict is if we were - // to create a top-level package named exe. - name := "a.out" - if p.exeName != "" { - name = p.exeName - } else if goos == "darwin" && buildBuildmode == "c-shared" && p.target != "" { - // On OS X, the linker output name gets recorded in the - // shared library's LC_ID_DYLIB load command. - // The code invoking the linker knows to pass only the final - // path element. Arrange that the path element matches what - // we'll install it as; otherwise the library is only loadable as "a.out". - _, name = filepath.Split(p.target) - } - a.target = a.objdir + filepath.Join("exe", name) + exeSuffix - } - } - - return a -} - -func (b *builder) libaction(libname string, pkgs []*Package, mode, depMode buildMode) *action { - a := &action{} - switch mode { - default: - fatalf("unrecognized mode %v", mode) - - case modeBuild: - a.f = (*builder).linkShared - a.target = filepath.Join(b.work, libname) - for _, p := range pkgs { - if p.target == "" { - continue - } - a.deps = append(a.deps, b.action(depMode, depMode, p)) - } - - case modeInstall: - // Currently build mode shared forces external linking mode, and - // external linking mode forces an import of runtime/cgo (and - // math on arm). So if it was not passed on the command line and - // it is not present in another shared library, add it here. - _, gccgo := buildToolchain.(gccgoToolchain) - if !gccgo { - seencgo := false - for _, p := range pkgs { - seencgo = seencgo || (p.Standard && p.ImportPath == "runtime/cgo") - } - if !seencgo { - var stk importStack - p := loadPackage("runtime/cgo", &stk) - if p.Error != nil { - fatalf("load runtime/cgo: %v", p.Error) - } - computeStale(p) - // If runtime/cgo is in another shared library, then that's - // also the shared library that contains runtime, so - // something will depend on it and so runtime/cgo's staleness - // will be checked when processing that library. - if p.Shlib == "" || p.Shlib == libname { - pkgs = append([]*Package{}, pkgs...) - pkgs = append(pkgs, p) - } - } - if goarch == "arm" { - seenmath := false - for _, p := range pkgs { - seenmath = seenmath || (p.Standard && p.ImportPath == "math") - } - if !seenmath { - var stk importStack - p := loadPackage("math", &stk) - if p.Error != nil { - fatalf("load math: %v", p.Error) - } - computeStale(p) - // If math is in another shared library, then that's - // also the shared library that contains runtime, so - // something will depend on it and so math's staleness - // will be checked when processing that library. - if p.Shlib == "" || p.Shlib == libname { - pkgs = append([]*Package{}, pkgs...) - pkgs = append(pkgs, p) - } - } - } - } - - // Figure out where the library will go. - var libdir string - for _, p := range pkgs { - plibdir := p.build.PkgTargetRoot - if gccgo { - plibdir = filepath.Join(plibdir, "shlibs") - } - if libdir == "" { - libdir = plibdir - } else if libdir != plibdir { - fatalf("multiple roots %s & %s", libdir, plibdir) - } - } - a.target = filepath.Join(libdir, libname) - - // Now we can check whether we need to rebuild it. - stale := false - var built time.Time - if fi, err := os.Stat(a.target); err == nil { - built = fi.ModTime() - } - for _, p := range pkgs { - if p.target == "" { - continue - } - stale = stale || p.Stale - lstat, err := os.Stat(p.target) - if err != nil || lstat.ModTime().After(built) { - stale = true - } - a.deps = append(a.deps, b.action1(depMode, depMode, p, false, a.target)) - } - - if stale { - a.f = (*builder).install - buildAction := b.libaction(libname, pkgs, modeBuild, depMode) - a.deps = []*action{buildAction} - for _, p := range pkgs { - if p.target == "" { - continue - } - shlibnameaction := &action{} - shlibnameaction.f = (*builder).installShlibname - shlibnameaction.target = p.target[:len(p.target)-2] + ".shlibname" - a.deps = append(a.deps, shlibnameaction) - shlibnameaction.deps = append(shlibnameaction.deps, buildAction) - } - } - } - return a -} - -// actionList returns the list of actions in the dag rooted at root -// as visited in a depth-first post-order traversal. -func actionList(root *action) []*action { - seen := map[*action]bool{} - all := []*action{} - var walk func(*action) - walk = func(a *action) { - if seen[a] { - return - } - seen[a] = true - for _, a1 := range a.deps { - walk(a1) - } - all = append(all, a) - } - walk(root) - return all -} - -// allArchiveActions returns a list of the archive dependencies of root. -// This is needed because if package p depends on package q that is in libr.so, the -// action graph looks like p->libr.so->q and so just scanning through p's -// dependencies does not find the import dir for q. -func allArchiveActions(root *action) []*action { - seen := map[*action]bool{} - r := []*action{} - var walk func(*action) - walk = func(a *action) { - if seen[a] { - return - } - seen[a] = true - if strings.HasSuffix(a.target, ".so") || a == root { - for _, a1 := range a.deps { - walk(a1) - } - } else if strings.HasSuffix(a.target, ".a") { - r = append(r, a) - } - } - walk(root) - return r -} - -// do runs the action graph rooted at root. -func (b *builder) do(root *action) { - /* Commented out for gccgo, which does not have osArchSupportsCgo. - - if _, ok := osArchSupportsCgo[goos+"/"+goarch]; !ok && buildContext.Compiler == "gc" { - fmt.Fprintf(os.Stderr, "cmd/go: unsupported GOOS/GOARCH pair %s/%s\n", goos, goarch) - os.Exit(2) - } - */ - - // Build list of all actions, assigning depth-first post-order priority. - // The original implementation here was a true queue - // (using a channel) but it had the effect of getting - // distracted by low-level leaf actions to the detriment - // of completing higher-level actions. The order of - // work does not matter much to overall execution time, - // but when running "go test std" it is nice to see each test - // results as soon as possible. The priorities assigned - // ensure that, all else being equal, the execution prefers - // to do what it would have done first in a simple depth-first - // dependency order traversal. - all := actionList(root) - for i, a := range all { - a.priority = i - } - - b.readySema = make(chan bool, len(all)) - - // Initialize per-action execution state. - for _, a := range all { - for _, a1 := range a.deps { - a1.triggers = append(a1.triggers, a) - } - a.pending = len(a.deps) - if a.pending == 0 { - b.ready.push(a) - b.readySema <- true - } - } - - // Handle runs a single action and takes care of triggering - // any actions that are runnable as a result. - handle := func(a *action) { - var err error - if a.f != nil && (!a.failed || a.ignoreFail) { - err = a.f(b, a) - } - - // The actions run in parallel but all the updates to the - // shared work state are serialized through b.exec. - b.exec.Lock() - defer b.exec.Unlock() - - if err != nil { - if err == errPrintedOutput { - setExitStatus(2) - } else if _, ok := err.(*build.NoGoError); ok && len(a.p.TestGoFiles) > 0 && b.testFilesOnlyOK { - // Ignore the "no buildable Go source files" error for a package with only test files. - } else { - errorf("%s", err) - } - a.failed = true - } - - for _, a0 := range a.triggers { - if a.failed { - a0.failed = true - } - if a0.pending--; a0.pending == 0 { - b.ready.push(a0) - b.readySema <- true - } - } - - if a == root { - close(b.readySema) - } - } - - var wg sync.WaitGroup - - // Kick off goroutines according to parallelism. - // If we are using the -n flag (just printing commands) - // drop the parallelism to 1, both to make the output - // deterministic and because there is no real work anyway. - par := buildP - if buildN { - par = 1 - } - for i := 0; i < par; i++ { - wg.Add(1) - go func() { - defer wg.Done() - for { - select { - case _, ok := <-b.readySema: - if !ok { - return - } - // Receiving a value from b.readySema entitles - // us to take from the ready queue. - b.exec.Lock() - a := b.ready.pop() - b.exec.Unlock() - handle(a) - case <-interrupted: - setExitStatus(1) - return - } - } - }() - } - - wg.Wait() -} - -// build is the action for building a single package or command. -func (b *builder) build(a *action) (err error) { - // Return an error for binary-only package. - // We only reach this if isStale believes the binary form is - // either not present or not usable. - if a.p.BinaryOnly { - return fmt.Errorf("missing or invalid package binary for binary-only package %s", a.p.ImportPath) - } - - // Return an error if the package has CXX files but it's not using - // cgo nor SWIG, since the CXX files can only be processed by cgo - // and SWIG. - if len(a.p.CXXFiles) > 0 && !a.p.usesCgo() && !a.p.usesSwig() { - return fmt.Errorf("can't build package %s because it contains C++ files (%s) but it's not using cgo nor SWIG", - a.p.ImportPath, strings.Join(a.p.CXXFiles, ",")) - } - // Same as above for Objective-C files - if len(a.p.MFiles) > 0 && !a.p.usesCgo() && !a.p.usesSwig() { - return fmt.Errorf("can't build package %s because it contains Objective-C files (%s) but it's not using cgo nor SWIG", - a.p.ImportPath, strings.Join(a.p.MFiles, ",")) - } - // Same as above for Fortran files - if len(a.p.FFiles) > 0 && !a.p.usesCgo() && !a.p.usesSwig() { - return fmt.Errorf("can't build package %s because it contains Fortran files (%s) but it's not using cgo nor SWIG", - a.p.ImportPath, strings.Join(a.p.FFiles, ",")) - } - - defer func() { - if _, ok := err.(*build.NoGoError); err != nil && err != errPrintedOutput && !(ok && b.testFilesOnlyOK && len(a.p.TestGoFiles) > 0) { - err = fmt.Errorf("go build %s: %v", a.p.ImportPath, err) - } - }() - if buildN { - // In -n mode, print a banner between packages. - // The banner is five lines so that when changes to - // different sections of the bootstrap script have to - // be merged, the banners give patch something - // to use to find its context. - b.print("\n#\n# " + a.p.ImportPath + "\n#\n\n") - } - - if buildV { - b.print(a.p.ImportPath + "\n") - } - - // Make build directory. - obj := a.objdir - if err := b.mkdir(obj); err != nil { - return err - } - - // make target directory - dir, _ := filepath.Split(a.target) - if dir != "" { - if err := b.mkdir(dir); err != nil { - return err - } - } - - var gofiles, cgofiles, objdirCgofiles, cfiles, sfiles, cxxfiles, objects, cgoObjects, pcCFLAGS, pcLDFLAGS []string - - gofiles = append(gofiles, a.p.GoFiles...) - cgofiles = append(cgofiles, a.p.CgoFiles...) - cfiles = append(cfiles, a.p.CFiles...) - sfiles = append(sfiles, a.p.SFiles...) - cxxfiles = append(cxxfiles, a.p.CXXFiles...) - - if a.p.usesCgo() || a.p.usesSwig() { - if pcCFLAGS, pcLDFLAGS, err = b.getPkgConfigFlags(a.p); err != nil { - return - } - } - - // Run SWIG on each .swig and .swigcxx file. - // Each run will generate two files, a .go file and a .c or .cxx file. - // The .go file will use import "C" and is to be processed by cgo. - if a.p.usesSwig() { - outGo, outC, outCXX, err := b.swig(a.p, obj, pcCFLAGS) - if err != nil { - return err - } - objdirCgofiles = append(objdirCgofiles, outGo...) - cfiles = append(cfiles, outC...) - cxxfiles = append(cxxfiles, outCXX...) - } - - // Run cgo. - if a.p.usesCgo() || a.p.usesSwig() { - // In a package using cgo, cgo compiles the C, C++ and assembly files with gcc. - // There is one exception: runtime/cgo's job is to bridge the - // cgo and non-cgo worlds, so it necessarily has files in both. - // In that case gcc only gets the gcc_* files. - var gccfiles []string - gccfiles = append(gccfiles, cfiles...) - cfiles = nil - if a.p.Standard && a.p.ImportPath == "runtime/cgo" { - filter := func(files, nongcc, gcc []string) ([]string, []string) { - for _, f := range files { - if strings.HasPrefix(f, "gcc_") { - gcc = append(gcc, f) - } else { - nongcc = append(nongcc, f) - } - } - return nongcc, gcc - } - sfiles, gccfiles = filter(sfiles, sfiles[:0], gccfiles) - } else { - gccfiles = append(gccfiles, sfiles...) - sfiles = nil - } - - cgoExe := tool("cgo") - if a.cgo != nil && a.cgo.target != "" { - cgoExe = a.cgo.target - } - outGo, outObj, err := b.cgo(a, cgoExe, obj, pcCFLAGS, pcLDFLAGS, cgofiles, objdirCgofiles, gccfiles, cxxfiles, a.p.MFiles, a.p.FFiles) - if err != nil { - return err - } - if _, ok := buildToolchain.(gccgoToolchain); ok { - cgoObjects = append(cgoObjects, filepath.Join(a.objdir, "_cgo_flags")) - } - cgoObjects = append(cgoObjects, outObj...) - gofiles = append(gofiles, outGo...) - } - - if len(gofiles) == 0 { - return &build.NoGoError{Dir: a.p.Dir} - } - - // If we're doing coverage, preprocess the .go files and put them in the work directory - if a.p.coverMode != "" { - for i, file := range gofiles { - var sourceFile string - var coverFile string - var key string - if strings.HasSuffix(file, ".cgo1.go") { - // cgo files have absolute paths - base := filepath.Base(file) - sourceFile = file - coverFile = filepath.Join(obj, base) - key = strings.TrimSuffix(base, ".cgo1.go") + ".go" - } else { - sourceFile = filepath.Join(a.p.Dir, file) - coverFile = filepath.Join(obj, file) - key = file - } - cover := a.p.coverVars[key] - if cover == nil || isTestFile(file) { - // Not covering this file. - continue - } - if err := b.cover(a, coverFile, sourceFile, 0666, cover.Var); err != nil { - return err - } - gofiles[i] = coverFile - } - } - - // Prepare Go import path list. - inc := b.includeArgs("-I", allArchiveActions(a)) - - // Compile Go. - ofile, out, err := buildToolchain.gc(b, a.p, a.objpkg, obj, len(sfiles) > 0, inc, gofiles) - if len(out) > 0 { - b.showOutput(a.p.Dir, a.p.ImportPath, b.processOutput(out)) - if err != nil { - return errPrintedOutput - } - } - if err != nil { - return err - } - if ofile != a.objpkg { - objects = append(objects, ofile) - } - - // Copy .h files named for goos or goarch or goos_goarch - // to names using GOOS and GOARCH. - // For example, defs_linux_amd64.h becomes defs_GOOS_GOARCH.h. - _goos_goarch := "_" + goos + "_" + goarch - _goos := "_" + goos - _goarch := "_" + goarch - for _, file := range a.p.HFiles { - name, ext := fileExtSplit(file) - switch { - case strings.HasSuffix(name, _goos_goarch): - targ := file[:len(name)-len(_goos_goarch)] + "_GOOS_GOARCH." + ext - if err := b.copyFile(a, obj+targ, filepath.Join(a.p.Dir, file), 0666, true); err != nil { - return err - } - case strings.HasSuffix(name, _goarch): - targ := file[:len(name)-len(_goarch)] + "_GOARCH." + ext - if err := b.copyFile(a, obj+targ, filepath.Join(a.p.Dir, file), 0666, true); err != nil { - return err - } - case strings.HasSuffix(name, _goos): - targ := file[:len(name)-len(_goos)] + "_GOOS." + ext - if err := b.copyFile(a, obj+targ, filepath.Join(a.p.Dir, file), 0666, true); err != nil { - return err - } - } - } - - for _, file := range cfiles { - out := file[:len(file)-len(".c")] + ".o" - if err := buildToolchain.cc(b, a.p, obj, obj+out, file); err != nil { - return err - } - objects = append(objects, out) - } - - // Assemble .s files. - if len(sfiles) > 0 { - ofiles, err := buildToolchain.asm(b, a.p, obj, sfiles) - if err != nil { - return err - } - objects = append(objects, ofiles...) - } - - // NOTE(rsc): On Windows, it is critically important that the - // gcc-compiled objects (cgoObjects) be listed after the ordinary - // objects in the archive. I do not know why this is. - // https://golang.org/issue/2601 - objects = append(objects, cgoObjects...) - - // Add system object files. - for _, syso := range a.p.SysoFiles { - objects = append(objects, filepath.Join(a.p.Dir, syso)) - } - - // Pack into archive in obj directory. - // If the Go compiler wrote an archive, we only need to add the - // object files for non-Go sources to the archive. - // If the Go compiler wrote an archive and the package is entirely - // Go sources, there is no pack to execute at all. - if len(objects) > 0 { - if err := buildToolchain.pack(b, a.p, obj, a.objpkg, objects); err != nil { - return err - } - } - - // Link if needed. - if a.link { - // The compiler only cares about direct imports, but the - // linker needs the whole dependency tree. - all := actionList(a) - all = all[:len(all)-1] // drop a - if err := buildToolchain.ld(b, a, a.target, all, a.objpkg, objects); err != nil { - return err - } - } - - return nil -} - -// pkgconfigCmd returns a pkg-config binary name -// defaultPkgConfig is defined in zdefaultcc.go, written by cmd/dist. -func (b *builder) pkgconfigCmd() string { - return envList("PKG_CONFIG", defaultPkgConfig)[0] -} - -// splitPkgConfigOutput parses the pkg-config output into a slice of -// flags. pkg-config always uses \ to escape special characters. -func splitPkgConfigOutput(out []byte) []string { - if len(out) == 0 { - return nil - } - var flags []string - flag := make([]byte, len(out)) - r, w := 0, 0 - for r < len(out) { - switch out[r] { - case ' ', '\t', '\r', '\n': - if w > 0 { - flags = append(flags, string(flag[:w])) - } - w = 0 - case '\\': - r++ - fallthrough - default: - if r < len(out) { - flag[w] = out[r] - w++ - } - } - r++ - } - if w > 0 { - flags = append(flags, string(flag[:w])) - } - return flags -} - -// Calls pkg-config if needed and returns the cflags/ldflags needed to build the package. -func (b *builder) getPkgConfigFlags(p *Package) (cflags, ldflags []string, err error) { - if pkgs := p.CgoPkgConfig; len(pkgs) > 0 { - var out []byte - out, err = b.runOut(p.Dir, p.ImportPath, nil, b.pkgconfigCmd(), "--cflags", pkgs) - if err != nil { - b.showOutput(p.Dir, b.pkgconfigCmd()+" --cflags "+strings.Join(pkgs, " "), string(out)) - b.print(err.Error() + "\n") - err = errPrintedOutput - return - } - if len(out) > 0 { - cflags = splitPkgConfigOutput(out) - } - out, err = b.runOut(p.Dir, p.ImportPath, nil, b.pkgconfigCmd(), "--libs", pkgs) - if err != nil { - b.showOutput(p.Dir, b.pkgconfigCmd()+" --libs "+strings.Join(pkgs, " "), string(out)) - b.print(err.Error() + "\n") - err = errPrintedOutput - return - } - if len(out) > 0 { - ldflags = strings.Fields(string(out)) - } - } - return -} - -func (b *builder) installShlibname(a *action) error { - a1 := a.deps[0] - err := ioutil.WriteFile(a.target, []byte(filepath.Base(a1.target)+"\n"), 0666) - if err != nil { - return err - } - if buildX { - b.showcmd("", "echo '%s' > %s # internal", filepath.Base(a1.target), a.target) - } - return nil -} - -func (b *builder) linkShared(a *action) (err error) { - allactions := actionList(a) - allactions = allactions[:len(allactions)-1] - return buildToolchain.ldShared(b, a.deps, a.target, allactions) -} - -// install is the action for installing a single package or executable. -func (b *builder) install(a *action) (err error) { - defer func() { - if err != nil && err != errPrintedOutput { - err = fmt.Errorf("go install %s: %v", a.p.ImportPath, err) - } - }() - a1 := a.deps[0] - perm := os.FileMode(0666) - if a1.link { - switch buildBuildmode { - case "c-archive", "c-shared", "plugin": - default: - perm = 0777 - } - } - - // make target directory - dir, _ := filepath.Split(a.target) - if dir != "" { - if err := b.mkdir(dir); err != nil { - return err - } - } - - // remove object dir to keep the amount of - // garbage down in a large build. On an operating system - // with aggressive buffering, cleaning incrementally like - // this keeps the intermediate objects from hitting the disk. - if !buildWork { - defer os.RemoveAll(a1.objdir) - defer os.Remove(a1.target) - } - - return b.moveOrCopyFile(a, a.target, a1.target, perm, false) -} - -// includeArgs returns the -I or -L directory list for access -// to the results of the list of actions. -func (b *builder) includeArgs(flag string, all []*action) []string { - inc := []string{} - incMap := map[string]bool{ - b.work: true, // handled later - gorootPkg: true, - "": true, // ignore empty strings - } - - // Look in the temporary space for results of test-specific actions. - // This is the $WORK/my/package/_test directory for the - // package being built, so there are few of these. - for _, a1 := range all { - if a1.p == nil { - continue - } - if dir := a1.pkgdir; dir != a1.p.build.PkgRoot && !incMap[dir] { - incMap[dir] = true - inc = append(inc, flag, dir) - } - } - - // Also look in $WORK for any non-test packages that have - // been built but not installed. - inc = append(inc, flag, b.work) - - // Finally, look in the installed package directories for each action. - // First add the package dirs corresponding to GOPATH entries - // in the original GOPATH order. - need := map[string]*build.Package{} - for _, a1 := range all { - if a1.p != nil && a1.pkgdir == a1.p.build.PkgRoot { - need[a1.p.build.Root] = a1.p.build - } - } - for _, root := range gopath { - if p := need[root]; p != nil && !incMap[p.PkgRoot] { - incMap[p.PkgRoot] = true - inc = append(inc, flag, p.PkgTargetRoot) - } - } - - // Then add anything that's left. - for _, a1 := range all { - if a1.p == nil { - continue - } - if dir := a1.pkgdir; dir == a1.p.build.PkgRoot && !incMap[dir] { - incMap[dir] = true - inc = append(inc, flag, a1.p.build.PkgTargetRoot) - } - } - - return inc -} - -// moveOrCopyFile is like 'mv src dst' or 'cp src dst'. -func (b *builder) moveOrCopyFile(a *action, dst, src string, perm os.FileMode, force bool) error { - if buildN { - b.showcmd("", "mv %s %s", src, dst) - return nil - } - - // If we can update the mode and rename to the dst, do it. - // Otherwise fall back to standard copy. - - // The perm argument is meant to be adjusted according to umask, - // but we don't know what the umask is. - // Create a dummy file to find out. - // This avoids build tags and works even on systems like Plan 9 - // where the file mask computation incorporates other information. - mode := perm - f, err := os.OpenFile(filepath.Clean(dst)+"-go-tmp-umask", os.O_WRONLY|os.O_CREATE|os.O_EXCL, perm) - if err == nil { - fi, err := f.Stat() - if err == nil { - mode = fi.Mode() & 0777 - } - name := f.Name() - f.Close() - os.Remove(name) - } - - if err := os.Chmod(src, mode); err == nil { - if err := os.Rename(src, dst); err == nil { - if buildX { - b.showcmd("", "mv %s %s", src, dst) - } - return nil - } - } - - return b.copyFile(a, dst, src, perm, force) -} - -// copyFile is like 'cp src dst'. -func (b *builder) copyFile(a *action, dst, src string, perm os.FileMode, force bool) error { - if buildN || buildX { - b.showcmd("", "cp %s %s", src, dst) - if buildN { - return nil - } - } - - sf, err := os.Open(src) - if err != nil { - return err - } - defer sf.Close() - - // Be careful about removing/overwriting dst. - // Do not remove/overwrite if dst exists and is a directory - // or a non-object file. - if fi, err := os.Stat(dst); err == nil { - if fi.IsDir() { - return fmt.Errorf("build output %q already exists and is a directory", dst) - } - if !force && fi.Mode().IsRegular() && !isObject(dst) { - return fmt.Errorf("build output %q already exists and is not an object file", dst) - } - } - - // On Windows, remove lingering ~ file from last attempt. - if toolIsWindows { - if _, err := os.Stat(dst + "~"); err == nil { - os.Remove(dst + "~") - } - } - - mayberemovefile(dst) - df, err := os.OpenFile(dst, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, perm) - if err != nil && toolIsWindows { - // Windows does not allow deletion of a binary file - // while it is executing. Try to move it out of the way. - // If the move fails, which is likely, we'll try again the - // next time we do an install of this binary. - if err := os.Rename(dst, dst+"~"); err == nil { - os.Remove(dst + "~") - } - df, err = os.OpenFile(dst, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, perm) - } - if err != nil { - return err - } - - _, err = io.Copy(df, sf) - df.Close() - if err != nil { - mayberemovefile(dst) - return fmt.Errorf("copying %s to %s: %v", src, dst, err) - } - return nil -} - -// Install the cgo export header file, if there is one. -func (b *builder) installHeader(a *action) error { - src := a.objdir + "_cgo_install.h" - if _, err := os.Stat(src); os.IsNotExist(err) { - // If the file does not exist, there are no exported - // functions, and we do not install anything. - return nil - } - - dir, _ := filepath.Split(a.target) - if dir != "" { - if err := b.mkdir(dir); err != nil { - return err - } - } - - return b.moveOrCopyFile(a, a.target, src, 0666, true) -} - -// cover runs, in effect, -// go tool cover -mode=b.coverMode -var="varName" -o dst.go src.go -func (b *builder) cover(a *action, dst, src string, perm os.FileMode, varName string) error { - return b.run(a.objdir, "cover "+a.p.ImportPath, nil, - buildToolExec, - tool("cover"), - "-mode", a.p.coverMode, - "-var", varName, - "-o", dst, - src) -} - -var objectMagic = [][]byte{ - {'!', '<', 'a', 'r', 'c', 'h', '>', '\n'}, // Package archive - {'\x7F', 'E', 'L', 'F'}, // ELF - {0xFE, 0xED, 0xFA, 0xCE}, // Mach-O big-endian 32-bit - {0xFE, 0xED, 0xFA, 0xCF}, // Mach-O big-endian 64-bit - {0xCE, 0xFA, 0xED, 0xFE}, // Mach-O little-endian 32-bit - {0xCF, 0xFA, 0xED, 0xFE}, // Mach-O little-endian 64-bit - {0x4d, 0x5a, 0x90, 0x00, 0x03, 0x00}, // PE (Windows) as generated by 6l/8l and gcc - {0x00, 0x00, 0x01, 0xEB}, // Plan 9 i386 - {0x00, 0x00, 0x8a, 0x97}, // Plan 9 amd64 - {0x00, 0x00, 0x06, 0x47}, // Plan 9 arm -} - -func isObject(s string) bool { - f, err := os.Open(s) - if err != nil { - return false - } - defer f.Close() - buf := make([]byte, 64) - io.ReadFull(f, buf) - for _, magic := range objectMagic { - if bytes.HasPrefix(buf, magic) { - return true - } - } - return false -} - -// mayberemovefile removes a file only if it is a regular file -// When running as a user with sufficient privileges, we may delete -// even device files, for example, which is not intended. -func mayberemovefile(s string) { - if fi, err := os.Lstat(s); err == nil && !fi.Mode().IsRegular() { - return - } - os.Remove(s) -} - -// fmtcmd formats a command in the manner of fmt.Sprintf but also: -// -// If dir is non-empty and the script is not in dir right now, -// fmtcmd inserts "cd dir\n" before the command. -// -// fmtcmd replaces the value of b.work with $WORK. -// fmtcmd replaces the value of goroot with $GOROOT. -// fmtcmd replaces the value of b.gobin with $GOBIN. -// -// fmtcmd replaces the name of the current directory with dot (.) -// but only when it is at the beginning of a space-separated token. -// -func (b *builder) fmtcmd(dir string, format string, args ...interface{}) string { - cmd := fmt.Sprintf(format, args...) - if dir != "" && dir != "/" { - cmd = strings.Replace(" "+cmd, " "+dir, " .", -1)[1:] - if b.scriptDir != dir { - b.scriptDir = dir - cmd = "cd " + dir + "\n" + cmd - } - } - if b.work != "" { - cmd = strings.Replace(cmd, b.work, "$WORK", -1) - } - return cmd -} - -// showcmd prints the given command to standard output -// for the implementation of -n or -x. -func (b *builder) showcmd(dir string, format string, args ...interface{}) { - b.output.Lock() - defer b.output.Unlock() - b.print(b.fmtcmd(dir, format, args...) + "\n") -} - -// showOutput prints "# desc" followed by the given output. -// The output is expected to contain references to 'dir', usually -// the source directory for the package that has failed to build. -// showOutput rewrites mentions of dir with a relative path to dir -// when the relative path is shorter. This is usually more pleasant. -// For example, if fmt doesn't compile and we are in src/html, -// the output is -// -// $ go build -// # fmt -// ../fmt/print.go:1090: undefined: asdf -// $ -// -// instead of -// -// $ go build -// # fmt -// /usr/gopher/go/src/fmt/print.go:1090: undefined: asdf -// $ -// -// showOutput also replaces references to the work directory with $WORK. -// -func (b *builder) showOutput(dir, desc, out string) { - prefix := "# " + desc - suffix := "\n" + out - if reldir := shortPath(dir); reldir != dir { - suffix = strings.Replace(suffix, " "+dir, " "+reldir, -1) - suffix = strings.Replace(suffix, "\n"+dir, "\n"+reldir, -1) - } - suffix = strings.Replace(suffix, " "+b.work, " $WORK", -1) - - b.output.Lock() - defer b.output.Unlock() - b.print(prefix, suffix) -} - -// shortPath returns an absolute or relative name for path, whatever is shorter. -func shortPath(path string) string { - if rel, err := filepath.Rel(cwd, path); err == nil && len(rel) < len(path) { - return rel - } - return path -} - -// relPaths returns a copy of paths with absolute paths -// made relative to the current directory if they would be shorter. -func relPaths(paths []string) []string { - var out []string - pwd, _ := os.Getwd() - for _, p := range paths { - rel, err := filepath.Rel(pwd, p) - if err == nil && len(rel) < len(p) { - p = rel - } - out = append(out, p) - } - return out -} - -// errPrintedOutput is a special error indicating that a command failed -// but that it generated output as well, and that output has already -// been printed, so there's no point showing 'exit status 1' or whatever -// the wait status was. The main executor, builder.do, knows not to -// print this error. -var errPrintedOutput = errors.New("already printed output - no need to show error") - -var cgoLine = regexp.MustCompile(`\[[^\[\]]+\.cgo1\.go:[0-9]+\]`) -var cgoTypeSigRe = regexp.MustCompile(`\b_Ctype_\B`) - -// run runs the command given by cmdline in the directory dir. -// If the command fails, run prints information about the failure -// and returns a non-nil error. -func (b *builder) run(dir string, desc string, env []string, cmdargs ...interface{}) error { - out, err := b.runOut(dir, desc, env, cmdargs...) - if len(out) > 0 { - if desc == "" { - desc = b.fmtcmd(dir, "%s", strings.Join(stringList(cmdargs...), " ")) - } - b.showOutput(dir, desc, b.processOutput(out)) - if err != nil { - err = errPrintedOutput - } - } - return err -} - -// processOutput prepares the output of runOut to be output to the console. -func (b *builder) processOutput(out []byte) string { - if out[len(out)-1] != '\n' { - out = append(out, '\n') - } - messages := string(out) - // Fix up output referring to cgo-generated code to be more readable. - // Replace x.go:19[/tmp/.../x.cgo1.go:18] with x.go:19. - // Replace *[100]_Ctype_foo with *[100]C.foo. - // If we're using -x, assume we're debugging and want the full dump, so disable the rewrite. - if !buildX && cgoLine.MatchString(messages) { - messages = cgoLine.ReplaceAllString(messages, "") - messages = cgoTypeSigRe.ReplaceAllString(messages, "C.") - } - return messages -} - -// runOut runs the command given by cmdline in the directory dir. -// It returns the command output and any errors that occurred. -func (b *builder) runOut(dir string, desc string, env []string, cmdargs ...interface{}) ([]byte, error) { - cmdline := stringList(cmdargs...) - if buildN || buildX { - var envcmdline string - for i := range env { - envcmdline += env[i] - envcmdline += " " - } - envcmdline += joinUnambiguously(cmdline) - b.showcmd(dir, "%s", envcmdline) - if buildN { - return nil, nil - } - } - - nbusy := 0 - for { - var buf bytes.Buffer - cmd := exec.Command(cmdline[0], cmdline[1:]...) - cmd.Stdout = &buf - cmd.Stderr = &buf - cmd.Dir = dir - cmd.Env = mergeEnvLists(env, envForDir(cmd.Dir, os.Environ())) - err := cmd.Run() - - // cmd.Run will fail on Unix if some other process has the binary - // we want to run open for writing. This can happen here because - // we build and install the cgo command and then run it. - // If another command was kicked off while we were writing the - // cgo binary, the child process for that command may be holding - // a reference to the fd, keeping us from running exec. - // - // But, you might reasonably wonder, how can this happen? - // The cgo fd, like all our fds, is close-on-exec, so that we need - // not worry about other processes inheriting the fd accidentally. - // The answer is that running a command is fork and exec. - // A child forked while the cgo fd is open inherits that fd. - // Until the child has called exec, it holds the fd open and the - // kernel will not let us run cgo. Even if the child were to close - // the fd explicitly, it would still be open from the time of the fork - // until the time of the explicit close, and the race would remain. - // - // On Unix systems, this results in ETXTBSY, which formats - // as "text file busy". Rather than hard-code specific error cases, - // we just look for that string. If this happens, sleep a little - // and try again. We let this happen three times, with increasing - // sleep lengths: 100+200+400 ms = 0.7 seconds. - // - // An alternate solution might be to split the cmd.Run into - // separate cmd.Start and cmd.Wait, and then use an RWLock - // to make sure that copyFile only executes when no cmd.Start - // call is in progress. However, cmd.Start (really syscall.forkExec) - // only guarantees that when it returns, the exec is committed to - // happen and succeed. It uses a close-on-exec file descriptor - // itself to determine this, so we know that when cmd.Start returns, - // at least one close-on-exec file descriptor has been closed. - // However, we cannot be sure that all of them have been closed, - // so the program might still encounter ETXTBSY even with such - // an RWLock. The race window would be smaller, perhaps, but not - // guaranteed to be gone. - // - // Sleeping when we observe the race seems to be the most reliable - // option we have. - // - // https://golang.org/issue/3001 - // - if err != nil && nbusy < 3 && strings.Contains(err.Error(), "text file busy") { - time.Sleep(100 * time.Millisecond << uint(nbusy)) - nbusy++ - continue - } - - // err can be something like 'exit status 1'. - // Add information about what program was running. - // Note that if buf.Bytes() is non-empty, the caller usually - // shows buf.Bytes() and does not print err at all, so the - // prefix here does not make most output any more verbose. - if err != nil { - err = errors.New(cmdline[0] + ": " + err.Error()) - } - return buf.Bytes(), err - } -} - -// joinUnambiguously prints the slice, quoting where necessary to make the -// output unambiguous. -// TODO: See issue 5279. The printing of commands needs a complete redo. -func joinUnambiguously(a []string) string { - var buf bytes.Buffer - for i, s := range a { - if i > 0 { - buf.WriteByte(' ') - } - q := strconv.Quote(s) - if s == "" || strings.Contains(s, " ") || len(q) > len(s)+2 { - buf.WriteString(q) - } else { - buf.WriteString(s) - } - } - return buf.String() -} - -// mkdir makes the named directory. -func (b *builder) mkdir(dir string) error { - b.exec.Lock() - defer b.exec.Unlock() - // We can be a little aggressive about being - // sure directories exist. Skip repeated calls. - if b.mkdirCache[dir] { - return nil - } - b.mkdirCache[dir] = true - - if buildN || buildX { - b.showcmd("", "mkdir -p %s", dir) - if buildN { - return nil - } - } - - if err := os.MkdirAll(dir, 0777); err != nil { - return err - } - return nil -} - -// mkAbs returns an absolute path corresponding to -// evaluating f in the directory dir. -// We always pass absolute paths of source files so that -// the error messages will include the full path to a file -// in need of attention. -func mkAbs(dir, f string) string { - // Leave absolute paths alone. - // Also, during -n mode we use the pseudo-directory $WORK - // instead of creating an actual work directory that won't be used. - // Leave paths beginning with $WORK alone too. - if filepath.IsAbs(f) || strings.HasPrefix(f, "$WORK") { - return f - } - return filepath.Join(dir, f) -} - -type toolchain interface { - // gc runs the compiler in a specific directory on a set of files - // and returns the name of the generated output file. - gc(b *builder, p *Package, archive, obj string, asmhdr bool, importArgs []string, gofiles []string) (ofile string, out []byte, err error) - // cc runs the toolchain's C compiler in a directory on a C file - // to produce an output file. - cc(b *builder, p *Package, objdir, ofile, cfile string) error - // asm runs the assembler in a specific directory on specific files - // and returns a list of named output files. - asm(b *builder, p *Package, obj string, sfiles []string) ([]string, error) - // pkgpath builds an appropriate path for a temporary package file. - pkgpath(basedir string, p *Package) string - // pack runs the archive packer in a specific directory to create - // an archive from a set of object files. - // typically it is run in the object directory. - pack(b *builder, p *Package, objDir, afile string, ofiles []string) error - // ld runs the linker to create an executable starting at mainpkg. - ld(b *builder, root *action, out string, allactions []*action, mainpkg string, ofiles []string) error - // ldShared runs the linker to create a shared library containing the pkgs built by toplevelactions - ldShared(b *builder, toplevelactions []*action, out string, allactions []*action) error - - compiler() string - linker() string -} - -type noToolchain struct{} - -func noCompiler() error { - log.Fatalf("unknown compiler %q", buildContext.Compiler) - return nil -} - -func (noToolchain) compiler() string { - noCompiler() - return "" -} - -func (noToolchain) linker() string { - noCompiler() - return "" -} - -func (noToolchain) gc(b *builder, p *Package, archive, obj string, asmhdr bool, importArgs []string, gofiles []string) (ofile string, out []byte, err error) { - return "", nil, noCompiler() -} - -func (noToolchain) asm(b *builder, p *Package, obj string, sfiles []string) ([]string, error) { - return nil, noCompiler() -} - -func (noToolchain) pkgpath(basedir string, p *Package) string { - noCompiler() - return "" -} - -func (noToolchain) pack(b *builder, p *Package, objDir, afile string, ofiles []string) error { - return noCompiler() -} - -func (noToolchain) ld(b *builder, root *action, out string, allactions []*action, mainpkg string, ofiles []string) error { - return noCompiler() -} - -func (noToolchain) ldShared(b *builder, toplevelactions []*action, out string, allactions []*action) error { - return noCompiler() -} - -func (noToolchain) cc(b *builder, p *Package, objdir, ofile, cfile string) error { - return noCompiler() -} - -// The Go toolchain. -type gcToolchain struct{} - -func (gcToolchain) compiler() string { - return tool("compile") -} - -func (gcToolchain) linker() string { - return tool("link") -} - -func (gcToolchain) gc(b *builder, p *Package, archive, obj string, asmhdr bool, importArgs []string, gofiles []string) (ofile string, output []byte, err error) { - if archive != "" { - ofile = archive - } else { - out := "_go_.o" - ofile = obj + out - } - - gcargs := []string{"-p", p.ImportPath} - if p.Name == "main" { - gcargs[1] = "main" - } - if p.Standard && (p.ImportPath == "runtime" || strings.HasPrefix(p.ImportPath, "runtime/internal")) { - // runtime compiles with a special gc flag to emit - // additional reflect type data. - gcargs = append(gcargs, "-+") - } - - // If we're giving the compiler the entire package (no C etc files), tell it that, - // so that it can give good error messages about forward declarations. - // Exceptions: a few standard packages have forward declarations for - // pieces supplied behind-the-scenes by package runtime. - extFiles := len(p.CgoFiles) + len(p.CFiles) + len(p.CXXFiles) + len(p.MFiles) + len(p.FFiles) + len(p.SFiles) + len(p.SysoFiles) + len(p.SwigFiles) + len(p.SwigCXXFiles) - if p.Standard { - switch p.ImportPath { - case "bytes", "net", "os", "runtime/pprof", "sync", "time": - extFiles++ - } - } - if extFiles == 0 { - gcargs = append(gcargs, "-complete") - } - if buildContext.InstallSuffix != "" { - gcargs = append(gcargs, "-installsuffix", buildContext.InstallSuffix) - } - if p.buildID != "" { - gcargs = append(gcargs, "-buildid", p.buildID) - } - - for _, path := range p.Imports { - if i := strings.LastIndex(path, "/vendor/"); i >= 0 { - gcargs = append(gcargs, "-importmap", path[i+len("/vendor/"):]+"="+path) - } else if strings.HasPrefix(path, "vendor/") { - gcargs = append(gcargs, "-importmap", path[len("vendor/"):]+"="+path) - } - } - - args := []interface{}{buildToolExec, tool("compile"), "-o", ofile, "-trimpath", b.work, buildGcflags, gcargs, "-D", p.localPrefix, importArgs} - if ofile == archive { - args = append(args, "-pack") - } - if asmhdr { - args = append(args, "-asmhdr", obj+"go_asm.h") - } - for _, f := range gofiles { - args = append(args, mkAbs(p.Dir, f)) - } - - output, err = b.runOut(p.Dir, p.ImportPath, nil, args...) - return ofile, output, err -} - -func (gcToolchain) asm(b *builder, p *Package, obj string, sfiles []string) ([]string, error) { - // Add -I pkg/GOOS_GOARCH so #include "textflag.h" works in .s files. - inc := filepath.Join(goroot, "pkg", "include") - args := []interface{}{buildToolExec, tool("asm"), "-trimpath", b.work, "-I", obj, "-I", inc, "-D", "GOOS_" + goos, "-D", "GOARCH_" + goarch, buildAsmflags} - if p.ImportPath == "runtime" && goarch == "386" { - for _, arg := range buildAsmflags { - if arg == "-dynlink" { - args = append(args, "-D=GOBUILDMODE_shared=1") - } - } - } - var ofiles []string - for _, sfile := range sfiles { - ofile := obj + sfile[:len(sfile)-len(".s")] + ".o" - ofiles = append(ofiles, ofile) - a := append(args, "-o", ofile, mkAbs(p.Dir, sfile)) - if err := b.run(p.Dir, p.ImportPath, nil, a...); err != nil { - return nil, err - } - } - return ofiles, nil -} - -// toolVerify checks that the command line args writes the same output file -// if run using newTool instead. -// Unused now but kept around for future use. -func toolVerify(b *builder, p *Package, newTool string, ofile string, args []interface{}) error { - newArgs := make([]interface{}, len(args)) - copy(newArgs, args) - newArgs[1] = tool(newTool) - newArgs[3] = ofile + ".new" // x.6 becomes x.6.new - if err := b.run(p.Dir, p.ImportPath, nil, newArgs...); err != nil { - return err - } - data1, err := ioutil.ReadFile(ofile) - if err != nil { - return err - } - data2, err := ioutil.ReadFile(ofile + ".new") - if err != nil { - return err - } - if !bytes.Equal(data1, data2) { - return fmt.Errorf("%s and %s produced different output files:\n%s\n%s", filepath.Base(args[1].(string)), newTool, strings.Join(stringList(args...), " "), strings.Join(stringList(newArgs...), " ")) - } - os.Remove(ofile + ".new") - return nil -} - -func (gcToolchain) pkgpath(basedir string, p *Package) string { - end := filepath.FromSlash(p.ImportPath + ".a") - return filepath.Join(basedir, end) -} - -func (gcToolchain) pack(b *builder, p *Package, objDir, afile string, ofiles []string) error { - var absOfiles []string - for _, f := range ofiles { - absOfiles = append(absOfiles, mkAbs(objDir, f)) - } - absAfile := mkAbs(objDir, afile) - - // The archive file should have been created by the compiler. - // Since it used to not work that way, verify. - if !buildN { - if _, err := os.Stat(absAfile); err != nil { - fatalf("os.Stat of archive file failed: %v", err) - } - } - - if buildN || buildX { - cmdline := stringList("pack", "r", absAfile, absOfiles) - b.showcmd(p.Dir, "%s # internal", joinUnambiguously(cmdline)) - } - if buildN { - return nil - } - if err := packInternal(b, absAfile, absOfiles); err != nil { - b.showOutput(p.Dir, p.ImportPath, err.Error()+"\n") - return errPrintedOutput - } - return nil -} - -func packInternal(b *builder, afile string, ofiles []string) error { - dst, err := os.OpenFile(afile, os.O_WRONLY|os.O_APPEND, 0) - if err != nil { - return err - } - defer dst.Close() // only for error returns or panics - w := bufio.NewWriter(dst) - - for _, ofile := range ofiles { - src, err := os.Open(ofile) - if err != nil { - return err - } - fi, err := src.Stat() - if err != nil { - src.Close() - return err - } - // Note: Not using %-16.16s format because we care - // about bytes, not runes. - name := fi.Name() - if len(name) > 16 { - name = name[:16] - } else { - name += strings.Repeat(" ", 16-len(name)) - } - size := fi.Size() - fmt.Fprintf(w, "%s%-12d%-6d%-6d%-8o%-10d`\n", - name, 0, 0, 0, 0644, size) - n, err := io.Copy(w, src) - src.Close() - if err == nil && n < size { - err = io.ErrUnexpectedEOF - } else if err == nil && n > size { - err = fmt.Errorf("file larger than size reported by stat") - } - if err != nil { - return fmt.Errorf("copying %s to %s: %v", ofile, afile, err) - } - if size&1 != 0 { - w.WriteByte(0) - } - } - - if err := w.Flush(); err != nil { - return err - } - return dst.Close() -} - -// setextld sets the appropriate linker flags for the specified compiler. -func setextld(ldflags []string, compiler []string) []string { - for _, f := range ldflags { - if f == "-extld" || strings.HasPrefix(f, "-extld=") { - // don't override -extld if supplied - return ldflags - } - } - ldflags = append(ldflags, "-extld="+compiler[0]) - if len(compiler) > 1 { - extldflags := false - add := strings.Join(compiler[1:], " ") - for i, f := range ldflags { - if f == "-extldflags" && i+1 < len(ldflags) { - ldflags[i+1] = add + " " + ldflags[i+1] - extldflags = true - break - } else if strings.HasPrefix(f, "-extldflags=") { - ldflags[i] = "-extldflags=" + add + " " + ldflags[i][len("-extldflags="):] - extldflags = true - break - } - } - if !extldflags { - ldflags = append(ldflags, "-extldflags="+add) - } - } - return ldflags -} - -func (gcToolchain) ld(b *builder, root *action, out string, allactions []*action, mainpkg string, ofiles []string) error { - importArgs := b.includeArgs("-L", allactions) - cxx := len(root.p.CXXFiles) > 0 || len(root.p.SwigCXXFiles) > 0 - for _, a := range allactions { - if a.p != nil && (len(a.p.CXXFiles) > 0 || len(a.p.SwigCXXFiles) > 0) { - cxx = true - } - } - var ldflags []string - if buildContext.InstallSuffix != "" { - ldflags = append(ldflags, "-installsuffix", buildContext.InstallSuffix) - } - if root.p.omitDWARF { - ldflags = append(ldflags, "-w") - } - if buildBuildmode == "plugin" { - pluginpath := root.p.ImportPath - if pluginpath == "command-line-arguments" { - pluginpath = "plugin/unnamed-" + root.p.buildID - } - ldflags = append(ldflags, "-pluginpath", pluginpath) - } - - // If the user has not specified the -extld option, then specify the - // appropriate linker. In case of C++ code, use the compiler named - // by the CXX environment variable or defaultCXX if CXX is not set. - // Else, use the CC environment variable and defaultCC as fallback. - var compiler []string - if cxx { - compiler = envList("CXX", defaultCXX) - } else { - compiler = envList("CC", defaultCC) - } - ldflags = setextld(ldflags, compiler) - ldflags = append(ldflags, "-buildmode="+ldBuildmode) - if root.p.buildID != "" { - ldflags = append(ldflags, "-buildid="+root.p.buildID) - } - ldflags = append(ldflags, buildLdflags...) - - // On OS X when using external linking to build a shared library, - // the argument passed here to -o ends up recorded in the final - // shared library in the LC_ID_DYLIB load command. - // To avoid putting the temporary output directory name there - // (and making the resulting shared library useless), - // run the link in the output directory so that -o can name - // just the final path element. - dir := "." - if goos == "darwin" && buildBuildmode == "c-shared" { - dir, out = filepath.Split(out) - } - - return b.run(dir, root.p.ImportPath, nil, buildToolExec, tool("link"), "-o", out, importArgs, ldflags, mainpkg) -} - -func (gcToolchain) ldShared(b *builder, toplevelactions []*action, out string, allactions []*action) error { - importArgs := b.includeArgs("-L", allactions) - ldflags := []string{"-installsuffix", buildContext.InstallSuffix} - ldflags = append(ldflags, "-buildmode=shared") - ldflags = append(ldflags, buildLdflags...) - cxx := false - for _, a := range allactions { - if a.p != nil && (len(a.p.CXXFiles) > 0 || len(a.p.SwigCXXFiles) > 0) { - cxx = true - } - } - // If the user has not specified the -extld option, then specify the - // appropriate linker. In case of C++ code, use the compiler named - // by the CXX environment variable or defaultCXX if CXX is not set. - // Else, use the CC environment variable and defaultCC as fallback. - var compiler []string - if cxx { - compiler = envList("CXX", defaultCXX) - } else { - compiler = envList("CC", defaultCC) - } - ldflags = setextld(ldflags, compiler) - for _, d := range toplevelactions { - if !strings.HasSuffix(d.target, ".a") { // omit unsafe etc and actions for other shared libraries - continue - } - ldflags = append(ldflags, d.p.ImportPath+"="+d.target) - } - return b.run(".", out, nil, buildToolExec, tool("link"), "-o", out, importArgs, ldflags) -} - -func (gcToolchain) cc(b *builder, p *Package, objdir, ofile, cfile string) error { - return fmt.Errorf("%s: C source files not supported without cgo", mkAbs(p.Dir, cfile)) -} - -// The Gccgo toolchain. -type gccgoToolchain struct{} - -var gccgoName, gccgoBin string - -func init() { - gccgoName = os.Getenv("GCCGO") - if gccgoName == "" { - gccgoName = defaultGCCGO - } - gccgoBin, _ = exec.LookPath(gccgoName) -} - -func (gccgoToolchain) compiler() string { - return gccgoBin -} - -func (gccgoToolchain) linker() string { - return gccgoBin -} - -func (tools gccgoToolchain) gc(b *builder, p *Package, archive, obj string, asmhdr bool, importArgs []string, gofiles []string) (ofile string, output []byte, err error) { - out := "_go_.o" - ofile = obj + out - gcargs := []string{"-g"} - gcargs = append(gcargs, b.gccArchArgs()...) - gcargs = append(gcargs, "-fdebug-prefix-map="+b.work+"=/tmp/go-build") - gcargs = append(gcargs, "-gno-record-gcc-switches") - if pkgpath := gccgoPkgpath(p); pkgpath != "" { - gcargs = append(gcargs, "-fgo-pkgpath="+pkgpath) - } - if p.localPrefix != "" { - gcargs = append(gcargs, "-fgo-relative-import-path="+p.localPrefix) - } - savedirs := []string{} - for _, incdir := range importArgs { - if incdir != "-I" { - savedirs = append(savedirs, incdir) - } - } - - for _, path := range p.Imports { - // If this is a new vendor path, add it to the list of importArgs - if i := strings.LastIndex(path, "/vendor/"); i >= 0 { - for _, dir := range savedirs { - // Check if the vendor path is already included in dir - if strings.HasSuffix(dir, path[:i+len("/vendor/")]) { - continue - } - // Make sure this vendor path is not already in the list for importArgs - vendorPath := dir + "/" + path[:i+len("/vendor/")] - for _, imp := range importArgs { - if imp == "-I" { - continue - } - // This vendorPath is already in the list - if imp == vendorPath { - goto nextSuffixPath - } - } - // New vendorPath not yet in the importArgs list, so add it - importArgs = append(importArgs, "-I", vendorPath) - nextSuffixPath: - } - } else if strings.HasPrefix(path, "vendor/") { - for _, dir := range savedirs { - // Make sure this vendor path is not already in the list for importArgs - vendorPath := dir + "/" + path[len("/vendor"):] - for _, imp := range importArgs { - if imp == "-I" { - continue - } - if imp == vendorPath { - goto nextPrefixPath - } - } - // This vendor path is needed and not already in the list, so add it - importArgs = append(importArgs, "-I", vendorPath) - nextPrefixPath: - } - } - } - - args := stringList(tools.compiler(), importArgs, "-c", gcargs, "-o", ofile, buildGccgoflags) - for _, f := range gofiles { - args = append(args, mkAbs(p.Dir, f)) - } - - output, err = b.runOut(p.Dir, p.ImportPath, nil, args) - return ofile, output, err -} - -func (tools gccgoToolchain) asm(b *builder, p *Package, obj string, sfiles []string) ([]string, error) { - var ofiles []string - for _, sfile := range sfiles { - ofile := obj + sfile[:len(sfile)-len(".s")] + ".o" - ofiles = append(ofiles, ofile) - sfile = mkAbs(p.Dir, sfile) - defs := []string{"-D", "GOOS_" + goos, "-D", "GOARCH_" + goarch} - if pkgpath := gccgoCleanPkgpath(p); pkgpath != "" { - defs = append(defs, `-D`, `GOPKGPATH=`+pkgpath) - } - defs = tools.maybePIC(defs) - defs = append(defs, b.gccArchArgs()...) - err := b.run(p.Dir, p.ImportPath, nil, tools.compiler(), "-xassembler-with-cpp", "-I", obj, "-c", "-o", ofile, defs, sfile) - if err != nil { - return nil, err - } - } - return ofiles, nil -} - -func (gccgoToolchain) pkgpath(basedir string, p *Package) string { - end := filepath.FromSlash(p.ImportPath + ".a") - afile := filepath.Join(basedir, end) - // add "lib" to the final element - return filepath.Join(filepath.Dir(afile), "lib"+filepath.Base(afile)) -} - -func (gccgoToolchain) pack(b *builder, p *Package, objDir, afile string, ofiles []string) error { - var absOfiles []string - for _, f := range ofiles { - absOfiles = append(absOfiles, mkAbs(objDir, f)) - } - absAfile := mkAbs(objDir, afile) - // Try with D modifier first, then without if that fails. - if b.run(p.Dir, p.ImportPath, nil, "ar", "rcD", absAfile, absOfiles) != nil { - return b.run(p.Dir, p.ImportPath, nil, "ar", "rc", absAfile, absOfiles) - } - return nil -} - -func (tools gccgoToolchain) link(b *builder, root *action, out string, allactions []*action, mainpkg string, ofiles []string, buildmode, desc string) error { - // gccgo needs explicit linking with all package dependencies, - // and all LDFLAGS from cgo dependencies. - apackagePathsSeen := make(map[string]bool) - afiles := []string{} - shlibs := []string{} - ldflags := b.gccArchArgs() - cgoldflags := []string{} - usesCgo := false - cxx := false - objc := false - fortran := false - if root.p != nil { - cxx = len(root.p.CXXFiles) > 0 || len(root.p.SwigCXXFiles) > 0 - objc = len(root.p.MFiles) > 0 - fortran = len(root.p.FFiles) > 0 - } - - readCgoFlags := func(flagsFile string) error { - flags, err := ioutil.ReadFile(flagsFile) - if err != nil { - return err - } - const ldflagsPrefix = "_CGO_LDFLAGS=" - for _, line := range strings.Split(string(flags), "\n") { - if strings.HasPrefix(line, ldflagsPrefix) { - newFlags := strings.Fields(line[len(ldflagsPrefix):]) - for _, flag := range newFlags { - // Every _cgo_flags file has -g and -O2 in _CGO_LDFLAGS - // but they don't mean anything to the linker so filter - // them out. - if flag != "-g" && !strings.HasPrefix(flag, "-O") { - cgoldflags = append(cgoldflags, flag) - } - } - } - } - return nil - } - - readAndRemoveCgoFlags := func(archive string) (string, error) { - newa, err := ioutil.TempFile(b.work, filepath.Base(archive)) - if err != nil { - return "", err - } - olda, err := os.Open(archive) - if err != nil { - return "", err - } - _, err = io.Copy(newa, olda) - if err != nil { - return "", err - } - err = olda.Close() - if err != nil { - return "", err - } - err = newa.Close() - if err != nil { - return "", err - } - - newarchive := newa.Name() - err = b.run(b.work, desc, nil, "ar", "x", newarchive, "_cgo_flags") - if err != nil { - return "", err - } - err = b.run(".", desc, nil, "ar", "d", newarchive, "_cgo_flags") - if err != nil { - return "", err - } - err = readCgoFlags(filepath.Join(b.work, "_cgo_flags")) - if err != nil { - return "", err - } - return newarchive, nil - } - - actionsSeen := make(map[*action]bool) - // Make a pre-order depth-first traversal of the action graph, taking note of - // whether a shared library action has been seen on the way to an action (the - // construction of the graph means that if any path to a node passes through - // a shared library action, they all do). - var walk func(a *action, seenShlib bool) - var err error - walk = func(a *action, seenShlib bool) { - if actionsSeen[a] { - return - } - actionsSeen[a] = true - if a.p != nil && !seenShlib { - if a.p.Standard { - return - } - // We record the target of the first time we see a .a file - // for a package to make sure that we prefer the 'install' - // rather than the 'build' location (which may not exist any - // more). We still need to traverse the dependencies of the - // build action though so saying - // if apackagePathsSeen[a.p.ImportPath] { return } - // doesn't work. - if !apackagePathsSeen[a.p.ImportPath] { - apackagePathsSeen[a.p.ImportPath] = true - target := a.target - if len(a.p.CgoFiles) > 0 || a.p.usesSwig() { - target, err = readAndRemoveCgoFlags(target) - if err != nil { - return - } - } - afiles = append(afiles, target) - } - } - if strings.HasSuffix(a.target, ".so") { - shlibs = append(shlibs, a.target) - seenShlib = true - } - for _, a1 := range a.deps { - walk(a1, seenShlib) - if err != nil { - return - } - } - } - for _, a1 := range root.deps { - walk(a1, false) - if err != nil { - return err - } - } - - for _, a := range allactions { - // Gather CgoLDFLAGS, but not from standard packages. - // The go tool can dig up runtime/cgo from GOROOT and - // think that it should use its CgoLDFLAGS, but gccgo - // doesn't use runtime/cgo. - if a.p == nil { - continue - } - if !a.p.Standard { - cgoldflags = append(cgoldflags, a.p.CgoLDFLAGS...) - } - if len(a.p.CgoFiles) > 0 { - usesCgo = true - } - if a.p.usesSwig() { - usesCgo = true - } - if len(a.p.CXXFiles) > 0 || len(a.p.SwigCXXFiles) > 0 { - cxx = true - } - if len(a.p.MFiles) > 0 { - objc = true - } - if len(a.p.FFiles) > 0 { - fortran = true - } - } - - for i, o := range ofiles { - if filepath.Base(o) == "_cgo_flags" { - readCgoFlags(o) - ofiles = append(ofiles[:i], ofiles[i+1:]...) - break - } - } - - ldflags = append(ldflags, "-Wl,--whole-archive") - ldflags = append(ldflags, afiles...) - ldflags = append(ldflags, "-Wl,--no-whole-archive") - - ldflags = append(ldflags, cgoldflags...) - ldflags = append(ldflags, envList("CGO_LDFLAGS", "")...) - if root.p != nil { - ldflags = append(ldflags, root.p.CgoLDFLAGS...) - } - - ldflags = stringList("-Wl,-(", ldflags, "-Wl,-)") - - for _, shlib := range shlibs { - ldflags = append( - ldflags, - "-L"+filepath.Dir(shlib), - "-Wl,-rpath="+filepath.Dir(shlib), - "-l"+strings.TrimSuffix( - strings.TrimPrefix(filepath.Base(shlib), "lib"), - ".so")) - } - - var realOut string - switch buildmode { - case "exe": - if usesCgo && goos == "linux" { - ldflags = append(ldflags, "-Wl,-E") - } - - case "c-archive": - // Link the Go files into a single .o, and also link - // in -lgolibbegin. - // - // We need to use --whole-archive with -lgolibbegin - // because it doesn't define any symbols that will - // cause the contents to be pulled in; it's just - // initialization code. - // - // The user remains responsible for linking against - // -lgo -lpthread -lm in the final link. We can't use - // -r to pick them up because we can't combine - // split-stack and non-split-stack code in a single -r - // link, and libgo picks up non-split-stack code from - // libffi. - ldflags = append(ldflags, "-Wl,-r", "-nostdlib", "-Wl,--whole-archive", "-lgolibbegin", "-Wl,--no-whole-archive") - - if b.gccSupportsNoPie() { - ldflags = append(ldflags, "-no-pie") - } - - // We are creating an object file, so we don't want a build ID. - ldflags = b.disableBuildID(ldflags) - - realOut = out - out = out + ".o" - - case "c-shared": - ldflags = append(ldflags, "-shared", "-nostdlib", "-Wl,--whole-archive", "-lgolibbegin", "-Wl,--no-whole-archive", "-lgo", "-lgcc_s", "-lgcc", "-lc", "-lgcc") - case "shared": - ldflags = append(ldflags, "-zdefs", "-shared", "-nostdlib", "-lgo", "-lgcc_s", "-lgcc", "-lc") - case "pie": - ldflags = append(ldflags, "-pie") - - default: - fatalf("-buildmode=%s not supported for gccgo", buildmode) - } - - switch buildmode { - case "exe", "c-shared": - if cxx { - ldflags = append(ldflags, "-lstdc++") - } - if objc { - ldflags = append(ldflags, "-lobjc") - } - if fortran { - fc := os.Getenv("FC") - if fc == "" { - fc = "gfortran" - } - // support gfortran out of the box and let others pass the correct link options - // via CGO_LDFLAGS - if strings.Contains(fc, "gfortran") { - ldflags = append(ldflags, "-lgfortran") - } - } - } - - if err := b.run(".", desc, nil, tools.linker(), "-o", out, ofiles, ldflags, buildGccgoflags); err != nil { - return err - } - - switch buildmode { - case "c-archive": - if err := b.run(".", desc, nil, "ar", "rc", realOut, out); err != nil { - return err - } - } - return nil -} - -func (tools gccgoToolchain) ld(b *builder, root *action, out string, allactions []*action, mainpkg string, ofiles []string) error { - return tools.link(b, root, out, allactions, mainpkg, ofiles, ldBuildmode, root.p.ImportPath) -} - -func (tools gccgoToolchain) ldShared(b *builder, toplevelactions []*action, out string, allactions []*action) error { - fakeRoot := &action{} - fakeRoot.deps = toplevelactions - return tools.link(b, fakeRoot, out, allactions, "", nil, "shared", out) -} - -func (tools gccgoToolchain) cc(b *builder, p *Package, objdir, ofile, cfile string) error { - inc := filepath.Join(goroot, "pkg", "include") - cfile = mkAbs(p.Dir, cfile) - defs := []string{"-D", "GOOS_" + goos, "-D", "GOARCH_" + goarch} - defs = append(defs, b.gccArchArgs()...) - if pkgpath := gccgoCleanPkgpath(p); pkgpath != "" { - defs = append(defs, `-D`, `GOPKGPATH="`+pkgpath+`"`) - } - if b.gccSupportsFlag("-fsplit-stack") { - defs = append(defs, "-fsplit-stack") - } - defs = tools.maybePIC(defs) - if b.gccSupportsFlag("-fdebug-prefix-map=a=b") { - defs = append(defs, "-fdebug-prefix-map="+b.work+"=/tmp/go-build") - } - if b.gccSupportsFlag("-gno-record-gcc-switches") { - defs = append(defs, "-gno-record-gcc-switches") - } - return b.run(p.Dir, p.ImportPath, nil, envList("CC", defaultCC), "-Wall", "-g", - "-I", objdir, "-I", inc, "-o", ofile, defs, "-c", cfile) -} - -// maybePIC adds -fPIC to the list of arguments if needed. -func (tools gccgoToolchain) maybePIC(args []string) []string { - switch buildBuildmode { - case "c-archive", "c-shared", "shared", "plugin": - args = append(args, "-fPIC") - } - return args -} - -func gccgoPkgpath(p *Package) string { - if p.build.IsCommand() && !p.forceLibrary { - return "" - } - return p.ImportPath -} - -func gccgoCleanPkgpath(p *Package) string { - clean := func(r rune) rune { - switch { - case 'A' <= r && r <= 'Z', 'a' <= r && r <= 'z', - '0' <= r && r <= '9': - return r - } - return '_' - } - return strings.Map(clean, gccgoPkgpath(p)) -} - -// gcc runs the gcc C compiler to create an object from a single C file. -func (b *builder) gcc(p *Package, out string, flags []string, cfile string) error { - return b.ccompile(p, out, flags, cfile, b.gccCmd(p.Dir)) -} - -// gxx runs the g++ C++ compiler to create an object from a single C++ file. -func (b *builder) gxx(p *Package, out string, flags []string, cxxfile string) error { - return b.ccompile(p, out, flags, cxxfile, b.gxxCmd(p.Dir)) -} - -// gfortran runs the gfortran Fortran compiler to create an object from a single Fortran file. -func (b *builder) gfortran(p *Package, out string, flags []string, ffile string) error { - return b.ccompile(p, out, flags, ffile, b.gfortranCmd(p.Dir)) -} - -// ccompile runs the given C or C++ compiler and creates an object from a single source file. -func (b *builder) ccompile(p *Package, outfile string, flags []string, file string, compiler []string) error { - file = mkAbs(p.Dir, file) - desc := p.ImportPath - output, err := b.runOut(p.Dir, desc, nil, compiler, flags, "-o", outfile, "-c", file) - if len(output) > 0 { - // On FreeBSD 11, when we pass -g to clang 3.8 it - // invokes its internal assembler with -dwarf-version=2. - // When it sees .section .note.GNU-stack, it warns - // "DWARF2 only supports one section per compilation unit". - // This warning makes no sense, since the section is empty, - // but it confuses people. - // We work around the problem by detecting the warning - // and dropping -g and trying again. - if bytes.Contains(output, []byte("DWARF2 only supports one section per compilation unit")) { - newFlags := make([]string, 0, len(flags)) - for _, f := range flags { - if !strings.HasPrefix(f, "-g") { - newFlags = append(newFlags, f) - } - } - if len(newFlags) < len(flags) { - return b.ccompile(p, outfile, newFlags, file, compiler) - } - } - - b.showOutput(p.Dir, desc, b.processOutput(output)) - if err != nil { - err = errPrintedOutput - } else if os.Getenv("GO_BUILDER_NAME") != "" { - return errors.New("C compiler warning promoted to error on Go builders") - } - } - return err -} - -// gccld runs the gcc linker to create an executable from a set of object files. -func (b *builder) gccld(p *Package, out string, flags []string, obj []string) error { - var cmd []string - if len(p.CXXFiles) > 0 || len(p.SwigCXXFiles) > 0 { - cmd = b.gxxCmd(p.Dir) - } else { - cmd = b.gccCmd(p.Dir) - } - return b.run(p.Dir, p.ImportPath, nil, cmd, "-o", out, obj, flags) -} - -// gccCmd returns a gcc command line prefix -// defaultCC is defined in zdefaultcc.go, written by cmd/dist. -func (b *builder) gccCmd(objdir string) []string { - return b.ccompilerCmd("CC", defaultCC, objdir) -} - -// gxxCmd returns a g++ command line prefix -// defaultCXX is defined in zdefaultcc.go, written by cmd/dist. -func (b *builder) gxxCmd(objdir string) []string { - return b.ccompilerCmd("CXX", defaultCXX, objdir) -} - -// gfortranCmd returns a gfortran command line prefix. -func (b *builder) gfortranCmd(objdir string) []string { - return b.ccompilerCmd("FC", "gfortran", objdir) -} - -// ccompilerCmd returns a command line prefix for the given environment -// variable and using the default command when the variable is empty. -func (b *builder) ccompilerCmd(envvar, defcmd, objdir string) []string { - // NOTE: env.go's mkEnv knows that the first three - // strings returned are "gcc", "-I", objdir (and cuts them off). - - compiler := envList(envvar, defcmd) - a := []string{compiler[0], "-I", objdir} - a = append(a, compiler[1:]...) - - // Definitely want -fPIC but on Windows gcc complains - // "-fPIC ignored for target (all code is position independent)" - if goos != "windows" { - a = append(a, "-fPIC") - } - a = append(a, b.gccArchArgs()...) - // gcc-4.5 and beyond require explicit "-pthread" flag - // for multithreading with pthread library. - if buildContext.CgoEnabled { - switch goos { - case "windows": - a = append(a, "-mthreads") - default: - a = append(a, "-pthread") - } - } - - if strings.Contains(a[0], "clang") { - // disable ASCII art in clang errors, if possible - a = append(a, "-fno-caret-diagnostics") - // clang is too smart about command-line arguments - a = append(a, "-Qunused-arguments") - } - - // disable word wrapping in error messages - a = append(a, "-fmessage-length=0") - - // Tell gcc not to include the work directory in object files. - if b.gccSupportsFlag("-fdebug-prefix-map=a=b") { - a = append(a, "-fdebug-prefix-map="+b.work+"=/tmp/go-build") - } - - // Tell gcc not to include flags in object files, which defeats the - // point of -fdebug-prefix-map above. - if b.gccSupportsFlag("-gno-record-gcc-switches") { - a = append(a, "-gno-record-gcc-switches") - } - - // On OS X, some of the compilers behave as if -fno-common - // is always set, and the Mach-O linker in 6l/8l assumes this. - // See https://golang.org/issue/3253. - if goos == "darwin" { - a = append(a, "-fno-common") - } - - // gccgo uses the language-independent exception mechanism to - // handle panics, so it always needs unwind tables. - if _, ok := buildToolchain.(gccgoToolchain); ok { - a = append(a, "-funwind-tables") - } - - return a -} - -// On systems with PIE (position independent executables) enabled by default, -// -no-pie must be passed when doing a partial link with -Wl,-r. But -no-pie is -// not supported by all compilers. -func (b *builder) gccSupportsNoPie() bool { - return b.gccSupportsFlag("-no-pie") -} - -// gccSupportsFlag checks to see if the compiler supports a flag. -func (b *builder) gccSupportsFlag(flag string) bool { - b.exec.Lock() - defer b.exec.Unlock() - if b, ok := b.flagCache[flag]; ok { - return b - } - if b.flagCache == nil { - src := filepath.Join(b.work, "trivial.c") - if err := ioutil.WriteFile(src, []byte{}, 0666); err != nil { - return false - } - b.flagCache = make(map[string]bool) - } - cmdArgs := append(envList("CC", defaultCC), flag, "-c", "trivial.c") - if buildN || buildX { - b.showcmd(b.work, "%s", joinUnambiguously(cmdArgs)) - if buildN { - return false - } - } - cmd := exec.Command(cmdArgs[0], cmdArgs[1:]...) - cmd.Dir = b.work - cmd.Env = mergeEnvLists([]string{"LC_ALL=C"}, envForDir(cmd.Dir, os.Environ())) - out, err := cmd.CombinedOutput() - supported := err == nil && !bytes.Contains(out, []byte("unrecognized")) - b.flagCache[flag] = supported - return supported -} - -// gccArchArgs returns arguments to pass to gcc based on the architecture. -func (b *builder) gccArchArgs() []string { - switch goarch { - case "386": - return []string{"-m32"} - case "amd64", "amd64p32": - return []string{"-m64"} - case "arm": - return []string{"-marm"} // not thumb - case "s390x": - return []string{"-m64", "-march=z196"} - case "mips64", "mips64le": - return []string{"-mabi=64"} - case "mips", "mipsle": - return []string{"-mabi=32", "-march=mips32"} - } - return nil -} - -// envList returns the value of the given environment variable broken -// into fields, using the default value when the variable is empty. -func envList(key, def string) []string { - v := os.Getenv(key) - if v == "" { - v = def - } - return strings.Fields(v) -} - -// Return the flags to use when invoking the C, C++ or Fortran compilers, or cgo. -func (b *builder) cflags(p *Package) (cppflags, cflags, cxxflags, fflags, ldflags []string) { - defaults := "-g -O2" - - cppflags = stringList(envList("CGO_CPPFLAGS", ""), p.CgoCPPFLAGS) - cflags = stringList(envList("CGO_CFLAGS", defaults), p.CgoCFLAGS) - cxxflags = stringList(envList("CGO_CXXFLAGS", defaults), p.CgoCXXFLAGS) - fflags = stringList(envList("CGO_FFLAGS", defaults), p.CgoFFLAGS) - ldflags = stringList(envList("CGO_LDFLAGS", defaults), p.CgoLDFLAGS) - return -} - -var cgoRe = regexp.MustCompile(`[/\\:]`) - -func (b *builder) cgo(a *action, cgoExe, obj string, pcCFLAGS, pcLDFLAGS, cgofiles, objdirCgofiles, gccfiles, gxxfiles, mfiles, ffiles []string) (outGo, outObj []string, err error) { - p := a.p - cgoCPPFLAGS, cgoCFLAGS, cgoCXXFLAGS, cgoFFLAGS, cgoLDFLAGS := b.cflags(p) - cgoCPPFLAGS = append(cgoCPPFLAGS, pcCFLAGS...) - cgoLDFLAGS = append(cgoLDFLAGS, pcLDFLAGS...) - // If we are compiling Objective-C code, then we need to link against libobjc - if len(mfiles) > 0 { - cgoLDFLAGS = append(cgoLDFLAGS, "-lobjc") - } - - // Likewise for Fortran, except there are many Fortran compilers. - // Support gfortran out of the box and let others pass the correct link options - // via CGO_LDFLAGS - if len(ffiles) > 0 { - fc := os.Getenv("FC") - if fc == "" { - fc = "gfortran" - } - if strings.Contains(fc, "gfortran") { - cgoLDFLAGS = append(cgoLDFLAGS, "-lgfortran") - } - } - - if buildMSan { - cgoCFLAGS = append([]string{"-fsanitize=memory"}, cgoCFLAGS...) - cgoLDFLAGS = append([]string{"-fsanitize=memory"}, cgoLDFLAGS...) - } - - // Allows including _cgo_export.h from .[ch] files in the package. - cgoCPPFLAGS = append(cgoCPPFLAGS, "-I", obj) - - // If we have cgo files in the object directory, then copy any - // other cgo files into the object directory, and pass a - // -srcdir option to cgo. - var srcdirarg []string - if len(objdirCgofiles) > 0 { - for _, fn := range cgofiles { - if err := b.copyFile(a, obj+filepath.Base(fn), filepath.Join(p.Dir, fn), 0666, false); err != nil { - return nil, nil, err - } - } - cgofiles = append(cgofiles, objdirCgofiles...) - srcdirarg = []string{"-srcdir", obj} - } - - // cgo - // TODO: CGO_FLAGS? - gofiles := []string{obj + "_cgo_gotypes.go"} - cfiles := []string{"_cgo_export.c"} - for _, fn := range cgofiles { - f := cgoRe.ReplaceAllString(fn[:len(fn)-2], "_") - gofiles = append(gofiles, obj+f+"cgo1.go") - cfiles = append(cfiles, f+"cgo2.c") - } - - // TODO: make cgo not depend on $GOARCH? - - cgoflags := []string{} - if p.Standard && p.ImportPath == "runtime/cgo" { - cgoflags = append(cgoflags, "-import_runtime_cgo=false") - } - if p.Standard && (p.ImportPath == "runtime/race" || p.ImportPath == "runtime/msan" || p.ImportPath == "runtime/cgo") { - cgoflags = append(cgoflags, "-import_syscall=false") - } - - // Update $CGO_LDFLAGS with p.CgoLDFLAGS. - var cgoenv []string - if len(cgoLDFLAGS) > 0 { - flags := make([]string, len(cgoLDFLAGS)) - for i, f := range cgoLDFLAGS { - flags[i] = strconv.Quote(f) - } - cgoenv = []string{"CGO_LDFLAGS=" + strings.Join(flags, " ")} - } - - if _, ok := buildToolchain.(gccgoToolchain); ok { - if b.gccSupportsFlag("-fsplit-stack") { - cgoCFLAGS = append(cgoCFLAGS, "-fsplit-stack") - } - cgoflags = append(cgoflags, "-gccgo") - if pkgpath := gccgoPkgpath(p); pkgpath != "" { - cgoflags = append(cgoflags, "-gccgopkgpath="+pkgpath) - } - } - - switch buildBuildmode { - case "c-archive", "c-shared": - // Tell cgo that if there are any exported functions - // it should generate a header file that C code can - // #include. - cgoflags = append(cgoflags, "-exportheader="+obj+"_cgo_install.h") - } - - if err := b.run(p.Dir, p.ImportPath, cgoenv, buildToolExec, cgoExe, srcdirarg, "-objdir", obj, "-importpath", p.ImportPath, cgoflags, "--", cgoCPPFLAGS, cgoCFLAGS, cgofiles); err != nil { - return nil, nil, err - } - outGo = append(outGo, gofiles...) - - // gcc - cflags := stringList(cgoCPPFLAGS, cgoCFLAGS) - for _, cfile := range cfiles { - ofile := obj + cfile[:len(cfile)-1] + "o" - if err := b.gcc(p, ofile, cflags, obj+cfile); err != nil { - return nil, nil, err - } - outObj = append(outObj, ofile) - } - - for _, file := range gccfiles { - base := filepath.Base(file) - ofile := obj + cgoRe.ReplaceAllString(base[:len(base)-1], "_") + "o" - if err := b.gcc(p, ofile, cflags, file); err != nil { - return nil, nil, err - } - outObj = append(outObj, ofile) - } - - cxxflags := stringList(cgoCPPFLAGS, cgoCXXFLAGS) - for _, file := range gxxfiles { - // Append .o to the file, just in case the pkg has file.c and file.cpp - ofile := obj + cgoRe.ReplaceAllString(filepath.Base(file), "_") + ".o" - if err := b.gxx(p, ofile, cxxflags, file); err != nil { - return nil, nil, err - } - outObj = append(outObj, ofile) - } - - for _, file := range mfiles { - // Append .o to the file, just in case the pkg has file.c and file.m - ofile := obj + cgoRe.ReplaceAllString(filepath.Base(file), "_") + ".o" - if err := b.gcc(p, ofile, cflags, file); err != nil { - return nil, nil, err - } - outObj = append(outObj, ofile) - } - - fflags := stringList(cgoCPPFLAGS, cgoFFLAGS) - for _, file := range ffiles { - // Append .o to the file, just in case the pkg has file.c and file.f - ofile := obj + cgoRe.ReplaceAllString(filepath.Base(file), "_") + ".o" - if err := b.gfortran(p, ofile, fflags, file); err != nil { - return nil, nil, err - } - outObj = append(outObj, ofile) - } - - switch buildToolchain.(type) { - case gcToolchain: - importGo := obj + "_cgo_import.go" - if err := b.dynimport(p, obj, importGo, cgoExe, cflags, cgoLDFLAGS, outObj); err != nil { - return nil, nil, err - } - outGo = append(outGo, importGo) - - ofile := obj + "_all.o" - if err := b.collect(p, obj, ofile, cgoLDFLAGS, outObj); err != nil { - return nil, nil, err - } - outObj = []string{ofile} - - case gccgoToolchain: - defunC := obj + "_cgo_defun.c" - defunObj := obj + "_cgo_defun.o" - if err := buildToolchain.cc(b, p, obj, defunObj, defunC); err != nil { - return nil, nil, err - } - outObj = append(outObj, defunObj) - - default: - noCompiler() - } - - return outGo, outObj, nil -} - -// dynimport creates a Go source file named importGo containing -// //go:cgo_import_dynamic directives for each symbol or library -// dynamically imported by the object files outObj. -func (b *builder) dynimport(p *Package, obj, importGo, cgoExe string, cflags, cgoLDFLAGS, outObj []string) error { - cfile := obj + "_cgo_main.c" - ofile := obj + "_cgo_main.o" - if err := b.gcc(p, ofile, cflags, cfile); err != nil { - return err - } - - linkobj := stringList(ofile, outObj, p.SysoFiles) - dynobj := obj + "_cgo_.o" - - // we need to use -pie for Linux/ARM to get accurate imported sym - ldflags := cgoLDFLAGS - if (goarch == "arm" && goos == "linux") || goos == "android" { - ldflags = append(ldflags, "-pie") - } - if err := b.gccld(p, dynobj, ldflags, linkobj); err != nil { - return err - } - - // cgo -dynimport - var cgoflags []string - if p.Standard && p.ImportPath == "runtime/cgo" { - cgoflags = []string{"-dynlinker"} // record path to dynamic linker - } - return b.run(p.Dir, p.ImportPath, nil, buildToolExec, cgoExe, "-dynpackage", p.Name, "-dynimport", dynobj, "-dynout", importGo, cgoflags) -} - -// collect partially links the object files outObj into a single -// relocatable object file named ofile. -func (b *builder) collect(p *Package, obj, ofile string, cgoLDFLAGS, outObj []string) error { - // When linking relocatable objects, various flags need to be - // filtered out as they are inapplicable and can cause some linkers - // to fail. - var ldflags []string - for i := 0; i < len(cgoLDFLAGS); i++ { - f := cgoLDFLAGS[i] - switch { - // skip "-lc" or "-l somelib" - case strings.HasPrefix(f, "-l"): - if f == "-l" { - i++ - } - // skip "-framework X" on Darwin - case goos == "darwin" && f == "-framework": - i++ - // skip "*.{dylib,so,dll,o,a}" - case strings.HasSuffix(f, ".dylib"), - strings.HasSuffix(f, ".so"), - strings.HasSuffix(f, ".dll"), - strings.HasSuffix(f, ".o"), - strings.HasSuffix(f, ".a"): - // Remove any -fsanitize=foo flags. - // Otherwise the compiler driver thinks that we are doing final link - // and links sanitizer runtime into the object file. But we are not doing - // the final link, we will link the resulting object file again. And - // so the program ends up with two copies of sanitizer runtime. - // See issue 8788 for details. - case strings.HasPrefix(f, "-fsanitize="): - continue - // runpath flags not applicable unless building a shared - // object or executable; see issue 12115 for details. This - // is necessary as Go currently does not offer a way to - // specify the set of LDFLAGS that only apply to shared - // objects. - case strings.HasPrefix(f, "-Wl,-rpath"): - if f == "-Wl,-rpath" || f == "-Wl,-rpath-link" { - // Skip following argument to -rpath* too. - i++ - } - default: - ldflags = append(ldflags, f) - } - } - - ldflags = append(ldflags, "-Wl,-r", "-nostdlib") - - if b.gccSupportsNoPie() { - ldflags = append(ldflags, "-no-pie") - } - - // We are creating an object file, so we don't want a build ID. - ldflags = b.disableBuildID(ldflags) - - return b.gccld(p, ofile, ldflags, outObj) -} - -// Run SWIG on all SWIG input files. -// TODO: Don't build a shared library, once SWIG emits the necessary -// pragmas for external linking. -func (b *builder) swig(p *Package, obj string, pcCFLAGS []string) (outGo, outC, outCXX []string, err error) { - if err := b.swigVersionCheck(); err != nil { - return nil, nil, nil, err - } - - intgosize, err := b.swigIntSize(obj) - if err != nil { - return nil, nil, nil, err - } - - for _, f := range p.SwigFiles { - goFile, cFile, err := b.swigOne(p, f, obj, pcCFLAGS, false, intgosize) - if err != nil { - return nil, nil, nil, err - } - if goFile != "" { - outGo = append(outGo, goFile) - } - if cFile != "" { - outC = append(outC, cFile) - } - } - for _, f := range p.SwigCXXFiles { - goFile, cxxFile, err := b.swigOne(p, f, obj, pcCFLAGS, true, intgosize) - if err != nil { - return nil, nil, nil, err - } - if goFile != "" { - outGo = append(outGo, goFile) - } - if cxxFile != "" { - outCXX = append(outCXX, cxxFile) - } - } - return outGo, outC, outCXX, nil -} - -// Make sure SWIG is new enough. -var ( - swigCheckOnce sync.Once - swigCheck error -) - -func (b *builder) swigDoVersionCheck() error { - out, err := b.runOut("", "", nil, "swig", "-version") - if err != nil { - return err - } - re := regexp.MustCompile(`[vV]ersion +([\d]+)([.][\d]+)?([.][\d]+)?`) - matches := re.FindSubmatch(out) - if matches == nil { - // Can't find version number; hope for the best. - return nil - } - - major, err := strconv.Atoi(string(matches[1])) - if err != nil { - // Can't find version number; hope for the best. - return nil - } - const errmsg = "must have SWIG version >= 3.0.6" - if major < 3 { - return errors.New(errmsg) - } - if major > 3 { - // 4.0 or later - return nil - } - - // We have SWIG version 3.x. - if len(matches[2]) > 0 { - minor, err := strconv.Atoi(string(matches[2][1:])) - if err != nil { - return nil - } - if minor > 0 { - // 3.1 or later - return nil - } - } - - // We have SWIG version 3.0.x. - if len(matches[3]) > 0 { - patch, err := strconv.Atoi(string(matches[3][1:])) - if err != nil { - return nil - } - if patch < 6 { - // Before 3.0.6. - return errors.New(errmsg) - } - } - - return nil -} - -func (b *builder) swigVersionCheck() error { - swigCheckOnce.Do(func() { - swigCheck = b.swigDoVersionCheck() - }) - return swigCheck -} - -// Find the value to pass for the -intgosize option to swig. -var ( - swigIntSizeOnce sync.Once - swigIntSize string - swigIntSizeError error -) - -// This code fails to build if sizeof(int) <= 32 -const swigIntSizeCode = ` -package main -const i int = 1 << 32 -` - -// Determine the size of int on the target system for the -intgosize option -// of swig >= 2.0.9. Run only once. -func (b *builder) swigDoIntSize(obj string) (intsize string, err error) { - if buildN { - return "$INTBITS", nil - } - src := filepath.Join(b.work, "swig_intsize.go") - if err = ioutil.WriteFile(src, []byte(swigIntSizeCode), 0666); err != nil { - return - } - srcs := []string{src} - - p := goFilesPackage(srcs) - - if _, _, e := buildToolchain.gc(b, p, "", obj, false, nil, srcs); e != nil { - return "32", nil - } - return "64", nil -} - -// Determine the size of int on the target system for the -intgosize option -// of swig >= 2.0.9. -func (b *builder) swigIntSize(obj string) (intsize string, err error) { - swigIntSizeOnce.Do(func() { - swigIntSize, swigIntSizeError = b.swigDoIntSize(obj) - }) - return swigIntSize, swigIntSizeError -} - -// Run SWIG on one SWIG input file. -func (b *builder) swigOne(p *Package, file, obj string, pcCFLAGS []string, cxx bool, intgosize string) (outGo, outC string, err error) { - cgoCPPFLAGS, cgoCFLAGS, cgoCXXFLAGS, _, _ := b.cflags(p) - var cflags []string - if cxx { - cflags = stringList(cgoCPPFLAGS, pcCFLAGS, cgoCXXFLAGS) - } else { - cflags = stringList(cgoCPPFLAGS, pcCFLAGS, cgoCFLAGS) - } - - n := 5 // length of ".swig" - if cxx { - n = 8 // length of ".swigcxx" - } - base := file[:len(file)-n] - goFile := base + ".go" - gccBase := base + "_wrap." - gccExt := "c" - if cxx { - gccExt = "cxx" - } - - _, gccgo := buildToolchain.(gccgoToolchain) - - // swig - args := []string{ - "-go", - "-cgo", - "-intgosize", intgosize, - "-module", base, - "-o", obj + gccBase + gccExt, - "-outdir", obj, - } - - for _, f := range cflags { - if len(f) > 3 && f[:2] == "-I" { - args = append(args, f) - } - } - - if gccgo { - args = append(args, "-gccgo") - if pkgpath := gccgoPkgpath(p); pkgpath != "" { - args = append(args, "-go-pkgpath", pkgpath) - } - } - if cxx { - args = append(args, "-c++") - } - - out, err := b.runOut(p.Dir, p.ImportPath, nil, "swig", args, file) - if err != nil { - if len(out) > 0 { - if bytes.Contains(out, []byte("-intgosize")) || bytes.Contains(out, []byte("-cgo")) { - return "", "", errors.New("must have SWIG version >= 3.0.6") - } - b.showOutput(p.Dir, p.ImportPath, b.processOutput(out)) // swig error - return "", "", errPrintedOutput - } - return "", "", err - } - if len(out) > 0 { - b.showOutput(p.Dir, p.ImportPath, b.processOutput(out)) // swig warning - } - - return goFile, obj + gccBase + gccExt, nil -} - -// disableBuildID adjusts a linker command line to avoid creating a -// build ID when creating an object file rather than an executable or -// shared library. Some systems, such as Ubuntu, always add -// --build-id to every link, but we don't want a build ID when we are -// producing an object file. On some of those system a plain -r (not -// -Wl,-r) will turn off --build-id, but clang 3.0 doesn't support a -// plain -r. I don't know how to turn off --build-id when using clang -// other than passing a trailing --build-id=none. So that is what we -// do, but only on systems likely to support it, which is to say, -// systems that normally use gold or the GNU linker. -func (b *builder) disableBuildID(ldflags []string) []string { - switch goos { - case "android", "dragonfly", "linux", "netbsd": - ldflags = append(ldflags, "-Wl,--build-id=none") - } - return ldflags -} - -// An actionQueue is a priority queue of actions. -type actionQueue []*action - -// Implement heap.Interface -func (q *actionQueue) Len() int { return len(*q) } -func (q *actionQueue) Swap(i, j int) { (*q)[i], (*q)[j] = (*q)[j], (*q)[i] } -func (q *actionQueue) Less(i, j int) bool { return (*q)[i].priority < (*q)[j].priority } -func (q *actionQueue) Push(x interface{}) { *q = append(*q, x.(*action)) } -func (q *actionQueue) Pop() interface{} { - n := len(*q) - 1 - x := (*q)[n] - *q = (*q)[:n] - return x -} - -func (q *actionQueue) push(a *action) { - heap.Push(q, a) -} - -func (q *actionQueue) pop() *action { - return heap.Pop(q).(*action) -} - -func instrumentInit() { - if !buildRace && !buildMSan { - return - } - if buildRace && buildMSan { - fmt.Fprintf(os.Stderr, "go %s: may not use -race and -msan simultaneously\n", flag.Args()[0]) - os.Exit(2) - } - if buildMSan && (goos != "linux" || goarch != "amd64") { - fmt.Fprintf(os.Stderr, "-msan is not supported on %s/%s\n", goos, goarch) - os.Exit(2) - } - if goarch != "amd64" || goos != "linux" && goos != "freebsd" && goos != "darwin" && goos != "windows" { - fmt.Fprintf(os.Stderr, "go %s: -race and -msan are only supported on linux/amd64, freebsd/amd64, darwin/amd64 and windows/amd64\n", flag.Args()[0]) - os.Exit(2) - } - if !buildContext.CgoEnabled { - fmt.Fprintf(os.Stderr, "go %s: -race requires cgo; enable cgo by setting CGO_ENABLED=1\n", flag.Args()[0]) - os.Exit(2) - } - if buildRace { - buildGcflags = append(buildGcflags, "-race") - buildLdflags = append(buildLdflags, "-race") - } else { - buildGcflags = append(buildGcflags, "-msan") - buildLdflags = append(buildLdflags, "-msan") - } - if buildContext.InstallSuffix != "" { - buildContext.InstallSuffix += "_" - } - - if buildRace { - buildContext.InstallSuffix += "race" - buildContext.BuildTags = append(buildContext.BuildTags, "race") - } else { - buildContext.InstallSuffix += "msan" - buildContext.BuildTags = append(buildContext.BuildTags, "msan") - } -} diff --git a/libgo/go/cmd/go/build_test.go b/libgo/go/cmd/go/build_test.go deleted file mode 100644 index 79bbd545915..00000000000 --- a/libgo/go/cmd/go/build_test.go +++ /dev/null @@ -1,44 +0,0 @@ -// Copyright 2016 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package main - -import ( - "os" - "reflect" - "testing" -) - -func TestRemoveDevNull(t *testing.T) { - fi, err := os.Lstat(os.DevNull) - if err != nil { - t.Skip(err) - } - if fi.Mode().IsRegular() { - t.Errorf("Lstat(%s).Mode().IsRegular() = true; expected false", os.DevNull) - } - mayberemovefile(os.DevNull) - _, err = os.Lstat(os.DevNull) - if err != nil { - t.Errorf("mayberemovefile(%s) did remove it; oops", os.DevNull) - } -} - -func TestSplitPkgConfigOutput(t *testing.T) { - for _, test := range []struct { - in []byte - want []string - }{ - {[]byte(`-r:foo -L/usr/white\ space/lib -lfoo\ bar -lbar\ baz`), []string{"-r:foo", "-L/usr/white space/lib", "-lfoo bar", "-lbar baz"}}, - {[]byte(`-lextra\ fun\ arg\\`), []string{`-lextra fun arg\`}}, - {[]byte(`broken flag\`), []string{"broken", "flag"}}, - {[]byte("\textra whitespace\r\n"), []string{"extra", "whitespace"}}, - {[]byte(" \r\n "), nil}, - } { - got := splitPkgConfigOutput(test.in) - if !reflect.DeepEqual(got, test.want) { - t.Errorf("splitPkgConfigOutput(%v) = %v; want %v", test.in, got, test.want) - } - } -} diff --git a/libgo/go/cmd/go/clean.go b/libgo/go/cmd/go/clean.go deleted file mode 100644 index 7b07150743a..00000000000 --- a/libgo/go/cmd/go/clean.go +++ /dev/null @@ -1,248 +0,0 @@ -// Copyright 2012 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package main - -import ( - "fmt" - "io/ioutil" - "os" - "path/filepath" - "strings" -) - -var cmdClean = &Command{ - UsageLine: "clean [-i] [-r] [-n] [-x] [build flags] [packages]", - Short: "remove object files", - Long: ` -Clean removes object files from package source directories. -The go command builds most objects in a temporary directory, -so go clean is mainly concerned with object files left by other -tools or by manual invocations of go build. - -Specifically, clean removes the following files from each of the -source directories corresponding to the import paths: - - _obj/ old object directory, left from Makefiles - _test/ old test directory, left from Makefiles - _testmain.go old gotest file, left from Makefiles - test.out old test log, left from Makefiles - build.out old test log, left from Makefiles - *.[568ao] object files, left from Makefiles - - DIR(.exe) from go build - DIR.test(.exe) from go test -c - MAINFILE(.exe) from go build MAINFILE.go - *.so from SWIG - -In the list, DIR represents the final path element of the -directory, and MAINFILE is the base name of any Go source -file in the directory that is not included when building -the package. - -The -i flag causes clean to remove the corresponding installed -archive or binary (what 'go install' would create). - -The -n flag causes clean to print the remove commands it would execute, -but not run them. - -The -r flag causes clean to be applied recursively to all the -dependencies of the packages named by the import paths. - -The -x flag causes clean to print remove commands as it executes them. - -For more about build flags, see 'go help build'. - -For more about specifying packages, see 'go help packages'. - `, -} - -var cleanI bool // clean -i flag -var cleanR bool // clean -r flag - -func init() { - // break init cycle - cmdClean.Run = runClean - - cmdClean.Flag.BoolVar(&cleanI, "i", false, "") - cmdClean.Flag.BoolVar(&cleanR, "r", false, "") - // -n and -x are important enough to be - // mentioned explicitly in the docs but they - // are part of the build flags. - - addBuildFlags(cmdClean) -} - -func runClean(cmd *Command, args []string) { - for _, pkg := range packagesAndErrors(args) { - clean(pkg) - } -} - -var cleaned = map[*Package]bool{} - -// TODO: These are dregs left by Makefile-based builds. -// Eventually, can stop deleting these. -var cleanDir = map[string]bool{ - "_test": true, - "_obj": true, -} - -var cleanFile = map[string]bool{ - "_testmain.go": true, - "test.out": true, - "build.out": true, - "a.out": true, -} - -var cleanExt = map[string]bool{ - ".5": true, - ".6": true, - ".8": true, - ".a": true, - ".o": true, - ".so": true, -} - -func clean(p *Package) { - if cleaned[p] { - return - } - cleaned[p] = true - - if p.Dir == "" { - errorf("can't load package: %v", p.Error) - return - } - dirs, err := ioutil.ReadDir(p.Dir) - if err != nil { - errorf("go clean %s: %v", p.Dir, err) - return - } - - var b builder - b.print = fmt.Print - - packageFile := map[string]bool{} - if p.Name != "main" { - // Record which files are not in package main. - // The others are. - keep := func(list []string) { - for _, f := range list { - packageFile[f] = true - } - } - keep(p.GoFiles) - keep(p.CgoFiles) - keep(p.TestGoFiles) - keep(p.XTestGoFiles) - } - - _, elem := filepath.Split(p.Dir) - var allRemove []string - - // Remove dir-named executable only if this is package main. - if p.Name == "main" { - allRemove = append(allRemove, - elem, - elem+".exe", - ) - } - - // Remove package test executables. - allRemove = append(allRemove, - elem+".test", - elem+".test.exe", - ) - - // Remove a potential executable for each .go file in the directory that - // is not part of the directory's package. - for _, dir := range dirs { - name := dir.Name() - if packageFile[name] { - continue - } - if !dir.IsDir() && strings.HasSuffix(name, ".go") { - // TODO(adg,rsc): check that this .go file is actually - // in "package main", and therefore capable of building - // to an executable file. - base := name[:len(name)-len(".go")] - allRemove = append(allRemove, base, base+".exe") - } - } - - if buildN || buildX { - b.showcmd(p.Dir, "rm -f %s", strings.Join(allRemove, " ")) - } - - toRemove := map[string]bool{} - for _, name := range allRemove { - toRemove[name] = true - } - for _, dir := range dirs { - name := dir.Name() - if dir.IsDir() { - // TODO: Remove once Makefiles are forgotten. - if cleanDir[name] { - if buildN || buildX { - b.showcmd(p.Dir, "rm -r %s", name) - if buildN { - continue - } - } - if err := os.RemoveAll(filepath.Join(p.Dir, name)); err != nil { - errorf("go clean: %v", err) - } - } - continue - } - - if buildN { - continue - } - - if cleanFile[name] || cleanExt[filepath.Ext(name)] || toRemove[name] { - removeFile(filepath.Join(p.Dir, name)) - } - } - - if cleanI && p.target != "" { - if buildN || buildX { - b.showcmd("", "rm -f %s", p.target) - } - if !buildN { - removeFile(p.target) - } - } - - if cleanR { - for _, p1 := range p.imports { - clean(p1) - } - } -} - -// removeFile tries to remove file f, if error other than file doesn't exist -// occurs, it will report the error. -func removeFile(f string) { - err := os.Remove(f) - if err == nil || os.IsNotExist(err) { - return - } - // Windows does not allow deletion of a binary file while it is executing. - if toolIsWindows { - // Remove lingering ~ file from last attempt. - if _, err2 := os.Stat(f + "~"); err2 == nil { - os.Remove(f + "~") - } - // Try to move it out of the way. If the move fails, - // which is likely, we'll try again the - // next time we do an install of this binary. - if err2 := os.Rename(f, f+"~"); err2 == nil { - os.Remove(f + "~") - return - } - } - errorf("go clean: %v", err) -} diff --git a/libgo/go/cmd/go/context.go b/libgo/go/cmd/go/context.go deleted file mode 100644 index 94cd54d00d8..00000000000 --- a/libgo/go/cmd/go/context.go +++ /dev/null @@ -1,37 +0,0 @@ -// Copyright 2014 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package main - -import ( - "go/build" -) - -type Context struct { - GOARCH string `json:",omitempty"` // target architecture - GOOS string `json:",omitempty"` // target operating system - GOROOT string `json:",omitempty"` // Go root - GOPATH string `json:",omitempty"` // Go path - CgoEnabled bool `json:",omitempty"` // whether cgo can be used - UseAllFiles bool `json:",omitempty"` // use files regardless of +build lines, file names - Compiler string `json:",omitempty"` // compiler to assume when computing target paths - BuildTags []string `json:",omitempty"` // build constraints to match in +build lines - ReleaseTags []string `json:",omitempty"` // releases the current release is compatible with - InstallSuffix string `json:",omitempty"` // suffix to use in the name of the install dir -} - -func newContext(c *build.Context) *Context { - return &Context{ - GOARCH: c.GOARCH, - GOOS: c.GOOS, - GOROOT: c.GOROOT, - GOPATH: c.GOPATH, - CgoEnabled: c.CgoEnabled, - UseAllFiles: c.UseAllFiles, - Compiler: c.Compiler, - BuildTags: c.BuildTags, - ReleaseTags: c.ReleaseTags, - InstallSuffix: c.InstallSuffix, - } -} diff --git a/libgo/go/cmd/go/discovery.go b/libgo/go/cmd/go/discovery.go deleted file mode 100644 index b60eaef7398..00000000000 --- a/libgo/go/cmd/go/discovery.go +++ /dev/null @@ -1,83 +0,0 @@ -// Copyright 2012 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// +build !cmd_go_bootstrap - -// This code is compiled into the real 'go' binary, but it is not -// compiled into the binary that is built during all.bash, so as -// to avoid needing to build net (and thus use cgo) during the -// bootstrap process. - -package main - -import ( - "encoding/xml" - "fmt" - "io" - "strings" -) - -// charsetReader returns a reader for the given charset. Currently -// it only supports UTF-8 and ASCII. Otherwise, it returns a meaningful -// error which is printed by go get, so the user can find why the package -// wasn't downloaded if the encoding is not supported. Note that, in -// order to reduce potential errors, ASCII is treated as UTF-8 (i.e. characters -// greater than 0x7f are not rejected). -func charsetReader(charset string, input io.Reader) (io.Reader, error) { - switch strings.ToLower(charset) { - case "ascii": - return input, nil - default: - return nil, fmt.Errorf("can't decode XML document using charset %q", charset) - } -} - -// parseMetaGoImports returns meta imports from the HTML in r. -// Parsing ends at the end of the section or the beginning of the . -func parseMetaGoImports(r io.Reader) (imports []metaImport, err error) { - d := xml.NewDecoder(r) - d.CharsetReader = charsetReader - d.Strict = false - var t xml.Token - for { - t, err = d.RawToken() - if err != nil { - if err == io.EOF || len(imports) > 0 { - err = nil - } - return - } - if e, ok := t.(xml.StartElement); ok && strings.EqualFold(e.Name.Local, "body") { - return - } - if e, ok := t.(xml.EndElement); ok && strings.EqualFold(e.Name.Local, "head") { - return - } - e, ok := t.(xml.StartElement) - if !ok || !strings.EqualFold(e.Name.Local, "meta") { - continue - } - if attrValue(e.Attr, "name") != "go-import" { - continue - } - if f := strings.Fields(attrValue(e.Attr, "content")); len(f) == 3 { - imports = append(imports, metaImport{ - Prefix: f[0], - VCS: f[1], - RepoRoot: f[2], - }) - } - } -} - -// attrValue returns the attribute value for the case-insensitive key -// `name', or the empty string if nothing is found. -func attrValue(attrs []xml.Attr, name string) string { - for _, a := range attrs { - if strings.EqualFold(a.Name.Local, name) { - return a.Value - } - } - return "" -} diff --git a/libgo/go/cmd/go/doc.go b/libgo/go/cmd/go/doc.go deleted file mode 100644 index 829983950f8..00000000000 --- a/libgo/go/cmd/go/doc.go +++ /dev/null @@ -1,118 +0,0 @@ -// Copyright 2015 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -//go:generate ./mkalldocs.sh - -package main - -var cmdDoc = &Command{ - Run: runDoc, - UsageLine: "doc [-u] [-c] [package|[package.]symbol[.method]]", - CustomFlags: true, - Short: "show documentation for package or symbol", - Long: ` -Doc prints the documentation comments associated with the item identified by its -arguments (a package, const, func, type, var, or method) followed by a one-line -summary of each of the first-level items "under" that item (package-level -declarations for a package, methods for a type, etc.). - -Doc accepts zero, one, or two arguments. - -Given no arguments, that is, when run as - - go doc - -it prints the package documentation for the package in the current directory. -If the package is a command (package main), the exported symbols of the package -are elided from the presentation unless the -cmd flag is provided. - -When run with one argument, the argument is treated as a Go-syntax-like -representation of the item to be documented. What the argument selects depends -on what is installed in GOROOT and GOPATH, as well as the form of the argument, -which is schematically one of these: - - go doc - go doc [.] - go doc [.][.] - go doc [.][.] - -The first item in this list matched by the argument is the one whose documentation -is printed. (See the examples below.) However, if the argument starts with a capital -letter it is assumed to identify a symbol or method in the current directory. - -For packages, the order of scanning is determined lexically in breadth-first order. -That is, the package presented is the one that matches the search and is nearest -the root and lexically first at its level of the hierarchy. The GOROOT tree is -always scanned in its entirety before GOPATH. - -If there is no package specified or matched, the package in the current -directory is selected, so "go doc Foo" shows the documentation for symbol Foo in -the current package. - -The package path must be either a qualified path or a proper suffix of a -path. The go tool's usual package mechanism does not apply: package path -elements like . and ... are not implemented by go doc. - -When run with two arguments, the first must be a full package path (not just a -suffix), and the second is a symbol or symbol and method; this is similar to the -syntax accepted by godoc: - - go doc [.] - -In all forms, when matching symbols, lower-case letters in the argument match -either case but upper-case letters match exactly. This means that there may be -multiple matches of a lower-case argument in a package if different symbols have -different cases. If this occurs, documentation for all matches is printed. - -Examples: - go doc - Show documentation for current package. - go doc Foo - Show documentation for Foo in the current package. - (Foo starts with a capital letter so it cannot match - a package path.) - go doc encoding/json - Show documentation for the encoding/json package. - go doc json - Shorthand for encoding/json. - go doc json.Number (or go doc json.number) - Show documentation and method summary for json.Number. - go doc json.Number.Int64 (or go doc json.number.int64) - Show documentation for json.Number's Int64 method. - go doc cmd/doc - Show package docs for the doc command. - go doc -cmd cmd/doc - Show package docs and exported symbols within the doc command. - go doc template.new - Show documentation for html/template's New function. - (html/template is lexically before text/template) - go doc text/template.new # One argument - Show documentation for text/template's New function. - go doc text/template new # Two arguments - Show documentation for text/template's New function. - - At least in the current tree, these invocations all print the - documentation for json.Decoder's Decode method: - - go doc json.Decoder.Decode - go doc json.decoder.decode - go doc json.decode - cd go/src/encoding/json; go doc decode - -Flags: - -c - Respect case when matching symbols. - -cmd - Treat a command (package main) like a regular package. - Otherwise package main's exported symbols are hidden - when showing the package's top-level documentation. - -u - Show documentation for unexported as well as exported - symbols and methods. -`, -} - -func runDoc(cmd *Command, args []string) { - run(buildToolExec, tool("doc"), args) -} diff --git a/libgo/go/cmd/go/env.go b/libgo/go/cmd/go/env.go deleted file mode 100644 index 31710b7e6d6..00000000000 --- a/libgo/go/cmd/go/env.go +++ /dev/null @@ -1,138 +0,0 @@ -// Copyright 2012 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package main - -import ( - "fmt" - "os" - "runtime" - "strings" -) - -var cmdEnv = &Command{ - Run: runEnv, - UsageLine: "env [var ...]", - Short: "print Go environment information", - Long: ` -Env prints Go environment information. - -By default env prints information as a shell script -(on Windows, a batch file). If one or more variable -names is given as arguments, env prints the value of -each named variable on its own line. - `, -} - -type envVar struct { - name, value string -} - -func mkEnv() []envVar { - var b builder - b.init() - - env := []envVar{ - {"GOARCH", goarch}, - {"GOBIN", gobin}, - {"GOEXE", exeSuffix}, - {"GOHOSTARCH", runtime.GOARCH}, - {"GOHOSTOS", runtime.GOOS}, - {"GOOS", goos}, - {"GOPATH", buildContext.GOPATH}, - {"GORACE", os.Getenv("GORACE")}, - {"GOROOT", goroot}, - {"GOTOOLDIR", toolDir}, - - // disable escape codes in clang errors - {"TERM", "dumb"}, - } - - if gccgoBin != "" { - env = append(env, envVar{"GCCGO", gccgoBin}) - } else { - env = append(env, envVar{"GCCGO", gccgoName}) - } - - switch goarch { - case "arm": - env = append(env, envVar{"GOARM", os.Getenv("GOARM")}) - case "386": - env = append(env, envVar{"GO386", os.Getenv("GO386")}) - } - - cmd := b.gccCmd(".") - env = append(env, envVar{"CC", cmd[0]}) - env = append(env, envVar{"GOGCCFLAGS", strings.Join(cmd[3:], " ")}) - cmd = b.gxxCmd(".") - env = append(env, envVar{"CXX", cmd[0]}) - - if buildContext.CgoEnabled { - env = append(env, envVar{"CGO_ENABLED", "1"}) - } else { - env = append(env, envVar{"CGO_ENABLED", "0"}) - } - - return env -} - -func findEnv(env []envVar, name string) string { - for _, e := range env { - if e.name == name { - return e.value - } - } - return "" -} - -// extraEnvVars returns environment variables that should not leak into child processes. -func extraEnvVars() []envVar { - var b builder - b.init() - cppflags, cflags, cxxflags, fflags, ldflags := b.cflags(&Package{}) - return []envVar{ - {"PKG_CONFIG", b.pkgconfigCmd()}, - {"CGO_CFLAGS", strings.Join(cflags, " ")}, - {"CGO_CPPFLAGS", strings.Join(cppflags, " ")}, - {"CGO_CXXFLAGS", strings.Join(cxxflags, " ")}, - {"CGO_FFLAGS", strings.Join(fflags, " ")}, - {"CGO_LDFLAGS", strings.Join(ldflags, " ")}, - } -} - -func runEnv(cmd *Command, args []string) { - env := newEnv - env = append(env, extraEnvVars()...) - if len(args) > 0 { - for _, name := range args { - fmt.Printf("%s\n", findEnv(env, name)) - } - return - } - - for _, e := range env { - if e.name != "TERM" { - switch runtime.GOOS { - default: - fmt.Printf("%s=\"%s\"\n", e.name, e.value) - case "plan9": - if strings.IndexByte(e.value, '\x00') < 0 { - fmt.Printf("%s='%s'\n", e.name, strings.Replace(e.value, "'", "''", -1)) - } else { - v := strings.Split(e.value, "\x00") - fmt.Printf("%s=(", e.name) - for x, s := range v { - if x > 0 { - fmt.Printf(" ") - } - fmt.Printf("%s", s) - } - fmt.Printf(")\n") - } - case "windows": - fmt.Printf("set %s=%s\n", e.name, e.value) - } - } - } -} diff --git a/libgo/go/cmd/go/fix.go b/libgo/go/cmd/go/fix.go deleted file mode 100644 index 3af7adb4e1c..00000000000 --- a/libgo/go/cmd/go/fix.go +++ /dev/null @@ -1,30 +0,0 @@ -// Copyright 2011 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package main - -var cmdFix = &Command{ - Run: runFix, - UsageLine: "fix [packages]", - Short: "run go tool fix on packages", - Long: ` -Fix runs the Go fix command on the packages named by the import paths. - -For more about fix, see 'go doc cmd/fix'. -For more about specifying packages, see 'go help packages'. - -To run fix with specific options, run 'go tool fix'. - -See also: go fmt, go vet. - `, -} - -func runFix(cmd *Command, args []string) { - for _, pkg := range packages(args) { - // Use pkg.gofiles instead of pkg.Dir so that - // the command only applies to this package, - // not to packages in subdirectories. - run(stringList(buildToolExec, tool("fix"), relPaths(pkg.allgofiles))) - } -} diff --git a/libgo/go/cmd/go/fmt.go b/libgo/go/cmd/go/fmt.go deleted file mode 100644 index 4ed7722575e..00000000000 --- a/libgo/go/cmd/go/fmt.go +++ /dev/null @@ -1,64 +0,0 @@ -// Copyright 2011 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package main - -import ( - "os" - "path/filepath" -) - -func init() { - addBuildFlagsNX(cmdFmt) -} - -var cmdFmt = &Command{ - Run: runFmt, - UsageLine: "fmt [-n] [-x] [packages]", - Short: "run gofmt on package sources", - Long: ` -Fmt runs the command 'gofmt -l -w' on the packages named -by the import paths. It prints the names of the files that are modified. - -For more about gofmt, see 'go doc cmd/gofmt'. -For more about specifying packages, see 'go help packages'. - -The -n flag prints commands that would be executed. -The -x flag prints commands as they are executed. - -To run gofmt with specific options, run gofmt itself. - -See also: go fix, go vet. - `, -} - -func runFmt(cmd *Command, args []string) { - gofmt := gofmtPath() - for _, pkg := range packages(args) { - // Use pkg.gofiles instead of pkg.Dir so that - // the command only applies to this package, - // not to packages in subdirectories. - run(stringList(gofmt, "-l", "-w", relPaths(pkg.allgofiles))) - } -} - -func gofmtPath() string { - gofmt := "gofmt" - if toolIsWindows { - gofmt += toolWindowsExtension - } - - gofmtPath := filepath.Join(gobin, gofmt) - if _, err := os.Stat(gofmtPath); err == nil { - return gofmtPath - } - - gofmtPath = filepath.Join(goroot, "bin", gofmt) - if _, err := os.Stat(gofmtPath); err == nil { - return gofmtPath - } - - // fallback to looking for gofmt in $PATH - return "gofmt" -} diff --git a/libgo/go/cmd/go/generate.go b/libgo/go/cmd/go/generate.go deleted file mode 100644 index 2d92a0c100c..00000000000 --- a/libgo/go/cmd/go/generate.go +++ /dev/null @@ -1,401 +0,0 @@ -// Copyright 2011 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package main - -import ( - "bufio" - "bytes" - "fmt" - "io" - "log" - "os" - "os/exec" - "path/filepath" - "regexp" - "strconv" - "strings" -) - -var cmdGenerate = &Command{ - Run: runGenerate, - UsageLine: "generate [-run regexp] [-n] [-v] [-x] [build flags] [file.go... | packages]", - Short: "generate Go files by processing source", - Long: ` -Generate runs commands described by directives within existing -files. Those commands can run any process but the intent is to -create or update Go source files. - -Go generate is never run automatically by go build, go get, go test, -and so on. It must be run explicitly. - -Go generate scans the file for directives, which are lines of -the form, - - //go:generate command argument... - -(note: no leading spaces and no space in "//go") where command -is the generator to be run, corresponding to an executable file -that can be run locally. It must either be in the shell path -(gofmt), a fully qualified path (/usr/you/bin/mytool), or a -command alias, described below. - -Note that go generate does not parse the file, so lines that look -like directives in comments or multiline strings will be treated -as directives. - -The arguments to the directive are space-separated tokens or -double-quoted strings passed to the generator as individual -arguments when it is run. - -Quoted strings use Go syntax and are evaluated before execution; a -quoted string appears as a single argument to the generator. - -Go generate sets several variables when it runs the generator: - - $GOARCH - The execution architecture (arm, amd64, etc.) - $GOOS - The execution operating system (linux, windows, etc.) - $GOFILE - The base name of the file. - $GOLINE - The line number of the directive in the source file. - $GOPACKAGE - The name of the package of the file containing the directive. - $DOLLAR - A dollar sign. - -Other than variable substitution and quoted-string evaluation, no -special processing such as "globbing" is performed on the command -line. - -As a last step before running the command, any invocations of any -environment variables with alphanumeric names, such as $GOFILE or -$HOME, are expanded throughout the command line. The syntax for -variable expansion is $NAME on all operating systems. Due to the -order of evaluation, variables are expanded even inside quoted -strings. If the variable NAME is not set, $NAME expands to the -empty string. - -A directive of the form, - - //go:generate -command xxx args... - -specifies, for the remainder of this source file only, that the -string xxx represents the command identified by the arguments. This -can be used to create aliases or to handle multiword generators. -For example, - - //go:generate -command foo go tool foo - -specifies that the command "foo" represents the generator -"go tool foo". - -Generate processes packages in the order given on the command line, -one at a time. If the command line lists .go files, they are treated -as a single package. Within a package, generate processes the -source files in a package in file name order, one at a time. Within -a source file, generate runs generators in the order they appear -in the file, one at a time. - -If any generator returns an error exit status, "go generate" skips -all further processing for that package. - -The generator is run in the package's source directory. - -Go generate accepts one specific flag: - - -run="" - if non-empty, specifies a regular expression to select - directives whose full original source text (excluding - any trailing spaces and final newline) matches the - expression. - -It also accepts the standard build flags including -v, -n, and -x. -The -v flag prints the names of packages and files as they are -processed. -The -n flag prints commands that would be executed. -The -x flag prints commands as they are executed. - -For more about build flags, see 'go help build'. - -For more about specifying packages, see 'go help packages'. - `, -} - -var ( - generateRunFlag string // generate -run flag - generateRunRE *regexp.Regexp // compiled expression for -run -) - -func init() { - addBuildFlags(cmdGenerate) - cmdGenerate.Flag.StringVar(&generateRunFlag, "run", "", "") -} - -func runGenerate(cmd *Command, args []string) { - ignoreImports = true - - if generateRunFlag != "" { - var err error - generateRunRE, err = regexp.Compile(generateRunFlag) - if err != nil { - log.Fatalf("generate: %s", err) - } - } - // Even if the arguments are .go files, this loop suffices. - for _, pkg := range packages(args) { - for _, file := range pkg.gofiles { - if !generate(pkg.Name, file) { - break - } - } - } -} - -// generate runs the generation directives for a single file. -func generate(pkg, absFile string) bool { - fd, err := os.Open(absFile) - if err != nil { - log.Fatalf("generate: %s", err) - } - defer fd.Close() - g := &Generator{ - r: fd, - path: absFile, - pkg: pkg, - commands: make(map[string][]string), - } - return g.run() -} - -// A Generator represents the state of a single Go source file -// being scanned for generator commands. -type Generator struct { - r io.Reader - path string // full rooted path name. - dir string // full rooted directory of file. - file string // base name of file. - pkg string - commands map[string][]string - lineNum int // current line number. - env []string -} - -// run runs the generators in the current file. -func (g *Generator) run() (ok bool) { - // Processing below here calls g.errorf on failure, which does panic(stop). - // If we encounter an error, we abort the package. - defer func() { - e := recover() - if e != nil { - ok = false - if e != stop { - panic(e) - } - setExitStatus(1) - } - }() - g.dir, g.file = filepath.Split(g.path) - g.dir = filepath.Clean(g.dir) // No final separator please. - if buildV { - fmt.Fprintf(os.Stderr, "%s\n", shortPath(g.path)) - } - - // Scan for lines that start "//go:generate". - // Can't use bufio.Scanner because it can't handle long lines, - // which are likely to appear when using generate. - input := bufio.NewReader(g.r) - var err error - // One line per loop. - for { - g.lineNum++ // 1-indexed. - var buf []byte - buf, err = input.ReadSlice('\n') - if err == bufio.ErrBufferFull { - // Line too long - consume and ignore. - if isGoGenerate(buf) { - g.errorf("directive too long") - } - for err == bufio.ErrBufferFull { - _, err = input.ReadSlice('\n') - } - if err != nil { - break - } - continue - } - - if err != nil { - // Check for marker at EOF without final \n. - if err == io.EOF && isGoGenerate(buf) { - err = io.ErrUnexpectedEOF - } - break - } - - if !isGoGenerate(buf) { - continue - } - if generateRunFlag != "" { - if !generateRunRE.Match(bytes.TrimSpace(buf)) { - continue - } - } - - g.setEnv() - words := g.split(string(buf)) - if len(words) == 0 { - g.errorf("no arguments to directive") - } - if words[0] == "-command" { - g.setShorthand(words) - continue - } - // Run the command line. - if buildN || buildX { - fmt.Fprintf(os.Stderr, "%s\n", strings.Join(words, " ")) - } - if buildN { - continue - } - g.exec(words) - } - if err != nil && err != io.EOF { - g.errorf("error reading %s: %s", shortPath(g.path), err) - } - return true -} - -func isGoGenerate(buf []byte) bool { - return bytes.HasPrefix(buf, []byte("//go:generate ")) || bytes.HasPrefix(buf, []byte("//go:generate\t")) -} - -// setEnv sets the extra environment variables used when executing a -// single go:generate command. -func (g *Generator) setEnv() { - g.env = []string{ - "GOARCH=" + buildContext.GOARCH, - "GOOS=" + buildContext.GOOS, - "GOFILE=" + g.file, - "GOLINE=" + strconv.Itoa(g.lineNum), - "GOPACKAGE=" + g.pkg, - "DOLLAR=" + "$", - } -} - -// split breaks the line into words, evaluating quoted -// strings and evaluating environment variables. -// The initial //go:generate element is present in line. -func (g *Generator) split(line string) []string { - // Parse line, obeying quoted strings. - var words []string - line = line[len("//go:generate ") : len(line)-1] // Drop preamble and final newline. - // There may still be a carriage return. - if len(line) > 0 && line[len(line)-1] == '\r' { - line = line[:len(line)-1] - } - // One (possibly quoted) word per iteration. -Words: - for { - line = strings.TrimLeft(line, " \t") - if len(line) == 0 { - break - } - if line[0] == '"' { - for i := 1; i < len(line); i++ { - c := line[i] // Only looking for ASCII so this is OK. - switch c { - case '\\': - if i+1 == len(line) { - g.errorf("bad backslash") - } - i++ // Absorb next byte (If it's a multibyte we'll get an error in Unquote). - case '"': - word, err := strconv.Unquote(line[0 : i+1]) - if err != nil { - g.errorf("bad quoted string") - } - words = append(words, word) - line = line[i+1:] - // Check the next character is space or end of line. - if len(line) > 0 && line[0] != ' ' && line[0] != '\t' { - g.errorf("expect space after quoted argument") - } - continue Words - } - } - g.errorf("mismatched quoted string") - } - i := strings.IndexAny(line, " \t") - if i < 0 { - i = len(line) - } - words = append(words, line[0:i]) - line = line[i:] - } - // Substitute command if required. - if len(words) > 0 && g.commands[words[0]] != nil { - // Replace 0th word by command substitution. - words = append(g.commands[words[0]], words[1:]...) - } - // Substitute environment variables. - for i, word := range words { - words[i] = os.Expand(word, g.expandVar) - } - return words -} - -var stop = fmt.Errorf("error in generation") - -// errorf logs an error message prefixed with the file and line number. -// It then exits the program (with exit status 1) because generation stops -// at the first error. -func (g *Generator) errorf(format string, args ...interface{}) { - fmt.Fprintf(os.Stderr, "%s:%d: %s\n", shortPath(g.path), g.lineNum, - fmt.Sprintf(format, args...)) - panic(stop) -} - -// expandVar expands the $XXX invocation in word. It is called -// by os.Expand. -func (g *Generator) expandVar(word string) string { - w := word + "=" - for _, e := range g.env { - if strings.HasPrefix(e, w) { - return e[len(w):] - } - } - return os.Getenv(word) -} - -// setShorthand installs a new shorthand as defined by a -command directive. -func (g *Generator) setShorthand(words []string) { - // Create command shorthand. - if len(words) == 1 { - g.errorf("no command specified for -command") - } - command := words[1] - if g.commands[command] != nil { - g.errorf("command %q defined multiply defined", command) - } - g.commands[command] = words[2:len(words):len(words)] // force later append to make copy -} - -// exec runs the command specified by the argument. The first word is -// the command name itself. -func (g *Generator) exec(words []string) { - cmd := exec.Command(words[0], words[1:]...) - // Standard in and out of generator should be the usual. - cmd.Stdout = os.Stdout - cmd.Stderr = os.Stderr - // Run the command in the package directory. - cmd.Dir = g.dir - cmd.Env = mergeEnvLists(g.env, origEnv) - err := cmd.Run() - if err != nil { - g.errorf("running %q: %s", words[0], err) - } -} diff --git a/libgo/go/cmd/go/generate_test.go b/libgo/go/cmd/go/generate_test.go deleted file mode 100644 index dd116e6cc61..00000000000 --- a/libgo/go/cmd/go/generate_test.go +++ /dev/null @@ -1,56 +0,0 @@ -// Copyright 2011 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package main - -import ( - "reflect" - "runtime" - "testing" -) - -type splitTest struct { - in string - out []string -} - -var splitTests = []splitTest{ - {"", nil}, - {"x", []string{"x"}}, - {" a b\tc ", []string{"a", "b", "c"}}, - {` " a " `, []string{" a "}}, - {"$GOARCH", []string{runtime.GOARCH}}, - {"$GOOS", []string{runtime.GOOS}}, - {"$GOFILE", []string{"proc.go"}}, - {"$GOPACKAGE", []string{"sys"}}, - {"a $XXNOTDEFINEDXX b", []string{"a", "", "b"}}, - {"/$XXNOTDEFINED/", []string{"//"}}, - {"/$DOLLAR/", []string{"/$/"}}, - {"yacc -o $GOARCH/yacc_$GOFILE", []string{"go", "tool", "yacc", "-o", runtime.GOARCH + "/yacc_proc.go"}}, -} - -func TestGenerateCommandParse(t *testing.T) { - g := &Generator{ - r: nil, // Unused here. - path: "/usr/ken/sys/proc.go", - dir: "/usr/ken/sys", - file: "proc.go", - pkg: "sys", - commands: make(map[string][]string), - } - g.setEnv() - g.setShorthand([]string{"-command", "yacc", "go", "tool", "yacc"}) - for _, test := range splitTests { - // First with newlines. - got := g.split("//go:generate " + test.in + "\n") - if !reflect.DeepEqual(got, test.out) { - t.Errorf("split(%q): got %q expected %q", test.in, got, test.out) - } - // Then with CRLFs, thank you Windows. - got = g.split("//go:generate " + test.in + "\r\n") - if !reflect.DeepEqual(got, test.out) { - t.Errorf("split(%q): got %q expected %q", test.in, got, test.out) - } - } -} diff --git a/libgo/go/cmd/go/get.go b/libgo/go/cmd/go/get.go deleted file mode 100644 index 6fb4235068e..00000000000 --- a/libgo/go/cmd/go/get.go +++ /dev/null @@ -1,588 +0,0 @@ -// Copyright 2011 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package main - -import ( - "fmt" - "go/build" - "os" - "path/filepath" - "regexp" - "runtime" - "strconv" - "strings" -) - -var cmdGet = &Command{ - UsageLine: "get [-d] [-f] [-fix] [-insecure] [-t] [-u] [build flags] [packages]", - Short: "download and install packages and dependencies", - Long: ` -Get downloads the packages named by the import paths, along with their -dependencies. It then installs the named packages, like 'go install'. - -The -d flag instructs get to stop after downloading the packages; that is, -it instructs get not to install the packages. - -The -f flag, valid only when -u is set, forces get -u not to verify that -each package has been checked out from the source control repository -implied by its import path. This can be useful if the source is a local fork -of the original. - -The -fix flag instructs get to run the fix tool on the downloaded packages -before resolving dependencies or building the code. - -The -insecure flag permits fetching from repositories and resolving -custom domains using insecure schemes such as HTTP. Use with caution. - -The -t flag instructs get to also download the packages required to build -the tests for the specified packages. - -The -u flag instructs get to use the network to update the named packages -and their dependencies. By default, get uses the network to check out -missing packages but does not use it to look for updates to existing packages. - -The -v flag enables verbose progress and debug output. - -Get also accepts build flags to control the installation. See 'go help build'. - -When checking out a new package, get creates the target directory -GOPATH/src/. If the GOPATH contains multiple entries, -get uses the first one. For more details see: 'go help gopath'. - -When checking out or updating a package, get looks for a branch or tag -that matches the locally installed version of Go. The most important -rule is that if the local installation is running version "go1", get -searches for a branch or tag named "go1". If no such version exists it -retrieves the most recent version of the package. - -When go get checks out or updates a Git repository, -it also updates any git submodules referenced by the repository. - -Get never checks out or updates code stored in vendor directories. - -For more about specifying packages, see 'go help packages'. - -For more about how 'go get' finds source code to -download, see 'go help importpath'. - -See also: go build, go install, go clean. - `, -} - -var getD = cmdGet.Flag.Bool("d", false, "") -var getF = cmdGet.Flag.Bool("f", false, "") -var getT = cmdGet.Flag.Bool("t", false, "") -var getU = cmdGet.Flag.Bool("u", false, "") -var getFix = cmdGet.Flag.Bool("fix", false, "") -var getInsecure = cmdGet.Flag.Bool("insecure", false, "") - -func init() { - addBuildFlags(cmdGet) - cmdGet.Run = runGet // break init loop -} - -func runGet(cmd *Command, args []string) { - if *getF && !*getU { - fatalf("go get: cannot use -f flag without -u") - } - - // Disable any prompting for passwords by Git. - // Only has an effect for 2.3.0 or later, but avoiding - // the prompt in earlier versions is just too hard. - // If user has explicitly set GIT_TERMINAL_PROMPT=1, keep - // prompting. - // See golang.org/issue/9341 and golang.org/issue/12706. - if os.Getenv("GIT_TERMINAL_PROMPT") == "" { - os.Setenv("GIT_TERMINAL_PROMPT", "0") - } - - // Disable any ssh connection pooling by Git. - // If a Git subprocess forks a child into the background to cache a new connection, - // that child keeps stdout/stderr open. After the Git subprocess exits, - // os /exec expects to be able to read from the stdout/stderr pipe - // until EOF to get all the data that the Git subprocess wrote before exiting. - // The EOF doesn't come until the child exits too, because the child - // is holding the write end of the pipe. - // This is unfortunate, but it has come up at least twice - // (see golang.org/issue/13453 and golang.org/issue/16104) - // and confuses users when it does. - // If the user has explicitly set GIT_SSH or GIT_SSH_COMMAND, - // assume they know what they are doing and don't step on it. - // But default to turning off ControlMaster. - if os.Getenv("GIT_SSH") == "" && os.Getenv("GIT_SSH_COMMAND") == "" { - os.Setenv("GIT_SSH_COMMAND", "ssh -o ControlMaster=no") - } - - // Phase 1. Download/update. - var stk importStack - mode := 0 - if *getT { - mode |= getTestDeps - } - args = downloadPaths(args) - for _, arg := range args { - download(arg, nil, &stk, mode) - } - exitIfErrors() - - // Phase 2. Rescan packages and re-evaluate args list. - - // Code we downloaded and all code that depends on it - // needs to be evicted from the package cache so that - // the information will be recomputed. Instead of keeping - // track of the reverse dependency information, evict - // everything. - for name := range packageCache { - delete(packageCache, name) - } - - // In order to rebuild packages information completely, - // we need to clear commands cache. Command packages are - // referring to evicted packages from the package cache. - // This leads to duplicated loads of the standard packages. - for name := range cmdCache { - delete(cmdCache, name) - } - - args = importPaths(args) - packagesForBuild(args) - - // Phase 3. Install. - if *getD { - // Download only. - // Check delayed until now so that importPaths - // and packagesForBuild have a chance to print errors. - return - } - - installPackages(args, true) -} - -// downloadPaths prepares the list of paths to pass to download. -// It expands ... patterns that can be expanded. If there is no match -// for a particular pattern, downloadPaths leaves it in the result list, -// in the hope that we can figure out the repository from the -// initial ...-free prefix. -func downloadPaths(args []string) []string { - args = importPathsNoDotExpansion(args) - var out []string - for _, a := range args { - if strings.Contains(a, "...") { - var expand []string - // Use matchPackagesInFS to avoid printing - // warnings. They will be printed by the - // eventual call to importPaths instead. - if build.IsLocalImport(a) { - expand = matchPackagesInFS(a) - } else { - expand = matchPackages(a) - } - if len(expand) > 0 { - out = append(out, expand...) - continue - } - } - out = append(out, a) - } - return out -} - -// downloadCache records the import paths we have already -// considered during the download, to avoid duplicate work when -// there is more than one dependency sequence leading to -// a particular package. -var downloadCache = map[string]bool{} - -// downloadRootCache records the version control repository -// root directories we have already considered during the download. -// For example, all the packages in the github.com/google/codesearch repo -// share the same root (the directory for that path), and we only need -// to run the hg commands to consider each repository once. -var downloadRootCache = map[string]bool{} - -// download runs the download half of the get command -// for the package named by the argument. -func download(arg string, parent *Package, stk *importStack, mode int) { - if mode&useVendor != 0 { - // Caller is responsible for expanding vendor paths. - panic("internal error: download mode has useVendor set") - } - load := func(path string, mode int) *Package { - if parent == nil { - return loadPackage(path, stk) - } - return loadImport(path, parent.Dir, parent, stk, nil, mode) - } - - p := load(arg, mode) - if p.Error != nil && p.Error.hard { - errorf("%s", p.Error) - return - } - - // loadPackage inferred the canonical ImportPath from arg. - // Use that in the following to prevent hysteresis effects - // in e.g. downloadCache and packageCache. - // This allows invocations such as: - // mkdir -p $GOPATH/src/github.com/user - // cd $GOPATH/src/github.com/user - // go get ./foo - // see: golang.org/issue/9767 - arg = p.ImportPath - - // There's nothing to do if this is a package in the standard library. - if p.Standard { - return - } - - // Only process each package once. - // (Unless we're fetching test dependencies for this package, - // in which case we want to process it again.) - if downloadCache[arg] && mode&getTestDeps == 0 { - return - } - downloadCache[arg] = true - - pkgs := []*Package{p} - wildcardOkay := len(*stk) == 0 - isWildcard := false - - // Download if the package is missing, or update if we're using -u. - if p.Dir == "" || *getU { - // The actual download. - stk.push(arg) - err := downloadPackage(p) - if err != nil { - errorf("%s", &PackageError{ImportStack: stk.copy(), Err: err.Error()}) - stk.pop() - return - } - stk.pop() - - args := []string{arg} - // If the argument has a wildcard in it, re-evaluate the wildcard. - // We delay this until after reloadPackage so that the old entry - // for p has been replaced in the package cache. - if wildcardOkay && strings.Contains(arg, "...") { - if build.IsLocalImport(arg) { - args = matchPackagesInFS(arg) - } else { - args = matchPackages(arg) - } - isWildcard = true - } - - // Clear all relevant package cache entries before - // doing any new loads. - for _, arg := range args { - p := packageCache[arg] - if p != nil { - delete(packageCache, p.Dir) - delete(packageCache, p.ImportPath) - } - } - - pkgs = pkgs[:0] - for _, arg := range args { - // Note: load calls loadPackage or loadImport, - // which push arg onto stk already. - // Do not push here too, or else stk will say arg imports arg. - p := load(arg, mode) - if p.Error != nil { - errorf("%s", p.Error) - continue - } - pkgs = append(pkgs, p) - } - } - - // Process package, which might now be multiple packages - // due to wildcard expansion. - for _, p := range pkgs { - if *getFix { - run(buildToolExec, stringList(tool("fix"), relPaths(p.allgofiles))) - - // The imports might have changed, so reload again. - p = reloadPackage(arg, stk) - if p.Error != nil { - errorf("%s", p.Error) - return - } - } - - if isWildcard { - // Report both the real package and the - // wildcard in any error message. - stk.push(p.ImportPath) - } - - // Process dependencies, now that we know what they are. - imports := p.Imports - if mode&getTestDeps != 0 { - // Process test dependencies when -t is specified. - // (But don't get test dependencies for test dependencies: - // we always pass mode 0 to the recursive calls below.) - imports = stringList(imports, p.TestImports, p.XTestImports) - } - for i, path := range imports { - if path == "C" { - continue - } - // Fail fast on import naming full vendor path. - // Otherwise expand path as needed for test imports. - // Note that p.Imports can have additional entries beyond p.build.Imports. - orig := path - if i < len(p.build.Imports) { - orig = p.build.Imports[i] - } - if j, ok := findVendor(orig); ok { - stk.push(path) - err := &PackageError{ - ImportStack: stk.copy(), - Err: "must be imported as " + path[j+len("vendor/"):], - } - stk.pop() - errorf("%s", err) - continue - } - // If this is a test import, apply vendor lookup now. - // We cannot pass useVendor to download, because - // download does caching based on the value of path, - // so it must be the fully qualified path already. - if i >= len(p.Imports) { - path = vendoredImportPath(p, path) - } - download(path, p, stk, 0) - } - - if isWildcard { - stk.pop() - } - } -} - -// downloadPackage runs the create or download command -// to make the first copy of or update a copy of the given package. -func downloadPackage(p *Package) error { - var ( - vcs *vcsCmd - repo, rootPath string - err error - ) - - security := secure - if *getInsecure { - security = insecure - } - - if p.build.SrcRoot != "" { - // Directory exists. Look for checkout along path to src. - vcs, rootPath, err = vcsFromDir(p.Dir, p.build.SrcRoot) - if err != nil { - return err - } - repo = "" // should be unused; make distinctive - - // Double-check where it came from. - if *getU && vcs.remoteRepo != nil { - dir := filepath.Join(p.build.SrcRoot, filepath.FromSlash(rootPath)) - remote, err := vcs.remoteRepo(vcs, dir) - if err != nil { - return err - } - repo = remote - if !*getF { - if rr, err := repoRootForImportPath(p.ImportPath, security); err == nil { - repo := rr.repo - if rr.vcs.resolveRepo != nil { - resolved, err := rr.vcs.resolveRepo(rr.vcs, dir, repo) - if err == nil { - repo = resolved - } - } - if remote != repo && rr.isCustom { - return fmt.Errorf("%s is a custom import path for %s, but %s is checked out from %s", rr.root, repo, dir, remote) - } - } - } - } - } else { - // Analyze the import path to determine the version control system, - // repository, and the import path for the root of the repository. - rr, err := repoRootForImportPath(p.ImportPath, security) - if err != nil { - return err - } - vcs, repo, rootPath = rr.vcs, rr.repo, rr.root - } - if !vcs.isSecure(repo) && !*getInsecure { - return fmt.Errorf("cannot download, %v uses insecure protocol", repo) - } - - if p.build.SrcRoot == "" { - // Package not found. Put in first directory of $GOPATH. - list := filepath.SplitList(buildContext.GOPATH) - if len(list) == 0 { - return fmt.Errorf("cannot download, $GOPATH not set. For more details see: 'go help gopath'") - } - // Guard against people setting GOPATH=$GOROOT. - if filepath.Clean(list[0]) == filepath.Clean(goroot) { - return fmt.Errorf("cannot download, $GOPATH must not be set to $GOROOT. For more details see: 'go help gopath'") - } - if _, err := os.Stat(filepath.Join(list[0], "src/cmd/go/alldocs.go")); err == nil { - return fmt.Errorf("cannot download, %s is a GOROOT, not a GOPATH. For more details see: 'go help gopath'", list[0]) - } - p.build.Root = list[0] - p.build.SrcRoot = filepath.Join(list[0], "src") - p.build.PkgRoot = filepath.Join(list[0], "pkg") - } - root := filepath.Join(p.build.SrcRoot, filepath.FromSlash(rootPath)) - // If we've considered this repository already, don't do it again. - if downloadRootCache[root] { - return nil - } - downloadRootCache[root] = true - - if buildV { - fmt.Fprintf(os.Stderr, "%s (download)\n", rootPath) - } - - // Check that this is an appropriate place for the repo to be checked out. - // The target directory must either not exist or have a repo checked out already. - meta := filepath.Join(root, "."+vcs.cmd) - st, err := os.Stat(meta) - if err == nil && !st.IsDir() { - return fmt.Errorf("%s exists but is not a directory", meta) - } - if err != nil { - // Metadata directory does not exist. Prepare to checkout new copy. - // Some version control tools require the target directory not to exist. - // We require that too, just to avoid stepping on existing work. - if _, err := os.Stat(root); err == nil { - return fmt.Errorf("%s exists but %s does not - stale checkout?", root, meta) - } - - _, err := os.Stat(p.build.Root) - gopathExisted := err == nil - - // Some version control tools require the parent of the target to exist. - parent, _ := filepath.Split(root) - if err = os.MkdirAll(parent, 0777); err != nil { - return err - } - if buildV && !gopathExisted && p.build.Root == buildContext.GOPATH { - fmt.Fprintf(os.Stderr, "created GOPATH=%s; see 'go help gopath'\n", p.build.Root) - } - - if err = vcs.create(root, repo); err != nil { - return err - } - } else { - // Metadata directory does exist; download incremental updates. - if err = vcs.download(root); err != nil { - return err - } - } - - if buildN { - // Do not show tag sync in -n; it's noise more than anything, - // and since we're not running commands, no tag will be found. - // But avoid printing nothing. - fmt.Fprintf(os.Stderr, "# cd %s; %s sync/update\n", root, vcs.cmd) - return nil - } - - // Select and sync to appropriate version of the repository. - tags, err := vcs.tags(root) - if err != nil { - return err - } - vers := runtime.Version() - if i := strings.Index(vers, " "); i >= 0 { - vers = vers[:i] - } - if err := vcs.tagSync(root, selectTag(vers, tags)); err != nil { - return err - } - - return nil -} - -// goTag matches go release tags such as go1 and go1.2.3. -// The numbers involved must be small (at most 4 digits), -// have no unnecessary leading zeros, and the version cannot -// end in .0 - it is go1, not go1.0 or go1.0.0. -var goTag = regexp.MustCompile( - `^go((0|[1-9][0-9]{0,3})\.)*([1-9][0-9]{0,3})$`, -) - -// selectTag returns the closest matching tag for a given version. -// Closest means the latest one that is not after the current release. -// Version "goX" (or "goX.Y" or "goX.Y.Z") matches tags of the same form. -// Version "release.rN" matches tags of the form "go.rN" (N being a floating-point number). -// Version "weekly.YYYY-MM-DD" matches tags like "go.weekly.YYYY-MM-DD". -// -// NOTE(rsc): Eventually we will need to decide on some logic here. -// For now, there is only "go1". This matches the docs in go help get. -func selectTag(goVersion string, tags []string) (match string) { - for _, t := range tags { - if t == "go1" { - return "go1" - } - } - return "" - - /* - if goTag.MatchString(goVersion) { - v := goVersion - for _, t := range tags { - if !goTag.MatchString(t) { - continue - } - if cmpGoVersion(match, t) < 0 && cmpGoVersion(t, v) <= 0 { - match = t - } - } - } - - return match - */ -} - -// cmpGoVersion returns -1, 0, +1 reporting whether -// x < y, x == y, or x > y. -func cmpGoVersion(x, y string) int { - // Malformed strings compare less than well-formed strings. - if !goTag.MatchString(x) { - return -1 - } - if !goTag.MatchString(y) { - return +1 - } - - // Compare numbers in sequence. - xx := strings.Split(x[len("go"):], ".") - yy := strings.Split(y[len("go"):], ".") - - for i := 0; i < len(xx) && i < len(yy); i++ { - // The Atoi are guaranteed to succeed - // because the versions match goTag. - xi, _ := strconv.Atoi(xx[i]) - yi, _ := strconv.Atoi(yy[i]) - if xi < yi { - return -1 - } else if xi > yi { - return +1 - } - } - - if len(xx) < len(yy) { - return -1 - } - if len(xx) > len(yy) { - return +1 - } - return 0 -} diff --git a/libgo/go/cmd/go/go_test.go b/libgo/go/cmd/go/go_test.go index eaa8685e31e..28fb5efe328 100644 --- a/libgo/go/cmd/go/go_test.go +++ b/libgo/go/cmd/go/go_test.go @@ -7,7 +7,6 @@ package main_test import ( "bytes" "fmt" - "go/build" "go/format" "internal/race" "internal/testenv" @@ -74,6 +73,13 @@ func init() { } } +// testGOROOT is the GOROOT to use when running testgo, a cmd/go binary +// build from this process's current GOROOT, but run from a different +// (temp) directory. +var testGOROOT string + +var testCC string + // The TestMain function creates a go command for testing purposes and // deletes it after the tests have been run. func TestMain(m *testing.M) { @@ -88,6 +94,20 @@ func TestMain(m *testing.M) { os.Exit(2) } + out, err = exec.Command("go", "env", "GOROOT").CombinedOutput() + if err != nil { + fmt.Fprintf(os.Stderr, "could not find testing GOROOT: %v\n%s", err, out) + os.Exit(2) + } + testGOROOT = strings.TrimSpace(string(out)) + + out, err = exec.Command("go", "env", "CC").CombinedOutput() + if err != nil { + fmt.Fprintf(os.Stderr, "could not find testing CC: %v\n%s", err, out) + os.Exit(2) + } + testCC = strings.TrimSpace(string(out)) + if out, err := exec.Command("./testgo"+exeSuffix, "env", "CGO_ENABLED").Output(); err != nil { fmt.Fprintf(os.Stderr, "running testgo failed: %v\n", err) canRun = false @@ -100,7 +120,9 @@ func TestMain(m *testing.M) { switch runtime.GOOS { case "linux", "darwin", "freebsd", "windows": - canRace = canCgo && runtime.GOARCH == "amd64" && runtime.Compiler != "gccgo" + // The race detector doesn't work on Alpine Linux: + // golang.org/issue/14481 + canRace = canCgo && runtime.GOARCH == "amd64" && !isAlpineLinux() && runtime.Compiler != "gccgo" } } @@ -111,7 +133,7 @@ func TestMain(m *testing.M) { if home, ccacheDir := os.Getenv("HOME"), os.Getenv("CCACHE_DIR"); home != "" && ccacheDir == "" { // On some systems the default C compiler is ccache. // Setting HOME to a non-existent directory will break - // those systems. Set CCACHE_DIR to cope. Issue 17668. + // those systems. Set CCACHE_DIR to cope. Issue 17668. os.Setenv("CCACHE_DIR", filepath.Join(home, ".ccache")) } os.Setenv("HOME", "/test-go-home-does-not-exist") @@ -125,6 +147,14 @@ func TestMain(m *testing.M) { os.Exit(r) } +func isAlpineLinux() bool { + if runtime.GOOS != "linux" { + return false + } + fi, err := os.Lstat("/etc/alpine-release") + return err == nil && fi.Mode().IsRegular() +} + // The length of an mtime tick on this system. This is an estimate of // how long we need to sleep to ensure that the mtime of two files is // different. @@ -251,6 +281,13 @@ func (tg *testgoData) unsetenv(name string) { } } +func (tg *testgoData) goTool() string { + if tg.wd == "" { + return "./testgo" + exeSuffix + } + return filepath.Join(tg.wd, "testgo"+exeSuffix) +} + // doRun runs the test go command, recording stdout and stderr and // returning exit status. func (tg *testgoData) doRun(args []string) error { @@ -264,13 +301,20 @@ func (tg *testgoData) doRun(args []string) error { } } } - tg.t.Logf("running testgo %v", args) - var prog string - if tg.wd == "" { - prog = "./testgo" + exeSuffix - } else { - prog = filepath.Join(tg.wd, "testgo"+exeSuffix) + + hasGoroot := false + for _, v := range tg.env { + if strings.HasPrefix(v, "GOROOT=") { + hasGoroot = true + break + } + } + prog := tg.goTool() + if !hasGoroot { + tg.setenv("GOROOT", testGOROOT) } + + tg.t.Logf("running testgo %v", args) cmd := exec.Command(prog, args...) tg.stdout.Reset() tg.stderr.Reset() @@ -365,7 +409,7 @@ func (tg *testgoData) doGrepMatch(match string, b *bytes.Buffer) bool { // doGrep looks for a regular expression in a buffer and fails if it // is not found. The name argument is the name of the output we are -// searching, "output" or "error". The msg argument is logged on +// searching, "output" or "error". The msg argument is logged on // failure. func (tg *testgoData) doGrep(match string, b *bytes.Buffer, name, msg string) { if !tg.doGrepMatch(match, b) { @@ -1354,7 +1398,7 @@ func TestInstallFailsWithNoBuildableFiles(t *testing.T) { tg.setenv("GOPATH", filepath.Join(tg.pwd(), "testdata")) tg.setenv("CGO_ENABLED", "0") tg.runFail("install", "cgotest") - tg.grepStderr("no buildable Go source files", "go install cgotest did not report 'no buildable Go Source files'") + tg.grepStderr("build constraints exclude all Go files", "go install cgotest did not report 'build constraints exclude all Go files'") } func TestRelativeGOBINFail(t *testing.T) { @@ -1483,11 +1527,11 @@ func TestGoGetNonPkg(t *testing.T) { tg.setenv("GOPATH", tg.path(".")) tg.setenv("GOBIN", tg.path("gobin")) tg.runFail("get", "-d", "golang.org/x/tools") - tg.grepStderr("golang.org/x/tools: no buildable Go source files", "missing error") + tg.grepStderr("golang.org/x/tools: no Go files", "missing error") tg.runFail("get", "-d", "-u", "golang.org/x/tools") - tg.grepStderr("golang.org/x/tools: no buildable Go source files", "missing error") + tg.grepStderr("golang.org/x/tools: no Go files", "missing error") tg.runFail("get", "-d", "golang.org/x/tools") - tg.grepStderr("golang.org/x/tools: no buildable Go source files", "missing error") + tg.grepStderr("golang.org/x/tools: no Go files", "missing error") } func TestGoGetTestOnlyPkg(t *testing.T) { @@ -1753,7 +1797,7 @@ func TestDefaultGOPATHPrintedSearchList(t *testing.T) { tg.grepStderr(regexp.QuoteMeta(tg.path("home/go/src/github.com/golang/example/hello"))+`.*from \$GOPATH`, "expected default GOPATH") } -// Issue 4186. go get cannot be used to download packages to $GOROOT. +// Issue 4186. go get cannot be used to download packages to $GOROOT. // Test that without GOPATH set, go get should fail. func TestGoGetIntoGOROOT(t *testing.T) { testenv.MustHaveExternalNetwork(t) @@ -1895,10 +1939,7 @@ func TestGoTestDashIDashOWritesBinary(t *testing.T) { // Issue 4568. func TestSymlinksList(t *testing.T) { - switch runtime.GOOS { - case "plan9", "windows": - t.Skipf("skipping symlink test on %s", runtime.GOOS) - } + testenv.MustHaveSymlink(t) tg := testgo(t) defer tg.cleanup() @@ -1916,10 +1957,7 @@ func TestSymlinksList(t *testing.T) { // Issue 14054. func TestSymlinksVendor(t *testing.T) { - switch runtime.GOOS { - case "plan9", "windows": - t.Skipf("skipping symlink test on %s", runtime.GOOS) - } + testenv.MustHaveSymlink(t) tg := testgo(t) defer tg.cleanup() @@ -1943,10 +1981,7 @@ func TestSymlinksVendor(t *testing.T) { // Issue 15201. func TestSymlinksVendor15201(t *testing.T) { - switch runtime.GOOS { - case "plan9", "windows": - t.Skipf("skipping symlink test on %s", runtime.GOOS) - } + testenv.MustHaveSymlink(t) tg := testgo(t) defer tg.cleanup() @@ -1963,10 +1998,7 @@ func TestSymlinksVendor15201(t *testing.T) { } func TestSymlinksInternal(t *testing.T) { - switch runtime.GOOS { - case "plan9", "windows": - t.Skipf("skipping symlink test on %s", runtime.GOOS) - } + testenv.MustHaveSymlink(t) tg := testgo(t) defer tg.cleanup() @@ -2030,8 +2062,10 @@ func TestCaseCollisions(t *testing.T) { )`) tg.tempFile("src/example/a/pkg/pkg.go", `package pkg`) tg.tempFile("src/example/a/Pkg/pkg.go", `package pkg`) - tg.runFail("list", "example/a") - tg.grepStderr("case-insensitive import collision", "go list example/a did not report import collision") + tg.run("list", "-json", "example/a") + tg.grepStdout("case-insensitive import collision", "go list -json example/a did not report import collision") + tg.runFail("build", "example/a") + tg.grepStderr("case-insensitive import collision", "go build example/a did not report import collision") tg.tempFile("src/example/b/file.go", `package b`) tg.tempFile("src/example/b/FILE.go", `package b`) f, err := os.Open(tg.path("src/example/b")) @@ -2049,6 +2083,38 @@ func TestCaseCollisions(t *testing.T) { } tg.runFail(args...) tg.grepStderr("case-insensitive file name collision", "go list example/b did not report file name collision") + + tg.runFail("list", "example/a/pkg", "example/a/Pkg") + tg.grepStderr("case-insensitive import collision", "go list example/a/pkg example/a/Pkg did not report import collision") + tg.run("list", "-json", "-e", "example/a/pkg", "example/a/Pkg") + tg.grepStdout("case-insensitive import collision", "go list -json -e example/a/pkg example/a/Pkg did not report import collision") + tg.runFail("build", "example/a/pkg", "example/a/Pkg") + tg.grepStderr("case-insensitive import collision", "go build example/a/pkg example/a/Pkg did not report import collision") +} + +// Issue 17451, 17662. +func TestSymlinkWarning(t *testing.T) { + tg := testgo(t) + defer tg.cleanup() + tg.parallel() + tg.makeTempdir() + tg.setenv("GOPATH", tg.path(".")) + + tg.tempDir("src/example/xx") + tg.tempDir("yy/zz") + tg.tempFile("yy/zz/zz.go", "package zz\n") + if err := os.Symlink(tg.path("yy"), tg.path("src/example/xx/yy")); err != nil { + t.Skip("symlink failed: %v", err) + } + tg.run("list", "example/xx/z...") + tg.grepStdoutNot(".", "list should not have matched anything") + tg.grepStderr("matched no packages", "list should have reported that pattern matched no packages") + tg.grepStderrNot("symlink", "list should not have reported symlink") + + tg.run("list", "example/xx/...") + tg.grepStdoutNot(".", "list should not have matched anything") + tg.grepStderr("matched no packages", "list should have reported that pattern matched no packages") + tg.grepStderr("ignoring symlink", "list should have reported symlink") } // Issue 8181. @@ -2195,29 +2261,6 @@ func TestCoverageUsesAtomicModeForRace(t *testing.T) { checkCoverage(tg, data) } -func TestCoverageUsesActualSettingToOverrideEvenForRace(t *testing.T) { - if testing.Short() { - t.Skip("don't build libraries for coverage in short mode") - } - if !canRace { - t.Skip("skipping because race detector not supported") - } - - tg := testgo(t) - defer tg.cleanup() - tg.creatingTemp("testdata/cover.out") - tg.run("test", "-short", "-race", "-cover", "encoding/binary", "-covermode=count", "-coverprofile=testdata/cover.out") - data := tg.getStdout() + tg.getStderr() - if out, err := ioutil.ReadFile("testdata/cover.out"); err != nil { - t.Error(err) - } else { - if !bytes.Contains(out, []byte("mode: count")) { - t.Error("missing mode: count") - } - } - checkCoverage(tg, data) -} - func TestCoverageImportMainLoop(t *testing.T) { tg := testgo(t) defer tg.cleanup() @@ -2228,6 +2271,20 @@ func TestCoverageImportMainLoop(t *testing.T) { tg.grepStderr("not an importable package", "did not detect import main") } +func TestPluginNonMain(t *testing.T) { + wd, err := os.Getwd() + if err != nil { + t.Fatal(err) + } + + pkg := filepath.Join(wd, "testdata", "testdep", "p2") + + tg := testgo(t) + defer tg.cleanup() + + tg.runFail("build", "-buildmode=plugin", pkg) +} + func TestTestEmpty(t *testing.T) { if !canRace { t.Skip("no race detector") @@ -2235,7 +2292,6 @@ func TestTestEmpty(t *testing.T) { wd, _ := os.Getwd() testdata := filepath.Join(wd, "testdata") - for _, dir := range []string{"pkg", "test", "xtest", "pkgtest", "pkgxtest", "pkgtestxtest", "testxtest"} { t.Run(dir, func(t *testing.T) { tg := testgo(t) @@ -2250,10 +2306,36 @@ func TestTestEmpty(t *testing.T) { } } +func TestNoGoError(t *testing.T) { + wd, _ := os.Getwd() + testdata := filepath.Join(wd, "testdata") + for _, dir := range []string{"empty/test", "empty/xtest", "empty/testxtest", "exclude", "exclude/ignore", "exclude/empty"} { + t.Run(dir, func(t *testing.T) { + tg := testgo(t) + defer tg.cleanup() + tg.setenv("GOPATH", testdata) + tg.cd(filepath.Join(testdata, "src")) + tg.runFail("build", "./"+dir) + var want string + if strings.Contains(dir, "test") { + want = "no non-test Go files in " + } else if dir == "exclude" { + want = "build constraints exclude all Go files in " + } else { + want = "no Go files in " + } + tg.grepStderr(want, "wrong reason for failure") + }) + } +} + func TestTestRaceInstall(t *testing.T) { if !canRace { t.Skip("no race detector") } + if testing.Short() && testenv.Builder() == "" { + t.Skip("don't rebuild the standard library in short mode") + } tg := testgo(t) defer tg.cleanup() @@ -2308,6 +2390,19 @@ func TestCoverageWithCgo(t *testing.T) { } } +func TestCgoAsmError(t *testing.T) { + if !canCgo { + t.Skip("skipping because cgo not enabled") + } + + tg := testgo(t) + tg.parallel() + defer tg.cleanup() + tg.setenv("GOPATH", filepath.Join(tg.pwd(), "testdata")) + tg.runFail("build", "cgoasm") + tg.grepBoth("package using cgo has Go assembly file", "did not detect Go assembly file") +} + func TestCgoDependsOnSyscall(t *testing.T) { if testing.Short() { t.Skip("skipping test that removes $GOROOT/pkg/*_race in short mode") @@ -2445,7 +2540,7 @@ import "C" func main() { C.f() }`) tg.setenv("GOPATH", tg.path(".")) tg.run("build", "-n", "-compiler", "gccgo", "cgoref") - tg.grepStderr(`gccgo.*\-L alibpath \-lalib`, `no Go-inline "#cgo LDFLAGS:" ("-L alibpath -lalib") passed to gccgo linking stage`) + tg.grepStderr(`gccgo.*\-L [^ ]*alibpath \-lalib`, `no Go-inline "#cgo LDFLAGS:" ("-L alibpath -lalib") passed to gccgo linking stage`) } func TestListTemplateContextFunction(t *testing.T) { @@ -2535,7 +2630,7 @@ func TestGoBuildInTestOnlyDirectoryFailsWithAGoodError(t *testing.T) { tg := testgo(t) defer tg.cleanup() tg.runFail("build", "./testdata/testonly") - tg.grepStderr("no buildable Go", "go build ./testdata/testonly produced unexpected error") + tg.grepStderr("no non-test Go files in", "go build ./testdata/testonly produced unexpected error") } func TestGoTestDetectsTestOnlyImportCycles(t *testing.T) { @@ -2674,8 +2769,7 @@ func TestGoGetInternalWildcard(t *testing.T) { } func TestGoVetWithExternalTests(t *testing.T) { - testenv.MustHaveExternalNetwork(t) - + skipIfGccgo(t, "gccgo does not have vet") tg := testgo(t) defer tg.cleanup() tg.makeTempdir() @@ -2686,19 +2780,39 @@ func TestGoVetWithExternalTests(t *testing.T) { } func TestGoVetWithTags(t *testing.T) { - testenv.MustHaveExternalNetwork(t) - + skipIfGccgo(t, "gccgo does not have vet") tg := testgo(t) defer tg.cleanup() tg.makeTempdir() tg.run("install", "cmd/vet") tg.setenv("GOPATH", filepath.Join(tg.pwd(), "testdata")) tg.runFail("vet", "-tags", "tagtest", "vetpkg") - tg.grepBoth(`c\.go.*wrong number of args for format`, "go get vetpkg did not run scan tagged file") + tg.grepBoth(`c\.go.*wrong number of args for format`, "go vet vetpkg did not run scan tagged file") +} + +func TestGoVetWithFlagsOn(t *testing.T) { + skipIfGccgo(t, "gccgo does not have vet") + tg := testgo(t) + defer tg.cleanup() + tg.makeTempdir() + tg.run("install", "cmd/vet") + tg.setenv("GOPATH", filepath.Join(tg.pwd(), "testdata")) + tg.runFail("vet", "-printf", "vetpkg") + tg.grepBoth("missing argument for Printf", "go vet -printf vetpkg did not find missing argument for Printf") +} + +func TestGoVetWithFlagsOff(t *testing.T) { + skipIfGccgo(t, "gccgo does not have vet") + tg := testgo(t) + defer tg.cleanup() + tg.makeTempdir() + tg.run("install", "cmd/vet") + tg.setenv("GOPATH", filepath.Join(tg.pwd(), "testdata")) + tg.run("vet", "-printf=false", "vetpkg") } -// Issue 9767. -func TestGoGetRscIoToolstash(t *testing.T) { +// Issue 9767, 19769. +func TestGoGetDotSlashDownload(t *testing.T) { testenv.MustHaveExternalNetwork(t) tg := testgo(t) @@ -2706,7 +2820,7 @@ func TestGoGetRscIoToolstash(t *testing.T) { tg.tempDir("src/rsc.io") tg.setenv("GOPATH", tg.path(".")) tg.cd(tg.path("src/rsc.io")) - tg.run("get", "./toolstash") + tg.run("get", "./pprof_mac_fix") } // Issue 13037: Was not parsing tags in 404 served over HTTPS @@ -3047,17 +3161,8 @@ func TestGoInstallPkgdir(t *testing.T) { } func TestGoTestRaceInstallCgo(t *testing.T) { - skipIfGccgo(t, "gccgo has no race detector") - - switch sys := runtime.GOOS + "/" + runtime.GOARCH; sys { - case "darwin/amd64", "freebsd/amd64", "linux/amd64", "windows/amd64": - // ok - default: - t.Skip("no race detector on %s", sys) - } - - if !build.Default.CgoEnabled { - t.Skip("no race detector without cgo") + if !canRace { + t.Skip("skipping because race detector not supported") } // golang.org/issue/10500. @@ -3071,7 +3176,7 @@ func TestGoTestRaceInstallCgo(t *testing.T) { tg.run("test", "-race", "-i", "runtime/race") new, err := os.Stat(cgo) tg.must(err) - if new.ModTime() != old.ModTime() { + if !new.ModTime().Equal(old.ModTime()) { t.Fatalf("go test -i runtime/race reinstalled cmd/cgo") } } @@ -3100,7 +3205,7 @@ func TestGoTestRaceFailures(t *testing.T) { func TestGoTestImportErrorStack(t *testing.T) { const out = `package testdep/p1 (test) imports testdep/p2 - imports testdep/p3: no buildable Go source files` + imports testdep/p3: build constraints exclude all Go files ` tg := testgo(t) defer tg.cleanup() @@ -3141,6 +3246,20 @@ func TestGoGetUpdate(t *testing.T) { tg.run("get", "-d", "-u", "github.com/rsc/go-get-issue-9224-cmd") } +// Issue #20512. +func TestGoGetRace(t *testing.T) { + testenv.MustHaveExternalNetwork(t) + if !canRace { + t.Skip("skipping because race detector not supported") + } + + tg := testgo(t) + defer tg.cleanup() + tg.makeTempdir() + tg.setenv("GOPATH", tg.path(".")) + tg.run("get", "-race", "github.com/rsc/go-get-issue-9224-cmd") +} + func TestGoGetDomainRoot(t *testing.T) { // golang.org/issue/9357. // go get foo.io (not foo.io/subdir) was not working consistently. @@ -3477,7 +3596,7 @@ func TestBinaryOnlyPackages(t *testing.T) { func F() { p1.F(true) } `) tg.runFail("install", "p2") - tg.grepStderr("no buildable Go source files", "did not complain about missing sources") + tg.grepStderr("no Go files", "did not complain about missing sources") tg.tempFile("src/p1/missing.go", `//go:binary-only-package @@ -3675,6 +3794,28 @@ func TestMatchesOnlyBenchmarkIsOK(t *testing.T) { tg.grepBoth(okPattern, "go test did not say ok") } +func TestBenchmarkLabels(t *testing.T) { + tg := testgo(t) + defer tg.cleanup() + // TODO: tg.parallel() + tg.setenv("GOPATH", filepath.Join(tg.pwd(), "testdata")) + tg.run("test", "-run", "^$", "-bench", ".", "bench") + tg.grepStdout(`(?m)^goos: `+runtime.GOOS, "go test did not print goos") + tg.grepStdout(`(?m)^goarch: `+runtime.GOARCH, "go test did not print goarch") + tg.grepStdout(`(?m)^pkg: bench`, "go test did not say pkg: bench") + tg.grepBothNot(`(?s)pkg:.*pkg:`, "go test said pkg multiple times") +} + +func TestBenchmarkLabelsOutsideGOPATH(t *testing.T) { + tg := testgo(t) + defer tg.cleanup() + // TODO: tg.parallel() + tg.run("test", "-run", "^$", "-bench", ".", "testdata/standalone_benchmark_test.go") + tg.grepStdout(`(?m)^goos: `+runtime.GOOS, "go test did not print goos") + tg.grepStdout(`(?m)^goarch: `+runtime.GOARCH, "go test did not print goarch") + tg.grepBothNot(`(?m)^pkg:`, "go test did say pkg:") +} + func TestMatchesOnlyTestIsOK(t *testing.T) { tg := testgo(t) defer tg.cleanup() @@ -3804,3 +3945,431 @@ func TestA(t *testing.T) {}`) tg.grepStdout("pkgs$", "expected package not listed") tg.grepStdout("pkgs/a", "expected package not listed") } + +// Issue 18975. +func TestFFLAGS(t *testing.T) { + if !canCgo { + t.Skip("skipping because cgo not enabled") + } + + tg := testgo(t) + defer tg.cleanup() + tg.parallel() + + tg.tempFile("p/src/p/main.go", `package main + // #cgo FFLAGS: -no-such-fortran-flag + import "C" + func main() {} + `) + tg.tempFile("p/src/p/a.f", `! comment`) + tg.setenv("GOPATH", tg.path("p")) + + // This should normally fail because we are passing an unknown flag, + // but issue #19080 points to Fortran compilers that succeed anyhow. + // To work either way we call doRun directly rather than run or runFail. + tg.doRun([]string{"build", "-x", "p"}) + + tg.grepStderr("no-such-fortran-flag", `missing expected "-no-such-fortran-flag"`) +} + +// Issue 19198. +// This is really a cmd/link issue but this is a convenient place to test it. +func TestDuplicateGlobalAsmSymbols(t *testing.T) { + skipIfGccgo(t, "gccgo does not use cmd/asm") + if runtime.GOARCH != "386" && runtime.GOARCH != "amd64" { + t.Skipf("skipping test on %s", runtime.GOARCH) + } + if !canCgo { + t.Skip("skipping because cgo not enabled") + } + + tg := testgo(t) + defer tg.cleanup() + tg.parallel() + + asm := ` +#include "textflag.h" + +DATA sym<>+0x0(SB)/8,$0 +GLOBL sym<>(SB),(NOPTR+RODATA),$8 + +TEXT ·Data(SB),NOSPLIT,$0 + MOVB sym<>(SB), AX + MOVB AX, ret+0(FP) + RET +` + tg.tempFile("go/src/a/a.s", asm) + tg.tempFile("go/src/a/a.go", `package a; func Data() uint8`) + tg.tempFile("go/src/b/b.s", asm) + tg.tempFile("go/src/b/b.go", `package b; func Data() uint8`) + tg.tempFile("go/src/p/p.go", ` +package main +import "a" +import "b" +import "C" +func main() { + _ = a.Data() + b.Data() +} +`) + tg.setenv("GOPATH", tg.path("go")) + exe := filepath.Join(tg.tempdir, "p.exe") + tg.creatingTemp(exe) + tg.run("build", "-o", exe, "p") +} + +func TestBuildTagsNoComma(t *testing.T) { + skipIfGccgo(t, "gccgo has no standard packages") + tg := testgo(t) + defer tg.cleanup() + tg.makeTempdir() + tg.setenv("GOPATH", tg.path("go")) + tg.run("install", "-tags", "tag1 tag2", "math") + tg.runFail("install", "-tags", "tag1,tag2", "math") + tg.grepBoth("space-separated list contains comma", "-tags with a comma-separated list didn't error") + tg.runFail("build", "-tags", "tag1,tag2", "math") + tg.grepBoth("space-separated list contains comma", "-tags with a comma-separated list didn't error") +} + +func copyFile(src, dst string, perm os.FileMode) error { + sf, err := os.Open(src) + if err != nil { + return err + } + defer sf.Close() + + df, err := os.OpenFile(dst, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, perm) + if err != nil { + return err + } + + _, err = io.Copy(df, sf) + err2 := df.Close() + if err != nil { + return err + } + return err2 +} + +func TestExecutableGOROOT(t *testing.T) { + skipIfGccgo(t, "gccgo has no GOROOT") + if runtime.GOOS == "openbsd" { + t.Skipf("test case does not work on %s, missing os.Executable", runtime.GOOS) + } + + // Env with no GOROOT. + var env []string + for _, e := range os.Environ() { + if !strings.HasPrefix(e, "GOROOT=") { + env = append(env, e) + } + } + + check := func(t *testing.T, exe, want string) { + cmd := exec.Command(exe, "env", "GOROOT") + cmd.Env = env + out, err := cmd.CombinedOutput() + if err != nil { + t.Fatalf("%s env GOROOT: %v, %s", exe, err, out) + } + goroot, err := filepath.EvalSymlinks(strings.TrimSpace(string(out))) + if err != nil { + t.Fatal(err) + } + want, err = filepath.EvalSymlinks(want) + if err != nil { + t.Fatal(err) + } + if !strings.EqualFold(goroot, want) { + t.Errorf("go env GOROOT:\nhave %s\nwant %s", goroot, want) + } else { + t.Logf("go env GOROOT: %s", goroot) + } + } + + // Note: Must not call tg methods inside subtests: tg is attached to outer t. + tg := testgo(t) + defer tg.cleanup() + + tg.makeTempdir() + tg.tempDir("new/bin") + newGoTool := tg.path("new/bin/go" + exeSuffix) + tg.must(copyFile(tg.goTool(), newGoTool, 0775)) + newRoot := tg.path("new") + + t.Run("RelocatedExe", func(t *testing.T) { + t.Skip("TODO: skipping known broken test; see golang.org/issue/20284") + + // Should fall back to default location in binary. + // No way to dig out other than look at source code. + data, err := ioutil.ReadFile("../../runtime/internal/sys/zversion.go") + if err != nil { + t.Fatal(err) + } + m := regexp.MustCompile("const DefaultGoroot = `([^`]+)`").FindStringSubmatch(string(data)) + if m == nil { + t.Fatal("cannot find DefaultGoroot in ../../runtime/internal/sys/zversion.go") + } + check(t, newGoTool, m[1]) + }) + + // If the binary is sitting in a bin dir next to ../pkg/tool, that counts as a GOROOT, + // so it should find the new tree. + tg.tempDir("new/pkg/tool") + t.Run("RelocatedTree", func(t *testing.T) { + check(t, newGoTool, newRoot) + }) + + tg.tempDir("other/bin") + symGoTool := tg.path("other/bin/go" + exeSuffix) + + // Symlink into go tree should still find go tree. + t.Run("SymlinkedExe", func(t *testing.T) { + testenv.MustHaveSymlink(t) + if err := os.Symlink(newGoTool, symGoTool); err != nil { + t.Fatal(err) + } + check(t, symGoTool, newRoot) + }) +} + +func TestNeedVersion(t *testing.T) { + skipIfGccgo(t, "gccgo does not use cmd/compile") + tg := testgo(t) + defer tg.cleanup() + tg.parallel() + tg.tempFile("goversion.go", `package main; func main() {}`) + path := tg.path("goversion.go") + tg.setenv("TESTGO_VERSION", "go1.testgo") + tg.runFail("run", path) + tg.grepStderr("compile", "does not match go tool version") +} + +// Test that user can override default code generation flags. +func TestUserOverrideFlags(t *testing.T) { + skipIfGccgo(t, "gccgo does not use -gcflags") + if !canCgo { + t.Skip("skipping because cgo not enabled") + } + if runtime.GOOS != "linux" { + // We are testing platform-independent code, so it's + // OK to skip cases that work differently. + t.Skipf("skipping on %s because test only works if c-archive implies -shared", runtime.GOOS) + } + + tg := testgo(t) + defer tg.cleanup() + tg.parallel() + tg.tempFile("override.go", `package main + +import "C" + +//export GoFunc +func GoFunc() {} + +func main() {}`) + tg.creatingTemp("override.a") + tg.creatingTemp("override.h") + tg.run("build", "-x", "-buildmode=c-archive", "-gcflags=-shared=false", tg.path("override.go")) + tg.grepStderr("compile .*-shared .*-shared=false", "user can not override code generation flag") +} + +func TestCgoFlagContainsSpace(t *testing.T) { + if !canCgo { + t.Skip("skipping because cgo not enabled") + } + + tg := testgo(t) + defer tg.cleanup() + + ccName := filepath.Base(testCC) + + tg.tempFile(fmt.Sprintf("src/%s/main.go", ccName), fmt.Sprintf(`package main + import ( + "os" + "os/exec" + "path/filepath" + "strings" + ) + + func main() { + cmd := exec.Command(%q, os.Args[1:]...) + cmd.Stdin = os.Stdin + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + err := cmd.Run() + if err != nil { + panic(err) + } + + if os.Args[len(os.Args)-1] == "trivial.c" { + return + } + if filepath.Base(os.Args[len(os.Args)-1]) == "_cgo_defun.c" { + return + } + + var success bool + for _, arg := range os.Args { + switch { + case strings.Contains(arg, "c flags"): + if success { + panic("duplicate CFLAGS") + } + success = true + case strings.Contains(arg, "ld flags"): + if success { + panic("duplicate LDFLAGS") + } + success = true + } + } + if !success { + panic("args should contains '-Ic flags' or '-Lld flags'") + } + } + `, testCC)) + tg.cd(tg.path(fmt.Sprintf("src/%s", ccName))) + tg.run("build") + tg.setenv("CC", tg.path(fmt.Sprintf("src/%s/%s", ccName, ccName))) + + tg.tempFile("src/cgo/main.go", `package main + // #cgo CFLAGS: -I"c flags" + // #cgo LDFLAGS: -L"ld flags" + import "C" + func main() {} + `) + tg.cd(tg.path("src/cgo")) + tg.run("run", "main.go") +} + +// Issue #20435. +func TestGoTestRaceCoverModeFailures(t *testing.T) { + if !canRace { + t.Skip("skipping because race detector not supported") + } + + tg := testgo(t) + tg.parallel() + defer tg.cleanup() + tg.setenv("GOPATH", filepath.Join(tg.pwd(), "testdata")) + + tg.run("test", "testrace") + + tg.runFail("test", "-race", "-covermode=set", "testrace") + tg.grepStderr(`-covermode must be "atomic", not "set", when -race is enabled`, "-race -covermode=set was allowed") + tg.grepBothNot("PASS", "something passed") +} + +// Issue 9737: verify that GOARM and GO386 affect the computed build ID. +func TestBuildIDContainsArchModeEnv(t *testing.T) { + if testing.Short() { + t.Skip("skipping in short mode") + } + + var tg *testgoData + testWith := func(before, after func()) func(*testing.T) { + return func(t *testing.T) { + tg = testgo(t) + defer tg.cleanup() + tg.tempFile("src/mycmd/x.go", `package main +func main() {}`) + tg.setenv("GOPATH", tg.path(".")) + + tg.cd(tg.path("src/mycmd")) + tg.setenv("GOOS", "linux") + before() + tg.run("install", "mycmd") + after() + tg.wantStale("mycmd", "build ID mismatch", "should be stale after environment variable change") + } + } + + t.Run("386", testWith(func() { + tg.setenv("GOARCH", "386") + tg.setenv("GO386", "387") + }, func() { + tg.setenv("GO386", "sse2") + })) + + t.Run("arm", testWith(func() { + tg.setenv("GOARCH", "arm") + tg.setenv("GOARM", "5") + }, func() { + tg.setenv("GOARM", "7") + })) +} + +func TestTestRegexps(t *testing.T) { + tg := testgo(t) + defer tg.cleanup() + tg.setenv("GOPATH", filepath.Join(tg.pwd(), "testdata")) + tg.run("test", "-cpu=1", "-run=X/Y", "-bench=X/Y", "-count=2", "-v", "testregexp") + var lines []string + for _, line := range strings.SplitAfter(tg.getStdout(), "\n") { + if strings.Contains(line, "=== RUN") || strings.Contains(line, "--- BENCH") || strings.Contains(line, "LOG") { + lines = append(lines, line) + } + } + + // Important parts: + // TestX is run, twice + // TestX/Y is run, twice + // TestXX is run, twice + // TestZ is not run + // BenchmarkX is run but only with N=1, once + // BenchmarkXX is run but only with N=1, once + // BenchmarkX/Y is run in full, twice + want := `=== RUN TestX +=== RUN TestX/Y + x_test.go:6: LOG: X running + x_test.go:8: LOG: Y running +=== RUN TestXX + z_test.go:10: LOG: XX running +=== RUN TestX +=== RUN TestX/Y + x_test.go:6: LOG: X running + x_test.go:8: LOG: Y running +=== RUN TestXX + z_test.go:10: LOG: XX running +--- BENCH: BenchmarkX/Y + x_test.go:15: LOG: Y running N=1 + x_test.go:15: LOG: Y running N=100 + x_test.go:15: LOG: Y running N=10000 + x_test.go:15: LOG: Y running N=1000000 + x_test.go:15: LOG: Y running N=100000000 + x_test.go:15: LOG: Y running N=2000000000 +--- BENCH: BenchmarkX/Y + x_test.go:15: LOG: Y running N=1 + x_test.go:15: LOG: Y running N=100 + x_test.go:15: LOG: Y running N=10000 + x_test.go:15: LOG: Y running N=1000000 + x_test.go:15: LOG: Y running N=100000000 + x_test.go:15: LOG: Y running N=2000000000 +--- BENCH: BenchmarkX + x_test.go:13: LOG: X running N=1 +--- BENCH: BenchmarkXX + z_test.go:18: LOG: XX running N=1 +` + + have := strings.Join(lines, "") + if have != want { + t.Errorf("reduced output:<<<\n%s>>> want:<<<\n%s>>>", have, want) + } +} + +func TestListTests(t *testing.T) { + var tg *testgoData + testWith := func(listName, expected string) func(*testing.T) { + return func(t *testing.T) { + tg = testgo(t) + defer tg.cleanup() + tg.run("test", "./testdata/src/testlist/...", fmt.Sprintf("-list=%s", listName)) + tg.grepStdout(expected, fmt.Sprintf("-test.list=%s returned %q, expected %s", listName, tg.getStdout(), expected)) + } + } + + t.Run("Test", testWith("Test", "TestSimple")) + t.Run("Bench", testWith("Benchmark", "BenchmarkSimple")) + t.Run("Example1", testWith("Example", "ExampleSimple")) + t.Run("Example2", testWith("Example", "ExampleWithEmptyOutput")) +} diff --git a/libgo/go/cmd/go/go_unix_test.go b/libgo/go/cmd/go/go_unix_test.go index c445a2ec73b..f6e10ca59c7 100644 --- a/libgo/go/cmd/go/go_unix_test.go +++ b/libgo/go/cmd/go/go_unix_test.go @@ -19,9 +19,13 @@ func TestGoBuildUmask(t *testing.T) { tg := testgo(t) defer tg.cleanup() tg.tempFile("x.go", `package main; func main() {}`) - tg.creatingTemp("x") - tg.run("build", tg.path("x.go")) - fi, err := os.Stat("x") + // Make sure artifact will be output to /tmp/... in case the user + // has POSIX acl's on their go source tree. + // See issue 17909. + exe := tg.path("x") + tg.creatingTemp(exe) + tg.run("build", "-o", exe, tg.path("x.go")) + fi, err := os.Stat(exe) if err != nil { t.Fatal(err) } diff --git a/libgo/go/cmd/go/help.go b/libgo/go/cmd/go/help.go deleted file mode 100644 index 6b7422c95c6..00000000000 --- a/libgo/go/cmd/go/help.go +++ /dev/null @@ -1,610 +0,0 @@ -// Copyright 2011 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package main - -var helpC = &Command{ - UsageLine: "c", - Short: "calling between Go and C", - Long: ` -There are two different ways to call between Go and C/C++ code. - -The first is the cgo tool, which is part of the Go distribution. For -information on how to use it see the cgo documentation (go doc cmd/cgo). - -The second is the SWIG program, which is a general tool for -interfacing between languages. For information on SWIG see -http://swig.org/. When running go build, any file with a .swig -extension will be passed to SWIG. Any file with a .swigcxx extension -will be passed to SWIG with the -c++ option. - -When either cgo or SWIG is used, go build will pass any .c, .m, .s, -or .S files to the C compiler, and any .cc, .cpp, .cxx files to the C++ -compiler. The CC or CXX environment variables may be set to determine -the C or C++ compiler, respectively, to use. - `, -} - -var helpPackages = &Command{ - UsageLine: "packages", - Short: "description of package lists", - Long: ` -Many commands apply to a set of packages: - - go action [packages] - -Usually, [packages] is a list of import paths. - -An import path that is a rooted path or that begins with -a . or .. element is interpreted as a file system path and -denotes the package in that directory. - -Otherwise, the import path P denotes the package found in -the directory DIR/src/P for some DIR listed in the GOPATH -environment variable (For more details see: 'go help gopath'). - -If no import paths are given, the action applies to the -package in the current directory. - -There are four reserved names for paths that should not be used -for packages to be built with the go tool: - -- "main" denotes the top-level package in a stand-alone executable. - -- "all" expands to all package directories found in all the GOPATH -trees. For example, 'go list all' lists all the packages on the local -system. - -- "std" is like all but expands to just the packages in the standard -Go library. - -- "cmd" expands to the Go repository's commands and their -internal libraries. - -Import paths beginning with "cmd/" only match source code in -the Go repository. - -An import path is a pattern if it includes one or more "..." wildcards, -each of which can match any string, including the empty string and -strings containing slashes. Such a pattern expands to all package -directories found in the GOPATH trees with names matching the -patterns. As a special case, x/... matches x as well as x's subdirectories. -For example, net/... expands to net and packages in its subdirectories. - -An import path can also name a package to be downloaded from -a remote repository. Run 'go help importpath' for details. - -Every package in a program must have a unique import path. -By convention, this is arranged by starting each path with a -unique prefix that belongs to you. For example, paths used -internally at Google all begin with 'google', and paths -denoting remote repositories begin with the path to the code, -such as 'github.com/user/repo'. - -Packages in a program need not have unique package names, -but there are two reserved package names with special meaning. -The name main indicates a command, not a library. -Commands are built into binaries and cannot be imported. -The name documentation indicates documentation for -a non-Go program in the directory. Files in package documentation -are ignored by the go command. - -As a special case, if the package list is a list of .go files from a -single directory, the command is applied to a single synthesized -package made up of exactly those files, ignoring any build constraints -in those files and ignoring any other files in the directory. - -Directory and file names that begin with "." or "_" are ignored -by the go tool, as are directories named "testdata". - `, -} - -var helpImportPath = &Command{ - UsageLine: "importpath", - Short: "import path syntax", - Long: ` - -An import path (see 'go help packages') denotes a package stored in the local -file system. In general, an import path denotes either a standard package (such -as "unicode/utf8") or a package found in one of the work spaces (For more -details see: 'go help gopath'). - -Relative import paths - -An import path beginning with ./ or ../ is called a relative path. -The toolchain supports relative import paths as a shortcut in two ways. - -First, a relative path can be used as a shorthand on the command line. -If you are working in the directory containing the code imported as -"unicode" and want to run the tests for "unicode/utf8", you can type -"go test ./utf8" instead of needing to specify the full path. -Similarly, in the reverse situation, "go test .." will test "unicode" from -the "unicode/utf8" directory. Relative patterns are also allowed, like -"go test ./..." to test all subdirectories. See 'go help packages' for details -on the pattern syntax. - -Second, if you are compiling a Go program not in a work space, -you can use a relative path in an import statement in that program -to refer to nearby code also not in a work space. -This makes it easy to experiment with small multipackage programs -outside of the usual work spaces, but such programs cannot be -installed with "go install" (there is no work space in which to install them), -so they are rebuilt from scratch each time they are built. -To avoid ambiguity, Go programs cannot use relative import paths -within a work space. - -Remote import paths - -Certain import paths also -describe how to obtain the source code for the package using -a revision control system. - -A few common code hosting sites have special syntax: - - Bitbucket (Git, Mercurial) - - import "bitbucket.org/user/project" - import "bitbucket.org/user/project/sub/directory" - - GitHub (Git) - - import "github.com/user/project" - import "github.com/user/project/sub/directory" - - Launchpad (Bazaar) - - import "launchpad.net/project" - import "launchpad.net/project/series" - import "launchpad.net/project/series/sub/directory" - - import "launchpad.net/~user/project/branch" - import "launchpad.net/~user/project/branch/sub/directory" - - IBM DevOps Services (Git) - - import "hub.jazz.net/git/user/project" - import "hub.jazz.net/git/user/project/sub/directory" - -For code hosted on other servers, import paths may either be qualified -with the version control type, or the go tool can dynamically fetch -the import path over https/http and discover where the code resides -from a tag in the HTML. - -To declare the code location, an import path of the form - - repository.vcs/path - -specifies the given repository, with or without the .vcs suffix, -using the named version control system, and then the path inside -that repository. The supported version control systems are: - - Bazaar .bzr - Git .git - Mercurial .hg - Subversion .svn - -For example, - - import "example.org/user/foo.hg" - -denotes the root directory of the Mercurial repository at -example.org/user/foo or foo.hg, and - - import "example.org/repo.git/foo/bar" - -denotes the foo/bar directory of the Git repository at -example.org/repo or repo.git. - -When a version control system supports multiple protocols, -each is tried in turn when downloading. For example, a Git -download tries https://, then git+ssh://. - -By default, downloads are restricted to known secure protocols -(e.g. https, ssh). To override this setting for Git downloads, the -GIT_ALLOW_PROTOCOL environment variable can be set (For more details see: -'go help environment'). - -If the import path is not a known code hosting site and also lacks a -version control qualifier, the go tool attempts to fetch the import -over https/http and looks for a tag in the document's HTML -. - -The meta tag has the form: - - - -The import-prefix is the import path corresponding to the repository -root. It must be a prefix or an exact match of the package being -fetched with "go get". If it's not an exact match, another http -request is made at the prefix to verify the tags match. - -The meta tag should appear as early in the file as possible. -In particular, it should appear before any raw JavaScript or CSS, -to avoid confusing the go command's restricted parser. - -The vcs is one of "git", "hg", "svn", etc, - -The repo-root is the root of the version control system -containing a scheme and not containing a .vcs qualifier. - -For example, - - import "example.org/pkg/foo" - -will result in the following requests: - - https://example.org/pkg/foo?go-get=1 (preferred) - http://example.org/pkg/foo?go-get=1 (fallback, only with -insecure) - -If that page contains the meta tag - - - -the go tool will verify that https://example.org/?go-get=1 contains the -same meta tag and then git clone https://code.org/r/p/exproj into -GOPATH/src/example.org. - -New downloaded packages are written to the first directory listed in the GOPATH -environment variable (For more details see: 'go help gopath'). - -The go command attempts to download the version of the -package appropriate for the Go release being used. -Run 'go help get' for more. - -Import path checking - -When the custom import path feature described above redirects to a -known code hosting site, each of the resulting packages has two possible -import paths, using the custom domain or the known hosting site. - -A package statement is said to have an "import comment" if it is immediately -followed (before the next newline) by a comment of one of these two forms: - - package math // import "path" - package math /* import "path" */ - -The go command will refuse to install a package with an import comment -unless it is being referred to by that import path. In this way, import comments -let package authors make sure the custom import path is used and not a -direct path to the underlying code hosting site. - -Import path checking is disabled for code found within vendor trees. -This makes it possible to copy code into alternate locations in vendor trees -without needing to update import comments. - -See https://golang.org/s/go14customimport for details. - `, -} - -var helpGopath = &Command{ - UsageLine: "gopath", - Short: "GOPATH environment variable", - Long: ` -The Go path is used to resolve import statements. -It is implemented by and documented in the go/build package. - -The GOPATH environment variable lists places to look for Go code. -On Unix, the value is a colon-separated string. -On Windows, the value is a semicolon-separated string. -On Plan 9, the value is a list. - -If the environment variable is unset, GOPATH defaults -to a subdirectory named "go" in the user's home directory -($HOME/go on Unix, %USERPROFILE%\go on Windows), -unless that directory holds a Go distribution. -Run "go env GOPATH" to see the current GOPATH. - -See https://golang.org/wiki/SettingGOPATH to set a custom GOPATH. - -Each directory listed in GOPATH must have a prescribed structure: - -The src directory holds source code. The path below src -determines the import path or executable name. - -The pkg directory holds installed package objects. -As in the Go tree, each target operating system and -architecture pair has its own subdirectory of pkg -(pkg/GOOS_GOARCH). - -If DIR is a directory listed in the GOPATH, a package with -source in DIR/src/foo/bar can be imported as "foo/bar" and -has its compiled form installed to "DIR/pkg/GOOS_GOARCH/foo/bar.a". - -The bin directory holds compiled commands. -Each command is named for its source directory, but only -the final element, not the entire path. That is, the -command with source in DIR/src/foo/quux is installed into -DIR/bin/quux, not DIR/bin/foo/quux. The "foo/" prefix is stripped -so that you can add DIR/bin to your PATH to get at the -installed commands. If the GOBIN environment variable is -set, commands are installed to the directory it names instead -of DIR/bin. GOBIN must be an absolute path. - -Here's an example directory layout: - - GOPATH=/home/user/go - - /home/user/go/ - src/ - foo/ - bar/ (go code in package bar) - x.go - quux/ (go code in package main) - y.go - bin/ - quux (installed command) - pkg/ - linux_amd64/ - foo/ - bar.a (installed package object) - -Go searches each directory listed in GOPATH to find source code, -but new packages are always downloaded into the first directory -in the list. - -See https://golang.org/doc/code.html for an example. - -Internal Directories - -Code in or below a directory named "internal" is importable only -by code in the directory tree rooted at the parent of "internal". -Here's an extended version of the directory layout above: - - /home/user/go/ - src/ - crash/ - bang/ (go code in package bang) - b.go - foo/ (go code in package foo) - f.go - bar/ (go code in package bar) - x.go - internal/ - baz/ (go code in package baz) - z.go - quux/ (go code in package main) - y.go - - -The code in z.go is imported as "foo/internal/baz", but that -import statement can only appear in source files in the subtree -rooted at foo. The source files foo/f.go, foo/bar/x.go, and -foo/quux/y.go can all import "foo/internal/baz", but the source file -crash/bang/b.go cannot. - -See https://golang.org/s/go14internal for details. - -Vendor Directories - -Go 1.6 includes support for using local copies of external dependencies -to satisfy imports of those dependencies, often referred to as vendoring. - -Code below a directory named "vendor" is importable only -by code in the directory tree rooted at the parent of "vendor", -and only using an import path that omits the prefix up to and -including the vendor element. - -Here's the example from the previous section, -but with the "internal" directory renamed to "vendor" -and a new foo/vendor/crash/bang directory added: - - /home/user/go/ - src/ - crash/ - bang/ (go code in package bang) - b.go - foo/ (go code in package foo) - f.go - bar/ (go code in package bar) - x.go - vendor/ - crash/ - bang/ (go code in package bang) - b.go - baz/ (go code in package baz) - z.go - quux/ (go code in package main) - y.go - -The same visibility rules apply as for internal, but the code -in z.go is imported as "baz", not as "foo/vendor/baz". - -Code in vendor directories deeper in the source tree shadows -code in higher directories. Within the subtree rooted at foo, an import -of "crash/bang" resolves to "foo/vendor/crash/bang", not the -top-level "crash/bang". - -Code in vendor directories is not subject to import path -checking (see 'go help importpath'). - -When 'go get' checks out or updates a git repository, it now also -updates submodules. - -Vendor directories do not affect the placement of new repositories -being checked out for the first time by 'go get': those are always -placed in the main GOPATH, never in a vendor subtree. - -See https://golang.org/s/go15vendor for details. - `, -} - -var helpEnvironment = &Command{ - UsageLine: "environment", - Short: "environment variables", - Long: ` - -The go command, and the tools it invokes, examine a few different -environment variables. For many of these, you can see the default -value of on your system by running 'go env NAME', where NAME is the -name of the variable. - -General-purpose environment variables: - - GCCGO - The gccgo command to run for 'go build -compiler=gccgo'. - GOARCH - The architecture, or processor, for which to compile code. - Examples are amd64, 386, arm, ppc64. - GOBIN - The directory where 'go install' will install a command. - GOOS - The operating system for which to compile code. - Examples are linux, darwin, windows, netbsd. - GOPATH - For more details see: 'go help gopath'. - GORACE - Options for the race detector. - See https://golang.org/doc/articles/race_detector.html. - GOROOT - The root of the go tree. - -Environment variables for use with cgo: - - CC - The command to use to compile C code. - CGO_ENABLED - Whether the cgo command is supported. Either 0 or 1. - CGO_CFLAGS - Flags that cgo will pass to the compiler when compiling - C code. - CGO_CPPFLAGS - Flags that cgo will pass to the compiler when compiling - C or C++ code. - CGO_CXXFLAGS - Flags that cgo will pass to the compiler when compiling - C++ code. - CGO_FFLAGS - Flags that cgo will pass to the compiler when compiling - Fortran code. - CGO_LDFLAGS - Flags that cgo will pass to the compiler when linking. - CXX - The command to use to compile C++ code. - PKG_CONFIG - Path to pkg-config tool. - -Architecture-specific environment variables: - - GOARM - For GOARCH=arm, the ARM architecture for which to compile. - Valid values are 5, 6, 7. - GO386 - For GOARCH=386, the floating point instruction set. - Valid values are 387, sse2. - -Special-purpose environment variables: - - GOROOT_FINAL - The root of the installed Go tree, when it is - installed in a location other than where it is built. - File names in stack traces are rewritten from GOROOT to - GOROOT_FINAL. - GO_EXTLINK_ENABLED - Whether the linker should use external linking mode - when using -linkmode=auto with code that uses cgo. - Set to 0 to disable external linking mode, 1 to enable it. - GIT_ALLOW_PROTOCOL - Defined by Git. A colon-separated list of schemes that are allowed to be used - with git fetch/clone. If set, any scheme not explicitly mentioned will be - considered insecure by 'go get'. - GCCGOTOOLDIR - If set, where to find gccgo tools, such as cgo. - The default is based on how gccgo was configured. - `, -} - -var helpFileType = &Command{ - UsageLine: "filetype", - Short: "file types", - Long: ` -The go command examines the contents of a restricted set of files -in each directory. It identifies which files to examine based on -the extension of the file name. These extensions are: - - .go - Go source files. - .c, .h - C source files. - If the package uses cgo or SWIG, these will be compiled with the - OS-native compiler (typically gcc); otherwise they will - trigger an error. - .cc, .cpp, .cxx, .hh, .hpp, .hxx - C++ source files. Only useful with cgo or SWIG, and always - compiled with the OS-native compiler. - .m - Objective-C source files. Only useful with cgo, and always - compiled with the OS-native compiler. - .s, .S - Assembler source files. - If the package uses cgo or SWIG, these will be assembled with the - OS-native assembler (typically gcc (sic)); otherwise they - will be assembled with the Go assembler. - .swig, .swigcxx - SWIG definition files. - .syso - System object files. - -Files of each of these types except .syso may contain build -constraints, but the go command stops scanning for build constraints -at the first item in the file that is not a blank line or //-style -line comment. See the go/build package documentation for -more details. - -Non-test Go source files can also include a //go:binary-only-package -comment, indicating that the package sources are included -for documentation only and must not be used to build the -package binary. This enables distribution of Go packages in -their compiled form alone. See the go/build package documentation -for more details. - `, -} - -var helpBuildmode = &Command{ - UsageLine: "buildmode", - Short: "description of build modes", - Long: ` -The 'go build' and 'go install' commands take a -buildmode argument which -indicates which kind of object file is to be built. Currently supported values -are: - - -buildmode=archive - Build the listed non-main packages into .a files. Packages named - main are ignored. - - -buildmode=c-archive - Build the listed main package, plus all packages it imports, - into a C archive file. The only callable symbols will be those - functions exported using a cgo //export comment. Requires - exactly one main package to be listed. - - -buildmode=c-shared - Build the listed main packages, plus all packages that they - import, into C shared libraries. The only callable symbols will - be those functions exported using a cgo //export comment. - Non-main packages are ignored. - - -buildmode=default - Listed main packages are built into executables and listed - non-main packages are built into .a files (the default - behavior). - - -buildmode=shared - Combine all the listed non-main packages into a single shared - library that will be used when building with the -linkshared - option. Packages named main are ignored. - - -buildmode=exe - Build the listed main packages and everything they import into - executables. Packages not named main are ignored. - - -buildmode=pie - Build the listed main packages and everything they import into - position independent executables (PIE). Packages not named - main are ignored. - - -buildmode=plugin - Build the listed main packages, plus all packages that they - import, into a Go plugin. Packages not named main are ignored. -`, -} diff --git a/libgo/go/cmd/go/http.go b/libgo/go/cmd/go/http.go deleted file mode 100644 index dcb4e9fea5f..00000000000 --- a/libgo/go/cmd/go/http.go +++ /dev/null @@ -1,120 +0,0 @@ -// Copyright 2012 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// +build !cmd_go_bootstrap - -// This code is compiled into the real 'go' binary, but it is not -// compiled into the binary that is built during all.bash, so as -// to avoid needing to build net (and thus use cgo) during the -// bootstrap process. - -package main - -import ( - "cmd/internal/browser" - "crypto/tls" - "fmt" - "io" - "io/ioutil" - "log" - "net/http" - "net/url" - "time" -) - -// httpClient is the default HTTP client, but a variable so it can be -// changed by tests, without modifying http.DefaultClient. -var httpClient = http.DefaultClient - -// impatientInsecureHTTPClient is used in -insecure mode, -// when we're connecting to https servers that might not be there -// or might be using self-signed certificates. -var impatientInsecureHTTPClient = &http.Client{ - Timeout: 5 * time.Second, - Transport: &http.Transport{ - Proxy: http.ProxyFromEnvironment, - TLSClientConfig: &tls.Config{ - InsecureSkipVerify: true, - }, - }, -} - -type httpError struct { - status string - statusCode int - url string -} - -func (e *httpError) Error() string { - return fmt.Sprintf("%s: %s", e.url, e.status) -} - -// httpGET returns the data from an HTTP GET request for the given URL. -func httpGET(url string) ([]byte, error) { - resp, err := httpClient.Get(url) - if err != nil { - return nil, err - } - defer resp.Body.Close() - if resp.StatusCode != 200 { - err := &httpError{status: resp.Status, statusCode: resp.StatusCode, url: url} - - return nil, err - } - b, err := ioutil.ReadAll(resp.Body) - if err != nil { - return nil, fmt.Errorf("%s: %v", url, err) - } - return b, nil -} - -// httpsOrHTTP returns the body of either the importPath's -// https resource or, if unavailable, the http resource. -func httpsOrHTTP(importPath string, security securityMode) (urlStr string, body io.ReadCloser, err error) { - fetch := func(scheme string) (urlStr string, res *http.Response, err error) { - u, err := url.Parse(scheme + "://" + importPath) - if err != nil { - return "", nil, err - } - u.RawQuery = "go-get=1" - urlStr = u.String() - if buildV { - log.Printf("Fetching %s", urlStr) - } - if security == insecure && scheme == "https" { // fail earlier - res, err = impatientInsecureHTTPClient.Get(urlStr) - } else { - res, err = httpClient.Get(urlStr) - } - return - } - closeBody := func(res *http.Response) { - if res != nil { - res.Body.Close() - } - } - urlStr, res, err := fetch("https") - if err != nil { - if buildV { - log.Printf("https fetch failed: %v", err) - } - if security == insecure { - closeBody(res) - urlStr, res, err = fetch("http") - } - } - if err != nil { - closeBody(res) - return "", nil, err - } - // Note: accepting a non-200 OK here, so people can serve a - // meta import in their http 404 page. - if buildV { - log.Printf("Parsing meta tags from %s (status code %d)", urlStr, res.StatusCode) - } - return urlStr, res.Body, nil -} - -func queryEscape(s string) string { return url.QueryEscape(s) } -func openBrowser(url string) bool { return browser.Open(url) } diff --git a/libgo/go/cmd/go/internal/base/base.go b/libgo/go/cmd/go/internal/base/base.go new file mode 100644 index 00000000000..aff33f70d8d --- /dev/null +++ b/libgo/go/cmd/go/internal/base/base.go @@ -0,0 +1,173 @@ +// Copyright 2017 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package base defines shared basic pieces of the go command, +// in particular logging and the Command structure. +package base + +import ( + "bytes" + "errors" + "flag" + "fmt" + "go/scanner" + "log" + "os" + "os/exec" + "strings" + "sync" + + "cmd/go/internal/cfg" + "cmd/go/internal/str" +) + +// A Command is an implementation of a go command +// like go build or go fix. +type Command struct { + // Run runs the command. + // The args are the arguments after the command name. + Run func(cmd *Command, args []string) + + // UsageLine is the one-line usage message. + // The first word in the line is taken to be the command name. + UsageLine string + + // Short is the short description shown in the 'go help' output. + Short string + + // Long is the long message shown in the 'go help ' output. + Long string + + // Flag is a set of flags specific to this command. + Flag flag.FlagSet + + // CustomFlags indicates that the command will do its own + // flag parsing. + CustomFlags bool +} + +// Commands lists the available commands and help topics. +// The order here is the order in which they are printed by 'go help'. +var Commands []*Command + +// Name returns the command's name: the first word in the usage line. +func (c *Command) Name() string { + name := c.UsageLine + i := strings.Index(name, " ") + if i >= 0 { + name = name[:i] + } + return name +} + +func (c *Command) Usage() { + fmt.Fprintf(os.Stderr, "usage: %s\n\n", c.UsageLine) + fmt.Fprintf(os.Stderr, "%s\n", strings.TrimSpace(c.Long)) + os.Exit(2) +} + +// Runnable reports whether the command can be run; otherwise +// it is a documentation pseudo-command such as importpath. +func (c *Command) Runnable() bool { + return c.Run != nil +} + +var atExitFuncs []func() + +func AtExit(f func()) { + atExitFuncs = append(atExitFuncs, f) +} + +func Exit() { + for _, f := range atExitFuncs { + f() + } + os.Exit(exitStatus) +} + +func Fatalf(format string, args ...interface{}) { + Errorf(format, args...) + Exit() +} + +func Errorf(format string, args ...interface{}) { + log.Printf(format, args...) + SetExitStatus(1) +} + +func ExitIfErrors() { + if exitStatus != 0 { + Exit() + } +} + +var exitStatus = 0 +var exitMu sync.Mutex + +func SetExitStatus(n int) { + exitMu.Lock() + if exitStatus < n { + exitStatus = n + } + exitMu.Unlock() +} + +// Run runs the command, with stdout and stderr +// connected to the go command's own stdout and stderr. +// If the command fails, Run reports the error using Errorf. +func Run(cmdargs ...interface{}) { + cmdline := str.StringList(cmdargs...) + if cfg.BuildN || cfg.BuildX { + fmt.Printf("%s\n", strings.Join(cmdline, " ")) + if cfg.BuildN { + return + } + } + + cmd := exec.Command(cmdline[0], cmdline[1:]...) + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + if err := cmd.Run(); err != nil { + Errorf("%v", err) + } +} + +// RunStdin is like run but connects Stdin. +func RunStdin(cmdline []string) { + cmd := exec.Command(cmdline[0], cmdline[1:]...) + cmd.Stdin = os.Stdin + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + cmd.Env = cfg.OrigEnv + StartSigHandlers() + if err := cmd.Run(); err != nil { + Errorf("%v", err) + } +} + +// Usage is the usage-reporting function, filled in by package main +// but here for reference by other packages. +var Usage func() + +// ExpandScanner expands a scanner.List error into all the errors in the list. +// The default Error method only shows the first error +// and does not shorten paths. +func ExpandScanner(err error) error { + // Look for parser errors. + if err, ok := err.(scanner.ErrorList); ok { + // Prepare error with \n before each message. + // When printed in something like context: %v + // this will put the leading file positions each on + // its own line. It will also show all the errors + // instead of just the first, as err.Error does. + var buf bytes.Buffer + for _, e := range err { + e.Pos.Filename = ShortPath(e.Pos.Filename) + buf.WriteString("\n") + buf.WriteString(e.Error()) + } + return errors.New(buf.String()) + } + return err +} diff --git a/libgo/go/cmd/go/internal/base/env.go b/libgo/go/cmd/go/internal/base/env.go new file mode 100644 index 00000000000..fcade9d84e2 --- /dev/null +++ b/libgo/go/cmd/go/internal/base/env.go @@ -0,0 +1,37 @@ +// Copyright 2017 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package base + +import "strings" + +// EnvForDir returns a copy of the environment +// suitable for running in the given directory. +// The environment is the current process's environment +// but with an updated $PWD, so that an os.Getwd in the +// child will be faster. +func EnvForDir(dir string, base []string) []string { + // Internally we only use rooted paths, so dir is rooted. + // Even if dir is not rooted, no harm done. + return MergeEnvLists([]string{"PWD=" + dir}, base) +} + +// MergeEnvLists merges the two environment lists such that +// variables with the same name in "in" replace those in "out". +// This always returns a newly allocated slice. +func MergeEnvLists(in, out []string) []string { + out = append([]string(nil), out...) +NextVar: + for _, inkv := range in { + k := strings.SplitAfterN(inkv, "=", 2)[0] + for i, outkv := range out { + if strings.HasPrefix(outkv, k) { + out[i] = inkv + continue NextVar + } + } + out = append(out, inkv) + } + return out +} diff --git a/libgo/go/cmd/go/internal/base/flag.go b/libgo/go/cmd/go/internal/base/flag.go new file mode 100644 index 00000000000..5e03e640588 --- /dev/null +++ b/libgo/go/cmd/go/internal/base/flag.go @@ -0,0 +1,35 @@ +// Copyright 2017 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package base + +import ( + "flag" + + "cmd/go/internal/cfg" + "cmd/go/internal/str" +) + +// A StringsFlag is a command-line flag that interprets its argument +// as a space-separated list of possibly-quoted strings. +type StringsFlag []string + +func (v *StringsFlag) Set(s string) error { + var err error + *v, err = str.SplitQuotedFields(s) + if *v == nil { + *v = []string{} + } + return err +} + +func (v *StringsFlag) String() string { + return "" +} + +// AddBuildFlagsNX adds the -n and -x build flags to the flag set. +func AddBuildFlagsNX(flags *flag.FlagSet) { + flags.BoolVar(&cfg.BuildN, "n", false, "") + flags.BoolVar(&cfg.BuildX, "x", false, "") +} diff --git a/libgo/go/cmd/go/internal/base/path.go b/libgo/go/cmd/go/internal/base/path.go new file mode 100644 index 00000000000..4f12fa8c28c --- /dev/null +++ b/libgo/go/cmd/go/internal/base/path.go @@ -0,0 +1,74 @@ +// Copyright 2017 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package base + +import ( + "os" + "path/filepath" + "strings" +) + +func getwd() string { + wd, err := os.Getwd() + if err != nil { + Fatalf("cannot determine current directory: %v", err) + } + return wd +} + +var Cwd = getwd() + +// ShortPath returns an absolute or relative name for path, whatever is shorter. +func ShortPath(path string) string { + if rel, err := filepath.Rel(Cwd, path); err == nil && len(rel) < len(path) { + return rel + } + return path +} + +// RelPaths returns a copy of paths with absolute paths +// made relative to the current directory if they would be shorter. +func RelPaths(paths []string) []string { + var out []string + // TODO(rsc): Can this use Cwd from above? + pwd, _ := os.Getwd() + for _, p := range paths { + rel, err := filepath.Rel(pwd, p) + if err == nil && len(rel) < len(p) { + p = rel + } + out = append(out, p) + } + return out +} + +// FilterDotUnderscoreFiles returns a slice containing all elements +// of path whose base name doesn't begin with "." or "_". +func FilterDotUnderscoreFiles(path []string) []string { + var out []string // lazily initialized + for i, p := range path { + base := filepath.Base(p) + if strings.HasPrefix(base, ".") || strings.HasPrefix(base, "_") { + if out == nil { + out = append(make([]string, 0, len(path)), path[:i]...) + } + continue + } + if out != nil { + out = append(out, p) + } + } + if out == nil { + return path + } + return out +} + +// IsTestFile reports whether the source file is a set of tests and should therefore +// be excluded from coverage analysis. +func IsTestFile(file string) bool { + // We don't cover tests, only the code they test. + return strings.HasSuffix(file, "_test.go") +} diff --git a/libgo/go/cmd/go/internal/base/signal.go b/libgo/go/cmd/go/internal/base/signal.go new file mode 100644 index 00000000000..54d11876d07 --- /dev/null +++ b/libgo/go/cmd/go/internal/base/signal.go @@ -0,0 +1,31 @@ +// Copyright 2012 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package base + +import ( + "os" + "os/signal" + "sync" +) + +// Interrupted is closed when the go command receives an interrupt signal. +var Interrupted = make(chan struct{}) + +// processSignals setups signal handler. +func processSignals() { + sig := make(chan os.Signal) + signal.Notify(sig, signalsToIgnore...) + go func() { + <-sig + close(Interrupted) + }() +} + +var onceProcessSignals sync.Once + +// StartSigHandlers starts the signal handlers. +func StartSigHandlers() { + onceProcessSignals.Do(processSignals) +} diff --git a/libgo/go/cmd/go/internal/base/signal_notunix.go b/libgo/go/cmd/go/internal/base/signal_notunix.go new file mode 100644 index 00000000000..9e869b03ea8 --- /dev/null +++ b/libgo/go/cmd/go/internal/base/signal_notunix.go @@ -0,0 +1,17 @@ +// Copyright 2012 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build plan9 windows + +package base + +import ( + "os" +) + +var signalsToIgnore = []os.Signal{os.Interrupt} + +// SignalTrace is the signal to send to make a Go program +// crash with a stack trace (no such signal in this case). +var SignalTrace os.Signal = nil diff --git a/libgo/go/cmd/go/internal/base/signal_unix.go b/libgo/go/cmd/go/internal/base/signal_unix.go new file mode 100644 index 00000000000..4ca3da99226 --- /dev/null +++ b/libgo/go/cmd/go/internal/base/signal_unix.go @@ -0,0 +1,18 @@ +// Copyright 2012 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build darwin dragonfly freebsd linux nacl netbsd openbsd solaris + +package base + +import ( + "os" + "syscall" +) + +var signalsToIgnore = []os.Signal{os.Interrupt, syscall.SIGQUIT} + +// SignalTrace is the signal to send to make a Go program +// crash with a stack trace. +var SignalTrace os.Signal = syscall.SIGQUIT diff --git a/libgo/go/cmd/go/internal/base/tool.go b/libgo/go/cmd/go/internal/base/tool.go new file mode 100644 index 00000000000..c907772c00a --- /dev/null +++ b/libgo/go/cmd/go/internal/base/tool.go @@ -0,0 +1,53 @@ +// Copyright 2017 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package base + +import ( + "fmt" + "go/build" + "os" + "path/filepath" + "runtime" + + "cmd/go/internal/cfg" +) + +// Configuration for finding tool binaries. +var ( + ToolGOOS = runtime.GOOS + ToolGOARCH = runtime.GOARCH + ToolIsWindows = ToolGOOS == "windows" + ToolDir = build.ToolDir +) + +const ToolWindowsExtension = ".exe" + +// Tool returns the path to the named tool (for example, "vet"). +// If the tool cannot be found, Tool exits the process. +func Tool(toolName string) string { + toolPath := filepath.Join(ToolDir, toolName) + if ToolIsWindows { + toolPath += ToolWindowsExtension + } + if len(cfg.BuildToolexec) > 0 { + return toolPath + } + // Give a nice message if there is no tool with that name. + if _, err := os.Stat(toolPath); err != nil { + if isInGoToolsRepo(toolName) { + fmt.Fprintf(os.Stderr, "go tool: no such tool %q; to install:\n\tgo get golang.org/x/tools/cmd/%s\n", toolName, toolName) + } else { + fmt.Fprintf(os.Stderr, "go tool: no such tool %q\n", toolName) + } + SetExitStatus(2) + Exit() + } + return toolPath +} + +// TODO: Delete. +func isInGoToolsRepo(toolName string) bool { + return false +} diff --git a/libgo/go/cmd/go/internal/bug/bug.go b/libgo/go/cmd/go/internal/bug/bug.go new file mode 100644 index 00000000000..963da94c493 --- /dev/null +++ b/libgo/go/cmd/go/internal/bug/bug.go @@ -0,0 +1,218 @@ +// Copyright 2016 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package bug implements the ``go bug'' command. +package bug + +import ( + "bytes" + "fmt" + "io" + "io/ioutil" + "os" + "os/exec" + "path/filepath" + "regexp" + "runtime" + "strings" + + "cmd/go/internal/base" + "cmd/go/internal/cfg" + "cmd/go/internal/envcmd" + "cmd/go/internal/web" +) + +var CmdBug = &base.Command{ + Run: runBug, + UsageLine: "bug", + Short: "start a bug report", + Long: ` +Bug opens the default browser and starts a new bug report. +The report includes useful system information. + `, +} + +func init() { + CmdBug.Flag.BoolVar(&cfg.BuildV, "v", false, "") +} + +func runBug(cmd *base.Command, args []string) { + var buf bytes.Buffer + buf.WriteString(bugHeader) + inspectGoVersion(&buf) + fmt.Fprint(&buf, "#### System details\n\n") + fmt.Fprintln(&buf, "```") + fmt.Fprintf(&buf, "go version %s %s/%s\n", runtime.Version(), runtime.GOOS, runtime.GOARCH) + env := cfg.CmdEnv + env = append(env, envcmd.ExtraEnvVars()...) + for _, e := range env { + // Hide the TERM environment variable from "go bug". + // See issue #18128 + if e.Name != "TERM" { + fmt.Fprintf(&buf, "%s=\"%s\"\n", e.Name, e.Value) + } + } + printGoDetails(&buf) + printOSDetails(&buf) + printCDetails(&buf) + fmt.Fprintln(&buf, "```") + + body := buf.String() + url := "https://github.com/golang/go/issues/new?body=" + web.QueryEscape(body) + if !web.OpenBrowser(url) { + fmt.Print("Please file a new issue at golang.org/issue/new using this template:\n\n") + fmt.Print(body) + } +} + +const bugHeader = `Please answer these questions before submitting your issue. Thanks! + +#### What did you do? +If possible, provide a recipe for reproducing the error. +A complete runnable program is good. +A link on play.golang.org is best. + + +#### What did you expect to see? + + +#### What did you see instead? + + +` + +func printGoDetails(w io.Writer) { + printCmdOut(w, "GOROOT/bin/go version: ", filepath.Join(runtime.GOROOT(), "bin/go"), "version") + printCmdOut(w, "GOROOT/bin/go tool compile -V: ", filepath.Join(runtime.GOROOT(), "bin/go"), "tool", "compile", "-V") +} + +func printOSDetails(w io.Writer) { + switch runtime.GOOS { + case "darwin": + printCmdOut(w, "uname -v: ", "uname", "-v") + printCmdOut(w, "", "sw_vers") + case "linux": + printCmdOut(w, "uname -sr: ", "uname", "-sr") + printCmdOut(w, "", "lsb_release", "-a") + printGlibcVersion(w) + case "openbsd", "netbsd", "freebsd", "dragonfly": + printCmdOut(w, "uname -v: ", "uname", "-v") + case "solaris": + out, err := ioutil.ReadFile("/etc/release") + if err == nil { + fmt.Fprintf(w, "/etc/release: %s\n", out) + } else { + if cfg.BuildV { + fmt.Printf("failed to read /etc/release: %v\n", err) + } + } + } +} + +func printCDetails(w io.Writer) { + printCmdOut(w, "lldb --version: ", "lldb", "--version") + cmd := exec.Command("gdb", "--version") + out, err := cmd.Output() + if err == nil { + // There's apparently no combination of command line flags + // to get gdb to spit out its version without the license and warranty. + // Print up to the first newline. + fmt.Fprintf(w, "gdb --version: %s\n", firstLine(out)) + } else { + if cfg.BuildV { + fmt.Printf("failed to run gdb --version: %v\n", err) + } + } +} + +func inspectGoVersion(w io.Writer) { + data, err := web.Get("https://golang.org/VERSION?m=text") + if err != nil { + if cfg.BuildV { + fmt.Printf("failed to read from golang.org/VERSION: %v\n", err) + } + return + } + + // golang.org/VERSION currently returns a whitespace-free string, + // but just in case, protect against that changing. + // Similarly so for runtime.Version. + release := string(bytes.TrimSpace(data)) + vers := strings.TrimSpace(runtime.Version()) + + if vers == release { + // Up to date + return + } + + // Devel version or outdated release. Either way, this request is apropos. + fmt.Fprintf(w, "#### Does this issue reproduce with the latest release (%s)?\n\n\n", release) +} + +// printCmdOut prints the output of running the given command. +// It ignores failures; 'go bug' is best effort. +func printCmdOut(w io.Writer, prefix, path string, args ...string) { + cmd := exec.Command(path, args...) + out, err := cmd.Output() + if err != nil { + if cfg.BuildV { + fmt.Printf("%s %s: %v\n", path, strings.Join(args, " "), err) + } + return + } + fmt.Fprintf(w, "%s%s\n", prefix, bytes.TrimSpace(out)) +} + +// firstLine returns the first line of a given byte slice. +func firstLine(buf []byte) []byte { + idx := bytes.IndexByte(buf, '\n') + if idx > 0 { + buf = buf[:idx] + } + return bytes.TrimSpace(buf) +} + +// printGlibcVersion prints information about the glibc version. +// It ignores failures. +func printGlibcVersion(w io.Writer) { + tempdir := os.TempDir() + if tempdir == "" { + return + } + src := []byte(`int main() {}`) + srcfile := filepath.Join(tempdir, "go-bug.c") + outfile := filepath.Join(tempdir, "go-bug") + err := ioutil.WriteFile(srcfile, src, 0644) + if err != nil { + return + } + defer os.Remove(srcfile) + cmd := exec.Command("gcc", "-o", outfile, srcfile) + if _, err = cmd.CombinedOutput(); err != nil { + return + } + defer os.Remove(outfile) + + cmd = exec.Command("ldd", outfile) + out, err := cmd.CombinedOutput() + if err != nil { + return + } + re := regexp.MustCompile(`libc\.so[^ ]* => ([^ ]+)`) + m := re.FindStringSubmatch(string(out)) + if m == nil { + return + } + cmd = exec.Command(m[1]) + out, err = cmd.Output() + if err != nil { + return + } + fmt.Fprintf(w, "%s: %s\n", m[1], firstLine(out)) + + // print another line (the one containing version string) in case of musl libc + if idx := bytes.IndexByte(out, '\n'); bytes.Index(out, []byte("musl")) != -1 && idx > -1 { + fmt.Fprintf(w, "%s\n", firstLine(out[idx+1:])) + } +} diff --git a/libgo/go/cmd/go/internal/buildid/buildid.go b/libgo/go/cmd/go/internal/buildid/buildid.go new file mode 100644 index 00000000000..091c9090c86 --- /dev/null +++ b/libgo/go/cmd/go/internal/buildid/buildid.go @@ -0,0 +1,201 @@ +// Copyright 2017 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package buildid + +import ( + "bytes" + "cmd/go/internal/cfg" + "fmt" + "io" + "os" + "strconv" + "strings" +) + +var ( + errBuildIDToolchain = fmt.Errorf("build ID only supported in gc toolchain") + errBuildIDMalformed = fmt.Errorf("malformed object file") + errBuildIDUnknown = fmt.Errorf("lost build ID") +) + +var ( + bangArch = []byte("!") + pkgdef = []byte("__.PKGDEF") + goobject = []byte("go object ") + buildid = []byte("build id ") +) + +// ReadBuildID reads the build ID from an archive or binary. +// It only supports the gc toolchain. +// Other toolchain maintainers should adjust this function. +func ReadBuildID(name, target string) (id string, err error) { + if cfg.BuildToolchainName != "gc" { + return "", errBuildIDToolchain + } + + // For commands, read build ID directly from binary. + if name == "main" { + return ReadBuildIDFromBinary(target) + } + + // Otherwise, we expect to have an archive (.a) file, + // and we can read the build ID from the Go export data. + if !strings.HasSuffix(target, ".a") { + return "", &os.PathError{Op: "parse", Path: target, Err: errBuildIDUnknown} + } + + // Read just enough of the target to fetch the build ID. + // The archive is expected to look like: + // + // ! + // __.PKGDEF 0 0 0 644 7955 ` + // go object darwin amd64 devel X:none + // build id "b41e5c45250e25c9fd5e9f9a1de7857ea0d41224" + // + // The variable-sized strings are GOOS, GOARCH, and the experiment list (X:none). + // Reading the first 1024 bytes should be plenty. + f, err := os.Open(target) + if err != nil { + return "", err + } + data := make([]byte, 1024) + n, err := io.ReadFull(f, data) + f.Close() + + if err != nil && n == 0 { + return "", err + } + + bad := func() (string, error) { + return "", &os.PathError{Op: "parse", Path: target, Err: errBuildIDMalformed} + } + + // Archive header. + for i := 0; ; i++ { // returns during i==3 + j := bytes.IndexByte(data, '\n') + if j < 0 { + return bad() + } + line := data[:j] + data = data[j+1:] + switch i { + case 0: + if !bytes.Equal(line, bangArch) { + return bad() + } + case 1: + if !bytes.HasPrefix(line, pkgdef) { + return bad() + } + case 2: + if !bytes.HasPrefix(line, goobject) { + return bad() + } + case 3: + if !bytes.HasPrefix(line, buildid) { + // Found the object header, just doesn't have a build id line. + // Treat as successful, with empty build id. + return "", nil + } + id, err := strconv.Unquote(string(line[len(buildid):])) + if err != nil { + return bad() + } + return id, nil + } + } +} + +var ( + goBuildPrefix = []byte("\xff Go build ID: \"") + goBuildEnd = []byte("\"\n \xff") + + elfPrefix = []byte("\x7fELF") + + machoPrefixes = [][]byte{ + {0xfe, 0xed, 0xfa, 0xce}, + {0xfe, 0xed, 0xfa, 0xcf}, + {0xce, 0xfa, 0xed, 0xfe}, + {0xcf, 0xfa, 0xed, 0xfe}, + } +) + +var BuildIDReadSize = 32 * 1024 // changed for testing + +// ReadBuildIDFromBinary reads the build ID from a binary. +// +// ELF binaries store the build ID in a proper PT_NOTE section. +// +// Other binary formats are not so flexible. For those, the linker +// stores the build ID as non-instruction bytes at the very beginning +// of the text segment, which should appear near the beginning +// of the file. This is clumsy but fairly portable. Custom locations +// can be added for other binary types as needed, like we did for ELF. +func ReadBuildIDFromBinary(filename string) (id string, err error) { + if filename == "" { + return "", &os.PathError{Op: "parse", Path: filename, Err: errBuildIDUnknown} + } + + // Read the first 32 kB of the binary file. + // That should be enough to find the build ID. + // In ELF files, the build ID is in the leading headers, + // which are typically less than 4 kB, not to mention 32 kB. + // In Mach-O files, there's no limit, so we have to parse the file. + // On other systems, we're trying to read enough that + // we get the beginning of the text segment in the read. + // The offset where the text segment begins in a hello + // world compiled for each different object format today: + // + // Plan 9: 0x20 + // Windows: 0x600 + // + f, err := os.Open(filename) + if err != nil { + return "", err + } + defer f.Close() + + data := make([]byte, BuildIDReadSize) + _, err = io.ReadFull(f, data) + if err == io.ErrUnexpectedEOF { + err = nil + } + if err != nil { + return "", err + } + + if bytes.HasPrefix(data, elfPrefix) { + return readELFGoBuildID(filename, f, data) + } + for _, m := range machoPrefixes { + if bytes.HasPrefix(data, m) { + return readMachoGoBuildID(filename, f, data) + } + } + + return readRawGoBuildID(filename, data) +} + +// readRawGoBuildID finds the raw build ID stored in text segment data. +func readRawGoBuildID(filename string, data []byte) (id string, err error) { + i := bytes.Index(data, goBuildPrefix) + if i < 0 { + // Missing. Treat as successful but build ID empty. + return "", nil + } + + j := bytes.Index(data[i+len(goBuildPrefix):], goBuildEnd) + if j < 0 { + return "", &os.PathError{Op: "parse", Path: filename, Err: errBuildIDMalformed} + } + + quoted := data[i+len(goBuildPrefix)-1 : i+len(goBuildPrefix)+j+1] + id, err = strconv.Unquote(string(quoted)) + if err != nil { + return "", &os.PathError{Op: "parse", Path: filename, Err: errBuildIDMalformed} + } + + return id, nil +} diff --git a/libgo/go/cmd/go/internal/buildid/note.go b/libgo/go/cmd/go/internal/buildid/note.go new file mode 100644 index 00000000000..68c91e27047 --- /dev/null +++ b/libgo/go/cmd/go/internal/buildid/note.go @@ -0,0 +1,187 @@ +// Copyright 2015 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package buildid + +import ( + "bytes" + "debug/elf" + "debug/macho" + "encoding/binary" + "fmt" + "io" + "os" +) + +func readAligned4(r io.Reader, sz int32) ([]byte, error) { + full := (sz + 3) &^ 3 + data := make([]byte, full) + _, err := io.ReadFull(r, data) + if err != nil { + return nil, err + } + data = data[:sz] + return data, nil +} + +func ReadELFNote(filename, name string, typ int32) ([]byte, error) { + f, err := elf.Open(filename) + if err != nil { + return nil, err + } + for _, sect := range f.Sections { + if sect.Type != elf.SHT_NOTE { + continue + } + r := sect.Open() + for { + var namesize, descsize, noteType int32 + err = binary.Read(r, f.ByteOrder, &namesize) + if err != nil { + if err == io.EOF { + break + } + return nil, fmt.Errorf("read namesize failed: %v", err) + } + err = binary.Read(r, f.ByteOrder, &descsize) + if err != nil { + return nil, fmt.Errorf("read descsize failed: %v", err) + } + err = binary.Read(r, f.ByteOrder, ¬eType) + if err != nil { + return nil, fmt.Errorf("read type failed: %v", err) + } + noteName, err := readAligned4(r, namesize) + if err != nil { + return nil, fmt.Errorf("read name failed: %v", err) + } + desc, err := readAligned4(r, descsize) + if err != nil { + return nil, fmt.Errorf("read desc failed: %v", err) + } + if name == string(noteName) && typ == noteType { + return desc, nil + } + } + } + return nil, nil +} + +var elfGoNote = []byte("Go\x00\x00") + +// The Go build ID is stored in a note described by an ELF PT_NOTE prog +// header. The caller has already opened filename, to get f, and read +// at least 4 kB out, in data. +func readELFGoBuildID(filename string, f *os.File, data []byte) (buildid string, err error) { + // Assume the note content is in the data, already read. + // Rewrite the ELF header to set shnum to 0, so that we can pass + // the data to elf.NewFile and it will decode the Prog list but not + // try to read the section headers and the string table from disk. + // That's a waste of I/O when all we care about is the Prog list + // and the one ELF note. + switch elf.Class(data[elf.EI_CLASS]) { + case elf.ELFCLASS32: + data[48] = 0 + data[49] = 0 + case elf.ELFCLASS64: + data[60] = 0 + data[61] = 0 + } + + const elfGoBuildIDTag = 4 + + ef, err := elf.NewFile(bytes.NewReader(data)) + if err != nil { + return "", &os.PathError{Path: filename, Op: "parse", Err: err} + } + for _, p := range ef.Progs { + if p.Type != elf.PT_NOTE || p.Filesz < 16 { + continue + } + + var note []byte + if p.Off+p.Filesz < uint64(len(data)) { + note = data[p.Off : p.Off+p.Filesz] + } else { + // For some linkers, such as the Solaris linker, + // the buildid may not be found in data (which + // likely contains the first 16kB of the file) + // or even the first few megabytes of the file + // due to differences in note segment placement; + // in that case, extract the note data manually. + _, err = f.Seek(int64(p.Off), io.SeekStart) + if err != nil { + return "", err + } + + note = make([]byte, p.Filesz) + _, err = io.ReadFull(f, note) + if err != nil { + return "", err + } + } + + filesz := p.Filesz + for filesz >= 16 { + nameSize := ef.ByteOrder.Uint32(note) + valSize := ef.ByteOrder.Uint32(note[4:]) + tag := ef.ByteOrder.Uint32(note[8:]) + name := note[12:16] + if nameSize == 4 && 16+valSize <= uint32(len(note)) && tag == elfGoBuildIDTag && bytes.Equal(name, elfGoNote) { + return string(note[16 : 16+valSize]), nil + } + + nameSize = (nameSize + 3) &^ 3 + valSize = (valSize + 3) &^ 3 + notesz := uint64(12 + nameSize + valSize) + if filesz <= notesz { + break + } + filesz -= notesz + note = note[notesz:] + } + } + + // No note. Treat as successful but build ID empty. + return "", nil +} + +// The Go build ID is stored at the beginning of the Mach-O __text segment. +// The caller has already opened filename, to get f, and read a few kB out, in data. +// Sadly, that's not guaranteed to hold the note, because there is an arbitrary amount +// of other junk placed in the file ahead of the main text. +func readMachoGoBuildID(filename string, f *os.File, data []byte) (buildid string, err error) { + // If the data we want has already been read, don't worry about Mach-O parsing. + // This is both an optimization and a hedge against the Mach-O parsing failing + // in the future due to, for example, the name of the __text section changing. + if b, err := readRawGoBuildID(filename, data); b != "" && err == nil { + return b, err + } + + mf, err := macho.NewFile(f) + if err != nil { + return "", &os.PathError{Path: filename, Op: "parse", Err: err} + } + + sect := mf.Section("__text") + if sect == nil { + // Every binary has a __text section. Something is wrong. + return "", &os.PathError{Path: filename, Op: "parse", Err: fmt.Errorf("cannot find __text section")} + } + + // It should be in the first few bytes, but read a lot just in case, + // especially given our past problems on OS X with the build ID moving. + // There shouldn't be much difference between reading 4kB and 32kB: + // the hard part is getting to the data, not transferring it. + n := sect.Size + if n > uint64(BuildIDReadSize) { + n = uint64(BuildIDReadSize) + } + buf := make([]byte, n) + if _, err := f.ReadAt(buf, int64(sect.Offset)); err != nil { + return "", err + } + + return readRawGoBuildID(filename, buf) +} diff --git a/libgo/go/cmd/go/internal/cfg/cfg.go b/libgo/go/cmd/go/internal/cfg/cfg.go new file mode 100644 index 00000000000..8257a0e5116 --- /dev/null +++ b/libgo/go/cmd/go/internal/cfg/cfg.go @@ -0,0 +1,134 @@ +// Copyright 2017 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package cfg holds configuration shared by multiple parts +// of the go command. +package cfg + +import ( + "fmt" + "go/build" + "os" + "path/filepath" + "runtime" + + "cmd/internal/objabi" +) + +// These are general "build flags" used by build and other commands. +var ( + BuildA bool // -a flag + BuildBuildmode string // -buildmode flag + BuildContext = build.Default + BuildI bool // -i flag + BuildLdflags []string // -ldflags flag + BuildLinkshared bool // -linkshared flag + BuildMSan bool // -msan flag + BuildN bool // -n flag + BuildO string // -o flag + BuildP = runtime.NumCPU() // -p flag + BuildPkgdir string // -pkgdir flag + BuildRace bool // -race flag + BuildToolexec []string // -toolexec flag + BuildToolchainName string + BuildToolchainCompiler func() string + BuildToolchainLinker func() string + BuildV bool // -v flag + BuildWork bool // -work flag + BuildX bool // -x flag +) + +func init() { + BuildToolchainCompiler = func() string { return "missing-compiler" } + BuildToolchainLinker = func() string { return "missing-linker" } +} + +// An EnvVar is an environment variable Name=Value. +type EnvVar struct { + Name string + Value string +} + +// OrigEnv is the original environment of the program at startup. +var OrigEnv []string + +// CmdEnv is the new environment for running go tool commands. +// User binaries (during go test or go run) are run with OrigEnv, +// not CmdEnv. +var CmdEnv []EnvVar + +// Global build parameters (used during package load) +var ( + Goarch = BuildContext.GOARCH + Goos = BuildContext.GOOS + ExeSuffix string + Gopath = filepath.SplitList(BuildContext.GOPATH) +) + +func init() { + if Goos == "windows" { + ExeSuffix = ".exe" + } +} + +var ( + GOROOT = findGOROOT() + GOBIN = os.Getenv("GOBIN") + GOROOTbin = filepath.Join(GOROOT, "bin") + GOROOTpkg = filepath.Join(GOROOT, "pkg") + GOROOTsrc = filepath.Join(GOROOT, "src") + + // Used in envcmd.MkEnv and build ID computations. + GOARM = fmt.Sprint(objabi.GOARM) + GO386 = objabi.GO386 +) + +// Update build context to use our computed GOROOT. +func init() { + BuildContext.GOROOT = GOROOT + // Note that we must use runtime.GOOS and runtime.GOARCH here, + // as the tool directory does not move based on environment variables. + // This matches the initialization of ToolDir in go/build, + // except for using GOROOT rather than runtime.GOROOT(). + if runtime.Compiler != "gccgo" { + build.ToolDir = filepath.Join(GOROOT, "pkg/tool/"+runtime.GOOS+"_"+runtime.GOARCH) + } +} + +func findGOROOT() string { + if env := os.Getenv("GOROOT"); env != "" { + return filepath.Clean(env) + } + if runtime.Compiler != "gccgo" { + exe, err := os.Executable() + if err == nil { + exe, err = filepath.Abs(exe) + if err == nil { + if dir := filepath.Join(exe, "../.."); isGOROOT(dir) { + return dir + } + exe, err = filepath.EvalSymlinks(exe) + if err == nil { + if dir := filepath.Join(exe, "../.."); isGOROOT(dir) { + return dir + } + } + } + } + } + return filepath.Clean(runtime.GOROOT()) +} + +// isGOROOT reports whether path looks like a GOROOT. +// +// It does this by looking for the path/pkg/tool directory, +// which is necessary for useful operation of the cmd/go tool, +// and is not typically present in a GOPATH. +func isGOROOT(path string) bool { + stat, err := os.Stat(filepath.Join(path, "pkg", "tool")) + if err != nil { + return false + } + return stat.IsDir() +} diff --git a/libgo/go/cmd/go/internal/clean/clean.go b/libgo/go/cmd/go/internal/clean/clean.go new file mode 100644 index 00000000000..454cac1f47b --- /dev/null +++ b/libgo/go/cmd/go/internal/clean/clean.go @@ -0,0 +1,254 @@ +// Copyright 2012 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package clean implements the ``go clean'' command. +package clean + +import ( + "fmt" + "io/ioutil" + "os" + "path/filepath" + "strings" + + "cmd/go/internal/base" + "cmd/go/internal/cfg" + "cmd/go/internal/load" + "cmd/go/internal/work" +) + +var CmdClean = &base.Command{ + UsageLine: "clean [-i] [-r] [-n] [-x] [build flags] [packages]", + Short: "remove object files", + Long: ` +Clean removes object files from package source directories. +The go command builds most objects in a temporary directory, +so go clean is mainly concerned with object files left by other +tools or by manual invocations of go build. + +Specifically, clean removes the following files from each of the +source directories corresponding to the import paths: + + _obj/ old object directory, left from Makefiles + _test/ old test directory, left from Makefiles + _testmain.go old gotest file, left from Makefiles + test.out old test log, left from Makefiles + build.out old test log, left from Makefiles + *.[568ao] object files, left from Makefiles + + DIR(.exe) from go build + DIR.test(.exe) from go test -c + MAINFILE(.exe) from go build MAINFILE.go + *.so from SWIG + +In the list, DIR represents the final path element of the +directory, and MAINFILE is the base name of any Go source +file in the directory that is not included when building +the package. + +The -i flag causes clean to remove the corresponding installed +archive or binary (what 'go install' would create). + +The -n flag causes clean to print the remove commands it would execute, +but not run them. + +The -r flag causes clean to be applied recursively to all the +dependencies of the packages named by the import paths. + +The -x flag causes clean to print remove commands as it executes them. + +For more about build flags, see 'go help build'. + +For more about specifying packages, see 'go help packages'. + `, +} + +var cleanI bool // clean -i flag +var cleanR bool // clean -r flag + +func init() { + // break init cycle + CmdClean.Run = runClean + + CmdClean.Flag.BoolVar(&cleanI, "i", false, "") + CmdClean.Flag.BoolVar(&cleanR, "r", false, "") + // -n and -x are important enough to be + // mentioned explicitly in the docs but they + // are part of the build flags. + + work.AddBuildFlags(CmdClean) +} + +func runClean(cmd *base.Command, args []string) { + for _, pkg := range load.PackagesAndErrors(args) { + clean(pkg) + } +} + +var cleaned = map[*load.Package]bool{} + +// TODO: These are dregs left by Makefile-based builds. +// Eventually, can stop deleting these. +var cleanDir = map[string]bool{ + "_test": true, + "_obj": true, +} + +var cleanFile = map[string]bool{ + "_testmain.go": true, + "test.out": true, + "build.out": true, + "a.out": true, +} + +var cleanExt = map[string]bool{ + ".5": true, + ".6": true, + ".8": true, + ".a": true, + ".o": true, + ".so": true, +} + +func clean(p *load.Package) { + if cleaned[p] { + return + } + cleaned[p] = true + + if p.Dir == "" { + base.Errorf("can't load package: %v", p.Error) + return + } + dirs, err := ioutil.ReadDir(p.Dir) + if err != nil { + base.Errorf("go clean %s: %v", p.Dir, err) + return + } + + var b work.Builder + b.Print = fmt.Print + + packageFile := map[string]bool{} + if p.Name != "main" { + // Record which files are not in package main. + // The others are. + keep := func(list []string) { + for _, f := range list { + packageFile[f] = true + } + } + keep(p.GoFiles) + keep(p.CgoFiles) + keep(p.TestGoFiles) + keep(p.XTestGoFiles) + } + + _, elem := filepath.Split(p.Dir) + var allRemove []string + + // Remove dir-named executable only if this is package main. + if p.Name == "main" { + allRemove = append(allRemove, + elem, + elem+".exe", + ) + } + + // Remove package test executables. + allRemove = append(allRemove, + elem+".test", + elem+".test.exe", + ) + + // Remove a potential executable for each .go file in the directory that + // is not part of the directory's package. + for _, dir := range dirs { + name := dir.Name() + if packageFile[name] { + continue + } + if !dir.IsDir() && strings.HasSuffix(name, ".go") { + // TODO(adg,rsc): check that this .go file is actually + // in "package main", and therefore capable of building + // to an executable file. + base := name[:len(name)-len(".go")] + allRemove = append(allRemove, base, base+".exe") + } + } + + if cfg.BuildN || cfg.BuildX { + b.Showcmd(p.Dir, "rm -f %s", strings.Join(allRemove, " ")) + } + + toRemove := map[string]bool{} + for _, name := range allRemove { + toRemove[name] = true + } + for _, dir := range dirs { + name := dir.Name() + if dir.IsDir() { + // TODO: Remove once Makefiles are forgotten. + if cleanDir[name] { + if cfg.BuildN || cfg.BuildX { + b.Showcmd(p.Dir, "rm -r %s", name) + if cfg.BuildN { + continue + } + } + if err := os.RemoveAll(filepath.Join(p.Dir, name)); err != nil { + base.Errorf("go clean: %v", err) + } + } + continue + } + + if cfg.BuildN { + continue + } + + if cleanFile[name] || cleanExt[filepath.Ext(name)] || toRemove[name] { + removeFile(filepath.Join(p.Dir, name)) + } + } + + if cleanI && p.Internal.Target != "" { + if cfg.BuildN || cfg.BuildX { + b.Showcmd("", "rm -f %s", p.Internal.Target) + } + if !cfg.BuildN { + removeFile(p.Internal.Target) + } + } + + if cleanR { + for _, p1 := range p.Internal.Imports { + clean(p1) + } + } +} + +// removeFile tries to remove file f, if error other than file doesn't exist +// occurs, it will report the error. +func removeFile(f string) { + err := os.Remove(f) + if err == nil || os.IsNotExist(err) { + return + } + // Windows does not allow deletion of a binary file while it is executing. + if base.ToolIsWindows { + // Remove lingering ~ file from last attempt. + if _, err2 := os.Stat(f + "~"); err2 == nil { + os.Remove(f + "~") + } + // Try to move it out of the way. If the move fails, + // which is likely, we'll try again the + // next time we do an install of this binary. + if err2 := os.Rename(f, f+"~"); err2 == nil { + os.Remove(f + "~") + return + } + } + base.Errorf("go clean: %v", err) +} diff --git a/libgo/go/cmd/go/internal/cmdflag/flag.go b/libgo/go/cmd/go/internal/cmdflag/flag.go new file mode 100644 index 00000000000..7ab30221279 --- /dev/null +++ b/libgo/go/cmd/go/internal/cmdflag/flag.go @@ -0,0 +1,123 @@ +// Copyright 2017 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package cmdflag handles flag processing common to several go tools. +package cmdflag + +import ( + "flag" + "fmt" + "os" + "strconv" + "strings" + + "cmd/go/internal/base" +) + +// The flag handling part of go commands such as test is large and distracting. +// We can't use the standard flag package because some of the flags from +// our command line are for us, and some are for the binary we're running, +// and some are for both. + +// Defn defines a flag we know about. +type Defn struct { + Name string // Name on command line. + BoolVar *bool // If it's a boolean flag, this points to it. + Value flag.Value // The flag.Value represented. + PassToTest bool // Pass to the test binary? Used only by go test. + Present bool // Flag has been seen. +} + +// IsBool reports whether v is a bool flag. +func IsBool(v flag.Value) bool { + vv, ok := v.(interface { + IsBoolFlag() bool + }) + if ok { + return vv.IsBoolFlag() + } + return false +} + +// SetBool sets the addressed boolean to the value. +func SetBool(cmd string, flag *bool, value string) { + x, err := strconv.ParseBool(value) + if err != nil { + SyntaxError(cmd, "illegal bool flag value "+value) + } + *flag = x +} + +// SetInt sets the addressed integer to the value. +func SetInt(cmd string, flag *int, value string) { + x, err := strconv.Atoi(value) + if err != nil { + SyntaxError(cmd, "illegal int flag value "+value) + } + *flag = x +} + +// SyntaxError reports an argument syntax error and exits the program. +func SyntaxError(cmd, msg string) { + fmt.Fprintf(os.Stderr, "go %s: %s\n", cmd, msg) + if cmd == "test" { + fmt.Fprintf(os.Stderr, `run "go help %s" or "go help testflag" for more information`+"\n", cmd) + } else { + fmt.Fprintf(os.Stderr, `run "go help %s" for more information`+"\n", cmd) + } + os.Exit(2) +} + +// Parse sees if argument i is present in the definitions and if so, +// returns its definition, value, and whether it consumed an extra word. +// If the flag begins (cmd+".") it is ignored for the purpose of this function. +func Parse(cmd string, defns []*Defn, args []string, i int) (f *Defn, value string, extra bool) { + arg := args[i] + if strings.HasPrefix(arg, "--") { // reduce two minuses to one + arg = arg[1:] + } + switch arg { + case "-?", "-h", "-help": + base.Usage() + } + if arg == "" || arg[0] != '-' { + return + } + name := arg[1:] + // If there's already a prefix such as "test.", drop it for now. + name = strings.TrimPrefix(name, cmd+".") + equals := strings.Index(name, "=") + if equals >= 0 { + value = name[equals+1:] + name = name[:equals] + } + for _, f = range defns { + if name == f.Name { + // Booleans are special because they have modes -x, -x=true, -x=false. + if f.BoolVar != nil || IsBool(f.Value) { + if equals < 0 { // Otherwise, it's been set and will be verified in SetBool. + value = "true" + } else { + // verify it parses + SetBool(cmd, new(bool), value) + } + } else { // Non-booleans must have a value. + extra = equals < 0 + if extra { + if i+1 >= len(args) { + SyntaxError(cmd, "missing argument for flag "+f.Name) + } + value = args[i+1] + } + } + if f.Present { + SyntaxError(cmd, f.Name+" flag may be set only once") + } + f.Present = true + return + } + } + f = nil + return +} diff --git a/libgo/go/cmd/go/internal/doc/doc.go b/libgo/go/cmd/go/internal/doc/doc.go new file mode 100644 index 00000000000..d73dd9ad914 --- /dev/null +++ b/libgo/go/cmd/go/internal/doc/doc.go @@ -0,0 +1,123 @@ +// Copyright 2015 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package doc implements the ``go doc'' command. +package doc + +import ( + "cmd/go/internal/base" + "cmd/go/internal/cfg" +) + +var CmdDoc = &base.Command{ + Run: runDoc, + UsageLine: "doc [-u] [-c] [package|[package.]symbol[.methodOrField]]", + CustomFlags: true, + Short: "show documentation for package or symbol", + Long: ` +Doc prints the documentation comments associated with the item identified by its +arguments (a package, const, func, type, var, method, or struct field) +followed by a one-line summary of each of the first-level items "under" +that item (package-level declarations for a package, methods for a type, +etc.). + +Doc accepts zero, one, or two arguments. + +Given no arguments, that is, when run as + + go doc + +it prints the package documentation for the package in the current directory. +If the package is a command (package main), the exported symbols of the package +are elided from the presentation unless the -cmd flag is provided. + +When run with one argument, the argument is treated as a Go-syntax-like +representation of the item to be documented. What the argument selects depends +on what is installed in GOROOT and GOPATH, as well as the form of the argument, +which is schematically one of these: + + go doc + go doc [.] + go doc [.][.] + go doc [.][.] + +The first item in this list matched by the argument is the one whose documentation +is printed. (See the examples below.) However, if the argument starts with a capital +letter it is assumed to identify a symbol or method in the current directory. + +For packages, the order of scanning is determined lexically in breadth-first order. +That is, the package presented is the one that matches the search and is nearest +the root and lexically first at its level of the hierarchy. The GOROOT tree is +always scanned in its entirety before GOPATH. + +If there is no package specified or matched, the package in the current +directory is selected, so "go doc Foo" shows the documentation for symbol Foo in +the current package. + +The package path must be either a qualified path or a proper suffix of a +path. The go tool's usual package mechanism does not apply: package path +elements like . and ... are not implemented by go doc. + +When run with two arguments, the first must be a full package path (not just a +suffix), and the second is a symbol, or symbol with method or struct field. +This is similar to the syntax accepted by godoc: + + go doc [.] + +In all forms, when matching symbols, lower-case letters in the argument match +either case but upper-case letters match exactly. This means that there may be +multiple matches of a lower-case argument in a package if different symbols have +different cases. If this occurs, documentation for all matches is printed. + +Examples: + go doc + Show documentation for current package. + go doc Foo + Show documentation for Foo in the current package. + (Foo starts with a capital letter so it cannot match + a package path.) + go doc encoding/json + Show documentation for the encoding/json package. + go doc json + Shorthand for encoding/json. + go doc json.Number (or go doc json.number) + Show documentation and method summary for json.Number. + go doc json.Number.Int64 (or go doc json.number.int64) + Show documentation for json.Number's Int64 method. + go doc cmd/doc + Show package docs for the doc command. + go doc -cmd cmd/doc + Show package docs and exported symbols within the doc command. + go doc template.new + Show documentation for html/template's New function. + (html/template is lexically before text/template) + go doc text/template.new # One argument + Show documentation for text/template's New function. + go doc text/template new # Two arguments + Show documentation for text/template's New function. + + At least in the current tree, these invocations all print the + documentation for json.Decoder's Decode method: + + go doc json.Decoder.Decode + go doc json.decoder.decode + go doc json.decode + cd go/src/encoding/json; go doc decode + +Flags: + -c + Respect case when matching symbols. + -cmd + Treat a command (package main) like a regular package. + Otherwise package main's exported symbols are hidden + when showing the package's top-level documentation. + -u + Show documentation for unexported as well as exported + symbols, methods, and fields. +`, +} + +func runDoc(cmd *base.Command, args []string) { + base.Run(cfg.BuildToolexec, base.Tool("doc"), args) +} diff --git a/libgo/go/cmd/go/internal/envcmd/env.go b/libgo/go/cmd/go/internal/envcmd/env.go new file mode 100644 index 00000000000..43d4334f060 --- /dev/null +++ b/libgo/go/cmd/go/internal/envcmd/env.go @@ -0,0 +1,178 @@ +// Copyright 2012 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package envcmd implements the ``go env'' command. +package envcmd + +import ( + "encoding/json" + "fmt" + "os" + "runtime" + "strings" + + "cmd/go/internal/base" + "cmd/go/internal/cfg" + "cmd/go/internal/load" + "cmd/go/internal/work" +) + +var CmdEnv = &base.Command{ + UsageLine: "env [-json] [var ...]", + Short: "print Go environment information", + Long: ` +Env prints Go environment information. + +By default env prints information as a shell script +(on Windows, a batch file). If one or more variable +names is given as arguments, env prints the value of +each named variable on its own line. + +The -json flag prints the environment in JSON format +instead of as a shell script. + `, +} + +func init() { + CmdEnv.Run = runEnv // break init cycle +} + +var envJson = CmdEnv.Flag.Bool("json", false, "") + +func MkEnv() []cfg.EnvVar { + var b work.Builder + b.Init() + + env := []cfg.EnvVar{ + {Name: "GOARCH", Value: cfg.Goarch}, + {Name: "GOBIN", Value: cfg.GOBIN}, + {Name: "GOEXE", Value: cfg.ExeSuffix}, + {Name: "GOHOSTARCH", Value: runtime.GOARCH}, + {Name: "GOHOSTOS", Value: runtime.GOOS}, + {Name: "GOOS", Value: cfg.Goos}, + {Name: "GOPATH", Value: cfg.BuildContext.GOPATH}, + {Name: "GORACE", Value: os.Getenv("GORACE")}, + {Name: "GOROOT", Value: cfg.GOROOT}, + {Name: "GOTOOLDIR", Value: base.ToolDir}, + + // disable escape codes in clang errors + {Name: "TERM", Value: "dumb"}, + } + + if work.GccgoBin != "" { + env = append(env, cfg.EnvVar{Name: "GCCGO", Value: work.GccgoBin}) + } else { + env = append(env, cfg.EnvVar{Name: "GCCGO", Value: work.GccgoName}) + } + + switch cfg.Goarch { + case "arm": + env = append(env, cfg.EnvVar{Name: "GOARM", Value: cfg.GOARM}) + case "386": + env = append(env, cfg.EnvVar{Name: "GO386", Value: cfg.GO386}) + } + + cmd := b.GccCmd(".") + env = append(env, cfg.EnvVar{Name: "CC", Value: cmd[0]}) + env = append(env, cfg.EnvVar{Name: "GOGCCFLAGS", Value: strings.Join(cmd[3:], " ")}) + cmd = b.GxxCmd(".") + env = append(env, cfg.EnvVar{Name: "CXX", Value: cmd[0]}) + + if cfg.BuildContext.CgoEnabled { + env = append(env, cfg.EnvVar{Name: "CGO_ENABLED", Value: "1"}) + } else { + env = append(env, cfg.EnvVar{Name: "CGO_ENABLED", Value: "0"}) + } + + return env +} + +func findEnv(env []cfg.EnvVar, name string) string { + for _, e := range env { + if e.Name == name { + return e.Value + } + } + return "" +} + +// ExtraEnvVars returns environment variables that should not leak into child processes. +func ExtraEnvVars() []cfg.EnvVar { + var b work.Builder + b.Init() + cppflags, cflags, cxxflags, fflags, ldflags := b.CFlags(&load.Package{}) + return []cfg.EnvVar{ + {Name: "CGO_CFLAGS", Value: strings.Join(cflags, " ")}, + {Name: "CGO_CPPFLAGS", Value: strings.Join(cppflags, " ")}, + {Name: "CGO_CXXFLAGS", Value: strings.Join(cxxflags, " ")}, + {Name: "CGO_FFLAGS", Value: strings.Join(fflags, " ")}, + {Name: "CGO_LDFLAGS", Value: strings.Join(ldflags, " ")}, + {Name: "PKG_CONFIG", Value: b.PkgconfigCmd()}, + } +} + +func runEnv(cmd *base.Command, args []string) { + env := cfg.CmdEnv + env = append(env, ExtraEnvVars()...) + if len(args) > 0 { + if *envJson { + var es []cfg.EnvVar + for _, name := range args { + e := cfg.EnvVar{Name: name, Value: findEnv(env, name)} + es = append(es, e) + } + printEnvAsJSON(es) + } else { + for _, name := range args { + fmt.Printf("%s\n", findEnv(env, name)) + } + } + return + } + + if *envJson { + printEnvAsJSON(env) + return + } + + for _, e := range env { + if e.Name != "TERM" { + switch runtime.GOOS { + default: + fmt.Printf("%s=\"%s\"\n", e.Name, e.Value) + case "plan9": + if strings.IndexByte(e.Value, '\x00') < 0 { + fmt.Printf("%s='%s'\n", e.Name, strings.Replace(e.Value, "'", "''", -1)) + } else { + v := strings.Split(e.Value, "\x00") + fmt.Printf("%s=(", e.Name) + for x, s := range v { + if x > 0 { + fmt.Printf(" ") + } + fmt.Printf("%s", s) + } + fmt.Printf(")\n") + } + case "windows": + fmt.Printf("set %s=%s\n", e.Name, e.Value) + } + } + } +} + +func printEnvAsJSON(env []cfg.EnvVar) { + m := make(map[string]string) + for _, e := range env { + if e.Name == "TERM" { + continue + } + m[e.Name] = e.Value + } + enc := json.NewEncoder(os.Stdout) + enc.SetIndent("", "\t") + if err := enc.Encode(m); err != nil { + base.Fatalf("%s", err) + } +} diff --git a/libgo/go/cmd/go/internal/fix/fix.go b/libgo/go/cmd/go/internal/fix/fix.go new file mode 100644 index 00000000000..788d49bcb60 --- /dev/null +++ b/libgo/go/cmd/go/internal/fix/fix.go @@ -0,0 +1,39 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package fix implements the ``go fix'' command. +package fix + +import ( + "cmd/go/internal/base" + "cmd/go/internal/cfg" + "cmd/go/internal/load" + "cmd/go/internal/str" +) + +var CmdFix = &base.Command{ + Run: runFix, + UsageLine: "fix [packages]", + Short: "run go tool fix on packages", + Long: ` +Fix runs the Go fix command on the packages named by the import paths. + +For more about fix, see 'go doc cmd/fix'. +For more about specifying packages, see 'go help packages'. + +To run fix with specific options, run 'go tool fix'. + +See also: go fmt, go vet. + `, +} + +func runFix(cmd *base.Command, args []string) { + for _, pkg := range load.Packages(args) { + // Use pkg.gofiles instead of pkg.Dir so that + // the command only applies to this package, + // not to packages in subdirectories. + files := base.FilterDotUnderscoreFiles(base.RelPaths(pkg.Internal.AllGoFiles)) + base.Run(str.StringList(cfg.BuildToolexec, base.Tool("fix"), files)) + } +} diff --git a/libgo/go/cmd/go/internal/fmtcmd/fmt.go b/libgo/go/cmd/go/internal/fmtcmd/fmt.go new file mode 100644 index 00000000000..0563a0410b3 --- /dev/null +++ b/libgo/go/cmd/go/internal/fmtcmd/fmt.go @@ -0,0 +1,71 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package fmtcmd implements the ``go fmt'' command. +package fmtcmd + +import ( + "os" + "path/filepath" + + "cmd/go/internal/base" + "cmd/go/internal/cfg" + "cmd/go/internal/load" + "cmd/go/internal/str" +) + +func init() { + base.AddBuildFlagsNX(&CmdFmt.Flag) +} + +var CmdFmt = &base.Command{ + Run: runFmt, + UsageLine: "fmt [-n] [-x] [packages]", + Short: "run gofmt on package sources", + Long: ` +Fmt runs the command 'gofmt -l -w' on the packages named +by the import paths. It prints the names of the files that are modified. + +For more about gofmt, see 'go doc cmd/gofmt'. +For more about specifying packages, see 'go help packages'. + +The -n flag prints commands that would be executed. +The -x flag prints commands as they are executed. + +To run gofmt with specific options, run gofmt itself. + +See also: go fix, go vet. + `, +} + +func runFmt(cmd *base.Command, args []string) { + gofmt := gofmtPath() + for _, pkg := range load.Packages(args) { + // Use pkg.gofiles instead of pkg.Dir so that + // the command only applies to this package, + // not to packages in subdirectories. + files := base.FilterDotUnderscoreFiles(base.RelPaths(pkg.Internal.AllGoFiles)) + base.Run(str.StringList(gofmt, "-l", "-w", files)) + } +} + +func gofmtPath() string { + gofmt := "gofmt" + if base.ToolIsWindows { + gofmt += base.ToolWindowsExtension + } + + gofmtPath := filepath.Join(cfg.GOBIN, gofmt) + if _, err := os.Stat(gofmtPath); err == nil { + return gofmtPath + } + + gofmtPath = filepath.Join(cfg.GOROOT, "bin", gofmt) + if _, err := os.Stat(gofmtPath); err == nil { + return gofmtPath + } + + // fallback to looking for gofmt in $PATH + return "gofmt" +} diff --git a/libgo/go/cmd/go/internal/generate/generate.go b/libgo/go/cmd/go/internal/generate/generate.go new file mode 100644 index 00000000000..d47c9b73236 --- /dev/null +++ b/libgo/go/cmd/go/internal/generate/generate.go @@ -0,0 +1,407 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package generate implements the ``go generate'' command. +package generate + +import ( + "bufio" + "bytes" + "fmt" + "io" + "log" + "os" + "os/exec" + "path/filepath" + "regexp" + "strconv" + "strings" + + "cmd/go/internal/base" + "cmd/go/internal/cfg" + "cmd/go/internal/load" + "cmd/go/internal/work" +) + +var CmdGenerate = &base.Command{ + Run: runGenerate, + UsageLine: "generate [-run regexp] [-n] [-v] [-x] [build flags] [file.go... | packages]", + Short: "generate Go files by processing source", + Long: ` +Generate runs commands described by directives within existing +files. Those commands can run any process but the intent is to +create or update Go source files. + +Go generate is never run automatically by go build, go get, go test, +and so on. It must be run explicitly. + +Go generate scans the file for directives, which are lines of +the form, + + //go:generate command argument... + +(note: no leading spaces and no space in "//go") where command +is the generator to be run, corresponding to an executable file +that can be run locally. It must either be in the shell path +(gofmt), a fully qualified path (/usr/you/bin/mytool), or a +command alias, described below. + +Note that go generate does not parse the file, so lines that look +like directives in comments or multiline strings will be treated +as directives. + +The arguments to the directive are space-separated tokens or +double-quoted strings passed to the generator as individual +arguments when it is run. + +Quoted strings use Go syntax and are evaluated before execution; a +quoted string appears as a single argument to the generator. + +Go generate sets several variables when it runs the generator: + + $GOARCH + The execution architecture (arm, amd64, etc.) + $GOOS + The execution operating system (linux, windows, etc.) + $GOFILE + The base name of the file. + $GOLINE + The line number of the directive in the source file. + $GOPACKAGE + The name of the package of the file containing the directive. + $DOLLAR + A dollar sign. + +Other than variable substitution and quoted-string evaluation, no +special processing such as "globbing" is performed on the command +line. + +As a last step before running the command, any invocations of any +environment variables with alphanumeric names, such as $GOFILE or +$HOME, are expanded throughout the command line. The syntax for +variable expansion is $NAME on all operating systems. Due to the +order of evaluation, variables are expanded even inside quoted +strings. If the variable NAME is not set, $NAME expands to the +empty string. + +A directive of the form, + + //go:generate -command xxx args... + +specifies, for the remainder of this source file only, that the +string xxx represents the command identified by the arguments. This +can be used to create aliases or to handle multiword generators. +For example, + + //go:generate -command foo go tool foo + +specifies that the command "foo" represents the generator +"go tool foo". + +Generate processes packages in the order given on the command line, +one at a time. If the command line lists .go files, they are treated +as a single package. Within a package, generate processes the +source files in a package in file name order, one at a time. Within +a source file, generate runs generators in the order they appear +in the file, one at a time. + +If any generator returns an error exit status, "go generate" skips +all further processing for that package. + +The generator is run in the package's source directory. + +Go generate accepts one specific flag: + + -run="" + if non-empty, specifies a regular expression to select + directives whose full original source text (excluding + any trailing spaces and final newline) matches the + expression. + +It also accepts the standard build flags including -v, -n, and -x. +The -v flag prints the names of packages and files as they are +processed. +The -n flag prints commands that would be executed. +The -x flag prints commands as they are executed. + +For more about build flags, see 'go help build'. + +For more about specifying packages, see 'go help packages'. + `, +} + +var ( + generateRunFlag string // generate -run flag + generateRunRE *regexp.Regexp // compiled expression for -run +) + +func init() { + work.AddBuildFlags(CmdGenerate) + CmdGenerate.Flag.StringVar(&generateRunFlag, "run", "", "") +} + +func runGenerate(cmd *base.Command, args []string) { + load.IgnoreImports = true + + if generateRunFlag != "" { + var err error + generateRunRE, err = regexp.Compile(generateRunFlag) + if err != nil { + log.Fatalf("generate: %s", err) + } + } + // Even if the arguments are .go files, this loop suffices. + for _, pkg := range load.Packages(args) { + for _, file := range pkg.Internal.GoFiles { + if !generate(pkg.Name, file) { + break + } + } + } +} + +// generate runs the generation directives for a single file. +func generate(pkg, absFile string) bool { + fd, err := os.Open(absFile) + if err != nil { + log.Fatalf("generate: %s", err) + } + defer fd.Close() + g := &Generator{ + r: fd, + path: absFile, + pkg: pkg, + commands: make(map[string][]string), + } + return g.run() +} + +// A Generator represents the state of a single Go source file +// being scanned for generator commands. +type Generator struct { + r io.Reader + path string // full rooted path name. + dir string // full rooted directory of file. + file string // base name of file. + pkg string + commands map[string][]string + lineNum int // current line number. + env []string +} + +// run runs the generators in the current file. +func (g *Generator) run() (ok bool) { + // Processing below here calls g.errorf on failure, which does panic(stop). + // If we encounter an error, we abort the package. + defer func() { + e := recover() + if e != nil { + ok = false + if e != stop { + panic(e) + } + base.SetExitStatus(1) + } + }() + g.dir, g.file = filepath.Split(g.path) + g.dir = filepath.Clean(g.dir) // No final separator please. + if cfg.BuildV { + fmt.Fprintf(os.Stderr, "%s\n", base.ShortPath(g.path)) + } + + // Scan for lines that start "//go:generate". + // Can't use bufio.Scanner because it can't handle long lines, + // which are likely to appear when using generate. + input := bufio.NewReader(g.r) + var err error + // One line per loop. + for { + g.lineNum++ // 1-indexed. + var buf []byte + buf, err = input.ReadSlice('\n') + if err == bufio.ErrBufferFull { + // Line too long - consume and ignore. + if isGoGenerate(buf) { + g.errorf("directive too long") + } + for err == bufio.ErrBufferFull { + _, err = input.ReadSlice('\n') + } + if err != nil { + break + } + continue + } + + if err != nil { + // Check for marker at EOF without final \n. + if err == io.EOF && isGoGenerate(buf) { + err = io.ErrUnexpectedEOF + } + break + } + + if !isGoGenerate(buf) { + continue + } + if generateRunFlag != "" { + if !generateRunRE.Match(bytes.TrimSpace(buf)) { + continue + } + } + + g.setEnv() + words := g.split(string(buf)) + if len(words) == 0 { + g.errorf("no arguments to directive") + } + if words[0] == "-command" { + g.setShorthand(words) + continue + } + // Run the command line. + if cfg.BuildN || cfg.BuildX { + fmt.Fprintf(os.Stderr, "%s\n", strings.Join(words, " ")) + } + if cfg.BuildN { + continue + } + g.exec(words) + } + if err != nil && err != io.EOF { + g.errorf("error reading %s: %s", base.ShortPath(g.path), err) + } + return true +} + +func isGoGenerate(buf []byte) bool { + return bytes.HasPrefix(buf, []byte("//go:generate ")) || bytes.HasPrefix(buf, []byte("//go:generate\t")) +} + +// setEnv sets the extra environment variables used when executing a +// single go:generate command. +func (g *Generator) setEnv() { + g.env = []string{ + "GOARCH=" + cfg.BuildContext.GOARCH, + "GOOS=" + cfg.BuildContext.GOOS, + "GOFILE=" + g.file, + "GOLINE=" + strconv.Itoa(g.lineNum), + "GOPACKAGE=" + g.pkg, + "DOLLAR=" + "$", + } +} + +// split breaks the line into words, evaluating quoted +// strings and evaluating environment variables. +// The initial //go:generate element is present in line. +func (g *Generator) split(line string) []string { + // Parse line, obeying quoted strings. + var words []string + line = line[len("//go:generate ") : len(line)-1] // Drop preamble and final newline. + // There may still be a carriage return. + if len(line) > 0 && line[len(line)-1] == '\r' { + line = line[:len(line)-1] + } + // One (possibly quoted) word per iteration. +Words: + for { + line = strings.TrimLeft(line, " \t") + if len(line) == 0 { + break + } + if line[0] == '"' { + for i := 1; i < len(line); i++ { + c := line[i] // Only looking for ASCII so this is OK. + switch c { + case '\\': + if i+1 == len(line) { + g.errorf("bad backslash") + } + i++ // Absorb next byte (If it's a multibyte we'll get an error in Unquote). + case '"': + word, err := strconv.Unquote(line[0 : i+1]) + if err != nil { + g.errorf("bad quoted string") + } + words = append(words, word) + line = line[i+1:] + // Check the next character is space or end of line. + if len(line) > 0 && line[0] != ' ' && line[0] != '\t' { + g.errorf("expect space after quoted argument") + } + continue Words + } + } + g.errorf("mismatched quoted string") + } + i := strings.IndexAny(line, " \t") + if i < 0 { + i = len(line) + } + words = append(words, line[0:i]) + line = line[i:] + } + // Substitute command if required. + if len(words) > 0 && g.commands[words[0]] != nil { + // Replace 0th word by command substitution. + words = append(g.commands[words[0]], words[1:]...) + } + // Substitute environment variables. + for i, word := range words { + words[i] = os.Expand(word, g.expandVar) + } + return words +} + +var stop = fmt.Errorf("error in generation") + +// errorf logs an error message prefixed with the file and line number. +// It then exits the program (with exit status 1) because generation stops +// at the first error. +func (g *Generator) errorf(format string, args ...interface{}) { + fmt.Fprintf(os.Stderr, "%s:%d: %s\n", base.ShortPath(g.path), g.lineNum, + fmt.Sprintf(format, args...)) + panic(stop) +} + +// expandVar expands the $XXX invocation in word. It is called +// by os.Expand. +func (g *Generator) expandVar(word string) string { + w := word + "=" + for _, e := range g.env { + if strings.HasPrefix(e, w) { + return e[len(w):] + } + } + return os.Getenv(word) +} + +// setShorthand installs a new shorthand as defined by a -command directive. +func (g *Generator) setShorthand(words []string) { + // Create command shorthand. + if len(words) == 1 { + g.errorf("no command specified for -command") + } + command := words[1] + if g.commands[command] != nil { + g.errorf("command %q defined multiply defined", command) + } + g.commands[command] = words[2:len(words):len(words)] // force later append to make copy +} + +// exec runs the command specified by the argument. The first word is +// the command name itself. +func (g *Generator) exec(words []string) { + cmd := exec.Command(words[0], words[1:]...) + // Standard in and out of generator should be the usual. + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + // Run the command in the package directory. + cmd.Dir = g.dir + cmd.Env = base.MergeEnvLists(g.env, cfg.OrigEnv) + err := cmd.Run() + if err != nil { + g.errorf("running %q: %s", words[0], err) + } +} diff --git a/libgo/go/cmd/go/internal/generate/generate_test.go b/libgo/go/cmd/go/internal/generate/generate_test.go new file mode 100644 index 00000000000..defc15387f3 --- /dev/null +++ b/libgo/go/cmd/go/internal/generate/generate_test.go @@ -0,0 +1,56 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package generate + +import ( + "reflect" + "runtime" + "testing" +) + +type splitTest struct { + in string + out []string +} + +var splitTests = []splitTest{ + {"", nil}, + {"x", []string{"x"}}, + {" a b\tc ", []string{"a", "b", "c"}}, + {` " a " `, []string{" a "}}, + {"$GOARCH", []string{runtime.GOARCH}}, + {"$GOOS", []string{runtime.GOOS}}, + {"$GOFILE", []string{"proc.go"}}, + {"$GOPACKAGE", []string{"sys"}}, + {"a $XXNOTDEFINEDXX b", []string{"a", "", "b"}}, + {"/$XXNOTDEFINED/", []string{"//"}}, + {"/$DOLLAR/", []string{"/$/"}}, + {"yacc -o $GOARCH/yacc_$GOFILE", []string{"go", "tool", "yacc", "-o", runtime.GOARCH + "/yacc_proc.go"}}, +} + +func TestGenerateCommandParse(t *testing.T) { + g := &Generator{ + r: nil, // Unused here. + path: "/usr/ken/sys/proc.go", + dir: "/usr/ken/sys", + file: "proc.go", + pkg: "sys", + commands: make(map[string][]string), + } + g.setEnv() + g.setShorthand([]string{"-command", "yacc", "go", "tool", "yacc"}) + for _, test := range splitTests { + // First with newlines. + got := g.split("//go:generate " + test.in + "\n") + if !reflect.DeepEqual(got, test.out) { + t.Errorf("split(%q): got %q expected %q", test.in, got, test.out) + } + // Then with CRLFs, thank you Windows. + got = g.split("//go:generate " + test.in + "\r\n") + if !reflect.DeepEqual(got, test.out) { + t.Errorf("split(%q): got %q expected %q", test.in, got, test.out) + } + } +} diff --git a/libgo/go/cmd/go/internal/get/discovery.go b/libgo/go/cmd/go/internal/get/discovery.go new file mode 100644 index 00000000000..b2918dbb4f3 --- /dev/null +++ b/libgo/go/cmd/go/internal/get/discovery.go @@ -0,0 +1,76 @@ +// Copyright 2012 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package get + +import ( + "encoding/xml" + "fmt" + "io" + "strings" +) + +// charsetReader returns a reader for the given charset. Currently +// it only supports UTF-8 and ASCII. Otherwise, it returns a meaningful +// error which is printed by go get, so the user can find why the package +// wasn't downloaded if the encoding is not supported. Note that, in +// order to reduce potential errors, ASCII is treated as UTF-8 (i.e. characters +// greater than 0x7f are not rejected). +func charsetReader(charset string, input io.Reader) (io.Reader, error) { + switch strings.ToLower(charset) { + case "ascii": + return input, nil + default: + return nil, fmt.Errorf("can't decode XML document using charset %q", charset) + } +} + +// parseMetaGoImports returns meta imports from the HTML in r. +// Parsing ends at the end of the section or the beginning of the . +func parseMetaGoImports(r io.Reader) (imports []metaImport, err error) { + d := xml.NewDecoder(r) + d.CharsetReader = charsetReader + d.Strict = false + var t xml.Token + for { + t, err = d.RawToken() + if err != nil { + if err == io.EOF || len(imports) > 0 { + err = nil + } + return + } + if e, ok := t.(xml.StartElement); ok && strings.EqualFold(e.Name.Local, "body") { + return + } + if e, ok := t.(xml.EndElement); ok && strings.EqualFold(e.Name.Local, "head") { + return + } + e, ok := t.(xml.StartElement) + if !ok || !strings.EqualFold(e.Name.Local, "meta") { + continue + } + if attrValue(e.Attr, "name") != "go-import" { + continue + } + if f := strings.Fields(attrValue(e.Attr, "content")); len(f) == 3 { + imports = append(imports, metaImport{ + Prefix: f[0], + VCS: f[1], + RepoRoot: f[2], + }) + } + } +} + +// attrValue returns the attribute value for the case-insensitive key +// `name', or the empty string if nothing is found. +func attrValue(attrs []xml.Attr, name string) string { + for _, a := range attrs { + if strings.EqualFold(a.Name.Local, name) { + return a.Value + } + } + return "" +} diff --git a/libgo/go/cmd/go/internal/get/get.go b/libgo/go/cmd/go/internal/get/get.go new file mode 100644 index 00000000000..550321198d1 --- /dev/null +++ b/libgo/go/cmd/go/internal/get/get.go @@ -0,0 +1,528 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package get implements the ``go get'' command. +package get + +import ( + "fmt" + "go/build" + "os" + "path/filepath" + "runtime" + "strings" + + "cmd/go/internal/base" + "cmd/go/internal/cfg" + "cmd/go/internal/load" + "cmd/go/internal/str" + "cmd/go/internal/web" + "cmd/go/internal/work" +) + +var CmdGet = &base.Command{ + UsageLine: "get [-d] [-f] [-fix] [-insecure] [-t] [-u] [build flags] [packages]", + Short: "download and install packages and dependencies", + Long: ` +Get downloads the packages named by the import paths, along with their +dependencies. It then installs the named packages, like 'go install'. + +The -d flag instructs get to stop after downloading the packages; that is, +it instructs get not to install the packages. + +The -f flag, valid only when -u is set, forces get -u not to verify that +each package has been checked out from the source control repository +implied by its import path. This can be useful if the source is a local fork +of the original. + +The -fix flag instructs get to run the fix tool on the downloaded packages +before resolving dependencies or building the code. + +The -insecure flag permits fetching from repositories and resolving +custom domains using insecure schemes such as HTTP. Use with caution. + +The -t flag instructs get to also download the packages required to build +the tests for the specified packages. + +The -u flag instructs get to use the network to update the named packages +and their dependencies. By default, get uses the network to check out +missing packages but does not use it to look for updates to existing packages. + +The -v flag enables verbose progress and debug output. + +Get also accepts build flags to control the installation. See 'go help build'. + +When checking out a new package, get creates the target directory +GOPATH/src/. If the GOPATH contains multiple entries, +get uses the first one. For more details see: 'go help gopath'. + +When checking out or updating a package, get looks for a branch or tag +that matches the locally installed version of Go. The most important +rule is that if the local installation is running version "go1", get +searches for a branch or tag named "go1". If no such version exists +it retrieves the default branch of the package. + +When go get checks out or updates a Git repository, +it also updates any git submodules referenced by the repository. + +Get never checks out or updates code stored in vendor directories. + +For more about specifying packages, see 'go help packages'. + +For more about how 'go get' finds source code to +download, see 'go help importpath'. + +See also: go build, go install, go clean. + `, +} + +var getD = CmdGet.Flag.Bool("d", false, "") +var getF = CmdGet.Flag.Bool("f", false, "") +var getT = CmdGet.Flag.Bool("t", false, "") +var getU = CmdGet.Flag.Bool("u", false, "") +var getFix = CmdGet.Flag.Bool("fix", false, "") +var getInsecure = CmdGet.Flag.Bool("insecure", false, "") + +func init() { + work.AddBuildFlags(CmdGet) + CmdGet.Run = runGet // break init loop +} + +func runGet(cmd *base.Command, args []string) { + work.InstrumentInit() + work.BuildModeInit() + + if *getF && !*getU { + base.Fatalf("go get: cannot use -f flag without -u") + } + + // Disable any prompting for passwords by Git. + // Only has an effect for 2.3.0 or later, but avoiding + // the prompt in earlier versions is just too hard. + // If user has explicitly set GIT_TERMINAL_PROMPT=1, keep + // prompting. + // See golang.org/issue/9341 and golang.org/issue/12706. + if os.Getenv("GIT_TERMINAL_PROMPT") == "" { + os.Setenv("GIT_TERMINAL_PROMPT", "0") + } + + // Disable any ssh connection pooling by Git. + // If a Git subprocess forks a child into the background to cache a new connection, + // that child keeps stdout/stderr open. After the Git subprocess exits, + // os /exec expects to be able to read from the stdout/stderr pipe + // until EOF to get all the data that the Git subprocess wrote before exiting. + // The EOF doesn't come until the child exits too, because the child + // is holding the write end of the pipe. + // This is unfortunate, but it has come up at least twice + // (see golang.org/issue/13453 and golang.org/issue/16104) + // and confuses users when it does. + // If the user has explicitly set GIT_SSH or GIT_SSH_COMMAND, + // assume they know what they are doing and don't step on it. + // But default to turning off ControlMaster. + if os.Getenv("GIT_SSH") == "" && os.Getenv("GIT_SSH_COMMAND") == "" { + os.Setenv("GIT_SSH_COMMAND", "ssh -o ControlMaster=no") + } + + // Phase 1. Download/update. + var stk load.ImportStack + mode := 0 + if *getT { + mode |= load.GetTestDeps + } + args = downloadPaths(args) + for _, arg := range args { + download(arg, nil, &stk, mode) + } + base.ExitIfErrors() + + // Phase 2. Rescan packages and re-evaluate args list. + + // Code we downloaded and all code that depends on it + // needs to be evicted from the package cache so that + // the information will be recomputed. Instead of keeping + // track of the reverse dependency information, evict + // everything. + load.ClearPackageCache() + + // In order to rebuild packages information completely, + // we need to clear commands cache. Command packages are + // referring to evicted packages from the package cache. + // This leads to duplicated loads of the standard packages. + load.ClearCmdCache() + + args = load.ImportPaths(args) + load.PackagesForBuild(args) + + // Phase 3. Install. + if *getD { + // Download only. + // Check delayed until now so that importPaths + // and packagesForBuild have a chance to print errors. + return + } + + work.InstallPackages(args, true) +} + +// downloadPaths prepares the list of paths to pass to download. +// It expands ... patterns that can be expanded. If there is no match +// for a particular pattern, downloadPaths leaves it in the result list, +// in the hope that we can figure out the repository from the +// initial ...-free prefix. +func downloadPaths(args []string) []string { + args = load.ImportPathsNoDotExpansion(args) + var out []string + for _, a := range args { + if strings.Contains(a, "...") { + var expand []string + // Use matchPackagesInFS to avoid printing + // warnings. They will be printed by the + // eventual call to importPaths instead. + if build.IsLocalImport(a) { + expand = load.MatchPackagesInFS(a) + } else { + expand = load.MatchPackages(a) + } + if len(expand) > 0 { + out = append(out, expand...) + continue + } + } + out = append(out, a) + } + return out +} + +// downloadCache records the import paths we have already +// considered during the download, to avoid duplicate work when +// there is more than one dependency sequence leading to +// a particular package. +var downloadCache = map[string]bool{} + +// downloadRootCache records the version control repository +// root directories we have already considered during the download. +// For example, all the packages in the github.com/google/codesearch repo +// share the same root (the directory for that path), and we only need +// to run the hg commands to consider each repository once. +var downloadRootCache = map[string]bool{} + +// download runs the download half of the get command +// for the package named by the argument. +func download(arg string, parent *load.Package, stk *load.ImportStack, mode int) { + if mode&load.UseVendor != 0 { + // Caller is responsible for expanding vendor paths. + panic("internal error: download mode has useVendor set") + } + load1 := func(path string, mode int) *load.Package { + if parent == nil { + return load.LoadPackage(path, stk) + } + return load.LoadImport(path, parent.Dir, parent, stk, nil, mode) + } + + p := load1(arg, mode) + if p.Error != nil && p.Error.Hard { + base.Errorf("%s", p.Error) + return + } + + // loadPackage inferred the canonical ImportPath from arg. + // Use that in the following to prevent hysteresis effects + // in e.g. downloadCache and packageCache. + // This allows invocations such as: + // mkdir -p $GOPATH/src/github.com/user + // cd $GOPATH/src/github.com/user + // go get ./foo + // see: golang.org/issue/9767 + arg = p.ImportPath + + // There's nothing to do if this is a package in the standard library. + if p.Standard { + return + } + + // Only process each package once. + // (Unless we're fetching test dependencies for this package, + // in which case we want to process it again.) + if downloadCache[arg] && mode&load.GetTestDeps == 0 { + return + } + downloadCache[arg] = true + + pkgs := []*load.Package{p} + wildcardOkay := len(*stk) == 0 + isWildcard := false + + // Download if the package is missing, or update if we're using -u. + if p.Dir == "" || *getU { + // The actual download. + stk.Push(arg) + err := downloadPackage(p) + if err != nil { + base.Errorf("%s", &load.PackageError{ImportStack: stk.Copy(), Err: err.Error()}) + stk.Pop() + return + } + stk.Pop() + + args := []string{arg} + // If the argument has a wildcard in it, re-evaluate the wildcard. + // We delay this until after reloadPackage so that the old entry + // for p has been replaced in the package cache. + if wildcardOkay && strings.Contains(arg, "...") { + if build.IsLocalImport(arg) { + args = load.MatchPackagesInFS(arg) + } else { + args = load.MatchPackages(arg) + } + isWildcard = true + } + + // Clear all relevant package cache entries before + // doing any new loads. + load.ClearPackageCachePartial(args) + + pkgs = pkgs[:0] + for _, arg := range args { + // Note: load calls loadPackage or loadImport, + // which push arg onto stk already. + // Do not push here too, or else stk will say arg imports arg. + p := load1(arg, mode) + if p.Error != nil { + base.Errorf("%s", p.Error) + continue + } + pkgs = append(pkgs, p) + } + } + + // Process package, which might now be multiple packages + // due to wildcard expansion. + for _, p := range pkgs { + if *getFix { + files := base.FilterDotUnderscoreFiles(base.RelPaths(p.Internal.AllGoFiles)) + base.Run(cfg.BuildToolexec, str.StringList(base.Tool("fix"), files)) + + // The imports might have changed, so reload again. + p = load.ReloadPackage(arg, stk) + if p.Error != nil { + base.Errorf("%s", p.Error) + return + } + } + + if isWildcard { + // Report both the real package and the + // wildcard in any error message. + stk.Push(p.ImportPath) + } + + // Process dependencies, now that we know what they are. + imports := p.Imports + if mode&load.GetTestDeps != 0 { + // Process test dependencies when -t is specified. + // (But don't get test dependencies for test dependencies: + // we always pass mode 0 to the recursive calls below.) + imports = str.StringList(imports, p.TestImports, p.XTestImports) + } + for i, path := range imports { + if path == "C" { + continue + } + // Fail fast on import naming full vendor path. + // Otherwise expand path as needed for test imports. + // Note that p.Imports can have additional entries beyond p.Internal.Build.Imports. + orig := path + if i < len(p.Internal.Build.Imports) { + orig = p.Internal.Build.Imports[i] + } + if j, ok := load.FindVendor(orig); ok { + stk.Push(path) + err := &load.PackageError{ + ImportStack: stk.Copy(), + Err: "must be imported as " + path[j+len("vendor/"):], + } + stk.Pop() + base.Errorf("%s", err) + continue + } + // If this is a test import, apply vendor lookup now. + // We cannot pass useVendor to download, because + // download does caching based on the value of path, + // so it must be the fully qualified path already. + if i >= len(p.Imports) { + path = load.VendoredImportPath(p, path) + } + download(path, p, stk, 0) + } + + if isWildcard { + stk.Pop() + } + } +} + +// downloadPackage runs the create or download command +// to make the first copy of or update a copy of the given package. +func downloadPackage(p *load.Package) error { + var ( + vcs *vcsCmd + repo, rootPath string + err error + ) + + security := web.Secure + if *getInsecure { + security = web.Insecure + } + + if p.Internal.Build.SrcRoot != "" { + // Directory exists. Look for checkout along path to src. + vcs, rootPath, err = vcsFromDir(p.Dir, p.Internal.Build.SrcRoot) + if err != nil { + return err + } + repo = "" // should be unused; make distinctive + + // Double-check where it came from. + if *getU && vcs.remoteRepo != nil { + dir := filepath.Join(p.Internal.Build.SrcRoot, filepath.FromSlash(rootPath)) + remote, err := vcs.remoteRepo(vcs, dir) + if err != nil { + return err + } + repo = remote + if !*getF { + if rr, err := repoRootForImportPath(p.ImportPath, security); err == nil { + repo := rr.repo + if rr.vcs.resolveRepo != nil { + resolved, err := rr.vcs.resolveRepo(rr.vcs, dir, repo) + if err == nil { + repo = resolved + } + } + if remote != repo && rr.isCustom { + return fmt.Errorf("%s is a custom import path for %s, but %s is checked out from %s", rr.root, repo, dir, remote) + } + } + } + } + } else { + // Analyze the import path to determine the version control system, + // repository, and the import path for the root of the repository. + rr, err := repoRootForImportPath(p.ImportPath, security) + if err != nil { + return err + } + vcs, repo, rootPath = rr.vcs, rr.repo, rr.root + } + if !vcs.isSecure(repo) && !*getInsecure { + return fmt.Errorf("cannot download, %v uses insecure protocol", repo) + } + + if p.Internal.Build.SrcRoot == "" { + // Package not found. Put in first directory of $GOPATH. + list := filepath.SplitList(cfg.BuildContext.GOPATH) + if len(list) == 0 { + return fmt.Errorf("cannot download, $GOPATH not set. For more details see: 'go help gopath'") + } + // Guard against people setting GOPATH=$GOROOT. + if filepath.Clean(list[0]) == filepath.Clean(cfg.GOROOT) { + return fmt.Errorf("cannot download, $GOPATH must not be set to $GOROOT. For more details see: 'go help gopath'") + } + if _, err := os.Stat(filepath.Join(list[0], "src/cmd/go/alldocs.go")); err == nil { + return fmt.Errorf("cannot download, %s is a GOROOT, not a GOPATH. For more details see: 'go help gopath'", list[0]) + } + p.Internal.Build.Root = list[0] + p.Internal.Build.SrcRoot = filepath.Join(list[0], "src") + p.Internal.Build.PkgRoot = filepath.Join(list[0], "pkg") + } + root := filepath.Join(p.Internal.Build.SrcRoot, filepath.FromSlash(rootPath)) + // If we've considered this repository already, don't do it again. + if downloadRootCache[root] { + return nil + } + downloadRootCache[root] = true + + if cfg.BuildV { + fmt.Fprintf(os.Stderr, "%s (download)\n", rootPath) + } + + // Check that this is an appropriate place for the repo to be checked out. + // The target directory must either not exist or have a repo checked out already. + meta := filepath.Join(root, "."+vcs.cmd) + st, err := os.Stat(meta) + if err == nil && !st.IsDir() { + return fmt.Errorf("%s exists but is not a directory", meta) + } + if err != nil { + // Metadata directory does not exist. Prepare to checkout new copy. + // Some version control tools require the target directory not to exist. + // We require that too, just to avoid stepping on existing work. + if _, err := os.Stat(root); err == nil { + return fmt.Errorf("%s exists but %s does not - stale checkout?", root, meta) + } + + _, err := os.Stat(p.Internal.Build.Root) + gopathExisted := err == nil + + // Some version control tools require the parent of the target to exist. + parent, _ := filepath.Split(root) + if err = os.MkdirAll(parent, 0777); err != nil { + return err + } + if cfg.BuildV && !gopathExisted && p.Internal.Build.Root == cfg.BuildContext.GOPATH { + fmt.Fprintf(os.Stderr, "created GOPATH=%s; see 'go help gopath'\n", p.Internal.Build.Root) + } + + if err = vcs.create(root, repo); err != nil { + return err + } + } else { + // Metadata directory does exist; download incremental updates. + if err = vcs.download(root); err != nil { + return err + } + } + + if cfg.BuildN { + // Do not show tag sync in -n; it's noise more than anything, + // and since we're not running commands, no tag will be found. + // But avoid printing nothing. + fmt.Fprintf(os.Stderr, "# cd %s; %s sync/update\n", root, vcs.cmd) + return nil + } + + // Select and sync to appropriate version of the repository. + tags, err := vcs.tags(root) + if err != nil { + return err + } + vers := runtime.Version() + if i := strings.Index(vers, " "); i >= 0 { + vers = vers[:i] + } + if err := vcs.tagSync(root, selectTag(vers, tags)); err != nil { + return err + } + + return nil +} + +// selectTag returns the closest matching tag for a given version. +// Closest means the latest one that is not after the current release. +// Version "goX" (or "goX.Y" or "goX.Y.Z") matches tags of the same form. +// Version "release.rN" matches tags of the form "go.rN" (N being a floating-point number). +// Version "weekly.YYYY-MM-DD" matches tags like "go.weekly.YYYY-MM-DD". +// +// NOTE(rsc): Eventually we will need to decide on some logic here. +// For now, there is only "go1". This matches the docs in go help get. +func selectTag(goVersion string, tags []string) (match string) { + for _, t := range tags { + if t == "go1" { + return "go1" + } + } + return "" +} diff --git a/libgo/go/cmd/go/internal/get/pkg_test.go b/libgo/go/cmd/go/internal/get/pkg_test.go new file mode 100644 index 00000000000..b8937a57ec9 --- /dev/null +++ b/libgo/go/cmd/go/internal/get/pkg_test.go @@ -0,0 +1,83 @@ +// Copyright 2014 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package get + +import ( + "cmd/go/internal/str" + "reflect" + "strings" + "testing" +) + +var foldDupTests = []struct { + list []string + f1, f2 string +}{ + {str.StringList("math/rand", "math/big"), "", ""}, + {str.StringList("math", "strings"), "", ""}, + {str.StringList("strings"), "", ""}, + {str.StringList("strings", "strings"), "strings", "strings"}, + {str.StringList("Rand", "rand", "math", "math/rand", "math/Rand"), "Rand", "rand"}, +} + +func TestFoldDup(t *testing.T) { + for _, tt := range foldDupTests { + f1, f2 := str.FoldDup(tt.list) + if f1 != tt.f1 || f2 != tt.f2 { + t.Errorf("foldDup(%q) = %q, %q, want %q, %q", tt.list, f1, f2, tt.f1, tt.f2) + } + } +} + +var parseMetaGoImportsTests = []struct { + in string + out []metaImport +}{ + { + ``, + []metaImport{{"foo/bar", "git", "https://github.com/rsc/foo/bar"}}, + }, + { + ` + `, + []metaImport{ + {"foo/bar", "git", "https://github.com/rsc/foo/bar"}, + {"baz/quux", "git", "http://github.com/rsc/baz/quux"}, + }, + }, + { + ` + + `, + []metaImport{{"foo/bar", "git", "https://github.com/rsc/foo/bar"}}, + }, + { + ` + `, + []metaImport{{"foo/bar", "git", "https://github.com/rsc/foo/bar"}}, + }, + { + ``, + []metaImport{{"foo/bar", "git", "https://github.com/rsc/foo/bar"}}, + }, + { + // XML doesn't like
. + `Page Not Found
DRAFT
`, + []metaImport{{"chitin.io/chitin", "git", "https://github.com/chitin-io/chitin"}}, + }, +} + +func TestParseMetaGoImports(t *testing.T) { + for i, tt := range parseMetaGoImportsTests { + out, err := parseMetaGoImports(strings.NewReader(tt.in)) + if err != nil { + t.Errorf("test#%d: %v", i, err) + continue + } + if !reflect.DeepEqual(out, tt.out) { + t.Errorf("test#%d:\n\thave %q\n\twant %q", i, out, tt.out) + } + } +} diff --git a/libgo/go/cmd/go/internal/get/tag_test.go b/libgo/go/cmd/go/internal/get/tag_test.go new file mode 100644 index 00000000000..9a25dfafbc3 --- /dev/null +++ b/libgo/go/cmd/go/internal/get/tag_test.go @@ -0,0 +1,100 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package get + +import "testing" + +var selectTagTestTags = []string{ + "go.r58", + "go.r58.1", + "go.r59", + "go.r59.1", + "go.r61", + "go.r61.1", + "go.weekly.2010-01-02", + "go.weekly.2011-10-12", + "go.weekly.2011-10-12.1", + "go.weekly.2011-10-14", + "go.weekly.2011-11-01", + "go1", + "go1.0.1", + "go1.999", + "go1.9.2", + "go5", + + // these should be ignored: + "release.r59", + "release.r59.1", + "release", + "weekly.2011-10-12", + "weekly.2011-10-12.1", + "weekly", + "foo", + "bar", + "go.f00", + "go!r60", + "go.1999-01-01", + "go.2x", + "go.20000000000000", + "go.2.", + "go.2.0", + "go2x", + "go20000000000000", + "go2.", + "go2.0", +} + +var selectTagTests = []struct { + version string + selected string +}{ + /* + {"release.r57", ""}, + {"release.r58.2", "go.r58.1"}, + {"release.r59", "go.r59"}, + {"release.r59.1", "go.r59.1"}, + {"release.r60", "go.r59.1"}, + {"release.r60.1", "go.r59.1"}, + {"release.r61", "go.r61"}, + {"release.r66", "go.r61.1"}, + {"weekly.2010-01-01", ""}, + {"weekly.2010-01-02", "go.weekly.2010-01-02"}, + {"weekly.2010-01-02.1", "go.weekly.2010-01-02"}, + {"weekly.2010-01-03", "go.weekly.2010-01-02"}, + {"weekly.2011-10-12", "go.weekly.2011-10-12"}, + {"weekly.2011-10-12.1", "go.weekly.2011-10-12.1"}, + {"weekly.2011-10-13", "go.weekly.2011-10-12.1"}, + {"weekly.2011-10-14", "go.weekly.2011-10-14"}, + {"weekly.2011-10-14.1", "go.weekly.2011-10-14"}, + {"weekly.2011-11-01", "go.weekly.2011-11-01"}, + {"weekly.2014-01-01", "go.weekly.2011-11-01"}, + {"weekly.3000-01-01", "go.weekly.2011-11-01"}, + {"go1", "go1"}, + {"go1.1", "go1.0.1"}, + {"go1.998", "go1.9.2"}, + {"go1.1000", "go1.999"}, + {"go6", "go5"}, + + // faulty versions: + {"release.f00", ""}, + {"weekly.1999-01-01", ""}, + {"junk", ""}, + {"", ""}, + {"go2x", ""}, + {"go200000000000", ""}, + {"go2.", ""}, + {"go2.0", ""}, + */ + {"anything", "go1"}, +} + +func TestSelectTag(t *testing.T) { + for _, c := range selectTagTests { + selected := selectTag(c.version, selectTagTestTags) + if selected != c.selected { + t.Errorf("selectTag(%q) = %q, want %q", c.version, selected, c.selected) + } + } +} diff --git a/libgo/go/cmd/go/internal/get/vcs.go b/libgo/go/cmd/go/internal/get/vcs.go new file mode 100644 index 00000000000..71d0b51344a --- /dev/null +++ b/libgo/go/cmd/go/internal/get/vcs.go @@ -0,0 +1,1005 @@ +// Copyright 2012 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package get + +import ( + "bytes" + "encoding/json" + "errors" + "fmt" + "internal/singleflight" + "log" + "net/url" + "os" + "os/exec" + "path/filepath" + "regexp" + "strings" + "sync" + + "cmd/go/internal/base" + "cmd/go/internal/cfg" + "cmd/go/internal/web" +) + +// A vcsCmd describes how to use a version control system +// like Mercurial, Git, or Subversion. +type vcsCmd struct { + name string + cmd string // name of binary to invoke command + + createCmd []string // commands to download a fresh copy of a repository + downloadCmd []string // commands to download updates into an existing repository + + tagCmd []tagCmd // commands to list tags + tagLookupCmd []tagCmd // commands to lookup tags before running tagSyncCmd + tagSyncCmd []string // commands to sync to specific tag + tagSyncDefault []string // commands to sync to default tag + + scheme []string + pingCmd string + + remoteRepo func(v *vcsCmd, rootDir string) (remoteRepo string, err error) + resolveRepo func(v *vcsCmd, rootDir, remoteRepo string) (realRepo string, err error) +} + +var defaultSecureScheme = map[string]bool{ + "https": true, + "git+ssh": true, + "bzr+ssh": true, + "svn+ssh": true, + "ssh": true, +} + +func (v *vcsCmd) isSecure(repo string) bool { + u, err := url.Parse(repo) + if err != nil { + // If repo is not a URL, it's not secure. + return false + } + return v.isSecureScheme(u.Scheme) +} + +func (v *vcsCmd) isSecureScheme(scheme string) bool { + switch v.cmd { + case "git": + // GIT_ALLOW_PROTOCOL is an environment variable defined by Git. It is a + // colon-separated list of schemes that are allowed to be used with git + // fetch/clone. Any scheme not mentioned will be considered insecure. + if allow := os.Getenv("GIT_ALLOW_PROTOCOL"); allow != "" { + for _, s := range strings.Split(allow, ":") { + if s == scheme { + return true + } + } + return false + } + } + return defaultSecureScheme[scheme] +} + +// A tagCmd describes a command to list available tags +// that can be passed to tagSyncCmd. +type tagCmd struct { + cmd string // command to list tags + pattern string // regexp to extract tags from list +} + +// vcsList lists the known version control systems +var vcsList = []*vcsCmd{ + vcsHg, + vcsGit, + vcsSvn, + vcsBzr, +} + +// vcsByCmd returns the version control system for the given +// command name (hg, git, svn, bzr). +func vcsByCmd(cmd string) *vcsCmd { + for _, vcs := range vcsList { + if vcs.cmd == cmd { + return vcs + } + } + return nil +} + +// vcsHg describes how to use Mercurial. +var vcsHg = &vcsCmd{ + name: "Mercurial", + cmd: "hg", + + createCmd: []string{"clone -U {repo} {dir}"}, + downloadCmd: []string{"pull"}, + + // We allow both tag and branch names as 'tags' + // for selecting a version. This lets people have + // a go.release.r60 branch and a go1 branch + // and make changes in both, without constantly + // editing .hgtags. + tagCmd: []tagCmd{ + {"tags", `^(\S+)`}, + {"branches", `^(\S+)`}, + }, + tagSyncCmd: []string{"update -r {tag}"}, + tagSyncDefault: []string{"update default"}, + + scheme: []string{"https", "http", "ssh"}, + pingCmd: "identify {scheme}://{repo}", + remoteRepo: hgRemoteRepo, +} + +func hgRemoteRepo(vcsHg *vcsCmd, rootDir string) (remoteRepo string, err error) { + out, err := vcsHg.runOutput(rootDir, "paths default") + if err != nil { + return "", err + } + return strings.TrimSpace(string(out)), nil +} + +// vcsGit describes how to use Git. +var vcsGit = &vcsCmd{ + name: "Git", + cmd: "git", + + createCmd: []string{"clone {repo} {dir}", "-go-internal-cd {dir} submodule update --init --recursive"}, + downloadCmd: []string{"pull --ff-only", "submodule update --init --recursive"}, + + tagCmd: []tagCmd{ + // tags/xxx matches a git tag named xxx + // origin/xxx matches a git branch named xxx on the default remote repository + {"show-ref", `(?:tags|origin)/(\S+)$`}, + }, + tagLookupCmd: []tagCmd{ + {"show-ref tags/{tag} origin/{tag}", `((?:tags|origin)/\S+)$`}, + }, + tagSyncCmd: []string{"checkout {tag}", "submodule update --init --recursive"}, + // both createCmd and downloadCmd update the working dir. + // No need to do more here. We used to 'checkout master' + // but that doesn't work if the default branch is not named master. + // DO NOT add 'checkout master' here. + // See golang.org/issue/9032. + tagSyncDefault: []string{"submodule update --init --recursive"}, + + scheme: []string{"git", "https", "http", "git+ssh", "ssh"}, + pingCmd: "ls-remote {scheme}://{repo}", + remoteRepo: gitRemoteRepo, +} + +// scpSyntaxRe matches the SCP-like addresses used by Git to access +// repositories by SSH. +var scpSyntaxRe = regexp.MustCompile(`^([a-zA-Z0-9_]+)@([a-zA-Z0-9._-]+):(.*)$`) + +func gitRemoteRepo(vcsGit *vcsCmd, rootDir string) (remoteRepo string, err error) { + cmd := "config remote.origin.url" + errParse := errors.New("unable to parse output of git " + cmd) + errRemoteOriginNotFound := errors.New("remote origin not found") + outb, err := vcsGit.run1(rootDir, cmd, nil, false) + if err != nil { + // if it doesn't output any message, it means the config argument is correct, + // but the config value itself doesn't exist + if outb != nil && len(outb) == 0 { + return "", errRemoteOriginNotFound + } + return "", err + } + out := strings.TrimSpace(string(outb)) + + var repoURL *url.URL + if m := scpSyntaxRe.FindStringSubmatch(out); m != nil { + // Match SCP-like syntax and convert it to a URL. + // Eg, "git@github.com:user/repo" becomes + // "ssh://git@github.com/user/repo". + repoURL = &url.URL{ + Scheme: "ssh", + User: url.User(m[1]), + Host: m[2], + Path: m[3], + } + } else { + repoURL, err = url.Parse(out) + if err != nil { + return "", err + } + } + + // Iterate over insecure schemes too, because this function simply + // reports the state of the repo. If we can't see insecure schemes then + // we can't report the actual repo URL. + for _, s := range vcsGit.scheme { + if repoURL.Scheme == s { + return repoURL.String(), nil + } + } + return "", errParse +} + +// vcsBzr describes how to use Bazaar. +var vcsBzr = &vcsCmd{ + name: "Bazaar", + cmd: "bzr", + + createCmd: []string{"branch {repo} {dir}"}, + + // Without --overwrite bzr will not pull tags that changed. + // Replace by --overwrite-tags after http://pad.lv/681792 goes in. + downloadCmd: []string{"pull --overwrite"}, + + tagCmd: []tagCmd{{"tags", `^(\S+)`}}, + tagSyncCmd: []string{"update -r {tag}"}, + tagSyncDefault: []string{"update -r revno:-1"}, + + scheme: []string{"https", "http", "bzr", "bzr+ssh"}, + pingCmd: "info {scheme}://{repo}", + remoteRepo: bzrRemoteRepo, + resolveRepo: bzrResolveRepo, +} + +func bzrRemoteRepo(vcsBzr *vcsCmd, rootDir string) (remoteRepo string, err error) { + outb, err := vcsBzr.runOutput(rootDir, "config parent_location") + if err != nil { + return "", err + } + return strings.TrimSpace(string(outb)), nil +} + +func bzrResolveRepo(vcsBzr *vcsCmd, rootDir, remoteRepo string) (realRepo string, err error) { + outb, err := vcsBzr.runOutput(rootDir, "info "+remoteRepo) + if err != nil { + return "", err + } + out := string(outb) + + // Expect: + // ... + // (branch root|repository branch): + // ... + + found := false + for _, prefix := range []string{"\n branch root: ", "\n repository branch: "} { + i := strings.Index(out, prefix) + if i >= 0 { + out = out[i+len(prefix):] + found = true + break + } + } + if !found { + return "", fmt.Errorf("unable to parse output of bzr info") + } + + i := strings.Index(out, "\n") + if i < 0 { + return "", fmt.Errorf("unable to parse output of bzr info") + } + out = out[:i] + return strings.TrimSpace(out), nil +} + +// vcsSvn describes how to use Subversion. +var vcsSvn = &vcsCmd{ + name: "Subversion", + cmd: "svn", + + createCmd: []string{"checkout {repo} {dir}"}, + downloadCmd: []string{"update"}, + + // There is no tag command in subversion. + // The branch information is all in the path names. + + scheme: []string{"https", "http", "svn", "svn+ssh"}, + pingCmd: "info {scheme}://{repo}", + remoteRepo: svnRemoteRepo, +} + +func svnRemoteRepo(vcsSvn *vcsCmd, rootDir string) (remoteRepo string, err error) { + outb, err := vcsSvn.runOutput(rootDir, "info") + if err != nil { + return "", err + } + out := string(outb) + + // Expect: + // + // ... + // URL: + // ... + // + // Note that we're not using the Repository Root line, + // because svn allows checking out subtrees. + // The URL will be the URL of the subtree (what we used with 'svn co') + // while the Repository Root may be a much higher parent. + i := strings.Index(out, "\nURL: ") + if i < 0 { + return "", fmt.Errorf("unable to parse output of svn info") + } + out = out[i+len("\nURL: "):] + i = strings.Index(out, "\n") + if i < 0 { + return "", fmt.Errorf("unable to parse output of svn info") + } + out = out[:i] + return strings.TrimSpace(out), nil +} + +func (v *vcsCmd) String() string { + return v.name +} + +// run runs the command line cmd in the given directory. +// keyval is a list of key, value pairs. run expands +// instances of {key} in cmd into value, but only after +// splitting cmd into individual arguments. +// If an error occurs, run prints the command line and the +// command's combined stdout+stderr to standard error. +// Otherwise run discards the command's output. +func (v *vcsCmd) run(dir string, cmd string, keyval ...string) error { + _, err := v.run1(dir, cmd, keyval, true) + return err +} + +// runVerboseOnly is like run but only generates error output to standard error in verbose mode. +func (v *vcsCmd) runVerboseOnly(dir string, cmd string, keyval ...string) error { + _, err := v.run1(dir, cmd, keyval, false) + return err +} + +// runOutput is like run but returns the output of the command. +func (v *vcsCmd) runOutput(dir string, cmd string, keyval ...string) ([]byte, error) { + return v.run1(dir, cmd, keyval, true) +} + +// run1 is the generalized implementation of run and runOutput. +func (v *vcsCmd) run1(dir string, cmdline string, keyval []string, verbose bool) ([]byte, error) { + m := make(map[string]string) + for i := 0; i < len(keyval); i += 2 { + m[keyval[i]] = keyval[i+1] + } + args := strings.Fields(cmdline) + for i, arg := range args { + args[i] = expand(m, arg) + } + + if len(args) >= 2 && args[0] == "-go-internal-cd" { + if filepath.IsAbs(args[1]) { + dir = args[1] + } else { + dir = filepath.Join(dir, args[1]) + } + args = args[2:] + } + + _, err := exec.LookPath(v.cmd) + if err != nil { + fmt.Fprintf(os.Stderr, + "go: missing %s command. See https://golang.org/s/gogetcmd\n", + v.name) + return nil, err + } + + cmd := exec.Command(v.cmd, args...) + cmd.Dir = dir + cmd.Env = base.EnvForDir(cmd.Dir, os.Environ()) + if cfg.BuildX { + fmt.Printf("cd %s\n", dir) + fmt.Printf("%s %s\n", v.cmd, strings.Join(args, " ")) + } + var buf bytes.Buffer + cmd.Stdout = &buf + cmd.Stderr = &buf + err = cmd.Run() + out := buf.Bytes() + if err != nil { + if verbose || cfg.BuildV { + fmt.Fprintf(os.Stderr, "# cd %s; %s %s\n", dir, v.cmd, strings.Join(args, " ")) + os.Stderr.Write(out) + } + return out, err + } + return out, nil +} + +// ping pings to determine scheme to use. +func (v *vcsCmd) ping(scheme, repo string) error { + return v.runVerboseOnly(".", v.pingCmd, "scheme", scheme, "repo", repo) +} + +// create creates a new copy of repo in dir. +// The parent of dir must exist; dir must not. +func (v *vcsCmd) create(dir, repo string) error { + for _, cmd := range v.createCmd { + if err := v.run(".", cmd, "dir", dir, "repo", repo); err != nil { + return err + } + } + return nil +} + +// download downloads any new changes for the repo in dir. +func (v *vcsCmd) download(dir string) error { + for _, cmd := range v.downloadCmd { + if err := v.run(dir, cmd); err != nil { + return err + } + } + return nil +} + +// tags returns the list of available tags for the repo in dir. +func (v *vcsCmd) tags(dir string) ([]string, error) { + var tags []string + for _, tc := range v.tagCmd { + out, err := v.runOutput(dir, tc.cmd) + if err != nil { + return nil, err + } + re := regexp.MustCompile(`(?m-s)` + tc.pattern) + for _, m := range re.FindAllStringSubmatch(string(out), -1) { + tags = append(tags, m[1]) + } + } + return tags, nil +} + +// tagSync syncs the repo in dir to the named tag, +// which either is a tag returned by tags or is v.tagDefault. +func (v *vcsCmd) tagSync(dir, tag string) error { + if v.tagSyncCmd == nil { + return nil + } + if tag != "" { + for _, tc := range v.tagLookupCmd { + out, err := v.runOutput(dir, tc.cmd, "tag", tag) + if err != nil { + return err + } + re := regexp.MustCompile(`(?m-s)` + tc.pattern) + m := re.FindStringSubmatch(string(out)) + if len(m) > 1 { + tag = m[1] + break + } + } + } + + if tag == "" && v.tagSyncDefault != nil { + for _, cmd := range v.tagSyncDefault { + if err := v.run(dir, cmd); err != nil { + return err + } + } + return nil + } + + for _, cmd := range v.tagSyncCmd { + if err := v.run(dir, cmd, "tag", tag); err != nil { + return err + } + } + return nil +} + +// A vcsPath describes how to convert an import path into a +// version control system and repository name. +type vcsPath struct { + prefix string // prefix this description applies to + re string // pattern for import path + repo string // repository to use (expand with match of re) + vcs string // version control system to use (expand with match of re) + check func(match map[string]string) error // additional checks + ping bool // ping for scheme to use to download repo + + regexp *regexp.Regexp // cached compiled form of re +} + +// vcsFromDir inspects dir and its parents to determine the +// version control system and code repository to use. +// On return, root is the import path +// corresponding to the root of the repository. +func vcsFromDir(dir, srcRoot string) (vcs *vcsCmd, root string, err error) { + // Clean and double-check that dir is in (a subdirectory of) srcRoot. + dir = filepath.Clean(dir) + srcRoot = filepath.Clean(srcRoot) + if len(dir) <= len(srcRoot) || dir[len(srcRoot)] != filepath.Separator { + return nil, "", fmt.Errorf("directory %q is outside source root %q", dir, srcRoot) + } + + origDir := dir + for len(dir) > len(srcRoot) { + for _, vcs := range vcsList { + if _, err := os.Stat(filepath.Join(dir, "."+vcs.cmd)); err == nil { + return vcs, filepath.ToSlash(dir[len(srcRoot)+1:]), nil + } + } + + // Move to parent. + ndir := filepath.Dir(dir) + if len(ndir) >= len(dir) { + // Shouldn't happen, but just in case, stop. + break + } + dir = ndir + } + + return nil, "", fmt.Errorf("directory %q is not using a known version control system", origDir) +} + +// repoRoot represents a version control system, a repo, and a root of +// where to put it on disk. +type repoRoot struct { + vcs *vcsCmd + + // repo is the repository URL, including scheme + repo string + + // root is the import path corresponding to the root of the + // repository + root string + + // isCustom is true for custom import paths (those defined by HTML meta tags) + isCustom bool +} + +var httpPrefixRE = regexp.MustCompile(`^https?:`) + +// repoRootForImportPath analyzes importPath to determine the +// version control system, and code repository to use. +func repoRootForImportPath(importPath string, security web.SecurityMode) (*repoRoot, error) { + rr, err := repoRootFromVCSPaths(importPath, "", security, vcsPaths) + if err == errUnknownSite { + // If there are wildcards, look up the thing before the wildcard, + // hoping it applies to the wildcarded parts too. + // This makes 'go get rsc.io/pdf/...' work in a fresh GOPATH. + lookup := strings.TrimSuffix(importPath, "/...") + if i := strings.Index(lookup, "/.../"); i >= 0 { + lookup = lookup[:i] + } + rr, err = repoRootForImportDynamic(lookup, security) + if err != nil { + err = fmt.Errorf("unrecognized import path %q (%v)", importPath, err) + } + } + if err != nil { + rr1, err1 := repoRootFromVCSPaths(importPath, "", security, vcsPathsAfterDynamic) + if err1 == nil { + rr = rr1 + err = nil + } + } + + if err == nil && strings.Contains(importPath, "...") && strings.Contains(rr.root, "...") { + // Do not allow wildcards in the repo root. + rr = nil + err = fmt.Errorf("cannot expand ... in %q", importPath) + } + return rr, err +} + +var errUnknownSite = errors.New("dynamic lookup required to find mapping") + +// repoRootFromVCSPaths attempts to map importPath to a repoRoot +// using the mappings defined in vcsPaths. +// If scheme is non-empty, that scheme is forced. +func repoRootFromVCSPaths(importPath, scheme string, security web.SecurityMode, vcsPaths []*vcsPath) (*repoRoot, error) { + // A common error is to use https://packagepath because that's what + // hg and git require. Diagnose this helpfully. + if loc := httpPrefixRE.FindStringIndex(importPath); loc != nil { + // The importPath has been cleaned, so has only one slash. The pattern + // ignores the slashes; the error message puts them back on the RHS at least. + return nil, fmt.Errorf("%q not allowed in import path", importPath[loc[0]:loc[1]]+"//") + } + for _, srv := range vcsPaths { + if !strings.HasPrefix(importPath, srv.prefix) { + continue + } + m := srv.regexp.FindStringSubmatch(importPath) + if m == nil { + if srv.prefix != "" { + return nil, fmt.Errorf("invalid %s import path %q", srv.prefix, importPath) + } + continue + } + + // Build map of named subexpression matches for expand. + match := map[string]string{ + "prefix": srv.prefix, + "import": importPath, + } + for i, name := range srv.regexp.SubexpNames() { + if name != "" && match[name] == "" { + match[name] = m[i] + } + } + if srv.vcs != "" { + match["vcs"] = expand(match, srv.vcs) + } + if srv.repo != "" { + match["repo"] = expand(match, srv.repo) + } + if srv.check != nil { + if err := srv.check(match); err != nil { + return nil, err + } + } + vcs := vcsByCmd(match["vcs"]) + if vcs == nil { + return nil, fmt.Errorf("unknown version control system %q", match["vcs"]) + } + if srv.ping { + if scheme != "" { + match["repo"] = scheme + "://" + match["repo"] + } else { + for _, scheme := range vcs.scheme { + if security == web.Secure && !vcs.isSecureScheme(scheme) { + continue + } + if vcs.ping(scheme, match["repo"]) == nil { + match["repo"] = scheme + "://" + match["repo"] + break + } + } + } + } + rr := &repoRoot{ + vcs: vcs, + repo: match["repo"], + root: match["root"], + } + return rr, nil + } + return nil, errUnknownSite +} + +// repoRootForImportDynamic finds a *repoRoot for a custom domain that's not +// statically known by repoRootForImportPathStatic. +// +// This handles custom import paths like "name.tld/pkg/foo" or just "name.tld". +func repoRootForImportDynamic(importPath string, security web.SecurityMode) (*repoRoot, error) { + slash := strings.Index(importPath, "/") + if slash < 0 { + slash = len(importPath) + } + host := importPath[:slash] + if !strings.Contains(host, ".") { + return nil, errors.New("import path does not begin with hostname") + } + urlStr, body, err := web.GetMaybeInsecure(importPath, security) + if err != nil { + msg := "https fetch: %v" + if security == web.Insecure { + msg = "http/" + msg + } + return nil, fmt.Errorf(msg, err) + } + defer body.Close() + imports, err := parseMetaGoImports(body) + if err != nil { + return nil, fmt.Errorf("parsing %s: %v", importPath, err) + } + // Find the matched meta import. + mmi, err := matchGoImport(imports, importPath) + if err != nil { + if _, ok := err.(ImportMismatchError); !ok { + return nil, fmt.Errorf("parse %s: %v", urlStr, err) + } + return nil, fmt.Errorf("parse %s: no go-import meta tags (%s)", urlStr, err) + } + if cfg.BuildV { + log.Printf("get %q: found meta tag %#v at %s", importPath, mmi, urlStr) + } + // If the import was "uni.edu/bob/project", which said the + // prefix was "uni.edu" and the RepoRoot was "evilroot.com", + // make sure we don't trust Bob and check out evilroot.com to + // "uni.edu" yet (possibly overwriting/preempting another + // non-evil student). Instead, first verify the root and see + // if it matches Bob's claim. + if mmi.Prefix != importPath { + if cfg.BuildV { + log.Printf("get %q: verifying non-authoritative meta tag", importPath) + } + urlStr0 := urlStr + var imports []metaImport + urlStr, imports, err = metaImportsForPrefix(mmi.Prefix, security) + if err != nil { + return nil, err + } + metaImport2, err := matchGoImport(imports, importPath) + if err != nil || mmi != metaImport2 { + return nil, fmt.Errorf("%s and %s disagree about go-import for %s", urlStr0, urlStr, mmi.Prefix) + } + } + + if !strings.Contains(mmi.RepoRoot, "://") { + return nil, fmt.Errorf("%s: invalid repo root %q; no scheme", urlStr, mmi.RepoRoot) + } + rr := &repoRoot{ + vcs: vcsByCmd(mmi.VCS), + repo: mmi.RepoRoot, + root: mmi.Prefix, + isCustom: true, + } + if rr.vcs == nil { + return nil, fmt.Errorf("%s: unknown vcs %q", urlStr, mmi.VCS) + } + return rr, nil +} + +var fetchGroup singleflight.Group +var ( + fetchCacheMu sync.Mutex + fetchCache = map[string]fetchResult{} // key is metaImportsForPrefix's importPrefix +) + +// metaImportsForPrefix takes a package's root import path as declared in a tag +// and returns its HTML discovery URL and the parsed metaImport lines +// found on the page. +// +// The importPath is of the form "golang.org/x/tools". +// It is an error if no imports are found. +// urlStr will still be valid if err != nil. +// The returned urlStr will be of the form "https://golang.org/x/tools?go-get=1" +func metaImportsForPrefix(importPrefix string, security web.SecurityMode) (urlStr string, imports []metaImport, err error) { + setCache := func(res fetchResult) (fetchResult, error) { + fetchCacheMu.Lock() + defer fetchCacheMu.Unlock() + fetchCache[importPrefix] = res + return res, nil + } + + resi, _, _ := fetchGroup.Do(importPrefix, func() (resi interface{}, err error) { + fetchCacheMu.Lock() + if res, ok := fetchCache[importPrefix]; ok { + fetchCacheMu.Unlock() + return res, nil + } + fetchCacheMu.Unlock() + + urlStr, body, err := web.GetMaybeInsecure(importPrefix, security) + if err != nil { + return setCache(fetchResult{urlStr: urlStr, err: fmt.Errorf("fetch %s: %v", urlStr, err)}) + } + imports, err := parseMetaGoImports(body) + if err != nil { + return setCache(fetchResult{urlStr: urlStr, err: fmt.Errorf("parsing %s: %v", urlStr, err)}) + } + if len(imports) == 0 { + err = fmt.Errorf("fetch %s: no go-import meta tag", urlStr) + } + return setCache(fetchResult{urlStr: urlStr, imports: imports, err: err}) + }) + res := resi.(fetchResult) + return res.urlStr, res.imports, res.err +} + +type fetchResult struct { + urlStr string // e.g. "https://foo.com/x/bar?go-get=1" + imports []metaImport + err error +} + +// metaImport represents the parsed tags from HTML files. +type metaImport struct { + Prefix, VCS, RepoRoot string +} + +func splitPathHasPrefix(path, prefix []string) bool { + if len(path) < len(prefix) { + return false + } + for i, p := range prefix { + if path[i] != p { + return false + } + } + return true +} + +// A ImportMismatchError is returned where metaImport/s are present +// but none match our import path. +type ImportMismatchError struct { + importPath string + mismatches []string // the meta imports that were discarded for not matching our importPath +} + +func (m ImportMismatchError) Error() string { + formattedStrings := make([]string, len(m.mismatches)) + for i, pre := range m.mismatches { + formattedStrings[i] = fmt.Sprintf("meta tag %s did not match import path %s", pre, m.importPath) + } + return strings.Join(formattedStrings, ", ") +} + +// matchGoImport returns the metaImport from imports matching importPath. +// An error is returned if there are multiple matches. +// errNoMatch is returned if none match. +func matchGoImport(imports []metaImport, importPath string) (metaImport, error) { + match := -1 + imp := strings.Split(importPath, "/") + + errImportMismatch := ImportMismatchError{importPath: importPath} + for i, im := range imports { + pre := strings.Split(im.Prefix, "/") + + if !splitPathHasPrefix(imp, pre) { + errImportMismatch.mismatches = append(errImportMismatch.mismatches, im.Prefix) + continue + } + + if match != -1 { + return metaImport{}, fmt.Errorf("multiple meta tags match import path %q", importPath) + } + match = i + } + + if match == -1 { + return metaImport{}, errImportMismatch + } + return imports[match], nil +} + +// expand rewrites s to replace {k} with match[k] for each key k in match. +func expand(match map[string]string, s string) string { + for k, v := range match { + s = strings.Replace(s, "{"+k+"}", v, -1) + } + return s +} + +// vcsPaths defines the meaning of import paths referring to +// commonly-used VCS hosting sites (github.com/user/dir) +// and import paths referring to a fully-qualified importPath +// containing a VCS type (foo.com/repo.git/dir) +var vcsPaths = []*vcsPath{ + // Github + { + prefix: "github.com/", + re: `^(?Pgithub\.com/[A-Za-z0-9_.\-]+/[A-Za-z0-9_.\-]+)(/[\p{L}0-9_.\-]+)*$`, + vcs: "git", + repo: "https://{root}", + check: noVCSSuffix, + }, + + // Bitbucket + { + prefix: "bitbucket.org/", + re: `^(?Pbitbucket\.org/(?P[A-Za-z0-9_.\-]+/[A-Za-z0-9_.\-]+))(/[A-Za-z0-9_.\-]+)*$`, + repo: "https://{root}", + check: bitbucketVCS, + }, + + // IBM DevOps Services (JazzHub) + { + prefix: "hub.jazz.net/git", + re: `^(?Phub.jazz.net/git/[a-z0-9]+/[A-Za-z0-9_.\-]+)(/[A-Za-z0-9_.\-]+)*$`, + vcs: "git", + repo: "https://{root}", + check: noVCSSuffix, + }, + + // Git at Apache + { + prefix: "git.apache.org", + re: `^(?Pgit.apache.org/[a-z0-9_.\-]+\.git)(/[A-Za-z0-9_.\-]+)*$`, + vcs: "git", + repo: "https://{root}", + }, + + // Git at OpenStack + { + prefix: "git.openstack.org", + re: `^(?Pgit\.openstack\.org/[A-Za-z0-9_.\-]+/[A-Za-z0-9_.\-]+)(\.git)?(/[A-Za-z0-9_.\-]+)*$`, + vcs: "git", + repo: "https://{root}", + }, + + // General syntax for any server. + // Must be last. + { + re: `^(?P(?P([a-z0-9.\-]+\.)+[a-z0-9.\-]+(:[0-9]+)?(/~?[A-Za-z0-9_.\-]+)+?)\.(?Pbzr|git|hg|svn))(/~?[A-Za-z0-9_.\-]+)*$`, + ping: true, + }, +} + +// vcsPathsAfterDynamic gives additional vcsPaths entries +// to try after the dynamic HTML check. +// This gives those sites a chance to introduce tags +// as part of a graceful transition away from the hard-coded logic. +var vcsPathsAfterDynamic = []*vcsPath{ + // Launchpad. See golang.org/issue/11436. + { + prefix: "launchpad.net/", + re: `^(?Plaunchpad\.net/((?P[A-Za-z0-9_.\-]+)(?P/[A-Za-z0-9_.\-]+)?|~[A-Za-z0-9_.\-]+/(\+junk|[A-Za-z0-9_.\-]+)/[A-Za-z0-9_.\-]+))(/[A-Za-z0-9_.\-]+)*$`, + vcs: "bzr", + repo: "https://{root}", + check: launchpadVCS, + }, +} + +func init() { + // fill in cached regexps. + // Doing this eagerly discovers invalid regexp syntax + // without having to run a command that needs that regexp. + for _, srv := range vcsPaths { + srv.regexp = regexp.MustCompile(srv.re) + } + for _, srv := range vcsPathsAfterDynamic { + srv.regexp = regexp.MustCompile(srv.re) + } +} + +// noVCSSuffix checks that the repository name does not +// end in .foo for any version control system foo. +// The usual culprit is ".git". +func noVCSSuffix(match map[string]string) error { + repo := match["repo"] + for _, vcs := range vcsList { + if strings.HasSuffix(repo, "."+vcs.cmd) { + return fmt.Errorf("invalid version control suffix in %s path", match["prefix"]) + } + } + return nil +} + +// bitbucketVCS determines the version control system for a +// Bitbucket repository, by using the Bitbucket API. +func bitbucketVCS(match map[string]string) error { + if err := noVCSSuffix(match); err != nil { + return err + } + + var resp struct { + SCM string `json:"scm"` + } + url := expand(match, "https://api.bitbucket.org/2.0/repositories/{bitname}?fields=scm") + data, err := web.Get(url) + if err != nil { + if httpErr, ok := err.(*web.HTTPError); ok && httpErr.StatusCode == 403 { + // this may be a private repository. If so, attempt to determine which + // VCS it uses. See issue 5375. + root := match["root"] + for _, vcs := range []string{"git", "hg"} { + if vcsByCmd(vcs).ping("https", root) == nil { + resp.SCM = vcs + break + } + } + } + + if resp.SCM == "" { + return err + } + } else { + if err := json.Unmarshal(data, &resp); err != nil { + return fmt.Errorf("decoding %s: %v", url, err) + } + } + + if vcsByCmd(resp.SCM) != nil { + match["vcs"] = resp.SCM + if resp.SCM == "git" { + match["repo"] += ".git" + } + return nil + } + + return fmt.Errorf("unable to detect version control system for bitbucket.org/ path") +} + +// launchpadVCS solves the ambiguity for "lp.net/project/foo". In this case, +// "foo" could be a series name registered in Launchpad with its own branch, +// and it could also be the name of a directory within the main project +// branch one level up. +func launchpadVCS(match map[string]string) error { + if match["project"] == "" || match["series"] == "" { + return nil + } + _, err := web.Get(expand(match, "https://code.launchpad.net/{project}{series}/.bzr/branch-format")) + if err != nil { + match["root"] = expand(match, "launchpad.net/{project}") + match["repo"] = expand(match, "https://{root}") + } + return nil +} diff --git a/libgo/go/cmd/go/internal/get/vcs_test.go b/libgo/go/cmd/go/internal/get/vcs_test.go new file mode 100644 index 00000000000..62d352ae575 --- /dev/null +++ b/libgo/go/cmd/go/internal/get/vcs_test.go @@ -0,0 +1,392 @@ +// Copyright 2014 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package get + +import ( + "errors" + "internal/testenv" + "io/ioutil" + "os" + "path" + "path/filepath" + "testing" + + "cmd/go/internal/web" +) + +// Test that RepoRootForImportPath creates the correct RepoRoot for a given importPath. +// TODO(cmang): Add tests for SVN and BZR. +func TestRepoRootForImportPath(t *testing.T) { + testenv.MustHaveExternalNetwork(t) + + tests := []struct { + path string + want *repoRoot + }{ + { + "github.com/golang/groupcache", + &repoRoot{ + vcs: vcsGit, + repo: "https://github.com/golang/groupcache", + }, + }, + // Unicode letters in directories (issue 18660). + { + "github.com/user/unicode/испытание", + &repoRoot{ + vcs: vcsGit, + repo: "https://github.com/user/unicode", + }, + }, + // IBM DevOps Services tests + { + "hub.jazz.net/git/user1/pkgname", + &repoRoot{ + vcs: vcsGit, + repo: "https://hub.jazz.net/git/user1/pkgname", + }, + }, + { + "hub.jazz.net/git/user1/pkgname/submodule/submodule/submodule", + &repoRoot{ + vcs: vcsGit, + repo: "https://hub.jazz.net/git/user1/pkgname", + }, + }, + { + "hub.jazz.net", + nil, + }, + { + "hub2.jazz.net", + nil, + }, + { + "hub.jazz.net/someotherprefix", + nil, + }, + { + "hub.jazz.net/someotherprefix/user1/pkgname", + nil, + }, + // Spaces are not valid in user names or package names + { + "hub.jazz.net/git/User 1/pkgname", + nil, + }, + { + "hub.jazz.net/git/user1/pkg name", + nil, + }, + // Dots are not valid in user names + { + "hub.jazz.net/git/user.1/pkgname", + nil, + }, + { + "hub.jazz.net/git/user/pkg.name", + &repoRoot{ + vcs: vcsGit, + repo: "https://hub.jazz.net/git/user/pkg.name", + }, + }, + // User names cannot have uppercase letters + { + "hub.jazz.net/git/USER/pkgname", + nil, + }, + // OpenStack tests + { + "git.openstack.org/openstack/swift", + &repoRoot{ + vcs: vcsGit, + repo: "https://git.openstack.org/openstack/swift", + }, + }, + // Trailing .git is less preferred but included for + // compatibility purposes while the same source needs to + // be compilable on both old and new go + { + "git.openstack.org/openstack/swift.git", + &repoRoot{ + vcs: vcsGit, + repo: "https://git.openstack.org/openstack/swift.git", + }, + }, + { + "git.openstack.org/openstack/swift/go/hummingbird", + &repoRoot{ + vcs: vcsGit, + repo: "https://git.openstack.org/openstack/swift", + }, + }, + { + "git.openstack.org", + nil, + }, + { + "git.openstack.org/openstack", + nil, + }, + // Spaces are not valid in package name + { + "git.apache.org/package name/path/to/lib", + nil, + }, + // Should have ".git" suffix + { + "git.apache.org/package-name/path/to/lib", + nil, + }, + { + "git.apache.org/package-name.git", + &repoRoot{ + vcs: vcsGit, + repo: "https://git.apache.org/package-name.git", + }, + }, + { + "git.apache.org/package-name_2.x.git/path/to/lib", + &repoRoot{ + vcs: vcsGit, + repo: "https://git.apache.org/package-name_2.x.git", + }, + }, + } + + for _, test := range tests { + got, err := repoRootForImportPath(test.path, web.Secure) + want := test.want + + if want == nil { + if err == nil { + t.Errorf("repoRootForImportPath(%q): Error expected but not received", test.path) + } + continue + } + if err != nil { + t.Errorf("repoRootForImportPath(%q): %v", test.path, err) + continue + } + if got.vcs.name != want.vcs.name || got.repo != want.repo { + t.Errorf("repoRootForImportPath(%q) = VCS(%s) Repo(%s), want VCS(%s) Repo(%s)", test.path, got.vcs, got.repo, want.vcs, want.repo) + } + } +} + +// Test that vcsFromDir correctly inspects a given directory and returns the right VCS and root. +func TestFromDir(t *testing.T) { + tempDir, err := ioutil.TempDir("", "vcstest") + if err != nil { + t.Fatal(err) + } + defer os.RemoveAll(tempDir) + + for j, vcs := range vcsList { + dir := filepath.Join(tempDir, "example.com", vcs.name, "."+vcs.cmd) + if j&1 == 0 { + err := os.MkdirAll(dir, 0755) + if err != nil { + t.Fatal(err) + } + } else { + err := os.MkdirAll(filepath.Dir(dir), 0755) + if err != nil { + t.Fatal(err) + } + f, err := os.Create(dir) + if err != nil { + t.Fatal(err) + } + f.Close() + } + + want := repoRoot{ + vcs: vcs, + root: path.Join("example.com", vcs.name), + } + var got repoRoot + got.vcs, got.root, err = vcsFromDir(dir, tempDir) + if err != nil { + t.Errorf("FromDir(%q, %q): %v", dir, tempDir, err) + continue + } + if got.vcs.name != want.vcs.name || got.root != want.root { + t.Errorf("FromDir(%q, %q) = VCS(%s) Root(%s), want VCS(%s) Root(%s)", dir, tempDir, got.vcs, got.root, want.vcs, want.root) + } + } +} + +func TestIsSecure(t *testing.T) { + tests := []struct { + vcs *vcsCmd + url string + secure bool + }{ + {vcsGit, "http://example.com/foo.git", false}, + {vcsGit, "https://example.com/foo.git", true}, + {vcsBzr, "http://example.com/foo.bzr", false}, + {vcsBzr, "https://example.com/foo.bzr", true}, + {vcsSvn, "http://example.com/svn", false}, + {vcsSvn, "https://example.com/svn", true}, + {vcsHg, "http://example.com/foo.hg", false}, + {vcsHg, "https://example.com/foo.hg", true}, + {vcsGit, "ssh://user@example.com/foo.git", true}, + {vcsGit, "user@server:path/to/repo.git", false}, + {vcsGit, "user@server:", false}, + {vcsGit, "server:repo.git", false}, + {vcsGit, "server:path/to/repo.git", false}, + {vcsGit, "example.com:path/to/repo.git", false}, + {vcsGit, "path/that/contains/a:colon/repo.git", false}, + {vcsHg, "ssh://user@example.com/path/to/repo.hg", true}, + } + + for _, test := range tests { + secure := test.vcs.isSecure(test.url) + if secure != test.secure { + t.Errorf("%s isSecure(%q) = %t; want %t", test.vcs, test.url, secure, test.secure) + } + } +} + +func TestIsSecureGitAllowProtocol(t *testing.T) { + tests := []struct { + vcs *vcsCmd + url string + secure bool + }{ + // Same as TestIsSecure to verify same behavior. + {vcsGit, "http://example.com/foo.git", false}, + {vcsGit, "https://example.com/foo.git", true}, + {vcsBzr, "http://example.com/foo.bzr", false}, + {vcsBzr, "https://example.com/foo.bzr", true}, + {vcsSvn, "http://example.com/svn", false}, + {vcsSvn, "https://example.com/svn", true}, + {vcsHg, "http://example.com/foo.hg", false}, + {vcsHg, "https://example.com/foo.hg", true}, + {vcsGit, "user@server:path/to/repo.git", false}, + {vcsGit, "user@server:", false}, + {vcsGit, "server:repo.git", false}, + {vcsGit, "server:path/to/repo.git", false}, + {vcsGit, "example.com:path/to/repo.git", false}, + {vcsGit, "path/that/contains/a:colon/repo.git", false}, + {vcsHg, "ssh://user@example.com/path/to/repo.hg", true}, + // New behavior. + {vcsGit, "ssh://user@example.com/foo.git", false}, + {vcsGit, "foo://example.com/bar.git", true}, + {vcsHg, "foo://example.com/bar.hg", false}, + {vcsSvn, "foo://example.com/svn", false}, + {vcsBzr, "foo://example.com/bar.bzr", false}, + } + + defer os.Unsetenv("GIT_ALLOW_PROTOCOL") + os.Setenv("GIT_ALLOW_PROTOCOL", "https:foo") + for _, test := range tests { + secure := test.vcs.isSecure(test.url) + if secure != test.secure { + t.Errorf("%s isSecure(%q) = %t; want %t", test.vcs, test.url, secure, test.secure) + } + } +} + +func TestMatchGoImport(t *testing.T) { + tests := []struct { + imports []metaImport + path string + mi metaImport + err error + }{ + { + imports: []metaImport{ + {Prefix: "example.com/user/foo", VCS: "git", RepoRoot: "https://example.com/repo/target"}, + }, + path: "example.com/user/foo", + mi: metaImport{Prefix: "example.com/user/foo", VCS: "git", RepoRoot: "https://example.com/repo/target"}, + }, + { + imports: []metaImport{ + {Prefix: "example.com/user/foo", VCS: "git", RepoRoot: "https://example.com/repo/target"}, + }, + path: "example.com/user/foo/", + mi: metaImport{Prefix: "example.com/user/foo", VCS: "git", RepoRoot: "https://example.com/repo/target"}, + }, + { + imports: []metaImport{ + {Prefix: "example.com/user/foo", VCS: "git", RepoRoot: "https://example.com/repo/target"}, + {Prefix: "example.com/user/fooa", VCS: "git", RepoRoot: "https://example.com/repo/target"}, + }, + path: "example.com/user/foo", + mi: metaImport{Prefix: "example.com/user/foo", VCS: "git", RepoRoot: "https://example.com/repo/target"}, + }, + { + imports: []metaImport{ + {Prefix: "example.com/user/foo", VCS: "git", RepoRoot: "https://example.com/repo/target"}, + {Prefix: "example.com/user/fooa", VCS: "git", RepoRoot: "https://example.com/repo/target"}, + }, + path: "example.com/user/fooa", + mi: metaImport{Prefix: "example.com/user/fooa", VCS: "git", RepoRoot: "https://example.com/repo/target"}, + }, + { + imports: []metaImport{ + {Prefix: "example.com/user/foo", VCS: "git", RepoRoot: "https://example.com/repo/target"}, + {Prefix: "example.com/user/foo/bar", VCS: "git", RepoRoot: "https://example.com/repo/target"}, + }, + path: "example.com/user/foo/bar", + err: errors.New("should not be allowed to create nested repo"), + }, + { + imports: []metaImport{ + {Prefix: "example.com/user/foo", VCS: "git", RepoRoot: "https://example.com/repo/target"}, + {Prefix: "example.com/user/foo/bar", VCS: "git", RepoRoot: "https://example.com/repo/target"}, + }, + path: "example.com/user/foo/bar/baz", + err: errors.New("should not be allowed to create nested repo"), + }, + { + imports: []metaImport{ + {Prefix: "example.com/user/foo", VCS: "git", RepoRoot: "https://example.com/repo/target"}, + {Prefix: "example.com/user/foo/bar", VCS: "git", RepoRoot: "https://example.com/repo/target"}, + }, + path: "example.com/user/foo/bar/baz/qux", + err: errors.New("should not be allowed to create nested repo"), + }, + { + imports: []metaImport{ + {Prefix: "example.com/user/foo", VCS: "git", RepoRoot: "https://example.com/repo/target"}, + {Prefix: "example.com/user/foo/bar", VCS: "git", RepoRoot: "https://example.com/repo/target"}, + }, + path: "example.com/user/foo/bar/baz/", + err: errors.New("should not be allowed to create nested repo"), + }, + { + imports: []metaImport{ + {Prefix: "example.com/user/foo", VCS: "git", RepoRoot: "https://example.com/repo/target"}, + {Prefix: "example.com/user/foo/bar", VCS: "git", RepoRoot: "https://example.com/repo/target"}, + }, + path: "example.com", + err: errors.New("pathologically short path"), + }, + { + imports: []metaImport{ + {Prefix: "example.com/user/foo", VCS: "git", RepoRoot: "https://example.com/repo/target"}, + }, + path: "different.example.com/user/foo", + err: errors.New("meta tags do not match import path"), + }, + } + + for _, test := range tests { + mi, err := matchGoImport(test.imports, test.path) + if mi != test.mi { + t.Errorf("unexpected metaImport; got %v, want %v", mi, test.mi) + } + + got := err + want := test.err + if (got == nil) != (want == nil) { + t.Errorf("unexpected error; got %v, want %v", got, want) + } + } +} diff --git a/libgo/go/cmd/go/internal/help/help.go b/libgo/go/cmd/go/internal/help/help.go new file mode 100644 index 00000000000..b4c5217f836 --- /dev/null +++ b/libgo/go/cmd/go/internal/help/help.go @@ -0,0 +1,178 @@ +// Copyright 2017 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package help implements the ``go help'' command. +package help + +import ( + "bufio" + "bytes" + "fmt" + "io" + "os" + "strings" + "text/template" + "unicode" + "unicode/utf8" + + "cmd/go/internal/base" +) + +// Help implements the 'help' command. +func Help(args []string) { + if len(args) == 0 { + PrintUsage(os.Stdout) + // not exit 2: succeeded at 'go help'. + return + } + if len(args) != 1 { + fmt.Fprintf(os.Stderr, "usage: go help command\n\nToo many arguments given.\n") + os.Exit(2) // failed at 'go help' + } + + arg := args[0] + + // 'go help documentation' generates doc.go. + if arg == "documentation" { + fmt.Println("// Copyright 2011 The Go Authors. All rights reserved.") + fmt.Println("// Use of this source code is governed by a BSD-style") + fmt.Println("// license that can be found in the LICENSE file.") + fmt.Println() + fmt.Println("// DO NOT EDIT THIS FILE. GENERATED BY mkalldocs.sh.") + fmt.Println("// Edit the documentation in other files and rerun mkalldocs.sh to generate this one.") + fmt.Println() + buf := new(bytes.Buffer) + PrintUsage(buf) + usage := &base.Command{Long: buf.String()} + tmpl(&commentWriter{W: os.Stdout}, documentationTemplate, append([]*base.Command{usage}, base.Commands...)) + fmt.Println("package main") + return + } + + for _, cmd := range base.Commands { + if cmd.Name() == arg { + tmpl(os.Stdout, helpTemplate, cmd) + // not exit 2: succeeded at 'go help cmd'. + return + } + } + + fmt.Fprintf(os.Stderr, "Unknown help topic %#q. Run 'go help'.\n", arg) + os.Exit(2) // failed at 'go help cmd' +} + +var usageTemplate = `Go is a tool for managing Go source code. + +Usage: + + go command [arguments] + +The commands are: +{{range .}}{{if .Runnable}} + {{.Name | printf "%-11s"}} {{.Short}}{{end}}{{end}} + +Use "go help [command]" for more information about a command. + +Additional help topics: +{{range .}}{{if not .Runnable}} + {{.Name | printf "%-11s"}} {{.Short}}{{end}}{{end}} + +Use "go help [topic]" for more information about that topic. + +` + +var helpTemplate = `{{if .Runnable}}usage: go {{.UsageLine}} + +{{end}}{{.Long | trim}} +` + +var documentationTemplate = `{{range .}}{{if .Short}}{{.Short | capitalize}} + +{{end}}{{if .Runnable}}Usage: + + go {{.UsageLine}} + +{{end}}{{.Long | trim}} + + +{{end}}` + +// commentWriter writes a Go comment to the underlying io.Writer, +// using line comment form (//). +type commentWriter struct { + W io.Writer + wroteSlashes bool // Wrote "//" at the beginning of the current line. +} + +func (c *commentWriter) Write(p []byte) (int, error) { + var n int + for i, b := range p { + if !c.wroteSlashes { + s := "//" + if b != '\n' { + s = "// " + } + if _, err := io.WriteString(c.W, s); err != nil { + return n, err + } + c.wroteSlashes = true + } + n0, err := c.W.Write(p[i : i+1]) + n += n0 + if err != nil { + return n, err + } + if b == '\n' { + c.wroteSlashes = false + } + } + return len(p), nil +} + +// An errWriter wraps a writer, recording whether a write error occurred. +type errWriter struct { + w io.Writer + err error +} + +func (w *errWriter) Write(b []byte) (int, error) { + n, err := w.w.Write(b) + if err != nil { + w.err = err + } + return n, err +} + +// tmpl executes the given template text on data, writing the result to w. +func tmpl(w io.Writer, text string, data interface{}) { + t := template.New("top") + t.Funcs(template.FuncMap{"trim": strings.TrimSpace, "capitalize": capitalize}) + template.Must(t.Parse(text)) + ew := &errWriter{w: w} + err := t.Execute(ew, data) + if ew.err != nil { + // I/O error writing. Ignore write on closed pipe. + if strings.Contains(ew.err.Error(), "pipe") { + os.Exit(1) + } + base.Fatalf("writing output: %v", ew.err) + } + if err != nil { + panic(err) + } +} + +func capitalize(s string) string { + if s == "" { + return s + } + r, n := utf8.DecodeRuneInString(s) + return string(unicode.ToTitle(r)) + s[n:] +} + +func PrintUsage(w io.Writer) { + bw := bufio.NewWriter(w) + tmpl(bw, usageTemplate, base.Commands) + bw.Flush() +} diff --git a/libgo/go/cmd/go/internal/help/helpdoc.go b/libgo/go/cmd/go/internal/help/helpdoc.go new file mode 100644 index 00000000000..516fff3528e --- /dev/null +++ b/libgo/go/cmd/go/internal/help/helpdoc.go @@ -0,0 +1,623 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package help + +import "cmd/go/internal/base" + +var HelpC = &base.Command{ + UsageLine: "c", + Short: "calling between Go and C", + Long: ` +There are two different ways to call between Go and C/C++ code. + +The first is the cgo tool, which is part of the Go distribution. For +information on how to use it see the cgo documentation (go doc cmd/cgo). + +The second is the SWIG program, which is a general tool for +interfacing between languages. For information on SWIG see +http://swig.org/. When running go build, any file with a .swig +extension will be passed to SWIG. Any file with a .swigcxx extension +will be passed to SWIG with the -c++ option. + +When either cgo or SWIG is used, go build will pass any .c, .m, .s, +or .S files to the C compiler, and any .cc, .cpp, .cxx files to the C++ +compiler. The CC or CXX environment variables may be set to determine +the C or C++ compiler, respectively, to use. + `, +} + +var HelpPackages = &base.Command{ + UsageLine: "packages", + Short: "description of package lists", + Long: ` +Many commands apply to a set of packages: + + go action [packages] + +Usually, [packages] is a list of import paths. + +An import path that is a rooted path or that begins with +a . or .. element is interpreted as a file system path and +denotes the package in that directory. + +Otherwise, the import path P denotes the package found in +the directory DIR/src/P for some DIR listed in the GOPATH +environment variable (For more details see: 'go help gopath'). + +If no import paths are given, the action applies to the +package in the current directory. + +There are four reserved names for paths that should not be used +for packages to be built with the go tool: + +- "main" denotes the top-level package in a stand-alone executable. + +- "all" expands to all package directories found in all the GOPATH +trees. For example, 'go list all' lists all the packages on the local +system. + +- "std" is like all but expands to just the packages in the standard +Go library. + +- "cmd" expands to the Go repository's commands and their +internal libraries. + +Import paths beginning with "cmd/" only match source code in +the Go repository. + +An import path is a pattern if it includes one or more "..." wildcards, +each of which can match any string, including the empty string and +strings containing slashes. Such a pattern expands to all package +directories found in the GOPATH trees with names matching the +patterns. + +To make common patterns more convenient, there are two special cases. +First, /... at the end of the pattern can match an empty string, +so that net/... matches both net and packages in its subdirectories, like net/http. +Second, any slash-separated pattern element containing a wildcard never +participates in a match of the "vendor" element in the path of a vendored +package, so that ./... does not match packages in subdirectories of +./vendor or ./mycode/vendor, but ./vendor/... and ./mycode/vendor/... do. +Note, however, that a directory named vendor that itself contains code +is not a vendored package: cmd/vendor would be a command named vendor, +and the pattern cmd/... matches it. +See golang.org/s/go15vendor for more about vendoring. + +An import path can also name a package to be downloaded from +a remote repository. Run 'go help importpath' for details. + +Every package in a program must have a unique import path. +By convention, this is arranged by starting each path with a +unique prefix that belongs to you. For example, paths used +internally at Google all begin with 'google', and paths +denoting remote repositories begin with the path to the code, +such as 'github.com/user/repo'. + +Packages in a program need not have unique package names, +but there are two reserved package names with special meaning. +The name main indicates a command, not a library. +Commands are built into binaries and cannot be imported. +The name documentation indicates documentation for +a non-Go program in the directory. Files in package documentation +are ignored by the go command. + +As a special case, if the package list is a list of .go files from a +single directory, the command is applied to a single synthesized +package made up of exactly those files, ignoring any build constraints +in those files and ignoring any other files in the directory. + +Directory and file names that begin with "." or "_" are ignored +by the go tool, as are directories named "testdata". + `, +} + +var HelpImportPath = &base.Command{ + UsageLine: "importpath", + Short: "import path syntax", + Long: ` + +An import path (see 'go help packages') denotes a package stored in the local +file system. In general, an import path denotes either a standard package (such +as "unicode/utf8") or a package found in one of the work spaces (For more +details see: 'go help gopath'). + +Relative import paths + +An import path beginning with ./ or ../ is called a relative path. +The toolchain supports relative import paths as a shortcut in two ways. + +First, a relative path can be used as a shorthand on the command line. +If you are working in the directory containing the code imported as +"unicode" and want to run the tests for "unicode/utf8", you can type +"go test ./utf8" instead of needing to specify the full path. +Similarly, in the reverse situation, "go test .." will test "unicode" from +the "unicode/utf8" directory. Relative patterns are also allowed, like +"go test ./..." to test all subdirectories. See 'go help packages' for details +on the pattern syntax. + +Second, if you are compiling a Go program not in a work space, +you can use a relative path in an import statement in that program +to refer to nearby code also not in a work space. +This makes it easy to experiment with small multipackage programs +outside of the usual work spaces, but such programs cannot be +installed with "go install" (there is no work space in which to install them), +so they are rebuilt from scratch each time they are built. +To avoid ambiguity, Go programs cannot use relative import paths +within a work space. + +Remote import paths + +Certain import paths also +describe how to obtain the source code for the package using +a revision control system. + +A few common code hosting sites have special syntax: + + Bitbucket (Git, Mercurial) + + import "bitbucket.org/user/project" + import "bitbucket.org/user/project/sub/directory" + + GitHub (Git) + + import "github.com/user/project" + import "github.com/user/project/sub/directory" + + Launchpad (Bazaar) + + import "launchpad.net/project" + import "launchpad.net/project/series" + import "launchpad.net/project/series/sub/directory" + + import "launchpad.net/~user/project/branch" + import "launchpad.net/~user/project/branch/sub/directory" + + IBM DevOps Services (Git) + + import "hub.jazz.net/git/user/project" + import "hub.jazz.net/git/user/project/sub/directory" + +For code hosted on other servers, import paths may either be qualified +with the version control type, or the go tool can dynamically fetch +the import path over https/http and discover where the code resides +from a tag in the HTML. + +To declare the code location, an import path of the form + + repository.vcs/path + +specifies the given repository, with or without the .vcs suffix, +using the named version control system, and then the path inside +that repository. The supported version control systems are: + + Bazaar .bzr + Git .git + Mercurial .hg + Subversion .svn + +For example, + + import "example.org/user/foo.hg" + +denotes the root directory of the Mercurial repository at +example.org/user/foo or foo.hg, and + + import "example.org/repo.git/foo/bar" + +denotes the foo/bar directory of the Git repository at +example.org/repo or repo.git. + +When a version control system supports multiple protocols, +each is tried in turn when downloading. For example, a Git +download tries https://, then git+ssh://. + +By default, downloads are restricted to known secure protocols +(e.g. https, ssh). To override this setting for Git downloads, the +GIT_ALLOW_PROTOCOL environment variable can be set (For more details see: +'go help environment'). + +If the import path is not a known code hosting site and also lacks a +version control qualifier, the go tool attempts to fetch the import +over https/http and looks for a tag in the document's HTML +. + +The meta tag has the form: + + + +The import-prefix is the import path corresponding to the repository +root. It must be a prefix or an exact match of the package being +fetched with "go get". If it's not an exact match, another http +request is made at the prefix to verify the tags match. + +The meta tag should appear as early in the file as possible. +In particular, it should appear before any raw JavaScript or CSS, +to avoid confusing the go command's restricted parser. + +The vcs is one of "git", "hg", "svn", etc, + +The repo-root is the root of the version control system +containing a scheme and not containing a .vcs qualifier. + +For example, + + import "example.org/pkg/foo" + +will result in the following requests: + + https://example.org/pkg/foo?go-get=1 (preferred) + http://example.org/pkg/foo?go-get=1 (fallback, only with -insecure) + +If that page contains the meta tag + + + +the go tool will verify that https://example.org/?go-get=1 contains the +same meta tag and then git clone https://code.org/r/p/exproj into +GOPATH/src/example.org. + +New downloaded packages are written to the first directory listed in the GOPATH +environment variable (For more details see: 'go help gopath'). + +The go command attempts to download the version of the +package appropriate for the Go release being used. +Run 'go help get' for more. + +Import path checking + +When the custom import path feature described above redirects to a +known code hosting site, each of the resulting packages has two possible +import paths, using the custom domain or the known hosting site. + +A package statement is said to have an "import comment" if it is immediately +followed (before the next newline) by a comment of one of these two forms: + + package math // import "path" + package math /* import "path" */ + +The go command will refuse to install a package with an import comment +unless it is being referred to by that import path. In this way, import comments +let package authors make sure the custom import path is used and not a +direct path to the underlying code hosting site. + +Import path checking is disabled for code found within vendor trees. +This makes it possible to copy code into alternate locations in vendor trees +without needing to update import comments. + +See https://golang.org/s/go14customimport for details. + `, +} + +var HelpGopath = &base.Command{ + UsageLine: "gopath", + Short: "GOPATH environment variable", + Long: ` +The Go path is used to resolve import statements. +It is implemented by and documented in the go/build package. + +The GOPATH environment variable lists places to look for Go code. +On Unix, the value is a colon-separated string. +On Windows, the value is a semicolon-separated string. +On Plan 9, the value is a list. + +If the environment variable is unset, GOPATH defaults +to a subdirectory named "go" in the user's home directory +($HOME/go on Unix, %USERPROFILE%\go on Windows), +unless that directory holds a Go distribution. +Run "go env GOPATH" to see the current GOPATH. + +See https://golang.org/wiki/SettingGOPATH to set a custom GOPATH. + +Each directory listed in GOPATH must have a prescribed structure: + +The src directory holds source code. The path below src +determines the import path or executable name. + +The pkg directory holds installed package objects. +As in the Go tree, each target operating system and +architecture pair has its own subdirectory of pkg +(pkg/GOOS_GOARCH). + +If DIR is a directory listed in the GOPATH, a package with +source in DIR/src/foo/bar can be imported as "foo/bar" and +has its compiled form installed to "DIR/pkg/GOOS_GOARCH/foo/bar.a". + +The bin directory holds compiled commands. +Each command is named for its source directory, but only +the final element, not the entire path. That is, the +command with source in DIR/src/foo/quux is installed into +DIR/bin/quux, not DIR/bin/foo/quux. The "foo/" prefix is stripped +so that you can add DIR/bin to your PATH to get at the +installed commands. If the GOBIN environment variable is +set, commands are installed to the directory it names instead +of DIR/bin. GOBIN must be an absolute path. + +Here's an example directory layout: + + GOPATH=/home/user/go + + /home/user/go/ + src/ + foo/ + bar/ (go code in package bar) + x.go + quux/ (go code in package main) + y.go + bin/ + quux (installed command) + pkg/ + linux_amd64/ + foo/ + bar.a (installed package object) + +Go searches each directory listed in GOPATH to find source code, +but new packages are always downloaded into the first directory +in the list. + +See https://golang.org/doc/code.html for an example. + +Internal Directories + +Code in or below a directory named "internal" is importable only +by code in the directory tree rooted at the parent of "internal". +Here's an extended version of the directory layout above: + + /home/user/go/ + src/ + crash/ + bang/ (go code in package bang) + b.go + foo/ (go code in package foo) + f.go + bar/ (go code in package bar) + x.go + internal/ + baz/ (go code in package baz) + z.go + quux/ (go code in package main) + y.go + + +The code in z.go is imported as "foo/internal/baz", but that +import statement can only appear in source files in the subtree +rooted at foo. The source files foo/f.go, foo/bar/x.go, and +foo/quux/y.go can all import "foo/internal/baz", but the source file +crash/bang/b.go cannot. + +See https://golang.org/s/go14internal for details. + +Vendor Directories + +Go 1.6 includes support for using local copies of external dependencies +to satisfy imports of those dependencies, often referred to as vendoring. + +Code below a directory named "vendor" is importable only +by code in the directory tree rooted at the parent of "vendor", +and only using an import path that omits the prefix up to and +including the vendor element. + +Here's the example from the previous section, +but with the "internal" directory renamed to "vendor" +and a new foo/vendor/crash/bang directory added: + + /home/user/go/ + src/ + crash/ + bang/ (go code in package bang) + b.go + foo/ (go code in package foo) + f.go + bar/ (go code in package bar) + x.go + vendor/ + crash/ + bang/ (go code in package bang) + b.go + baz/ (go code in package baz) + z.go + quux/ (go code in package main) + y.go + +The same visibility rules apply as for internal, but the code +in z.go is imported as "baz", not as "foo/vendor/baz". + +Code in vendor directories deeper in the source tree shadows +code in higher directories. Within the subtree rooted at foo, an import +of "crash/bang" resolves to "foo/vendor/crash/bang", not the +top-level "crash/bang". + +Code in vendor directories is not subject to import path +checking (see 'go help importpath'). + +When 'go get' checks out or updates a git repository, it now also +updates submodules. + +Vendor directories do not affect the placement of new repositories +being checked out for the first time by 'go get': those are always +placed in the main GOPATH, never in a vendor subtree. + +See https://golang.org/s/go15vendor for details. + `, +} + +var HelpEnvironment = &base.Command{ + UsageLine: "environment", + Short: "environment variables", + Long: ` + +The go command, and the tools it invokes, examine a few different +environment variables. For many of these, you can see the default +value of on your system by running 'go env NAME', where NAME is the +name of the variable. + +General-purpose environment variables: + + GCCGO + The gccgo command to run for 'go build -compiler=gccgo'. + GOARCH + The architecture, or processor, for which to compile code. + Examples are amd64, 386, arm, ppc64. + GOBIN + The directory where 'go install' will install a command. + GOOS + The operating system for which to compile code. + Examples are linux, darwin, windows, netbsd. + GOPATH + For more details see: 'go help gopath'. + GORACE + Options for the race detector. + See https://golang.org/doc/articles/race_detector.html. + GOROOT + The root of the go tree. + +Environment variables for use with cgo: + + CC + The command to use to compile C code. + CGO_ENABLED + Whether the cgo command is supported. Either 0 or 1. + CGO_CFLAGS + Flags that cgo will pass to the compiler when compiling + C code. + CGO_CPPFLAGS + Flags that cgo will pass to the compiler when compiling + C or C++ code. + CGO_CXXFLAGS + Flags that cgo will pass to the compiler when compiling + C++ code. + CGO_FFLAGS + Flags that cgo will pass to the compiler when compiling + Fortran code. + CGO_LDFLAGS + Flags that cgo will pass to the compiler when linking. + CXX + The command to use to compile C++ code. + PKG_CONFIG + Path to pkg-config tool. + +Architecture-specific environment variables: + + GOARM + For GOARCH=arm, the ARM architecture for which to compile. + Valid values are 5, 6, 7. + GO386 + For GOARCH=386, the floating point instruction set. + Valid values are 387, sse2. + +Special-purpose environment variables: + + GOROOT_FINAL + The root of the installed Go tree, when it is + installed in a location other than where it is built. + File names in stack traces are rewritten from GOROOT to + GOROOT_FINAL. + GO_EXTLINK_ENABLED + Whether the linker should use external linking mode + when using -linkmode=auto with code that uses cgo. + Set to 0 to disable external linking mode, 1 to enable it. + GIT_ALLOW_PROTOCOL + Defined by Git. A colon-separated list of schemes that are allowed to be used + with git fetch/clone. If set, any scheme not explicitly mentioned will be + considered insecure by 'go get'. + GCCGOTOOLDIR + If set, where to find gccgo tools, such as cgo. + The default is based on how gccgo was configured. + `, +} + +var HelpFileType = &base.Command{ + UsageLine: "filetype", + Short: "file types", + Long: ` +The go command examines the contents of a restricted set of files +in each directory. It identifies which files to examine based on +the extension of the file name. These extensions are: + + .go + Go source files. + .c, .h + C source files. + If the package uses cgo or SWIG, these will be compiled with the + OS-native compiler (typically gcc); otherwise they will + trigger an error. + .cc, .cpp, .cxx, .hh, .hpp, .hxx + C++ source files. Only useful with cgo or SWIG, and always + compiled with the OS-native compiler. + .m + Objective-C source files. Only useful with cgo, and always + compiled with the OS-native compiler. + .s, .S + Assembler source files. + If the package uses cgo or SWIG, these will be assembled with the + OS-native assembler (typically gcc (sic)); otherwise they + will be assembled with the Go assembler. + .swig, .swigcxx + SWIG definition files. + .syso + System object files. + +Files of each of these types except .syso may contain build +constraints, but the go command stops scanning for build constraints +at the first item in the file that is not a blank line or //-style +line comment. See the go/build package documentation for +more details. + +Non-test Go source files can also include a //go:binary-only-package +comment, indicating that the package sources are included +for documentation only and must not be used to build the +package binary. This enables distribution of Go packages in +their compiled form alone. See the go/build package documentation +for more details. + `, +} + +var HelpBuildmode = &base.Command{ + UsageLine: "buildmode", + Short: "description of build modes", + Long: ` +The 'go build' and 'go install' commands take a -buildmode argument which +indicates which kind of object file is to be built. Currently supported values +are: + + -buildmode=archive + Build the listed non-main packages into .a files. Packages named + main are ignored. + + -buildmode=c-archive + Build the listed main package, plus all packages it imports, + into a C archive file. The only callable symbols will be those + functions exported using a cgo //export comment. Requires + exactly one main package to be listed. + + -buildmode=c-shared + Build the listed main package, plus all packages it imports, + into a C shared library. The only callable symbols will + be those functions exported using a cgo //export comment. + Requires exactly one main package to be listed. + + -buildmode=default + Listed main packages are built into executables and listed + non-main packages are built into .a files (the default + behavior). + + -buildmode=shared + Combine all the listed non-main packages into a single shared + library that will be used when building with the -linkshared + option. Packages named main are ignored. + + -buildmode=exe + Build the listed main packages and everything they import into + executables. Packages not named main are ignored. + + -buildmode=pie + Build the listed main packages and everything they import into + position independent executables (PIE). Packages not named + main are ignored. + + -buildmode=plugin + Build the listed main packages, plus all packages that they + import, into a Go plugin. Packages not named main are ignored. +`, +} diff --git a/libgo/go/cmd/go/internal/list/context.go b/libgo/go/cmd/go/internal/list/context.go new file mode 100644 index 00000000000..68d691ebe2e --- /dev/null +++ b/libgo/go/cmd/go/internal/list/context.go @@ -0,0 +1,37 @@ +// Copyright 2014 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package list + +import ( + "go/build" +) + +type Context struct { + GOARCH string `json:",omitempty"` // target architecture + GOOS string `json:",omitempty"` // target operating system + GOROOT string `json:",omitempty"` // Go root + GOPATH string `json:",omitempty"` // Go path + CgoEnabled bool `json:",omitempty"` // whether cgo can be used + UseAllFiles bool `json:",omitempty"` // use files regardless of +build lines, file names + Compiler string `json:",omitempty"` // compiler to assume when computing target paths + BuildTags []string `json:",omitempty"` // build constraints to match in +build lines + ReleaseTags []string `json:",omitempty"` // releases the current release is compatible with + InstallSuffix string `json:",omitempty"` // suffix to use in the name of the install dir +} + +func newContext(c *build.Context) *Context { + return &Context{ + GOARCH: c.GOARCH, + GOOS: c.GOOS, + GOROOT: c.GOROOT, + GOPATH: c.GOPATH, + CgoEnabled: c.CgoEnabled, + UseAllFiles: c.UseAllFiles, + Compiler: c.Compiler, + BuildTags: c.BuildTags, + ReleaseTags: c.ReleaseTags, + InstallSuffix: c.InstallSuffix, + } +} diff --git a/libgo/go/cmd/go/internal/list/list.go b/libgo/go/cmd/go/internal/list/list.go new file mode 100644 index 00000000000..241d0894c01 --- /dev/null +++ b/libgo/go/cmd/go/internal/list/list.go @@ -0,0 +1,240 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package list implements the ``go list'' command. +package list + +import ( + "bufio" + "encoding/json" + "io" + "os" + "strings" + "text/template" + + "cmd/go/internal/base" + "cmd/go/internal/cfg" + "cmd/go/internal/load" + "cmd/go/internal/work" +) + +var CmdList = &base.Command{ + UsageLine: "list [-e] [-f format] [-json] [build flags] [packages]", + Short: "list packages", + Long: ` +List lists the packages named by the import paths, one per line. + +The default output shows the package import path: + + bytes + encoding/json + github.com/gorilla/mux + golang.org/x/net/html + +The -f flag specifies an alternate format for the list, using the +syntax of package template. The default output is equivalent to -f +'{{.ImportPath}}'. The struct being passed to the template is: + + type Package struct { + Dir string // directory containing package sources + ImportPath string // import path of package in dir + ImportComment string // path in import comment on package statement + Name string // package name + Doc string // package documentation string + Target string // install path + Shlib string // the shared library that contains this package (only set when -linkshared) + Goroot bool // is this package in the Go root? + Standard bool // is this package part of the standard Go library? + Stale bool // would 'go install' do anything for this package? + StaleReason string // explanation for Stale==true + Root string // Go root or Go path dir containing this package + ConflictDir string // this directory shadows Dir in $GOPATH + BinaryOnly bool // binary-only package: cannot be recompiled from sources + + // Source files + GoFiles []string // .go source files (excluding CgoFiles, TestGoFiles, XTestGoFiles) + CgoFiles []string // .go sources files that import "C" + IgnoredGoFiles []string // .go sources ignored due to build constraints + CFiles []string // .c source files + CXXFiles []string // .cc, .cxx and .cpp source files + MFiles []string // .m source files + HFiles []string // .h, .hh, .hpp and .hxx source files + FFiles []string // .f, .F, .for and .f90 Fortran source files + SFiles []string // .s source files + SwigFiles []string // .swig files + SwigCXXFiles []string // .swigcxx files + SysoFiles []string // .syso object files to add to archive + TestGoFiles []string // _test.go files in package + XTestGoFiles []string // _test.go files outside package + + // Cgo directives + CgoCFLAGS []string // cgo: flags for C compiler + CgoCPPFLAGS []string // cgo: flags for C preprocessor + CgoCXXFLAGS []string // cgo: flags for C++ compiler + CgoFFLAGS []string // cgo: flags for Fortran compiler + CgoLDFLAGS []string // cgo: flags for linker + CgoPkgConfig []string // cgo: pkg-config names + + // Dependency information + Imports []string // import paths used by this package + Deps []string // all (recursively) imported dependencies + TestImports []string // imports from TestGoFiles + XTestImports []string // imports from XTestGoFiles + + // Error information + Incomplete bool // this package or a dependency has an error + Error *PackageError // error loading package + DepsErrors []*PackageError // errors loading dependencies + } + +Packages stored in vendor directories report an ImportPath that includes the +path to the vendor directory (for example, "d/vendor/p" instead of "p"), +so that the ImportPath uniquely identifies a given copy of a package. +The Imports, Deps, TestImports, and XTestImports lists also contain these +expanded imports paths. See golang.org/s/go15vendor for more about vendoring. + +The error information, if any, is + + type PackageError struct { + ImportStack []string // shortest path from package named on command line to this one + Pos string // position of error (if present, file:line:col) + Err string // the error itself + } + +The template function "join" calls strings.Join. + +The template function "context" returns the build context, defined as: + + type Context struct { + GOARCH string // target architecture + GOOS string // target operating system + GOROOT string // Go root + GOPATH string // Go path + CgoEnabled bool // whether cgo can be used + UseAllFiles bool // use files regardless of +build lines, file names + Compiler string // compiler to assume when computing target paths + BuildTags []string // build constraints to match in +build lines + ReleaseTags []string // releases the current release is compatible with + InstallSuffix string // suffix to use in the name of the install dir + } + +For more information about the meaning of these fields see the documentation +for the go/build package's Context type. + +The -json flag causes the package data to be printed in JSON format +instead of using the template format. + +The -e flag changes the handling of erroneous packages, those that +cannot be found or are malformed. By default, the list command +prints an error to standard error for each erroneous package and +omits the packages from consideration during the usual printing. +With the -e flag, the list command never prints errors to standard +error and instead processes the erroneous packages with the usual +printing. Erroneous packages will have a non-empty ImportPath and +a non-nil Error field; other information may or may not be missing +(zeroed). + +For more about build flags, see 'go help build'. + +For more about specifying packages, see 'go help packages'. + `, +} + +func init() { + CmdList.Run = runList // break init cycle + work.AddBuildFlags(CmdList) +} + +var listE = CmdList.Flag.Bool("e", false, "") +var listFmt = CmdList.Flag.String("f", "{{.ImportPath}}", "") +var listJson = CmdList.Flag.Bool("json", false, "") +var nl = []byte{'\n'} + +func runList(cmd *base.Command, args []string) { + work.BuildModeInit() + out := newTrackingWriter(os.Stdout) + defer out.w.Flush() + + var do func(*load.PackagePublic) + if *listJson { + do = func(p *load.PackagePublic) { + b, err := json.MarshalIndent(p, "", "\t") + if err != nil { + out.Flush() + base.Fatalf("%s", err) + } + out.Write(b) + out.Write(nl) + } + } else { + var cachedCtxt *Context + context := func() *Context { + if cachedCtxt == nil { + cachedCtxt = newContext(&cfg.BuildContext) + } + return cachedCtxt + } + fm := template.FuncMap{ + "join": strings.Join, + "context": context, + } + tmpl, err := template.New("main").Funcs(fm).Parse(*listFmt) + if err != nil { + base.Fatalf("%s", err) + } + do = func(p *load.PackagePublic) { + if err := tmpl.Execute(out, p); err != nil { + out.Flush() + base.Fatalf("%s", err) + } + if out.NeedNL() { + out.Write(nl) + } + } + } + + loadpkgs := load.Packages + if *listE { + loadpkgs = load.PackagesAndErrors + } + + for _, pkg := range loadpkgs(args) { + // Show vendor-expanded paths in listing + pkg.TestImports = pkg.Vendored(pkg.TestImports) + pkg.XTestImports = pkg.Vendored(pkg.XTestImports) + + do(&pkg.PackagePublic) + } +} + +// TrackingWriter tracks the last byte written on every write so +// we can avoid printing a newline if one was already written or +// if there is no output at all. +type TrackingWriter struct { + w *bufio.Writer + last byte +} + +func newTrackingWriter(w io.Writer) *TrackingWriter { + return &TrackingWriter{ + w: bufio.NewWriter(w), + last: '\n', + } +} + +func (t *TrackingWriter) Write(p []byte) (n int, err error) { + n, err = t.w.Write(p) + if n > 0 { + t.last = p[n-1] + } + return +} + +func (t *TrackingWriter) Flush() { + t.w.Flush() +} + +func (t *TrackingWriter) NeedNL() bool { + return t.last != '\n' +} diff --git a/libgo/go/cmd/go/internal/load/match_test.go b/libgo/go/cmd/go/internal/load/match_test.go new file mode 100644 index 00000000000..b8d67dac742 --- /dev/null +++ b/libgo/go/cmd/go/internal/load/match_test.go @@ -0,0 +1,165 @@ +// Copyright 2012 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package load + +import ( + "strings" + "testing" +) + +var matchPatternTests = ` + pattern ... + match foo + + pattern net + match net + not net/http + + pattern net/http + match net/http + not net + + pattern net... + match net net/http netchan + not not/http not/net/http + + # Special cases. Quoting docs: + + # First, /... at the end of the pattern can match an empty string, + # so that net/... matches both net and packages in its subdirectories, like net/http. + pattern net/... + match net net/http + not not/http not/net/http netchan + + # Second, any slash-separted pattern element containing a wildcard never + # participates in a match of the "vendor" element in the path of a vendored + # package, so that ./... does not match packages in subdirectories of + # ./vendor or ./mycode/vendor, but ./vendor/... and ./mycode/vendor/... do. + # Note, however, that a directory named vendor that itself contains code + # is not a vendored package: cmd/vendor would be a command named vendor, + # and the pattern cmd/... matches it. + pattern ./... + match ./vendor ./mycode/vendor + not ./vendor/foo ./mycode/vendor/foo + + pattern ./vendor/... + match ./vendor/foo ./vendor/foo/vendor + not ./vendor/foo/vendor/bar + + pattern mycode/vendor/... + match mycode/vendor mycode/vendor/foo mycode/vendor/foo/vendor + not mycode/vendor/foo/vendor/bar + + pattern x/vendor/y + match x/vendor/y + not x/vendor + + pattern x/vendor/y/... + match x/vendor/y x/vendor/y/z x/vendor/y/vendor x/vendor/y/z/vendor + not x/vendor/y/vendor/z + + pattern .../vendor/... + match x/vendor/y x/vendor/y/z x/vendor/y/vendor x/vendor/y/z/vendor +` + +func TestMatchPattern(t *testing.T) { + testPatterns(t, "matchPattern", matchPatternTests, func(pattern, name string) bool { + return matchPattern(pattern)(name) + }) +} + +var treeCanMatchPatternTests = ` + pattern ... + match foo + + pattern net + match net + not net/http + + pattern net/http + match net net/http + + pattern net... + match net netchan net/http + not not/http not/net/http + + pattern net/... + match net net/http + not not/http netchan + + pattern abc.../def + match abcxyz + not xyzabc + + pattern x/y/z/... + match x x/y x/y/z x/y/z/w + + pattern x/y/z + match x x/y x/y/z + not x/y/z/w + + pattern x/.../y/z + match x/a/b/c + not y/x/a/b/c +` + +func TestTreeCanMatchPattern(t *testing.T) { + testPatterns(t, "treeCanMatchPattern", treeCanMatchPatternTests, func(pattern, name string) bool { + return treeCanMatchPattern(pattern)(name) + }) +} + +var hasPathPrefixTests = []stringPairTest{ + {"abc", "a", false}, + {"a/bc", "a", true}, + {"a", "a", true}, + {"a/bc", "a/", true}, +} + +func TestHasPathPrefix(t *testing.T) { + testStringPairs(t, "hasPathPrefix", hasPathPrefixTests, hasPathPrefix) +} + +type stringPairTest struct { + in1 string + in2 string + out bool +} + +func testStringPairs(t *testing.T, name string, tests []stringPairTest, f func(string, string) bool) { + for _, tt := range tests { + if out := f(tt.in1, tt.in2); out != tt.out { + t.Errorf("%s(%q, %q) = %v, want %v", name, tt.in1, tt.in2, out, tt.out) + } + } +} + +func testPatterns(t *testing.T, name, tests string, fn func(string, string) bool) { + var patterns []string + for _, line := range strings.Split(tests, "\n") { + if i := strings.Index(line, "#"); i >= 0 { + line = line[:i] + } + f := strings.Fields(line) + if len(f) == 0 { + continue + } + switch f[0] { + default: + t.Fatalf("unknown directive %q", f[0]) + case "pattern": + patterns = f[1:] + case "match", "not": + want := f[0] == "match" + for _, pattern := range patterns { + for _, in := range f[1:] { + if fn(pattern, in) != want { + t.Errorf("%s(%q, %q) = %v, want %v", name, pattern, in, !want, want) + } + } + } + } + } +} diff --git a/libgo/go/cmd/go/internal/load/path.go b/libgo/go/cmd/go/internal/load/path.go new file mode 100644 index 00000000000..9cc85dd757b --- /dev/null +++ b/libgo/go/cmd/go/internal/load/path.go @@ -0,0 +1,80 @@ +// Copyright 2017 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package load + +import ( + "path/filepath" + "strings" +) + +// hasSubdir reports whether dir is a subdirectory of +// (possibly multiple levels below) root. +// If so, it sets rel to the path fragment that must be +// appended to root to reach dir. +func hasSubdir(root, dir string) (rel string, ok bool) { + if p, err := filepath.EvalSymlinks(root); err == nil { + root = p + } + if p, err := filepath.EvalSymlinks(dir); err == nil { + dir = p + } + const sep = string(filepath.Separator) + root = filepath.Clean(root) + if !strings.HasSuffix(root, sep) { + root += sep + } + dir = filepath.Clean(dir) + if !strings.HasPrefix(dir, root) { + return "", false + } + return filepath.ToSlash(dir[len(root):]), true +} + +// hasPathPrefix reports whether the path s begins with the +// elements in prefix. +func hasPathPrefix(s, prefix string) bool { + switch { + default: + return false + case len(s) == len(prefix): + return s == prefix + case len(s) > len(prefix): + if prefix != "" && prefix[len(prefix)-1] == '/' { + return strings.HasPrefix(s, prefix) + } + return s[len(prefix)] == '/' && s[:len(prefix)] == prefix + } +} + +// expandPath returns the symlink-expanded form of path. +func expandPath(p string) string { + x, err := filepath.EvalSymlinks(p) + if err == nil { + return x + } + return p +} + +// hasFilePathPrefix reports whether the filesystem path s begins with the +// elements in prefix. +func hasFilePathPrefix(s, prefix string) bool { + sv := strings.ToUpper(filepath.VolumeName(s)) + pv := strings.ToUpper(filepath.VolumeName(prefix)) + s = s[len(sv):] + prefix = prefix[len(pv):] + switch { + default: + return false + case sv != pv: + return false + case len(s) == len(prefix): + return s == prefix + case len(s) > len(prefix): + if prefix != "" && prefix[len(prefix)-1] == filepath.Separator { + return strings.HasPrefix(s, prefix) + } + return s[len(prefix)] == filepath.Separator && s[:len(prefix)] == prefix + } +} diff --git a/libgo/go/cmd/go/internal/load/pkg.go b/libgo/go/cmd/go/internal/load/pkg.go new file mode 100644 index 00000000000..acb20fe03e4 --- /dev/null +++ b/libgo/go/cmd/go/internal/load/pkg.go @@ -0,0 +1,1971 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package load loads packages. +package load + +import ( + "crypto/sha1" + "fmt" + "go/build" + "go/token" + "io/ioutil" + "os" + pathpkg "path" + "path/filepath" + "runtime" + "sort" + "strings" + "unicode" + + "cmd/go/internal/base" + "cmd/go/internal/buildid" + "cmd/go/internal/cfg" + "cmd/go/internal/str" +) + +var IgnoreImports bool // control whether we ignore imports in packages + +// A Package describes a single package found in a directory. +type Package struct { + PackagePublic // visible in 'go list' + Internal PackageInternal // for use inside go command only +} + +type PackagePublic struct { + // Note: These fields are part of the go command's public API. + // See list.go. It is okay to add fields, but not to change or + // remove existing ones. Keep in sync with list.go + Dir string `json:",omitempty"` // directory containing package sources + ImportPath string `json:",omitempty"` // import path of package in dir + ImportComment string `json:",omitempty"` // path in import comment on package statement + Name string `json:",omitempty"` // package name + Doc string `json:",omitempty"` // package documentation string + Target string `json:",omitempty"` // install path + Shlib string `json:",omitempty"` // the shared library that contains this package (only set when -linkshared) + Goroot bool `json:",omitempty"` // is this package found in the Go root? + Standard bool `json:",omitempty"` // is this package part of the standard Go library? + Stale bool `json:",omitempty"` // would 'go install' do anything for this package? + StaleReason string `json:",omitempty"` // why is Stale true? + Root string `json:",omitempty"` // Go root or Go path dir containing this package + ConflictDir string `json:",omitempty"` // Dir is hidden by this other directory + BinaryOnly bool `json:",omitempty"` // package cannot be recompiled + + // Source files + GoFiles []string `json:",omitempty"` // .go source files (excluding CgoFiles, TestGoFiles, XTestGoFiles) + CgoFiles []string `json:",omitempty"` // .go sources files that import "C" + IgnoredGoFiles []string `json:",omitempty"` // .go sources ignored due to build constraints + CFiles []string `json:",omitempty"` // .c source files + CXXFiles []string `json:",omitempty"` // .cc, .cpp and .cxx source files + MFiles []string `json:",omitempty"` // .m source files + HFiles []string `json:",omitempty"` // .h, .hh, .hpp and .hxx source files + FFiles []string `json:",omitempty"` // .f, .F, .for and .f90 Fortran source files + SFiles []string `json:",omitempty"` // .s source files + SwigFiles []string `json:",omitempty"` // .swig files + SwigCXXFiles []string `json:",omitempty"` // .swigcxx files + SysoFiles []string `json:",omitempty"` // .syso system object files added to package + + // Cgo directives + CgoCFLAGS []string `json:",omitempty"` // cgo: flags for C compiler + CgoCPPFLAGS []string `json:",omitempty"` // cgo: flags for C preprocessor + CgoCXXFLAGS []string `json:",omitempty"` // cgo: flags for C++ compiler + CgoFFLAGS []string `json:",omitempty"` // cgo: flags for Fortran compiler + CgoLDFLAGS []string `json:",omitempty"` // cgo: flags for linker + CgoPkgConfig []string `json:",omitempty"` // cgo: pkg-config names + + // Dependency information + Imports []string `json:",omitempty"` // import paths used by this package + Deps []string `json:",omitempty"` // all (recursively) imported dependencies + + // Error information + Incomplete bool `json:",omitempty"` // was there an error loading this package or dependencies? + Error *PackageError `json:",omitempty"` // error loading this package (not dependencies) + DepsErrors []*PackageError `json:",omitempty"` // errors loading dependencies + + // Test information + TestGoFiles []string `json:",omitempty"` // _test.go files in package + TestImports []string `json:",omitempty"` // imports from TestGoFiles + XTestGoFiles []string `json:",omitempty"` // _test.go files outside package + XTestImports []string `json:",omitempty"` // imports from XTestGoFiles +} + +type PackageInternal struct { + // Unexported fields are not part of the public API. + Build *build.Package + Pkgdir string // overrides build.PkgDir + Imports []*Package + Deps []*Package + GoFiles []string // GoFiles+CgoFiles+TestGoFiles+XTestGoFiles files, absolute paths + SFiles []string + AllGoFiles []string // gofiles + IgnoredGoFiles, absolute paths + Target string // installed file for this package (may be executable) + Fake bool // synthesized package + External bool // synthesized external test package + ForceLibrary bool // this package is a library (even if named "main") + Cmdline bool // defined by files listed on command line + Local bool // imported via local path (./ or ../) + LocalPrefix string // interpret ./ and ../ imports relative to this prefix + ExeName string // desired name for temporary executable + CoverMode string // preprocess Go source files with the coverage tool in this mode + CoverVars map[string]*CoverVar // variables created by coverage analysis + OmitDebug bool // tell linker not to write debug information + BuildID string // expected build ID for generated package + GobinSubdir bool // install target would be subdir of GOBIN +} + +type NoGoError struct { + Package *Package +} + +func (e *NoGoError) Error() string { + // Count files beginning with _ and ., which we will pretend don't exist at all. + dummy := 0 + for _, name := range e.Package.IgnoredGoFiles { + if strings.HasPrefix(name, "_") || strings.HasPrefix(name, ".") { + dummy++ + } + } + + if len(e.Package.IgnoredGoFiles) > dummy { + // Go files exist, but they were ignored due to build constraints. + return "build constraints exclude all Go files in " + e.Package.Dir + } + if len(e.Package.TestGoFiles)+len(e.Package.XTestGoFiles) > 0 { + // Test Go files exist, but we're not interested in them. + // The double-negative is unfortunate but we want e.Package.Dir + // to appear at the end of error message. + return "no non-test Go files in " + e.Package.Dir + } + return "no Go files in " + e.Package.Dir +} + +// Vendored returns the vendor-resolved version of imports, +// which should be p.TestImports or p.XTestImports, NOT p.Imports. +// The imports in p.TestImports and p.XTestImports are not recursively +// loaded during the initial load of p, so they list the imports found in +// the source file, but most processing should be over the vendor-resolved +// import paths. We do this resolution lazily both to avoid file system work +// and because the eventual real load of the test imports (during 'go test') +// can produce better error messages if it starts with the original paths. +// The initial load of p loads all the non-test imports and rewrites +// the vendored paths, so nothing should ever call p.vendored(p.Imports). +func (p *Package) Vendored(imports []string) []string { + if len(imports) > 0 && len(p.Imports) > 0 && &imports[0] == &p.Imports[0] { + panic("internal error: p.vendored(p.Imports) called") + } + seen := make(map[string]bool) + var all []string + for _, path := range imports { + path = VendoredImportPath(p, path) + if !seen[path] { + seen[path] = true + all = append(all, path) + } + } + sort.Strings(all) + return all +} + +// CoverVar holds the name of the generated coverage variables targeting the named file. +type CoverVar struct { + File string // local file name + Var string // name of count struct +} + +func (p *Package) copyBuild(pp *build.Package) { + p.Internal.Build = pp + + if pp.PkgTargetRoot != "" && cfg.BuildPkgdir != "" { + old := pp.PkgTargetRoot + pp.PkgRoot = cfg.BuildPkgdir + pp.PkgTargetRoot = cfg.BuildPkgdir + pp.PkgObj = filepath.Join(cfg.BuildPkgdir, strings.TrimPrefix(pp.PkgObj, old)) + } + + p.Dir = pp.Dir + p.ImportPath = pp.ImportPath + p.ImportComment = pp.ImportComment + p.Name = pp.Name + p.Doc = pp.Doc + p.Root = pp.Root + p.ConflictDir = pp.ConflictDir + p.BinaryOnly = pp.BinaryOnly + + // TODO? Target + p.Goroot = pp.Goroot + p.Standard = p.Goroot && p.ImportPath != "" && isStandardImportPath(p.ImportPath) + if cfg.BuildToolchainName == "gccgo" { + p.Standard = stdpkg[p.ImportPath] + } + p.GoFiles = pp.GoFiles + p.CgoFiles = pp.CgoFiles + p.IgnoredGoFiles = pp.IgnoredGoFiles + p.CFiles = pp.CFiles + p.CXXFiles = pp.CXXFiles + p.MFiles = pp.MFiles + p.HFiles = pp.HFiles + p.FFiles = pp.FFiles + p.SFiles = pp.SFiles + p.SwigFiles = pp.SwigFiles + p.SwigCXXFiles = pp.SwigCXXFiles + p.SysoFiles = pp.SysoFiles + p.CgoCFLAGS = pp.CgoCFLAGS + p.CgoCPPFLAGS = pp.CgoCPPFLAGS + p.CgoCXXFLAGS = pp.CgoCXXFLAGS + p.CgoFFLAGS = pp.CgoFFLAGS + p.CgoLDFLAGS = pp.CgoLDFLAGS + p.CgoPkgConfig = pp.CgoPkgConfig + // We modify p.Imports in place, so make copy now. + p.Imports = make([]string, len(pp.Imports)) + copy(p.Imports, pp.Imports) + p.TestGoFiles = pp.TestGoFiles + p.TestImports = pp.TestImports + p.XTestGoFiles = pp.XTestGoFiles + p.XTestImports = pp.XTestImports + if IgnoreImports { + p.Imports = nil + p.TestImports = nil + p.XTestImports = nil + } +} + +// isStandardImportPath reports whether $GOROOT/src/path should be considered +// part of the standard distribution. For historical reasons we allow people to add +// their own code to $GOROOT instead of using $GOPATH, but we assume that +// code will start with a domain name (dot in the first element). +func isStandardImportPath(path string) bool { + i := strings.Index(path, "/") + if i < 0 { + i = len(path) + } + elem := path[:i] + return !strings.Contains(elem, ".") +} + +// A PackageError describes an error loading information about a package. +type PackageError struct { + ImportStack []string // shortest path from package named on command line to this one + Pos string // position of error + Err string // the error itself + IsImportCycle bool `json:"-"` // the error is an import cycle + Hard bool `json:"-"` // whether the error is soft or hard; soft errors are ignored in some places +} + +func (p *PackageError) Error() string { + // Import cycles deserve special treatment. + if p.IsImportCycle { + return fmt.Sprintf("%s\npackage %s\n", p.Err, strings.Join(p.ImportStack, "\n\timports ")) + } + if p.Pos != "" { + // Omit import stack. The full path to the file where the error + // is the most important thing. + return p.Pos + ": " + p.Err + } + if len(p.ImportStack) == 0 { + return p.Err + } + return "package " + strings.Join(p.ImportStack, "\n\timports ") + ": " + p.Err +} + +// An ImportStack is a stack of import paths. +type ImportStack []string + +func (s *ImportStack) Push(p string) { + *s = append(*s, p) +} + +func (s *ImportStack) Pop() { + *s = (*s)[0 : len(*s)-1] +} + +func (s *ImportStack) Copy() []string { + return append([]string{}, *s...) +} + +// shorterThan reports whether sp is shorter than t. +// We use this to record the shortest import sequence +// that leads to a particular package. +func (sp *ImportStack) shorterThan(t []string) bool { + s := *sp + if len(s) != len(t) { + return len(s) < len(t) + } + // If they are the same length, settle ties using string ordering. + for i := range s { + if s[i] != t[i] { + return s[i] < t[i] + } + } + return false // they are equal +} + +// packageCache is a lookup cache for loadPackage, +// so that if we look up a package multiple times +// we return the same pointer each time. +var packageCache = map[string]*Package{} + +func ClearPackageCache() { + for name := range packageCache { + delete(packageCache, name) + } +} + +func ClearPackageCachePartial(args []string) { + for _, arg := range args { + p := packageCache[arg] + if p != nil { + delete(packageCache, p.Dir) + delete(packageCache, p.ImportPath) + } + } +} + +// reloadPackage is like loadPackage but makes sure +// not to use the package cache. +func ReloadPackage(arg string, stk *ImportStack) *Package { + p := packageCache[arg] + if p != nil { + delete(packageCache, p.Dir) + delete(packageCache, p.ImportPath) + } + return LoadPackage(arg, stk) +} + +// dirToImportPath returns the pseudo-import path we use for a package +// outside the Go path. It begins with _/ and then contains the full path +// to the directory. If the package lives in c:\home\gopher\my\pkg then +// the pseudo-import path is _/c_/home/gopher/my/pkg. +// Using a pseudo-import path like this makes the ./ imports no longer +// a special case, so that all the code to deal with ordinary imports works +// automatically. +func dirToImportPath(dir string) string { + return pathpkg.Join("_", strings.Map(makeImportValid, filepath.ToSlash(dir))) +} + +func makeImportValid(r rune) rune { + // Should match Go spec, compilers, and ../../go/parser/parser.go:/isValidImport. + const illegalChars = `!"#$%&'()*,:;<=>?[\]^{|}` + "`\uFFFD" + if !unicode.IsGraphic(r) || unicode.IsSpace(r) || strings.ContainsRune(illegalChars, r) { + return '_' + } + return r +} + +// Mode flags for loadImport and download (in get.go). +const ( + // useVendor means that loadImport should do vendor expansion + // (provided the vendoring experiment is enabled). + // That is, useVendor means that the import path came from + // a source file and has not been vendor-expanded yet. + // Every import path should be loaded initially with useVendor, + // and then the expanded version (with the /vendor/ in it) gets + // recorded as the canonical import path. At that point, future loads + // of that package must not pass useVendor, because + // disallowVendor will reject direct use of paths containing /vendor/. + UseVendor = 1 << iota + + // getTestDeps is for download (part of "go get") and indicates + // that test dependencies should be fetched too. + GetTestDeps +) + +// loadImport scans the directory named by path, which must be an import path, +// but possibly a local import path (an absolute file system path or one beginning +// with ./ or ../). A local relative path is interpreted relative to srcDir. +// It returns a *Package describing the package found in that directory. +func LoadImport(path, srcDir string, parent *Package, stk *ImportStack, importPos []token.Position, mode int) *Package { + stk.Push(path) + defer stk.Pop() + + // Determine canonical identifier for this package. + // For a local import the identifier is the pseudo-import path + // we create from the full directory to the package. + // Otherwise it is the usual import path. + // For vendored imports, it is the expanded form. + importPath := path + origPath := path + isLocal := build.IsLocalImport(path) + if isLocal { + importPath = dirToImportPath(filepath.Join(srcDir, path)) + } else if mode&UseVendor != 0 { + // We do our own vendor resolution, because we want to + // find out the key to use in packageCache without the + // overhead of repeated calls to buildContext.Import. + // The code is also needed in a few other places anyway. + path = VendoredImportPath(parent, path) + importPath = path + } + + p := packageCache[importPath] + if p != nil { + p = reusePackage(p, stk) + } else { + p = new(Package) + p.Internal.Local = isLocal + p.ImportPath = importPath + packageCache[importPath] = p + + // Load package. + // Import always returns bp != nil, even if an error occurs, + // in order to return partial information. + // + // TODO: After Go 1, decide when to pass build.AllowBinary here. + // See issue 3268 for mistakes to avoid. + buildMode := build.ImportComment + if mode&UseVendor == 0 || path != origPath { + // Not vendoring, or we already found the vendored path. + buildMode |= build.IgnoreVendor + } + bp, err := cfg.BuildContext.Import(path, srcDir, buildMode) + bp.ImportPath = importPath + if cfg.GOBIN != "" { + bp.BinDir = cfg.GOBIN + } + if err == nil && !isLocal && bp.ImportComment != "" && bp.ImportComment != path && + !strings.Contains(path, "/vendor/") && !strings.HasPrefix(path, "vendor/") { + err = fmt.Errorf("code in directory %s expects import %q", bp.Dir, bp.ImportComment) + } + p.load(stk, bp, err) + if p.Error != nil && p.Error.Pos == "" { + p = setErrorPos(p, importPos) + } + + if origPath != cleanImport(origPath) { + p.Error = &PackageError{ + ImportStack: stk.Copy(), + Err: fmt.Sprintf("non-canonical import path: %q should be %q", origPath, pathpkg.Clean(origPath)), + } + p.Incomplete = true + } + } + + // Checked on every import because the rules depend on the code doing the importing. + if perr := disallowInternal(srcDir, p, stk); perr != p { + return setErrorPos(perr, importPos) + } + if mode&UseVendor != 0 { + if perr := disallowVendor(srcDir, origPath, p, stk); perr != p { + return setErrorPos(perr, importPos) + } + } + + if p.Name == "main" && parent != nil && parent.Dir != p.Dir { + perr := *p + perr.Error = &PackageError{ + ImportStack: stk.Copy(), + Err: fmt.Sprintf("import %q is a program, not an importable package", path), + } + return setErrorPos(&perr, importPos) + } + + if p.Internal.Local && parent != nil && !parent.Internal.Local { + perr := *p + perr.Error = &PackageError{ + ImportStack: stk.Copy(), + Err: fmt.Sprintf("local import %q in non-local package", path), + } + return setErrorPos(&perr, importPos) + } + + return p +} + +func setErrorPos(p *Package, importPos []token.Position) *Package { + if len(importPos) > 0 { + pos := importPos[0] + pos.Filename = base.ShortPath(pos.Filename) + p.Error.Pos = pos.String() + } + return p +} + +func cleanImport(path string) string { + orig := path + path = pathpkg.Clean(path) + if strings.HasPrefix(orig, "./") && path != ".." && !strings.HasPrefix(path, "../") { + path = "./" + path + } + return path +} + +var isDirCache = map[string]bool{} + +func isDir(path string) bool { + result, ok := isDirCache[path] + if ok { + return result + } + + fi, err := os.Stat(path) + result = err == nil && fi.IsDir() + isDirCache[path] = result + return result +} + +// VendoredImportPath returns the expansion of path when it appears in parent. +// If parent is x/y/z, then path might expand to x/y/z/vendor/path, x/y/vendor/path, +// x/vendor/path, vendor/path, or else stay path if none of those exist. +// VendoredImportPath returns the expanded path or, if no expansion is found, the original. +func VendoredImportPath(parent *Package, path string) (found string) { + if parent == nil || parent.Root == "" { + return path + } + + dir := filepath.Clean(parent.Dir) + root := filepath.Join(parent.Root, "src") + if !hasFilePathPrefix(dir, root) || parent.ImportPath != "command-line-arguments" && filepath.Join(root, parent.ImportPath) != dir { + // Look for symlinks before reporting error. + dir = expandPath(dir) + root = expandPath(root) + } + + if !hasFilePathPrefix(dir, root) || len(dir) <= len(root) || dir[len(root)] != filepath.Separator || parent.ImportPath != "command-line-arguments" && !parent.Internal.Local && filepath.Join(root, parent.ImportPath) != dir { + base.Fatalf("unexpected directory layout:\n"+ + " import path: %s\n"+ + " root: %s\n"+ + " dir: %s\n"+ + " expand root: %s\n"+ + " expand dir: %s\n"+ + " separator: %s", + parent.ImportPath, + filepath.Join(parent.Root, "src"), + filepath.Clean(parent.Dir), + root, + dir, + string(filepath.Separator)) + } + + vpath := "vendor/" + path + for i := len(dir); i >= len(root); i-- { + if i < len(dir) && dir[i] != filepath.Separator { + continue + } + // Note: checking for the vendor directory before checking + // for the vendor/path directory helps us hit the + // isDir cache more often. It also helps us prepare a more useful + // list of places we looked, to report when an import is not found. + if !isDir(filepath.Join(dir[:i], "vendor")) { + continue + } + targ := filepath.Join(dir[:i], vpath) + if isDir(targ) && hasGoFiles(targ) { + importPath := parent.ImportPath + if importPath == "command-line-arguments" { + // If parent.ImportPath is 'command-line-arguments'. + // set to relative directory to root (also chopped root directory) + importPath = dir[len(root)+1:] + } + // We started with parent's dir c:\gopath\src\foo\bar\baz\quux\xyzzy. + // We know the import path for parent's dir. + // We chopped off some number of path elements and + // added vendor\path to produce c:\gopath\src\foo\bar\baz\vendor\path. + // Now we want to know the import path for that directory. + // Construct it by chopping the same number of path elements + // (actually the same number of bytes) from parent's import path + // and then append /vendor/path. + chopped := len(dir) - i + if chopped == len(importPath)+1 { + // We walked up from c:\gopath\src\foo\bar + // and found c:\gopath\src\vendor\path. + // We chopped \foo\bar (length 8) but the import path is "foo/bar" (length 7). + // Use "vendor/path" without any prefix. + return vpath + } + return importPath[:len(importPath)-chopped] + "/" + vpath + } + } + return path +} + +// hasGoFiles reports whether dir contains any files with names ending in .go. +// For a vendor check we must exclude directories that contain no .go files. +// Otherwise it is not possible to vendor just a/b/c and still import the +// non-vendored a/b. See golang.org/issue/13832. +func hasGoFiles(dir string) bool { + fis, _ := ioutil.ReadDir(dir) + for _, fi := range fis { + if !fi.IsDir() && strings.HasSuffix(fi.Name(), ".go") { + return true + } + } + return false +} + +// reusePackage reuses package p to satisfy the import at the top +// of the import stack stk. If this use causes an import loop, +// reusePackage updates p's error information to record the loop. +func reusePackage(p *Package, stk *ImportStack) *Package { + // We use p.Internal.Imports==nil to detect a package that + // is in the midst of its own loadPackage call + // (all the recursion below happens before p.Internal.Imports gets set). + if p.Internal.Imports == nil { + if p.Error == nil { + p.Error = &PackageError{ + ImportStack: stk.Copy(), + Err: "import cycle not allowed", + IsImportCycle: true, + } + } + p.Incomplete = true + } + // Don't rewrite the import stack in the error if we have an import cycle. + // If we do, we'll lose the path that describes the cycle. + if p.Error != nil && !p.Error.IsImportCycle && stk.shorterThan(p.Error.ImportStack) { + p.Error.ImportStack = stk.Copy() + } + return p +} + +// disallowInternal checks that srcDir is allowed to import p. +// If the import is allowed, disallowInternal returns the original package p. +// If not, it returns a new package containing just an appropriate error. +func disallowInternal(srcDir string, p *Package, stk *ImportStack) *Package { + // golang.org/s/go14internal: + // An import of a path containing the element “internal” + // is disallowed if the importing code is outside the tree + // rooted at the parent of the “internal” directory. + + // There was an error loading the package; stop here. + if p.Error != nil { + return p + } + + // The generated 'testmain' package is allowed to access testing/internal/..., + // as if it were generated into the testing directory tree + // (it's actually in a temporary directory outside any Go tree). + // This cleans up a former kludge in passing functionality to the testing package. + if strings.HasPrefix(p.ImportPath, "testing/internal") && len(*stk) >= 2 && (*stk)[len(*stk)-2] == "testmain" { + return p + } + + // We can't check standard packages with gccgo. + if cfg.BuildContext.Compiler == "gccgo" && p.Standard { + return p + } + + // The stack includes p.ImportPath. + // If that's the only thing on the stack, we started + // with a name given on the command line, not an + // import. Anything listed on the command line is fine. + if len(*stk) == 1 { + return p + } + + // Check for "internal" element: three cases depending on begin of string and/or end of string. + i, ok := findInternal(p.ImportPath) + if !ok { + return p + } + + // Internal is present. + // Map import path back to directory corresponding to parent of internal. + if i > 0 { + i-- // rewind over slash in ".../internal" + } + parent := p.Dir[:i+len(p.Dir)-len(p.ImportPath)] + if hasFilePathPrefix(filepath.Clean(srcDir), filepath.Clean(parent)) { + return p + } + + // Look for symlinks before reporting error. + srcDir = expandPath(srcDir) + parent = expandPath(parent) + if hasFilePathPrefix(filepath.Clean(srcDir), filepath.Clean(parent)) { + return p + } + + // Internal is present, and srcDir is outside parent's tree. Not allowed. + perr := *p + perr.Error = &PackageError{ + ImportStack: stk.Copy(), + Err: "use of internal package not allowed", + } + perr.Incomplete = true + return &perr +} + +// findInternal looks for the final "internal" path element in the given import path. +// If there isn't one, findInternal returns ok=false. +// Otherwise, findInternal returns ok=true and the index of the "internal". +func findInternal(path string) (index int, ok bool) { + // Three cases, depending on internal at start/end of string or not. + // The order matters: we must return the index of the final element, + // because the final one produces the most restrictive requirement + // on the importer. + switch { + case strings.HasSuffix(path, "/internal"): + return len(path) - len("internal"), true + case strings.Contains(path, "/internal/"): + return strings.LastIndex(path, "/internal/") + 1, true + case path == "internal", strings.HasPrefix(path, "internal/"): + return 0, true + } + return 0, false +} + +// disallowVendor checks that srcDir is allowed to import p as path. +// If the import is allowed, disallowVendor returns the original package p. +// If not, it returns a new package containing just an appropriate error. +func disallowVendor(srcDir, path string, p *Package, stk *ImportStack) *Package { + // The stack includes p.ImportPath. + // If that's the only thing on the stack, we started + // with a name given on the command line, not an + // import. Anything listed on the command line is fine. + if len(*stk) == 1 { + return p + } + + if perr := disallowVendorVisibility(srcDir, p, stk); perr != p { + return perr + } + + // Paths like x/vendor/y must be imported as y, never as x/vendor/y. + if i, ok := FindVendor(path); ok { + perr := *p + perr.Error = &PackageError{ + ImportStack: stk.Copy(), + Err: "must be imported as " + path[i+len("vendor/"):], + } + perr.Incomplete = true + return &perr + } + + return p +} + +// disallowVendorVisibility checks that srcDir is allowed to import p. +// The rules are the same as for /internal/ except that a path ending in /vendor +// is not subject to the rules, only subdirectories of vendor. +// This allows people to have packages and commands named vendor, +// for maximal compatibility with existing source trees. +func disallowVendorVisibility(srcDir string, p *Package, stk *ImportStack) *Package { + // The stack includes p.ImportPath. + // If that's the only thing on the stack, we started + // with a name given on the command line, not an + // import. Anything listed on the command line is fine. + if len(*stk) == 1 { + return p + } + + // Check for "vendor" element. + i, ok := FindVendor(p.ImportPath) + if !ok { + return p + } + + // Vendor is present. + // Map import path back to directory corresponding to parent of vendor. + if i > 0 { + i-- // rewind over slash in ".../vendor" + } + truncateTo := i + len(p.Dir) - len(p.ImportPath) + if truncateTo < 0 || len(p.Dir) < truncateTo { + return p + } + parent := p.Dir[:truncateTo] + if hasFilePathPrefix(filepath.Clean(srcDir), filepath.Clean(parent)) { + return p + } + + // Look for symlinks before reporting error. + srcDir = expandPath(srcDir) + parent = expandPath(parent) + if hasFilePathPrefix(filepath.Clean(srcDir), filepath.Clean(parent)) { + return p + } + + // Vendor is present, and srcDir is outside parent's tree. Not allowed. + perr := *p + perr.Error = &PackageError{ + ImportStack: stk.Copy(), + Err: "use of vendored package not allowed", + } + perr.Incomplete = true + return &perr +} + +// FindVendor looks for the last non-terminating "vendor" path element in the given import path. +// If there isn't one, FindVendor returns ok=false. +// Otherwise, FindVendor returns ok=true and the index of the "vendor". +// +// Note that terminating "vendor" elements don't count: "x/vendor" is its own package, +// not the vendored copy of an import "" (the empty import path). +// This will allow people to have packages or commands named vendor. +// This may help reduce breakage, or it may just be confusing. We'll see. +func FindVendor(path string) (index int, ok bool) { + // Two cases, depending on internal at start of string or not. + // The order matters: we must return the index of the final element, + // because the final one is where the effective import path starts. + switch { + case strings.Contains(path, "/vendor/"): + return strings.LastIndex(path, "/vendor/") + 1, true + case strings.HasPrefix(path, "vendor/"): + return 0, true + } + return 0, false +} + +type targetDir int + +const ( + ToRoot targetDir = iota // to bin dir inside package root (default) + ToTool // GOROOT/pkg/tool + StalePath // the old import path; fail to build +) + +// goTools is a map of Go program import path to install target directory. +var GoTools = map[string]targetDir{ + "cmd/addr2line": ToTool, + "cmd/api": ToTool, + "cmd/asm": ToTool, + "cmd/compile": ToTool, + "cmd/cgo": ToTool, + "cmd/cover": ToTool, + "cmd/dist": ToTool, + "cmd/doc": ToTool, + "cmd/fix": ToTool, + "cmd/link": ToTool, + "cmd/newlink": ToTool, + "cmd/nm": ToTool, + "cmd/objdump": ToTool, + "cmd/pack": ToTool, + "cmd/pprof": ToTool, + "cmd/trace": ToTool, + "cmd/vet": ToTool, + "code.google.com/p/go.tools/cmd/cover": StalePath, + "code.google.com/p/go.tools/cmd/godoc": StalePath, + "code.google.com/p/go.tools/cmd/vet": StalePath, +} + +var raceExclude = map[string]bool{ + "runtime/race": true, + "runtime/msan": true, + "runtime/cgo": true, + "cmd/cgo": true, + "syscall": true, + "errors": true, +} + +var cgoExclude = map[string]bool{ + "runtime/cgo": true, +} + +var cgoSyscallExclude = map[string]bool{ + "runtime/cgo": true, + "runtime/race": true, + "runtime/msan": true, +} + +var foldPath = make(map[string]string) + +// load populates p using information from bp, err, which should +// be the result of calling build.Context.Import. +func (p *Package) load(stk *ImportStack, bp *build.Package, err error) *Package { + p.copyBuild(bp) + + // When using gccgo the go/build package will not be able to + // find a standard package. It would be nicer to not get that + // error, but go/build doesn't know stdpkg. + if cfg.BuildToolchainName == "gccgo" && err != nil && p.Standard { + err = nil + } + + // The localPrefix is the path we interpret ./ imports relative to. + // Synthesized main packages sometimes override this. + p.Internal.LocalPrefix = dirToImportPath(p.Dir) + + if err != nil { + if _, ok := err.(*build.NoGoError); ok { + err = &NoGoError{Package: p} + } + p.Incomplete = true + err = base.ExpandScanner(err) + p.Error = &PackageError{ + ImportStack: stk.Copy(), + Err: err.Error(), + } + return p + } + + useBindir := p.Name == "main" + if !p.Standard { + switch cfg.BuildBuildmode { + case "c-archive", "c-shared", "plugin": + useBindir = false + } + } + + if useBindir { + // Report an error when the old code.google.com/p/go.tools paths are used. + if GoTools[p.ImportPath] == StalePath { + newPath := strings.Replace(p.ImportPath, "code.google.com/p/go.", "golang.org/x/", 1) + e := fmt.Sprintf("the %v command has moved; use %v instead.", p.ImportPath, newPath) + p.Error = &PackageError{Err: e} + return p + } + _, elem := filepath.Split(p.Dir) + full := cfg.BuildContext.GOOS + "_" + cfg.BuildContext.GOARCH + "/" + elem + if cfg.BuildContext.GOOS != base.ToolGOOS || cfg.BuildContext.GOARCH != base.ToolGOARCH { + // Install cross-compiled binaries to subdirectories of bin. + elem = full + } + if p.Internal.Build.BinDir != "" { + // Install to GOBIN or bin of GOPATH entry. + p.Internal.Target = filepath.Join(p.Internal.Build.BinDir, elem) + if !p.Goroot && strings.Contains(elem, "/") && cfg.GOBIN != "" { + // Do not create $GOBIN/goos_goarch/elem. + p.Internal.Target = "" + p.Internal.GobinSubdir = true + } + } + if GoTools[p.ImportPath] == ToTool { + // This is for 'go tool'. + // Override all the usual logic and force it into the tool directory. + if cfg.BuildToolchainName == "gccgo" { + p.Internal.Target = filepath.Join(runtime.GCCGOTOOLDIR, elem) + } else { + p.Internal.Target = filepath.Join(cfg.GOROOTpkg, "tool", full) + } + } + if p.Internal.Target != "" && cfg.BuildContext.GOOS == "windows" { + p.Internal.Target += ".exe" + } + } else if p.Internal.Local { + // Local import turned into absolute path. + // No permanent install target. + p.Internal.Target = "" + } else { + p.Internal.Target = p.Internal.Build.PkgObj + if cfg.BuildLinkshared { + shlibnamefile := p.Internal.Target[:len(p.Internal.Target)-2] + ".shlibname" + shlib, err := ioutil.ReadFile(shlibnamefile) + if err == nil { + libname := strings.TrimSpace(string(shlib)) + if cfg.BuildContext.Compiler == "gccgo" { + p.Shlib = filepath.Join(p.Internal.Build.PkgTargetRoot, "shlibs", libname) + } else { + p.Shlib = filepath.Join(p.Internal.Build.PkgTargetRoot, libname) + + } + } else if !os.IsNotExist(err) { + base.Fatalf("unexpected error reading %s: %v", shlibnamefile, err) + } + } + } + + ImportPaths := p.Imports + // Packages that use cgo import runtime/cgo implicitly. + // Packages that use cgo also import syscall implicitly, + // to wrap errno. + // Exclude certain packages to avoid circular dependencies. + if len(p.CgoFiles) > 0 && (!p.Standard || !cgoExclude[p.ImportPath]) { + ImportPaths = append(ImportPaths, "runtime/cgo") + } + if len(p.CgoFiles) > 0 && (!p.Standard || !cgoSyscallExclude[p.ImportPath]) { + ImportPaths = append(ImportPaths, "syscall") + } + + if cfg.BuildContext.CgoEnabled && p.Name == "main" && !p.Goroot { + // Currently build modes c-shared, pie (on systems that do not + // support PIE with internal linking mode), plugin, and + // -linkshared force external linking mode, as of course does + // -ldflags=-linkmode=external. External linking mode forces + // an import of runtime/cgo. + pieCgo := cfg.BuildBuildmode == "pie" && (cfg.BuildContext.GOOS != "linux" || cfg.BuildContext.GOARCH != "amd64") + linkmodeExternal := false + for i, a := range cfg.BuildLdflags { + if a == "-linkmode=external" { + linkmodeExternal = true + } + if a == "-linkmode" && i+1 < len(cfg.BuildLdflags) && cfg.BuildLdflags[i+1] == "external" { + linkmodeExternal = true + } + } + if cfg.BuildBuildmode == "c-shared" || cfg.BuildBuildmode == "plugin" || pieCgo || cfg.BuildLinkshared || linkmodeExternal { + ImportPaths = append(ImportPaths, "runtime/cgo") + } + } + + // Everything depends on runtime, except runtime, its internal + // subpackages, and unsafe. + if !p.Standard || (p.ImportPath != "runtime" && !strings.HasPrefix(p.ImportPath, "runtime/internal/") && p.ImportPath != "unsafe") { + ImportPaths = append(ImportPaths, "runtime") + // When race detection enabled everything depends on runtime/race. + // Exclude certain packages to avoid circular dependencies. + if cfg.BuildRace && (!p.Standard || !raceExclude[p.ImportPath]) { + ImportPaths = append(ImportPaths, "runtime/race") + } + // MSan uses runtime/msan. + if cfg.BuildMSan && (!p.Standard || !raceExclude[p.ImportPath]) { + ImportPaths = append(ImportPaths, "runtime/msan") + } + // On ARM with GOARM=5, everything depends on math for the link. + if p.Name == "main" && cfg.Goarch == "arm" { + ImportPaths = append(ImportPaths, "math") + } + } + + // Runtime and its internal packages depend on runtime/internal/sys, + // so that they pick up the generated zversion.go file. + // This can be an issue particularly for runtime/internal/atomic; + // see issue 13655. + if p.Standard && (p.ImportPath == "runtime" || strings.HasPrefix(p.ImportPath, "runtime/internal/")) && p.ImportPath != "runtime/internal/sys" { + ImportPaths = append(ImportPaths, "runtime/internal/sys") + } + + // Build list of full paths to all Go files in the package, + // for use by commands like go fmt. + p.Internal.GoFiles = str.StringList(p.GoFiles, p.CgoFiles, p.TestGoFiles, p.XTestGoFiles) + for i := range p.Internal.GoFiles { + p.Internal.GoFiles[i] = filepath.Join(p.Dir, p.Internal.GoFiles[i]) + } + sort.Strings(p.Internal.GoFiles) + + p.Internal.SFiles = str.StringList(p.SFiles) + for i := range p.Internal.SFiles { + p.Internal.SFiles[i] = filepath.Join(p.Dir, p.Internal.SFiles[i]) + } + sort.Strings(p.Internal.SFiles) + + p.Internal.AllGoFiles = str.StringList(p.IgnoredGoFiles) + for i := range p.Internal.AllGoFiles { + p.Internal.AllGoFiles[i] = filepath.Join(p.Dir, p.Internal.AllGoFiles[i]) + } + p.Internal.AllGoFiles = append(p.Internal.AllGoFiles, p.Internal.GoFiles...) + sort.Strings(p.Internal.AllGoFiles) + + // Check for case-insensitive collision of input files. + // To avoid problems on case-insensitive files, we reject any package + // where two different input files have equal names under a case-insensitive + // comparison. + f1, f2 := str.FoldDup(str.StringList( + p.GoFiles, + p.CgoFiles, + p.IgnoredGoFiles, + p.CFiles, + p.CXXFiles, + p.MFiles, + p.HFiles, + p.FFiles, + p.SFiles, + p.SysoFiles, + p.SwigFiles, + p.SwigCXXFiles, + p.TestGoFiles, + p.XTestGoFiles, + )) + if f1 != "" { + p.Error = &PackageError{ + ImportStack: stk.Copy(), + Err: fmt.Sprintf("case-insensitive file name collision: %q and %q", f1, f2), + } + return p + } + + // Build list of imported packages and full dependency list. + imports := make([]*Package, 0, len(p.Imports)) + deps := make(map[string]*Package) + save := func(path string, p1 *Package) { + // The same import path could produce an error or not, + // depending on what tries to import it. + // Prefer to record entries with errors, so we can report them. + p0 := deps[path] + if p0 == nil || p1.Error != nil && (p0.Error == nil || len(p0.Error.ImportStack) > len(p1.Error.ImportStack)) { + deps[path] = p1 + } + } + + for i, path := range ImportPaths { + if path == "C" { + continue + } + p1 := LoadImport(path, p.Dir, p, stk, p.Internal.Build.ImportPos[path], UseVendor) + if cfg.BuildToolchainName == "gccgo" && p1.Standard { + continue + } + if p.Standard && p.Error == nil && !p1.Standard && p1.Error == nil { + p.Error = &PackageError{ + ImportStack: stk.Copy(), + Err: fmt.Sprintf("non-standard import %q in standard package %q", path, p.ImportPath), + } + pos := p.Internal.Build.ImportPos[path] + if len(pos) > 0 { + p.Error.Pos = pos[0].String() + } + } + + path = p1.ImportPath + ImportPaths[i] = path + if i < len(p.Imports) { + p.Imports[i] = path + } + + save(path, p1) + imports = append(imports, p1) + for _, dep := range p1.Internal.Deps { + save(dep.ImportPath, dep) + } + if p1.Incomplete { + p.Incomplete = true + } + } + p.Internal.Imports = imports + + p.Deps = make([]string, 0, len(deps)) + for dep := range deps { + p.Deps = append(p.Deps, dep) + } + sort.Strings(p.Deps) + for _, dep := range p.Deps { + p1 := deps[dep] + if p1 == nil { + panic("impossible: missing entry in package cache for " + dep + " imported by " + p.ImportPath) + } + p.Internal.Deps = append(p.Internal.Deps, p1) + if p1.Error != nil { + p.DepsErrors = append(p.DepsErrors, p1.Error) + } + } + + // unsafe is a fake package. + if p.Standard && (p.ImportPath == "unsafe" || cfg.BuildContext.Compiler == "gccgo") { + p.Internal.Target = "" + } + p.Target = p.Internal.Target + + // If cgo is not enabled, ignore cgo supporting sources + // just as we ignore go files containing import "C". + if !cfg.BuildContext.CgoEnabled { + p.CFiles = nil + p.CXXFiles = nil + p.MFiles = nil + p.SwigFiles = nil + p.SwigCXXFiles = nil + // Note that SFiles are okay (they go to the Go assembler) + // and HFiles are okay (they might be used by the SFiles). + // Also Sysofiles are okay (they might not contain object + // code; see issue #16050). + } + + // The gc toolchain only permits C source files with cgo. + if len(p.CFiles) > 0 && !p.UsesCgo() && !p.UsesSwig() && cfg.BuildContext.Compiler == "gc" { + p.Error = &PackageError{ + ImportStack: stk.Copy(), + Err: fmt.Sprintf("C source files not allowed when not using cgo or SWIG: %s", strings.Join(p.CFiles, " ")), + } + return p + } + + // Check for case-insensitive collisions of import paths. + fold := str.ToFold(p.ImportPath) + if other := foldPath[fold]; other == "" { + foldPath[fold] = p.ImportPath + } else if other != p.ImportPath { + p.Error = &PackageError{ + ImportStack: stk.Copy(), + Err: fmt.Sprintf("case-insensitive import collision: %q and %q", p.ImportPath, other), + } + return p + } + + if p.BinaryOnly { + // For binary-only package, use build ID from supplied package binary. + buildID, err := buildid.ReadBuildID(p.Name, p.Target) + if err == nil { + p.Internal.BuildID = buildID + } + } else { + computeBuildID(p) + } + return p +} + +// usesSwig reports whether the package needs to run SWIG. +func (p *Package) UsesSwig() bool { + return len(p.SwigFiles) > 0 || len(p.SwigCXXFiles) > 0 +} + +// usesCgo reports whether the package needs to run cgo +func (p *Package) UsesCgo() bool { + return len(p.CgoFiles) > 0 +} + +// packageList returns the list of packages in the dag rooted at roots +// as visited in a depth-first post-order traversal. +func PackageList(roots []*Package) []*Package { + seen := map[*Package]bool{} + all := []*Package{} + var walk func(*Package) + walk = func(p *Package) { + if seen[p] { + return + } + seen[p] = true + for _, p1 := range p.Internal.Imports { + walk(p1) + } + all = append(all, p) + } + for _, root := range roots { + walk(root) + } + return all +} + +// computeStale computes the Stale flag in the package dag that starts +// at the named pkgs (command-line arguments). +func ComputeStale(pkgs ...*Package) { + for _, p := range PackageList(pkgs) { + p.Stale, p.StaleReason = isStale(p) + } +} + +// The runtime version string takes one of two forms: +// "go1.X[.Y]" for Go releases, and "devel +hash" at tip. +// Determine whether we are in a released copy by +// inspecting the version. +var isGoRelease = strings.HasPrefix(runtime.Version(), "go1") + +// isStale and computeBuildID +// +// Theory of Operation +// +// There is an installed copy of the package (or binary). +// Can we reuse the installed copy, or do we need to build a new one? +// +// We can use the installed copy if it matches what we'd get +// by building a new one. The hard part is predicting that without +// actually running a build. +// +// To start, we must know the set of inputs to the build process that can +// affect the generated output. At a minimum, that includes the source +// files for the package and also any compiled packages imported by those +// source files. The *Package has these, and we use them. One might also +// argue for including in the input set: the build tags, whether the race +// detector is in use, the target operating system and architecture, the +// compiler and linker binaries being used, the additional flags being +// passed to those, the cgo binary being used, the additional flags cgo +// passes to the host C compiler, the host C compiler being used, the set +// of host C include files and installed C libraries, and so on. +// We include some but not all of this information. +// +// Once we have decided on a set of inputs, we must next decide how to +// tell whether the content of that set has changed since the last build +// of p. If there have been no changes, then we assume a new build would +// produce the same result and reuse the installed package or binary. +// But if there have been changes, then we assume a new build might not +// produce the same result, so we rebuild. +// +// There are two common ways to decide whether the content of the set has +// changed: modification times and content hashes. We use a mixture of both. +// +// The use of modification times (mtimes) was pioneered by make: +// assuming that a file's mtime is an accurate record of when that file was last written, +// and assuming that the modification time of an installed package or +// binary is the time that it was built, if the mtimes of the inputs +// predate the mtime of the installed object, then the build of that +// object saw those versions of the files, and therefore a rebuild using +// those same versions would produce the same object. In contrast, if any +// mtime of an input is newer than the mtime of the installed object, a +// change has occurred since the build, and the build should be redone. +// +// Modification times are attractive because the logic is easy to +// understand and the file system maintains the mtimes automatically +// (less work for us). Unfortunately, there are a variety of ways in +// which the mtime approach fails to detect a change and reuses a stale +// object file incorrectly. (Making the opposite mistake, rebuilding +// unnecessarily, is only a performance problem and not a correctness +// problem, so we ignore that one.) +// +// As a warmup, one problem is that to be perfectly precise, we need to +// compare the input mtimes against the time at the beginning of the +// build, but the object file time is the time at the end of the build. +// If an input file changes after being read but before the object is +// written, the next build will see an object newer than the input and +// will incorrectly decide that the object is up to date. We make no +// attempt to detect or solve this problem. +// +// Another problem is that due to file system imprecision, an input and +// output that are actually ordered in time have the same mtime. +// This typically happens on file systems with 1-second (or, worse, +// 2-second) mtime granularity and with automated scripts that write an +// input and then immediately run a build, or vice versa. If an input and +// an output have the same mtime, the conservative behavior is to treat +// the output as out-of-date and rebuild. This can cause one or more +// spurious rebuilds, but only for 1 second, until the object finally has +// an mtime later than the input. +// +// Another problem is that binary distributions often set the mtime on +// all files to the same time. If the distribution includes both inputs +// and cached build outputs, the conservative solution to the previous +// problem will cause unnecessary rebuilds. Worse, in such a binary +// distribution, those rebuilds might not even have permission to update +// the cached build output. To avoid these write errors, if an input and +// output have the same mtime, we assume the output is up-to-date. +// This is the opposite of what the previous problem would have us do, +// but binary distributions are more common than instances of the +// previous problem. +// +// A variant of the last problem is that some binary distributions do not +// set the mtime on all files to the same time. Instead they let the file +// system record mtimes as the distribution is unpacked. If the outputs +// are unpacked before the inputs, they'll be older and a build will try +// to rebuild them. That rebuild might hit the same write errors as in +// the last scenario. We don't make any attempt to solve this, and we +// haven't had many reports of it. Perhaps the only time this happens is +// when people manually unpack the distribution, and most of the time +// that's done as the same user who will be using it, so an initial +// rebuild on first use succeeds quietly. +// +// More generally, people and programs change mtimes on files. The last +// few problems were specific examples of this, but it's a general problem. +// For example, instead of a binary distribution, copying a home +// directory from one directory or machine to another might copy files +// but not preserve mtimes. If the inputs are new than the outputs on the +// first machine but copied first, they end up older than the outputs on +// the second machine. +// +// Because many other build systems have the same sensitivity to mtimes, +// most programs manipulating source code take pains not to break the +// mtime assumptions. For example, Git does not set the mtime of files +// during a checkout operation, even when checking out an old version of +// the code. This decision was made specifically to work well with +// mtime-based build systems. +// +// The killer problem, though, for mtime-based build systems is that the +// build only has access to the mtimes of the inputs that still exist. +// If it is possible to remove an input without changing any other inputs, +// a later build will think the object is up-to-date when it is not. +// This happens for Go because a package is made up of all source +// files in a directory. If a source file is removed, there is no newer +// mtime available recording that fact. The mtime on the directory could +// be used, but it also changes when unrelated files are added to or +// removed from the directory, so including the directory mtime would +// cause unnecessary rebuilds, possibly many. It would also exacerbate +// the problems mentioned earlier, since even programs that are careful +// to maintain mtimes on files rarely maintain mtimes on directories. +// +// A variant of the last problem is when the inputs change for other +// reasons. For example, Go 1.4 and Go 1.5 both install $GOPATH/src/mypkg +// into the same target, $GOPATH/pkg/$GOOS_$GOARCH/mypkg.a. +// If Go 1.4 has built mypkg into mypkg.a, a build using Go 1.5 must +// rebuild mypkg.a, but from mtimes alone mypkg.a looks up-to-date. +// If Go 1.5 has just been installed, perhaps the compiler will have a +// newer mtime; since the compiler is considered an input, that would +// trigger a rebuild. But only once, and only the last Go 1.4 build of +// mypkg.a happened before Go 1.5 was installed. If a user has the two +// versions installed in different locations and flips back and forth, +// mtimes alone cannot tell what to do. Changing the toolchain is +// changing the set of inputs, without affecting any mtimes. +// +// To detect the set of inputs changing, we turn away from mtimes and to +// an explicit data comparison. Specifically, we build a list of the +// inputs to the build, compute its SHA1 hash, and record that as the +// ``build ID'' in the generated object. At the next build, we can +// recompute the build ID and compare it to the one in the generated +// object. If they differ, the list of inputs has changed, so the object +// is out of date and must be rebuilt. +// +// Because this build ID is computed before the build begins, the +// comparison does not have the race that mtime comparison does. +// +// Making the build sensitive to changes in other state is +// straightforward: include the state in the build ID hash, and if it +// changes, so does the build ID, triggering a rebuild. +// +// To detect changes in toolchain, we include the toolchain version in +// the build ID hash for package runtime, and then we include the build +// IDs of all imported packages in the build ID for p. +// +// It is natural to think about including build tags in the build ID, but +// the naive approach of just dumping the tags into the hash would cause +// spurious rebuilds. For example, 'go install' and 'go install -tags neverusedtag' +// produce the same binaries (assuming neverusedtag is never used). +// A more precise approach would be to include only tags that have an +// effect on the build. But the effect of a tag on the build is to +// include or exclude a file from the compilation, and that file list is +// already in the build ID hash. So the build ID is already tag-sensitive +// in a perfectly precise way. So we do NOT explicitly add build tags to +// the build ID hash. +// +// We do not include as part of the build ID the operating system, +// architecture, or whether the race detector is enabled, even though all +// three have an effect on the output, because that information is used +// to decide the install location. Binaries for linux and binaries for +// darwin are written to different directory trees; including that +// information in the build ID is unnecessary (although it would be +// harmless). +// +// TODO(rsc): Investigate the cost of putting source file content into +// the build ID hash as a replacement for the use of mtimes. Using the +// file content would avoid all the mtime problems, but it does require +// reading all the source files, something we avoid today (we read the +// beginning to find the build tags and the imports, but we stop as soon +// as we see the import block is over). If the package is stale, the compiler +// is going to read the files anyway. But if the package is up-to-date, the +// read is overhead. +// +// TODO(rsc): Investigate the complexity of making the build more +// precise about when individual results are needed. To be fully precise, +// there are two results of a compilation: the entire .a file used by the link +// and the subpiece used by later compilations (__.PKGDEF only). +// If a rebuild is needed but produces the previous __.PKGDEF, then +// no more recompilation due to the rebuilt package is needed, only +// relinking. To date, there is nothing in the Go command to express this. +// +// Special Cases +// +// When the go command makes the wrong build decision and does not +// rebuild something it should, users fall back to adding the -a flag. +// Any common use of the -a flag should be considered prima facie evidence +// that isStale is returning an incorrect false result in some important case. +// Bugs reported in the behavior of -a itself should prompt the question +// ``Why is -a being used at all? What bug does that indicate?'' +// +// There is a long history of changes to isStale to try to make -a into a +// suitable workaround for bugs in the mtime-based decisions. +// It is worth recording that history to inform (and, as much as possible, deter) future changes. +// +// (1) Before the build IDs were introduced, building with alternate tags +// would happily reuse installed objects built without those tags. +// For example, "go build -tags netgo myprog.go" would use the installed +// copy of package net, even if that copy had been built without netgo. +// (The netgo tag controls whether package net uses cgo or pure Go for +// functionality such as name resolution.) +// Using the installed non-netgo package defeats the purpose. +// +// Users worked around this with "go build -tags netgo -a myprog.go". +// +// Build IDs have made that workaround unnecessary: +// "go build -tags netgo myprog.go" +// cannot use a non-netgo copy of package net. +// +// (2) Before the build IDs were introduced, building with different toolchains, +// especially changing between toolchains, tried to reuse objects stored in +// $GOPATH/pkg, resulting in link-time errors about object file mismatches. +// +// Users worked around this with "go install -a ./...". +// +// Build IDs have made that workaround unnecessary: +// "go install ./..." will rebuild any objects it finds that were built against +// a different toolchain. +// +// (3) The common use of "go install -a ./..." led to reports of problems +// when the -a forced the rebuild of the standard library, which for some +// users was not writable. Because we didn't understand that the real +// problem was the bug -a was working around, we changed -a not to +// apply to the standard library. +// +// (4) The common use of "go build -tags netgo -a myprog.go" broke +// when we changed -a not to apply to the standard library, because +// if go build doesn't rebuild package net, it uses the non-netgo version. +// +// Users worked around this with "go build -tags netgo -installsuffix barf myprog.go". +// The -installsuffix here is making the go command look for packages +// in pkg/$GOOS_$GOARCH_barf instead of pkg/$GOOS_$GOARCH. +// Since the former presumably doesn't exist, go build decides to rebuild +// everything, including the standard library. Since go build doesn't +// install anything it builds, nothing is ever written to pkg/$GOOS_$GOARCH_barf, +// so repeated invocations continue to work. +// +// If the use of -a wasn't a red flag, the use of -installsuffix to point to +// a non-existent directory in a command that installs nothing should +// have been. +// +// (5) Now that (1) and (2) no longer need -a, we have removed the kludge +// introduced in (3): once again, -a means ``rebuild everything,'' not +// ``rebuild everything except the standard library.'' Only Go 1.4 had +// the restricted meaning. +// +// In addition to these cases trying to trigger rebuilds, there are +// special cases trying NOT to trigger rebuilds. The main one is that for +// a variety of reasons (see above), the install process for a Go release +// cannot be relied upon to set the mtimes such that the go command will +// think the standard library is up to date. So the mtime evidence is +// ignored for the standard library if we find ourselves in a release +// version of Go. Build ID-based staleness checks still apply to the +// standard library, even in release versions. This makes +// 'go build -tags netgo' work, among other things. + +// isStale reports whether package p needs to be rebuilt, +// along with the reason why. +func isStale(p *Package) (bool, string) { + if p.Standard && (p.ImportPath == "unsafe" || cfg.BuildContext.Compiler == "gccgo") { + // fake, builtin package + return false, "builtin package" + } + if p.Error != nil { + return true, "errors loading package" + } + if p.Stale { + return true, p.StaleReason + } + + // If this is a package with no source code, it cannot be rebuilt. + // If the binary is missing, we mark the package stale so that + // if a rebuild is needed, that rebuild attempt will produce a useful error. + // (Some commands, such as 'go list', do not attempt to rebuild.) + if p.BinaryOnly { + if p.Internal.Target == "" { + // Fail if a build is attempted. + return true, "no source code for package, but no install target" + } + if _, err := os.Stat(p.Internal.Target); err != nil { + // Fail if a build is attempted. + return true, "no source code for package, but cannot access install target: " + err.Error() + } + return false, "no source code for package" + } + + // If the -a flag is given, rebuild everything. + if cfg.BuildA { + return true, "build -a flag in use" + } + + // If there's no install target, we have to rebuild. + if p.Internal.Target == "" { + return true, "no install target" + } + + // Package is stale if completely unbuilt. + fi, err := os.Stat(p.Internal.Target) + if err != nil { + return true, "cannot stat install target" + } + + // Package is stale if the expected build ID differs from the + // recorded build ID. This catches changes like a source file + // being removed from a package directory. See issue 3895. + // It also catches changes in build tags that affect the set of + // files being compiled. See issue 9369. + // It also catches changes in toolchain, like when flipping between + // two versions of Go compiling a single GOPATH. + // See issue 8290 and issue 10702. + targetBuildID, err := buildid.ReadBuildID(p.Name, p.Target) + if err == nil && targetBuildID != p.Internal.BuildID { + return true, "build ID mismatch" + } + + // Package is stale if a dependency is. + for _, p1 := range p.Internal.Deps { + if p1.Stale { + return true, "stale dependency" + } + } + + // The checks above are content-based staleness. + // We assume they are always accurate. + // + // The checks below are mtime-based staleness. + // We hope they are accurate, but we know that they fail in the case of + // prebuilt Go installations that don't preserve the build mtimes + // (for example, if the pkg/ mtimes are before the src/ mtimes). + // See the large comment above isStale for details. + + // If we are running a release copy of Go and didn't find a content-based + // reason to rebuild the standard packages, do not rebuild them. + // They may not be writable anyway, but they are certainly not changing. + // This makes 'go build' skip the standard packages when + // using an official release, even when the mtimes have been changed. + // See issue 3036, issue 3149, issue 4106, issue 8290. + // (If a change to a release tree must be made by hand, the way to force the + // install is to run make.bash, which will remove the old package archives + // before rebuilding.) + if p.Standard && isGoRelease { + return false, "standard package in Go release distribution" + } + + // Time-based staleness. + + built := fi.ModTime() + + olderThan := func(file string) bool { + fi, err := os.Stat(file) + return err != nil || fi.ModTime().After(built) + } + + // Package is stale if a dependency is, or if a dependency is newer. + for _, p1 := range p.Internal.Deps { + if p1.Internal.Target != "" && olderThan(p1.Internal.Target) { + return true, "newer dependency" + } + } + + // As a courtesy to developers installing new versions of the compiler + // frequently, define that packages are stale if they are + // older than the compiler, and commands if they are older than + // the linker. This heuristic will not work if the binaries are + // back-dated, as some binary distributions may do, but it does handle + // a very common case. + // See issue 3036. + // Exclude $GOROOT, under the assumption that people working on + // the compiler may want to control when everything gets rebuilt, + // and people updating the Go repository will run make.bash or all.bash + // and get a full rebuild anyway. + // Excluding $GOROOT used to also fix issue 4106, but that's now + // taken care of above (at least when the installed Go is a released version). + if p.Root != cfg.GOROOT { + if olderThan(cfg.BuildToolchainCompiler()) { + return true, "newer compiler" + } + if p.Internal.Build.IsCommand() && olderThan(cfg.BuildToolchainLinker()) { + return true, "newer linker" + } + } + + // Note: Until Go 1.5, we had an additional shortcut here. + // We built a list of the workspace roots ($GOROOT, each $GOPATH) + // containing targets directly named on the command line, + // and if p were not in any of those, it would be treated as up-to-date + // as long as it is built. The goal was to avoid rebuilding a system-installed + // $GOROOT, unless something from $GOROOT were explicitly named + // on the command line (like go install math). + // That's now handled by the isGoRelease clause above. + // The other effect of the shortcut was to isolate different entries in + // $GOPATH from each other. This had the unfortunate effect that + // if you had (say), GOPATH listing two entries, one for commands + // and one for libraries, and you did a 'git pull' in the library one + // and then tried 'go install commands/...', it would build the new libraries + // during the first build (because they wouldn't have been installed at all) + // but then subsequent builds would not rebuild the libraries, even if the + // mtimes indicate they are stale, because the different GOPATH entries + // were treated differently. This behavior was confusing when using + // non-trivial GOPATHs, which were particularly common with some + // code management conventions, like the original godep. + // Since the $GOROOT case (the original motivation) is handled separately, + // we no longer put a barrier between the different $GOPATH entries. + // + // One implication of this is that if there is a system directory for + // non-standard Go packages that is included in $GOPATH, the mtimes + // on those compiled packages must be no earlier than the mtimes + // on the source files. Since most distributions use the same mtime + // for all files in a tree, they will be unaffected. People using plain + // tar x to extract system-installed packages will need to adjust mtimes, + // but it's better to force them to get the mtimes right than to ignore + // the mtimes and thereby do the wrong thing in common use cases. + // + // So there is no GOPATH vs GOPATH shortcut here anymore. + // + // If something needs to come back here, we could try writing a dummy + // file with a random name to the $GOPATH/pkg directory (and removing it) + // to test for write access, and then skip GOPATH roots we don't have write + // access to. But hopefully we can just use the mtimes always. + + srcs := str.StringList(p.GoFiles, p.CFiles, p.CXXFiles, p.MFiles, p.HFiles, p.FFiles, p.SFiles, p.CgoFiles, p.SysoFiles, p.SwigFiles, p.SwigCXXFiles) + for _, src := range srcs { + if olderThan(filepath.Join(p.Dir, src)) { + return true, "newer source file" + } + } + + return false, "" +} + +// computeBuildID computes the build ID for p, leaving it in p.Internal.BuildID. +// Build ID is a hash of the information we want to detect changes in. +// See the long comment in isStale for details. +func computeBuildID(p *Package) { + h := sha1.New() + + // Include the list of files compiled as part of the package. + // This lets us detect removed files. See issue 3895. + inputFiles := str.StringList( + p.GoFiles, + p.CgoFiles, + p.CFiles, + p.CXXFiles, + p.FFiles, + p.MFiles, + p.HFiles, + p.SFiles, + p.SysoFiles, + p.SwigFiles, + p.SwigCXXFiles, + ) + for _, file := range inputFiles { + fmt.Fprintf(h, "file %s\n", file) + } + + // Include the content of runtime/internal/sys/zversion.go in the hash + // for package runtime. This will give package runtime a + // different build ID in each Go release. + if p.Standard && p.ImportPath == "runtime/internal/sys" && cfg.BuildContext.Compiler != "gccgo" { + data, err := ioutil.ReadFile(filepath.Join(p.Dir, "zversion.go")) + if os.IsNotExist(err) { + p.Stale = true + p.StaleReason = fmt.Sprintf("missing zversion.go") + } else if err != nil { + base.Fatalf("go: %s", err) + } + fmt.Fprintf(h, "zversion %q\n", string(data)) + + // Add environment variables that affect code generation. + switch cfg.BuildContext.GOARCH { + case "arm": + fmt.Fprintf(h, "GOARM=%s\n", cfg.GOARM) + case "386": + fmt.Fprintf(h, "GO386=%s\n", cfg.GO386) + } + } + + // Include the build IDs of any dependencies in the hash. + // This, combined with the runtime/zversion content, + // will cause packages to have different build IDs when + // compiled with different Go releases. + // This helps the go command know to recompile when + // people use the same GOPATH but switch between + // different Go releases. See issue 10702. + // This is also a better fix for issue 8290. + for _, p1 := range p.Internal.Deps { + fmt.Fprintf(h, "dep %s %s\n", p1.ImportPath, p1.Internal.BuildID) + } + + p.Internal.BuildID = fmt.Sprintf("%x", h.Sum(nil)) +} + +var cmdCache = map[string]*Package{} + +func ClearCmdCache() { + for name := range cmdCache { + delete(cmdCache, name) + } +} + +// loadPackage is like loadImport but is used for command-line arguments, +// not for paths found in import statements. In addition to ordinary import paths, +// loadPackage accepts pseudo-paths beginning with cmd/ to denote commands +// in the Go command directory, as well as paths to those directories. +func LoadPackage(arg string, stk *ImportStack) *Package { + if build.IsLocalImport(arg) { + dir := arg + if !filepath.IsAbs(dir) { + if abs, err := filepath.Abs(dir); err == nil { + // interpret relative to current directory + dir = abs + } + } + if sub, ok := hasSubdir(cfg.GOROOTsrc, dir); ok && strings.HasPrefix(sub, "cmd/") && !strings.Contains(sub[4:], "/") { + arg = sub + } + } + if strings.HasPrefix(arg, "cmd/") && !strings.Contains(arg[4:], "/") { + if p := cmdCache[arg]; p != nil { + return p + } + stk.Push(arg) + defer stk.Pop() + + bp, err := cfg.BuildContext.ImportDir(filepath.Join(cfg.GOROOTsrc, arg), 0) + bp.ImportPath = arg + bp.Goroot = true + bp.BinDir = cfg.GOROOTbin + if cfg.GOROOTbin != "" { + bp.BinDir = cfg.GOROOTbin + } + bp.Root = cfg.GOROOT + bp.SrcRoot = cfg.GOROOTsrc + p := new(Package) + cmdCache[arg] = p + p.load(stk, bp, err) + if p.Error == nil && p.Name != "main" { + p.Error = &PackageError{ + ImportStack: stk.Copy(), + Err: fmt.Sprintf("expected package main but found package %s in %s", p.Name, p.Dir), + } + } + return p + } + + // Wasn't a command; must be a package. + // If it is a local import path but names a standard package, + // we treat it as if the user specified the standard package. + // This lets you run go test ./ioutil in package io and be + // referring to io/ioutil rather than a hypothetical import of + // "./ioutil". + if build.IsLocalImport(arg) { + bp, _ := cfg.BuildContext.ImportDir(filepath.Join(base.Cwd, arg), build.FindOnly) + if bp.ImportPath != "" && bp.ImportPath != "." { + arg = bp.ImportPath + } + } + + return LoadImport(arg, base.Cwd, nil, stk, nil, 0) +} + +// packages returns the packages named by the +// command line arguments 'args'. If a named package +// cannot be loaded at all (for example, if the directory does not exist), +// then packages prints an error and does not include that +// package in the results. However, if errors occur trying +// to load dependencies of a named package, the named +// package is still returned, with p.Incomplete = true +// and details in p.DepsErrors. +func Packages(args []string) []*Package { + var pkgs []*Package + for _, pkg := range PackagesAndErrors(args) { + if pkg.Error != nil { + base.Errorf("can't load package: %s", pkg.Error) + continue + } + pkgs = append(pkgs, pkg) + } + return pkgs +} + +// packagesAndErrors is like 'packages' but returns a +// *Package for every argument, even the ones that +// cannot be loaded at all. +// The packages that fail to load will have p.Error != nil. +func PackagesAndErrors(args []string) []*Package { + if len(args) > 0 && strings.HasSuffix(args[0], ".go") { + return []*Package{GoFilesPackage(args)} + } + + args = ImportPaths(args) + var ( + pkgs []*Package + stk ImportStack + seenArg = make(map[string]bool) + seenPkg = make(map[*Package]bool) + ) + + for _, arg := range args { + if seenArg[arg] { + continue + } + seenArg[arg] = true + pkg := LoadPackage(arg, &stk) + if seenPkg[pkg] { + continue + } + seenPkg[pkg] = true + pkgs = append(pkgs, pkg) + } + ComputeStale(pkgs...) + + return pkgs +} + +// packagesForBuild is like 'packages' but fails if any of +// the packages or their dependencies have errors +// (cannot be built). +func PackagesForBuild(args []string) []*Package { + pkgs := PackagesAndErrors(args) + printed := map[*PackageError]bool{} + for _, pkg := range pkgs { + if pkg.Error != nil { + base.Errorf("can't load package: %s", pkg.Error) + } + for _, err := range pkg.DepsErrors { + // Since these are errors in dependencies, + // the same error might show up multiple times, + // once in each package that depends on it. + // Only print each once. + if !printed[err] { + printed[err] = true + base.Errorf("%s", err) + } + } + } + base.ExitIfErrors() + + // Check for duplicate loads of the same package. + // That should be impossible, but if it does happen then + // we end up trying to build the same package twice, + // usually in parallel overwriting the same files, + // which doesn't work very well. + seen := map[string]bool{} + reported := map[string]bool{} + for _, pkg := range PackageList(pkgs) { + if seen[pkg.ImportPath] && !reported[pkg.ImportPath] { + reported[pkg.ImportPath] = true + base.Errorf("internal error: duplicate loads of %s", pkg.ImportPath) + } + seen[pkg.ImportPath] = true + } + base.ExitIfErrors() + + return pkgs +} + +// GoFilesPackage creates a package for building a collection of Go files +// (typically named on the command line). The target is named p.a for +// package p or named after the first Go file for package main. +func GoFilesPackage(gofiles []string) *Package { + // TODO: Remove this restriction. + for _, f := range gofiles { + if !strings.HasSuffix(f, ".go") { + base.Fatalf("named files must be .go files") + } + } + + var stk ImportStack + ctxt := cfg.BuildContext + ctxt.UseAllFiles = true + + // Synthesize fake "directory" that only shows the named files, + // to make it look like this is a standard package or + // command directory. So that local imports resolve + // consistently, the files must all be in the same directory. + var dirent []os.FileInfo + var dir string + for _, file := range gofiles { + fi, err := os.Stat(file) + if err != nil { + base.Fatalf("%s", err) + } + if fi.IsDir() { + base.Fatalf("%s is a directory, should be a Go file", file) + } + dir1, _ := filepath.Split(file) + if dir1 == "" { + dir1 = "./" + } + if dir == "" { + dir = dir1 + } else if dir != dir1 { + base.Fatalf("named files must all be in one directory; have %s and %s", dir, dir1) + } + dirent = append(dirent, fi) + } + ctxt.ReadDir = func(string) ([]os.FileInfo, error) { return dirent, nil } + + var err error + if dir == "" { + dir = base.Cwd + } + dir, err = filepath.Abs(dir) + if err != nil { + base.Fatalf("%s", err) + } + + bp, err := ctxt.ImportDir(dir, 0) + pkg := new(Package) + pkg.Internal.Local = true + pkg.Internal.Cmdline = true + stk.Push("main") + pkg.load(&stk, bp, err) + stk.Pop() + pkg.Internal.LocalPrefix = dirToImportPath(dir) + pkg.ImportPath = "command-line-arguments" + pkg.Internal.Target = "" + + if pkg.Name == "main" { + _, elem := filepath.Split(gofiles[0]) + exe := elem[:len(elem)-len(".go")] + cfg.ExeSuffix + if cfg.BuildO == "" { + cfg.BuildO = exe + } + if cfg.GOBIN != "" { + pkg.Internal.Target = filepath.Join(cfg.GOBIN, exe) + } + } + + pkg.Target = pkg.Internal.Target + pkg.Stale = true + pkg.StaleReason = "files named on command line" + + ComputeStale(pkg) + return pkg +} diff --git a/libgo/go/cmd/go/internal/load/search.go b/libgo/go/cmd/go/internal/load/search.go new file mode 100644 index 00000000000..0c7d9ce0e6c --- /dev/null +++ b/libgo/go/cmd/go/internal/load/search.go @@ -0,0 +1,338 @@ +// Copyright 2017 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package load + +import ( + "cmd/go/internal/cfg" + "fmt" + "go/build" + "log" + "os" + "path" + "path/filepath" + "regexp" + "strings" +) + +// allPackages returns all the packages that can be found +// under the $GOPATH directories and $GOROOT matching pattern. +// The pattern is either "all" (all packages), "std" (standard packages), +// "cmd" (standard commands), or a path including "...". +func allPackages(pattern string) []string { + pkgs := MatchPackages(pattern) + if len(pkgs) == 0 { + fmt.Fprintf(os.Stderr, "warning: %q matched no packages\n", pattern) + } + return pkgs +} + +// allPackagesInFS is like allPackages but is passed a pattern +// beginning ./ or ../, meaning it should scan the tree rooted +// at the given directory. There are ... in the pattern too. +func allPackagesInFS(pattern string) []string { + pkgs := MatchPackagesInFS(pattern) + if len(pkgs) == 0 { + fmt.Fprintf(os.Stderr, "warning: %q matched no packages\n", pattern) + } + return pkgs +} + +// MatchPackages returns a list of package paths matching pattern +// (see go help packages for pattern syntax). +func MatchPackages(pattern string) []string { + match := func(string) bool { return true } + treeCanMatch := func(string) bool { return true } + if !IsMetaPackage(pattern) { + match = matchPattern(pattern) + treeCanMatch = treeCanMatchPattern(pattern) + } + + have := map[string]bool{ + "builtin": true, // ignore pseudo-package that exists only for documentation + } + if !cfg.BuildContext.CgoEnabled { + have["runtime/cgo"] = true // ignore during walk + } + var pkgs []string + + for _, src := range cfg.BuildContext.SrcDirs() { + if (pattern == "std" || pattern == "cmd") && src != cfg.GOROOTsrc { + continue + } + src = filepath.Clean(src) + string(filepath.Separator) + root := src + if pattern == "cmd" { + root += "cmd" + string(filepath.Separator) + } + filepath.Walk(root, func(path string, fi os.FileInfo, err error) error { + if err != nil || path == src { + return nil + } + + want := true + // Avoid .foo, _foo, and testdata directory trees. + _, elem := filepath.Split(path) + if strings.HasPrefix(elem, ".") || strings.HasPrefix(elem, "_") || elem == "testdata" { + want = false + } + + name := filepath.ToSlash(path[len(src):]) + if pattern == "std" && (!isStandardImportPath(name) || name == "cmd") { + // The name "std" is only the standard library. + // If the name is cmd, it's the root of the command tree. + want = false + } + if !treeCanMatch(name) { + want = false + } + + if !fi.IsDir() { + if fi.Mode()&os.ModeSymlink != 0 && want { + if target, err := os.Stat(path); err == nil && target.IsDir() { + fmt.Fprintf(os.Stderr, "warning: ignoring symlink %s\n", path) + } + } + return nil + } + if !want { + return filepath.SkipDir + } + + if have[name] { + return nil + } + have[name] = true + if !match(name) { + return nil + } + pkg, err := cfg.BuildContext.ImportDir(path, 0) + if err != nil { + if _, noGo := err.(*build.NoGoError); noGo { + return nil + } + } + + // If we are expanding "cmd", skip main + // packages under cmd/vendor. At least as of + // March, 2017, there is one there for the + // vendored pprof tool. + if pattern == "cmd" && strings.HasPrefix(pkg.ImportPath, "cmd/vendor") && pkg.Name == "main" { + return nil + } + + pkgs = append(pkgs, name) + return nil + }) + } + return pkgs +} + +// MatchPackagesInFS returns a list of package paths matching pattern, +// which must begin with ./ or ../ +// (see go help packages for pattern syntax). +func MatchPackagesInFS(pattern string) []string { + // Find directory to begin the scan. + // Could be smarter but this one optimization + // is enough for now, since ... is usually at the + // end of a path. + i := strings.Index(pattern, "...") + dir, _ := path.Split(pattern[:i]) + + // pattern begins with ./ or ../. + // path.Clean will discard the ./ but not the ../. + // We need to preserve the ./ for pattern matching + // and in the returned import paths. + prefix := "" + if strings.HasPrefix(pattern, "./") { + prefix = "./" + } + match := matchPattern(pattern) + + var pkgs []string + filepath.Walk(dir, func(path string, fi os.FileInfo, err error) error { + if err != nil || !fi.IsDir() { + return nil + } + if path == dir { + // filepath.Walk starts at dir and recurses. For the recursive case, + // the path is the result of filepath.Join, which calls filepath.Clean. + // The initial case is not Cleaned, though, so we do this explicitly. + // + // This converts a path like "./io/" to "io". Without this step, running + // "cd $GOROOT/src; go list ./io/..." would incorrectly skip the io + // package, because prepending the prefix "./" to the unclean path would + // result in "././io", and match("././io") returns false. + path = filepath.Clean(path) + } + + // Avoid .foo, _foo, and testdata directory trees, but do not avoid "." or "..". + _, elem := filepath.Split(path) + dot := strings.HasPrefix(elem, ".") && elem != "." && elem != ".." + if dot || strings.HasPrefix(elem, "_") || elem == "testdata" { + return filepath.SkipDir + } + + name := prefix + filepath.ToSlash(path) + if !match(name) { + return nil + } + + // We keep the directory if we can import it, or if we can't import it + // due to invalid Go source files. This means that directories containing + // parse errors will be built (and fail) instead of being silently skipped + // as not matching the pattern. Go 1.5 and earlier skipped, but that + // behavior means people miss serious mistakes. + // See golang.org/issue/11407. + if p, err := cfg.BuildContext.ImportDir(path, 0); err != nil && (p == nil || len(p.InvalidGoFiles) == 0) { + if _, noGo := err.(*build.NoGoError); !noGo { + log.Print(err) + } + return nil + } + pkgs = append(pkgs, name) + return nil + }) + return pkgs +} + +// treeCanMatchPattern(pattern)(name) reports whether +// name or children of name can possibly match pattern. +// Pattern is the same limited glob accepted by matchPattern. +func treeCanMatchPattern(pattern string) func(name string) bool { + wildCard := false + if i := strings.Index(pattern, "..."); i >= 0 { + wildCard = true + pattern = pattern[:i] + } + return func(name string) bool { + return len(name) <= len(pattern) && hasPathPrefix(pattern, name) || + wildCard && strings.HasPrefix(name, pattern) + } +} + +// matchPattern(pattern)(name) reports whether +// name matches pattern. Pattern is a limited glob +// pattern in which '...' means 'any string' and there +// is no other special syntax. +// Unfortunately, there are two special cases. Quoting "go help packages": +// +// First, /... at the end of the pattern can match an empty string, +// so that net/... matches both net and packages in its subdirectories, like net/http. +// Second, any slash-separted pattern element containing a wildcard never +// participates in a match of the "vendor" element in the path of a vendored +// package, so that ./... does not match packages in subdirectories of +// ./vendor or ./mycode/vendor, but ./vendor/... and ./mycode/vendor/... do. +// Note, however, that a directory named vendor that itself contains code +// is not a vendored package: cmd/vendor would be a command named vendor, +// and the pattern cmd/... matches it. +func matchPattern(pattern string) func(name string) bool { + // Convert pattern to regular expression. + // The strategy for the trailing /... is to nest it in an explicit ? expression. + // The strategy for the vendor exclusion is to change the unmatchable + // vendor strings to a disallowed code point (vendorChar) and to use + // "(anything but that codepoint)*" as the implementation of the ... wildcard. + // This is a bit complicated but the obvious alternative, + // namely a hand-written search like in most shell glob matchers, + // is too easy to make accidentally exponential. + // Using package regexp guarantees linear-time matching. + + const vendorChar = "\x00" + + if strings.Contains(pattern, vendorChar) { + return func(name string) bool { return false } + } + + re := regexp.QuoteMeta(pattern) + re = replaceVendor(re, vendorChar) + switch { + case strings.HasSuffix(re, `/`+vendorChar+`/\.\.\.`): + re = strings.TrimSuffix(re, `/`+vendorChar+`/\.\.\.`) + `(/vendor|/` + vendorChar + `/\.\.\.)` + case re == vendorChar+`/\.\.\.`: + re = `(/vendor|/` + vendorChar + `/\.\.\.)` + case strings.HasSuffix(re, `/\.\.\.`): + re = strings.TrimSuffix(re, `/\.\.\.`) + `(/\.\.\.)?` + } + re = strings.Replace(re, `\.\.\.`, `[^`+vendorChar+`]*`, -1) + + reg := regexp.MustCompile(`^` + re + `$`) + + return func(name string) bool { + if strings.Contains(name, vendorChar) { + return false + } + return reg.MatchString(replaceVendor(name, vendorChar)) + } +} + +// replaceVendor returns the result of replacing +// non-trailing vendor path elements in x with repl. +func replaceVendor(x, repl string) string { + if !strings.Contains(x, "vendor") { + return x + } + elem := strings.Split(x, "/") + for i := 0; i < len(elem)-1; i++ { + if elem[i] == "vendor" { + elem[i] = repl + } + } + return strings.Join(elem, "/") +} + +// ImportPaths returns the import paths to use for the given command line. +func ImportPaths(args []string) []string { + args = ImportPathsNoDotExpansion(args) + var out []string + for _, a := range args { + if strings.Contains(a, "...") { + if build.IsLocalImport(a) { + out = append(out, allPackagesInFS(a)...) + } else { + out = append(out, allPackages(a)...) + } + continue + } + out = append(out, a) + } + return out +} + +// ImportPathsNoDotExpansion returns the import paths to use for the given +// command line, but it does no ... expansion. +func ImportPathsNoDotExpansion(args []string) []string { + if len(args) == 0 { + return []string{"."} + } + var out []string + for _, a := range args { + // Arguments are supposed to be import paths, but + // as a courtesy to Windows developers, rewrite \ to / + // in command-line arguments. Handles .\... and so on. + if filepath.Separator == '\\' { + a = strings.Replace(a, `\`, `/`, -1) + } + + // Put argument in canonical form, but preserve leading ./. + if strings.HasPrefix(a, "./") { + a = "./" + path.Clean(a) + if a == "./." { + a = "." + } + } else { + a = path.Clean(a) + } + if IsMetaPackage(a) { + out = append(out, allPackages(a)...) + continue + } + out = append(out, a) + } + return out +} + +// isMetaPackage checks if name is a reserved package name that expands to multiple packages. +func IsMetaPackage(name string) bool { + return name == "std" || name == "cmd" || name == "all" +} diff --git a/libgo/go/cmd/go/internal/load/testgo.go b/libgo/go/cmd/go/internal/load/testgo.go new file mode 100644 index 00000000000..7734048f5c9 --- /dev/null +++ b/libgo/go/cmd/go/internal/load/testgo.go @@ -0,0 +1,21 @@ +// Copyright 2014 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// This file contains extra hooks for testing the go command. +// It is compiled into the Go binary only when building the +// test copy; it does not get compiled into the standard go +// command, so these testing hooks are not present in the +// go command that everyone uses. + +// +build testgo + +package load + +import "os" + +func init() { + if v := os.Getenv("TESTGO_IS_GO_RELEASE"); v != "" { + isGoRelease = v == "1" + } +} diff --git a/libgo/go/cmd/go/internal/run/run.go b/libgo/go/cmd/go/internal/run/run.go new file mode 100644 index 00000000000..6e276c28ec1 --- /dev/null +++ b/libgo/go/cmd/go/internal/run/run.go @@ -0,0 +1,131 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package run implements the ``go run'' command. +package run + +import ( + "fmt" + "os" + "strings" + + "cmd/go/internal/base" + "cmd/go/internal/cfg" + "cmd/go/internal/load" + "cmd/go/internal/str" + "cmd/go/internal/work" +) + +var CmdRun = &base.Command{ + UsageLine: "run [build flags] [-exec xprog] gofiles... [arguments...]", + Short: "compile and run Go program", + Long: ` +Run compiles and runs the main package comprising the named Go source files. +A Go source file is defined to be a file ending in a literal ".go" suffix. + +By default, 'go run' runs the compiled binary directly: 'a.out arguments...'. +If the -exec flag is given, 'go run' invokes the binary using xprog: + 'xprog a.out arguments...'. +If the -exec flag is not given, GOOS or GOARCH is different from the system +default, and a program named go_$GOOS_$GOARCH_exec can be found +on the current search path, 'go run' invokes the binary using that program, +for example 'go_nacl_386_exec a.out arguments...'. This allows execution of +cross-compiled programs when a simulator or other execution method is +available. + +For more about build flags, see 'go help build'. + +See also: go build. + `, +} + +func init() { + CmdRun.Run = runRun // break init loop + + work.AddBuildFlags(CmdRun) + CmdRun.Flag.Var((*base.StringsFlag)(&work.ExecCmd), "exec", "") +} + +func printStderr(args ...interface{}) (int, error) { + return fmt.Fprint(os.Stderr, args...) +} + +func runRun(cmd *base.Command, args []string) { + work.InstrumentInit() + work.BuildModeInit() + var b work.Builder + b.Init() + b.Print = printStderr + i := 0 + for i < len(args) && strings.HasSuffix(args[i], ".go") { + i++ + } + files, cmdArgs := args[:i], args[i:] + if len(files) == 0 { + base.Fatalf("go run: no go files listed") + } + for _, file := range files { + if strings.HasSuffix(file, "_test.go") { + // GoFilesPackage is going to assign this to TestGoFiles. + // Reject since it won't be part of the build. + base.Fatalf("go run: cannot run *_test.go files (%s)", file) + } + } + p := load.GoFilesPackage(files) + if p.Error != nil { + base.Fatalf("%s", p.Error) + } + p.Internal.OmitDebug = true + if len(p.DepsErrors) > 0 { + // Since these are errors in dependencies, + // the same error might show up multiple times, + // once in each package that depends on it. + // Only print each once. + printed := map[*load.PackageError]bool{} + for _, err := range p.DepsErrors { + if !printed[err] { + printed[err] = true + base.Errorf("%s", err) + } + } + } + base.ExitIfErrors() + if p.Name != "main" { + base.Fatalf("go run: cannot run non-main package") + } + p.Internal.Target = "" // must build - not up to date + var src string + if len(p.GoFiles) > 0 { + src = p.GoFiles[0] + } else if len(p.CgoFiles) > 0 { + src = p.CgoFiles[0] + } else { + // this case could only happen if the provided source uses cgo + // while cgo is disabled. + hint := "" + if !cfg.BuildContext.CgoEnabled { + hint = " (cgo is disabled)" + } + base.Fatalf("go run: no suitable source files%s", hint) + } + p.Internal.ExeName = src[:len(src)-len(".go")] // name temporary executable for first go file + a1 := b.Action(work.ModeBuild, work.ModeBuild, p) + a := &work.Action{Func: buildRunProgram, Args: cmdArgs, Deps: []*work.Action{a1}} + b.Do(a) +} + +// buildRunProgram is the action for running a binary that has already +// been compiled. We ignore exit status. +func buildRunProgram(b *work.Builder, a *work.Action) error { + cmdline := str.StringList(work.FindExecCmd(), a.Deps[0].Target, a.Args) + if cfg.BuildN || cfg.BuildX { + b.Showcmd("", "%s", strings.Join(cmdline, " ")) + if cfg.BuildN { + return nil + } + } + + base.RunStdin(cmdline) + return nil +} diff --git a/libgo/go/cmd/go/internal/str/str.go b/libgo/go/cmd/go/internal/str/str.go new file mode 100644 index 00000000000..0413ed8e69e --- /dev/null +++ b/libgo/go/cmd/go/internal/str/str.go @@ -0,0 +1,141 @@ +// Copyright 2017 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package str provides string manipulation utilities. +package str + +import ( + "bytes" + "fmt" + "unicode" + "unicode/utf8" +) + +// StringList flattens its arguments into a single []string. +// Each argument in args must have type string or []string. +func StringList(args ...interface{}) []string { + var x []string + for _, arg := range args { + switch arg := arg.(type) { + case []string: + x = append(x, arg...) + case string: + x = append(x, arg) + default: + panic("stringList: invalid argument of type " + fmt.Sprintf("%T", arg)) + } + } + return x +} + +// ToFold returns a string with the property that +// strings.EqualFold(s, t) iff ToFold(s) == ToFold(t) +// This lets us test a large set of strings for fold-equivalent +// duplicates without making a quadratic number of calls +// to EqualFold. Note that strings.ToUpper and strings.ToLower +// do not have the desired property in some corner cases. +func ToFold(s string) string { + // Fast path: all ASCII, no upper case. + // Most paths look like this already. + for i := 0; i < len(s); i++ { + c := s[i] + if c >= utf8.RuneSelf || 'A' <= c && c <= 'Z' { + goto Slow + } + } + return s + +Slow: + var buf bytes.Buffer + for _, r := range s { + // SimpleFold(x) cycles to the next equivalent rune > x + // or wraps around to smaller values. Iterate until it wraps, + // and we've found the minimum value. + for { + r0 := r + r = unicode.SimpleFold(r0) + if r <= r0 { + break + } + } + // Exception to allow fast path above: A-Z => a-z + if 'A' <= r && r <= 'Z' { + r += 'a' - 'A' + } + buf.WriteRune(r) + } + return buf.String() +} + +// FoldDup reports a pair of strings from the list that are +// equal according to strings.EqualFold. +// It returns "", "" if there are no such strings. +func FoldDup(list []string) (string, string) { + clash := map[string]string{} + for _, s := range list { + fold := ToFold(s) + if t := clash[fold]; t != "" { + if s > t { + s, t = t, s + } + return s, t + } + clash[fold] = s + } + return "", "" +} + +// Contains reports whether x contains s. +func Contains(x []string, s string) bool { + for _, t := range x { + if t == s { + return true + } + } + return false +} + +func isSpaceByte(c byte) bool { + return c == ' ' || c == '\t' || c == '\n' || c == '\r' +} + +// SplitQuotedFields splits s into a list of fields, +// allowing single or double quotes around elements. +// There is no unescaping or other processing within +// quoted fields. +func SplitQuotedFields(s string) ([]string, error) { + // Split fields allowing '' or "" around elements. + // Quotes further inside the string do not count. + var f []string + for len(s) > 0 { + for len(s) > 0 && isSpaceByte(s[0]) { + s = s[1:] + } + if len(s) == 0 { + break + } + // Accepted quoted string. No unescaping inside. + if s[0] == '"' || s[0] == '\'' { + quote := s[0] + s = s[1:] + i := 0 + for i < len(s) && s[i] != quote { + i++ + } + if i >= len(s) { + return nil, fmt.Errorf("unterminated %c string", quote) + } + f = append(f, s[:i]) + s = s[i+1:] + continue + } + i := 0 + for i < len(s) && !isSpaceByte(s[i]) { + i++ + } + f = append(f, s[:i]) + s = s[i:] + } + return f, nil +} diff --git a/libgo/go/cmd/go/internal/test/test.go b/libgo/go/cmd/go/internal/test/test.go new file mode 100644 index 00000000000..c1e66f9e889 --- /dev/null +++ b/libgo/go/cmd/go/internal/test/test.go @@ -0,0 +1,1625 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package test + +import ( + "bytes" + "errors" + "fmt" + "go/ast" + "go/build" + "go/doc" + "go/parser" + "go/token" + "os" + "os/exec" + "path" + "path/filepath" + "regexp" + "runtime" + "sort" + "strings" + "text/template" + "time" + "unicode" + "unicode/utf8" + + "cmd/go/internal/base" + "cmd/go/internal/cfg" + "cmd/go/internal/load" + "cmd/go/internal/str" + "cmd/go/internal/work" +) + +// Break init loop. +func init() { + CmdTest.Run = runTest +} + +const testUsage = "test [build/test flags] [packages] [build/test flags & test binary flags]" + +var CmdTest = &base.Command{ + CustomFlags: true, + UsageLine: testUsage, + Short: "test packages", + Long: ` +'Go test' automates testing the packages named by the import paths. +It prints a summary of the test results in the format: + + ok archive/tar 0.011s + FAIL archive/zip 0.022s + ok compress/gzip 0.033s + ... + +followed by detailed output for each failed package. + +'Go test' recompiles each package along with any files with names matching +the file pattern "*_test.go". +Files whose names begin with "_" (including "_test.go") or "." are ignored. +These additional files can contain test functions, benchmark functions, and +example functions. See 'go help testfunc' for more. +Each listed package causes the execution of a separate test binary. + +Test files that declare a package with the suffix "_test" will be compiled as a +separate package, and then linked and run with the main test binary. + +The go tool will ignore a directory named "testdata", making it available +to hold ancillary data needed by the tests. + +By default, go test needs no arguments. It compiles and tests the package +with source in the current directory, including tests, and runs the tests. + +The package is built in a temporary directory so it does not interfere with the +non-test installation. + +` + strings.TrimSpace(testFlag1) + ` See 'go help testflag' for details. + +For more about build flags, see 'go help build'. +For more about specifying packages, see 'go help packages'. + +See also: go build, go vet. +`, +} + +const testFlag1 = ` +In addition to the build flags, the flags handled by 'go test' itself are: + + -args + Pass the remainder of the command line (everything after -args) + to the test binary, uninterpreted and unchanged. + Because this flag consumes the remainder of the command line, + the package list (if present) must appear before this flag. + + -c + Compile the test binary to pkg.test but do not run it + (where pkg is the last element of the package's import path). + The file name can be changed with the -o flag. + + -exec xprog + Run the test binary using xprog. The behavior is the same as + in 'go run'. See 'go help run' for details. + + -i + Install packages that are dependencies of the test. + Do not run the test. + + -o file + Compile the test binary to the named file. + The test still runs (unless -c or -i is specified). + +The test binary also accepts flags that control execution of the test; these +flags are also accessible by 'go test'. +` + +// Usage prints the usage message for 'go test -h' and exits. +func Usage() { + os.Stderr.WriteString(testUsage + "\n\n" + + strings.TrimSpace(testFlag1) + "\n\n\t" + + strings.TrimSpace(testFlag2) + "\n") + os.Exit(2) +} + +var HelpTestflag = &base.Command{ + UsageLine: "testflag", + Short: "description of testing flags", + Long: ` +The 'go test' command takes both flags that apply to 'go test' itself +and flags that apply to the resulting test binary. + +Several of the flags control profiling and write an execution profile +suitable for "go tool pprof"; run "go tool pprof -h" for more +information. The --alloc_space, --alloc_objects, and --show_bytes +options of pprof control how the information is presented. + +The following flags are recognized by the 'go test' command and +control the execution of any test: + + ` + strings.TrimSpace(testFlag2) + ` +`, +} + +const testFlag2 = ` + -bench regexp + Run only those benchmarks matching a regular expression. + By default, no benchmarks are run. + To run all benchmarks, use '-bench .' or '-bench=.'. + The regular expression is split by unbracketed slash (/) + characters into a sequence of regular expressions, and each + part of a benchmark's identifier must match the corresponding + element in the sequence, if any. Possible parents of matches + are run with b.N=1 to identify sub-benchmarks. For example, + given -bench=X/Y, top-level benchmarks matching X are run + with b.N=1 to find any sub-benchmarks matching Y, which are + then run in full. + + -benchtime t + Run enough iterations of each benchmark to take t, specified + as a time.Duration (for example, -benchtime 1h30s). + The default is 1 second (1s). + + -count n + Run each test and benchmark n times (default 1). + If -cpu is set, run n times for each GOMAXPROCS value. + Examples are always run once. + + -cover + Enable coverage analysis. + Note that because coverage works by annotating the source + code before compilation, compilation and test failures with + coverage enabled may report line numbers that don't correspond + to the original sources. + + -covermode set,count,atomic + Set the mode for coverage analysis for the package[s] + being tested. The default is "set" unless -race is enabled, + in which case it is "atomic". + The values: + set: bool: does this statement run? + count: int: how many times does this statement run? + atomic: int: count, but correct in multithreaded tests; + significantly more expensive. + Sets -cover. + + -coverpkg pkg1,pkg2,pkg3 + Apply coverage analysis in each test to the given list of packages. + The default is for each test to analyze only the package being tested. + Packages are specified as import paths. + Sets -cover. + + -cpu 1,2,4 + Specify a list of GOMAXPROCS values for which the tests or + benchmarks should be executed. The default is the current value + of GOMAXPROCS. + + -list regexp + List tests, benchmarks, or examples matching the regular expression. + No tests, benchmarks or examples will be run. This will only + list top-level tests. No subtest or subbenchmarks will be shown. + + -parallel n + Allow parallel execution of test functions that call t.Parallel. + The value of this flag is the maximum number of tests to run + simultaneously; by default, it is set to the value of GOMAXPROCS. + Note that -parallel only applies within a single test binary. + The 'go test' command may run tests for different packages + in parallel as well, according to the setting of the -p flag + (see 'go help build'). + + -run regexp + Run only those tests and examples matching the regular expression. + For tests, the regular expression is split by unbracketed slash (/) + characters into a sequence of regular expressions, and each part + of a test's identifier must match the corresponding element in + the sequence, if any. Note that possible parents of matches are + run too, so that -run=X/Y matches and runs and reports the result + of all tests matching X, even those without sub-tests matching Y, + because it must run them to look for those sub-tests. + + -short + Tell long-running tests to shorten their run time. + It is off by default but set during all.bash so that installing + the Go tree can run a sanity check but not spend time running + exhaustive tests. + + -timeout d + If a test binary runs longer than duration d, panic. + The default is 10 minutes (10m). + + -v + Verbose output: log all tests as they are run. Also print all + text from Log and Logf calls even if the test succeeds. + +The following flags are also recognized by 'go test' and can be used to +profile the tests during execution: + + -benchmem + Print memory allocation statistics for benchmarks. + + -blockprofile block.out + Write a goroutine blocking profile to the specified file + when all tests are complete. + Writes test binary as -c would. + + -blockprofilerate n + Control the detail provided in goroutine blocking profiles by + calling runtime.SetBlockProfileRate with n. + See 'go doc runtime.SetBlockProfileRate'. + The profiler aims to sample, on average, one blocking event every + n nanoseconds the program spends blocked. By default, + if -test.blockprofile is set without this flag, all blocking events + are recorded, equivalent to -test.blockprofilerate=1. + + -coverprofile cover.out + Write a coverage profile to the file after all tests have passed. + Sets -cover. + + -cpuprofile cpu.out + Write a CPU profile to the specified file before exiting. + Writes test binary as -c would. + + -memprofile mem.out + Write a memory profile to the file after all tests have passed. + Writes test binary as -c would. + + -memprofilerate n + Enable more precise (and expensive) memory profiles by setting + runtime.MemProfileRate. See 'go doc runtime.MemProfileRate'. + To profile all memory allocations, use -test.memprofilerate=1 + and pass --alloc_space flag to the pprof tool. + + -mutexprofile mutex.out + Write a mutex contention profile to the specified file + when all tests are complete. + Writes test binary as -c would. + + -mutexprofilefraction n + Sample 1 in n stack traces of goroutines holding a + contended mutex. + + -outputdir directory + Place output files from profiling in the specified directory, + by default the directory in which "go test" is running. + + -trace trace.out + Write an execution trace to the specified file before exiting. + +Each of these flags is also recognized with an optional 'test.' prefix, +as in -test.v. When invoking the generated test binary (the result of +'go test -c') directly, however, the prefix is mandatory. + +The 'go test' command rewrites or removes recognized flags, +as appropriate, both before and after the optional package list, +before invoking the test binary. + +For instance, the command + + go test -v -myflag testdata -cpuprofile=prof.out -x + +will compile the test binary and then run it as + + pkg.test -test.v -myflag testdata -test.cpuprofile=prof.out + +(The -x flag is removed because it applies only to the go command's +execution, not to the test itself.) + +The test flags that generate profiles (other than for coverage) also +leave the test binary in pkg.test for use when analyzing the profiles. + +When 'go test' runs a test binary, it does so from within the +corresponding package's source code directory. Depending on the test, +it may be necessary to do the same when invoking a generated test +binary directly. + +The command-line package list, if present, must appear before any +flag not known to the go test command. Continuing the example above, +the package list would have to appear before -myflag, but could appear +on either side of -v. + +To keep an argument for a test binary from being interpreted as a +known flag or a package name, use -args (see 'go help test') which +passes the remainder of the command line through to the test binary +uninterpreted and unaltered. + +For instance, the command + + go test -v -args -x -v + +will compile the test binary and then run it as + + pkg.test -test.v -x -v + +Similarly, + + go test -args math + +will compile the test binary and then run it as + + pkg.test math + +In the first example, the -x and the second -v are passed through to the +test binary unchanged and with no effect on the go command itself. +In the second example, the argument math is passed through to the test +binary, instead of being interpreted as the package list. +` + +var HelpTestfunc = &base.Command{ + UsageLine: "testfunc", + Short: "description of testing functions", + Long: ` +The 'go test' command expects to find test, benchmark, and example functions +in the "*_test.go" files corresponding to the package under test. + +A test function is one named TestXXX (where XXX is any alphanumeric string +not starting with a lower case letter) and should have the signature, + + func TestXXX(t *testing.T) { ... } + +A benchmark function is one named BenchmarkXXX and should have the signature, + + func BenchmarkXXX(b *testing.B) { ... } + +An example function is similar to a test function but, instead of using +*testing.T to report success or failure, prints output to os.Stdout. +If the last comment in the function starts with "Output:" then the output +is compared exactly against the comment (see examples below). If the last +comment begins with "Unordered output:" then the output is compared to the +comment, however the order of the lines is ignored. An example with no such +comment is compiled but not executed. An example with no text after +"Output:" is compiled, executed, and expected to produce no output. + +Godoc displays the body of ExampleXXX to demonstrate the use +of the function, constant, or variable XXX. An example of a method M with +receiver type T or *T is named ExampleT_M. There may be multiple examples +for a given function, constant, or variable, distinguished by a trailing _xxx, +where xxx is a suffix not beginning with an upper case letter. + +Here is an example of an example: + + func ExamplePrintln() { + Println("The output of\nthis example.") + // Output: The output of + // this example. + } + +Here is another example where the ordering of the output is ignored: + + func ExamplePerm() { + for _, value := range Perm(4) { + fmt.Println(value) + } + + // Unordered output: 4 + // 2 + // 1 + // 3 + // 0 + } + +The entire test file is presented as the example when it contains a single +example function, at least one other function, type, variable, or constant +declaration, and no test or benchmark functions. + +See the documentation of the testing package for more information. +`, +} + +var ( + testC bool // -c flag + testCover bool // -cover flag + testCoverMode string // -covermode flag + testCoverPaths []string // -coverpkg flag + testCoverPkgs []*load.Package // -coverpkg flag + testO string // -o flag + testProfile bool // some profiling flag + testNeedBinary bool // profile needs to keep binary around + testV bool // -v flag + testTimeout string // -timeout flag + testArgs []string + testBench bool + testList bool + testStreamOutput bool // show output as it is generated + testShowPass bool // show passing output + + testKillTimeout = 10 * time.Minute +) + +var testMainDeps = map[string]bool{ + // Dependencies for testmain. + "testing": true, + "testing/internal/testdeps": true, + "os": true, +} + +func runTest(cmd *base.Command, args []string) { + var pkgArgs []string + pkgArgs, testArgs = testFlags(args) + + work.FindExecCmd() // initialize cached result + + work.InstrumentInit() + work.BuildModeInit() + pkgs := load.PackagesForBuild(pkgArgs) + if len(pkgs) == 0 { + base.Fatalf("no packages to test") + } + + if testC && len(pkgs) != 1 { + base.Fatalf("cannot use -c flag with multiple packages") + } + if testO != "" && len(pkgs) != 1 { + base.Fatalf("cannot use -o flag with multiple packages") + } + if testProfile && len(pkgs) != 1 { + base.Fatalf("cannot use test profile flag with multiple packages") + } + + // If a test timeout was given and is parseable, set our kill timeout + // to that timeout plus one minute. This is a backup alarm in case + // the test wedges with a goroutine spinning and its background + // timer does not get a chance to fire. + if dt, err := time.ParseDuration(testTimeout); err == nil && dt > 0 { + testKillTimeout = dt + 1*time.Minute + } + + // show passing test output (after buffering) with -v flag. + // must buffer because tests are running in parallel, and + // otherwise the output will get mixed. + testShowPass = testV || testList + + // stream test output (no buffering) when no package has + // been given on the command line (implicit current directory) + // or when benchmarking. + // Also stream if we're showing output anyway with a + // single package under test or if parallelism is set to 1. + // In these cases, streaming the output produces the same result + // as not streaming, just more immediately. + testStreamOutput = len(pkgArgs) == 0 || testBench || + (testShowPass && (len(pkgs) == 1 || cfg.BuildP == 1)) + + // For 'go test -i -o x.test', we want to build x.test. Imply -c to make the logic easier. + if cfg.BuildI && testO != "" { + testC = true + } + + var b work.Builder + b.Init() + + if cfg.BuildI { + cfg.BuildV = testV + + deps := make(map[string]bool) + for dep := range testMainDeps { + deps[dep] = true + } + + for _, p := range pkgs { + // Dependencies for each test. + for _, path := range p.Imports { + deps[path] = true + } + for _, path := range p.Vendored(p.TestImports) { + deps[path] = true + } + for _, path := range p.Vendored(p.XTestImports) { + deps[path] = true + } + } + + // translate C to runtime/cgo + if deps["C"] { + delete(deps, "C") + deps["runtime/cgo"] = true + if cfg.Goos == runtime.GOOS && cfg.Goarch == runtime.GOARCH && !cfg.BuildRace && !cfg.BuildMSan { + deps["cmd/cgo"] = true + } + } + // Ignore pseudo-packages. + delete(deps, "unsafe") + + all := []string{} + for path := range deps { + if !build.IsLocalImport(path) { + all = append(all, path) + } + } + sort.Strings(all) + + a := &work.Action{} + for _, p := range load.PackagesForBuild(all) { + if cfg.BuildToolchainName == "gccgo" && p.Standard { + continue + } + a.Deps = append(a.Deps, b.Action(work.ModeInstall, work.ModeInstall, p)) + } + b.Do(a) + if !testC || a.Failed { + return + } + b.Init() + } + + var builds, runs, prints []*work.Action + + if testCoverPaths != nil { + // Load packages that were asked about for coverage. + // packagesForBuild exits if the packages cannot be loaded. + testCoverPkgs = load.PackagesForBuild(testCoverPaths) + + // Warn about -coverpkg arguments that are not actually used. + used := make(map[string]bool) + for _, p := range pkgs { + used[p.ImportPath] = true + for _, dep := range p.Deps { + used[dep] = true + } + } + for _, p := range testCoverPkgs { + if !used[p.ImportPath] { + fmt.Fprintf(os.Stderr, "warning: no packages being tested depend on %s\n", p.ImportPath) + } + } + + // Mark all the coverage packages for rebuilding with coverage. + for _, p := range testCoverPkgs { + // There is nothing to cover in package unsafe; it comes from the compiler. + if p.ImportPath == "unsafe" { + continue + } + p.Stale = true // rebuild + p.StaleReason = "rebuild for coverage" + p.Internal.Fake = true // do not warn about rebuild + p.Internal.CoverMode = testCoverMode + var coverFiles []string + coverFiles = append(coverFiles, p.GoFiles...) + coverFiles = append(coverFiles, p.CgoFiles...) + coverFiles = append(coverFiles, p.TestGoFiles...) + p.Internal.CoverVars = declareCoverVars(p.ImportPath, coverFiles...) + } + } + + // Prepare build + run + print actions for all packages being tested. + for _, p := range pkgs { + // sync/atomic import is inserted by the cover tool. See #18486 + if testCover && testCoverMode == "atomic" { + ensureImport(p, "sync/atomic") + } + + buildTest, runTest, printTest, err := builderTest(&b, p) + if err != nil { + str := err.Error() + if strings.HasPrefix(str, "\n") { + str = str[1:] + } + failed := fmt.Sprintf("FAIL\t%s [setup failed]\n", p.ImportPath) + + if p.ImportPath != "" { + base.Errorf("# %s\n%s\n%s", p.ImportPath, str, failed) + } else { + base.Errorf("%s\n%s", str, failed) + } + continue + } + builds = append(builds, buildTest) + runs = append(runs, runTest) + prints = append(prints, printTest) + } + + // Ultimately the goal is to print the output. + root := &work.Action{Deps: prints} + + // Force the printing of results to happen in order, + // one at a time. + for i, a := range prints { + if i > 0 { + a.Deps = append(a.Deps, prints[i-1]) + } + } + + // Force benchmarks to run in serial. + if !testC && testBench { + // The first run must wait for all builds. + // Later runs must wait for the previous run's print. + for i, run := range runs { + if i == 0 { + run.Deps = append(run.Deps, builds...) + } else { + run.Deps = append(run.Deps, prints[i-1]) + } + } + } + + // If we are building any out-of-date packages other + // than those under test, warn. + okBuild := map[*load.Package]bool{} + for _, p := range pkgs { + okBuild[p] = true + } + warned := false + for _, a := range work.ActionList(root) { + if a.Package == nil || okBuild[a.Package] { + continue + } + okBuild[a.Package] = true // warn at most once + + // Don't warn about packages being rebuilt because of + // things like coverage analysis. + for _, p1 := range a.Package.Internal.Imports { + if p1.Internal.Fake { + a.Package.Internal.Fake = true + } + } + + if a.Func != nil && !okBuild[a.Package] && !a.Package.Internal.Fake && !a.Package.Internal.Local { + if !warned { + fmt.Fprintf(os.Stderr, "warning: building out-of-date packages:\n") + warned = true + } + fmt.Fprintf(os.Stderr, "\t%s\n", a.Package.ImportPath) + } + } + if warned { + args := strings.Join(pkgArgs, " ") + if args != "" { + args = " " + args + } + extraOpts := "" + if cfg.BuildRace { + extraOpts = "-race " + } + if cfg.BuildMSan { + extraOpts = "-msan " + } + fmt.Fprintf(os.Stderr, "installing these packages with 'go test %s-i%s' will speed future tests.\n\n", extraOpts, args) + } + + b.Do(root) +} + +// ensures that package p imports the named package +func ensureImport(p *load.Package, pkg string) { + for _, d := range p.Internal.Deps { + if d.Name == pkg { + return + } + } + + a := load.LoadPackage(pkg, &load.ImportStack{}) + if a.Error != nil { + base.Fatalf("load %s: %v", pkg, a.Error) + } + load.ComputeStale(a) + + p.Internal.Imports = append(p.Internal.Imports, a) +} + +var windowsBadWords = []string{ + "install", + "patch", + "setup", + "update", +} + +func builderTest(b *work.Builder, p *load.Package) (buildAction, runAction, printAction *work.Action, err error) { + if len(p.TestGoFiles)+len(p.XTestGoFiles) == 0 { + build := b.Action(work.ModeBuild, work.ModeBuild, p) + run := &work.Action{Package: p, Deps: []*work.Action{build}} + print := &work.Action{Func: builderNoTest, Package: p, Deps: []*work.Action{run}} + return build, run, print, nil + } + + // Build Package structs describing: + // ptest - package + test files + // pxtest - package of external test files + // pmain - pkg.test binary + var ptest, pxtest, pmain *load.Package + + var imports, ximports []*load.Package + var stk load.ImportStack + stk.Push(p.ImportPath + " (test)") + for i, path := range p.TestImports { + p1 := load.LoadImport(path, p.Dir, p, &stk, p.Internal.Build.TestImportPos[path], load.UseVendor) + if cfg.BuildToolchainName == "gccgo" && p1.Standard { + continue + } + if p1.Error != nil { + return nil, nil, nil, p1.Error + } + if len(p1.DepsErrors) > 0 { + err := p1.DepsErrors[0] + err.Pos = "" // show full import stack + return nil, nil, nil, err + } + if str.Contains(p1.Deps, p.ImportPath) || p1.ImportPath == p.ImportPath { + // Same error that loadPackage returns (via reusePackage) in pkg.go. + // Can't change that code, because that code is only for loading the + // non-test copy of a package. + err := &load.PackageError{ + ImportStack: testImportStack(stk[0], p1, p.ImportPath), + Err: "import cycle not allowed in test", + IsImportCycle: true, + } + return nil, nil, nil, err + } + p.TestImports[i] = p1.ImportPath + imports = append(imports, p1) + } + stk.Pop() + stk.Push(p.ImportPath + "_test") + pxtestNeedsPtest := false + for i, path := range p.XTestImports { + p1 := load.LoadImport(path, p.Dir, p, &stk, p.Internal.Build.XTestImportPos[path], load.UseVendor) + if cfg.BuildToolchainName == "gccgo" && p1.Standard { + continue + } + if p1.Error != nil { + return nil, nil, nil, p1.Error + } + if len(p1.DepsErrors) > 0 { + err := p1.DepsErrors[0] + err.Pos = "" // show full import stack + return nil, nil, nil, err + } + if p1.ImportPath == p.ImportPath { + pxtestNeedsPtest = true + } else { + ximports = append(ximports, p1) + } + p.XTestImports[i] = p1.ImportPath + } + stk.Pop() + + // Use last element of import path, not package name. + // They differ when package name is "main". + // But if the import path is "command-line-arguments", + // like it is during 'go run', use the package name. + var elem string + if p.ImportPath == "command-line-arguments" { + elem = p.Name + } else { + _, elem = path.Split(p.ImportPath) + } + testBinary := elem + ".test" + + // The ptest package needs to be importable under the + // same import path that p has, but we cannot put it in + // the usual place in the temporary tree, because then + // other tests will see it as the real package. + // Instead we make a _test directory under the import path + // and then repeat the import path there. We tell the + // compiler and linker to look in that _test directory first. + // + // That is, if the package under test is unicode/utf8, + // then the normal place to write the package archive is + // $WORK/unicode/utf8.a, but we write the test package archive to + // $WORK/unicode/utf8/_test/unicode/utf8.a. + // We write the external test package archive to + // $WORK/unicode/utf8/_test/unicode/utf8_test.a. + testDir := filepath.Join(b.WorkDir, filepath.FromSlash(p.ImportPath+"/_test")) + ptestObj := work.BuildToolchain.Pkgpath(testDir, p) + + // Create the directory for the .a files. + ptestDir, _ := filepath.Split(ptestObj) + if err := b.Mkdir(ptestDir); err != nil { + return nil, nil, nil, err + } + + // Should we apply coverage analysis locally, + // only for this package and only for this test? + // Yes, if -cover is on but -coverpkg has not specified + // a list of packages for global coverage. + localCover := testCover && testCoverPaths == nil + + // Test package. + if len(p.TestGoFiles) > 0 || localCover || p.Name == "main" { + ptest = new(load.Package) + *ptest = *p + ptest.GoFiles = nil + ptest.GoFiles = append(ptest.GoFiles, p.GoFiles...) + ptest.GoFiles = append(ptest.GoFiles, p.TestGoFiles...) + ptest.Internal.Target = "" + ptest.Imports = str.StringList(p.Imports, p.TestImports) + ptest.Internal.Imports = append(append([]*load.Package{}, p.Internal.Imports...), imports...) + ptest.Internal.Pkgdir = testDir + ptest.Internal.Fake = true + ptest.Internal.ForceLibrary = true + ptest.Stale = true + ptest.StaleReason = "rebuild for test" + ptest.Internal.Build = new(build.Package) + *ptest.Internal.Build = *p.Internal.Build + m := map[string][]token.Position{} + for k, v := range p.Internal.Build.ImportPos { + m[k] = append(m[k], v...) + } + for k, v := range p.Internal.Build.TestImportPos { + m[k] = append(m[k], v...) + } + ptest.Internal.Build.ImportPos = m + + if localCover { + ptest.Internal.CoverMode = testCoverMode + var coverFiles []string + coverFiles = append(coverFiles, ptest.GoFiles...) + coverFiles = append(coverFiles, ptest.CgoFiles...) + ptest.Internal.CoverVars = declareCoverVars(ptest.ImportPath, coverFiles...) + } + } else { + ptest = p + } + + // External test package. + if len(p.XTestGoFiles) > 0 { + pxtest = &load.Package{ + PackagePublic: load.PackagePublic{ + Name: p.Name + "_test", + ImportPath: p.ImportPath + "_test", + Root: p.Root, + Dir: p.Dir, + GoFiles: p.XTestGoFiles, + Imports: p.XTestImports, + Stale: true, + }, + Internal: load.PackageInternal{ + LocalPrefix: p.Internal.LocalPrefix, + Build: &build.Package{ + ImportPos: p.Internal.Build.XTestImportPos, + }, + Imports: ximports, + Pkgdir: testDir, + Fake: true, + External: true, + }, + } + if pxtestNeedsPtest { + pxtest.Internal.Imports = append(pxtest.Internal.Imports, ptest) + } + } + + // Action for building pkg.test. + pmain = &load.Package{ + PackagePublic: load.PackagePublic{ + Name: "main", + Dir: testDir, + GoFiles: []string{"_testmain.go"}, + ImportPath: "testmain", + Root: p.Root, + Stale: true, + }, + Internal: load.PackageInternal{ + Build: &build.Package{Name: "main"}, + Pkgdir: testDir, + Fake: true, + OmitDebug: !testC && !testNeedBinary, + }, + } + + // The generated main also imports testing, regexp, and os. + stk.Push("testmain") + for dep := range testMainDeps { + if dep == ptest.ImportPath { + pmain.Internal.Imports = append(pmain.Internal.Imports, ptest) + } else { + p1 := load.LoadImport(dep, "", nil, &stk, nil, 0) + if cfg.BuildToolchainName == "gccgo" && p1.Standard { + continue + } + if p1.Error != nil { + return nil, nil, nil, p1.Error + } + pmain.Internal.Imports = append(pmain.Internal.Imports, p1) + } + } + + if testCoverPkgs != nil { + // Add imports, but avoid duplicates. + seen := map[*load.Package]bool{p: true, ptest: true} + for _, p1 := range pmain.Internal.Imports { + seen[p1] = true + } + for _, p1 := range testCoverPkgs { + if !seen[p1] { + seen[p1] = true + pmain.Internal.Imports = append(pmain.Internal.Imports, p1) + } + } + } + + // Do initial scan for metadata needed for writing _testmain.go + // Use that metadata to update the list of imports for package main. + // The list of imports is used by recompileForTest and by the loop + // afterward that gathers t.Cover information. + t, err := loadTestFuncs(ptest) + if err != nil { + return nil, nil, nil, err + } + if len(ptest.GoFiles)+len(ptest.CgoFiles) > 0 { + pmain.Internal.Imports = append(pmain.Internal.Imports, ptest) + t.ImportTest = true + } + if pxtest != nil { + pmain.Internal.Imports = append(pmain.Internal.Imports, pxtest) + t.ImportXtest = true + } + + if ptest != p && localCover { + // We have made modifications to the package p being tested + // and are rebuilding p (as ptest), writing it to the testDir tree. + // Arrange to rebuild, writing to that same tree, all packages q + // such that the test depends on q, and q depends on p. + // This makes sure that q sees the modifications to p. + // Strictly speaking, the rebuild is only necessary if the + // modifications to p change its export metadata, but + // determining that is a bit tricky, so we rebuild always. + // + // This will cause extra compilation, so for now we only do it + // when testCover is set. The conditions are more general, though, + // and we may find that we need to do it always in the future. + recompileForTest(pmain, p, ptest, testDir) + } + + if cfg.BuildContext.GOOS == "darwin" { + if cfg.BuildContext.GOARCH == "arm" || cfg.BuildContext.GOARCH == "arm64" { + t.NeedCgo = true + } + } + + for _, cp := range pmain.Internal.Imports { + if len(cp.Internal.CoverVars) > 0 { + t.Cover = append(t.Cover, coverInfo{cp, cp.Internal.CoverVars}) + } + } + + if !cfg.BuildN { + // writeTestmain writes _testmain.go. This must happen after recompileForTest, + // because recompileForTest modifies XXX. + if err := writeTestmain(filepath.Join(testDir, "_testmain.go"), t); err != nil { + return nil, nil, nil, err + } + } + + load.ComputeStale(pmain) + + if ptest != p { + a := b.Action(work.ModeBuild, work.ModeBuild, ptest) + a.Objdir = testDir + string(filepath.Separator) + "_obj_test" + string(filepath.Separator) + a.Objpkg = ptestObj + a.Target = ptestObj + a.Link = false + } + + if pxtest != nil { + a := b.Action(work.ModeBuild, work.ModeBuild, pxtest) + a.Objdir = testDir + string(filepath.Separator) + "_obj_xtest" + string(filepath.Separator) + a.Objpkg = work.BuildToolchain.Pkgpath(testDir, pxtest) + a.Target = a.Objpkg + } + + a := b.Action(work.ModeBuild, work.ModeBuild, pmain) + a.Objdir = testDir + string(filepath.Separator) + a.Objpkg = filepath.Join(testDir, "main.a") + a.Target = filepath.Join(testDir, testBinary) + cfg.ExeSuffix + if cfg.Goos == "windows" { + // There are many reserved words on Windows that, + // if used in the name of an executable, cause Windows + // to try to ask for extra permissions. + // The word list includes setup, install, update, and patch, + // but it does not appear to be defined anywhere. + // We have run into this trying to run the + // go.codereview/patch tests. + // For package names containing those words, use test.test.exe + // instead of pkgname.test.exe. + // Note that this file name is only used in the Go command's + // temporary directory. If the -c or other flags are + // given, the code below will still use pkgname.test.exe. + // There are two user-visible effects of this change. + // First, you can actually run 'go test' in directories that + // have names that Windows thinks are installer-like, + // without getting a dialog box asking for more permissions. + // Second, in the Windows process listing during go test, + // the test shows up as test.test.exe, not pkgname.test.exe. + // That second one is a drawback, but it seems a small + // price to pay for the test running at all. + // If maintaining the list of bad words is too onerous, + // we could just do this always on Windows. + for _, bad := range windowsBadWords { + if strings.Contains(testBinary, bad) { + a.Target = filepath.Join(testDir, "test.test") + cfg.ExeSuffix + break + } + } + } + buildAction = a + + if testC || testNeedBinary { + // -c or profiling flag: create action to copy binary to ./test.out. + target := filepath.Join(base.Cwd, testBinary+cfg.ExeSuffix) + if testO != "" { + target = testO + if !filepath.IsAbs(target) { + target = filepath.Join(base.Cwd, target) + } + } + buildAction = &work.Action{ + Func: work.BuildInstallFunc, + Deps: []*work.Action{buildAction}, + Package: pmain, + Target: target, + } + runAction = buildAction // make sure runAction != nil even if not running test + } + if testC { + printAction = &work.Action{Package: p, Deps: []*work.Action{runAction}} // nop + } else { + // run test + runAction = &work.Action{ + Func: builderRunTest, + Deps: []*work.Action{buildAction}, + Package: p, + IgnoreFail: true, + } + cleanAction := &work.Action{ + Func: builderCleanTest, + Deps: []*work.Action{runAction}, + Package: p, + } + printAction = &work.Action{ + Func: builderPrintTest, + Deps: []*work.Action{cleanAction}, + Package: p, + } + } + + return buildAction, runAction, printAction, nil +} + +func testImportStack(top string, p *load.Package, target string) []string { + stk := []string{top, p.ImportPath} +Search: + for p.ImportPath != target { + for _, p1 := range p.Internal.Imports { + if p1.ImportPath == target || str.Contains(p1.Deps, target) { + stk = append(stk, p1.ImportPath) + p = p1 + continue Search + } + } + // Can't happen, but in case it does... + stk = append(stk, "") + break + } + return stk +} + +func recompileForTest(pmain, preal, ptest *load.Package, testDir string) { + // The "test copy" of preal is ptest. + // For each package that depends on preal, make a "test copy" + // that depends on ptest. And so on, up the dependency tree. + testCopy := map[*load.Package]*load.Package{preal: ptest} + for _, p := range load.PackageList([]*load.Package{pmain}) { + // Copy on write. + didSplit := false + split := func() { + if didSplit { + return + } + didSplit = true + if p.Internal.Pkgdir != testDir { + p1 := new(load.Package) + testCopy[p] = p1 + *p1 = *p + p1.Internal.Imports = make([]*load.Package, len(p.Internal.Imports)) + copy(p1.Internal.Imports, p.Internal.Imports) + p = p1 + p.Internal.Pkgdir = testDir + p.Internal.Target = "" + p.Internal.Fake = true + p.Stale = true + p.StaleReason = "depends on package being tested" + } + } + + // Update p.Deps and p.Internal.Imports to use at test copies. + for i, dep := range p.Internal.Deps { + if p1 := testCopy[dep]; p1 != nil && p1 != dep { + split() + p.Internal.Deps[i] = p1 + } + } + for i, imp := range p.Internal.Imports { + if p1 := testCopy[imp]; p1 != nil && p1 != imp { + split() + p.Internal.Imports[i] = p1 + } + } + } +} + +var coverIndex = 0 + +// isTestFile reports whether the source file is a set of tests and should therefore +// be excluded from coverage analysis. +func isTestFile(file string) bool { + // We don't cover tests, only the code they test. + return strings.HasSuffix(file, "_test.go") +} + +// declareCoverVars attaches the required cover variables names +// to the files, to be used when annotating the files. +func declareCoverVars(importPath string, files ...string) map[string]*load.CoverVar { + coverVars := make(map[string]*load.CoverVar) + for _, file := range files { + if isTestFile(file) { + continue + } + coverVars[file] = &load.CoverVar{ + File: filepath.Join(importPath, file), + Var: fmt.Sprintf("GoCover_%d", coverIndex), + } + coverIndex++ + } + return coverVars +} + +var noTestsToRun = []byte("\ntesting: warning: no tests to run\n") + +// builderRunTest is the action for running a test binary. +func builderRunTest(b *work.Builder, a *work.Action) error { + args := str.StringList(work.FindExecCmd(), a.Deps[0].Target, testArgs) + a.TestOutput = new(bytes.Buffer) + + if cfg.BuildN || cfg.BuildX { + b.Showcmd("", "%s", strings.Join(args, " ")) + if cfg.BuildN { + return nil + } + } + + if a.Failed { + // We were unable to build the binary. + a.Failed = false + fmt.Fprintf(a.TestOutput, "FAIL\t%s [build failed]\n", a.Package.ImportPath) + base.SetExitStatus(1) + return nil + } + + cmd := exec.Command(args[0], args[1:]...) + cmd.Dir = a.Package.Dir + cmd.Env = base.EnvForDir(cmd.Dir, cfg.OrigEnv) + var buf bytes.Buffer + if testStreamOutput { + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + } else { + cmd.Stdout = &buf + cmd.Stderr = &buf + } + + // If there are any local SWIG dependencies, we want to load + // the shared library from the build directory. + if a.Package.UsesSwig() { + env := cmd.Env + found := false + prefix := "LD_LIBRARY_PATH=" + for i, v := range env { + if strings.HasPrefix(v, prefix) { + env[i] = v + ":." + found = true + break + } + } + if !found { + env = append(env, "LD_LIBRARY_PATH=.") + } + cmd.Env = env + } + + t0 := time.Now() + err := cmd.Start() + + // This is a last-ditch deadline to detect and + // stop wedged test binaries, to keep the builders + // running. + if err == nil { + tick := time.NewTimer(testKillTimeout) + base.StartSigHandlers() + done := make(chan error) + go func() { + done <- cmd.Wait() + }() + Outer: + select { + case err = <-done: + // ok + case <-tick.C: + if base.SignalTrace != nil { + // Send a quit signal in the hope that the program will print + // a stack trace and exit. Give it five seconds before resorting + // to Kill. + cmd.Process.Signal(base.SignalTrace) + select { + case err = <-done: + fmt.Fprintf(&buf, "*** Test killed with %v: ran too long (%v).\n", base.SignalTrace, testKillTimeout) + break Outer + case <-time.After(5 * time.Second): + } + } + cmd.Process.Kill() + err = <-done + fmt.Fprintf(&buf, "*** Test killed: ran too long (%v).\n", testKillTimeout) + } + tick.Stop() + } + out := buf.Bytes() + t := fmt.Sprintf("%.3fs", time.Since(t0).Seconds()) + if err == nil { + norun := "" + if testShowPass { + a.TestOutput.Write(out) + } + if bytes.HasPrefix(out, noTestsToRun[1:]) || bytes.Contains(out, noTestsToRun) { + norun = " [no tests to run]" + } + fmt.Fprintf(a.TestOutput, "ok \t%s\t%s%s%s\n", a.Package.ImportPath, t, coveragePercentage(out), norun) + return nil + } + + base.SetExitStatus(1) + if len(out) > 0 { + a.TestOutput.Write(out) + // assume printing the test binary's exit status is superfluous + } else { + fmt.Fprintf(a.TestOutput, "%s\n", err) + } + fmt.Fprintf(a.TestOutput, "FAIL\t%s\t%s\n", a.Package.ImportPath, t) + + return nil +} + +// coveragePercentage returns the coverage results (if enabled) for the +// test. It uncovers the data by scanning the output from the test run. +func coveragePercentage(out []byte) string { + if !testCover { + return "" + } + // The string looks like + // test coverage for encoding/binary: 79.9% of statements + // Extract the piece from the percentage to the end of the line. + re := regexp.MustCompile(`coverage: (.*)\n`) + matches := re.FindSubmatch(out) + if matches == nil { + // Probably running "go test -cover" not "go test -cover fmt". + // The coverage output will appear in the output directly. + return "" + } + return fmt.Sprintf("\tcoverage: %s", matches[1]) +} + +// builderCleanTest is the action for cleaning up after a test. +func builderCleanTest(b *work.Builder, a *work.Action) error { + if cfg.BuildWork { + return nil + } + run := a.Deps[0] + testDir := filepath.Join(b.WorkDir, filepath.FromSlash(run.Package.ImportPath+"/_test")) + os.RemoveAll(testDir) + return nil +} + +// builderPrintTest is the action for printing a test result. +func builderPrintTest(b *work.Builder, a *work.Action) error { + clean := a.Deps[0] + run := clean.Deps[0] + os.Stdout.Write(run.TestOutput.Bytes()) + run.TestOutput = nil + return nil +} + +// builderNoTest is the action for testing a package with no test files. +func builderNoTest(b *work.Builder, a *work.Action) error { + fmt.Printf("? \t%s\t[no test files]\n", a.Package.ImportPath) + return nil +} + +// isTestFunc tells whether fn has the type of a testing function. arg +// specifies the parameter type we look for: B, M or T. +func isTestFunc(fn *ast.FuncDecl, arg string) bool { + if fn.Type.Results != nil && len(fn.Type.Results.List) > 0 || + fn.Type.Params.List == nil || + len(fn.Type.Params.List) != 1 || + len(fn.Type.Params.List[0].Names) > 1 { + return false + } + ptr, ok := fn.Type.Params.List[0].Type.(*ast.StarExpr) + if !ok { + return false + } + // We can't easily check that the type is *testing.M + // because we don't know how testing has been imported, + // but at least check that it's *M or *something.M. + // Same applies for B and T. + if name, ok := ptr.X.(*ast.Ident); ok && name.Name == arg { + return true + } + if sel, ok := ptr.X.(*ast.SelectorExpr); ok && sel.Sel.Name == arg { + return true + } + return false +} + +// isTest tells whether name looks like a test (or benchmark, according to prefix). +// It is a Test (say) if there is a character after Test that is not a lower-case letter. +// We don't want TesticularCancer. +func isTest(name, prefix string) bool { + if !strings.HasPrefix(name, prefix) { + return false + } + if len(name) == len(prefix) { // "Test" is ok + return true + } + rune, _ := utf8.DecodeRuneInString(name[len(prefix):]) + return !unicode.IsLower(rune) +} + +type coverInfo struct { + Package *load.Package + Vars map[string]*load.CoverVar +} + +// loadTestFuncs returns the testFuncs describing the tests that will be run. +func loadTestFuncs(ptest *load.Package) (*testFuncs, error) { + t := &testFuncs{ + Package: ptest, + } + for _, file := range ptest.TestGoFiles { + if err := t.load(filepath.Join(ptest.Dir, file), "_test", &t.ImportTest, &t.NeedTest); err != nil { + return nil, err + } + } + for _, file := range ptest.XTestGoFiles { + if err := t.load(filepath.Join(ptest.Dir, file), "_xtest", &t.ImportXtest, &t.NeedXtest); err != nil { + return nil, err + } + } + return t, nil +} + +// writeTestmain writes the _testmain.go file for t to the file named out. +func writeTestmain(out string, t *testFuncs) error { + f, err := os.Create(out) + if err != nil { + return err + } + defer f.Close() + + if err := testmainTmpl.Execute(f, t); err != nil { + return err + } + + return nil +} + +type testFuncs struct { + Tests []testFunc + Benchmarks []testFunc + Examples []testFunc + TestMain *testFunc + Package *load.Package + ImportTest bool + NeedTest bool + ImportXtest bool + NeedXtest bool + NeedCgo bool + Cover []coverInfo +} + +func (t *testFuncs) CoverMode() string { + return testCoverMode +} + +func (t *testFuncs) CoverEnabled() bool { + return testCover +} + +// ImportPath returns the import path of the package being tested, if it is within GOPATH. +// This is printed by the testing package when running benchmarks. +func (t *testFuncs) ImportPath() string { + pkg := t.Package.ImportPath + if strings.HasPrefix(pkg, "_/") { + return "" + } + if pkg == "command-line-arguments" { + return "" + } + return pkg +} + +// Covered returns a string describing which packages are being tested for coverage. +// If the covered package is the same as the tested package, it returns the empty string. +// Otherwise it is a comma-separated human-readable list of packages beginning with +// " in", ready for use in the coverage message. +func (t *testFuncs) Covered() string { + if testCoverPaths == nil { + return "" + } + return " in " + strings.Join(testCoverPaths, ", ") +} + +// Tested returns the name of the package being tested. +func (t *testFuncs) Tested() string { + return t.Package.Name +} + +type testFunc struct { + Package string // imported package name (_test or _xtest) + Name string // function name + Output string // output, for examples + Unordered bool // output is allowed to be unordered. +} + +var testFileSet = token.NewFileSet() + +func (t *testFuncs) load(filename, pkg string, doImport, seen *bool) error { + f, err := parser.ParseFile(testFileSet, filename, nil, parser.ParseComments) + if err != nil { + return base.ExpandScanner(err) + } + for _, d := range f.Decls { + n, ok := d.(*ast.FuncDecl) + if !ok { + continue + } + if n.Recv != nil { + continue + } + name := n.Name.String() + switch { + case name == "TestMain" && isTestFunc(n, "M"): + if t.TestMain != nil { + return errors.New("multiple definitions of TestMain") + } + t.TestMain = &testFunc{pkg, name, "", false} + *doImport, *seen = true, true + case isTest(name, "Test"): + err := checkTestFunc(n, "T") + if err != nil { + return err + } + t.Tests = append(t.Tests, testFunc{pkg, name, "", false}) + *doImport, *seen = true, true + case isTest(name, "Benchmark"): + err := checkTestFunc(n, "B") + if err != nil { + return err + } + t.Benchmarks = append(t.Benchmarks, testFunc{pkg, name, "", false}) + *doImport, *seen = true, true + } + } + ex := doc.Examples(f) + sort.Slice(ex, func(i, j int) bool { return ex[i].Order < ex[j].Order }) + for _, e := range ex { + *doImport = true // import test file whether executed or not + if e.Output == "" && !e.EmptyOutput { + // Don't run examples with no output. + continue + } + t.Examples = append(t.Examples, testFunc{pkg, "Example" + e.Name, e.Output, e.Unordered}) + *seen = true + } + return nil +} + +func checkTestFunc(fn *ast.FuncDecl, arg string) error { + if !isTestFunc(fn, arg) { + name := fn.Name.String() + pos := testFileSet.Position(fn.Pos()) + return fmt.Errorf("%s: wrong signature for %s, must be: func %s(%s *testing.%s)", pos, name, name, strings.ToLower(arg), arg) + } + return nil +} + +var testmainTmpl = template.Must(template.New("main").Parse(` +package main + +import ( +{{if not .TestMain}} + "os" +{{end}} + "testing" + "testing/internal/testdeps" + +{{if .ImportTest}} + {{if .NeedTest}}_test{{else}}_{{end}} {{.Package.ImportPath | printf "%q"}} +{{end}} +{{if .ImportXtest}} + {{if .NeedXtest}}_xtest{{else}}_{{end}} {{.Package.ImportPath | printf "%s_test" | printf "%q"}} +{{end}} +{{range $i, $p := .Cover}} + _cover{{$i}} {{$p.Package.ImportPath | printf "%q"}} +{{end}} + +{{if .NeedCgo}} + _ "runtime/cgo" +{{end}} +) + +var tests = []testing.InternalTest{ +{{range .Tests}} + {"{{.Name}}", {{.Package}}.{{.Name}}}, +{{end}} +} + +var benchmarks = []testing.InternalBenchmark{ +{{range .Benchmarks}} + {"{{.Name}}", {{.Package}}.{{.Name}}}, +{{end}} +} + +var examples = []testing.InternalExample{ +{{range .Examples}} + {"{{.Name}}", {{.Package}}.{{.Name}}, {{.Output | printf "%q"}}, {{.Unordered}}}, +{{end}} +} + +func init() { + testdeps.ImportPath = {{.ImportPath | printf "%q"}} +} + +{{if .CoverEnabled}} + +// Only updated by init functions, so no need for atomicity. +var ( + coverCounters = make(map[string][]uint32) + coverBlocks = make(map[string][]testing.CoverBlock) +) + +func init() { + {{range $i, $p := .Cover}} + {{range $file, $cover := $p.Vars}} + coverRegisterFile({{printf "%q" $cover.File}}, _cover{{$i}}.{{$cover.Var}}.Count[:], _cover{{$i}}.{{$cover.Var}}.Pos[:], _cover{{$i}}.{{$cover.Var}}.NumStmt[:]) + {{end}} + {{end}} +} + +func coverRegisterFile(fileName string, counter []uint32, pos []uint32, numStmts []uint16) { + if 3*len(counter) != len(pos) || len(counter) != len(numStmts) { + panic("coverage: mismatched sizes") + } + if coverCounters[fileName] != nil { + // Already registered. + return + } + coverCounters[fileName] = counter + block := make([]testing.CoverBlock, len(counter)) + for i := range counter { + block[i] = testing.CoverBlock{ + Line0: pos[3*i+0], + Col0: uint16(pos[3*i+2]), + Line1: pos[3*i+1], + Col1: uint16(pos[3*i+2]>>16), + Stmts: numStmts[i], + } + } + coverBlocks[fileName] = block +} +{{end}} + +func main() { +{{if .CoverEnabled}} + testing.RegisterCover(testing.Cover{ + Mode: {{printf "%q" .CoverMode}}, + Counters: coverCounters, + Blocks: coverBlocks, + CoveredPackages: {{printf "%q" .Covered}}, + }) +{{end}} + m := testing.MainStart(testdeps.TestDeps{}, tests, benchmarks, examples) +{{with .TestMain}} + {{.Package}}.{{.Name}}(m) +{{else}} + os.Exit(m.Run()) +{{end}} +} + +`)) diff --git a/libgo/go/cmd/go/internal/test/testflag.go b/libgo/go/cmd/go/internal/test/testflag.go new file mode 100644 index 00000000000..bff8656a4c1 --- /dev/null +++ b/libgo/go/cmd/go/internal/test/testflag.go @@ -0,0 +1,211 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package test + +import ( + "flag" + "os" + "strings" + + "cmd/go/internal/base" + "cmd/go/internal/cfg" + "cmd/go/internal/cmdflag" + "cmd/go/internal/str" + "cmd/go/internal/work" +) + +const cmd = "test" + +// The flag handling part of go test is large and distracting. +// We can't use the flag package because some of the flags from +// our command line are for us, and some are for 6.out, and +// some are for both. + +// testFlagDefn is the set of flags we process. +var testFlagDefn = []*cmdflag.Defn{ + // local. + {Name: "c", BoolVar: &testC}, + {Name: "i", BoolVar: &cfg.BuildI}, + {Name: "o"}, + {Name: "cover", BoolVar: &testCover}, + {Name: "covermode"}, + {Name: "coverpkg"}, + {Name: "exec"}, + + // Passed to 6.out, adding a "test." prefix to the name if necessary: -v becomes -test.v. + {Name: "bench", PassToTest: true}, + {Name: "benchmem", BoolVar: new(bool), PassToTest: true}, + {Name: "benchtime", PassToTest: true}, + {Name: "count", PassToTest: true}, + {Name: "coverprofile", PassToTest: true}, + {Name: "cpu", PassToTest: true}, + {Name: "cpuprofile", PassToTest: true}, + {Name: "list", PassToTest: true}, + {Name: "memprofile", PassToTest: true}, + {Name: "memprofilerate", PassToTest: true}, + {Name: "blockprofile", PassToTest: true}, + {Name: "blockprofilerate", PassToTest: true}, + {Name: "mutexprofile", PassToTest: true}, + {Name: "mutexprofilefraction", PassToTest: true}, + {Name: "outputdir", PassToTest: true}, + {Name: "parallel", PassToTest: true}, + {Name: "run", PassToTest: true}, + {Name: "short", BoolVar: new(bool), PassToTest: true}, + {Name: "timeout", PassToTest: true}, + {Name: "trace", PassToTest: true}, + {Name: "v", BoolVar: &testV, PassToTest: true}, +} + +// add build flags to testFlagDefn +func init() { + var cmd base.Command + work.AddBuildFlags(&cmd) + cmd.Flag.VisitAll(func(f *flag.Flag) { + if f.Name == "v" { + // test overrides the build -v flag + return + } + testFlagDefn = append(testFlagDefn, &cmdflag.Defn{ + Name: f.Name, + Value: f.Value, + }) + }) +} + +// testFlags processes the command line, grabbing -x and -c, rewriting known flags +// to have "test" before them, and reading the command line for the 6.out. +// Unfortunately for us, we need to do our own flag processing because go test +// grabs some flags but otherwise its command line is just a holding place for +// pkg.test's arguments. +// We allow known flags both before and after the package name list, +// to allow both +// go test fmt -custom-flag-for-fmt-test +// go test -x math +func testFlags(args []string) (packageNames, passToTest []string) { + inPkg := false + outputDir := "" + var explicitArgs []string + for i := 0; i < len(args); i++ { + if !strings.HasPrefix(args[i], "-") { + if !inPkg && packageNames == nil { + // First package name we've seen. + inPkg = true + } + if inPkg { + packageNames = append(packageNames, args[i]) + continue + } + } + + if inPkg { + // Found an argument beginning with "-"; end of package list. + inPkg = false + } + + f, value, extraWord := cmdflag.Parse(cmd, testFlagDefn, args, i) + if f == nil { + // This is a flag we do not know; we must assume + // that any args we see after this might be flag + // arguments, not package names. + inPkg = false + if packageNames == nil { + // make non-nil: we have seen the empty package list + packageNames = []string{} + } + if args[i] == "-args" || args[i] == "--args" { + // -args or --args signals that everything that follows + // should be passed to the test. + explicitArgs = args[i+1:] + break + } + passToTest = append(passToTest, args[i]) + continue + } + if f.Value != nil { + if err := f.Value.Set(value); err != nil { + base.Fatalf("invalid flag argument for -%s: %v", f.Name, err) + } + } else { + // Test-only flags. + // Arguably should be handled by f.Value, but aren't. + switch f.Name { + // bool flags. + case "c", "i", "v", "cover": + cmdflag.SetBool(cmd, f.BoolVar, value) + case "o": + testO = value + testNeedBinary = true + case "exec": + xcmd, err := str.SplitQuotedFields(value) + if err != nil { + base.Fatalf("invalid flag argument for -%s: %v", f.Name, err) + } + work.ExecCmd = xcmd + case "bench": + // record that we saw the flag; don't care about the value + testBench = true + case "list": + testList = true + case "timeout": + testTimeout = value + case "blockprofile", "cpuprofile", "memprofile", "mutexprofile": + testProfile = true + testNeedBinary = true + case "trace": + testProfile = true + case "coverpkg": + testCover = true + if value == "" { + testCoverPaths = nil + } else { + testCoverPaths = strings.Split(value, ",") + } + case "coverprofile": + testCover = true + testProfile = true + case "covermode": + switch value { + case "set", "count", "atomic": + testCoverMode = value + default: + base.Fatalf("invalid flag argument for -covermode: %q", value) + } + testCover = true + case "outputdir": + outputDir = value + } + } + if extraWord { + i++ + } + if f.PassToTest { + passToTest = append(passToTest, "-test."+f.Name+"="+value) + } + } + + if testCoverMode == "" { + testCoverMode = "set" + if cfg.BuildRace { + // Default coverage mode is atomic when -race is set. + testCoverMode = "atomic" + } + } + + if cfg.BuildRace && testCoverMode != "atomic" { + base.Fatalf(`-covermode must be "atomic", not %q, when -race is enabled`, testCoverMode) + } + + // Tell the test what directory we're running in, so it can write the profiles there. + if testProfile && outputDir == "" { + dir, err := os.Getwd() + if err != nil { + base.Fatalf("error from os.Getwd: %s", err) + } + passToTest = append(passToTest, "-test.outputdir", dir) + } + + passToTest = append(passToTest, explicitArgs...) + return +} diff --git a/libgo/go/cmd/go/internal/tool/tool.go b/libgo/go/cmd/go/internal/tool/tool.go new file mode 100644 index 00000000000..7dd5510328c --- /dev/null +++ b/libgo/go/cmd/go/internal/tool/tool.go @@ -0,0 +1,135 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package tool implements the ``go tool'' command. +package tool + +import ( + "fmt" + "os" + "os/exec" + "sort" + "strings" + + "cmd/go/internal/base" + "cmd/go/internal/cfg" +) + +var CmdTool = &base.Command{ + Run: runTool, + UsageLine: "tool [-n] command [args...]", + Short: "run specified go tool", + Long: ` +Tool runs the go tool command identified by the arguments. +With no arguments it prints the list of known tools. + +The -n flag causes tool to print the command that would be +executed but not execute it. + +For more about each tool command, see 'go tool command -h'. +`, +} + +var toolN bool + +// Return whether tool can be expected in the gccgo tool directory. +// Other binaries could be in the same directory so don't +// show those with the 'go tool' command. +func isGccgoTool(tool string) bool { + switch tool { + case "cgo", "fix", "cover", "godoc", "vet": + return true + } + return false +} + +func init() { + CmdTool.Flag.BoolVar(&toolN, "n", false, "") +} + +func runTool(cmd *base.Command, args []string) { + if len(args) == 0 { + listTools() + return + } + toolName := args[0] + // The tool name must be lower-case letters, numbers or underscores. + for _, c := range toolName { + switch { + case 'a' <= c && c <= 'z', '0' <= c && c <= '9', c == '_': + default: + fmt.Fprintf(os.Stderr, "go tool: bad tool name %q\n", toolName) + base.SetExitStatus(2) + return + } + } + toolPath := base.Tool(toolName) + if toolPath == "" { + return + } + if toolN { + cmd := toolPath + if len(args) > 1 { + cmd += " " + strings.Join(args[1:], " ") + } + fmt.Printf("%s\n", cmd) + return + } + args[0] = toolPath // in case the tool wants to re-exec itself, e.g. cmd/dist + toolCmd := &exec.Cmd{ + Path: toolPath, + Args: args, + Stdin: os.Stdin, + Stdout: os.Stdout, + Stderr: os.Stderr, + // Set $GOROOT, mainly for go tool dist. + Env: base.MergeEnvLists([]string{"GOROOT=" + cfg.GOROOT}, os.Environ()), + } + err := toolCmd.Run() + if err != nil { + // Only print about the exit status if the command + // didn't even run (not an ExitError) or it didn't exit cleanly + // or we're printing command lines too (-x mode). + // Assume if command exited cleanly (even with non-zero status) + // it printed any messages it wanted to print. + if e, ok := err.(*exec.ExitError); !ok || !e.Exited() || cfg.BuildX { + fmt.Fprintf(os.Stderr, "go tool %s: %s\n", toolName, err) + } + base.SetExitStatus(1) + return + } +} + +// listTools prints a list of the available tools in the tools directory. +func listTools() { + f, err := os.Open(base.ToolDir) + if err != nil { + fmt.Fprintf(os.Stderr, "go tool: no tool directory: %s\n", err) + base.SetExitStatus(2) + return + } + defer f.Close() + names, err := f.Readdirnames(-1) + if err != nil { + fmt.Fprintf(os.Stderr, "go tool: can't read directory: %s\n", err) + base.SetExitStatus(2) + return + } + + sort.Strings(names) + for _, name := range names { + // Unify presentation by going to lower case. + name = strings.ToLower(name) + // If it's windows, don't show the .exe suffix. + if base.ToolIsWindows && strings.HasSuffix(name, base.ToolWindowsExtension) { + name = name[:len(name)-len(base.ToolWindowsExtension)] + } + // The tool directory used by gccgo will have other binaries + // in addition to go tools. Only display go tools here. + if cfg.BuildToolchainName == "gccgo" && !isGccgoTool(name) { + continue + } + fmt.Println(name) + } +} diff --git a/libgo/go/cmd/go/internal/version/version.go b/libgo/go/cmd/go/internal/version/version.go new file mode 100644 index 00000000000..c3f7d73ff1d --- /dev/null +++ b/libgo/go/cmd/go/internal/version/version.go @@ -0,0 +1,28 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package version implements the ``go version'' command. +package version + +import ( + "fmt" + "runtime" + + "cmd/go/internal/base" +) + +var CmdVersion = &base.Command{ + Run: runVersion, + UsageLine: "version", + Short: "print Go version", + Long: `Version prints the Go version, as reported by runtime.Version.`, +} + +func runVersion(cmd *base.Command, args []string) { + if len(args) != 0 { + cmd.Usage() + } + + fmt.Printf("go version %s %s/%s\n", runtime.Version(), runtime.GOOS, runtime.GOARCH) +} diff --git a/libgo/go/cmd/go/internal/vet/vet.go b/libgo/go/cmd/go/internal/vet/vet.go new file mode 100644 index 00000000000..ddacd085b06 --- /dev/null +++ b/libgo/go/cmd/go/internal/vet/vet.go @@ -0,0 +1,56 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package vet implements the ``go vet'' command. +package vet + +import ( + "path/filepath" + + "cmd/go/internal/base" + "cmd/go/internal/cfg" + "cmd/go/internal/load" + "cmd/go/internal/str" +) + +var CmdVet = &base.Command{ + Run: runVet, + CustomFlags: true, + UsageLine: "vet [-n] [-x] [build flags] [vet flags] [packages]", + Short: "run go tool vet on packages", + Long: ` +Vet runs the Go vet command on the packages named by the import paths. + +For more about vet and its flags, see 'go doc cmd/vet'. +For more about specifying packages, see 'go help packages'. + +The -n flag prints commands that would be executed. +The -x flag prints commands as they are executed. + +For more about build flags, see 'go help build'. + +See also: go fmt, go fix. + `, +} + +func runVet(cmd *base.Command, args []string) { + vetFlags, packages := vetFlags(args) + for _, p := range load.Packages(packages) { + // Vet expects to be given a set of files all from the same package. + // Run once for package p and once for package p_test. + if len(p.GoFiles)+len(p.CgoFiles)+len(p.TestGoFiles) > 0 { + runVetFiles(p, vetFlags, str.StringList(p.GoFiles, p.CgoFiles, p.TestGoFiles, p.SFiles)) + } + if len(p.XTestGoFiles) > 0 { + runVetFiles(p, vetFlags, str.StringList(p.XTestGoFiles)) + } + } +} + +func runVetFiles(p *load.Package, flags, files []string) { + for i := range files { + files[i] = filepath.Join(p.Dir, files[i]) + } + base.Run(cfg.BuildToolexec, base.Tool("vet"), flags, base.RelPaths(files)) +} diff --git a/libgo/go/cmd/go/internal/vet/vetflag.go b/libgo/go/cmd/go/internal/vet/vetflag.go new file mode 100644 index 00000000000..8cd21bb72b7 --- /dev/null +++ b/libgo/go/cmd/go/internal/vet/vetflag.go @@ -0,0 +1,99 @@ +// Copyright 2017 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package vet + +import ( + "flag" + "fmt" + "os" + "strings" + + "cmd/go/internal/base" + "cmd/go/internal/cmdflag" + "cmd/go/internal/work" +) + +const cmd = "vet" + +// vetFlagDefn is the set of flags we process. +var vetFlagDefn = []*cmdflag.Defn{ + // Note: Some flags, in particular -tags and -v, are known to + // vet but also defined as build flags. This works fine, so we + // don't define them here but use AddBuildFlags to init them. + // However some, like -x, are known to the build but not + // to vet. We handle them in vetFlags. + + // local. + {Name: "all", BoolVar: new(bool)}, + {Name: "asmdecl", BoolVar: new(bool)}, + {Name: "assign", BoolVar: new(bool)}, + {Name: "atomic", BoolVar: new(bool)}, + {Name: "bool", BoolVar: new(bool)}, + {Name: "buildtags", BoolVar: new(bool)}, + {Name: "cgocall", BoolVar: new(bool)}, + {Name: "composites", BoolVar: new(bool)}, + {Name: "copylocks", BoolVar: new(bool)}, + {Name: "httpresponse", BoolVar: new(bool)}, + {Name: "lostcancel", BoolVar: new(bool)}, + {Name: "methods", BoolVar: new(bool)}, + {Name: "nilfunc", BoolVar: new(bool)}, + {Name: "printf", BoolVar: new(bool)}, + {Name: "printfuncs"}, + {Name: "rangeloops", BoolVar: new(bool)}, + {Name: "shadow", BoolVar: new(bool)}, + {Name: "shadowstrict", BoolVar: new(bool)}, + {Name: "source", BoolVar: new(bool)}, + {Name: "structtags", BoolVar: new(bool)}, + {Name: "tests", BoolVar: new(bool)}, + {Name: "unreachable", BoolVar: new(bool)}, + {Name: "unsafeptr", BoolVar: new(bool)}, + {Name: "unusedfuncs"}, + {Name: "unusedresult", BoolVar: new(bool)}, + {Name: "unusedstringmethods"}, +} + +// add build flags to vetFlagDefn. +func init() { + var cmd base.Command + work.AddBuildFlags(&cmd) + cmd.Flag.VisitAll(func(f *flag.Flag) { + vetFlagDefn = append(vetFlagDefn, &cmdflag.Defn{ + Name: f.Name, + Value: f.Value, + }) + }) +} + +// vetFlags processes the command line, splitting it at the first non-flag +// into the list of flags and list of packages. +func vetFlags(args []string) (passToVet, packageNames []string) { + for i := 0; i < len(args); i++ { + if !strings.HasPrefix(args[i], "-") { + return args[:i], args[i:] + } + + f, value, extraWord := cmdflag.Parse(cmd, vetFlagDefn, args, i) + if f == nil { + fmt.Fprintf(os.Stderr, "vet: flag %q not defined\n", args[i]) + fmt.Fprintf(os.Stderr, "Run \"go help vet\" for more information\n") + os.Exit(2) + } + if f.Value != nil { + if err := f.Value.Set(value); err != nil { + base.Fatalf("invalid flag argument for -%s: %v", f.Name, err) + } + switch f.Name { + // Flags known to the build but not to vet, so must be dropped. + case "x", "n": + args = append(args[:i], args[i+1:]...) + i-- + } + } + if extraWord { + i++ + } + } + return args, nil +} diff --git a/libgo/go/cmd/go/internal/web/bootstrap.go b/libgo/go/cmd/go/internal/web/bootstrap.go new file mode 100644 index 00000000000..d1d4621a44b --- /dev/null +++ b/libgo/go/cmd/go/internal/web/bootstrap.go @@ -0,0 +1,37 @@ +// Copyright 2012 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build cmd_go_bootstrap + +// This code is compiled only into the bootstrap 'go' binary. +// These stubs avoid importing packages with large dependency +// trees, like the use of "net/http" in vcs.go. + +package web + +import ( + "errors" + "io" +) + +var errHTTP = errors.New("no http in bootstrap go command") + +type HTTPError struct { + StatusCode int +} + +func (e *HTTPError) Error() string { + panic("unreachable") +} + +func Get(url string) ([]byte, error) { + return nil, errHTTP +} + +func GetMaybeInsecure(importPath string, security SecurityMode) (string, io.ReadCloser, error) { + return "", nil, errHTTP +} + +func QueryEscape(s string) string { panic("unreachable") } +func OpenBrowser(url string) bool { panic("unreachable") } diff --git a/libgo/go/cmd/go/internal/web/http.go b/libgo/go/cmd/go/internal/web/http.go new file mode 100644 index 00000000000..6e347fbf860 --- /dev/null +++ b/libgo/go/cmd/go/internal/web/http.go @@ -0,0 +1,122 @@ +// Copyright 2012 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build !cmd_go_bootstrap + +// This code is compiled into the real 'go' binary, but it is not +// compiled into the binary that is built during all.bash, so as +// to avoid needing to build net (and thus use cgo) during the +// bootstrap process. + +package web + +import ( + "crypto/tls" + "fmt" + "io" + "io/ioutil" + "log" + "net/http" + "net/url" + "time" + + "cmd/go/internal/cfg" + "cmd/internal/browser" +) + +// httpClient is the default HTTP client, but a variable so it can be +// changed by tests, without modifying http.DefaultClient. +var httpClient = http.DefaultClient + +// impatientInsecureHTTPClient is used in -insecure mode, +// when we're connecting to https servers that might not be there +// or might be using self-signed certificates. +var impatientInsecureHTTPClient = &http.Client{ + Timeout: 5 * time.Second, + Transport: &http.Transport{ + Proxy: http.ProxyFromEnvironment, + TLSClientConfig: &tls.Config{ + InsecureSkipVerify: true, + }, + }, +} + +type HTTPError struct { + status string + StatusCode int + url string +} + +func (e *HTTPError) Error() string { + return fmt.Sprintf("%s: %s", e.url, e.status) +} + +// Get returns the data from an HTTP GET request for the given URL. +func Get(url string) ([]byte, error) { + resp, err := httpClient.Get(url) + if err != nil { + return nil, err + } + defer resp.Body.Close() + if resp.StatusCode != 200 { + err := &HTTPError{status: resp.Status, StatusCode: resp.StatusCode, url: url} + + return nil, err + } + b, err := ioutil.ReadAll(resp.Body) + if err != nil { + return nil, fmt.Errorf("%s: %v", url, err) + } + return b, nil +} + +// GetMaybeInsecure returns the body of either the importPath's +// https resource or, if unavailable and permitted by the security mode, the http resource. +func GetMaybeInsecure(importPath string, security SecurityMode) (urlStr string, body io.ReadCloser, err error) { + fetch := func(scheme string) (urlStr string, res *http.Response, err error) { + u, err := url.Parse(scheme + "://" + importPath) + if err != nil { + return "", nil, err + } + u.RawQuery = "go-get=1" + urlStr = u.String() + if cfg.BuildV { + log.Printf("Fetching %s", urlStr) + } + if security == Insecure && scheme == "https" { // fail earlier + res, err = impatientInsecureHTTPClient.Get(urlStr) + } else { + res, err = httpClient.Get(urlStr) + } + return + } + closeBody := func(res *http.Response) { + if res != nil { + res.Body.Close() + } + } + urlStr, res, err := fetch("https") + if err != nil { + if cfg.BuildV { + log.Printf("https fetch failed: %v", err) + } + if security == Insecure { + closeBody(res) + urlStr, res, err = fetch("http") + } + } + if err != nil { + closeBody(res) + return "", nil, err + } + // Note: accepting a non-200 OK here, so people can serve a + // meta import in their http 404 page. + if cfg.BuildV { + log.Printf("Parsing meta tags from %s (status code %d)", urlStr, res.StatusCode) + } + return urlStr, res.Body, nil +} + +func QueryEscape(s string) string { return url.QueryEscape(s) } +func OpenBrowser(url string) bool { return browser.Open(url) } diff --git a/libgo/go/cmd/go/internal/web/security.go b/libgo/go/cmd/go/internal/web/security.go new file mode 100644 index 00000000000..1dc6f1b076f --- /dev/null +++ b/libgo/go/cmd/go/internal/web/security.go @@ -0,0 +1,16 @@ +// Copyright 2017 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package web defines helper routines for accessing HTTP/HTTPS resources. +package web + +// SecurityMode specifies whether a function should make network +// calls using insecure transports (eg, plain text HTTP). +// The zero value is "secure". +type SecurityMode int + +const ( + Secure SecurityMode = iota + Insecure +) diff --git a/libgo/go/cmd/go/internal/work/build.go b/libgo/go/cmd/go/internal/work/build.go new file mode 100644 index 00000000000..de8c4937a13 --- /dev/null +++ b/libgo/go/cmd/go/internal/work/build.go @@ -0,0 +1,3923 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package work + +import ( + "bufio" + "bytes" + "container/heap" + "debug/elf" + "errors" + "flag" + "fmt" + "go/build" + "io" + "io/ioutil" + "log" + "os" + "os/exec" + "path" + "path/filepath" + "regexp" + "runtime" + "strconv" + "strings" + "sync" + "time" + + "cmd/go/internal/base" + "cmd/go/internal/buildid" + "cmd/go/internal/cfg" + "cmd/go/internal/load" + "cmd/go/internal/str" +) + +var CmdBuild = &base.Command{ + UsageLine: "build [-o output] [-i] [build flags] [packages]", + Short: "compile packages and dependencies", + Long: ` +Build compiles the packages named by the import paths, +along with their dependencies, but it does not install the results. + +If the arguments to build are a list of .go files, build treats +them as a list of source files specifying a single package. + +When compiling a single main package, build writes +the resulting executable to an output file named after +the first source file ('go build ed.go rx.go' writes 'ed' or 'ed.exe') +or the source code directory ('go build unix/sam' writes 'sam' or 'sam.exe'). +The '.exe' suffix is added when writing a Windows executable. + +When compiling multiple packages or a single non-main package, +build compiles the packages but discards the resulting object, +serving only as a check that the packages can be built. + +When compiling packages, build ignores files that end in '_test.go'. + +The -o flag, only allowed when compiling a single package, +forces build to write the resulting executable or object +to the named output file, instead of the default behavior described +in the last two paragraphs. + +The -i flag installs the packages that are dependencies of the target. + +The build flags are shared by the build, clean, get, install, list, run, +and test commands: + + -a + force rebuilding of packages that are already up-to-date. + -n + print the commands but do not run them. + -p n + the number of programs, such as build commands or + test binaries, that can be run in parallel. + The default is the number of CPUs available. + -race + enable data race detection. + Supported only on linux/amd64, freebsd/amd64, darwin/amd64 and windows/amd64. + -msan + enable interoperation with memory sanitizer. + Supported only on linux/amd64, + and only with Clang/LLVM as the host C compiler. + -v + print the names of packages as they are compiled. + -work + print the name of the temporary work directory and + do not delete it when exiting. + -x + print the commands. + + -asmflags 'flag list' + arguments to pass on each go tool asm invocation. + -buildmode mode + build mode to use. See 'go help buildmode' for more. + -compiler name + name of compiler to use, as in runtime.Compiler (gccgo or gc). + -gccgoflags 'arg list' + arguments to pass on each gccgo compiler/linker invocation. + -gcflags 'arg list' + arguments to pass on each go tool compile invocation. + -installsuffix suffix + a suffix to use in the name of the package installation directory, + in order to keep output separate from default builds. + If using the -race flag, the install suffix is automatically set to race + or, if set explicitly, has _race appended to it. Likewise for the -msan + flag. Using a -buildmode option that requires non-default compile flags + has a similar effect. + -ldflags 'flag list' + arguments to pass on each go tool link invocation. + -linkshared + link against shared libraries previously created with + -buildmode=shared. + -pkgdir dir + install and load all packages from dir instead of the usual locations. + For example, when building with a non-standard configuration, + use -pkgdir to keep generated packages in a separate location. + -tags 'tag list' + a space-separated list of build tags to consider satisfied during the + build. For more information about build tags, see the description of + build constraints in the documentation for the go/build package. + -toolexec 'cmd args' + a program to use to invoke toolchain programs like vet and asm. + For example, instead of running asm, the go command will run + 'cmd args /path/to/asm '. + +All the flags that take a list of arguments accept a space-separated +list of strings. To embed spaces in an element in the list, surround +it with either single or double quotes. + +For more about specifying packages, see 'go help packages'. +For more about where packages and binaries are installed, +run 'go help gopath'. +For more about calling between Go and C/C++, run 'go help c'. + +Note: Build adheres to certain conventions such as those described +by 'go help gopath'. Not all projects can follow these conventions, +however. Installations that have their own conventions or that use +a separate software build system may choose to use lower-level +invocations such as 'go tool compile' and 'go tool link' to avoid +some of the overheads and design decisions of the build tool. + +See also: go install, go get, go clean. + `, +} + +const concurrentGCBackendCompilationEnabledByDefault = true + +func init() { + // break init cycle + CmdBuild.Run = runBuild + CmdInstall.Run = runInstall + + CmdBuild.Flag.BoolVar(&cfg.BuildI, "i", false, "") + CmdBuild.Flag.StringVar(&cfg.BuildO, "o", "", "output file") + + AddBuildFlags(CmdBuild) + AddBuildFlags(CmdInstall) +} + +// Note that flags consulted by other parts of the code +// (for example, buildV) are in cmd/go/internal/cfg. + +var buildAsmflags []string // -asmflags flag +var buildGcflags []string // -gcflags flag +var buildGccgoflags []string // -gccgoflags flag + +var BuildToolchain toolchain = noToolchain{} +var ldBuildmode string + +// buildCompiler implements flag.Var. +// It implements Set by updating both +// BuildToolchain and buildContext.Compiler. +type buildCompiler struct{} + +func (c buildCompiler) Set(value string) error { + switch value { + case "gc": + BuildToolchain = gcToolchain{} + case "gccgo": + BuildToolchain = gccgoToolchain{} + default: + return fmt.Errorf("unknown compiler %q", value) + } + cfg.BuildToolchainName = value + cfg.BuildToolchainCompiler = BuildToolchain.compiler + cfg.BuildToolchainLinker = BuildToolchain.linker + cfg.BuildContext.Compiler = value + return nil +} + +func (c buildCompiler) String() string { + return cfg.BuildContext.Compiler +} + +func init() { + switch build.Default.Compiler { + case "gc", "gccgo": + buildCompiler{}.Set(build.Default.Compiler) + } +} + +// addBuildFlags adds the flags common to the build, clean, get, +// install, list, run, and test commands. +func AddBuildFlags(cmd *base.Command) { + cmd.Flag.BoolVar(&cfg.BuildA, "a", false, "") + cmd.Flag.BoolVar(&cfg.BuildN, "n", false, "") + cmd.Flag.IntVar(&cfg.BuildP, "p", cfg.BuildP, "") + cmd.Flag.BoolVar(&cfg.BuildV, "v", false, "") + cmd.Flag.BoolVar(&cfg.BuildX, "x", false, "") + + cmd.Flag.Var((*base.StringsFlag)(&buildAsmflags), "asmflags", "") + cmd.Flag.Var(buildCompiler{}, "compiler", "") + cmd.Flag.StringVar(&cfg.BuildBuildmode, "buildmode", "default", "") + cmd.Flag.Var((*base.StringsFlag)(&buildGcflags), "gcflags", "") + cmd.Flag.Var((*base.StringsFlag)(&buildGccgoflags), "gccgoflags", "") + cmd.Flag.StringVar(&cfg.BuildContext.InstallSuffix, "installsuffix", "", "") + cmd.Flag.Var((*base.StringsFlag)(&cfg.BuildLdflags), "ldflags", "") + cmd.Flag.BoolVar(&cfg.BuildLinkshared, "linkshared", false, "") + cmd.Flag.StringVar(&cfg.BuildPkgdir, "pkgdir", "", "") + cmd.Flag.BoolVar(&cfg.BuildRace, "race", false, "") + cmd.Flag.BoolVar(&cfg.BuildMSan, "msan", false, "") + cmd.Flag.Var((*base.StringsFlag)(&cfg.BuildContext.BuildTags), "tags", "") + cmd.Flag.Var((*base.StringsFlag)(&cfg.BuildToolexec), "toolexec", "") + cmd.Flag.BoolVar(&cfg.BuildWork, "work", false, "") +} + +// fileExtSplit expects a filename and returns the name +// and ext (without the dot). If the file has no +// extension, ext will be empty. +func fileExtSplit(file string) (name, ext string) { + dotExt := filepath.Ext(file) + name = file[:len(file)-len(dotExt)] + if dotExt != "" { + ext = dotExt[1:] + } + return +} + +func pkgsMain(pkgs []*load.Package) (res []*load.Package) { + for _, p := range pkgs { + if p.Name == "main" { + res = append(res, p) + } + } + return res +} + +func pkgsNotMain(pkgs []*load.Package) (res []*load.Package) { + for _, p := range pkgs { + if p.Name != "main" { + res = append(res, p) + } + } + return res +} + +func oneMainPkg(pkgs []*load.Package) []*load.Package { + if len(pkgs) != 1 || pkgs[0].Name != "main" { + base.Fatalf("-buildmode=%s requires exactly one main package", cfg.BuildBuildmode) + } + return pkgs +} + +var pkgsFilter = func(pkgs []*load.Package) []*load.Package { return pkgs } + +func BuildModeInit() { + gccgo := cfg.BuildToolchainName == "gccgo" + var codegenArg string + platform := cfg.Goos + "/" + cfg.Goarch + switch cfg.BuildBuildmode { + case "archive": + pkgsFilter = pkgsNotMain + case "c-archive": + pkgsFilter = oneMainPkg + if gccgo { + codegenArg = "-fPIC" + } else { + switch platform { + case "darwin/arm", "darwin/arm64": + codegenArg = "-shared" + default: + switch cfg.Goos { + case "dragonfly", "freebsd", "linux", "netbsd", "openbsd", "solaris": + // Use -shared so that the result is + // suitable for inclusion in a PIE or + // shared library. + codegenArg = "-shared" + } + } + } + cfg.ExeSuffix = ".a" + ldBuildmode = "c-archive" + case "c-shared": + pkgsFilter = oneMainPkg + if gccgo { + codegenArg = "-fPIC" + } else { + switch platform { + case "linux/amd64", "linux/arm", "linux/arm64", "linux/386", + "android/amd64", "android/arm", "android/arm64", "android/386": + codegenArg = "-shared" + case "darwin/amd64", "darwin/386": + default: + base.Fatalf("-buildmode=c-shared not supported on %s\n", platform) + } + } + ldBuildmode = "c-shared" + case "default": + switch platform { + case "android/arm", "android/arm64", "android/amd64", "android/386": + if !gccgo { + codegenArg = "-shared" + } + ldBuildmode = "pie" + case "darwin/arm", "darwin/arm64": + if !gccgo { + codegenArg = "-shared" + } + fallthrough + default: + ldBuildmode = "exe" + } + case "exe": + pkgsFilter = pkgsMain + ldBuildmode = "exe" + case "pie": + if cfg.BuildRace { + base.Fatalf("-buildmode=pie not supported when -race is enabled") + } + if gccgo { + codegenArg = "-fPIE" + } else { + switch platform { + case "linux/386", "linux/amd64", "linux/arm", "linux/arm64", "linux/ppc64le", "linux/s390x", + "android/amd64", "android/arm", "android/arm64", "android/386": + codegenArg = "-shared" + default: + base.Fatalf("-buildmode=pie not supported on %s\n", platform) + } + } + ldBuildmode = "pie" + case "shared": + pkgsFilter = pkgsNotMain + if gccgo { + codegenArg = "-fPIC" + } else { + switch platform { + case "linux/386", "linux/amd64", "linux/arm", "linux/arm64", "linux/ppc64le", "linux/s390x": + default: + base.Fatalf("-buildmode=shared not supported on %s\n", platform) + } + codegenArg = "-dynlink" + } + if cfg.BuildO != "" { + base.Fatalf("-buildmode=shared and -o not supported together") + } + ldBuildmode = "shared" + case "plugin": + pkgsFilter = oneMainPkg + if gccgo { + codegenArg = "-fPIC" + } else { + switch platform { + case "linux/amd64", "linux/arm", "linux/arm64", "linux/386", "linux/s390x", + "android/amd64", "android/arm", "android/arm64", "android/386": + default: + base.Fatalf("-buildmode=plugin not supported on %s\n", platform) + } + codegenArg = "-dynlink" + } + cfg.ExeSuffix = ".so" + ldBuildmode = "plugin" + default: + base.Fatalf("buildmode=%s not supported", cfg.BuildBuildmode) + } + if cfg.BuildLinkshared { + if gccgo { + codegenArg = "-fPIC" + } else { + switch platform { + case "linux/386", "linux/amd64", "linux/arm", "linux/arm64", "linux/ppc64le", "linux/s390x": + buildAsmflags = append(buildAsmflags, "-D=GOBUILDMODE_shared=1") + default: + base.Fatalf("-linkshared not supported on %s\n", platform) + } + codegenArg = "-dynlink" + // TODO(mwhudson): remove -w when that gets fixed in linker. + cfg.BuildLdflags = append(cfg.BuildLdflags, "-linkshared", "-w") + } + } + if codegenArg != "" { + if gccgo { + buildGccgoflags = append([]string{codegenArg}, buildGccgoflags...) + } else { + buildAsmflags = append([]string{codegenArg}, buildAsmflags...) + buildGcflags = append([]string{codegenArg}, buildGcflags...) + } + // Don't alter InstallSuffix when modifying default codegen args. + if cfg.BuildBuildmode != "default" || cfg.BuildLinkshared { + if cfg.BuildContext.InstallSuffix != "" { + cfg.BuildContext.InstallSuffix += "_" + } + cfg.BuildContext.InstallSuffix += codegenArg[1:] + } + } + if strings.HasPrefix(runtimeVersion, "go1") && !strings.Contains(os.Args[0], "go_bootstrap") && !gccgo { + buildGcflags = append(buildGcflags, "-goversion", runtimeVersion) + } +} + +var runtimeVersion = runtime.Version() + +func runBuild(cmd *base.Command, args []string) { + InstrumentInit() + BuildModeInit() + var b Builder + b.Init() + + pkgs := load.PackagesForBuild(args) + + if len(pkgs) == 1 && pkgs[0].Name == "main" && cfg.BuildO == "" { + _, cfg.BuildO = path.Split(pkgs[0].ImportPath) + cfg.BuildO += cfg.ExeSuffix + } + + // Special case -o /dev/null by not writing at all. + if cfg.BuildO == os.DevNull { + cfg.BuildO = "" + } + + // sanity check some often mis-used options + switch cfg.BuildContext.Compiler { + case "gccgo": + if len(buildGcflags) != 0 { + fmt.Println("go build: when using gccgo toolchain, please pass compiler flags using -gccgoflags, not -gcflags") + } + if len(cfg.BuildLdflags) != 0 { + fmt.Println("go build: when using gccgo toolchain, please pass linker flags using -gccgoflags, not -ldflags") + } + case "gc": + if len(buildGccgoflags) != 0 { + fmt.Println("go build: when using gc toolchain, please pass compile flags using -gcflags, and linker flags using -ldflags") + } + } + + depMode := ModeBuild + if cfg.BuildI { + depMode = ModeInstall + } + + if cfg.BuildO != "" { + if len(pkgs) > 1 { + base.Fatalf("go build: cannot use -o with multiple packages") + } else if len(pkgs) == 0 { + base.Fatalf("no packages to build") + } + p := pkgs[0] + p.Internal.Target = cfg.BuildO + p.Stale = true // must build - not up to date + p.StaleReason = "build -o flag in use" + a := b.Action(ModeInstall, depMode, p) + b.Do(a) + return + } + + pkgs = pkgsFilter(load.Packages(args)) + + var a *Action + if cfg.BuildBuildmode == "shared" { + if libName, err := libname(args, pkgs); err != nil { + base.Fatalf("%s", err.Error()) + } else { + a = b.libaction(libName, pkgs, ModeBuild, depMode) + } + } else { + a = &Action{} + for _, p := range pkgs { + a.Deps = append(a.Deps, b.Action(ModeBuild, depMode, p)) + } + } + b.Do(a) +} + +var CmdInstall = &base.Command{ + UsageLine: "install [build flags] [packages]", + Short: "compile and install packages and dependencies", + Long: ` +Install compiles and installs the packages named by the import paths, +along with their dependencies. + +For more about the build flags, see 'go help build'. +For more about specifying packages, see 'go help packages'. + +See also: go build, go get, go clean. + `, +} + +// libname returns the filename to use for the shared library when using +// -buildmode=shared. The rules we use are: +// Use arguments for special 'meta' packages: +// std --> libstd.so +// std cmd --> libstd,cmd.so +// A single non-meta argument with trailing "/..." is special cased: +// foo/... --> libfoo.so +// (A relative path like "./..." expands the "." first) +// Use import paths for other cases, changing '/' to '-': +// somelib --> libsubdir-somelib.so +// ./ or ../ --> libsubdir-somelib.so +// gopkg.in/tomb.v2 -> libgopkg.in-tomb.v2.so +// a/... b/... ---> liba/c,b/d.so - all matching import paths +// Name parts are joined with ','. +func libname(args []string, pkgs []*load.Package) (string, error) { + var libname string + appendName := func(arg string) { + if libname == "" { + libname = arg + } else { + libname += "," + arg + } + } + var haveNonMeta bool + for _, arg := range args { + if load.IsMetaPackage(arg) { + appendName(arg) + } else { + haveNonMeta = true + } + } + if len(libname) == 0 { // non-meta packages only. use import paths + if len(args) == 1 && strings.HasSuffix(args[0], "/...") { + // Special case of "foo/..." as mentioned above. + arg := strings.TrimSuffix(args[0], "/...") + if build.IsLocalImport(arg) { + cwd, _ := os.Getwd() + bp, _ := cfg.BuildContext.ImportDir(filepath.Join(cwd, arg), build.FindOnly) + if bp.ImportPath != "" && bp.ImportPath != "." { + arg = bp.ImportPath + } + } + appendName(strings.Replace(arg, "/", "-", -1)) + } else { + for _, pkg := range pkgs { + appendName(strings.Replace(pkg.ImportPath, "/", "-", -1)) + } + } + } else if haveNonMeta { // have both meta package and a non-meta one + return "", errors.New("mixing of meta and non-meta packages is not allowed") + } + // TODO(mwhudson): Needs to change for platforms that use different naming + // conventions... + return "lib" + libname + ".so", nil +} + +func runInstall(cmd *base.Command, args []string) { + InstrumentInit() + BuildModeInit() + InstallPackages(args, false) +} + +func InstallPackages(args []string, forGet bool) { + if cfg.GOBIN != "" && !filepath.IsAbs(cfg.GOBIN) { + base.Fatalf("cannot install, GOBIN must be an absolute path") + } + + pkgs := pkgsFilter(load.PackagesForBuild(args)) + + for _, p := range pkgs { + if p.Target == "" && (!p.Standard || p.ImportPath != "unsafe") { + switch { + case p.Internal.GobinSubdir: + base.Errorf("go install: cannot install cross-compiled binaries when GOBIN is set") + case p.Internal.Cmdline: + base.Errorf("go install: no install location for .go files listed on command line (GOBIN not set)") + case p.ConflictDir != "": + base.Errorf("go install: no install location for %s: hidden by %s", p.Dir, p.ConflictDir) + default: + base.Errorf("go install: no install location for directory %s outside GOPATH\n"+ + "\tFor more details see: 'go help gopath'", p.Dir) + } + } + } + base.ExitIfErrors() + + var b Builder + b.Init() + var a *Action + if cfg.BuildBuildmode == "shared" { + if libName, err := libname(args, pkgs); err != nil { + base.Fatalf("%s", err.Error()) + } else { + a = b.libaction(libName, pkgs, ModeInstall, ModeInstall) + } + } else { + a = &Action{} + var tools []*Action + for _, p := range pkgs { + // During 'go get', don't attempt (and fail) to install packages with only tests. + // TODO(rsc): It's not clear why 'go get' should be different from 'go install' here. See #20760. + if forGet && len(p.GoFiles)+len(p.CgoFiles) == 0 && len(p.TestGoFiles)+len(p.XTestGoFiles) > 0 { + continue + } + // If p is a tool, delay the installation until the end of the build. + // This avoids installing assemblers/compilers that are being executed + // by other steps in the build. + // cmd/cgo is handled specially in b.Action, so that we can + // both build and use it in the same 'go install'. + Action := b.Action(ModeInstall, ModeInstall, p) + if load.GoTools[p.ImportPath] == load.ToTool && p.ImportPath != "cmd/cgo" { + a.Deps = append(a.Deps, Action.Deps...) + Action.Deps = append(Action.Deps, a) + tools = append(tools, Action) + continue + } + a.Deps = append(a.Deps, Action) + } + if len(tools) > 0 { + a = &Action{ + Deps: tools, + } + } + } + b.Do(a) + base.ExitIfErrors() + + // Success. If this command is 'go install' with no arguments + // and the current directory (the implicit argument) is a command, + // remove any leftover command binary from a previous 'go build'. + // The binary is installed; it's not needed here anymore. + // And worse it might be a stale copy, which you don't want to find + // instead of the installed one if $PATH contains dot. + // One way to view this behavior is that it is as if 'go install' first + // runs 'go build' and the moves the generated file to the install dir. + // See issue 9645. + if len(args) == 0 && len(pkgs) == 1 && pkgs[0].Name == "main" { + // Compute file 'go build' would have created. + // If it exists and is an executable file, remove it. + _, targ := filepath.Split(pkgs[0].ImportPath) + targ += cfg.ExeSuffix + if filepath.Join(pkgs[0].Dir, targ) != pkgs[0].Target { // maybe $GOBIN is the current directory + fi, err := os.Stat(targ) + if err == nil { + m := fi.Mode() + if m.IsRegular() { + if m&0111 != 0 || cfg.Goos == "windows" { // windows never sets executable bit + os.Remove(targ) + } + } + } + } + } +} + +// A Builder holds global state about a build. +// It does not hold per-package state, because we +// build packages in parallel, and the builder is shared. +type Builder struct { + WorkDir string // the temporary work directory (ends in filepath.Separator) + actionCache map[cacheKey]*Action // a cache of already-constructed actions + mkdirCache map[string]bool // a cache of created directories + flagCache map[string]bool // a cache of supported compiler flags + Print func(args ...interface{}) (int, error) + + output sync.Mutex + scriptDir string // current directory in printed script + + exec sync.Mutex + readySema chan bool + ready actionQueue +} + +// NOTE: Much of Action would not need to be exported if not for test. +// Maybe test functionality should move into this package too? + +// An Action represents a single action in the action graph. +type Action struct { + Package *load.Package // the package this action works on + Deps []*Action // actions that must happen before this one + Func func(*Builder, *Action) error // the action itself (nil = no-op) + IgnoreFail bool // whether to run f even if dependencies fail + TestOutput *bytes.Buffer // test output buffer + Args []string // additional args for runProgram + + triggers []*Action // inverse of deps + cgo *Action // action for cgo binary if needed + + // Generated files, directories. + Link bool // target is executable, not just package + Pkgdir string // the -I or -L argument to use when importing this package + Objdir string // directory for intermediate objects + Objpkg string // the intermediate package .a file created during the action + Target string // goal of the action: the created package or executable + + // Execution state. + pending int // number of deps yet to complete + priority int // relative execution priority + Failed bool // whether the action failed +} + +// cacheKey is the key for the action cache. +type cacheKey struct { + mode BuildMode + p *load.Package + shlib string +} + +// BuildMode specifies the build mode: +// are we just building things or also installing the results? +type BuildMode int + +const ( + ModeBuild BuildMode = iota + ModeInstall +) + +func (b *Builder) Init() { + var err error + b.Print = func(a ...interface{}) (int, error) { + return fmt.Fprint(os.Stderr, a...) + } + b.actionCache = make(map[cacheKey]*Action) + b.mkdirCache = make(map[string]bool) + + if cfg.BuildN { + b.WorkDir = "$WORK" + } else { + b.WorkDir, err = ioutil.TempDir("", "go-build") + if err != nil { + base.Fatalf("%s", err) + } + if cfg.BuildX || cfg.BuildWork { + fmt.Fprintf(os.Stderr, "WORK=%s\n", b.WorkDir) + } + if !cfg.BuildWork { + workdir := b.WorkDir + base.AtExit(func() { os.RemoveAll(workdir) }) + } + } +} + +// readpkglist returns the list of packages that were built into the shared library +// at shlibpath. For the native toolchain this list is stored, newline separated, in +// an ELF note with name "Go\x00\x00" and type 1. For GCCGO it is extracted from the +// .go_export section. +func readpkglist(shlibpath string) (pkgs []*load.Package) { + var stk load.ImportStack + if cfg.BuildToolchainName == "gccgo" { + f, _ := elf.Open(shlibpath) + sect := f.Section(".go_export") + data, _ := sect.Data() + scanner := bufio.NewScanner(bytes.NewBuffer(data)) + for scanner.Scan() { + t := scanner.Text() + if strings.HasPrefix(t, "pkgpath ") { + t = strings.TrimPrefix(t, "pkgpath ") + t = strings.TrimSuffix(t, ";") + pkgs = append(pkgs, load.LoadPackage(t, &stk)) + } + } + } else { + pkglistbytes, err := buildid.ReadELFNote(shlibpath, "Go\x00\x00", 1) + if err != nil { + base.Fatalf("readELFNote failed: %v", err) + } + scanner := bufio.NewScanner(bytes.NewBuffer(pkglistbytes)) + for scanner.Scan() { + t := scanner.Text() + pkgs = append(pkgs, load.LoadPackage(t, &stk)) + } + } + return +} + +// Action returns the action for applying the given operation (mode) to the package. +// depMode is the action to use when building dependencies. +// action never looks for p in a shared library, but may find p's dependencies in a +// shared library if buildLinkshared is true. +func (b *Builder) Action(mode BuildMode, depMode BuildMode, p *load.Package) *Action { + return b.action1(mode, depMode, p, false, "") +} + +// action1 returns the action for applying the given operation (mode) to the package. +// depMode is the action to use when building dependencies. +// action1 will look for p in a shared library if lookshared is true. +// forShlib is the shared library that p will become part of, if any. +func (b *Builder) action1(mode BuildMode, depMode BuildMode, p *load.Package, lookshared bool, forShlib string) *Action { + shlib := "" + if lookshared { + shlib = p.Shlib + } + key := cacheKey{mode, p, shlib} + + a := b.actionCache[key] + if a != nil { + return a + } + if shlib != "" { + key2 := cacheKey{ModeInstall, nil, shlib} + a = b.actionCache[key2] + if a != nil { + b.actionCache[key] = a + return a + } + pkgs := readpkglist(shlib) + a = b.libaction(filepath.Base(shlib), pkgs, ModeInstall, depMode) + b.actionCache[key2] = a + b.actionCache[key] = a + return a + } + + a = &Action{Package: p, Pkgdir: p.Internal.Build.PkgRoot} + if p.Internal.Pkgdir != "" { // overrides p.t + a.Pkgdir = p.Internal.Pkgdir + } + b.actionCache[key] = a + + for _, p1 := range p.Internal.Imports { + if forShlib != "" { + // p is part of a shared library. + if p1.Shlib != "" && p1.Shlib != forShlib { + // p1 is explicitly part of a different shared library. + // Put the action for that shared library into a.Deps. + a.Deps = append(a.Deps, b.action1(depMode, depMode, p1, true, p1.Shlib)) + } else { + // p1 is (implicitly or not) part of this shared library. + // Put the action for p1 into a.Deps. + a.Deps = append(a.Deps, b.action1(depMode, depMode, p1, false, forShlib)) + } + } else { + // p is not part of a shared library. + // If p1 is in a shared library, put the action for that into + // a.Deps, otherwise put the action for p1 into a.Deps. + a.Deps = append(a.Deps, b.action1(depMode, depMode, p1, cfg.BuildLinkshared, p1.Shlib)) + } + } + + // If we are not doing a cross-build, then record the binary we'll + // generate for cgo as a dependency of the build of any package + // using cgo, to make sure we do not overwrite the binary while + // a package is using it. If this is a cross-build, then the cgo we + // are writing is not the cgo we need to use. + if cfg.Goos == runtime.GOOS && cfg.Goarch == runtime.GOARCH && !cfg.BuildRace && !cfg.BuildMSan && cfg.BuildToolchainName != "gccgo" { + if (len(p.CgoFiles) > 0 || p.Standard && p.ImportPath == "runtime/cgo") && !cfg.BuildLinkshared && cfg.BuildBuildmode != "shared" { + var stk load.ImportStack + p1 := load.LoadPackage("cmd/cgo", &stk) + if p1.Error != nil { + base.Fatalf("load cmd/cgo: %v", p1.Error) + } + a.cgo = b.Action(depMode, depMode, p1) + a.Deps = append(a.Deps, a.cgo) + } + } + + if p.Standard { + switch p.ImportPath { + case "builtin", "unsafe": + // Fake packages - nothing to build. + return a + } + // gccgo standard library is "fake" too. + if cfg.BuildToolchainName == "gccgo" { + // the target name is needed for cgo. + a.Target = p.Internal.Target + return a + } + } + + if !p.Stale && p.Internal.Target != "" { + // p.Stale==false implies that p.Internal.Target is up-to-date. + // Record target name for use by actions depending on this one. + a.Target = p.Internal.Target + return a + } + + if p.Internal.Local && p.Internal.Target == "" { + // Imported via local path. No permanent target. + mode = ModeBuild + } + work := p.Internal.Pkgdir + if work == "" { + work = b.WorkDir + } + a.Objdir = filepath.Join(work, a.Package.ImportPath, "_obj") + string(filepath.Separator) + a.Objpkg = BuildToolchain.Pkgpath(work, a.Package) + a.Link = p.Name == "main" + + switch mode { + case ModeInstall: + a.Func = BuildInstallFunc + a.Deps = []*Action{b.action1(ModeBuild, depMode, p, lookshared, forShlib)} + a.Target = a.Package.Internal.Target + + // Install header for cgo in c-archive and c-shared modes. + if p.UsesCgo() && (cfg.BuildBuildmode == "c-archive" || cfg.BuildBuildmode == "c-shared") { + hdrTarget := a.Target[:len(a.Target)-len(filepath.Ext(a.Target))] + ".h" + if cfg.BuildContext.Compiler == "gccgo" && cfg.BuildO == "" { + // For the header file, remove the "lib" + // added by go/build, so we generate pkg.h + // rather than libpkg.h. + dir, file := filepath.Split(hdrTarget) + file = strings.TrimPrefix(file, "lib") + hdrTarget = filepath.Join(dir, file) + } + ah := &Action{ + Package: a.Package, + Deps: []*Action{a.Deps[0]}, + Func: (*Builder).installHeader, + Pkgdir: a.Pkgdir, + Objdir: a.Objdir, + Target: hdrTarget, + } + a.Deps = append(a.Deps, ah) + } + + case ModeBuild: + a.Func = (*Builder).build + a.Target = a.Objpkg + if a.Link { + // An executable file. (This is the name of a temporary file.) + // Because we run the temporary file in 'go run' and 'go test', + // the name will show up in ps listings. If the caller has specified + // a name, use that instead of a.out. The binary is generated + // in an otherwise empty subdirectory named exe to avoid + // naming conflicts. The only possible conflict is if we were + // to create a top-level package named exe. + name := "a.out" + if p.Internal.ExeName != "" { + name = p.Internal.ExeName + } else if cfg.Goos == "darwin" && cfg.BuildBuildmode == "c-shared" && p.Internal.Target != "" { + // On OS X, the linker output name gets recorded in the + // shared library's LC_ID_DYLIB load command. + // The code invoking the linker knows to pass only the final + // path element. Arrange that the path element matches what + // we'll install it as; otherwise the library is only loadable as "a.out". + _, name = filepath.Split(p.Internal.Target) + } + a.Target = a.Objdir + filepath.Join("exe", name) + cfg.ExeSuffix + } + } + + return a +} + +func (b *Builder) libaction(libname string, pkgs []*load.Package, mode, depMode BuildMode) *Action { + a := &Action{} + switch mode { + default: + base.Fatalf("unrecognized mode %v", mode) + + case ModeBuild: + a.Func = (*Builder).linkShared + a.Target = filepath.Join(b.WorkDir, libname) + for _, p := range pkgs { + if p.Internal.Target == "" { + continue + } + a.Deps = append(a.Deps, b.Action(depMode, depMode, p)) + } + + case ModeInstall: + // Currently build mode shared forces external linking mode, and + // external linking mode forces an import of runtime/cgo (and + // math on arm). So if it was not passed on the command line and + // it is not present in another shared library, add it here. + gccgo := cfg.BuildToolchainName == "gccgo" + if !gccgo { + seencgo := false + for _, p := range pkgs { + seencgo = seencgo || (p.Standard && p.ImportPath == "runtime/cgo") + } + if !seencgo { + var stk load.ImportStack + p := load.LoadPackage("runtime/cgo", &stk) + if p.Error != nil { + base.Fatalf("load runtime/cgo: %v", p.Error) + } + load.ComputeStale(p) + // If runtime/cgo is in another shared library, then that's + // also the shared library that contains runtime, so + // something will depend on it and so runtime/cgo's staleness + // will be checked when processing that library. + if p.Shlib == "" || p.Shlib == libname { + pkgs = append([]*load.Package{}, pkgs...) + pkgs = append(pkgs, p) + } + } + if cfg.Goarch == "arm" { + seenmath := false + for _, p := range pkgs { + seenmath = seenmath || (p.Standard && p.ImportPath == "math") + } + if !seenmath { + var stk load.ImportStack + p := load.LoadPackage("math", &stk) + if p.Error != nil { + base.Fatalf("load math: %v", p.Error) + } + load.ComputeStale(p) + // If math is in another shared library, then that's + // also the shared library that contains runtime, so + // something will depend on it and so math's staleness + // will be checked when processing that library. + if p.Shlib == "" || p.Shlib == libname { + pkgs = append([]*load.Package{}, pkgs...) + pkgs = append(pkgs, p) + } + } + } + } + + // Figure out where the library will go. + var libdir string + for _, p := range pkgs { + plibdir := p.Internal.Build.PkgTargetRoot + if gccgo { + plibdir = filepath.Join(plibdir, "shlibs") + } + if libdir == "" { + libdir = plibdir + } else if libdir != plibdir { + base.Fatalf("multiple roots %s & %s", libdir, plibdir) + } + } + a.Target = filepath.Join(libdir, libname) + + // Now we can check whether we need to rebuild it. + stale := false + var built time.Time + if fi, err := os.Stat(a.Target); err == nil { + built = fi.ModTime() + } + for _, p := range pkgs { + if p.Internal.Target == "" { + continue + } + stale = stale || p.Stale + lstat, err := os.Stat(p.Internal.Target) + if err != nil || lstat.ModTime().After(built) { + stale = true + } + a.Deps = append(a.Deps, b.action1(depMode, depMode, p, false, a.Target)) + } + + if stale { + a.Func = BuildInstallFunc + buildAction := b.libaction(libname, pkgs, ModeBuild, depMode) + a.Deps = []*Action{buildAction} + for _, p := range pkgs { + if p.Internal.Target == "" { + continue + } + shlibnameaction := &Action{} + shlibnameaction.Func = (*Builder).installShlibname + shlibnameaction.Target = p.Internal.Target[:len(p.Internal.Target)-2] + ".shlibname" + a.Deps = append(a.Deps, shlibnameaction) + shlibnameaction.Deps = append(shlibnameaction.Deps, buildAction) + } + } + } + return a +} + +// ActionList returns the list of actions in the dag rooted at root +// as visited in a depth-first post-order traversal. +func ActionList(root *Action) []*Action { + seen := map[*Action]bool{} + all := []*Action{} + var walk func(*Action) + walk = func(a *Action) { + if seen[a] { + return + } + seen[a] = true + for _, a1 := range a.Deps { + walk(a1) + } + all = append(all, a) + } + walk(root) + return all +} + +// allArchiveActions returns a list of the archive dependencies of root. +// This is needed because if package p depends on package q that is in libr.so, the +// action graph looks like p->libr.so->q and so just scanning through p's +// dependencies does not find the import dir for q. +func allArchiveActions(root *Action) []*Action { + seen := map[*Action]bool{} + r := []*Action{} + var walk func(*Action) + walk = func(a *Action) { + if seen[a] { + return + } + seen[a] = true + if strings.HasSuffix(a.Target, ".so") || a == root { + for _, a1 := range a.Deps { + walk(a1) + } + } else if strings.HasSuffix(a.Target, ".a") { + r = append(r, a) + } + } + walk(root) + return r +} + +// do runs the action graph rooted at root. +func (b *Builder) Do(root *Action) { + /* Commented for gccgo, which does not have OSArchSupportsCgo. + + if _, ok := cfg.OSArchSupportsCgo[cfg.Goos+"/"+cfg.Goarch]; !ok && cfg.BuildContext.Compiler == "gc" { + fmt.Fprintf(os.Stderr, "cmd/go: unsupported GOOS/GOARCH pair %s/%s\n", cfg.Goos, cfg.Goarch) + os.Exit(2) + } + */ + + for _, tag := range cfg.BuildContext.BuildTags { + if strings.Contains(tag, ",") { + fmt.Fprintf(os.Stderr, "cmd/go: -tags space-separated list contains comma\n") + os.Exit(2) + } + } + + // Build list of all actions, assigning depth-first post-order priority. + // The original implementation here was a true queue + // (using a channel) but it had the effect of getting + // distracted by low-level leaf actions to the detriment + // of completing higher-level actions. The order of + // work does not matter much to overall execution time, + // but when running "go test std" it is nice to see each test + // results as soon as possible. The priorities assigned + // ensure that, all else being equal, the execution prefers + // to do what it would have done first in a simple depth-first + // dependency order traversal. + all := ActionList(root) + for i, a := range all { + a.priority = i + } + + b.readySema = make(chan bool, len(all)) + + // Initialize per-action execution state. + for _, a := range all { + for _, a1 := range a.Deps { + a1.triggers = append(a1.triggers, a) + } + a.pending = len(a.Deps) + if a.pending == 0 { + b.ready.push(a) + b.readySema <- true + } + } + + // Handle runs a single action and takes care of triggering + // any actions that are runnable as a result. + handle := func(a *Action) { + var err error + if a.Func != nil && (!a.Failed || a.IgnoreFail) { + err = a.Func(b, a) + } + + // The actions run in parallel but all the updates to the + // shared work state are serialized through b.exec. + b.exec.Lock() + defer b.exec.Unlock() + + if err != nil { + if err == errPrintedOutput { + base.SetExitStatus(2) + } else { + base.Errorf("%s", err) + } + a.Failed = true + } + + for _, a0 := range a.triggers { + if a.Failed { + a0.Failed = true + } + if a0.pending--; a0.pending == 0 { + b.ready.push(a0) + b.readySema <- true + } + } + + if a == root { + close(b.readySema) + } + } + + var wg sync.WaitGroup + + // Kick off goroutines according to parallelism. + // If we are using the -n flag (just printing commands) + // drop the parallelism to 1, both to make the output + // deterministic and because there is no real work anyway. + par := cfg.BuildP + if cfg.BuildN { + par = 1 + } + for i := 0; i < par; i++ { + wg.Add(1) + go func() { + defer wg.Done() + for { + select { + case _, ok := <-b.readySema: + if !ok { + return + } + // Receiving a value from b.readySema entitles + // us to take from the ready queue. + b.exec.Lock() + a := b.ready.pop() + b.exec.Unlock() + handle(a) + case <-base.Interrupted: + base.SetExitStatus(1) + return + } + } + }() + } + + wg.Wait() +} + +// build is the action for building a single package or command. +func (b *Builder) build(a *Action) (err error) { + // Return an error for binary-only package. + // We only reach this if isStale believes the binary form is + // either not present or not usable. + if a.Package.BinaryOnly { + return fmt.Errorf("missing or invalid package binary for binary-only package %s", a.Package.ImportPath) + } + + // Return an error if the package has CXX files but it's not using + // cgo nor SWIG, since the CXX files can only be processed by cgo + // and SWIG. + if len(a.Package.CXXFiles) > 0 && !a.Package.UsesCgo() && !a.Package.UsesSwig() { + return fmt.Errorf("can't build package %s because it contains C++ files (%s) but it's not using cgo nor SWIG", + a.Package.ImportPath, strings.Join(a.Package.CXXFiles, ",")) + } + // Same as above for Objective-C files + if len(a.Package.MFiles) > 0 && !a.Package.UsesCgo() && !a.Package.UsesSwig() { + return fmt.Errorf("can't build package %s because it contains Objective-C files (%s) but it's not using cgo nor SWIG", + a.Package.ImportPath, strings.Join(a.Package.MFiles, ",")) + } + // Same as above for Fortran files + if len(a.Package.FFiles) > 0 && !a.Package.UsesCgo() && !a.Package.UsesSwig() { + return fmt.Errorf("can't build package %s because it contains Fortran files (%s) but it's not using cgo nor SWIG", + a.Package.ImportPath, strings.Join(a.Package.FFiles, ",")) + } + + defer func() { + if err != nil && err != errPrintedOutput { + err = fmt.Errorf("go build %s: %v", a.Package.ImportPath, err) + } + }() + if cfg.BuildN { + // In -n mode, print a banner between packages. + // The banner is five lines so that when changes to + // different sections of the bootstrap script have to + // be merged, the banners give patch something + // to use to find its context. + b.Print("\n#\n# " + a.Package.ImportPath + "\n#\n\n") + } + + if cfg.BuildV { + b.Print(a.Package.ImportPath + "\n") + } + + // Make build directory. + obj := a.Objdir + if err := b.Mkdir(obj); err != nil { + return err + } + + // make target directory + dir, _ := filepath.Split(a.Target) + if dir != "" { + if err := b.Mkdir(dir); err != nil { + return err + } + } + + var gofiles, cgofiles, objdirCgofiles, cfiles, sfiles, cxxfiles, objects, cgoObjects, pcCFLAGS, pcLDFLAGS []string + + gofiles = append(gofiles, a.Package.GoFiles...) + cgofiles = append(cgofiles, a.Package.CgoFiles...) + cfiles = append(cfiles, a.Package.CFiles...) + sfiles = append(sfiles, a.Package.SFiles...) + cxxfiles = append(cxxfiles, a.Package.CXXFiles...) + + if a.Package.UsesCgo() || a.Package.UsesSwig() { + if pcCFLAGS, pcLDFLAGS, err = b.getPkgConfigFlags(a.Package); err != nil { + return + } + } + + // Run SWIG on each .swig and .swigcxx file. + // Each run will generate two files, a .go file and a .c or .cxx file. + // The .go file will use import "C" and is to be processed by cgo. + if a.Package.UsesSwig() { + outGo, outC, outCXX, err := b.swig(a.Package, obj, pcCFLAGS) + if err != nil { + return err + } + objdirCgofiles = append(objdirCgofiles, outGo...) + cfiles = append(cfiles, outC...) + cxxfiles = append(cxxfiles, outCXX...) + } + + // Run cgo. + if a.Package.UsesCgo() || a.Package.UsesSwig() { + // In a package using cgo, cgo compiles the C, C++ and assembly files with gcc. + // There is one exception: runtime/cgo's job is to bridge the + // cgo and non-cgo worlds, so it necessarily has files in both. + // In that case gcc only gets the gcc_* files. + var gccfiles []string + gccfiles = append(gccfiles, cfiles...) + cfiles = nil + if a.Package.Standard && a.Package.ImportPath == "runtime/cgo" { + filter := func(files, nongcc, gcc []string) ([]string, []string) { + for _, f := range files { + if strings.HasPrefix(f, "gcc_") { + gcc = append(gcc, f) + } else { + nongcc = append(nongcc, f) + } + } + return nongcc, gcc + } + sfiles, gccfiles = filter(sfiles, sfiles[:0], gccfiles) + } else { + for _, sfile := range sfiles { + data, err := ioutil.ReadFile(filepath.Join(a.Package.Dir, sfile)) + if err == nil { + if bytes.HasPrefix(data, []byte("TEXT")) || bytes.Contains(data, []byte("\nTEXT")) || + bytes.HasPrefix(data, []byte("DATA")) || bytes.Contains(data, []byte("\nDATA")) || + bytes.HasPrefix(data, []byte("GLOBL")) || bytes.Contains(data, []byte("\nGLOBL")) { + return fmt.Errorf("package using cgo has Go assembly file %s", sfile) + } + } + } + gccfiles = append(gccfiles, sfiles...) + sfiles = nil + } + + var cgoExe string + if a.cgo != nil && a.cgo.Target != "" { + cgoExe = a.cgo.Target + } else { + cgoExe = base.Tool("cgo") + } + outGo, outObj, err := b.cgo(a, cgoExe, obj, pcCFLAGS, pcLDFLAGS, cgofiles, objdirCgofiles, gccfiles, cxxfiles, a.Package.MFiles, a.Package.FFiles) + if err != nil { + return err + } + if cfg.BuildToolchainName == "gccgo" { + cgoObjects = append(cgoObjects, filepath.Join(a.Objdir, "_cgo_flags")) + } + cgoObjects = append(cgoObjects, outObj...) + gofiles = append(gofiles, outGo...) + } + + if len(gofiles) == 0 { + return &load.NoGoError{Package: a.Package} + } + + // If we're doing coverage, preprocess the .go files and put them in the work directory + if a.Package.Internal.CoverMode != "" { + for i, file := range gofiles { + var sourceFile string + var coverFile string + var key string + if strings.HasSuffix(file, ".cgo1.go") { + // cgo files have absolute paths + base := filepath.Base(file) + sourceFile = file + coverFile = filepath.Join(obj, base) + key = strings.TrimSuffix(base, ".cgo1.go") + ".go" + } else { + sourceFile = filepath.Join(a.Package.Dir, file) + coverFile = filepath.Join(obj, file) + key = file + } + cover := a.Package.Internal.CoverVars[key] + if cover == nil || base.IsTestFile(file) { + // Not covering this file. + continue + } + if err := b.cover(a, coverFile, sourceFile, 0666, cover.Var); err != nil { + return err + } + gofiles[i] = coverFile + } + } + + // Prepare Go import path list. + inc := b.includeArgs("-I", allArchiveActions(a)) + + // Compile Go. + ofile, out, err := BuildToolchain.gc(b, a.Package, a.Objpkg, obj, len(sfiles) > 0, inc, gofiles) + if len(out) > 0 { + b.showOutput(a.Package.Dir, a.Package.ImportPath, b.processOutput(out)) + if err != nil { + return errPrintedOutput + } + } + if err != nil { + return err + } + if ofile != a.Objpkg { + objects = append(objects, ofile) + } + + // Copy .h files named for goos or goarch or goos_goarch + // to names using GOOS and GOARCH. + // For example, defs_linux_amd64.h becomes defs_GOOS_GOARCH.h. + _goos_goarch := "_" + cfg.Goos + "_" + cfg.Goarch + _goos := "_" + cfg.Goos + _goarch := "_" + cfg.Goarch + for _, file := range a.Package.HFiles { + name, ext := fileExtSplit(file) + switch { + case strings.HasSuffix(name, _goos_goarch): + targ := file[:len(name)-len(_goos_goarch)] + "_GOOS_GOARCH." + ext + if err := b.copyFile(a, obj+targ, filepath.Join(a.Package.Dir, file), 0666, true); err != nil { + return err + } + case strings.HasSuffix(name, _goarch): + targ := file[:len(name)-len(_goarch)] + "_GOARCH." + ext + if err := b.copyFile(a, obj+targ, filepath.Join(a.Package.Dir, file), 0666, true); err != nil { + return err + } + case strings.HasSuffix(name, _goos): + targ := file[:len(name)-len(_goos)] + "_GOOS." + ext + if err := b.copyFile(a, obj+targ, filepath.Join(a.Package.Dir, file), 0666, true); err != nil { + return err + } + } + } + + for _, file := range cfiles { + out := file[:len(file)-len(".c")] + ".o" + if err := BuildToolchain.cc(b, a.Package, obj, obj+out, file); err != nil { + return err + } + objects = append(objects, out) + } + + // Assemble .s files. + if len(sfiles) > 0 { + ofiles, err := BuildToolchain.asm(b, a.Package, obj, sfiles) + if err != nil { + return err + } + objects = append(objects, ofiles...) + } + + // NOTE(rsc): On Windows, it is critically important that the + // gcc-compiled objects (cgoObjects) be listed after the ordinary + // objects in the archive. I do not know why this is. + // https://golang.org/issue/2601 + objects = append(objects, cgoObjects...) + + // Add system object files. + for _, syso := range a.Package.SysoFiles { + objects = append(objects, filepath.Join(a.Package.Dir, syso)) + } + + // Pack into archive in obj directory. + // If the Go compiler wrote an archive, we only need to add the + // object files for non-Go sources to the archive. + // If the Go compiler wrote an archive and the package is entirely + // Go sources, there is no pack to execute at all. + if len(objects) > 0 { + if err := BuildToolchain.pack(b, a.Package, obj, a.Objpkg, objects); err != nil { + return err + } + } + + // Link if needed. + if a.Link { + // The compiler only cares about direct imports, but the + // linker needs the whole dependency tree. + all := ActionList(a) + all = all[:len(all)-1] // drop a + if err := BuildToolchain.ld(b, a, a.Target, all, a.Objpkg, objects); err != nil { + return err + } + } + + return nil +} + +// PkgconfigCmd returns a pkg-config binary name +// defaultPkgConfig is defined in zdefaultcc.go, written by cmd/dist. +func (b *Builder) PkgconfigCmd() string { + return envList("PKG_CONFIG", cfg.DefaultPkgConfig)[0] +} + +// splitPkgConfigOutput parses the pkg-config output into a slice of +// flags. pkg-config always uses \ to escape special characters. +func splitPkgConfigOutput(out []byte) []string { + if len(out) == 0 { + return nil + } + var flags []string + flag := make([]byte, len(out)) + r, w := 0, 0 + for r < len(out) { + switch out[r] { + case ' ', '\t', '\r', '\n': + if w > 0 { + flags = append(flags, string(flag[:w])) + } + w = 0 + case '\\': + r++ + fallthrough + default: + if r < len(out) { + flag[w] = out[r] + w++ + } + } + r++ + } + if w > 0 { + flags = append(flags, string(flag[:w])) + } + return flags +} + +// Calls pkg-config if needed and returns the cflags/ldflags needed to build the package. +func (b *Builder) getPkgConfigFlags(p *load.Package) (cflags, ldflags []string, err error) { + if pkgs := p.CgoPkgConfig; len(pkgs) > 0 { + var out []byte + out, err = b.runOut(p.Dir, p.ImportPath, nil, b.PkgconfigCmd(), "--cflags", pkgs) + if err != nil { + b.showOutput(p.Dir, b.PkgconfigCmd()+" --cflags "+strings.Join(pkgs, " "), string(out)) + b.Print(err.Error() + "\n") + err = errPrintedOutput + return + } + if len(out) > 0 { + cflags = splitPkgConfigOutput(out) + } + out, err = b.runOut(p.Dir, p.ImportPath, nil, b.PkgconfigCmd(), "--libs", pkgs) + if err != nil { + b.showOutput(p.Dir, b.PkgconfigCmd()+" --libs "+strings.Join(pkgs, " "), string(out)) + b.Print(err.Error() + "\n") + err = errPrintedOutput + return + } + if len(out) > 0 { + ldflags = strings.Fields(string(out)) + } + } + return +} + +func (b *Builder) installShlibname(a *Action) error { + a1 := a.Deps[0] + err := ioutil.WriteFile(a.Target, []byte(filepath.Base(a1.Target)+"\n"), 0666) + if err != nil { + return err + } + if cfg.BuildX { + b.Showcmd("", "echo '%s' > %s # internal", filepath.Base(a1.Target), a.Target) + } + return nil +} + +func (b *Builder) linkShared(a *Action) (err error) { + allactions := ActionList(a) + allactions = allactions[:len(allactions)-1] + return BuildToolchain.ldShared(b, a.Deps, a.Target, allactions) +} + +// BuildInstallFunc is the action for installing a single package or executable. +func BuildInstallFunc(b *Builder, a *Action) (err error) { + defer func() { + if err != nil && err != errPrintedOutput { + err = fmt.Errorf("go install %s: %v", a.Package.ImportPath, err) + } + }() + a1 := a.Deps[0] + perm := os.FileMode(0666) + if a1.Link { + switch cfg.BuildBuildmode { + case "c-archive", "c-shared", "plugin": + default: + perm = 0777 + } + } + + // make target directory + dir, _ := filepath.Split(a.Target) + if dir != "" { + if err := b.Mkdir(dir); err != nil { + return err + } + } + + // remove object dir to keep the amount of + // garbage down in a large build. On an operating system + // with aggressive buffering, cleaning incrementally like + // this keeps the intermediate objects from hitting the disk. + if !cfg.BuildWork { + defer os.RemoveAll(a1.Objdir) + defer os.Remove(a1.Target) + } + + return b.moveOrCopyFile(a, a.Target, a1.Target, perm, false) +} + +// includeArgs returns the -I or -L directory list for access +// to the results of the list of actions. +func (b *Builder) includeArgs(flag string, all []*Action) []string { + inc := []string{} + incMap := map[string]bool{ + b.WorkDir: true, // handled later + cfg.GOROOTpkg: true, + "": true, // ignore empty strings + } + + // Look in the temporary space for results of test-specific actions. + // This is the $WORK/my/package/_test directory for the + // package being built, so there are few of these. + for _, a1 := range all { + if a1.Package == nil { + continue + } + if dir := a1.Pkgdir; dir != a1.Package.Internal.Build.PkgRoot && !incMap[dir] { + incMap[dir] = true + inc = append(inc, flag, dir) + } + } + + // Also look in $WORK for any non-test packages that have + // been built but not installed. + inc = append(inc, flag, b.WorkDir) + + // Finally, look in the installed package directories for each action. + // First add the package dirs corresponding to GOPATH entries + // in the original GOPATH order. + need := map[string]*build.Package{} + for _, a1 := range all { + if a1.Package != nil && a1.Pkgdir == a1.Package.Internal.Build.PkgRoot { + need[a1.Package.Internal.Build.Root] = a1.Package.Internal.Build + } + } + for _, root := range cfg.Gopath { + if p := need[root]; p != nil && !incMap[p.PkgRoot] { + incMap[p.PkgRoot] = true + inc = append(inc, flag, p.PkgTargetRoot) + } + } + + // Then add anything that's left. + for _, a1 := range all { + if a1.Package == nil { + continue + } + if dir := a1.Pkgdir; dir == a1.Package.Internal.Build.PkgRoot && !incMap[dir] { + incMap[dir] = true + inc = append(inc, flag, a1.Package.Internal.Build.PkgTargetRoot) + } + } + + return inc +} + +// moveOrCopyFile is like 'mv src dst' or 'cp src dst'. +func (b *Builder) moveOrCopyFile(a *Action, dst, src string, perm os.FileMode, force bool) error { + if cfg.BuildN { + b.Showcmd("", "mv %s %s", src, dst) + return nil + } + + // If we can update the mode and rename to the dst, do it. + // Otherwise fall back to standard copy. + + // If the destination directory has the group sticky bit set, + // we have to copy the file to retain the correct permissions. + // https://golang.org/issue/18878 + if fi, err := os.Stat(filepath.Dir(dst)); err == nil { + if fi.IsDir() && (fi.Mode()&os.ModeSetgid) != 0 { + return b.copyFile(a, dst, src, perm, force) + } + } + + // The perm argument is meant to be adjusted according to umask, + // but we don't know what the umask is. + // Create a dummy file to find out. + // This avoids build tags and works even on systems like Plan 9 + // where the file mask computation incorporates other information. + mode := perm + f, err := os.OpenFile(filepath.Clean(dst)+"-go-tmp-umask", os.O_WRONLY|os.O_CREATE|os.O_EXCL, perm) + if err == nil { + fi, err := f.Stat() + if err == nil { + mode = fi.Mode() & 0777 + } + name := f.Name() + f.Close() + os.Remove(name) + } + + if err := os.Chmod(src, mode); err == nil { + if err := os.Rename(src, dst); err == nil { + if cfg.BuildX { + b.Showcmd("", "mv %s %s", src, dst) + } + return nil + } + } + + return b.copyFile(a, dst, src, perm, force) +} + +// copyFile is like 'cp src dst'. +func (b *Builder) copyFile(a *Action, dst, src string, perm os.FileMode, force bool) error { + if cfg.BuildN || cfg.BuildX { + b.Showcmd("", "cp %s %s", src, dst) + if cfg.BuildN { + return nil + } + } + + sf, err := os.Open(src) + if err != nil { + return err + } + defer sf.Close() + + // Be careful about removing/overwriting dst. + // Do not remove/overwrite if dst exists and is a directory + // or a non-object file. + if fi, err := os.Stat(dst); err == nil { + if fi.IsDir() { + return fmt.Errorf("build output %q already exists and is a directory", dst) + } + if !force && fi.Mode().IsRegular() && !isObject(dst) { + return fmt.Errorf("build output %q already exists and is not an object file", dst) + } + } + + // On Windows, remove lingering ~ file from last attempt. + if base.ToolIsWindows { + if _, err := os.Stat(dst + "~"); err == nil { + os.Remove(dst + "~") + } + } + + mayberemovefile(dst) + df, err := os.OpenFile(dst, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, perm) + if err != nil && base.ToolIsWindows { + // Windows does not allow deletion of a binary file + // while it is executing. Try to move it out of the way. + // If the move fails, which is likely, we'll try again the + // next time we do an install of this binary. + if err := os.Rename(dst, dst+"~"); err == nil { + os.Remove(dst + "~") + } + df, err = os.OpenFile(dst, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, perm) + } + if err != nil { + return err + } + + _, err = io.Copy(df, sf) + df.Close() + if err != nil { + mayberemovefile(dst) + return fmt.Errorf("copying %s to %s: %v", src, dst, err) + } + return nil +} + +// Install the cgo export header file, if there is one. +func (b *Builder) installHeader(a *Action) error { + src := a.Objdir + "_cgo_install.h" + if _, err := os.Stat(src); os.IsNotExist(err) { + // If the file does not exist, there are no exported + // functions, and we do not install anything. + return nil + } + + dir, _ := filepath.Split(a.Target) + if dir != "" { + if err := b.Mkdir(dir); err != nil { + return err + } + } + + return b.moveOrCopyFile(a, a.Target, src, 0666, true) +} + +// cover runs, in effect, +// go tool cover -mode=b.coverMode -var="varName" -o dst.go src.go +func (b *Builder) cover(a *Action, dst, src string, perm os.FileMode, varName string) error { + return b.run(a.Objdir, "cover "+a.Package.ImportPath, nil, + cfg.BuildToolexec, + base.Tool("cover"), + "-mode", a.Package.Internal.CoverMode, + "-var", varName, + "-o", dst, + src) +} + +var objectMagic = [][]byte{ + {'!', '<', 'a', 'r', 'c', 'h', '>', '\n'}, // Package archive + {'\x7F', 'E', 'L', 'F'}, // ELF + {0xFE, 0xED, 0xFA, 0xCE}, // Mach-O big-endian 32-bit + {0xFE, 0xED, 0xFA, 0xCF}, // Mach-O big-endian 64-bit + {0xCE, 0xFA, 0xED, 0xFE}, // Mach-O little-endian 32-bit + {0xCF, 0xFA, 0xED, 0xFE}, // Mach-O little-endian 64-bit + {0x4d, 0x5a, 0x90, 0x00, 0x03, 0x00}, // PE (Windows) as generated by 6l/8l and gcc + {0x00, 0x00, 0x01, 0xEB}, // Plan 9 i386 + {0x00, 0x00, 0x8a, 0x97}, // Plan 9 amd64 + {0x00, 0x00, 0x06, 0x47}, // Plan 9 arm +} + +func isObject(s string) bool { + f, err := os.Open(s) + if err != nil { + return false + } + defer f.Close() + buf := make([]byte, 64) + io.ReadFull(f, buf) + for _, magic := range objectMagic { + if bytes.HasPrefix(buf, magic) { + return true + } + } + return false +} + +// mayberemovefile removes a file only if it is a regular file +// When running as a user with sufficient privileges, we may delete +// even device files, for example, which is not intended. +func mayberemovefile(s string) { + if fi, err := os.Lstat(s); err == nil && !fi.Mode().IsRegular() { + return + } + os.Remove(s) +} + +// fmtcmd formats a command in the manner of fmt.Sprintf but also: +// +// If dir is non-empty and the script is not in dir right now, +// fmtcmd inserts "cd dir\n" before the command. +// +// fmtcmd replaces the value of b.WorkDir with $WORK. +// fmtcmd replaces the value of goroot with $GOROOT. +// fmtcmd replaces the value of b.gobin with $GOBIN. +// +// fmtcmd replaces the name of the current directory with dot (.) +// but only when it is at the beginning of a space-separated token. +// +func (b *Builder) fmtcmd(dir string, format string, args ...interface{}) string { + cmd := fmt.Sprintf(format, args...) + if dir != "" && dir != "/" { + cmd = strings.Replace(" "+cmd, " "+dir, " .", -1)[1:] + if b.scriptDir != dir { + b.scriptDir = dir + cmd = "cd " + dir + "\n" + cmd + } + } + if b.WorkDir != "" { + cmd = strings.Replace(cmd, b.WorkDir, "$WORK", -1) + } + return cmd +} + +// showcmd prints the given command to standard output +// for the implementation of -n or -x. +func (b *Builder) Showcmd(dir string, format string, args ...interface{}) { + b.output.Lock() + defer b.output.Unlock() + b.Print(b.fmtcmd(dir, format, args...) + "\n") +} + +// showOutput prints "# desc" followed by the given output. +// The output is expected to contain references to 'dir', usually +// the source directory for the package that has failed to build. +// showOutput rewrites mentions of dir with a relative path to dir +// when the relative path is shorter. This is usually more pleasant. +// For example, if fmt doesn't compile and we are in src/html, +// the output is +// +// $ go build +// # fmt +// ../fmt/print.go:1090: undefined: asdf +// $ +// +// instead of +// +// $ go build +// # fmt +// /usr/gopher/go/src/fmt/print.go:1090: undefined: asdf +// $ +// +// showOutput also replaces references to the work directory with $WORK. +// +func (b *Builder) showOutput(dir, desc, out string) { + prefix := "# " + desc + suffix := "\n" + out + if reldir := base.ShortPath(dir); reldir != dir { + suffix = strings.Replace(suffix, " "+dir, " "+reldir, -1) + suffix = strings.Replace(suffix, "\n"+dir, "\n"+reldir, -1) + } + suffix = strings.Replace(suffix, " "+b.WorkDir, " $WORK", -1) + + b.output.Lock() + defer b.output.Unlock() + b.Print(prefix, suffix) +} + +// errPrintedOutput is a special error indicating that a command failed +// but that it generated output as well, and that output has already +// been printed, so there's no point showing 'exit status 1' or whatever +// the wait status was. The main executor, builder.do, knows not to +// print this error. +var errPrintedOutput = errors.New("already printed output - no need to show error") + +var cgoLine = regexp.MustCompile(`\[[^\[\]]+\.cgo1\.go:[0-9]+(:[0-9]+)?\]`) +var cgoTypeSigRe = regexp.MustCompile(`\b_Ctype_\B`) + +// run runs the command given by cmdline in the directory dir. +// If the command fails, run prints information about the failure +// and returns a non-nil error. +func (b *Builder) run(dir string, desc string, env []string, cmdargs ...interface{}) error { + out, err := b.runOut(dir, desc, env, cmdargs...) + if len(out) > 0 { + if desc == "" { + desc = b.fmtcmd(dir, "%s", strings.Join(str.StringList(cmdargs...), " ")) + } + b.showOutput(dir, desc, b.processOutput(out)) + if err != nil { + err = errPrintedOutput + } + } + return err +} + +// processOutput prepares the output of runOut to be output to the console. +func (b *Builder) processOutput(out []byte) string { + if out[len(out)-1] != '\n' { + out = append(out, '\n') + } + messages := string(out) + // Fix up output referring to cgo-generated code to be more readable. + // Replace x.go:19[/tmp/.../x.cgo1.go:18] with x.go:19. + // Replace *[100]_Ctype_foo with *[100]C.foo. + // If we're using -x, assume we're debugging and want the full dump, so disable the rewrite. + if !cfg.BuildX && cgoLine.MatchString(messages) { + messages = cgoLine.ReplaceAllString(messages, "") + messages = cgoTypeSigRe.ReplaceAllString(messages, "C.") + } + return messages +} + +// runOut runs the command given by cmdline in the directory dir. +// It returns the command output and any errors that occurred. +func (b *Builder) runOut(dir string, desc string, env []string, cmdargs ...interface{}) ([]byte, error) { + cmdline := str.StringList(cmdargs...) + if cfg.BuildN || cfg.BuildX { + var envcmdline string + for i := range env { + envcmdline += env[i] + envcmdline += " " + } + envcmdline += joinUnambiguously(cmdline) + b.Showcmd(dir, "%s", envcmdline) + if cfg.BuildN { + return nil, nil + } + } + + nbusy := 0 + for { + var buf bytes.Buffer + cmd := exec.Command(cmdline[0], cmdline[1:]...) + cmd.Stdout = &buf + cmd.Stderr = &buf + cmd.Dir = dir + cmd.Env = base.MergeEnvLists(env, base.EnvForDir(cmd.Dir, os.Environ())) + err := cmd.Run() + + // cmd.Run will fail on Unix if some other process has the binary + // we want to run open for writing. This can happen here because + // we build and install the cgo command and then run it. + // If another command was kicked off while we were writing the + // cgo binary, the child process for that command may be holding + // a reference to the fd, keeping us from running exec. + // + // But, you might reasonably wonder, how can this happen? + // The cgo fd, like all our fds, is close-on-exec, so that we need + // not worry about other processes inheriting the fd accidentally. + // The answer is that running a command is fork and exec. + // A child forked while the cgo fd is open inherits that fd. + // Until the child has called exec, it holds the fd open and the + // kernel will not let us run cgo. Even if the child were to close + // the fd explicitly, it would still be open from the time of the fork + // until the time of the explicit close, and the race would remain. + // + // On Unix systems, this results in ETXTBSY, which formats + // as "text file busy". Rather than hard-code specific error cases, + // we just look for that string. If this happens, sleep a little + // and try again. We let this happen three times, with increasing + // sleep lengths: 100+200+400 ms = 0.7 seconds. + // + // An alternate solution might be to split the cmd.Run into + // separate cmd.Start and cmd.Wait, and then use an RWLock + // to make sure that copyFile only executes when no cmd.Start + // call is in progress. However, cmd.Start (really syscall.forkExec) + // only guarantees that when it returns, the exec is committed to + // happen and succeed. It uses a close-on-exec file descriptor + // itself to determine this, so we know that when cmd.Start returns, + // at least one close-on-exec file descriptor has been closed. + // However, we cannot be sure that all of them have been closed, + // so the program might still encounter ETXTBSY even with such + // an RWLock. The race window would be smaller, perhaps, but not + // guaranteed to be gone. + // + // Sleeping when we observe the race seems to be the most reliable + // option we have. + // + // https://golang.org/issue/3001 + // + if err != nil && nbusy < 3 && strings.Contains(err.Error(), "text file busy") { + time.Sleep(100 * time.Millisecond << uint(nbusy)) + nbusy++ + continue + } + + // err can be something like 'exit status 1'. + // Add information about what program was running. + // Note that if buf.Bytes() is non-empty, the caller usually + // shows buf.Bytes() and does not print err at all, so the + // prefix here does not make most output any more verbose. + if err != nil { + err = errors.New(cmdline[0] + ": " + err.Error()) + } + return buf.Bytes(), err + } +} + +// joinUnambiguously prints the slice, quoting where necessary to make the +// output unambiguous. +// TODO: See issue 5279. The printing of commands needs a complete redo. +func joinUnambiguously(a []string) string { + var buf bytes.Buffer + for i, s := range a { + if i > 0 { + buf.WriteByte(' ') + } + q := strconv.Quote(s) + if s == "" || strings.Contains(s, " ") || len(q) > len(s)+2 { + buf.WriteString(q) + } else { + buf.WriteString(s) + } + } + return buf.String() +} + +// mkdir makes the named directory. +func (b *Builder) Mkdir(dir string) error { + b.exec.Lock() + defer b.exec.Unlock() + // We can be a little aggressive about being + // sure directories exist. Skip repeated calls. + if b.mkdirCache[dir] { + return nil + } + b.mkdirCache[dir] = true + + if cfg.BuildN || cfg.BuildX { + b.Showcmd("", "mkdir -p %s", dir) + if cfg.BuildN { + return nil + } + } + + if err := os.MkdirAll(dir, 0777); err != nil { + return err + } + return nil +} + +// mkAbs returns an absolute path corresponding to +// evaluating f in the directory dir. +// We always pass absolute paths of source files so that +// the error messages will include the full path to a file +// in need of attention. +func mkAbs(dir, f string) string { + // Leave absolute paths alone. + // Also, during -n mode we use the pseudo-directory $WORK + // instead of creating an actual work directory that won't be used. + // Leave paths beginning with $WORK alone too. + if filepath.IsAbs(f) || strings.HasPrefix(f, "$WORK") { + return f + } + return filepath.Join(dir, f) +} + +type toolchain interface { + // gc runs the compiler in a specific directory on a set of files + // and returns the name of the generated output file. + gc(b *Builder, p *load.Package, archive, obj string, asmhdr bool, importArgs []string, gofiles []string) (ofile string, out []byte, err error) + // cc runs the toolchain's C compiler in a directory on a C file + // to produce an output file. + cc(b *Builder, p *load.Package, objdir, ofile, cfile string) error + // asm runs the assembler in a specific directory on specific files + // and returns a list of named output files. + asm(b *Builder, p *load.Package, obj string, sfiles []string) ([]string, error) + // pkgpath builds an appropriate path for a temporary package file. + Pkgpath(basedir string, p *load.Package) string + // pack runs the archive packer in a specific directory to create + // an archive from a set of object files. + // typically it is run in the object directory. + pack(b *Builder, p *load.Package, objDir, afile string, ofiles []string) error + // ld runs the linker to create an executable starting at mainpkg. + ld(b *Builder, root *Action, out string, allactions []*Action, mainpkg string, ofiles []string) error + // ldShared runs the linker to create a shared library containing the pkgs built by toplevelactions + ldShared(b *Builder, toplevelactions []*Action, out string, allactions []*Action) error + + compiler() string + linker() string +} + +type noToolchain struct{} + +func noCompiler() error { + log.Fatalf("unknown compiler %q", cfg.BuildContext.Compiler) + return nil +} + +func (noToolchain) compiler() string { + noCompiler() + return "" +} + +func (noToolchain) linker() string { + noCompiler() + return "" +} + +func (noToolchain) gc(b *Builder, p *load.Package, archive, obj string, asmhdr bool, importArgs []string, gofiles []string) (ofile string, out []byte, err error) { + return "", nil, noCompiler() +} + +func (noToolchain) asm(b *Builder, p *load.Package, obj string, sfiles []string) ([]string, error) { + return nil, noCompiler() +} + +func (noToolchain) Pkgpath(basedir string, p *load.Package) string { + noCompiler() + return "" +} + +func (noToolchain) pack(b *Builder, p *load.Package, objDir, afile string, ofiles []string) error { + return noCompiler() +} + +func (noToolchain) ld(b *Builder, root *Action, out string, allactions []*Action, mainpkg string, ofiles []string) error { + return noCompiler() +} + +func (noToolchain) ldShared(b *Builder, toplevelactions []*Action, out string, allactions []*Action) error { + return noCompiler() +} + +func (noToolchain) cc(b *Builder, p *load.Package, objdir, ofile, cfile string) error { + return noCompiler() +} + +// The Go toolchain. +type gcToolchain struct{} + +func (gcToolchain) compiler() string { + return base.Tool("compile") +} + +func (gcToolchain) linker() string { + return base.Tool("link") +} + +func (gcToolchain) gc(b *Builder, p *load.Package, archive, obj string, asmhdr bool, importArgs []string, gofiles []string) (ofile string, output []byte, err error) { + if archive != "" { + ofile = archive + } else { + out := "_go_.o" + ofile = obj + out + } + + gcargs := []string{"-p", p.ImportPath} + if p.Name == "main" { + gcargs[1] = "main" + } + if p.Standard { + gcargs = append(gcargs, "-std") + } + compilingRuntime := p.Standard && (p.ImportPath == "runtime" || strings.HasPrefix(p.ImportPath, "runtime/internal")) + if compilingRuntime { + // runtime compiles with a special gc flag to emit + // additional reflect type data. + gcargs = append(gcargs, "-+") + } + + // If we're giving the compiler the entire package (no C etc files), tell it that, + // so that it can give good error messages about forward declarations. + // Exceptions: a few standard packages have forward declarations for + // pieces supplied behind-the-scenes by package runtime. + extFiles := len(p.CgoFiles) + len(p.CFiles) + len(p.CXXFiles) + len(p.MFiles) + len(p.FFiles) + len(p.SFiles) + len(p.SysoFiles) + len(p.SwigFiles) + len(p.SwigCXXFiles) + if p.Standard { + switch p.ImportPath { + case "bytes", "internal/poll", "net", "os", "runtime/pprof", "sync", "syscall", "time": + extFiles++ + } + } + if extFiles == 0 { + gcargs = append(gcargs, "-complete") + } + if cfg.BuildContext.InstallSuffix != "" { + gcargs = append(gcargs, "-installsuffix", cfg.BuildContext.InstallSuffix) + } + if p.Internal.BuildID != "" { + gcargs = append(gcargs, "-buildid", p.Internal.BuildID) + } + platform := cfg.Goos + "/" + cfg.Goarch + if p.Internal.OmitDebug || platform == "nacl/amd64p32" || platform == "darwin/arm" || platform == "darwin/arm64" || cfg.Goos == "plan9" { + gcargs = append(gcargs, "-dwarf=false") + } + + for _, path := range p.Imports { + if i := strings.LastIndex(path, "/vendor/"); i >= 0 { + gcargs = append(gcargs, "-importmap", path[i+len("/vendor/"):]+"="+path) + } else if strings.HasPrefix(path, "vendor/") { + gcargs = append(gcargs, "-importmap", path[len("vendor/"):]+"="+path) + } + } + + gcflags := buildGcflags + if compilingRuntime { + // Remove -N, if present. + // It is not possible to build the runtime with no optimizations, + // because the compiler cannot eliminate enough write barriers. + gcflags = make([]string, len(buildGcflags)) + copy(gcflags, buildGcflags) + for i := 0; i < len(gcflags); i++ { + if gcflags[i] == "-N" { + copy(gcflags[i:], gcflags[i+1:]) + gcflags = gcflags[:len(gcflags)-1] + i-- + } + } + } + args := []interface{}{cfg.BuildToolexec, base.Tool("compile"), "-o", ofile, "-trimpath", b.WorkDir, gcflags, gcargs, "-D", p.Internal.LocalPrefix, importArgs} + if ofile == archive { + args = append(args, "-pack") + } + if asmhdr { + args = append(args, "-asmhdr", obj+"go_asm.h") + } + + // Add -c=N to use concurrent backend compilation, if possible. + if c := gcBackendConcurrency(gcflags); c > 1 { + args = append(args, fmt.Sprintf("-c=%d", c)) + } + + for _, f := range gofiles { + args = append(args, mkAbs(p.Dir, f)) + } + + output, err = b.runOut(p.Dir, p.ImportPath, nil, args...) + return ofile, output, err +} + +// gcBackendConcurrency returns the backend compiler concurrency level for a package compilation. +func gcBackendConcurrency(gcflags []string) int { + // First, check whether we can use -c at all for this compilation. + canDashC := concurrentGCBackendCompilationEnabledByDefault + + switch e := os.Getenv("GO19CONCURRENTCOMPILATION"); e { + case "0": + canDashC = false + case "1": + canDashC = true + case "": + // Not set. Use default. + default: + log.Fatalf("GO19CONCURRENTCOMPILATION must be 0, 1, or unset, got %q", e) + } + + if os.Getenv("GOEXPERIMENT") != "" { + // Concurrent compilation is presumed incompatible with GOEXPERIMENTs. + canDashC = false + } + +CheckFlags: + for _, flag := range gcflags { + // Concurrent compilation is presumed incompatible with any gcflags, + // except for a small whitelist of commonly used flags. + // If the user knows better, they can manually add their own -c to the gcflags. + switch flag { + case "-N", "-l", "-S", "-B", "-C", "-I": + // OK + default: + canDashC = false + break CheckFlags + } + } + + if !canDashC { + return 1 + } + + // Decide how many concurrent backend compilations to allow. + // + // If we allow too many, in theory we might end up with p concurrent processes, + // each with c concurrent backend compiles, all fighting over the same resources. + // However, in practice, that seems not to happen too much. + // Most build graphs are surprisingly serial, so p==1 for much of the build. + // Furthermore, concurrent backend compilation is only enabled for a part + // of the overall compiler execution, so c==1 for much of the build. + // So don't worry too much about that interaction for now. + // + // However, in practice, setting c above 4 tends not to help very much. + // See the analysis in CL 41192. + // + // TODO(josharian): attempt to detect whether this particular compilation + // is likely to be a bottleneck, e.g. when: + // - it has no successor packages to compile (usually package main) + // - all paths through the build graph pass through it + // - critical path scheduling says it is high priority + // and in such a case, set c to runtime.NumCPU. + // We do this now when p==1. + if cfg.BuildP == 1 { + // No process parallelism. Max out c. + return runtime.NumCPU() + } + // Some process parallelism. Set c to min(4, numcpu). + c := 4 + if ncpu := runtime.NumCPU(); ncpu < c { + c = ncpu + } + return c +} + +func (gcToolchain) asm(b *Builder, p *load.Package, obj string, sfiles []string) ([]string, error) { + // Add -I pkg/GOOS_GOARCH so #include "textflag.h" works in .s files. + inc := filepath.Join(cfg.GOROOT, "pkg", "include") + args := []interface{}{cfg.BuildToolexec, base.Tool("asm"), "-trimpath", b.WorkDir, "-I", obj, "-I", inc, "-D", "GOOS_" + cfg.Goos, "-D", "GOARCH_" + cfg.Goarch, buildAsmflags} + if p.ImportPath == "runtime" && cfg.Goarch == "386" { + for _, arg := range buildAsmflags { + if arg == "-dynlink" { + args = append(args, "-D=GOBUILDMODE_shared=1") + } + } + } + var ofiles []string + for _, sfile := range sfiles { + ofile := obj + sfile[:len(sfile)-len(".s")] + ".o" + ofiles = append(ofiles, ofile) + a := append(args, "-o", ofile, mkAbs(p.Dir, sfile)) + if err := b.run(p.Dir, p.ImportPath, nil, a...); err != nil { + return nil, err + } + } + return ofiles, nil +} + +// toolVerify checks that the command line args writes the same output file +// if run using newTool instead. +// Unused now but kept around for future use. +func toolVerify(b *Builder, p *load.Package, newTool string, ofile string, args []interface{}) error { + newArgs := make([]interface{}, len(args)) + copy(newArgs, args) + newArgs[1] = base.Tool(newTool) + newArgs[3] = ofile + ".new" // x.6 becomes x.6.new + if err := b.run(p.Dir, p.ImportPath, nil, newArgs...); err != nil { + return err + } + data1, err := ioutil.ReadFile(ofile) + if err != nil { + return err + } + data2, err := ioutil.ReadFile(ofile + ".new") + if err != nil { + return err + } + if !bytes.Equal(data1, data2) { + return fmt.Errorf("%s and %s produced different output files:\n%s\n%s", filepath.Base(args[1].(string)), newTool, strings.Join(str.StringList(args...), " "), strings.Join(str.StringList(newArgs...), " ")) + } + os.Remove(ofile + ".new") + return nil +} + +func (gcToolchain) Pkgpath(basedir string, p *load.Package) string { + end := filepath.FromSlash(p.ImportPath + ".a") + return filepath.Join(basedir, end) +} + +func (gcToolchain) pack(b *Builder, p *load.Package, objDir, afile string, ofiles []string) error { + var absOfiles []string + for _, f := range ofiles { + absOfiles = append(absOfiles, mkAbs(objDir, f)) + } + absAfile := mkAbs(objDir, afile) + + // The archive file should have been created by the compiler. + // Since it used to not work that way, verify. + if !cfg.BuildN { + if _, err := os.Stat(absAfile); err != nil { + base.Fatalf("os.Stat of archive file failed: %v", err) + } + } + + if cfg.BuildN || cfg.BuildX { + cmdline := str.StringList("pack", "r", absAfile, absOfiles) + b.Showcmd(p.Dir, "%s # internal", joinUnambiguously(cmdline)) + } + if cfg.BuildN { + return nil + } + if err := packInternal(b, absAfile, absOfiles); err != nil { + b.showOutput(p.Dir, p.ImportPath, err.Error()+"\n") + return errPrintedOutput + } + return nil +} + +func packInternal(b *Builder, afile string, ofiles []string) error { + dst, err := os.OpenFile(afile, os.O_WRONLY|os.O_APPEND, 0) + if err != nil { + return err + } + defer dst.Close() // only for error returns or panics + w := bufio.NewWriter(dst) + + for _, ofile := range ofiles { + src, err := os.Open(ofile) + if err != nil { + return err + } + fi, err := src.Stat() + if err != nil { + src.Close() + return err + } + // Note: Not using %-16.16s format because we care + // about bytes, not runes. + name := fi.Name() + if len(name) > 16 { + name = name[:16] + } else { + name += strings.Repeat(" ", 16-len(name)) + } + size := fi.Size() + fmt.Fprintf(w, "%s%-12d%-6d%-6d%-8o%-10d`\n", + name, 0, 0, 0, 0644, size) + n, err := io.Copy(w, src) + src.Close() + if err == nil && n < size { + err = io.ErrUnexpectedEOF + } else if err == nil && n > size { + err = fmt.Errorf("file larger than size reported by stat") + } + if err != nil { + return fmt.Errorf("copying %s to %s: %v", ofile, afile, err) + } + if size&1 != 0 { + w.WriteByte(0) + } + } + + if err := w.Flush(); err != nil { + return err + } + return dst.Close() +} + +// setextld sets the appropriate linker flags for the specified compiler. +func setextld(ldflags []string, compiler []string) []string { + for _, f := range ldflags { + if f == "-extld" || strings.HasPrefix(f, "-extld=") { + // don't override -extld if supplied + return ldflags + } + } + ldflags = append(ldflags, "-extld="+compiler[0]) + if len(compiler) > 1 { + extldflags := false + add := strings.Join(compiler[1:], " ") + for i, f := range ldflags { + if f == "-extldflags" && i+1 < len(ldflags) { + ldflags[i+1] = add + " " + ldflags[i+1] + extldflags = true + break + } else if strings.HasPrefix(f, "-extldflags=") { + ldflags[i] = "-extldflags=" + add + " " + ldflags[i][len("-extldflags="):] + extldflags = true + break + } + } + if !extldflags { + ldflags = append(ldflags, "-extldflags="+add) + } + } + return ldflags +} + +func (gcToolchain) ld(b *Builder, root *Action, out string, allactions []*Action, mainpkg string, ofiles []string) error { + importArgs := b.includeArgs("-L", allactions) + cxx := len(root.Package.CXXFiles) > 0 || len(root.Package.SwigCXXFiles) > 0 + for _, a := range allactions { + if a.Package != nil && (len(a.Package.CXXFiles) > 0 || len(a.Package.SwigCXXFiles) > 0) { + cxx = true + } + } + var ldflags []string + if cfg.BuildContext.InstallSuffix != "" { + ldflags = append(ldflags, "-installsuffix", cfg.BuildContext.InstallSuffix) + } + if root.Package.Internal.OmitDebug { + ldflags = append(ldflags, "-s", "-w") + } + if cfg.BuildBuildmode == "plugin" { + pluginpath := root.Package.ImportPath + if pluginpath == "command-line-arguments" { + pluginpath = "plugin/unnamed-" + root.Package.Internal.BuildID + } + ldflags = append(ldflags, "-pluginpath", pluginpath) + } + + // If the user has not specified the -extld option, then specify the + // appropriate linker. In case of C++ code, use the compiler named + // by the CXX environment variable or defaultCXX if CXX is not set. + // Else, use the CC environment variable and defaultCC as fallback. + var compiler []string + if cxx { + compiler = envList("CXX", cfg.DefaultCXX) + } else { + compiler = envList("CC", cfg.DefaultCC) + } + ldflags = setextld(ldflags, compiler) + ldflags = append(ldflags, "-buildmode="+ldBuildmode) + if root.Package.Internal.BuildID != "" { + ldflags = append(ldflags, "-buildid="+root.Package.Internal.BuildID) + } + ldflags = append(ldflags, cfg.BuildLdflags...) + + // On OS X when using external linking to build a shared library, + // the argument passed here to -o ends up recorded in the final + // shared library in the LC_ID_DYLIB load command. + // To avoid putting the temporary output directory name there + // (and making the resulting shared library useless), + // run the link in the output directory so that -o can name + // just the final path element. + dir := "." + if cfg.Goos == "darwin" && cfg.BuildBuildmode == "c-shared" { + dir, out = filepath.Split(out) + } + + return b.run(dir, root.Package.ImportPath, nil, cfg.BuildToolexec, base.Tool("link"), "-o", out, importArgs, ldflags, mainpkg) +} + +func (gcToolchain) ldShared(b *Builder, toplevelactions []*Action, out string, allactions []*Action) error { + importArgs := b.includeArgs("-L", allactions) + ldflags := []string{"-installsuffix", cfg.BuildContext.InstallSuffix} + ldflags = append(ldflags, "-buildmode=shared") + ldflags = append(ldflags, cfg.BuildLdflags...) + cxx := false + for _, a := range allactions { + if a.Package != nil && (len(a.Package.CXXFiles) > 0 || len(a.Package.SwigCXXFiles) > 0) { + cxx = true + } + } + // If the user has not specified the -extld option, then specify the + // appropriate linker. In case of C++ code, use the compiler named + // by the CXX environment variable or defaultCXX if CXX is not set. + // Else, use the CC environment variable and defaultCC as fallback. + var compiler []string + if cxx { + compiler = envList("CXX", cfg.DefaultCXX) + } else { + compiler = envList("CC", cfg.DefaultCC) + } + ldflags = setextld(ldflags, compiler) + for _, d := range toplevelactions { + if !strings.HasSuffix(d.Target, ".a") { // omit unsafe etc and actions for other shared libraries + continue + } + ldflags = append(ldflags, d.Package.ImportPath+"="+d.Target) + } + return b.run(".", out, nil, cfg.BuildToolexec, base.Tool("link"), "-o", out, importArgs, ldflags) +} + +func (gcToolchain) cc(b *Builder, p *load.Package, objdir, ofile, cfile string) error { + return fmt.Errorf("%s: C source files not supported without cgo", mkAbs(p.Dir, cfile)) +} + +// The Gccgo toolchain. +type gccgoToolchain struct{} + +var GccgoName, GccgoBin string +var gccgoErr error + +func init() { + GccgoName = os.Getenv("GCCGO") + if GccgoName == "" { + GccgoName = cfg.DefaultGCCGO + } + GccgoBin, gccgoErr = exec.LookPath(GccgoName) +} + +func (gccgoToolchain) compiler() string { + checkGccgoBin() + return GccgoBin +} + +func (gccgoToolchain) linker() string { + checkGccgoBin() + return GccgoBin +} + +func checkGccgoBin() { + if gccgoErr == nil { + return + } + fmt.Fprintf(os.Stderr, "cmd/go: gccgo: %s\n", gccgoErr) + os.Exit(2) +} + +func (tools gccgoToolchain) gc(b *Builder, p *load.Package, archive, obj string, asmhdr bool, importArgs []string, gofiles []string) (ofile string, output []byte, err error) { + out := "_go_.o" + ofile = obj + out + gcargs := []string{"-g"} + gcargs = append(gcargs, b.gccArchArgs()...) + gcargs = append(gcargs, "-fdebug-prefix-map="+b.WorkDir+"=/tmp/go-build") + gcargs = append(gcargs, "-gno-record-gcc-switches") + if pkgpath := gccgoPkgpath(p); pkgpath != "" { + gcargs = append(gcargs, "-fgo-pkgpath="+pkgpath) + } + if p.Internal.LocalPrefix != "" { + gcargs = append(gcargs, "-fgo-relative-import-path="+p.Internal.LocalPrefix) + } + + // Handle vendor directories + savedirs := []string{} + for _, incdir := range importArgs { + if incdir != "-I" { + savedirs = append(savedirs, incdir) + } + } + + for _, path := range p.Imports { + // If this is a new vendor path, add it to the list of importArgs + if i := strings.LastIndex(path, "/vendor/"); i >= 0 { + for _, dir := range savedirs { + // Check if the vendor path is already included in dir + if strings.HasSuffix(dir, path[:i+len("/vendor")]) { + continue + } + // Make sure this vendor path is not already in the list for importArgs + vendorPath := dir + "/" + path[:i+len("/vendor/")] + for _, imp := range importArgs { + if imp == "-I" { + continue + } + // This vendorPath is already in the list + if imp == vendorPath { + goto nextSuffixPath + } + } + // New vendorPath not yet in the importArgs list, so add it + importArgs = append(importArgs, "-I", vendorPath) + nextSuffixPath: + } + } else if strings.HasPrefix(path, "vendor/") { + for _, dir := range savedirs { + // Make sure this vendor path is not already in the list for importArgs + vendorPath := dir + "/" + path[len("/vendor"):] + for _, imp := range importArgs { + if imp == "-I" { + continue + } + if imp == vendorPath { + goto nextPrefixPath + } + } + // This vendor path is needed and not already in the list, so add it + importArgs = append(importArgs, "-I", vendorPath) + nextPrefixPath: + } + } + } + + args := str.StringList(tools.compiler(), importArgs, "-c", gcargs, "-o", ofile, buildGccgoflags) + for _, f := range gofiles { + args = append(args, mkAbs(p.Dir, f)) + } + + output, err = b.runOut(p.Dir, p.ImportPath, nil, args) + return ofile, output, err +} + +func (tools gccgoToolchain) asm(b *Builder, p *load.Package, obj string, sfiles []string) ([]string, error) { + var ofiles []string + for _, sfile := range sfiles { + ofile := obj + sfile[:len(sfile)-len(".s")] + ".o" + ofiles = append(ofiles, ofile) + sfile = mkAbs(p.Dir, sfile) + defs := []string{"-D", "GOOS_" + cfg.Goos, "-D", "GOARCH_" + cfg.Goarch} + if pkgpath := gccgoCleanPkgpath(p); pkgpath != "" { + defs = append(defs, `-D`, `GOPKGPATH=`+pkgpath) + } + defs = tools.maybePIC(defs) + defs = append(defs, b.gccArchArgs()...) + err := b.run(p.Dir, p.ImportPath, nil, tools.compiler(), "-xassembler-with-cpp", "-I", obj, "-c", "-o", ofile, defs, sfile) + if err != nil { + return nil, err + } + } + return ofiles, nil +} + +func (gccgoToolchain) Pkgpath(basedir string, p *load.Package) string { + end := filepath.FromSlash(p.ImportPath + ".a") + afile := filepath.Join(basedir, end) + // add "lib" to the final element + return filepath.Join(filepath.Dir(afile), "lib"+filepath.Base(afile)) +} + +func (gccgoToolchain) pack(b *Builder, p *load.Package, objDir, afile string, ofiles []string) error { + var absOfiles []string + for _, f := range ofiles { + absOfiles = append(absOfiles, mkAbs(objDir, f)) + } + absAfile := mkAbs(objDir, afile) + // Try with D modifier first, then without if that fails. + if b.run(p.Dir, p.ImportPath, nil, "ar", "rcD", absAfile, absOfiles) != nil { + return b.run(p.Dir, p.ImportPath, nil, "ar", "rc", absAfile, absOfiles) + } + return nil +} + +func (tools gccgoToolchain) link(b *Builder, root *Action, out string, allactions []*Action, mainpkg string, ofiles []string, buildmode, desc string) error { + // gccgo needs explicit linking with all package dependencies, + // and all LDFLAGS from cgo dependencies. + apackagePathsSeen := make(map[string]bool) + afiles := []string{} + shlibs := []string{} + ldflags := b.gccArchArgs() + cgoldflags := []string{} + usesCgo := false + cxx := false + objc := false + fortran := false + if root.Package != nil { + cxx = len(root.Package.CXXFiles) > 0 || len(root.Package.SwigCXXFiles) > 0 + objc = len(root.Package.MFiles) > 0 + fortran = len(root.Package.FFiles) > 0 + } + + readCgoFlags := func(flagsFile string) error { + flags, err := ioutil.ReadFile(flagsFile) + if err != nil { + return err + } + const ldflagsPrefix = "_CGO_LDFLAGS=" + for _, line := range strings.Split(string(flags), "\n") { + if strings.HasPrefix(line, ldflagsPrefix) { + line = line[len(ldflagsPrefix):] + quote := byte(0) + start := true + var nl []byte + for len(line) > 0 { + b := line[0] + line = line[1:] + if quote == 0 && (b == ' ' || b == '\t') { + if len(nl) > 0 { + cgoldflags = append(cgoldflags, string(nl)) + nl = nil + } + start = true + continue + } else if b == '"' || b == '\'' { + quote = b + } else if b == quote { + quote = 0 + } else if quote == 0 && start && b == '-' && strings.HasPrefix(line, "g") { + line = line[1:] + continue + } else if quote == 0 && start && b == '-' && strings.HasPrefix(line, "O") { + for len(line) > 0 && line[0] != ' ' && line[0] != '\t' { + line = line[1:] + } + continue + } + nl = append(nl, b) + start = false + } + } + } + return nil + } + + readAndRemoveCgoFlags := func(archive string) (string, error) { + newa, err := ioutil.TempFile(b.WorkDir, filepath.Base(archive)) + if err != nil { + return "", err + } + olda, err := os.Open(archive) + if err != nil { + return "", err + } + _, err = io.Copy(newa, olda) + if err != nil { + return "", err + } + err = olda.Close() + if err != nil { + return "", err + } + err = newa.Close() + if err != nil { + return "", err + } + + newarchive := newa.Name() + err = b.run(b.WorkDir, desc, nil, "ar", "x", newarchive, "_cgo_flags") + if err != nil { + return "", err + } + err = b.run(".", desc, nil, "ar", "d", newarchive, "_cgo_flags") + if err != nil { + return "", err + } + err = readCgoFlags(filepath.Join(b.WorkDir, "_cgo_flags")) + if err != nil { + return "", err + } + return newarchive, nil + } + + actionsSeen := make(map[*Action]bool) + // Make a pre-order depth-first traversal of the action graph, taking note of + // whether a shared library action has been seen on the way to an action (the + // construction of the graph means that if any path to a node passes through + // a shared library action, they all do). + var walk func(a *Action, seenShlib bool) + var err error + walk = func(a *Action, seenShlib bool) { + if actionsSeen[a] { + return + } + actionsSeen[a] = true + if a.Package != nil && !seenShlib { + if a.Package.Standard { + return + } + // We record the target of the first time we see a .a file + // for a package to make sure that we prefer the 'install' + // rather than the 'build' location (which may not exist any + // more). We still need to traverse the dependencies of the + // build action though so saying + // if apackagePathsSeen[a.Package.ImportPath] { return } + // doesn't work. + if !apackagePathsSeen[a.Package.ImportPath] { + apackagePathsSeen[a.Package.ImportPath] = true + target := a.Target + if len(a.Package.CgoFiles) > 0 || a.Package.UsesSwig() { + target, err = readAndRemoveCgoFlags(target) + if err != nil { + return + } + } + afiles = append(afiles, target) + } + } + if strings.HasSuffix(a.Target, ".so") { + shlibs = append(shlibs, a.Target) + seenShlib = true + } + for _, a1 := range a.Deps { + walk(a1, seenShlib) + if err != nil { + return + } + } + } + for _, a1 := range root.Deps { + walk(a1, false) + if err != nil { + return err + } + } + + for _, a := range allactions { + // Gather CgoLDFLAGS, but not from standard packages. + // The go tool can dig up runtime/cgo from GOROOT and + // think that it should use its CgoLDFLAGS, but gccgo + // doesn't use runtime/cgo. + if a.Package == nil { + continue + } + if !a.Package.Standard { + cgoldflags = append(cgoldflags, a.Package.CgoLDFLAGS...) + } + if len(a.Package.CgoFiles) > 0 { + usesCgo = true + } + if a.Package.UsesSwig() { + usesCgo = true + } + if len(a.Package.CXXFiles) > 0 || len(a.Package.SwigCXXFiles) > 0 { + cxx = true + } + if len(a.Package.MFiles) > 0 { + objc = true + } + if len(a.Package.FFiles) > 0 { + fortran = true + } + } + + for i, o := range ofiles { + if filepath.Base(o) == "_cgo_flags" { + readCgoFlags(o) + ofiles = append(ofiles[:i], ofiles[i+1:]...) + break + } + } + + ldflags = append(ldflags, "-Wl,--whole-archive") + ldflags = append(ldflags, afiles...) + ldflags = append(ldflags, "-Wl,--no-whole-archive") + + ldflags = append(ldflags, cgoldflags...) + ldflags = append(ldflags, envList("CGO_LDFLAGS", "")...) + if root.Package != nil { + ldflags = append(ldflags, root.Package.CgoLDFLAGS...) + } + + ldflags = str.StringList("-Wl,-(", ldflags, "-Wl,-)") + + for _, shlib := range shlibs { + ldflags = append( + ldflags, + "-L"+filepath.Dir(shlib), + "-Wl,-rpath="+filepath.Dir(shlib), + "-l"+strings.TrimSuffix( + strings.TrimPrefix(filepath.Base(shlib), "lib"), + ".so")) + } + + var realOut string + switch buildmode { + case "exe": + if usesCgo && cfg.Goos == "linux" { + ldflags = append(ldflags, "-Wl,-E") + } + + case "c-archive": + // Link the Go files into a single .o, and also link + // in -lgolibbegin. + // + // We need to use --whole-archive with -lgolibbegin + // because it doesn't define any symbols that will + // cause the contents to be pulled in; it's just + // initialization code. + // + // The user remains responsible for linking against + // -lgo -lpthread -lm in the final link. We can't use + // -r to pick them up because we can't combine + // split-stack and non-split-stack code in a single -r + // link, and libgo picks up non-split-stack code from + // libffi. + ldflags = append(ldflags, "-Wl,-r", "-nostdlib", "-Wl,--whole-archive", "-lgolibbegin", "-Wl,--no-whole-archive") + + if nopie := b.gccNoPie(); nopie != "" { + ldflags = append(ldflags, nopie) + } + + // We are creating an object file, so we don't want a build ID. + ldflags = b.disableBuildID(ldflags) + + realOut = out + out = out + ".o" + + case "c-shared": + ldflags = append(ldflags, "-shared", "-nostdlib", "-Wl,--whole-archive", "-lgolibbegin", "-Wl,--no-whole-archive", "-lgo", "-lgcc_s", "-lgcc", "-lc", "-lgcc") + case "shared": + ldflags = append(ldflags, "-zdefs", "-shared", "-nostdlib", "-lgo", "-lgcc_s", "-lgcc", "-lc") + case "pie": + ldflags = append(ldflags, "-pie") + + default: + base.Fatalf("-buildmode=%s not supported for gccgo", buildmode) + } + + switch buildmode { + case "exe", "c-shared": + if cxx { + ldflags = append(ldflags, "-lstdc++") + } + if objc { + ldflags = append(ldflags, "-lobjc") + } + if fortran { + fc := os.Getenv("FC") + if fc == "" { + fc = "gfortran" + } + // support gfortran out of the box and let others pass the correct link options + // via CGO_LDFLAGS + if strings.Contains(fc, "gfortran") { + ldflags = append(ldflags, "-lgfortran") + } + } + } + + if err := b.run(".", desc, nil, tools.linker(), "-o", out, ofiles, ldflags, buildGccgoflags); err != nil { + return err + } + + switch buildmode { + case "c-archive": + if err := b.run(".", desc, nil, "ar", "rc", realOut, out); err != nil { + return err + } + } + return nil +} + +func (tools gccgoToolchain) ld(b *Builder, root *Action, out string, allactions []*Action, mainpkg string, ofiles []string) error { + return tools.link(b, root, out, allactions, mainpkg, ofiles, ldBuildmode, root.Package.ImportPath) +} + +func (tools gccgoToolchain) ldShared(b *Builder, toplevelactions []*Action, out string, allactions []*Action) error { + fakeRoot := &Action{} + fakeRoot.Deps = toplevelactions + return tools.link(b, fakeRoot, out, allactions, "", nil, "shared", out) +} + +func (tools gccgoToolchain) cc(b *Builder, p *load.Package, objdir, ofile, cfile string) error { + inc := filepath.Join(cfg.GOROOT, "pkg", "include") + cfile = mkAbs(p.Dir, cfile) + defs := []string{"-D", "GOOS_" + cfg.Goos, "-D", "GOARCH_" + cfg.Goarch} + defs = append(defs, b.gccArchArgs()...) + if pkgpath := gccgoCleanPkgpath(p); pkgpath != "" { + defs = append(defs, `-D`, `GOPKGPATH="`+pkgpath+`"`) + } + if b.gccSupportsFlag("-fsplit-stack") { + defs = append(defs, "-fsplit-stack") + } + defs = tools.maybePIC(defs) + if b.gccSupportsFlag("-fdebug-prefix-map=a=b") { + defs = append(defs, "-fdebug-prefix-map="+b.WorkDir+"=/tmp/go-build") + } + if b.gccSupportsFlag("-gno-record-gcc-switches") { + defs = append(defs, "-gno-record-gcc-switches") + } + return b.run(p.Dir, p.ImportPath, nil, envList("CC", cfg.DefaultCC), "-Wall", "-g", + "-I", objdir, "-I", inc, "-o", ofile, defs, "-c", cfile) +} + +// maybePIC adds -fPIC to the list of arguments if needed. +func (tools gccgoToolchain) maybePIC(args []string) []string { + switch cfg.BuildBuildmode { + case "c-archive", "c-shared", "shared", "plugin": + args = append(args, "-fPIC") + } + return args +} + +func gccgoPkgpath(p *load.Package) string { + if p.Internal.Build.IsCommand() && !p.Internal.ForceLibrary { + return "" + } + return p.ImportPath +} + +func gccgoCleanPkgpath(p *load.Package) string { + clean := func(r rune) rune { + switch { + case 'A' <= r && r <= 'Z', 'a' <= r && r <= 'z', + '0' <= r && r <= '9': + return r + } + return '_' + } + return strings.Map(clean, gccgoPkgpath(p)) +} + +// gcc runs the gcc C compiler to create an object from a single C file. +func (b *Builder) gcc(p *load.Package, out string, flags []string, cfile string) error { + return b.ccompile(p, out, flags, cfile, b.GccCmd(p.Dir)) +} + +// gxx runs the g++ C++ compiler to create an object from a single C++ file. +func (b *Builder) gxx(p *load.Package, out string, flags []string, cxxfile string) error { + return b.ccompile(p, out, flags, cxxfile, b.GxxCmd(p.Dir)) +} + +// gfortran runs the gfortran Fortran compiler to create an object from a single Fortran file. +func (b *Builder) gfortran(p *load.Package, out string, flags []string, ffile string) error { + return b.ccompile(p, out, flags, ffile, b.gfortranCmd(p.Dir)) +} + +// ccompile runs the given C or C++ compiler and creates an object from a single source file. +func (b *Builder) ccompile(p *load.Package, outfile string, flags []string, file string, compiler []string) error { + file = mkAbs(p.Dir, file) + desc := p.ImportPath + if !filepath.IsAbs(outfile) { + outfile = filepath.Join(p.Dir, outfile) + } + output, err := b.runOut(filepath.Dir(file), desc, nil, compiler, flags, "-o", outfile, "-c", filepath.Base(file)) + if len(output) > 0 { + // On FreeBSD 11, when we pass -g to clang 3.8 it + // invokes its internal assembler with -dwarf-version=2. + // When it sees .section .note.GNU-stack, it warns + // "DWARF2 only supports one section per compilation unit". + // This warning makes no sense, since the section is empty, + // but it confuses people. + // We work around the problem by detecting the warning + // and dropping -g and trying again. + if bytes.Contains(output, []byte("DWARF2 only supports one section per compilation unit")) { + newFlags := make([]string, 0, len(flags)) + for _, f := range flags { + if !strings.HasPrefix(f, "-g") { + newFlags = append(newFlags, f) + } + } + if len(newFlags) < len(flags) { + return b.ccompile(p, outfile, newFlags, file, compiler) + } + } + + b.showOutput(p.Dir, desc, b.processOutput(output)) + if err != nil { + err = errPrintedOutput + } else if os.Getenv("GO_BUILDER_NAME") != "" { + return errors.New("C compiler warning promoted to error on Go builders") + } + } + return err +} + +// gccld runs the gcc linker to create an executable from a set of object files. +func (b *Builder) gccld(p *load.Package, out string, flags []string, obj []string) error { + var cmd []string + if len(p.CXXFiles) > 0 || len(p.SwigCXXFiles) > 0 { + cmd = b.GxxCmd(p.Dir) + } else { + cmd = b.GccCmd(p.Dir) + } + return b.run(p.Dir, p.ImportPath, nil, cmd, "-o", out, obj, flags) +} + +// gccCmd returns a gcc command line prefix +// defaultCC is defined in zdefaultcc.go, written by cmd/dist. +func (b *Builder) GccCmd(objdir string) []string { + return b.ccompilerCmd("CC", cfg.DefaultCC, objdir) +} + +// gxxCmd returns a g++ command line prefix +// defaultCXX is defined in zdefaultcc.go, written by cmd/dist. +func (b *Builder) GxxCmd(objdir string) []string { + return b.ccompilerCmd("CXX", cfg.DefaultCXX, objdir) +} + +// gfortranCmd returns a gfortran command line prefix. +func (b *Builder) gfortranCmd(objdir string) []string { + return b.ccompilerCmd("FC", "gfortran", objdir) +} + +// ccompilerCmd returns a command line prefix for the given environment +// variable and using the default command when the variable is empty. +func (b *Builder) ccompilerCmd(envvar, defcmd, objdir string) []string { + // NOTE: env.go's mkEnv knows that the first three + // strings returned are "gcc", "-I", objdir (and cuts them off). + + compiler := envList(envvar, defcmd) + a := []string{compiler[0], "-I", objdir} + a = append(a, compiler[1:]...) + + // Definitely want -fPIC but on Windows gcc complains + // "-fPIC ignored for target (all code is position independent)" + if cfg.Goos != "windows" { + a = append(a, "-fPIC") + } + a = append(a, b.gccArchArgs()...) + // gcc-4.5 and beyond require explicit "-pthread" flag + // for multithreading with pthread library. + if cfg.BuildContext.CgoEnabled { + switch cfg.Goos { + case "windows": + a = append(a, "-mthreads") + default: + a = append(a, "-pthread") + } + } + + if strings.Contains(a[0], "clang") { + // disable ASCII art in clang errors, if possible + a = append(a, "-fno-caret-diagnostics") + // clang is too smart about command-line arguments + a = append(a, "-Qunused-arguments") + } + + // disable word wrapping in error messages + a = append(a, "-fmessage-length=0") + + // Tell gcc not to include the work directory in object files. + if b.gccSupportsFlag("-fdebug-prefix-map=a=b") { + a = append(a, "-fdebug-prefix-map="+b.WorkDir+"=/tmp/go-build") + } + + // Tell gcc not to include flags in object files, which defeats the + // point of -fdebug-prefix-map above. + if b.gccSupportsFlag("-gno-record-gcc-switches") { + a = append(a, "-gno-record-gcc-switches") + } + + // On OS X, some of the compilers behave as if -fno-common + // is always set, and the Mach-O linker in 6l/8l assumes this. + // See https://golang.org/issue/3253. + if cfg.Goos == "darwin" { + a = append(a, "-fno-common") + } + + // gccgo uses the language-independent exception mechanism to + // handle panics, so it always needs unwind tables. + if cfg.BuildToolchainName == "gccgo" { + a = append(a, "-funwind-tables") + } + + return a +} + +// gccNoPie returns the flag to use to request non-PIE. On systems +// with PIE (position independent executables) enabled by default, +// -no-pie must be passed when doing a partial link with -Wl,-r. +// But -no-pie is not supported by all compilers, and clang spells it -nopie. +func (b *Builder) gccNoPie() string { + if b.gccSupportsFlag("-no-pie") { + return "-no-pie" + } + if b.gccSupportsFlag("-nopie") { + return "-nopie" + } + return "" +} + +// gccSupportsFlag checks to see if the compiler supports a flag. +func (b *Builder) gccSupportsFlag(flag string) bool { + b.exec.Lock() + defer b.exec.Unlock() + if b, ok := b.flagCache[flag]; ok { + return b + } + if b.flagCache == nil { + src := filepath.Join(b.WorkDir, "trivial.c") + if err := ioutil.WriteFile(src, []byte{}, 0666); err != nil { + return false + } + b.flagCache = make(map[string]bool) + } + cmdArgs := append(envList("CC", cfg.DefaultCC), flag, "-c", "trivial.c") + if cfg.BuildN || cfg.BuildX { + b.Showcmd(b.WorkDir, "%s", joinUnambiguously(cmdArgs)) + if cfg.BuildN { + return false + } + } + cmd := exec.Command(cmdArgs[0], cmdArgs[1:]...) + cmd.Dir = b.WorkDir + cmd.Env = base.MergeEnvLists([]string{"LC_ALL=C"}, base.EnvForDir(cmd.Dir, os.Environ())) + out, err := cmd.CombinedOutput() + supported := err == nil && !bytes.Contains(out, []byte("unrecognized")) + b.flagCache[flag] = supported + return supported +} + +// gccArchArgs returns arguments to pass to gcc based on the architecture. +func (b *Builder) gccArchArgs() []string { + switch cfg.Goarch { + case "386": + return []string{"-m32"} + case "amd64", "amd64p32": + return []string{"-m64"} + case "arm": + return []string{"-marm"} // not thumb + case "s390x": + return []string{"-m64", "-march=z196"} + case "mips64", "mips64le": + return []string{"-mabi=64"} + case "mips", "mipsle": + return []string{"-mabi=32", "-march=mips32"} + } + return nil +} + +// envList returns the value of the given environment variable broken +// into fields, using the default value when the variable is empty. +func envList(key, def string) []string { + v := os.Getenv(key) + if v == "" { + v = def + } + return strings.Fields(v) +} + +// CFlags returns the flags to use when invoking the C, C++ or Fortran compilers, or cgo. +func (b *Builder) CFlags(p *load.Package) (cppflags, cflags, cxxflags, fflags, ldflags []string) { + defaults := "-g -O2" + + cppflags = str.StringList(envList("CGO_CPPFLAGS", ""), p.CgoCPPFLAGS) + cflags = str.StringList(envList("CGO_CFLAGS", defaults), p.CgoCFLAGS) + cxxflags = str.StringList(envList("CGO_CXXFLAGS", defaults), p.CgoCXXFLAGS) + fflags = str.StringList(envList("CGO_FFLAGS", defaults), p.CgoFFLAGS) + ldflags = str.StringList(envList("CGO_LDFLAGS", defaults), p.CgoLDFLAGS) + return +} + +var cgoRe = regexp.MustCompile(`[/\\:]`) + +func (b *Builder) cgo(a *Action, cgoExe, obj string, pcCFLAGS, pcLDFLAGS, cgofiles, objdirCgofiles, gccfiles, gxxfiles, mfiles, ffiles []string) (outGo, outObj []string, err error) { + p := a.Package + cgoCPPFLAGS, cgoCFLAGS, cgoCXXFLAGS, cgoFFLAGS, cgoLDFLAGS := b.CFlags(p) + cgoCPPFLAGS = append(cgoCPPFLAGS, pcCFLAGS...) + cgoLDFLAGS = append(cgoLDFLAGS, pcLDFLAGS...) + // If we are compiling Objective-C code, then we need to link against libobjc + if len(mfiles) > 0 { + cgoLDFLAGS = append(cgoLDFLAGS, "-lobjc") + } + + // Likewise for Fortran, except there are many Fortran compilers. + // Support gfortran out of the box and let others pass the correct link options + // via CGO_LDFLAGS + if len(ffiles) > 0 { + fc := os.Getenv("FC") + if fc == "" { + fc = "gfortran" + } + if strings.Contains(fc, "gfortran") { + cgoLDFLAGS = append(cgoLDFLAGS, "-lgfortran") + } + } + + if cfg.BuildMSan { + cgoCFLAGS = append([]string{"-fsanitize=memory"}, cgoCFLAGS...) + cgoLDFLAGS = append([]string{"-fsanitize=memory"}, cgoLDFLAGS...) + } + + // Allows including _cgo_export.h from .[ch] files in the package. + cgoCPPFLAGS = append(cgoCPPFLAGS, "-I", obj) + + // If we have cgo files in the object directory, then copy any + // other cgo files into the object directory, and pass a + // -srcdir option to cgo. + var srcdirarg []string + if len(objdirCgofiles) > 0 { + for _, fn := range cgofiles { + if err := b.copyFile(a, obj+filepath.Base(fn), filepath.Join(p.Dir, fn), 0666, false); err != nil { + return nil, nil, err + } + } + cgofiles = append(cgofiles, objdirCgofiles...) + srcdirarg = []string{"-srcdir", obj} + } + + // cgo + // TODO: CGO_FLAGS? + gofiles := []string{obj + "_cgo_gotypes.go"} + cfiles := []string{"_cgo_export.c"} + for _, fn := range cgofiles { + f := cgoRe.ReplaceAllString(fn[:len(fn)-2], "_") + gofiles = append(gofiles, obj+f+"cgo1.go") + cfiles = append(cfiles, f+"cgo2.c") + } + + // TODO: make cgo not depend on $GOARCH? + + cgoflags := []string{} + if p.Standard && p.ImportPath == "runtime/cgo" { + cgoflags = append(cgoflags, "-import_runtime_cgo=false") + } + if p.Standard && (p.ImportPath == "runtime/race" || p.ImportPath == "runtime/msan" || p.ImportPath == "runtime/cgo") { + cgoflags = append(cgoflags, "-import_syscall=false") + } + + // Update $CGO_LDFLAGS with p.CgoLDFLAGS. + var cgoenv []string + if len(cgoLDFLAGS) > 0 { + flags := make([]string, len(cgoLDFLAGS)) + for i, f := range cgoLDFLAGS { + flags[i] = strconv.Quote(f) + } + cgoenv = []string{"CGO_LDFLAGS=" + strings.Join(flags, " ")} + } + + if cfg.BuildToolchainName == "gccgo" { + if b.gccSupportsFlag("-fsplit-stack") { + cgoCFLAGS = append(cgoCFLAGS, "-fsplit-stack") + } + cgoflags = append(cgoflags, "-gccgo") + if pkgpath := gccgoPkgpath(p); pkgpath != "" { + cgoflags = append(cgoflags, "-gccgopkgpath="+pkgpath) + } + } + + switch cfg.BuildBuildmode { + case "c-archive", "c-shared": + // Tell cgo that if there are any exported functions + // it should generate a header file that C code can + // #include. + cgoflags = append(cgoflags, "-exportheader="+obj+"_cgo_install.h") + } + + if err := b.run(p.Dir, p.ImportPath, cgoenv, cfg.BuildToolexec, cgoExe, srcdirarg, "-objdir", obj, "-importpath", p.ImportPath, cgoflags, "--", cgoCPPFLAGS, cgoCFLAGS, cgofiles); err != nil { + return nil, nil, err + } + outGo = append(outGo, gofiles...) + + // gcc + cflags := str.StringList(cgoCPPFLAGS, cgoCFLAGS) + for _, cfile := range cfiles { + ofile := obj + cfile[:len(cfile)-1] + "o" + if err := b.gcc(p, ofile, cflags, obj+cfile); err != nil { + return nil, nil, err + } + outObj = append(outObj, ofile) + } + + for _, file := range gccfiles { + base := filepath.Base(file) + ofile := obj + cgoRe.ReplaceAllString(base[:len(base)-1], "_") + "o" + if err := b.gcc(p, ofile, cflags, file); err != nil { + return nil, nil, err + } + outObj = append(outObj, ofile) + } + + cxxflags := str.StringList(cgoCPPFLAGS, cgoCXXFLAGS) + for _, file := range gxxfiles { + // Append .o to the file, just in case the pkg has file.c and file.cpp + ofile := obj + cgoRe.ReplaceAllString(filepath.Base(file), "_") + ".o" + if err := b.gxx(p, ofile, cxxflags, file); err != nil { + return nil, nil, err + } + outObj = append(outObj, ofile) + } + + for _, file := range mfiles { + // Append .o to the file, just in case the pkg has file.c and file.m + ofile := obj + cgoRe.ReplaceAllString(filepath.Base(file), "_") + ".o" + if err := b.gcc(p, ofile, cflags, file); err != nil { + return nil, nil, err + } + outObj = append(outObj, ofile) + } + + fflags := str.StringList(cgoCPPFLAGS, cgoFFLAGS) + for _, file := range ffiles { + // Append .o to the file, just in case the pkg has file.c and file.f + ofile := obj + cgoRe.ReplaceAllString(filepath.Base(file), "_") + ".o" + if err := b.gfortran(p, ofile, fflags, file); err != nil { + return nil, nil, err + } + outObj = append(outObj, ofile) + } + + switch cfg.BuildToolchainName { + case "gc": + importGo := obj + "_cgo_import.go" + if err := b.dynimport(p, obj, importGo, cgoExe, cflags, cgoLDFLAGS, outObj); err != nil { + return nil, nil, err + } + outGo = append(outGo, importGo) + + ofile := obj + "_all.o" + if err := b.collect(p, obj, ofile, cgoLDFLAGS, outObj); err != nil { + return nil, nil, err + } + outObj = []string{ofile} + + case "gccgo": + defunC := obj + "_cgo_defun.c" + defunObj := obj + "_cgo_defun.o" + if err := BuildToolchain.cc(b, p, obj, defunObj, defunC); err != nil { + return nil, nil, err + } + outObj = append(outObj, defunObj) + + default: + noCompiler() + } + + return outGo, outObj, nil +} + +// dynimport creates a Go source file named importGo containing +// //go:cgo_import_dynamic directives for each symbol or library +// dynamically imported by the object files outObj. +func (b *Builder) dynimport(p *load.Package, obj, importGo, cgoExe string, cflags, cgoLDFLAGS, outObj []string) error { + cfile := obj + "_cgo_main.c" + ofile := obj + "_cgo_main.o" + if err := b.gcc(p, ofile, cflags, cfile); err != nil { + return err + } + + linkobj := str.StringList(ofile, outObj, p.SysoFiles) + dynobj := obj + "_cgo_.o" + + // we need to use -pie for Linux/ARM to get accurate imported sym + ldflags := cgoLDFLAGS + if (cfg.Goarch == "arm" && cfg.Goos == "linux") || cfg.Goos == "android" { + ldflags = append(ldflags, "-pie") + } + if err := b.gccld(p, dynobj, ldflags, linkobj); err != nil { + return err + } + + // cgo -dynimport + var cgoflags []string + if p.Standard && p.ImportPath == "runtime/cgo" { + cgoflags = []string{"-dynlinker"} // record path to dynamic linker + } + return b.run(p.Dir, p.ImportPath, nil, cfg.BuildToolexec, cgoExe, "-dynpackage", p.Name, "-dynimport", dynobj, "-dynout", importGo, cgoflags) +} + +// collect partially links the object files outObj into a single +// relocatable object file named ofile. +func (b *Builder) collect(p *load.Package, obj, ofile string, cgoLDFLAGS, outObj []string) error { + // When linking relocatable objects, various flags need to be + // filtered out as they are inapplicable and can cause some linkers + // to fail. + var ldflags []string + for i := 0; i < len(cgoLDFLAGS); i++ { + f := cgoLDFLAGS[i] + switch { + // skip "-lc" or "-l somelib" + case strings.HasPrefix(f, "-l"): + if f == "-l" { + i++ + } + // skip "-framework X" on Darwin + case cfg.Goos == "darwin" && f == "-framework": + i++ + // skip "*.{dylib,so,dll,o,a}" + case strings.HasSuffix(f, ".dylib"), + strings.HasSuffix(f, ".so"), + strings.HasSuffix(f, ".dll"), + strings.HasSuffix(f, ".o"), + strings.HasSuffix(f, ".a"): + // Remove any -fsanitize=foo flags. + // Otherwise the compiler driver thinks that we are doing final link + // and links sanitizer runtime into the object file. But we are not doing + // the final link, we will link the resulting object file again. And + // so the program ends up with two copies of sanitizer runtime. + // See issue 8788 for details. + case strings.HasPrefix(f, "-fsanitize="): + continue + // runpath flags not applicable unless building a shared + // object or executable; see issue 12115 for details. This + // is necessary as Go currently does not offer a way to + // specify the set of LDFLAGS that only apply to shared + // objects. + case strings.HasPrefix(f, "-Wl,-rpath"): + if f == "-Wl,-rpath" || f == "-Wl,-rpath-link" { + // Skip following argument to -rpath* too. + i++ + } + default: + ldflags = append(ldflags, f) + } + } + + ldflags = append(ldflags, "-Wl,-r", "-nostdlib") + + if flag := b.gccNoPie(); flag != "" { + ldflags = append(ldflags, flag) + } + + // We are creating an object file, so we don't want a build ID. + ldflags = b.disableBuildID(ldflags) + + return b.gccld(p, ofile, ldflags, outObj) +} + +// Run SWIG on all SWIG input files. +// TODO: Don't build a shared library, once SWIG emits the necessary +// pragmas for external linking. +func (b *Builder) swig(p *load.Package, obj string, pcCFLAGS []string) (outGo, outC, outCXX []string, err error) { + if err := b.swigVersionCheck(); err != nil { + return nil, nil, nil, err + } + + intgosize, err := b.swigIntSize(obj) + if err != nil { + return nil, nil, nil, err + } + + for _, f := range p.SwigFiles { + goFile, cFile, err := b.swigOne(p, f, obj, pcCFLAGS, false, intgosize) + if err != nil { + return nil, nil, nil, err + } + if goFile != "" { + outGo = append(outGo, goFile) + } + if cFile != "" { + outC = append(outC, cFile) + } + } + for _, f := range p.SwigCXXFiles { + goFile, cxxFile, err := b.swigOne(p, f, obj, pcCFLAGS, true, intgosize) + if err != nil { + return nil, nil, nil, err + } + if goFile != "" { + outGo = append(outGo, goFile) + } + if cxxFile != "" { + outCXX = append(outCXX, cxxFile) + } + } + return outGo, outC, outCXX, nil +} + +// Make sure SWIG is new enough. +var ( + swigCheckOnce sync.Once + swigCheck error +) + +func (b *Builder) swigDoVersionCheck() error { + out, err := b.runOut("", "", nil, "swig", "-version") + if err != nil { + return err + } + re := regexp.MustCompile(`[vV]ersion +([\d]+)([.][\d]+)?([.][\d]+)?`) + matches := re.FindSubmatch(out) + if matches == nil { + // Can't find version number; hope for the best. + return nil + } + + major, err := strconv.Atoi(string(matches[1])) + if err != nil { + // Can't find version number; hope for the best. + return nil + } + const errmsg = "must have SWIG version >= 3.0.6" + if major < 3 { + return errors.New(errmsg) + } + if major > 3 { + // 4.0 or later + return nil + } + + // We have SWIG version 3.x. + if len(matches[2]) > 0 { + minor, err := strconv.Atoi(string(matches[2][1:])) + if err != nil { + return nil + } + if minor > 0 { + // 3.1 or later + return nil + } + } + + // We have SWIG version 3.0.x. + if len(matches[3]) > 0 { + patch, err := strconv.Atoi(string(matches[3][1:])) + if err != nil { + return nil + } + if patch < 6 { + // Before 3.0.6. + return errors.New(errmsg) + } + } + + return nil +} + +func (b *Builder) swigVersionCheck() error { + swigCheckOnce.Do(func() { + swigCheck = b.swigDoVersionCheck() + }) + return swigCheck +} + +// Find the value to pass for the -intgosize option to swig. +var ( + swigIntSizeOnce sync.Once + swigIntSize string + swigIntSizeError error +) + +// This code fails to build if sizeof(int) <= 32 +const swigIntSizeCode = ` +package main +const i int = 1 << 32 +` + +// Determine the size of int on the target system for the -intgosize option +// of swig >= 2.0.9. Run only once. +func (b *Builder) swigDoIntSize(obj string) (intsize string, err error) { + if cfg.BuildN { + return "$INTBITS", nil + } + src := filepath.Join(b.WorkDir, "swig_intsize.go") + if err = ioutil.WriteFile(src, []byte(swigIntSizeCode), 0666); err != nil { + return + } + srcs := []string{src} + + p := load.GoFilesPackage(srcs) + + if _, _, e := BuildToolchain.gc(b, p, "", obj, false, nil, srcs); e != nil { + return "32", nil + } + return "64", nil +} + +// Determine the size of int on the target system for the -intgosize option +// of swig >= 2.0.9. +func (b *Builder) swigIntSize(obj string) (intsize string, err error) { + swigIntSizeOnce.Do(func() { + swigIntSize, swigIntSizeError = b.swigDoIntSize(obj) + }) + return swigIntSize, swigIntSizeError +} + +// Run SWIG on one SWIG input file. +func (b *Builder) swigOne(p *load.Package, file, obj string, pcCFLAGS []string, cxx bool, intgosize string) (outGo, outC string, err error) { + cgoCPPFLAGS, cgoCFLAGS, cgoCXXFLAGS, _, _ := b.CFlags(p) + var cflags []string + if cxx { + cflags = str.StringList(cgoCPPFLAGS, pcCFLAGS, cgoCXXFLAGS) + } else { + cflags = str.StringList(cgoCPPFLAGS, pcCFLAGS, cgoCFLAGS) + } + + n := 5 // length of ".swig" + if cxx { + n = 8 // length of ".swigcxx" + } + base := file[:len(file)-n] + goFile := base + ".go" + gccBase := base + "_wrap." + gccExt := "c" + if cxx { + gccExt = "cxx" + } + + gccgo := cfg.BuildToolchainName == "gccgo" + + // swig + args := []string{ + "-go", + "-cgo", + "-intgosize", intgosize, + "-module", base, + "-o", obj + gccBase + gccExt, + "-outdir", obj, + } + + for _, f := range cflags { + if len(f) > 3 && f[:2] == "-I" { + args = append(args, f) + } + } + + if gccgo { + args = append(args, "-gccgo") + if pkgpath := gccgoPkgpath(p); pkgpath != "" { + args = append(args, "-go-pkgpath", pkgpath) + } + } + if cxx { + args = append(args, "-c++") + } + + out, err := b.runOut(p.Dir, p.ImportPath, nil, "swig", args, file) + if err != nil { + if len(out) > 0 { + if bytes.Contains(out, []byte("-intgosize")) || bytes.Contains(out, []byte("-cgo")) { + return "", "", errors.New("must have SWIG version >= 3.0.6") + } + b.showOutput(p.Dir, p.ImportPath, b.processOutput(out)) // swig error + return "", "", errPrintedOutput + } + return "", "", err + } + if len(out) > 0 { + b.showOutput(p.Dir, p.ImportPath, b.processOutput(out)) // swig warning + } + + return goFile, obj + gccBase + gccExt, nil +} + +// disableBuildID adjusts a linker command line to avoid creating a +// build ID when creating an object file rather than an executable or +// shared library. Some systems, such as Ubuntu, always add +// --build-id to every link, but we don't want a build ID when we are +// producing an object file. On some of those system a plain -r (not +// -Wl,-r) will turn off --build-id, but clang 3.0 doesn't support a +// plain -r. I don't know how to turn off --build-id when using clang +// other than passing a trailing --build-id=none. So that is what we +// do, but only on systems likely to support it, which is to say, +// systems that normally use gold or the GNU linker. +func (b *Builder) disableBuildID(ldflags []string) []string { + switch cfg.Goos { + case "android", "dragonfly", "linux", "netbsd": + ldflags = append(ldflags, "-Wl,--build-id=none") + } + return ldflags +} + +// An actionQueue is a priority queue of actions. +type actionQueue []*Action + +// Implement heap.Interface +func (q *actionQueue) Len() int { return len(*q) } +func (q *actionQueue) Swap(i, j int) { (*q)[i], (*q)[j] = (*q)[j], (*q)[i] } +func (q *actionQueue) Less(i, j int) bool { return (*q)[i].priority < (*q)[j].priority } +func (q *actionQueue) Push(x interface{}) { *q = append(*q, x.(*Action)) } +func (q *actionQueue) Pop() interface{} { + n := len(*q) - 1 + x := (*q)[n] + *q = (*q)[:n] + return x +} + +func (q *actionQueue) push(a *Action) { + heap.Push(q, a) +} + +func (q *actionQueue) pop() *Action { + return heap.Pop(q).(*Action) +} + +func InstrumentInit() { + if !cfg.BuildRace && !cfg.BuildMSan { + return + } + if cfg.BuildRace && cfg.BuildMSan { + fmt.Fprintf(os.Stderr, "go %s: may not use -race and -msan simultaneously\n", flag.Args()[0]) + os.Exit(2) + } + if cfg.BuildMSan && (cfg.Goos != "linux" || cfg.Goarch != "amd64") { + fmt.Fprintf(os.Stderr, "-msan is not supported on %s/%s\n", cfg.Goos, cfg.Goarch) + os.Exit(2) + } + if cfg.Goarch != "amd64" || cfg.Goos != "linux" && cfg.Goos != "freebsd" && cfg.Goos != "darwin" && cfg.Goos != "windows" { + fmt.Fprintf(os.Stderr, "go %s: -race and -msan are only supported on linux/amd64, freebsd/amd64, darwin/amd64 and windows/amd64\n", flag.Args()[0]) + os.Exit(2) + } + if !cfg.BuildContext.CgoEnabled { + fmt.Fprintf(os.Stderr, "go %s: -race requires cgo; enable cgo by setting CGO_ENABLED=1\n", flag.Args()[0]) + os.Exit(2) + } + if cfg.BuildRace { + buildGcflags = append(buildGcflags, "-race") + cfg.BuildLdflags = append(cfg.BuildLdflags, "-race") + } else { + buildGcflags = append(buildGcflags, "-msan") + cfg.BuildLdflags = append(cfg.BuildLdflags, "-msan") + } + if cfg.BuildContext.InstallSuffix != "" { + cfg.BuildContext.InstallSuffix += "_" + } + + if cfg.BuildRace { + cfg.BuildContext.InstallSuffix += "race" + cfg.BuildContext.BuildTags = append(cfg.BuildContext.BuildTags, "race") + } else { + cfg.BuildContext.InstallSuffix += "msan" + cfg.BuildContext.BuildTags = append(cfg.BuildContext.BuildTags, "msan") + } +} + +// ExecCmd is the command to use to run user binaries. +// Normally it is empty, meaning run the binaries directly. +// If cross-compiling and running on a remote system or +// simulator, it is typically go_GOOS_GOARCH_exec, with +// the target GOOS and GOARCH substituted. +// The -exec flag overrides these defaults. +var ExecCmd []string + +// FindExecCmd derives the value of ExecCmd to use. +// It returns that value and leaves ExecCmd set for direct use. +func FindExecCmd() []string { + if ExecCmd != nil { + return ExecCmd + } + ExecCmd = []string{} // avoid work the second time + if cfg.Goos == runtime.GOOS && cfg.Goarch == runtime.GOARCH { + return ExecCmd + } + path, err := exec.LookPath(fmt.Sprintf("go_%s_%s_exec", cfg.Goos, cfg.Goarch)) + if err == nil { + ExecCmd = []string{path} + } + return ExecCmd +} diff --git a/libgo/go/cmd/go/internal/work/build_test.go b/libgo/go/cmd/go/internal/work/build_test.go new file mode 100644 index 00000000000..294b83c6b2a --- /dev/null +++ b/libgo/go/cmd/go/internal/work/build_test.go @@ -0,0 +1,227 @@ +// Copyright 2016 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package work + +import ( + "bytes" + "fmt" + "io/ioutil" + "os" + "path/filepath" + "reflect" + "runtime" + "strings" + "testing" + + "cmd/go/internal/base" + "cmd/go/internal/cfg" + "cmd/go/internal/load" +) + +func TestRemoveDevNull(t *testing.T) { + fi, err := os.Lstat(os.DevNull) + if err != nil { + t.Skip(err) + } + if fi.Mode().IsRegular() { + t.Errorf("Lstat(%s).Mode().IsRegular() = true; expected false", os.DevNull) + } + mayberemovefile(os.DevNull) + _, err = os.Lstat(os.DevNull) + if err != nil { + t.Errorf("mayberemovefile(%s) did remove it; oops", os.DevNull) + } +} + +func TestSplitPkgConfigOutput(t *testing.T) { + for _, test := range []struct { + in []byte + want []string + }{ + {[]byte(`-r:foo -L/usr/white\ space/lib -lfoo\ bar -lbar\ baz`), []string{"-r:foo", "-L/usr/white space/lib", "-lfoo bar", "-lbar baz"}}, + {[]byte(`-lextra\ fun\ arg\\`), []string{`-lextra fun arg\`}}, + {[]byte(`broken flag\`), []string{"broken", "flag"}}, + {[]byte("\textra whitespace\r\n"), []string{"extra", "whitespace"}}, + {[]byte(" \r\n "), nil}, + } { + got := splitPkgConfigOutput(test.in) + if !reflect.DeepEqual(got, test.want) { + t.Errorf("splitPkgConfigOutput(%v) = %v; want %v", test.in, got, test.want) + } + } +} + +func TestSharedLibName(t *testing.T) { + // TODO(avdva) - make these values platform-specific + prefix := "lib" + suffix := ".so" + testData := []struct { + args []string + pkgs []*load.Package + expected string + expectErr bool + rootedAt string + }{ + { + args: []string{"std"}, + pkgs: []*load.Package{}, + expected: "std", + }, + { + args: []string{"std", "cmd"}, + pkgs: []*load.Package{}, + expected: "std,cmd", + }, + { + args: []string{}, + pkgs: []*load.Package{pkgImportPath("gopkg.in/somelib")}, + expected: "gopkg.in-somelib", + }, + { + args: []string{"./..."}, + pkgs: []*load.Package{pkgImportPath("somelib")}, + expected: "somelib", + rootedAt: "somelib", + }, + { + args: []string{"../somelib", "../somelib"}, + pkgs: []*load.Package{pkgImportPath("somelib")}, + expected: "somelib", + }, + { + args: []string{"../lib1", "../lib2"}, + pkgs: []*load.Package{pkgImportPath("gopkg.in/lib1"), pkgImportPath("gopkg.in/lib2")}, + expected: "gopkg.in-lib1,gopkg.in-lib2", + }, + { + args: []string{"./..."}, + pkgs: []*load.Package{ + pkgImportPath("gopkg.in/dir/lib1"), + pkgImportPath("gopkg.in/lib2"), + pkgImportPath("gopkg.in/lib3"), + }, + expected: "gopkg.in", + rootedAt: "gopkg.in", + }, + { + args: []string{"std", "../lib2"}, + pkgs: []*load.Package{}, + expectErr: true, + }, + { + args: []string{"all", "./"}, + pkgs: []*load.Package{}, + expectErr: true, + }, + { + args: []string{"cmd", "fmt"}, + pkgs: []*load.Package{}, + expectErr: true, + }, + } + for _, data := range testData { + func() { + if data.rootedAt != "" { + tmpGopath, err := ioutil.TempDir("", "gopath") + if err != nil { + t.Fatal(err) + } + oldGopath := cfg.BuildContext.GOPATH + defer func() { + cfg.BuildContext.GOPATH = oldGopath + os.Chdir(base.Cwd) + err := os.RemoveAll(tmpGopath) + if err != nil { + t.Error(err) + } + }() + root := filepath.Join(tmpGopath, "src", data.rootedAt) + err = os.MkdirAll(root, 0755) + if err != nil { + t.Fatal(err) + } + cfg.BuildContext.GOPATH = tmpGopath + os.Chdir(root) + } + computed, err := libname(data.args, data.pkgs) + if err != nil { + if !data.expectErr { + t.Errorf("libname returned an error %q, expected a name", err.Error()) + } + } else if data.expectErr { + t.Errorf("libname returned %q, expected an error", computed) + } else { + expected := prefix + data.expected + suffix + if expected != computed { + t.Errorf("libname returned %q, expected %q", computed, expected) + } + } + }() + } +} + +func pkgImportPath(pkgpath string) *load.Package { + return &load.Package{ + PackagePublic: load.PackagePublic{ + ImportPath: pkgpath, + }, + } +} + +// When installing packages, the installed package directory should +// respect the SetGID bit and group name of the destination +// directory. +// See https://golang.org/issue/18878. +func TestRespectSetgidDir(t *testing.T) { + if runtime.GOOS == "nacl" { + t.Skip("can't set SetGID bit with chmod on nacl") + } + + var b Builder + + // Check that `cp` is called instead of `mv` by looking at the output + // of `(*Builder).ShowCmd` afterwards as a sanity check. + cfg.BuildX = true + var cmdBuf bytes.Buffer + b.Print = func(a ...interface{}) (int, error) { + return cmdBuf.WriteString(fmt.Sprint(a...)) + } + + setgiddir, err := ioutil.TempDir("", "SetGroupID") + if err != nil { + t.Fatal(err) + } + defer os.RemoveAll(setgiddir) + + if runtime.GOOS == "freebsd" { + err = os.Chown(setgiddir, os.Getuid(), os.Getgid()) + if err != nil { + t.Fatal(err) + } + } + + // Change setgiddir's permissions to include the SetGID bit. + if err := os.Chmod(setgiddir, 0755|os.ModeSetgid); err != nil { + t.Fatal(err) + } + + pkgfile, err := ioutil.TempFile("", "pkgfile") + if err != nil { + t.Fatalf("ioutil.TempFile(\"\", \"pkgfile\"): %v", err) + } + defer os.Remove(pkgfile.Name()) + defer pkgfile.Close() + + dirGIDFile := filepath.Join(setgiddir, "setgid") + if err := b.moveOrCopyFile(nil, dirGIDFile, pkgfile.Name(), 0666, true); err != nil { + t.Fatalf("moveOrCopyFile: %v", err) + } + + got := strings.TrimSpace(cmdBuf.String()) + want := b.fmtcmd("", "cp %s %s", pkgfile.Name(), dirGIDFile) + if got != want { + t.Fatalf("moveOrCopyFile(%q, %q): want %q, got %q", dirGIDFile, pkgfile.Name(), want, got) + } +} diff --git a/libgo/go/cmd/go/internal/work/testgo.go b/libgo/go/cmd/go/internal/work/testgo.go new file mode 100644 index 00000000000..3e623c66210 --- /dev/null +++ b/libgo/go/cmd/go/internal/work/testgo.go @@ -0,0 +1,17 @@ +// Copyright 2017 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// This file contains extra hooks for testing the go command. + +// +build testgo + +package work + +import "os" + +func init() { + if v := os.Getenv("TESTGO_VERSION"); v != "" { + runtimeVersion = v + } +} diff --git a/libgo/go/cmd/go/list.go b/libgo/go/cmd/go/list.go deleted file mode 100644 index 2f240834b2a..00000000000 --- a/libgo/go/cmd/go/list.go +++ /dev/null @@ -1,234 +0,0 @@ -// Copyright 2011 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package main - -import ( - "bufio" - "encoding/json" - "io" - "os" - "strings" - "text/template" -) - -var cmdList = &Command{ - UsageLine: "list [-e] [-f format] [-json] [build flags] [packages]", - Short: "list packages", - Long: ` -List lists the packages named by the import paths, one per line. - -The default output shows the package import path: - - bytes - encoding/json - github.com/gorilla/mux - golang.org/x/net/html - -The -f flag specifies an alternate format for the list, using the -syntax of package template. The default output is equivalent to -f -'{{.ImportPath}}'. The struct being passed to the template is: - - type Package struct { - Dir string // directory containing package sources - ImportPath string // import path of package in dir - ImportComment string // path in import comment on package statement - Name string // package name - Doc string // package documentation string - Target string // install path - Shlib string // the shared library that contains this package (only set when -linkshared) - Goroot bool // is this package in the Go root? - Standard bool // is this package part of the standard Go library? - Stale bool // would 'go install' do anything for this package? - StaleReason string // explanation for Stale==true - Root string // Go root or Go path dir containing this package - ConflictDir string // this directory shadows Dir in $GOPATH - BinaryOnly bool // binary-only package: cannot be recompiled from sources - - // Source files - GoFiles []string // .go source files (excluding CgoFiles, TestGoFiles, XTestGoFiles) - CgoFiles []string // .go sources files that import "C" - IgnoredGoFiles []string // .go sources ignored due to build constraints - CFiles []string // .c source files - CXXFiles []string // .cc, .cxx and .cpp source files - MFiles []string // .m source files - HFiles []string // .h, .hh, .hpp and .hxx source files - FFiles []string // .f, .F, .for and .f90 Fortran source files - SFiles []string // .s source files - SwigFiles []string // .swig files - SwigCXXFiles []string // .swigcxx files - SysoFiles []string // .syso object files to add to archive - TestGoFiles []string // _test.go files in package - XTestGoFiles []string // _test.go files outside package - - // Cgo directives - CgoCFLAGS []string // cgo: flags for C compiler - CgoCPPFLAGS []string // cgo: flags for C preprocessor - CgoCXXFLAGS []string // cgo: flags for C++ compiler - CgoFFLAGS []string // cgo: flags for Fortran compiler - CgoLDFLAGS []string // cgo: flags for linker - CgoPkgConfig []string // cgo: pkg-config names - - // Dependency information - Imports []string // import paths used by this package - Deps []string // all (recursively) imported dependencies - TestImports []string // imports from TestGoFiles - XTestImports []string // imports from XTestGoFiles - - // Error information - Incomplete bool // this package or a dependency has an error - Error *PackageError // error loading package - DepsErrors []*PackageError // errors loading dependencies - } - -Packages stored in vendor directories report an ImportPath that includes the -path to the vendor directory (for example, "d/vendor/p" instead of "p"), -so that the ImportPath uniquely identifies a given copy of a package. -The Imports, Deps, TestImports, and XTestImports lists also contain these -expanded imports paths. See golang.org/s/go15vendor for more about vendoring. - -The error information, if any, is - - type PackageError struct { - ImportStack []string // shortest path from package named on command line to this one - Pos string // position of error (if present, file:line:col) - Err string // the error itself - } - -The template function "join" calls strings.Join. - -The template function "context" returns the build context, defined as: - - type Context struct { - GOARCH string // target architecture - GOOS string // target operating system - GOROOT string // Go root - GOPATH string // Go path - CgoEnabled bool // whether cgo can be used - UseAllFiles bool // use files regardless of +build lines, file names - Compiler string // compiler to assume when computing target paths - BuildTags []string // build constraints to match in +build lines - ReleaseTags []string // releases the current release is compatible with - InstallSuffix string // suffix to use in the name of the install dir - } - -For more information about the meaning of these fields see the documentation -for the go/build package's Context type. - -The -json flag causes the package data to be printed in JSON format -instead of using the template format. - -The -e flag changes the handling of erroneous packages, those that -cannot be found or are malformed. By default, the list command -prints an error to standard error for each erroneous package and -omits the packages from consideration during the usual printing. -With the -e flag, the list command never prints errors to standard -error and instead processes the erroneous packages with the usual -printing. Erroneous packages will have a non-empty ImportPath and -a non-nil Error field; other information may or may not be missing -(zeroed). - -For more about build flags, see 'go help build'. - -For more about specifying packages, see 'go help packages'. - `, -} - -func init() { - cmdList.Run = runList // break init cycle - addBuildFlags(cmdList) -} - -var listE = cmdList.Flag.Bool("e", false, "") -var listFmt = cmdList.Flag.String("f", "{{.ImportPath}}", "") -var listJson = cmdList.Flag.Bool("json", false, "") -var nl = []byte{'\n'} - -func runList(cmd *Command, args []string) { - buildModeInit() - out := newTrackingWriter(os.Stdout) - defer out.w.Flush() - - var do func(*Package) - if *listJson { - do = func(p *Package) { - b, err := json.MarshalIndent(p, "", "\t") - if err != nil { - out.Flush() - fatalf("%s", err) - } - out.Write(b) - out.Write(nl) - } - } else { - var cachedCtxt *Context - context := func() *Context { - if cachedCtxt == nil { - cachedCtxt = newContext(&buildContext) - } - return cachedCtxt - } - fm := template.FuncMap{ - "join": strings.Join, - "context": context, - } - tmpl, err := template.New("main").Funcs(fm).Parse(*listFmt) - if err != nil { - fatalf("%s", err) - } - do = func(p *Package) { - if err := tmpl.Execute(out, p); err != nil { - out.Flush() - fatalf("%s", err) - } - if out.NeedNL() { - out.Write(nl) - } - } - } - - load := packages - if *listE { - load = packagesAndErrors - } - - for _, pkg := range load(args) { - // Show vendor-expanded paths in listing - pkg.TestImports = pkg.vendored(pkg.TestImports) - pkg.XTestImports = pkg.vendored(pkg.XTestImports) - - do(pkg) - } -} - -// TrackingWriter tracks the last byte written on every write so -// we can avoid printing a newline if one was already written or -// if there is no output at all. -type TrackingWriter struct { - w *bufio.Writer - last byte -} - -func newTrackingWriter(w io.Writer) *TrackingWriter { - return &TrackingWriter{ - w: bufio.NewWriter(w), - last: '\n', - } -} - -func (t *TrackingWriter) Write(p []byte) (n int, err error) { - n, err = t.w.Write(p) - if n > 0 { - t.last = p[n-1] - } - return -} - -func (t *TrackingWriter) Flush() { - t.w.Flush() -} - -func (t *TrackingWriter) NeedNL() bool { - return t.last != '\n' -} diff --git a/libgo/go/cmd/go/main.go b/libgo/go/cmd/go/main.go index ceeac4d4a3a..794ce08b769 100644 --- a/libgo/go/cmd/go/main.go +++ b/libgo/go/cmd/go/main.go @@ -2,141 +2,90 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. +//go:generate ./mkalldocs.sh + package main import ( - "bufio" - "bytes" "flag" "fmt" - "go/build" - "io" "log" "os" - "os/exec" - "path" "path/filepath" - "regexp" "runtime" "strings" - "sync" - "text/template" - "unicode" - "unicode/utf8" -) - -// A Command is an implementation of a go command -// like go build or go fix. -type Command struct { - // Run runs the command. - // The args are the arguments after the command name. - Run func(cmd *Command, args []string) - - // UsageLine is the one-line usage message. - // The first word in the line is taken to be the command name. - UsageLine string - - // Short is the short description shown in the 'go help' output. - Short string - - // Long is the long message shown in the 'go help ' output. - Long string - - // Flag is a set of flags specific to this command. - Flag flag.FlagSet - // CustomFlags indicates that the command will do its own - // flag parsing. - CustomFlags bool -} - -// Name returns the command's name: the first word in the usage line. -func (c *Command) Name() string { - name := c.UsageLine - i := strings.Index(name, " ") - if i >= 0 { - name = name[:i] - } - return name -} - -func (c *Command) Usage() { - fmt.Fprintf(os.Stderr, "usage: %s\n\n", c.UsageLine) - fmt.Fprintf(os.Stderr, "%s\n", strings.TrimSpace(c.Long)) - os.Exit(2) -} - -// Runnable reports whether the command can be run; otherwise -// it is a documentation pseudo-command such as importpath. -func (c *Command) Runnable() bool { - return c.Run != nil -} - -// Commands lists the available commands and help topics. -// The order here is the order in which they are printed by 'go help'. -var commands = []*Command{ - cmdBuild, - cmdClean, - cmdDoc, - cmdEnv, - cmdBug, - cmdFix, - cmdFmt, - cmdGenerate, - cmdGet, - cmdInstall, - cmdList, - cmdRun, - cmdTest, - cmdTool, - cmdVersion, - cmdVet, - - helpC, - helpBuildmode, - helpFileType, - helpGopath, - helpEnvironment, - helpImportPath, - helpPackages, - helpTestflag, - helpTestfunc, -} - -var exitStatus = 0 -var exitMu sync.Mutex + "cmd/go/internal/base" + "cmd/go/internal/bug" + "cmd/go/internal/cfg" + "cmd/go/internal/clean" + "cmd/go/internal/doc" + "cmd/go/internal/envcmd" + "cmd/go/internal/fix" + "cmd/go/internal/fmtcmd" + "cmd/go/internal/generate" + "cmd/go/internal/get" + "cmd/go/internal/help" + "cmd/go/internal/list" + "cmd/go/internal/run" + "cmd/go/internal/test" + "cmd/go/internal/tool" + "cmd/go/internal/version" + "cmd/go/internal/vet" + "cmd/go/internal/work" +) -func setExitStatus(n int) { - exitMu.Lock() - if exitStatus < n { - exitStatus = n +func init() { + base.Commands = []*base.Command{ + work.CmdBuild, + clean.CmdClean, + doc.CmdDoc, + envcmd.CmdEnv, + bug.CmdBug, + fix.CmdFix, + fmtcmd.CmdFmt, + generate.CmdGenerate, + get.CmdGet, + work.CmdInstall, + list.CmdList, + run.CmdRun, + test.CmdTest, + tool.CmdTool, + version.CmdVersion, + vet.CmdVet, + + help.HelpC, + help.HelpBuildmode, + help.HelpFileType, + help.HelpGopath, + help.HelpEnvironment, + help.HelpImportPath, + help.HelpPackages, + test.HelpTestflag, + test.HelpTestfunc, } - exitMu.Unlock() } -var origEnv []string -var newEnv []envVar - func main() { _ = go11tag - flag.Usage = usage + flag.Usage = base.Usage flag.Parse() log.SetFlags(0) args := flag.Args() if len(args) < 1 { - usage() + base.Usage() } if args[0] == "help" { - help(args[1:]) + help.Help(args[1:]) return } // Diagnose common mistake: GOPATH==GOROOT. // This setting is equivalent to not setting GOPATH at all, // which is not what most people want when they do it. - if gopath := buildContext.GOPATH; filepath.Clean(gopath) == filepath.Clean(runtime.GOROOT()) { + if gopath := cfg.BuildContext.GOPATH; filepath.Clean(gopath) == filepath.Clean(runtime.GOROOT()) { fmt.Fprintf(os.Stderr, "warning: GOPATH set to GOROOT (%s) has no effect\n", gopath) } else { for _, p := range filepath.SplitList(gopath) { @@ -154,14 +103,12 @@ func main() { } } - if fi, err := os.Stat(goroot); err != nil || !fi.IsDir() { - // For gccgo this is fine, carry on. - // Note that this check is imperfect as we have not yet - // parsed the -compiler flag. - if runtime.Compiler != "gccgo" { - fmt.Fprintf(os.Stderr, "go: cannot find GOROOT directory: %v\n", goroot) - os.Exit(2) - } + // For gccgo this is fine, carry on. + // Note that this check is imperfect as we have not yet parsed + // the -compiler flag. + if fi, err := os.Stat(cfg.GOROOT); err != nil || !fi.IsDir() && runtime.Compiler != "gccgo" { + fmt.Fprintf(os.Stderr, "go: cannot find GOROOT directory: %v\n", cfg.GOROOT) + os.Exit(2) } // Set environment (GOOS, GOARCH, etc) explicitly. @@ -169,15 +116,15 @@ func main() { // the same default computation of these as we do, // but in practice there might be skew // This makes sure we all agree. - origEnv = os.Environ() - newEnv = mkEnv() - for _, env := range newEnv { - if os.Getenv(env.name) != env.value { - os.Setenv(env.name, env.value) + cfg.OrigEnv = os.Environ() + cfg.CmdEnv = envcmd.MkEnv() + for _, env := range cfg.CmdEnv { + if os.Getenv(env.Name) != env.Value { + os.Setenv(env.Name, env.Value) } } - for _, cmd := range commands { + for _, cmd := range base.Commands { if cmd.Name() == args[0] && cmd.Runnable() { cmd.Flag.Usage = func() { cmd.Usage() } if cmd.CustomFlags { @@ -187,615 +134,25 @@ func main() { args = cmd.Flag.Args() } cmd.Run(cmd, args) - exit() + base.Exit() return } } fmt.Fprintf(os.Stderr, "go: unknown subcommand %q\nRun 'go help' for usage.\n", args[0]) - setExitStatus(2) - exit() -} - -var usageTemplate = `Go is a tool for managing Go source code. - -Usage: - - go command [arguments] - -The commands are: -{{range .}}{{if .Runnable}} - {{.Name | printf "%-11s"}} {{.Short}}{{end}}{{end}} - -Use "go help [command]" for more information about a command. - -Additional help topics: -{{range .}}{{if not .Runnable}} - {{.Name | printf "%-11s"}} {{.Short}}{{end}}{{end}} - -Use "go help [topic]" for more information about that topic. - -` - -var helpTemplate = `{{if .Runnable}}usage: go {{.UsageLine}} - -{{end}}{{.Long | trim}} -` - -var documentationTemplate = `{{range .}}{{if .Short}}{{.Short | capitalize}} - -{{end}}{{if .Runnable}}Usage: - - go {{.UsageLine}} - -{{end}}{{.Long | trim}} - - -{{end}}` - -// commentWriter writes a Go comment to the underlying io.Writer, -// using line comment form (//). -type commentWriter struct { - W io.Writer - wroteSlashes bool // Wrote "//" at the beginning of the current line. -} - -func (c *commentWriter) Write(p []byte) (int, error) { - var n int - for i, b := range p { - if !c.wroteSlashes { - s := "//" - if b != '\n' { - s = "// " - } - if _, err := io.WriteString(c.W, s); err != nil { - return n, err - } - c.wroteSlashes = true - } - n0, err := c.W.Write(p[i : i+1]) - n += n0 - if err != nil { - return n, err - } - if b == '\n' { - c.wroteSlashes = false - } - } - return len(p), nil -} - -// An errWriter wraps a writer, recording whether a write error occurred. -type errWriter struct { - w io.Writer - err error + base.SetExitStatus(2) + base.Exit() } -func (w *errWriter) Write(b []byte) (int, error) { - n, err := w.w.Write(b) - if err != nil { - w.err = err - } - return n, err +func init() { + base.Usage = mainUsage } -// tmpl executes the given template text on data, writing the result to w. -func tmpl(w io.Writer, text string, data interface{}) { - t := template.New("top") - t.Funcs(template.FuncMap{"trim": strings.TrimSpace, "capitalize": capitalize}) - template.Must(t.Parse(text)) - ew := &errWriter{w: w} - err := t.Execute(ew, data) - if ew.err != nil { - // I/O error writing. Ignore write on closed pipe. - if strings.Contains(ew.err.Error(), "pipe") { - os.Exit(1) - } - fatalf("writing output: %v", ew.err) - } - if err != nil { - panic(err) - } -} - -func capitalize(s string) string { - if s == "" { - return s - } - r, n := utf8.DecodeRuneInString(s) - return string(unicode.ToTitle(r)) + s[n:] -} - -func printUsage(w io.Writer) { - bw := bufio.NewWriter(w) - tmpl(bw, usageTemplate, commands) - bw.Flush() -} - -func usage() { +func mainUsage() { // special case "go test -h" if len(os.Args) > 1 && os.Args[1] == "test" { - os.Stderr.WriteString(testUsage + "\n\n" + - strings.TrimSpace(testFlag1) + "\n\n\t" + - strings.TrimSpace(testFlag2) + "\n") - os.Exit(2) + test.Usage() } - printUsage(os.Stderr) + help.PrintUsage(os.Stderr) os.Exit(2) } - -// help implements the 'help' command. -func help(args []string) { - if len(args) == 0 { - printUsage(os.Stdout) - // not exit 2: succeeded at 'go help'. - return - } - if len(args) != 1 { - fmt.Fprintf(os.Stderr, "usage: go help command\n\nToo many arguments given.\n") - os.Exit(2) // failed at 'go help' - } - - arg := args[0] - - // 'go help documentation' generates doc.go. - if arg == "documentation" { - fmt.Println("// Copyright 2011 The Go Authors. All rights reserved.") - fmt.Println("// Use of this source code is governed by a BSD-style") - fmt.Println("// license that can be found in the LICENSE file.") - fmt.Println() - fmt.Println("// DO NOT EDIT THIS FILE. GENERATED BY mkalldocs.sh.") - fmt.Println("// Edit the documentation in other files and rerun mkalldocs.sh to generate this one.") - fmt.Println() - buf := new(bytes.Buffer) - printUsage(buf) - usage := &Command{Long: buf.String()} - tmpl(&commentWriter{W: os.Stdout}, documentationTemplate, append([]*Command{usage}, commands...)) - fmt.Println("package main") - return - } - - for _, cmd := range commands { - if cmd.Name() == arg { - tmpl(os.Stdout, helpTemplate, cmd) - // not exit 2: succeeded at 'go help cmd'. - return - } - } - - fmt.Fprintf(os.Stderr, "Unknown help topic %#q. Run 'go help'.\n", arg) - os.Exit(2) // failed at 'go help cmd' -} - -// importPathsNoDotExpansion returns the import paths to use for the given -// command line, but it does no ... expansion. -func importPathsNoDotExpansion(args []string) []string { - if len(args) == 0 { - return []string{"."} - } - var out []string - for _, a := range args { - // Arguments are supposed to be import paths, but - // as a courtesy to Windows developers, rewrite \ to / - // in command-line arguments. Handles .\... and so on. - if filepath.Separator == '\\' { - a = strings.Replace(a, `\`, `/`, -1) - } - - // Put argument in canonical form, but preserve leading ./. - if strings.HasPrefix(a, "./") { - a = "./" + path.Clean(a) - if a == "./." { - a = "." - } - } else { - a = path.Clean(a) - } - if isMetaPackage(a) { - out = append(out, allPackages(a)...) - continue - } - out = append(out, a) - } - return out -} - -// importPaths returns the import paths to use for the given command line. -func importPaths(args []string) []string { - args = importPathsNoDotExpansion(args) - var out []string - for _, a := range args { - if strings.Contains(a, "...") { - if build.IsLocalImport(a) { - out = append(out, allPackagesInFS(a)...) - } else { - out = append(out, allPackages(a)...) - } - continue - } - out = append(out, a) - } - return out -} - -var atexitFuncs []func() - -func atexit(f func()) { - atexitFuncs = append(atexitFuncs, f) -} - -func exit() { - for _, f := range atexitFuncs { - f() - } - os.Exit(exitStatus) -} - -func fatalf(format string, args ...interface{}) { - errorf(format, args...) - exit() -} - -func errorf(format string, args ...interface{}) { - log.Printf(format, args...) - setExitStatus(1) -} - -func exitIfErrors() { - if exitStatus != 0 { - exit() - } -} - -func run(cmdargs ...interface{}) { - cmdline := stringList(cmdargs...) - if buildN || buildX { - fmt.Printf("%s\n", strings.Join(cmdline, " ")) - if buildN { - return - } - } - - cmd := exec.Command(cmdline[0], cmdline[1:]...) - cmd.Stdout = os.Stdout - cmd.Stderr = os.Stderr - if err := cmd.Run(); err != nil { - errorf("%v", err) - } -} - -// envForDir returns a copy of the environment -// suitable for running in the given directory. -// The environment is the current process's environment -// but with an updated $PWD, so that an os.Getwd in the -// child will be faster. -func envForDir(dir string, base []string) []string { - // Internally we only use rooted paths, so dir is rooted. - // Even if dir is not rooted, no harm done. - return mergeEnvLists([]string{"PWD=" + dir}, base) -} - -// mergeEnvLists merges the two environment lists such that -// variables with the same name in "in" replace those in "out". -// This always returns a newly allocated slice. -func mergeEnvLists(in, out []string) []string { - out = append([]string(nil), out...) -NextVar: - for _, inkv := range in { - k := strings.SplitAfterN(inkv, "=", 2)[0] - for i, outkv := range out { - if strings.HasPrefix(outkv, k) { - out[i] = inkv - continue NextVar - } - } - out = append(out, inkv) - } - return out -} - -// matchPattern(pattern)(name) reports whether -// name matches pattern. Pattern is a limited glob -// pattern in which '...' means 'any string' and there -// is no other special syntax. -func matchPattern(pattern string) func(name string) bool { - re := regexp.QuoteMeta(pattern) - re = strings.Replace(re, `\.\.\.`, `.*`, -1) - // Special case: foo/... matches foo too. - if strings.HasSuffix(re, `/.*`) { - re = re[:len(re)-len(`/.*`)] + `(/.*)?` - } - reg := regexp.MustCompile(`^` + re + `$`) - return func(name string) bool { - return reg.MatchString(name) - } -} - -// hasPathPrefix reports whether the path s begins with the -// elements in prefix. -func hasPathPrefix(s, prefix string) bool { - switch { - default: - return false - case len(s) == len(prefix): - return s == prefix - case len(s) > len(prefix): - if prefix != "" && prefix[len(prefix)-1] == '/' { - return strings.HasPrefix(s, prefix) - } - return s[len(prefix)] == '/' && s[:len(prefix)] == prefix - } -} - -// hasFilePathPrefix reports whether the filesystem path s begins with the -// elements in prefix. -func hasFilePathPrefix(s, prefix string) bool { - sv := strings.ToUpper(filepath.VolumeName(s)) - pv := strings.ToUpper(filepath.VolumeName(prefix)) - s = s[len(sv):] - prefix = prefix[len(pv):] - switch { - default: - return false - case sv != pv: - return false - case len(s) == len(prefix): - return s == prefix - case len(s) > len(prefix): - if prefix != "" && prefix[len(prefix)-1] == filepath.Separator { - return strings.HasPrefix(s, prefix) - } - return s[len(prefix)] == filepath.Separator && s[:len(prefix)] == prefix - } -} - -// expandPath returns the symlink-expanded form of path. -func expandPath(p string) string { - x, err := filepath.EvalSymlinks(p) - if err == nil { - return x - } - return p -} - -// treeCanMatchPattern(pattern)(name) reports whether -// name or children of name can possibly match pattern. -// Pattern is the same limited glob accepted by matchPattern. -func treeCanMatchPattern(pattern string) func(name string) bool { - wildCard := false - if i := strings.Index(pattern, "..."); i >= 0 { - wildCard = true - pattern = pattern[:i] - } - return func(name string) bool { - return len(name) <= len(pattern) && hasPathPrefix(pattern, name) || - wildCard && strings.HasPrefix(name, pattern) - } -} - -// allPackages returns all the packages that can be found -// under the $GOPATH directories and $GOROOT matching pattern. -// The pattern is either "all" (all packages), "std" (standard packages), -// "cmd" (standard commands), or a path including "...". -func allPackages(pattern string) []string { - pkgs := matchPackages(pattern) - if len(pkgs) == 0 { - fmt.Fprintf(os.Stderr, "warning: %q matched no packages\n", pattern) - } - return pkgs -} - -func matchPackages(pattern string) []string { - match := func(string) bool { return true } - treeCanMatch := func(string) bool { return true } - if !isMetaPackage(pattern) { - match = matchPattern(pattern) - treeCanMatch = treeCanMatchPattern(pattern) - } - - have := map[string]bool{ - "builtin": true, // ignore pseudo-package that exists only for documentation - } - if !buildContext.CgoEnabled { - have["runtime/cgo"] = true // ignore during walk - } - var pkgs []string - - for _, src := range buildContext.SrcDirs() { - if (pattern == "std" || pattern == "cmd") && src != gorootSrc { - continue - } - src = filepath.Clean(src) + string(filepath.Separator) - root := src - if pattern == "cmd" { - root += "cmd" + string(filepath.Separator) - } - filepath.Walk(root, func(path string, fi os.FileInfo, err error) error { - if err != nil || !fi.IsDir() || path == src { - return nil - } - - // Avoid .foo, _foo, and testdata directory trees. - _, elem := filepath.Split(path) - if strings.HasPrefix(elem, ".") || strings.HasPrefix(elem, "_") || elem == "testdata" { - return filepath.SkipDir - } - - name := filepath.ToSlash(path[len(src):]) - if pattern == "std" && (!isStandardImportPath(name) || name == "cmd") { - // The name "std" is only the standard library. - // If the name is cmd, it's the root of the command tree. - return filepath.SkipDir - } - if !treeCanMatch(name) { - return filepath.SkipDir - } - if have[name] { - return nil - } - have[name] = true - if !match(name) { - return nil - } - _, err = buildContext.ImportDir(path, 0) - if err != nil { - if _, noGo := err.(*build.NoGoError); noGo { - return nil - } - } - pkgs = append(pkgs, name) - return nil - }) - } - return pkgs -} - -// allPackagesInFS is like allPackages but is passed a pattern -// beginning ./ or ../, meaning it should scan the tree rooted -// at the given directory. There are ... in the pattern too. -func allPackagesInFS(pattern string) []string { - pkgs := matchPackagesInFS(pattern) - if len(pkgs) == 0 { - fmt.Fprintf(os.Stderr, "warning: %q matched no packages\n", pattern) - } - return pkgs -} - -func matchPackagesInFS(pattern string) []string { - // Find directory to begin the scan. - // Could be smarter but this one optimization - // is enough for now, since ... is usually at the - // end of a path. - i := strings.Index(pattern, "...") - dir, _ := path.Split(pattern[:i]) - - // pattern begins with ./ or ../. - // path.Clean will discard the ./ but not the ../. - // We need to preserve the ./ for pattern matching - // and in the returned import paths. - prefix := "" - if strings.HasPrefix(pattern, "./") { - prefix = "./" - } - match := matchPattern(pattern) - - var pkgs []string - filepath.Walk(dir, func(path string, fi os.FileInfo, err error) error { - if err != nil || !fi.IsDir() { - return nil - } - if path == dir { - // filepath.Walk starts at dir and recurses. For the recursive case, - // the path is the result of filepath.Join, which calls filepath.Clean. - // The initial case is not Cleaned, though, so we do this explicitly. - // - // This converts a path like "./io/" to "io". Without this step, running - // "cd $GOROOT/src; go list ./io/..." would incorrectly skip the io - // package, because prepending the prefix "./" to the unclean path would - // result in "././io", and match("././io") returns false. - path = filepath.Clean(path) - } - - // Avoid .foo, _foo, and testdata directory trees, but do not avoid "." or "..". - _, elem := filepath.Split(path) - dot := strings.HasPrefix(elem, ".") && elem != "." && elem != ".." - if dot || strings.HasPrefix(elem, "_") || elem == "testdata" { - return filepath.SkipDir - } - - name := prefix + filepath.ToSlash(path) - if !match(name) { - return nil - } - - // We keep the directory if we can import it, or if we can't import it - // due to invalid Go source files. This means that directories containing - // parse errors will be built (and fail) instead of being silently skipped - // as not matching the pattern. Go 1.5 and earlier skipped, but that - // behavior means people miss serious mistakes. - // See golang.org/issue/11407. - if p, err := buildContext.ImportDir(path, 0); err != nil && (p == nil || len(p.InvalidGoFiles) == 0) { - if _, noGo := err.(*build.NoGoError); !noGo { - log.Print(err) - } - return nil - } - pkgs = append(pkgs, name) - return nil - }) - return pkgs -} - -// stringList's arguments should be a sequence of string or []string values. -// stringList flattens them into a single []string. -func stringList(args ...interface{}) []string { - var x []string - for _, arg := range args { - switch arg := arg.(type) { - case []string: - x = append(x, arg...) - case string: - x = append(x, arg) - default: - panic("stringList: invalid argument of type " + fmt.Sprintf("%T", arg)) - } - } - return x -} - -// toFold returns a string with the property that -// strings.EqualFold(s, t) iff toFold(s) == toFold(t) -// This lets us test a large set of strings for fold-equivalent -// duplicates without making a quadratic number of calls -// to EqualFold. Note that strings.ToUpper and strings.ToLower -// have the desired property in some corner cases. -func toFold(s string) string { - // Fast path: all ASCII, no upper case. - // Most paths look like this already. - for i := 0; i < len(s); i++ { - c := s[i] - if c >= utf8.RuneSelf || 'A' <= c && c <= 'Z' { - goto Slow - } - } - return s - -Slow: - var buf bytes.Buffer - for _, r := range s { - // SimpleFold(x) cycles to the next equivalent rune > x - // or wraps around to smaller values. Iterate until it wraps, - // and we've found the minimum value. - for { - r0 := r - r = unicode.SimpleFold(r0) - if r <= r0 { - break - } - } - // Exception to allow fast path above: A-Z => a-z - if 'A' <= r && r <= 'Z' { - r += 'a' - 'A' - } - buf.WriteRune(r) - } - return buf.String() -} - -// foldDup reports a pair of strings from the list that are -// equal according to strings.EqualFold. -// It returns "", "" if there are no such strings. -func foldDup(list []string) (string, string) { - clash := map[string]string{} - for _, s := range list { - fold := toFold(s) - if t := clash[fold]; t != "" { - if s > t { - s, t = t, s - } - return s, t - } - clash[fold] = s - } - return "", "" -} diff --git a/libgo/go/cmd/go/match_test.go b/libgo/go/cmd/go/match_test.go deleted file mode 100644 index e0ad562384d..00000000000 --- a/libgo/go/cmd/go/match_test.go +++ /dev/null @@ -1,88 +0,0 @@ -// Copyright 2012 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package main - -import "testing" - -var matchPatternTests = []stringPairTest{ - {"...", "foo", true}, - {"net", "net", true}, - {"net", "net/http", false}, - {"net/http", "net", false}, - {"net/http", "net/http", true}, - {"net...", "netchan", true}, - {"net...", "net", true}, - {"net...", "net/http", true}, - {"net...", "not/http", false}, - {"net/...", "netchan", false}, - {"net/...", "net", true}, - {"net/...", "net/http", true}, - {"net/...", "not/http", false}, -} - -func TestMatchPattern(t *testing.T) { - testStringPairs(t, "matchPattern", matchPatternTests, func(pattern, name string) bool { - return matchPattern(pattern)(name) - }) -} - -var treeCanMatchPatternTests = []stringPairTest{ - {"...", "foo", true}, - {"net", "net", true}, - {"net", "net/http", false}, - {"net/http", "net", true}, - {"net/http", "net/http", true}, - {"net...", "netchan", true}, - {"net...", "net", true}, - {"net...", "net/http", true}, - {"net...", "not/http", false}, - {"net/...", "netchan", false}, - {"net/...", "net", true}, - {"net/...", "net/http", true}, - {"net/...", "not/http", false}, - {"abc.../def", "abcxyz", true}, - {"abc.../def", "xyxabc", false}, - {"x/y/z/...", "x", true}, - {"x/y/z/...", "x/y", true}, - {"x/y/z/...", "x/y/z", true}, - {"x/y/z/...", "x/y/z/w", true}, - {"x/y/z", "x", true}, - {"x/y/z", "x/y", true}, - {"x/y/z", "x/y/z", true}, - {"x/y/z", "x/y/z/w", false}, - {"x/.../y/z", "x/a/b/c", true}, - {"x/.../y/z", "y/x/a/b/c", false}, -} - -func TestChildrenCanMatchPattern(t *testing.T) { - testStringPairs(t, "treeCanMatchPattern", treeCanMatchPatternTests, func(pattern, name string) bool { - return treeCanMatchPattern(pattern)(name) - }) -} - -var hasPathPrefixTests = []stringPairTest{ - {"abc", "a", false}, - {"a/bc", "a", true}, - {"a", "a", true}, - {"a/bc", "a/", true}, -} - -func TestHasPathPrefix(t *testing.T) { - testStringPairs(t, "hasPathPrefix", hasPathPrefixTests, hasPathPrefix) -} - -type stringPairTest struct { - in1 string - in2 string - out bool -} - -func testStringPairs(t *testing.T, name string, tests []stringPairTest, f func(string, string) bool) { - for _, tt := range tests { - if out := f(tt.in1, tt.in2); out != tt.out { - t.Errorf("%s(%q, %q) = %v, want %v", name, tt.in1, tt.in2, out, tt.out) - } - } -} diff --git a/libgo/go/cmd/go/note.go b/libgo/go/cmd/go/note.go deleted file mode 100644 index fae9536d13f..00000000000 --- a/libgo/go/cmd/go/note.go +++ /dev/null @@ -1,187 +0,0 @@ -// Copyright 2015 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package main - -import ( - "bytes" - "debug/elf" - "debug/macho" - "encoding/binary" - "fmt" - "io" - "os" -) - -func readAligned4(r io.Reader, sz int32) ([]byte, error) { - full := (sz + 3) &^ 3 - data := make([]byte, full) - _, err := io.ReadFull(r, data) - if err != nil { - return nil, err - } - data = data[:sz] - return data, nil -} - -func readELFNote(filename, name string, typ int32) ([]byte, error) { - f, err := elf.Open(filename) - if err != nil { - return nil, err - } - for _, sect := range f.Sections { - if sect.Type != elf.SHT_NOTE { - continue - } - r := sect.Open() - for { - var namesize, descsize, noteType int32 - err = binary.Read(r, f.ByteOrder, &namesize) - if err != nil { - if err == io.EOF { - break - } - return nil, fmt.Errorf("read namesize failed: %v", err) - } - err = binary.Read(r, f.ByteOrder, &descsize) - if err != nil { - return nil, fmt.Errorf("read descsize failed: %v", err) - } - err = binary.Read(r, f.ByteOrder, ¬eType) - if err != nil { - return nil, fmt.Errorf("read type failed: %v", err) - } - noteName, err := readAligned4(r, namesize) - if err != nil { - return nil, fmt.Errorf("read name failed: %v", err) - } - desc, err := readAligned4(r, descsize) - if err != nil { - return nil, fmt.Errorf("read desc failed: %v", err) - } - if name == string(noteName) && typ == noteType { - return desc, nil - } - } - } - return nil, nil -} - -var elfGoNote = []byte("Go\x00\x00") - -// The Go build ID is stored in a note described by an ELF PT_NOTE prog -// header. The caller has already opened filename, to get f, and read -// at least 4 kB out, in data. -func readELFGoBuildID(filename string, f *os.File, data []byte) (buildid string, err error) { - // Assume the note content is in the data, already read. - // Rewrite the ELF header to set shnum to 0, so that we can pass - // the data to elf.NewFile and it will decode the Prog list but not - // try to read the section headers and the string table from disk. - // That's a waste of I/O when all we care about is the Prog list - // and the one ELF note. - switch elf.Class(data[elf.EI_CLASS]) { - case elf.ELFCLASS32: - data[48] = 0 - data[49] = 0 - case elf.ELFCLASS64: - data[60] = 0 - data[61] = 0 - } - - const elfGoBuildIDTag = 4 - - ef, err := elf.NewFile(bytes.NewReader(data)) - if err != nil { - return "", &os.PathError{Path: filename, Op: "parse", Err: err} - } - for _, p := range ef.Progs { - if p.Type != elf.PT_NOTE || p.Filesz < 16 { - continue - } - - var note []byte - if p.Off+p.Filesz < uint64(len(data)) { - note = data[p.Off : p.Off+p.Filesz] - } else { - // For some linkers, such as the Solaris linker, - // the buildid may not be found in data (which - // likely contains the first 16kB of the file) - // or even the first few megabytes of the file - // due to differences in note segment placement; - // in that case, extract the note data manually. - _, err = f.Seek(int64(p.Off), io.SeekStart) - if err != nil { - return "", err - } - - note = make([]byte, p.Filesz) - _, err = io.ReadFull(f, note) - if err != nil { - return "", err - } - } - - filesz := p.Filesz - for filesz >= 16 { - nameSize := ef.ByteOrder.Uint32(note) - valSize := ef.ByteOrder.Uint32(note[4:]) - tag := ef.ByteOrder.Uint32(note[8:]) - name := note[12:16] - if nameSize == 4 && 16+valSize <= uint32(len(note)) && tag == elfGoBuildIDTag && bytes.Equal(name, elfGoNote) { - return string(note[16 : 16+valSize]), nil - } - - nameSize = (nameSize + 3) &^ 3 - valSize = (valSize + 3) &^ 3 - notesz := uint64(12 + nameSize + valSize) - if filesz <= notesz { - break - } - filesz -= notesz - note = note[notesz:] - } - } - - // No note. Treat as successful but build ID empty. - return "", nil -} - -// The Go build ID is stored at the beginning of the Mach-O __text segment. -// The caller has already opened filename, to get f, and read a few kB out, in data. -// Sadly, that's not guaranteed to hold the note, because there is an arbitrary amount -// of other junk placed in the file ahead of the main text. -func readMachoGoBuildID(filename string, f *os.File, data []byte) (buildid string, err error) { - // If the data we want has already been read, don't worry about Mach-O parsing. - // This is both an optimization and a hedge against the Mach-O parsing failing - // in the future due to, for example, the name of the __text section changing. - if b, err := readRawGoBuildID(filename, data); b != "" && err == nil { - return b, err - } - - mf, err := macho.NewFile(f) - if err != nil { - return "", &os.PathError{Path: filename, Op: "parse", Err: err} - } - - sect := mf.Section("__text") - if sect == nil { - // Every binary has a __text section. Something is wrong. - return "", &os.PathError{Path: filename, Op: "parse", Err: fmt.Errorf("cannot find __text section")} - } - - // It should be in the first few bytes, but read a lot just in case, - // especially given our past problems on OS X with the build ID moving. - // There shouldn't be much difference between reading 4kB and 32kB: - // the hard part is getting to the data, not transferring it. - n := sect.Size - if n > uint64(BuildIDReadSize) { - n = uint64(BuildIDReadSize) - } - buf := make([]byte, n) - if _, err := f.ReadAt(buf, int64(sect.Offset)); err != nil { - return "", err - } - - return readRawGoBuildID(filename, buf) -} diff --git a/libgo/go/cmd/go/note_test.go b/libgo/go/cmd/go/note_test.go index 7b5568f2b71..f33efdce226 100644 --- a/libgo/go/cmd/go/note_test.go +++ b/libgo/go/cmd/go/note_test.go @@ -7,10 +7,11 @@ package main_test import ( - main "cmd/go" "go/build" "runtime" "testing" + + "cmd/go/internal/buildid" ) func TestNoteReading(t *testing.T) { @@ -23,9 +24,9 @@ func TestNoteReading2K(t *testing.T) { } // Set BuildIDReadSize to 2kB to exercise Mach-O parsing more strictly. defer func(old int) { - main.BuildIDReadSize = old - }(main.BuildIDReadSize) - main.BuildIDReadSize = 2 * 1024 + buildid.BuildIDReadSize = old + }(buildid.BuildIDReadSize) + buildid.BuildIDReadSize = 2 * 1024 testNoteReading(t) } @@ -36,7 +37,7 @@ func testNoteReading(t *testing.T) { tg.tempFile("hello.go", `package main; func main() { print("hello, world\n") }`) const buildID = "TestNoteReading-Build-ID" tg.run("build", "-ldflags", "-buildid="+buildID, "-o", tg.path("hello.exe"), tg.path("hello.go")) - id, err := main.ReadBuildIDFromBinary(tg.path("hello.exe")) + id, err := buildid.ReadBuildIDFromBinary(tg.path("hello.exe")) if err != nil { t.Fatalf("reading build ID from hello binary: %v", err) } @@ -56,7 +57,7 @@ func testNoteReading(t *testing.T) { } tg.run("build", "-ldflags", "-buildid="+buildID+" -linkmode=external", "-o", tg.path("hello.exe"), tg.path("hello.go")) - id, err = main.ReadBuildIDFromBinary(tg.path("hello.exe")) + id, err = buildid.ReadBuildIDFromBinary(tg.path("hello.exe")) if err != nil { t.Fatalf("reading build ID from hello binary (linkmode=external): %v", err) } diff --git a/libgo/go/cmd/go/pkg.go b/libgo/go/cmd/go/pkg.go deleted file mode 100644 index f0dc95670f9..00000000000 --- a/libgo/go/cmd/go/pkg.go +++ /dev/null @@ -1,2050 +0,0 @@ -// Copyright 2011 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package main - -import ( - "bytes" - "crypto/sha1" - "errors" - "fmt" - "go/build" - "go/scanner" - "go/token" - "io" - "io/ioutil" - "os" - pathpkg "path" - "path/filepath" - "runtime" - "sort" - "strconv" - "strings" - "unicode" -) - -var ignoreImports bool // control whether we ignore imports in packages - -// A Package describes a single package found in a directory. -type Package struct { - // Note: These fields are part of the go command's public API. - // See list.go. It is okay to add fields, but not to change or - // remove existing ones. Keep in sync with list.go - Dir string `json:",omitempty"` // directory containing package sources - ImportPath string `json:",omitempty"` // import path of package in dir - ImportComment string `json:",omitempty"` // path in import comment on package statement - Name string `json:",omitempty"` // package name - Doc string `json:",omitempty"` // package documentation string - Target string `json:",omitempty"` // install path - Shlib string `json:",omitempty"` // the shared library that contains this package (only set when -linkshared) - Goroot bool `json:",omitempty"` // is this package found in the Go root? - Standard bool `json:",omitempty"` // is this package part of the standard Go library? - Stale bool `json:",omitempty"` // would 'go install' do anything for this package? - StaleReason string `json:",omitempty"` // why is Stale true? - Root string `json:",omitempty"` // Go root or Go path dir containing this package - ConflictDir string `json:",omitempty"` // Dir is hidden by this other directory - BinaryOnly bool `json:",omitempty"` // package cannot be recompiled - - // Source files - GoFiles []string `json:",omitempty"` // .go source files (excluding CgoFiles, TestGoFiles, XTestGoFiles) - CgoFiles []string `json:",omitempty"` // .go sources files that import "C" - IgnoredGoFiles []string `json:",omitempty"` // .go sources ignored due to build constraints - CFiles []string `json:",omitempty"` // .c source files - CXXFiles []string `json:",omitempty"` // .cc, .cpp and .cxx source files - MFiles []string `json:",omitempty"` // .m source files - HFiles []string `json:",omitempty"` // .h, .hh, .hpp and .hxx source files - FFiles []string `json:",omitempty"` // .f, .F, .for and .f90 Fortran source files - SFiles []string `json:",omitempty"` // .s source files - SwigFiles []string `json:",omitempty"` // .swig files - SwigCXXFiles []string `json:",omitempty"` // .swigcxx files - SysoFiles []string `json:",omitempty"` // .syso system object files added to package - - // Cgo directives - CgoCFLAGS []string `json:",omitempty"` // cgo: flags for C compiler - CgoCPPFLAGS []string `json:",omitempty"` // cgo: flags for C preprocessor - CgoCXXFLAGS []string `json:",omitempty"` // cgo: flags for C++ compiler - CgoFFLAGS []string `json:",omitempty"` // cgo: flags for Fortran compiler - CgoLDFLAGS []string `json:",omitempty"` // cgo: flags for linker - CgoPkgConfig []string `json:",omitempty"` // cgo: pkg-config names - - // Dependency information - Imports []string `json:",omitempty"` // import paths used by this package - Deps []string `json:",omitempty"` // all (recursively) imported dependencies - - // Error information - Incomplete bool `json:",omitempty"` // was there an error loading this package or dependencies? - Error *PackageError `json:",omitempty"` // error loading this package (not dependencies) - DepsErrors []*PackageError `json:",omitempty"` // errors loading dependencies - - // Test information - TestGoFiles []string `json:",omitempty"` // _test.go files in package - TestImports []string `json:",omitempty"` // imports from TestGoFiles - XTestGoFiles []string `json:",omitempty"` // _test.go files outside package - XTestImports []string `json:",omitempty"` // imports from XTestGoFiles - - // Unexported fields are not part of the public API. - build *build.Package - pkgdir string // overrides build.PkgDir - imports []*Package - deps []*Package - gofiles []string // GoFiles+CgoFiles+TestGoFiles+XTestGoFiles files, absolute paths - sfiles []string - allgofiles []string // gofiles + IgnoredGoFiles, absolute paths - target string // installed file for this package (may be executable) - fake bool // synthesized package - external bool // synthesized external test package - forceLibrary bool // this package is a library (even if named "main") - cmdline bool // defined by files listed on command line - local bool // imported via local path (./ or ../) - localPrefix string // interpret ./ and ../ imports relative to this prefix - exeName string // desired name for temporary executable - coverMode string // preprocess Go source files with the coverage tool in this mode - coverVars map[string]*CoverVar // variables created by coverage analysis - omitDWARF bool // tell linker not to write DWARF information - buildID string // expected build ID for generated package - gobinSubdir bool // install target would be subdir of GOBIN -} - -// vendored returns the vendor-resolved version of imports, -// which should be p.TestImports or p.XTestImports, NOT p.Imports. -// The imports in p.TestImports and p.XTestImports are not recursively -// loaded during the initial load of p, so they list the imports found in -// the source file, but most processing should be over the vendor-resolved -// import paths. We do this resolution lazily both to avoid file system work -// and because the eventual real load of the test imports (during 'go test') -// can produce better error messages if it starts with the original paths. -// The initial load of p loads all the non-test imports and rewrites -// the vendored paths, so nothing should ever call p.vendored(p.Imports). -func (p *Package) vendored(imports []string) []string { - if len(imports) > 0 && len(p.Imports) > 0 && &imports[0] == &p.Imports[0] { - panic("internal error: p.vendored(p.Imports) called") - } - seen := make(map[string]bool) - var all []string - for _, path := range imports { - path = vendoredImportPath(p, path) - if !seen[path] { - seen[path] = true - all = append(all, path) - } - } - sort.Strings(all) - return all -} - -// CoverVar holds the name of the generated coverage variables targeting the named file. -type CoverVar struct { - File string // local file name - Var string // name of count struct -} - -func (p *Package) copyBuild(pp *build.Package) { - p.build = pp - - if pp.PkgTargetRoot != "" && buildPkgdir != "" { - old := pp.PkgTargetRoot - pp.PkgRoot = buildPkgdir - pp.PkgTargetRoot = buildPkgdir - pp.PkgObj = filepath.Join(buildPkgdir, strings.TrimPrefix(pp.PkgObj, old)) - } - - p.Dir = pp.Dir - p.ImportPath = pp.ImportPath - p.ImportComment = pp.ImportComment - p.Name = pp.Name - p.Doc = pp.Doc - p.Root = pp.Root - p.ConflictDir = pp.ConflictDir - p.BinaryOnly = pp.BinaryOnly - - // TODO? Target - p.Goroot = pp.Goroot - if buildContext.Compiler == "gccgo" { - p.Standard = stdpkg[p.ImportPath] - } else { - p.Standard = p.Goroot && p.ImportPath != "" && isStandardImportPath(p.ImportPath) - } - p.GoFiles = pp.GoFiles - p.CgoFiles = pp.CgoFiles - p.IgnoredGoFiles = pp.IgnoredGoFiles - p.CFiles = pp.CFiles - p.CXXFiles = pp.CXXFiles - p.MFiles = pp.MFiles - p.HFiles = pp.HFiles - p.FFiles = pp.FFiles - p.SFiles = pp.SFiles - p.SwigFiles = pp.SwigFiles - p.SwigCXXFiles = pp.SwigCXXFiles - p.SysoFiles = pp.SysoFiles - p.CgoCFLAGS = pp.CgoCFLAGS - p.CgoCPPFLAGS = pp.CgoCPPFLAGS - p.CgoCXXFLAGS = pp.CgoCXXFLAGS - p.CgoLDFLAGS = pp.CgoLDFLAGS - p.CgoPkgConfig = pp.CgoPkgConfig - // We modify p.Imports in place, so make copy now. - p.Imports = make([]string, len(pp.Imports)) - copy(p.Imports, pp.Imports) - p.TestGoFiles = pp.TestGoFiles - p.TestImports = pp.TestImports - p.XTestGoFiles = pp.XTestGoFiles - p.XTestImports = pp.XTestImports - if ignoreImports { - p.Imports = nil - p.TestImports = nil - p.XTestImports = nil - } -} - -// isStandardImportPath reports whether $GOROOT/src/path should be considered -// part of the standard distribution. For historical reasons we allow people to add -// their own code to $GOROOT instead of using $GOPATH, but we assume that -// code will start with a domain name (dot in the first element). -func isStandardImportPath(path string) bool { - i := strings.Index(path, "/") - if i < 0 { - i = len(path) - } - elem := path[:i] - return !strings.Contains(elem, ".") -} - -// A PackageError describes an error loading information about a package. -type PackageError struct { - ImportStack []string // shortest path from package named on command line to this one - Pos string // position of error - Err string // the error itself - isImportCycle bool // the error is an import cycle - hard bool // whether the error is soft or hard; soft errors are ignored in some places -} - -func (p *PackageError) Error() string { - // Import cycles deserve special treatment. - if p.isImportCycle { - return fmt.Sprintf("%s\npackage %s\n", p.Err, strings.Join(p.ImportStack, "\n\timports ")) - } - if p.Pos != "" { - // Omit import stack. The full path to the file where the error - // is the most important thing. - return p.Pos + ": " + p.Err - } - if len(p.ImportStack) == 0 { - return p.Err - } - return "package " + strings.Join(p.ImportStack, "\n\timports ") + ": " + p.Err -} - -// An importStack is a stack of import paths. -type importStack []string - -func (s *importStack) push(p string) { - *s = append(*s, p) -} - -func (s *importStack) pop() { - *s = (*s)[0 : len(*s)-1] -} - -func (s *importStack) copy() []string { - return append([]string{}, *s...) -} - -// shorterThan reports whether sp is shorter than t. -// We use this to record the shortest import sequence -// that leads to a particular package. -func (sp *importStack) shorterThan(t []string) bool { - s := *sp - if len(s) != len(t) { - return len(s) < len(t) - } - // If they are the same length, settle ties using string ordering. - for i := range s { - if s[i] != t[i] { - return s[i] < t[i] - } - } - return false // they are equal -} - -// packageCache is a lookup cache for loadPackage, -// so that if we look up a package multiple times -// we return the same pointer each time. -var packageCache = map[string]*Package{} - -// reloadPackage is like loadPackage but makes sure -// not to use the package cache. -func reloadPackage(arg string, stk *importStack) *Package { - p := packageCache[arg] - if p != nil { - delete(packageCache, p.Dir) - delete(packageCache, p.ImportPath) - } - return loadPackage(arg, stk) -} - -// dirToImportPath returns the pseudo-import path we use for a package -// outside the Go path. It begins with _/ and then contains the full path -// to the directory. If the package lives in c:\home\gopher\my\pkg then -// the pseudo-import path is _/c_/home/gopher/my/pkg. -// Using a pseudo-import path like this makes the ./ imports no longer -// a special case, so that all the code to deal with ordinary imports works -// automatically. -func dirToImportPath(dir string) string { - return pathpkg.Join("_", strings.Map(makeImportValid, filepath.ToSlash(dir))) -} - -func makeImportValid(r rune) rune { - // Should match Go spec, compilers, and ../../go/parser/parser.go:/isValidImport. - const illegalChars = `!"#$%&'()*,:;<=>?[\]^{|}` + "`\uFFFD" - if !unicode.IsGraphic(r) || unicode.IsSpace(r) || strings.ContainsRune(illegalChars, r) { - return '_' - } - return r -} - -// Mode flags for loadImport and download (in get.go). -const ( - // useVendor means that loadImport should do vendor expansion - // (provided the vendoring experiment is enabled). - // That is, useVendor means that the import path came from - // a source file and has not been vendor-expanded yet. - // Every import path should be loaded initially with useVendor, - // and then the expanded version (with the /vendor/ in it) gets - // recorded as the canonical import path. At that point, future loads - // of that package must not pass useVendor, because - // disallowVendor will reject direct use of paths containing /vendor/. - useVendor = 1 << iota - - // getTestDeps is for download (part of "go get") and indicates - // that test dependencies should be fetched too. - getTestDeps -) - -// loadImport scans the directory named by path, which must be an import path, -// but possibly a local import path (an absolute file system path or one beginning -// with ./ or ../). A local relative path is interpreted relative to srcDir. -// It returns a *Package describing the package found in that directory. -func loadImport(path, srcDir string, parent *Package, stk *importStack, importPos []token.Position, mode int) *Package { - stk.push(path) - defer stk.pop() - - // Determine canonical identifier for this package. - // For a local import the identifier is the pseudo-import path - // we create from the full directory to the package. - // Otherwise it is the usual import path. - // For vendored imports, it is the expanded form. - importPath := path - origPath := path - isLocal := build.IsLocalImport(path) - if isLocal { - importPath = dirToImportPath(filepath.Join(srcDir, path)) - } else if mode&useVendor != 0 { - // We do our own vendor resolution, because we want to - // find out the key to use in packageCache without the - // overhead of repeated calls to buildContext.Import. - // The code is also needed in a few other places anyway. - path = vendoredImportPath(parent, path) - importPath = path - } - - p := packageCache[importPath] - if p != nil { - p = reusePackage(p, stk) - } else { - p = new(Package) - p.local = isLocal - p.ImportPath = importPath - packageCache[importPath] = p - - // Load package. - // Import always returns bp != nil, even if an error occurs, - // in order to return partial information. - // - // TODO: After Go 1, decide when to pass build.AllowBinary here. - // See issue 3268 for mistakes to avoid. - buildMode := build.ImportComment - if mode&useVendor == 0 || path != origPath { - // Not vendoring, or we already found the vendored path. - buildMode |= build.IgnoreVendor - } - bp, err := buildContext.Import(path, srcDir, buildMode) - bp.ImportPath = importPath - if gobin != "" { - bp.BinDir = gobin - } - if err == nil && !isLocal && bp.ImportComment != "" && bp.ImportComment != path && - !strings.Contains(path, "/vendor/") && !strings.HasPrefix(path, "vendor/") { - err = fmt.Errorf("code in directory %s expects import %q", bp.Dir, bp.ImportComment) - } - p.load(stk, bp, err) - if p.Error != nil && p.Error.Pos == "" { - p = setErrorPos(p, importPos) - } - - if origPath != cleanImport(origPath) { - p.Error = &PackageError{ - ImportStack: stk.copy(), - Err: fmt.Sprintf("non-canonical import path: %q should be %q", origPath, pathpkg.Clean(origPath)), - } - p.Incomplete = true - } - } - - // Checked on every import because the rules depend on the code doing the importing. - if perr := disallowInternal(srcDir, p, stk); perr != p { - return setErrorPos(perr, importPos) - } - if mode&useVendor != 0 { - if perr := disallowVendor(srcDir, origPath, p, stk); perr != p { - return setErrorPos(perr, importPos) - } - } - - if p.Name == "main" && parent != nil && parent.Dir != p.Dir { - perr := *p - perr.Error = &PackageError{ - ImportStack: stk.copy(), - Err: fmt.Sprintf("import %q is a program, not an importable package", path), - } - return setErrorPos(&perr, importPos) - } - - if p.local && parent != nil && !parent.local { - perr := *p - perr.Error = &PackageError{ - ImportStack: stk.copy(), - Err: fmt.Sprintf("local import %q in non-local package", path), - } - return setErrorPos(&perr, importPos) - } - - return p -} - -func setErrorPos(p *Package, importPos []token.Position) *Package { - if len(importPos) > 0 { - pos := importPos[0] - pos.Filename = shortPath(pos.Filename) - p.Error.Pos = pos.String() - } - return p -} - -func cleanImport(path string) string { - orig := path - path = pathpkg.Clean(path) - if strings.HasPrefix(orig, "./") && path != ".." && !strings.HasPrefix(path, "../") { - path = "./" + path - } - return path -} - -var isDirCache = map[string]bool{} - -func isDir(path string) bool { - result, ok := isDirCache[path] - if ok { - return result - } - - fi, err := os.Stat(path) - result = err == nil && fi.IsDir() - isDirCache[path] = result - return result -} - -// vendoredImportPath returns the expansion of path when it appears in parent. -// If parent is x/y/z, then path might expand to x/y/z/vendor/path, x/y/vendor/path, -// x/vendor/path, vendor/path, or else stay path if none of those exist. -// vendoredImportPath returns the expanded path or, if no expansion is found, the original. -func vendoredImportPath(parent *Package, path string) (found string) { - if parent == nil || parent.Root == "" { - return path - } - - dir := filepath.Clean(parent.Dir) - root := filepath.Join(parent.Root, "src") - if !hasFilePathPrefix(dir, root) || parent.ImportPath != "command-line-arguments" && filepath.Join(root, parent.ImportPath) != dir { - // Look for symlinks before reporting error. - dir = expandPath(dir) - root = expandPath(root) - } - - if !hasFilePathPrefix(dir, root) || len(dir) <= len(root) || dir[len(root)] != filepath.Separator || parent.ImportPath != "command-line-arguments" && !parent.local && filepath.Join(root, parent.ImportPath) != dir { - fatalf("unexpected directory layout:\n"+ - " import path: %s\n"+ - " root: %s\n"+ - " dir: %s\n"+ - " expand root: %s\n"+ - " expand dir: %s\n"+ - " separator: %s", - parent.ImportPath, - filepath.Join(parent.Root, "src"), - filepath.Clean(parent.Dir), - root, - dir, - string(filepath.Separator)) - } - - vpath := "vendor/" + path - for i := len(dir); i >= len(root); i-- { - if i < len(dir) && dir[i] != filepath.Separator { - continue - } - // Note: checking for the vendor directory before checking - // for the vendor/path directory helps us hit the - // isDir cache more often. It also helps us prepare a more useful - // list of places we looked, to report when an import is not found. - if !isDir(filepath.Join(dir[:i], "vendor")) { - continue - } - targ := filepath.Join(dir[:i], vpath) - if isDir(targ) && hasGoFiles(targ) { - importPath := parent.ImportPath - if importPath == "command-line-arguments" { - // If parent.ImportPath is 'command-line-arguments'. - // set to relative directory to root (also chopped root directory) - importPath = dir[len(root)+1:] - } - // We started with parent's dir c:\gopath\src\foo\bar\baz\quux\xyzzy. - // We know the import path for parent's dir. - // We chopped off some number of path elements and - // added vendor\path to produce c:\gopath\src\foo\bar\baz\vendor\path. - // Now we want to know the import path for that directory. - // Construct it by chopping the same number of path elements - // (actually the same number of bytes) from parent's import path - // and then append /vendor/path. - chopped := len(dir) - i - if chopped == len(importPath)+1 { - // We walked up from c:\gopath\src\foo\bar - // and found c:\gopath\src\vendor\path. - // We chopped \foo\bar (length 8) but the import path is "foo/bar" (length 7). - // Use "vendor/path" without any prefix. - return vpath - } - return importPath[:len(importPath)-chopped] + "/" + vpath - } - } - return path -} - -// hasGoFiles reports whether dir contains any files with names ending in .go. -// For a vendor check we must exclude directories that contain no .go files. -// Otherwise it is not possible to vendor just a/b/c and still import the -// non-vendored a/b. See golang.org/issue/13832. -func hasGoFiles(dir string) bool { - fis, _ := ioutil.ReadDir(dir) - for _, fi := range fis { - if !fi.IsDir() && strings.HasSuffix(fi.Name(), ".go") { - return true - } - } - return false -} - -// reusePackage reuses package p to satisfy the import at the top -// of the import stack stk. If this use causes an import loop, -// reusePackage updates p's error information to record the loop. -func reusePackage(p *Package, stk *importStack) *Package { - // We use p.imports==nil to detect a package that - // is in the midst of its own loadPackage call - // (all the recursion below happens before p.imports gets set). - if p.imports == nil { - if p.Error == nil { - p.Error = &PackageError{ - ImportStack: stk.copy(), - Err: "import cycle not allowed", - isImportCycle: true, - } - } - p.Incomplete = true - } - // Don't rewrite the import stack in the error if we have an import cycle. - // If we do, we'll lose the path that describes the cycle. - if p.Error != nil && !p.Error.isImportCycle && stk.shorterThan(p.Error.ImportStack) { - p.Error.ImportStack = stk.copy() - } - return p -} - -// disallowInternal checks that srcDir is allowed to import p. -// If the import is allowed, disallowInternal returns the original package p. -// If not, it returns a new package containing just an appropriate error. -func disallowInternal(srcDir string, p *Package, stk *importStack) *Package { - // golang.org/s/go14internal: - // An import of a path containing the element “internal” - // is disallowed if the importing code is outside the tree - // rooted at the parent of the “internal” directory. - - // There was an error loading the package; stop here. - if p.Error != nil { - return p - } - - // The generated 'testmain' package is allowed to access testing/internal/..., - // as if it were generated into the testing directory tree - // (it's actually in a temporary directory outside any Go tree). - // This cleans up a former kludge in passing functionality to the testing package. - if strings.HasPrefix(p.ImportPath, "testing/internal") && len(*stk) >= 2 && (*stk)[len(*stk)-2] == "testmain" { - return p - } - - // We can't check standard packages with gccgo. - if buildContext.Compiler == "gccgo" && p.Standard { - return p - } - - // The stack includes p.ImportPath. - // If that's the only thing on the stack, we started - // with a name given on the command line, not an - // import. Anything listed on the command line is fine. - if len(*stk) == 1 { - return p - } - - // Check for "internal" element: three cases depending on begin of string and/or end of string. - i, ok := findInternal(p.ImportPath) - if !ok { - return p - } - - // Internal is present. - // Map import path back to directory corresponding to parent of internal. - if i > 0 { - i-- // rewind over slash in ".../internal" - } - parent := p.Dir[:i+len(p.Dir)-len(p.ImportPath)] - if hasFilePathPrefix(filepath.Clean(srcDir), filepath.Clean(parent)) { - return p - } - - // Look for symlinks before reporting error. - srcDir = expandPath(srcDir) - parent = expandPath(parent) - if hasFilePathPrefix(filepath.Clean(srcDir), filepath.Clean(parent)) { - return p - } - - // Internal is present, and srcDir is outside parent's tree. Not allowed. - perr := *p - perr.Error = &PackageError{ - ImportStack: stk.copy(), - Err: "use of internal package not allowed", - } - perr.Incomplete = true - return &perr -} - -// findInternal looks for the final "internal" path element in the given import path. -// If there isn't one, findInternal returns ok=false. -// Otherwise, findInternal returns ok=true and the index of the "internal". -func findInternal(path string) (index int, ok bool) { - // Three cases, depending on internal at start/end of string or not. - // The order matters: we must return the index of the final element, - // because the final one produces the most restrictive requirement - // on the importer. - switch { - case strings.HasSuffix(path, "/internal"): - return len(path) - len("internal"), true - case strings.Contains(path, "/internal/"): - return strings.LastIndex(path, "/internal/") + 1, true - case path == "internal", strings.HasPrefix(path, "internal/"): - return 0, true - } - return 0, false -} - -// disallowVendor checks that srcDir is allowed to import p as path. -// If the import is allowed, disallowVendor returns the original package p. -// If not, it returns a new package containing just an appropriate error. -func disallowVendor(srcDir, path string, p *Package, stk *importStack) *Package { - // The stack includes p.ImportPath. - // If that's the only thing on the stack, we started - // with a name given on the command line, not an - // import. Anything listed on the command line is fine. - if len(*stk) == 1 { - return p - } - - if perr := disallowVendorVisibility(srcDir, p, stk); perr != p { - return perr - } - - // Paths like x/vendor/y must be imported as y, never as x/vendor/y. - if i, ok := findVendor(path); ok { - perr := *p - perr.Error = &PackageError{ - ImportStack: stk.copy(), - Err: "must be imported as " + path[i+len("vendor/"):], - } - perr.Incomplete = true - return &perr - } - - return p -} - -// disallowVendorVisibility checks that srcDir is allowed to import p. -// The rules are the same as for /internal/ except that a path ending in /vendor -// is not subject to the rules, only subdirectories of vendor. -// This allows people to have packages and commands named vendor, -// for maximal compatibility with existing source trees. -func disallowVendorVisibility(srcDir string, p *Package, stk *importStack) *Package { - // The stack includes p.ImportPath. - // If that's the only thing on the stack, we started - // with a name given on the command line, not an - // import. Anything listed on the command line is fine. - if len(*stk) == 1 { - return p - } - - // Check for "vendor" element. - i, ok := findVendor(p.ImportPath) - if !ok { - return p - } - - // Vendor is present. - // Map import path back to directory corresponding to parent of vendor. - if i > 0 { - i-- // rewind over slash in ".../vendor" - } - truncateTo := i + len(p.Dir) - len(p.ImportPath) - if truncateTo < 0 || len(p.Dir) < truncateTo { - return p - } - parent := p.Dir[:truncateTo] - if hasFilePathPrefix(filepath.Clean(srcDir), filepath.Clean(parent)) { - return p - } - - // Look for symlinks before reporting error. - srcDir = expandPath(srcDir) - parent = expandPath(parent) - if hasFilePathPrefix(filepath.Clean(srcDir), filepath.Clean(parent)) { - return p - } - - // Vendor is present, and srcDir is outside parent's tree. Not allowed. - perr := *p - perr.Error = &PackageError{ - ImportStack: stk.copy(), - Err: "use of vendored package not allowed", - } - perr.Incomplete = true - return &perr -} - -// findVendor looks for the last non-terminating "vendor" path element in the given import path. -// If there isn't one, findVendor returns ok=false. -// Otherwise, findVendor returns ok=true and the index of the "vendor". -// -// Note that terminating "vendor" elements don't count: "x/vendor" is its own package, -// not the vendored copy of an import "" (the empty import path). -// This will allow people to have packages or commands named vendor. -// This may help reduce breakage, or it may just be confusing. We'll see. -func findVendor(path string) (index int, ok bool) { - // Two cases, depending on internal at start of string or not. - // The order matters: we must return the index of the final element, - // because the final one is where the effective import path starts. - switch { - case strings.Contains(path, "/vendor/"): - return strings.LastIndex(path, "/vendor/") + 1, true - case strings.HasPrefix(path, "vendor/"): - return 0, true - } - return 0, false -} - -type targetDir int - -const ( - toRoot targetDir = iota // to bin dir inside package root (default) - toTool // GOROOT/pkg/tool - stalePath // the old import path; fail to build -) - -// goTools is a map of Go program import path to install target directory. -var goTools = map[string]targetDir{ - "cmd/addr2line": toTool, - "cmd/api": toTool, - "cmd/asm": toTool, - "cmd/compile": toTool, - "cmd/cgo": toTool, - "cmd/cover": toTool, - "cmd/dist": toTool, - "cmd/doc": toTool, - "cmd/fix": toTool, - "cmd/link": toTool, - "cmd/newlink": toTool, - "cmd/nm": toTool, - "cmd/objdump": toTool, - "cmd/pack": toTool, - "cmd/pprof": toTool, - "cmd/trace": toTool, - "cmd/vet": toTool, - "code.google.com/p/go.tools/cmd/cover": stalePath, - "code.google.com/p/go.tools/cmd/godoc": stalePath, - "code.google.com/p/go.tools/cmd/vet": stalePath, -} - -// expandScanner expands a scanner.List error into all the errors in the list. -// The default Error method only shows the first error. -func expandScanner(err error) error { - // Look for parser errors. - if err, ok := err.(scanner.ErrorList); ok { - // Prepare error with \n before each message. - // When printed in something like context: %v - // this will put the leading file positions each on - // its own line. It will also show all the errors - // instead of just the first, as err.Error does. - var buf bytes.Buffer - for _, e := range err { - e.Pos.Filename = shortPath(e.Pos.Filename) - buf.WriteString("\n") - buf.WriteString(e.Error()) - } - return errors.New(buf.String()) - } - return err -} - -var raceExclude = map[string]bool{ - "runtime/race": true, - "runtime/msan": true, - "runtime/cgo": true, - "cmd/cgo": true, - "syscall": true, - "errors": true, -} - -var cgoExclude = map[string]bool{ - "runtime/cgo": true, -} - -var cgoSyscallExclude = map[string]bool{ - "runtime/cgo": true, - "runtime/race": true, - "runtime/msan": true, -} - -// load populates p using information from bp, err, which should -// be the result of calling build.Context.Import. -func (p *Package) load(stk *importStack, bp *build.Package, err error) *Package { - p.copyBuild(bp) - - // When using gccgo the go/build package will not be able to - // find a standard package. It would be nicer to not get that - // error, but go/build doesn't know stdpkg. - if runtime.Compiler == "gccgo" && err != nil && p.Standard { - err = nil - } - - // The localPrefix is the path we interpret ./ imports relative to. - // Synthesized main packages sometimes override this. - p.localPrefix = dirToImportPath(p.Dir) - - if err != nil { - p.Incomplete = true - err = expandScanner(err) - p.Error = &PackageError{ - ImportStack: stk.copy(), - Err: err.Error(), - } - return p - } - - useBindir := p.Name == "main" - if !p.Standard { - switch buildBuildmode { - case "c-archive", "c-shared", "plugin": - useBindir = false - } - } - - if useBindir { - // Report an error when the old code.google.com/p/go.tools paths are used. - if goTools[p.ImportPath] == stalePath { - newPath := strings.Replace(p.ImportPath, "code.google.com/p/go.", "golang.org/x/", 1) - e := fmt.Sprintf("the %v command has moved; use %v instead.", p.ImportPath, newPath) - p.Error = &PackageError{Err: e} - return p - } - _, elem := filepath.Split(p.Dir) - full := buildContext.GOOS + "_" + buildContext.GOARCH + "/" + elem - if buildContext.GOOS != toolGOOS || buildContext.GOARCH != toolGOARCH { - // Install cross-compiled binaries to subdirectories of bin. - elem = full - } - if p.build.BinDir != "" { - // Install to GOBIN or bin of GOPATH entry. - p.target = filepath.Join(p.build.BinDir, elem) - if !p.Goroot && strings.Contains(elem, "/") && gobin != "" { - // Do not create $GOBIN/goos_goarch/elem. - p.target = "" - p.gobinSubdir = true - } - } - if goTools[p.ImportPath] == toTool { - // This is for 'go tool'. - // Override all the usual logic and force it into the tool directory. - if buildContext.Compiler == "gccgo" { - p.target = filepath.Join(runtime.GCCGOTOOLDIR, elem) - } else { - p.target = filepath.Join(gorootPkg, "tool", full) - } - } - if p.target != "" && buildContext.GOOS == "windows" { - p.target += ".exe" - } - } else if p.local { - // Local import turned into absolute path. - // No permanent install target. - p.target = "" - } else { - p.target = p.build.PkgObj - if buildLinkshared { - shlibnamefile := p.target[:len(p.target)-2] + ".shlibname" - shlib, err := ioutil.ReadFile(shlibnamefile) - if err == nil { - libname := strings.TrimSpace(string(shlib)) - if buildContext.Compiler == "gccgo" { - p.Shlib = filepath.Join(p.build.PkgTargetRoot, "shlibs", libname) - } else { - p.Shlib = filepath.Join(p.build.PkgTargetRoot, libname) - - } - } else if !os.IsNotExist(err) { - fatalf("unexpected error reading %s: %v", shlibnamefile, err) - } - } - } - - importPaths := p.Imports - // Packages that use cgo import runtime/cgo implicitly. - // Packages that use cgo also import syscall implicitly, - // to wrap errno. - // Exclude certain packages to avoid circular dependencies. - if len(p.CgoFiles) > 0 && (!p.Standard || !cgoExclude[p.ImportPath]) { - importPaths = append(importPaths, "runtime/cgo") - } - if len(p.CgoFiles) > 0 && (!p.Standard || !cgoSyscallExclude[p.ImportPath]) { - importPaths = append(importPaths, "syscall") - } - - if buildContext.CgoEnabled && p.Name == "main" && !p.Goroot { - // Currently build modes c-shared, pie (on systems that do not - // support PIE with internal linking mode), plugin, and - // -linkshared force external linking mode, as of course does - // -ldflags=-linkmode=external. External linking mode forces - // an import of runtime/cgo. - pieCgo := buildBuildmode == "pie" && (buildContext.GOOS != "linux" || buildContext.GOARCH != "amd64") - linkmodeExternal := false - for i, a := range buildLdflags { - if a == "-linkmode=external" { - linkmodeExternal = true - } - if a == "-linkmode" && i+1 < len(buildLdflags) && buildLdflags[i+1] == "external" { - linkmodeExternal = true - } - } - if buildBuildmode == "c-shared" || buildBuildmode == "plugin" || pieCgo || buildLinkshared || linkmodeExternal { - importPaths = append(importPaths, "runtime/cgo") - } - } - - // Everything depends on runtime, except runtime, its internal - // subpackages, and unsafe. - if !p.Standard || (p.ImportPath != "runtime" && !strings.HasPrefix(p.ImportPath, "runtime/internal/") && p.ImportPath != "unsafe") { - importPaths = append(importPaths, "runtime") - // When race detection enabled everything depends on runtime/race. - // Exclude certain packages to avoid circular dependencies. - if buildRace && (!p.Standard || !raceExclude[p.ImportPath]) { - importPaths = append(importPaths, "runtime/race") - } - // MSan uses runtime/msan. - if buildMSan && (!p.Standard || !raceExclude[p.ImportPath]) { - importPaths = append(importPaths, "runtime/msan") - } - // On ARM with GOARM=5, everything depends on math for the link. - if p.Name == "main" && goarch == "arm" { - importPaths = append(importPaths, "math") - } - } - - // Runtime and its internal packages depend on runtime/internal/sys, - // so that they pick up the generated zversion.go file. - // This can be an issue particularly for runtime/internal/atomic; - // see issue 13655. - if p.Standard && (p.ImportPath == "runtime" || strings.HasPrefix(p.ImportPath, "runtime/internal/")) && p.ImportPath != "runtime/internal/sys" { - importPaths = append(importPaths, "runtime/internal/sys") - } - - // Build list of full paths to all Go files in the package, - // for use by commands like go fmt. - p.gofiles = stringList(p.GoFiles, p.CgoFiles, p.TestGoFiles, p.XTestGoFiles) - for i := range p.gofiles { - p.gofiles[i] = filepath.Join(p.Dir, p.gofiles[i]) - } - sort.Strings(p.gofiles) - - p.sfiles = stringList(p.SFiles) - for i := range p.sfiles { - p.sfiles[i] = filepath.Join(p.Dir, p.sfiles[i]) - } - sort.Strings(p.sfiles) - - p.allgofiles = stringList(p.IgnoredGoFiles) - for i := range p.allgofiles { - p.allgofiles[i] = filepath.Join(p.Dir, p.allgofiles[i]) - } - p.allgofiles = append(p.allgofiles, p.gofiles...) - sort.Strings(p.allgofiles) - - // Check for case-insensitive collision of input files. - // To avoid problems on case-insensitive files, we reject any package - // where two different input files have equal names under a case-insensitive - // comparison. - f1, f2 := foldDup(stringList( - p.GoFiles, - p.CgoFiles, - p.IgnoredGoFiles, - p.CFiles, - p.CXXFiles, - p.MFiles, - p.HFiles, - p.FFiles, - p.SFiles, - p.SysoFiles, - p.SwigFiles, - p.SwigCXXFiles, - p.TestGoFiles, - p.XTestGoFiles, - )) - if f1 != "" { - p.Error = &PackageError{ - ImportStack: stk.copy(), - Err: fmt.Sprintf("case-insensitive file name collision: %q and %q", f1, f2), - } - return p - } - - // Build list of imported packages and full dependency list. - imports := make([]*Package, 0, len(p.Imports)) - deps := make(map[string]*Package) - save := func(path string, p1 *Package) { - // The same import path could produce an error or not, - // depending on what tries to import it. - // Prefer to record entries with errors, so we can report them. - p0 := deps[path] - if p0 == nil || p1.Error != nil && (p0.Error == nil || len(p0.Error.ImportStack) > len(p1.Error.ImportStack)) { - deps[path] = p1 - } - } - - for i, path := range importPaths { - if path == "C" { - continue - } - p1 := loadImport(path, p.Dir, p, stk, p.build.ImportPos[path], useVendor) - if !reqStdPkgSrc && p1.Standard { - continue - } - if p.Standard && p.Error == nil && !p1.Standard && p1.Error == nil { - p.Error = &PackageError{ - ImportStack: stk.copy(), - Err: fmt.Sprintf("non-standard import %q in standard package %q", path, p.ImportPath), - } - pos := p.build.ImportPos[path] - if len(pos) > 0 { - p.Error.Pos = pos[0].String() - } - } - - path = p1.ImportPath - importPaths[i] = path - if i < len(p.Imports) { - p.Imports[i] = path - } - - save(path, p1) - imports = append(imports, p1) - for _, dep := range p1.deps { - save(dep.ImportPath, dep) - } - if p1.Incomplete { - p.Incomplete = true - } - } - p.imports = imports - - p.Deps = make([]string, 0, len(deps)) - for dep := range deps { - p.Deps = append(p.Deps, dep) - } - sort.Strings(p.Deps) - for _, dep := range p.Deps { - p1 := deps[dep] - if p1 == nil { - panic("impossible: missing entry in package cache for " + dep + " imported by " + p.ImportPath) - } - p.deps = append(p.deps, p1) - if p1.Error != nil { - p.DepsErrors = append(p.DepsErrors, p1.Error) - } - } - - // unsafe is a fake package. - if p.Standard && (p.ImportPath == "unsafe" || buildContext.Compiler == "gccgo") { - p.target = "" - } - p.Target = p.target - - // If cgo is not enabled, ignore cgo supporting sources - // just as we ignore go files containing import "C". - if !buildContext.CgoEnabled { - p.CFiles = nil - p.CXXFiles = nil - p.MFiles = nil - p.SwigFiles = nil - p.SwigCXXFiles = nil - // Note that SFiles are okay (they go to the Go assembler) - // and HFiles are okay (they might be used by the SFiles). - // Also Sysofiles are okay (they might not contain object - // code; see issue #16050). - } - - // The gc toolchain only permits C source files with cgo. - if len(p.CFiles) > 0 && !p.usesCgo() && !p.usesSwig() && buildContext.Compiler == "gc" { - p.Error = &PackageError{ - ImportStack: stk.copy(), - Err: fmt.Sprintf("C source files not allowed when not using cgo or SWIG: %s", strings.Join(p.CFiles, " ")), - } - return p - } - - // In the absence of errors lower in the dependency tree, - // check for case-insensitive collisions of import paths. - if len(p.DepsErrors) == 0 { - dep1, dep2 := foldDup(p.Deps) - if dep1 != "" { - p.Error = &PackageError{ - ImportStack: stk.copy(), - Err: fmt.Sprintf("case-insensitive import collision: %q and %q", dep1, dep2), - } - return p - } - } - - if p.BinaryOnly { - // For binary-only package, use build ID from supplied package binary. - buildID, err := readBuildID(p) - if err == nil { - p.buildID = buildID - } - } else { - computeBuildID(p) - } - return p -} - -// usesSwig reports whether the package needs to run SWIG. -func (p *Package) usesSwig() bool { - return len(p.SwigFiles) > 0 || len(p.SwigCXXFiles) > 0 -} - -// usesCgo reports whether the package needs to run cgo -func (p *Package) usesCgo() bool { - return len(p.CgoFiles) > 0 -} - -// packageList returns the list of packages in the dag rooted at roots -// as visited in a depth-first post-order traversal. -func packageList(roots []*Package) []*Package { - seen := map[*Package]bool{} - all := []*Package{} - var walk func(*Package) - walk = func(p *Package) { - if seen[p] { - return - } - seen[p] = true - for _, p1 := range p.imports { - walk(p1) - } - all = append(all, p) - } - for _, root := range roots { - walk(root) - } - return all -} - -// computeStale computes the Stale flag in the package dag that starts -// at the named pkgs (command-line arguments). -func computeStale(pkgs ...*Package) { - for _, p := range packageList(pkgs) { - p.Stale, p.StaleReason = isStale(p) - } -} - -// The runtime version string takes one of two forms: -// "go1.X[.Y]" for Go releases, and "devel +hash" at tip. -// Determine whether we are in a released copy by -// inspecting the version. -var isGoRelease = strings.HasPrefix(runtime.Version(), "go1") - -// isStale and computeBuildID -// -// Theory of Operation -// -// There is an installed copy of the package (or binary). -// Can we reuse the installed copy, or do we need to build a new one? -// -// We can use the installed copy if it matches what we'd get -// by building a new one. The hard part is predicting that without -// actually running a build. -// -// To start, we must know the set of inputs to the build process that can -// affect the generated output. At a minimum, that includes the source -// files for the package and also any compiled packages imported by those -// source files. The *Package has these, and we use them. One might also -// argue for including in the input set: the build tags, whether the race -// detector is in use, the target operating system and architecture, the -// compiler and linker binaries being used, the additional flags being -// passed to those, the cgo binary being used, the additional flags cgo -// passes to the host C compiler, the host C compiler being used, the set -// of host C include files and installed C libraries, and so on. -// We include some but not all of this information. -// -// Once we have decided on a set of inputs, we must next decide how to -// tell whether the content of that set has changed since the last build -// of p. If there have been no changes, then we assume a new build would -// produce the same result and reuse the installed package or binary. -// But if there have been changes, then we assume a new build might not -// produce the same result, so we rebuild. -// -// There are two common ways to decide whether the content of the set has -// changed: modification times and content hashes. We use a mixture of both. -// -// The use of modification times (mtimes) was pioneered by make: -// assuming that a file's mtime is an accurate record of when that file was last written, -// and assuming that the modification time of an installed package or -// binary is the time that it was built, if the mtimes of the inputs -// predate the mtime of the installed object, then the build of that -// object saw those versions of the files, and therefore a rebuild using -// those same versions would produce the same object. In contrast, if any -// mtime of an input is newer than the mtime of the installed object, a -// change has occurred since the build, and the build should be redone. -// -// Modification times are attractive because the logic is easy to -// understand and the file system maintains the mtimes automatically -// (less work for us). Unfortunately, there are a variety of ways in -// which the mtime approach fails to detect a change and reuses a stale -// object file incorrectly. (Making the opposite mistake, rebuilding -// unnecessarily, is only a performance problem and not a correctness -// problem, so we ignore that one.) -// -// As a warmup, one problem is that to be perfectly precise, we need to -// compare the input mtimes against the time at the beginning of the -// build, but the object file time is the time at the end of the build. -// If an input file changes after being read but before the object is -// written, the next build will see an object newer than the input and -// will incorrectly decide that the object is up to date. We make no -// attempt to detect or solve this problem. -// -// Another problem is that due to file system imprecision, an input and -// output that are actually ordered in time have the same mtime. -// This typically happens on file systems with 1-second (or, worse, -// 2-second) mtime granularity and with automated scripts that write an -// input and then immediately run a build, or vice versa. If an input and -// an output have the same mtime, the conservative behavior is to treat -// the output as out-of-date and rebuild. This can cause one or more -// spurious rebuilds, but only for 1 second, until the object finally has -// an mtime later than the input. -// -// Another problem is that binary distributions often set the mtime on -// all files to the same time. If the distribution includes both inputs -// and cached build outputs, the conservative solution to the previous -// problem will cause unnecessary rebuilds. Worse, in such a binary -// distribution, those rebuilds might not even have permission to update -// the cached build output. To avoid these write errors, if an input and -// output have the same mtime, we assume the output is up-to-date. -// This is the opposite of what the previous problem would have us do, -// but binary distributions are more common than instances of the -// previous problem. -// -// A variant of the last problem is that some binary distributions do not -// set the mtime on all files to the same time. Instead they let the file -// system record mtimes as the distribution is unpacked. If the outputs -// are unpacked before the inputs, they'll be older and a build will try -// to rebuild them. That rebuild might hit the same write errors as in -// the last scenario. We don't make any attempt to solve this, and we -// haven't had many reports of it. Perhaps the only time this happens is -// when people manually unpack the distribution, and most of the time -// that's done as the same user who will be using it, so an initial -// rebuild on first use succeeds quietly. -// -// More generally, people and programs change mtimes on files. The last -// few problems were specific examples of this, but it's a general problem. -// For example, instead of a binary distribution, copying a home -// directory from one directory or machine to another might copy files -// but not preserve mtimes. If the inputs are new than the outputs on the -// first machine but copied first, they end up older than the outputs on -// the second machine. -// -// Because many other build systems have the same sensitivity to mtimes, -// most programs manipulating source code take pains not to break the -// mtime assumptions. For example, Git does not set the mtime of files -// during a checkout operation, even when checking out an old version of -// the code. This decision was made specifically to work well with -// mtime-based build systems. -// -// The killer problem, though, for mtime-based build systems is that the -// build only has access to the mtimes of the inputs that still exist. -// If it is possible to remove an input without changing any other inputs, -// a later build will think the object is up-to-date when it is not. -// This happens for Go because a package is made up of all source -// files in a directory. If a source file is removed, there is no newer -// mtime available recording that fact. The mtime on the directory could -// be used, but it also changes when unrelated files are added to or -// removed from the directory, so including the directory mtime would -// cause unnecessary rebuilds, possibly many. It would also exacerbate -// the problems mentioned earlier, since even programs that are careful -// to maintain mtimes on files rarely maintain mtimes on directories. -// -// A variant of the last problem is when the inputs change for other -// reasons. For example, Go 1.4 and Go 1.5 both install $GOPATH/src/mypkg -// into the same target, $GOPATH/pkg/$GOOS_$GOARCH/mypkg.a. -// If Go 1.4 has built mypkg into mypkg.a, a build using Go 1.5 must -// rebuild mypkg.a, but from mtimes alone mypkg.a looks up-to-date. -// If Go 1.5 has just been installed, perhaps the compiler will have a -// newer mtime; since the compiler is considered an input, that would -// trigger a rebuild. But only once, and only the last Go 1.4 build of -// mypkg.a happened before Go 1.5 was installed. If a user has the two -// versions installed in different locations and flips back and forth, -// mtimes alone cannot tell what to do. Changing the toolchain is -// changing the set of inputs, without affecting any mtimes. -// -// To detect the set of inputs changing, we turn away from mtimes and to -// an explicit data comparison. Specifically, we build a list of the -// inputs to the build, compute its SHA1 hash, and record that as the -// ``build ID'' in the generated object. At the next build, we can -// recompute the build ID and compare it to the one in the generated -// object. If they differ, the list of inputs has changed, so the object -// is out of date and must be rebuilt. -// -// Because this build ID is computed before the build begins, the -// comparison does not have the race that mtime comparison does. -// -// Making the build sensitive to changes in other state is -// straightforward: include the state in the build ID hash, and if it -// changes, so does the build ID, triggering a rebuild. -// -// To detect changes in toolchain, we include the toolchain version in -// the build ID hash for package runtime, and then we include the build -// IDs of all imported packages in the build ID for p. -// -// It is natural to think about including build tags in the build ID, but -// the naive approach of just dumping the tags into the hash would cause -// spurious rebuilds. For example, 'go install' and 'go install -tags neverusedtag' -// produce the same binaries (assuming neverusedtag is never used). -// A more precise approach would be to include only tags that have an -// effect on the build. But the effect of a tag on the build is to -// include or exclude a file from the compilation, and that file list is -// already in the build ID hash. So the build ID is already tag-sensitive -// in a perfectly precise way. So we do NOT explicitly add build tags to -// the build ID hash. -// -// We do not include as part of the build ID the operating system, -// architecture, or whether the race detector is enabled, even though all -// three have an effect on the output, because that information is used -// to decide the install location. Binaries for linux and binaries for -// darwin are written to different directory trees; including that -// information in the build ID is unnecessary (although it would be -// harmless). -// -// TODO(rsc): Investigate the cost of putting source file content into -// the build ID hash as a replacement for the use of mtimes. Using the -// file content would avoid all the mtime problems, but it does require -// reading all the source files, something we avoid today (we read the -// beginning to find the build tags and the imports, but we stop as soon -// as we see the import block is over). If the package is stale, the compiler -// is going to read the files anyway. But if the package is up-to-date, the -// read is overhead. -// -// TODO(rsc): Investigate the complexity of making the build more -// precise about when individual results are needed. To be fully precise, -// there are two results of a compilation: the entire .a file used by the link -// and the subpiece used by later compilations (__.PKGDEF only). -// If a rebuild is needed but produces the previous __.PKGDEF, then -// no more recompilation due to the rebuilt package is needed, only -// relinking. To date, there is nothing in the Go command to express this. -// -// Special Cases -// -// When the go command makes the wrong build decision and does not -// rebuild something it should, users fall back to adding the -a flag. -// Any common use of the -a flag should be considered prima facie evidence -// that isStale is returning an incorrect false result in some important case. -// Bugs reported in the behavior of -a itself should prompt the question -// ``Why is -a being used at all? What bug does that indicate?'' -// -// There is a long history of changes to isStale to try to make -a into a -// suitable workaround for bugs in the mtime-based decisions. -// It is worth recording that history to inform (and, as much as possible, deter) future changes. -// -// (1) Before the build IDs were introduced, building with alternate tags -// would happily reuse installed objects built without those tags. -// For example, "go build -tags netgo myprog.go" would use the installed -// copy of package net, even if that copy had been built without netgo. -// (The netgo tag controls whether package net uses cgo or pure Go for -// functionality such as name resolution.) -// Using the installed non-netgo package defeats the purpose. -// -// Users worked around this with "go build -tags netgo -a myprog.go". -// -// Build IDs have made that workaround unnecessary: -// "go build -tags netgo myprog.go" -// cannot use a non-netgo copy of package net. -// -// (2) Before the build IDs were introduced, building with different toolchains, -// especially changing between toolchains, tried to reuse objects stored in -// $GOPATH/pkg, resulting in link-time errors about object file mismatches. -// -// Users worked around this with "go install -a ./...". -// -// Build IDs have made that workaround unnecessary: -// "go install ./..." will rebuild any objects it finds that were built against -// a different toolchain. -// -// (3) The common use of "go install -a ./..." led to reports of problems -// when the -a forced the rebuild of the standard library, which for some -// users was not writable. Because we didn't understand that the real -// problem was the bug -a was working around, we changed -a not to -// apply to the standard library. -// -// (4) The common use of "go build -tags netgo -a myprog.go" broke -// when we changed -a not to apply to the standard library, because -// if go build doesn't rebuild package net, it uses the non-netgo version. -// -// Users worked around this with "go build -tags netgo -installsuffix barf myprog.go". -// The -installsuffix here is making the go command look for packages -// in pkg/$GOOS_$GOARCH_barf instead of pkg/$GOOS_$GOARCH. -// Since the former presumably doesn't exist, go build decides to rebuild -// everything, including the standard library. Since go build doesn't -// install anything it builds, nothing is ever written to pkg/$GOOS_$GOARCH_barf, -// so repeated invocations continue to work. -// -// If the use of -a wasn't a red flag, the use of -installsuffix to point to -// a non-existent directory in a command that installs nothing should -// have been. -// -// (5) Now that (1) and (2) no longer need -a, we have removed the kludge -// introduced in (3): once again, -a means ``rebuild everything,'' not -// ``rebuild everything except the standard library.'' Only Go 1.4 had -// the restricted meaning. -// -// In addition to these cases trying to trigger rebuilds, there are -// special cases trying NOT to trigger rebuilds. The main one is that for -// a variety of reasons (see above), the install process for a Go release -// cannot be relied upon to set the mtimes such that the go command will -// think the standard library is up to date. So the mtime evidence is -// ignored for the standard library if we find ourselves in a release -// version of Go. Build ID-based staleness checks still apply to the -// standard library, even in release versions. This makes -// 'go build -tags netgo' work, among other things. - -// isStale reports whether package p needs to be rebuilt, -// along with the reason why. -func isStale(p *Package) (bool, string) { - if p.Standard && (p.ImportPath == "unsafe" || buildContext.Compiler == "gccgo") { - // fake, builtin package - return false, "builtin package" - } - if p.Error != nil { - return true, "errors loading package" - } - if p.Stale { - return true, p.StaleReason - } - - // If this is a package with no source code, it cannot be rebuilt. - // If the binary is missing, we mark the package stale so that - // if a rebuild is needed, that rebuild attempt will produce a useful error. - // (Some commands, such as 'go list', do not attempt to rebuild.) - if p.BinaryOnly { - if p.target == "" { - // Fail if a build is attempted. - return true, "no source code for package, but no install target" - } - if _, err := os.Stat(p.target); err != nil { - // Fail if a build is attempted. - return true, "no source code for package, but cannot access install target: " + err.Error() - } - return false, "no source code for package" - } - - // If the -a flag is given, rebuild everything. - if buildA { - return true, "build -a flag in use" - } - - // If there's no install target, we have to rebuild. - if p.target == "" { - return true, "no install target" - } - - // Package is stale if completely unbuilt. - fi, err := os.Stat(p.target) - if err != nil { - return true, "cannot stat install target" - } - - // Package is stale if the expected build ID differs from the - // recorded build ID. This catches changes like a source file - // being removed from a package directory. See issue 3895. - // It also catches changes in build tags that affect the set of - // files being compiled. See issue 9369. - // It also catches changes in toolchain, like when flipping between - // two versions of Go compiling a single GOPATH. - // See issue 8290 and issue 10702. - targetBuildID, err := readBuildID(p) - if err == nil && targetBuildID != p.buildID { - return true, "build ID mismatch" - } - - // Package is stale if a dependency is. - for _, p1 := range p.deps { - if p1.Stale { - return true, "stale dependency" - } - } - - // The checks above are content-based staleness. - // We assume they are always accurate. - // - // The checks below are mtime-based staleness. - // We hope they are accurate, but we know that they fail in the case of - // prebuilt Go installations that don't preserve the build mtimes - // (for example, if the pkg/ mtimes are before the src/ mtimes). - // See the large comment above isStale for details. - - // If we are running a release copy of Go and didn't find a content-based - // reason to rebuild the standard packages, do not rebuild them. - // They may not be writable anyway, but they are certainly not changing. - // This makes 'go build' skip the standard packages when - // using an official release, even when the mtimes have been changed. - // See issue 3036, issue 3149, issue 4106, issue 8290. - // (If a change to a release tree must be made by hand, the way to force the - // install is to run make.bash, which will remove the old package archives - // before rebuilding.) - if p.Standard && isGoRelease { - return false, "standard package in Go release distribution" - } - - // Time-based staleness. - - built := fi.ModTime() - - olderThan := func(file string) bool { - fi, err := os.Stat(file) - return err != nil || fi.ModTime().After(built) - } - - // Package is stale if a dependency is, or if a dependency is newer. - for _, p1 := range p.deps { - if p1.target != "" && olderThan(p1.target) { - return true, "newer dependency" - } - } - - // As a courtesy to developers installing new versions of the compiler - // frequently, define that packages are stale if they are - // older than the compiler, and commands if they are older than - // the linker. This heuristic will not work if the binaries are - // back-dated, as some binary distributions may do, but it does handle - // a very common case. - // See issue 3036. - // Exclude $GOROOT, under the assumption that people working on - // the compiler may want to control when everything gets rebuilt, - // and people updating the Go repository will run make.bash or all.bash - // and get a full rebuild anyway. - // Excluding $GOROOT used to also fix issue 4106, but that's now - // taken care of above (at least when the installed Go is a released version). - if p.Root != goroot { - if olderThan(buildToolchain.compiler()) { - return true, "newer compiler" - } - if p.build.IsCommand() && olderThan(buildToolchain.linker()) { - return true, "newer linker" - } - } - - // Note: Until Go 1.5, we had an additional shortcut here. - // We built a list of the workspace roots ($GOROOT, each $GOPATH) - // containing targets directly named on the command line, - // and if p were not in any of those, it would be treated as up-to-date - // as long as it is built. The goal was to avoid rebuilding a system-installed - // $GOROOT, unless something from $GOROOT were explicitly named - // on the command line (like go install math). - // That's now handled by the isGoRelease clause above. - // The other effect of the shortcut was to isolate different entries in - // $GOPATH from each other. This had the unfortunate effect that - // if you had (say), GOPATH listing two entries, one for commands - // and one for libraries, and you did a 'git pull' in the library one - // and then tried 'go install commands/...', it would build the new libraries - // during the first build (because they wouldn't have been installed at all) - // but then subsequent builds would not rebuild the libraries, even if the - // mtimes indicate they are stale, because the different GOPATH entries - // were treated differently. This behavior was confusing when using - // non-trivial GOPATHs, which were particularly common with some - // code management conventions, like the original godep. - // Since the $GOROOT case (the original motivation) is handled separately, - // we no longer put a barrier between the different $GOPATH entries. - // - // One implication of this is that if there is a system directory for - // non-standard Go packages that is included in $GOPATH, the mtimes - // on those compiled packages must be no earlier than the mtimes - // on the source files. Since most distributions use the same mtime - // for all files in a tree, they will be unaffected. People using plain - // tar x to extract system-installed packages will need to adjust mtimes, - // but it's better to force them to get the mtimes right than to ignore - // the mtimes and thereby do the wrong thing in common use cases. - // - // So there is no GOPATH vs GOPATH shortcut here anymore. - // - // If something needs to come back here, we could try writing a dummy - // file with a random name to the $GOPATH/pkg directory (and removing it) - // to test for write access, and then skip GOPATH roots we don't have write - // access to. But hopefully we can just use the mtimes always. - - srcs := stringList(p.GoFiles, p.CFiles, p.CXXFiles, p.MFiles, p.HFiles, p.FFiles, p.SFiles, p.CgoFiles, p.SysoFiles, p.SwigFiles, p.SwigCXXFiles) - for _, src := range srcs { - if olderThan(filepath.Join(p.Dir, src)) { - return true, "newer source file" - } - } - - return false, "" -} - -// computeBuildID computes the build ID for p, leaving it in p.buildID. -// Build ID is a hash of the information we want to detect changes in. -// See the long comment in isStale for details. -func computeBuildID(p *Package) { - h := sha1.New() - - // Include the list of files compiled as part of the package. - // This lets us detect removed files. See issue 3895. - inputFiles := stringList( - p.GoFiles, - p.CgoFiles, - p.CFiles, - p.CXXFiles, - p.MFiles, - p.HFiles, - p.SFiles, - p.SysoFiles, - p.SwigFiles, - p.SwigCXXFiles, - ) - for _, file := range inputFiles { - fmt.Fprintf(h, "file %s\n", file) - } - - // Include the content of runtime/internal/sys/zversion.go in the hash - // for package runtime. This will give package runtime a - // different build ID in each Go release. - if p.Standard && p.ImportPath == "runtime/internal/sys" && buildContext.Compiler != "gccgo" { - data, err := ioutil.ReadFile(filepath.Join(p.Dir, "zversion.go")) - if err != nil { - fatalf("go: %s", err) - } - fmt.Fprintf(h, "zversion %q\n", string(data)) - } - - // Include the build IDs of any dependencies in the hash. - // This, combined with the runtime/zversion content, - // will cause packages to have different build IDs when - // compiled with different Go releases. - // This helps the go command know to recompile when - // people use the same GOPATH but switch between - // different Go releases. See issue 10702. - // This is also a better fix for issue 8290. - for _, p1 := range p.deps { - fmt.Fprintf(h, "dep %s %s\n", p1.ImportPath, p1.buildID) - } - - p.buildID = fmt.Sprintf("%x", h.Sum(nil)) -} - -var cwd, _ = os.Getwd() - -var cmdCache = map[string]*Package{} - -// loadPackage is like loadImport but is used for command-line arguments, -// not for paths found in import statements. In addition to ordinary import paths, -// loadPackage accepts pseudo-paths beginning with cmd/ to denote commands -// in the Go command directory, as well as paths to those directories. -func loadPackage(arg string, stk *importStack) *Package { - if build.IsLocalImport(arg) { - dir := arg - if !filepath.IsAbs(dir) { - if abs, err := filepath.Abs(dir); err == nil { - // interpret relative to current directory - dir = abs - } - } - if sub, ok := hasSubdir(gorootSrc, dir); ok && strings.HasPrefix(sub, "cmd/") && !strings.Contains(sub[4:], "/") { - arg = sub - } - } - if strings.HasPrefix(arg, "cmd/") && !strings.Contains(arg[4:], "/") { - if p := cmdCache[arg]; p != nil { - return p - } - stk.push(arg) - defer stk.pop() - - bp, err := buildContext.ImportDir(filepath.Join(gorootSrc, arg), 0) - bp.ImportPath = arg - bp.Goroot = true - bp.BinDir = gorootBin - if gobin != "" { - bp.BinDir = gobin - } - bp.Root = goroot - bp.SrcRoot = gorootSrc - p := new(Package) - cmdCache[arg] = p - p.load(stk, bp, err) - if p.Error == nil && p.Name != "main" { - p.Error = &PackageError{ - ImportStack: stk.copy(), - Err: fmt.Sprintf("expected package main but found package %s in %s", p.Name, p.Dir), - } - } - return p - } - - // Wasn't a command; must be a package. - // If it is a local import path but names a standard package, - // we treat it as if the user specified the standard package. - // This lets you run go test ./ioutil in package io and be - // referring to io/ioutil rather than a hypothetical import of - // "./ioutil". - if build.IsLocalImport(arg) { - bp, _ := buildContext.ImportDir(filepath.Join(cwd, arg), build.FindOnly) - if bp.ImportPath != "" && bp.ImportPath != "." { - arg = bp.ImportPath - } - } - - return loadImport(arg, cwd, nil, stk, nil, 0) -} - -// packages returns the packages named by the -// command line arguments 'args'. If a named package -// cannot be loaded at all (for example, if the directory does not exist), -// then packages prints an error and does not include that -// package in the results. However, if errors occur trying -// to load dependencies of a named package, the named -// package is still returned, with p.Incomplete = true -// and details in p.DepsErrors. -func packages(args []string) []*Package { - var pkgs []*Package - for _, pkg := range packagesAndErrors(args) { - if pkg.Error != nil { - errorf("can't load package: %s", pkg.Error) - continue - } - pkgs = append(pkgs, pkg) - } - return pkgs -} - -// packagesAndErrors is like 'packages' but returns a -// *Package for every argument, even the ones that -// cannot be loaded at all. -// The packages that fail to load will have p.Error != nil. -func packagesAndErrors(args []string) []*Package { - if len(args) > 0 && strings.HasSuffix(args[0], ".go") { - return []*Package{goFilesPackage(args)} - } - - args = importPaths(args) - var ( - pkgs []*Package - stk importStack - seenArg = make(map[string]bool) - seenPkg = make(map[*Package]bool) - ) - - for _, arg := range args { - if seenArg[arg] { - continue - } - seenArg[arg] = true - pkg := loadPackage(arg, &stk) - if seenPkg[pkg] { - continue - } - seenPkg[pkg] = true - pkgs = append(pkgs, pkg) - } - computeStale(pkgs...) - - return pkgs -} - -// packagesForBuild is like 'packages' but fails if any of -// the packages or their dependencies have errors -// (cannot be built). -func packagesForBuild(args []string) []*Package { - pkgs := packagesAndErrors(args) - printed := map[*PackageError]bool{} - for _, pkg := range pkgs { - if pkg.Error != nil { - errorf("can't load package: %s", pkg.Error) - } - for _, err := range pkg.DepsErrors { - // Since these are errors in dependencies, - // the same error might show up multiple times, - // once in each package that depends on it. - // Only print each once. - if !printed[err] { - printed[err] = true - errorf("%s", err) - } - } - } - exitIfErrors() - - // Check for duplicate loads of the same package. - // That should be impossible, but if it does happen then - // we end up trying to build the same package twice, - // usually in parallel overwriting the same files, - // which doesn't work very well. - seen := map[string]bool{} - reported := map[string]bool{} - for _, pkg := range packageList(pkgs) { - if seen[pkg.ImportPath] && !reported[pkg.ImportPath] { - reported[pkg.ImportPath] = true - errorf("internal error: duplicate loads of %s", pkg.ImportPath) - } - seen[pkg.ImportPath] = true - } - exitIfErrors() - - return pkgs -} - -// hasSubdir reports whether dir is a subdirectory of -// (possibly multiple levels below) root. -// If so, it sets rel to the path fragment that must be -// appended to root to reach dir. -func hasSubdir(root, dir string) (rel string, ok bool) { - if p, err := filepath.EvalSymlinks(root); err == nil { - root = p - } - if p, err := filepath.EvalSymlinks(dir); err == nil { - dir = p - } - const sep = string(filepath.Separator) - root = filepath.Clean(root) - if !strings.HasSuffix(root, sep) { - root += sep - } - dir = filepath.Clean(dir) - if !strings.HasPrefix(dir, root) { - return "", false - } - return filepath.ToSlash(dir[len(root):]), true -} - -var ( - errBuildIDToolchain = fmt.Errorf("build ID only supported in gc toolchain") - errBuildIDMalformed = fmt.Errorf("malformed object file") - errBuildIDUnknown = fmt.Errorf("lost build ID") -) - -var ( - bangArch = []byte("!") - pkgdef = []byte("__.PKGDEF") - goobject = []byte("go object ") - buildid = []byte("build id ") -) - -// readBuildID reads the build ID from an archive or binary. -// It only supports the gc toolchain. -// Other toolchain maintainers should adjust this function. -func readBuildID(p *Package) (id string, err error) { - if buildToolchain != (gcToolchain{}) { - return "", errBuildIDToolchain - } - - // For commands, read build ID directly from binary. - if p.Name == "main" { - return ReadBuildIDFromBinary(p.Target) - } - - // Otherwise, we expect to have an archive (.a) file, - // and we can read the build ID from the Go export data. - if !strings.HasSuffix(p.Target, ".a") { - return "", &os.PathError{Op: "parse", Path: p.Target, Err: errBuildIDUnknown} - } - - // Read just enough of the target to fetch the build ID. - // The archive is expected to look like: - // - // ! - // __.PKGDEF 0 0 0 644 7955 ` - // go object darwin amd64 devel X:none - // build id "b41e5c45250e25c9fd5e9f9a1de7857ea0d41224" - // - // The variable-sized strings are GOOS, GOARCH, and the experiment list (X:none). - // Reading the first 1024 bytes should be plenty. - f, err := os.Open(p.Target) - if err != nil { - return "", err - } - data := make([]byte, 1024) - n, err := io.ReadFull(f, data) - f.Close() - - if err != nil && n == 0 { - return "", err - } - - bad := func() (string, error) { - return "", &os.PathError{Op: "parse", Path: p.Target, Err: errBuildIDMalformed} - } - - // Archive header. - for i := 0; ; i++ { // returns during i==3 - j := bytes.IndexByte(data, '\n') - if j < 0 { - return bad() - } - line := data[:j] - data = data[j+1:] - switch i { - case 0: - if !bytes.Equal(line, bangArch) { - return bad() - } - case 1: - if !bytes.HasPrefix(line, pkgdef) { - return bad() - } - case 2: - if !bytes.HasPrefix(line, goobject) { - return bad() - } - case 3: - if !bytes.HasPrefix(line, buildid) { - // Found the object header, just doesn't have a build id line. - // Treat as successful, with empty build id. - return "", nil - } - id, err := strconv.Unquote(string(line[len(buildid):])) - if err != nil { - return bad() - } - return id, nil - } - } -} - -var ( - goBuildPrefix = []byte("\xff Go build ID: \"") - goBuildEnd = []byte("\"\n \xff") - - elfPrefix = []byte("\x7fELF") - - machoPrefixes = [][]byte{ - {0xfe, 0xed, 0xfa, 0xce}, - {0xfe, 0xed, 0xfa, 0xcf}, - {0xce, 0xfa, 0xed, 0xfe}, - {0xcf, 0xfa, 0xed, 0xfe}, - } -) - -var BuildIDReadSize = 32 * 1024 // changed for testing - -// ReadBuildIDFromBinary reads the build ID from a binary. -// -// ELF binaries store the build ID in a proper PT_NOTE section. -// -// Other binary formats are not so flexible. For those, the linker -// stores the build ID as non-instruction bytes at the very beginning -// of the text segment, which should appear near the beginning -// of the file. This is clumsy but fairly portable. Custom locations -// can be added for other binary types as needed, like we did for ELF. -func ReadBuildIDFromBinary(filename string) (id string, err error) { - if filename == "" { - return "", &os.PathError{Op: "parse", Path: filename, Err: errBuildIDUnknown} - } - - // Read the first 32 kB of the binary file. - // That should be enough to find the build ID. - // In ELF files, the build ID is in the leading headers, - // which are typically less than 4 kB, not to mention 32 kB. - // In Mach-O files, there's no limit, so we have to parse the file. - // On other systems, we're trying to read enough that - // we get the beginning of the text segment in the read. - // The offset where the text segment begins in a hello - // world compiled for each different object format today: - // - // Plan 9: 0x20 - // Windows: 0x600 - // - f, err := os.Open(filename) - if err != nil { - return "", err - } - defer f.Close() - - data := make([]byte, BuildIDReadSize) - _, err = io.ReadFull(f, data) - if err == io.ErrUnexpectedEOF { - err = nil - } - if err != nil { - return "", err - } - - if bytes.HasPrefix(data, elfPrefix) { - return readELFGoBuildID(filename, f, data) - } - for _, m := range machoPrefixes { - if bytes.HasPrefix(data, m) { - return readMachoGoBuildID(filename, f, data) - } - } - - return readRawGoBuildID(filename, data) -} - -// readRawGoBuildID finds the raw build ID stored in text segment data. -func readRawGoBuildID(filename string, data []byte) (id string, err error) { - i := bytes.Index(data, goBuildPrefix) - if i < 0 { - // Missing. Treat as successful but build ID empty. - return "", nil - } - - j := bytes.Index(data[i+len(goBuildPrefix):], goBuildEnd) - if j < 0 { - return "", &os.PathError{Op: "parse", Path: filename, Err: errBuildIDMalformed} - } - - quoted := data[i+len(goBuildPrefix)-1 : i+len(goBuildPrefix)+j+1] - id, err = strconv.Unquote(string(quoted)) - if err != nil { - return "", &os.PathError{Op: "parse", Path: filename, Err: errBuildIDMalformed} - } - - return id, nil -} diff --git a/libgo/go/cmd/go/pkg_test.go b/libgo/go/cmd/go/pkg_test.go deleted file mode 100644 index fba13636cdd..00000000000 --- a/libgo/go/cmd/go/pkg_test.go +++ /dev/null @@ -1,194 +0,0 @@ -// Copyright 2014 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package main - -import ( - "io/ioutil" - "os" - "path/filepath" - "reflect" - "strings" - "testing" -) - -var foldDupTests = []struct { - list []string - f1, f2 string -}{ - {stringList("math/rand", "math/big"), "", ""}, - {stringList("math", "strings"), "", ""}, - {stringList("strings"), "", ""}, - {stringList("strings", "strings"), "strings", "strings"}, - {stringList("Rand", "rand", "math", "math/rand", "math/Rand"), "Rand", "rand"}, -} - -func TestFoldDup(t *testing.T) { - for _, tt := range foldDupTests { - f1, f2 := foldDup(tt.list) - if f1 != tt.f1 || f2 != tt.f2 { - t.Errorf("foldDup(%q) = %q, %q, want %q, %q", tt.list, f1, f2, tt.f1, tt.f2) - } - } -} - -var parseMetaGoImportsTests = []struct { - in string - out []metaImport -}{ - { - ``, - []metaImport{{"foo/bar", "git", "https://github.com/rsc/foo/bar"}}, - }, - { - ` - `, - []metaImport{ - {"foo/bar", "git", "https://github.com/rsc/foo/bar"}, - {"baz/quux", "git", "http://github.com/rsc/baz/quux"}, - }, - }, - { - ` - - `, - []metaImport{{"foo/bar", "git", "https://github.com/rsc/foo/bar"}}, - }, - { - ` - `, - []metaImport{{"foo/bar", "git", "https://github.com/rsc/foo/bar"}}, - }, - { - ``, - []metaImport{{"foo/bar", "git", "https://github.com/rsc/foo/bar"}}, - }, - { - // XML doesn't like
. - `Page Not Found
DRAFT
`, - []metaImport{{"chitin.io/chitin", "git", "https://github.com/chitin-io/chitin"}}, - }, -} - -func TestParseMetaGoImports(t *testing.T) { - for i, tt := range parseMetaGoImportsTests { - out, err := parseMetaGoImports(strings.NewReader(tt.in)) - if err != nil { - t.Errorf("test#%d: %v", i, err) - continue - } - if !reflect.DeepEqual(out, tt.out) { - t.Errorf("test#%d:\n\thave %q\n\twant %q", i, out, tt.out) - } - } -} - -func TestSharedLibName(t *testing.T) { - // TODO(avdva) - make these values platform-specific - prefix := "lib" - suffix := ".so" - testData := []struct { - args []string - pkgs []*Package - expected string - expectErr bool - rootedAt string - }{ - { - args: []string{"std"}, - pkgs: []*Package{}, - expected: "std", - }, - { - args: []string{"std", "cmd"}, - pkgs: []*Package{}, - expected: "std,cmd", - }, - { - args: []string{}, - pkgs: []*Package{&Package{ImportPath: "gopkg.in/somelib"}}, - expected: "gopkg.in-somelib", - }, - { - args: []string{"./..."}, - pkgs: []*Package{&Package{ImportPath: "somelib"}}, - expected: "somelib", - rootedAt: "somelib", - }, - { - args: []string{"../somelib", "../somelib"}, - pkgs: []*Package{&Package{ImportPath: "somelib"}}, - expected: "somelib", - }, - { - args: []string{"../lib1", "../lib2"}, - pkgs: []*Package{&Package{ImportPath: "gopkg.in/lib1"}, &Package{ImportPath: "gopkg.in/lib2"}}, - expected: "gopkg.in-lib1,gopkg.in-lib2", - }, - { - args: []string{"./..."}, - pkgs: []*Package{ - &Package{ImportPath: "gopkg.in/dir/lib1"}, - &Package{ImportPath: "gopkg.in/lib2"}, - &Package{ImportPath: "gopkg.in/lib3"}, - }, - expected: "gopkg.in", - rootedAt: "gopkg.in", - }, - { - args: []string{"std", "../lib2"}, - pkgs: []*Package{}, - expectErr: true, - }, - { - args: []string{"all", "./"}, - pkgs: []*Package{}, - expectErr: true, - }, - { - args: []string{"cmd", "fmt"}, - pkgs: []*Package{}, - expectErr: true, - }, - } - for _, data := range testData { - func() { - if data.rootedAt != "" { - tmpGopath, err := ioutil.TempDir("", "gopath") - if err != nil { - t.Fatal(err) - } - oldGopath := buildContext.GOPATH - defer func() { - buildContext.GOPATH = oldGopath - os.Chdir(cwd) - err := os.RemoveAll(tmpGopath) - if err != nil { - t.Error(err) - } - }() - root := filepath.Join(tmpGopath, "src", data.rootedAt) - err = os.MkdirAll(root, 0755) - if err != nil { - t.Fatal(err) - } - buildContext.GOPATH = tmpGopath - os.Chdir(root) - } - computed, err := libname(data.args, data.pkgs) - if err != nil { - if !data.expectErr { - t.Errorf("libname returned an error %q, expected a name", err.Error()) - } - } else if data.expectErr { - t.Errorf("libname returned %q, expected an error", computed) - } else { - expected := prefix + data.expected + suffix - if expected != computed { - t.Errorf("libname returned %q, expected %q", computed, expected) - } - } - }() - } -} diff --git a/libgo/go/cmd/go/run.go b/libgo/go/cmd/go/run.go deleted file mode 100644 index 18387b5eafa..00000000000 --- a/libgo/go/cmd/go/run.go +++ /dev/null @@ -1,156 +0,0 @@ -// Copyright 2011 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package main - -import ( - "fmt" - "os" - "os/exec" - "runtime" - "strings" -) - -var execCmd []string // -exec flag, for run and test - -func findExecCmd() []string { - if execCmd != nil { - return execCmd - } - execCmd = []string{} // avoid work the second time - if goos == runtime.GOOS && goarch == runtime.GOARCH { - return execCmd - } - path, err := exec.LookPath(fmt.Sprintf("go_%s_%s_exec", goos, goarch)) - if err == nil { - execCmd = []string{path} - } - return execCmd -} - -var cmdRun = &Command{ - UsageLine: "run [build flags] [-exec xprog] gofiles... [arguments...]", - Short: "compile and run Go program", - Long: ` -Run compiles and runs the main package comprising the named Go source files. -A Go source file is defined to be a file ending in a literal ".go" suffix. - -By default, 'go run' runs the compiled binary directly: 'a.out arguments...'. -If the -exec flag is given, 'go run' invokes the binary using xprog: - 'xprog a.out arguments...'. -If the -exec flag is not given, GOOS or GOARCH is different from the system -default, and a program named go_$GOOS_$GOARCH_exec can be found -on the current search path, 'go run' invokes the binary using that program, -for example 'go_nacl_386_exec a.out arguments...'. This allows execution of -cross-compiled programs when a simulator or other execution method is -available. - -For more about build flags, see 'go help build'. - -See also: go build. - `, -} - -func init() { - cmdRun.Run = runRun // break init loop - - addBuildFlags(cmdRun) - cmdRun.Flag.Var((*stringsFlag)(&execCmd), "exec", "") -} - -func printStderr(args ...interface{}) (int, error) { - return fmt.Fprint(os.Stderr, args...) -} - -func runRun(cmd *Command, args []string) { - instrumentInit() - buildModeInit() - var b builder - b.init() - b.print = printStderr - i := 0 - for i < len(args) && strings.HasSuffix(args[i], ".go") { - i++ - } - files, cmdArgs := args[:i], args[i:] - if len(files) == 0 { - fatalf("go run: no go files listed") - } - for _, file := range files { - if strings.HasSuffix(file, "_test.go") { - // goFilesPackage is going to assign this to TestGoFiles. - // Reject since it won't be part of the build. - fatalf("go run: cannot run *_test.go files (%s)", file) - } - } - p := goFilesPackage(files) - if p.Error != nil { - fatalf("%s", p.Error) - } - p.omitDWARF = true - if len(p.DepsErrors) > 0 { - // Since these are errors in dependencies, - // the same error might show up multiple times, - // once in each package that depends on it. - // Only print each once. - printed := map[*PackageError]bool{} - for _, err := range p.DepsErrors { - if !printed[err] { - printed[err] = true - errorf("%s", err) - } - } - } - exitIfErrors() - if p.Name != "main" { - fatalf("go run: cannot run non-main package") - } - p.target = "" // must build - not up to date - var src string - if len(p.GoFiles) > 0 { - src = p.GoFiles[0] - } else if len(p.CgoFiles) > 0 { - src = p.CgoFiles[0] - } else { - // this case could only happen if the provided source uses cgo - // while cgo is disabled. - hint := "" - if !buildContext.CgoEnabled { - hint = " (cgo is disabled)" - } - fatalf("go run: no suitable source files%s", hint) - } - p.exeName = src[:len(src)-len(".go")] // name temporary executable for first go file - a1 := b.action(modeBuild, modeBuild, p) - a := &action{f: (*builder).runProgram, args: cmdArgs, deps: []*action{a1}} - b.do(a) -} - -// runProgram is the action for running a binary that has already -// been compiled. We ignore exit status. -func (b *builder) runProgram(a *action) error { - cmdline := stringList(findExecCmd(), a.deps[0].target, a.args) - if buildN || buildX { - b.showcmd("", "%s", strings.Join(cmdline, " ")) - if buildN { - return nil - } - } - - runStdin(cmdline) - return nil -} - -// runStdin is like run, but connects Stdin. -func runStdin(cmdline []string) { - cmd := exec.Command(cmdline[0], cmdline[1:]...) - cmd.Stdin = os.Stdin - cmd.Stdout = os.Stdout - cmd.Stderr = os.Stderr - cmd.Env = origEnv - startSigHandlers() - if err := cmd.Run(); err != nil { - errorf("%v", err) - } -} diff --git a/libgo/go/cmd/go/script b/libgo/go/cmd/go/script deleted file mode 100644 index 340a7e824ca..00000000000 --- a/libgo/go/cmd/go/script +++ /dev/null @@ -1,23 +0,0 @@ -#!/bin/sh - -x() { - echo '--- ' "$@" - "$@" - echo '---' - echo -} - -x go help -x go help build -x go help clean -x go help install -x go help fix -x go help fmt -x go help get -x go help list -x go help test -x go help version -x go help vet -x go help gopath -x go help importpath -x go help remote diff --git a/libgo/go/cmd/go/script.txt b/libgo/go/cmd/go/script.txt deleted file mode 100644 index a672146584e..00000000000 --- a/libgo/go/cmd/go/script.txt +++ /dev/null @@ -1,352 +0,0 @@ ---- go help -usage: go command [arguments] - -go manages Go source code. - -The commands are: - - build compile and install packages and dependencies - clean remove intermediate objects - fix run gofix on packages - fmt run gofmt -w on packages - get download and install packages and dependencies - install install packages and dependencies - list list packages - test test packages - version print Go version - vet run govet on packages - -Use "go help [command]" for more information about a command. - -Additional help topics: - - gopath GOPATH environment variable - importpath description of import paths - remote remote import path syntax - -Use "go help [topic]" for more information about that topic. - ---- - ---- go help build -usage: go build [-n] [-v] [importpath...] - -Build compiles the packages named by the import paths, -along with their dependencies, but it does not install the results. - -The -n flag prints the commands but does not run them. -The -v flag prints the commands. - -For more about import paths, see 'go help importpath'. - -See also: go install, go get, go clean. ---- - ---- go help clean -usage: go clean [-nuke] [importpath...] - -Clean removes intermediate object files generated during -the compilation of the packages named by the import paths, -but by default it does not remove the installed package binaries. - -The -nuke flag causes clean to remove the installed package binaries too. - -TODO: Clean does not clean dependencies of the packages. - -For more about import paths, see 'go help importpath'. ---- - ---- go help install -usage: go install [-n] [-v] [importpath...] - -Install compiles and installs the packages named by the import paths, -along with their dependencies. - -The -n flag prints the commands but does not run them. -The -v flag prints the commands. - -For more about import paths, see 'go help importpath'. - -See also: go build, go get, go clean. ---- - ---- go help fix -usage: go fix [importpath...] - -Fix runs the gofix command on the packages named by the import paths. - -For more about gofix, see 'godoc gofix'. -For more about import paths, see 'go help importpath'. - -To run gofix with specific options, run gofix itself. - -See also: go fmt, go vet. ---- - ---- go help fmt -usage: go fmt [importpath...] - -Fmt runs the command 'gofmt -w' on the packages named by the import paths. - -For more about gofmt, see 'godoc gofmt'. -For more about import paths, see 'go help importpath'. - -To run gofmt with specific options, run gofmt itself. - -See also: go fix, go vet. ---- - ---- go help get -usage: go get [importpath...] - -Get downloads and installs the packages named by the import paths, -along with their dependencies. - -After downloading the code, 'go get' looks for a tag beginning -with "go." that corresponds to the local Go version. -For Go "release.r58" it looks for a tag named "go.r58". -For "weekly.2011-06-03" it looks for "go.weekly.2011-06-03". -If the specific "go.X" tag is not found, it uses the latest earlier -version it can find. Otherwise, it uses the default version for -the version control system: HEAD for git, tip for Mercurial, -and so on. - -TODO: Explain versions better. - -For more about import paths, see 'go help importpath'. - -For more about how 'go get' finds source code to -download, see 'go help remote'. - -See also: go build, go install, go clean. ---- - ---- go help list -usage: go list [-f format] [-json] [importpath...] - -List lists the packages named by the import paths. - -The default output shows the package name and file system location: - - books /home/you/src/google-api-go-client.googlecode.com/hg/books/v1 - oauth /home/you/src/goauth2.googlecode.com/hg/oauth - sqlite /home/you/src/gosqlite.googlecode.com/hg/sqlite - -The -f flag specifies an alternate format for the list, -using the syntax of package template. The default output -is equivalent to -f '{{.Name}} {{.Dir}}' The struct -being passed to the template is: - - type Package struct { - Name string // package name - Doc string // package documentation string - GoFiles []string // names of Go source files in package - ImportPath string // import path denoting package - Imports []string // import paths used by this package - Deps []string // all (recursively) imported dependencies - Dir string // directory containing package sources - Version string // version of installed package - } - -The -json flag causes the package data to be printed in JSON format. - -For more about import paths, see 'go help importpath'. ---- - ---- go help test -usage: go test [importpath...] - -Test runs gotest to test the packages named by the import paths. -It prints a summary of the test results in the format: - - test archive/tar - FAIL archive/zip - test compress/gzip - ... - -followed by gotest output for each failed package. - -For more about import paths, see 'go help importpath'. - -See also: go build, go compile, go vet. ---- - ---- go help version -usage: go version - -Version prints the Go version, as reported by runtime.Version. ---- - ---- go help vet -usage: go vet [importpath...] - -Vet runs the govet command on the packages named by the import paths. - -For more about govet, see 'godoc govet'. -For more about import paths, see 'go help importpath'. - -To run govet with specific options, run govet itself. - -See also: go fmt, go fix. ---- - ---- go help gopath -The GOPATH environment variable lists places to look for Go code. -On Unix, the value is a colon-separated string. -On Windows, the value is a semicolon-separated string. -On Plan 9, the value is a list. - -GOPATH must be set to build and install packages outside the -standard Go tree. - -Each directory listed in GOPATH must have a prescribed structure: - -The src/ directory holds source code. The path below 'src' -determines the import path or executable name. - -The pkg/ directory holds installed package objects. -As in the Go tree, each target operating system and -architecture pair has its own subdirectory of pkg -(pkg/GOOS_GOARCH). - -If DIR is a directory listed in the GOPATH, a package with -source in DIR/src/foo/bar can be imported as "foo/bar" and -has its compiled form installed to "DIR/pkg/GOOS_GOARCH/foo/bar.a". - -The bin/ directory holds compiled commands. -Each command is named for its source directory, but only -the final element, not the entire path. That is, the -command with source in DIR/src/foo/quux is installed into -DIR/bin/quux, not DIR/bin/foo/quux. The foo/ is stripped -so that you can add DIR/bin to your PATH to get at the -installed commands. - -Here's an example directory layout: - - GOPATH=/home/user/gocode - - /home/user/gocode/ - src/ - foo/ - bar/ (go code in package bar) - x.go - quux/ (go code in package main) - y.go - bin/ - quux (installed command) - pkg/ - linux_amd64/ - foo/ - bar.a (installed package object) - -Go searches each directory listed in GOPATH to find source code, -but new packages are always downloaded into the first directory -in the list. ---- - ---- go help importpath -Many commands apply to a set of packages named by import paths: - - go action [importpath...] - -An import path that is a rooted path or that begins with -a . or .. element is interpreted as a file system path and -denotes the package in that directory. - -Otherwise, the import path P denotes the package found in -the directory DIR/src/P for some DIR listed in the GOPATH -environment variable (see 'go help gopath'). - -If no import paths are given, the action applies to the -package in the current directory. - -The special import path "all" expands to all package directories -found in all the GOPATH trees. For example, 'go list all' -lists all the packages on the local system. - -An import path can also name a package to be downloaded from -a remote repository. Run 'go help remote' for details. - -Every package in a program must have a unique import path. -By convention, this is arranged by starting each path with a -unique prefix that belongs to you. For example, paths used -internally at Google all begin with 'google', and paths -denoting remote repositories begin with the path to the code, -such as 'project.googlecode.com/'. ---- - ---- go help remote -An import path (see 'go help importpath') denotes a package -stored in the local file system. Certain import paths also -describe how to obtain the source code for the package using -a revision control system. - -A few common code hosting sites have special syntax: - - BitBucket (Mercurial) - - import "bitbucket.org/user/project" - import "bitbucket.org/user/project/sub/directory" - - GitHub (Git) - - import "github.com/user/project" - import "github.com/user/project/sub/directory" - - Google Code Project Hosting (Git, Mercurial, Subversion) - - import "project.googlecode.com/git" - import "project.googlecode.com/git/sub/directory" - - import "project.googlecode.com/hg" - import "project.googlecode.com/hg/sub/directory" - - import "project.googlecode.com/svn/trunk" - import "project.googlecode.com/svn/trunk/sub/directory" - - Launchpad (Bazaar) - - import "launchpad.net/project" - import "launchpad.net/project/series" - import "launchpad.net/project/series/sub/directory" - - import "launchpad.net/~user/project/branch" - import "launchpad.net/~user/project/branch/sub/directory" - -For code hosted on other servers, an import path of the form - - repository.vcs/path - -specifies the given repository, with or without the .vcs suffix, -using the named version control system, and then the path inside -that repository. The supported version control systems are: - - Bazaar .bzr - Git .git - Mercurial .hg - Subversion .svn - -For example, - - import "example.org/user/foo.hg" - -denotes the root directory of the Mercurial repository at -example.org/user/foo or foo.hg, and - - import "example.org/repo.git/foo/bar" - -denotes the foo/bar directory of the Git repository at -example.com/repo or repo.git. - -When a version control system supports multiple protocols, -each is tried in turn when downloading. For example, a Git -download tries git://, then https://, then http://. - -New downloaded packages are written to the first directory -listed in the GOPATH environment variable (see 'go help gopath'). - -The go command attempts to download the version of the -package appropriate for the Go release being used. -Run 'go help install' for more. ---- - diff --git a/libgo/go/cmd/go/signal.go b/libgo/go/cmd/go/signal.go deleted file mode 100644 index e8ba0d36556..00000000000 --- a/libgo/go/cmd/go/signal.go +++ /dev/null @@ -1,31 +0,0 @@ -// Copyright 2012 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package main - -import ( - "os" - "os/signal" - "sync" -) - -// interrupted is closed, if go process is interrupted. -var interrupted = make(chan struct{}) - -// processSignals setups signal handler. -func processSignals() { - sig := make(chan os.Signal) - signal.Notify(sig, signalsToIgnore...) - go func() { - <-sig - close(interrupted) - }() -} - -var onceProcessSignals sync.Once - -// startSigHandlers start signal handlers. -func startSigHandlers() { - onceProcessSignals.Do(processSignals) -} diff --git a/libgo/go/cmd/go/signal_notunix.go b/libgo/go/cmd/go/signal_notunix.go deleted file mode 100644 index 29aa9d8c209..00000000000 --- a/libgo/go/cmd/go/signal_notunix.go +++ /dev/null @@ -1,17 +0,0 @@ -// Copyright 2012 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// +build plan9 windows - -package main - -import ( - "os" -) - -var signalsToIgnore = []os.Signal{os.Interrupt} - -// signalTrace is the signal to send to make a Go program -// crash with a stack trace. -var signalTrace os.Signal = nil diff --git a/libgo/go/cmd/go/signal_unix.go b/libgo/go/cmd/go/signal_unix.go deleted file mode 100644 index e86cd465231..00000000000 --- a/libgo/go/cmd/go/signal_unix.go +++ /dev/null @@ -1,18 +0,0 @@ -// Copyright 2012 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// +build darwin dragonfly freebsd linux nacl netbsd openbsd solaris - -package main - -import ( - "os" - "syscall" -) - -var signalsToIgnore = []os.Signal{os.Interrupt, syscall.SIGQUIT} - -// signalTrace is the signal to send to make a Go program -// crash with a stack trace. -var signalTrace os.Signal = syscall.SIGQUIT diff --git a/libgo/go/cmd/go/tag_test.go b/libgo/go/cmd/go/tag_test.go deleted file mode 100644 index 6649bd61952..00000000000 --- a/libgo/go/cmd/go/tag_test.go +++ /dev/null @@ -1,100 +0,0 @@ -// Copyright 2011 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package main - -import "testing" - -var selectTagTestTags = []string{ - "go.r58", - "go.r58.1", - "go.r59", - "go.r59.1", - "go.r61", - "go.r61.1", - "go.weekly.2010-01-02", - "go.weekly.2011-10-12", - "go.weekly.2011-10-12.1", - "go.weekly.2011-10-14", - "go.weekly.2011-11-01", - "go1", - "go1.0.1", - "go1.999", - "go1.9.2", - "go5", - - // these should be ignored: - "release.r59", - "release.r59.1", - "release", - "weekly.2011-10-12", - "weekly.2011-10-12.1", - "weekly", - "foo", - "bar", - "go.f00", - "go!r60", - "go.1999-01-01", - "go.2x", - "go.20000000000000", - "go.2.", - "go.2.0", - "go2x", - "go20000000000000", - "go2.", - "go2.0", -} - -var selectTagTests = []struct { - version string - selected string -}{ - /* - {"release.r57", ""}, - {"release.r58.2", "go.r58.1"}, - {"release.r59", "go.r59"}, - {"release.r59.1", "go.r59.1"}, - {"release.r60", "go.r59.1"}, - {"release.r60.1", "go.r59.1"}, - {"release.r61", "go.r61"}, - {"release.r66", "go.r61.1"}, - {"weekly.2010-01-01", ""}, - {"weekly.2010-01-02", "go.weekly.2010-01-02"}, - {"weekly.2010-01-02.1", "go.weekly.2010-01-02"}, - {"weekly.2010-01-03", "go.weekly.2010-01-02"}, - {"weekly.2011-10-12", "go.weekly.2011-10-12"}, - {"weekly.2011-10-12.1", "go.weekly.2011-10-12.1"}, - {"weekly.2011-10-13", "go.weekly.2011-10-12.1"}, - {"weekly.2011-10-14", "go.weekly.2011-10-14"}, - {"weekly.2011-10-14.1", "go.weekly.2011-10-14"}, - {"weekly.2011-11-01", "go.weekly.2011-11-01"}, - {"weekly.2014-01-01", "go.weekly.2011-11-01"}, - {"weekly.3000-01-01", "go.weekly.2011-11-01"}, - {"go1", "go1"}, - {"go1.1", "go1.0.1"}, - {"go1.998", "go1.9.2"}, - {"go1.1000", "go1.999"}, - {"go6", "go5"}, - - // faulty versions: - {"release.f00", ""}, - {"weekly.1999-01-01", ""}, - {"junk", ""}, - {"", ""}, - {"go2x", ""}, - {"go200000000000", ""}, - {"go2.", ""}, - {"go2.0", ""}, - */ - {"anything", "go1"}, -} - -func TestSelectTag(t *testing.T) { - for _, c := range selectTagTests { - selected := selectTag(c.version, selectTagTestTags) - if selected != c.selected { - t.Errorf("selectTag(%q) = %q, want %q", c.version, selected, c.selected) - } - } -} diff --git a/libgo/go/cmd/go/test.bash b/libgo/go/cmd/go/test.bash deleted file mode 100644 index 0060ce2185f..00000000000 --- a/libgo/go/cmd/go/test.bash +++ /dev/null @@ -1,820 +0,0 @@ -#!/bin/bash -# Copyright 2012 The Go Authors. All rights reserved. -# Use of this source code is governed by a BSD-style -# license that can be found in the LICENSE file. - -set -e -go build -o testgo -go() { - echo TEST ERROR: ran go, not testgo: go "$@" >&2 - exit 2 -} - -started=false -TEST() { - if $started; then - stop - fi - echo TEST: "$@" - started=true - ok=true -} -stop() { - if ! $started; then - echo TEST ERROR: stop missing start >&2 - exit 2 - fi - started=false - if $ok; then - echo PASS - else - echo FAIL - allok=false - fi -} - -ok=true -allok=true - -unset GOBIN -unset GOPATH -unset GOROOT - -TEST 'file:line in error messages' -# Test that error messages have file:line information at beginning of -# the line. Also test issue 4917: that the error is on stderr. -d=$(TMPDIR=/var/tmp mktemp -d -t testgoXXX) -fn=$d/err.go -echo "package main" > $fn -echo 'import "bar"' >> $fn -./testgo run $fn 2>$d/err.out || true -if ! grep -q "^$fn:" $d/err.out; then - echo "missing file:line in error message" - cat $d/err.out - ok=false -fi -rm -r $d - -# Test local (./) imports. -testlocal() { - local="$1" - TEST local imports $2 '(easy)' - ./testgo build -o hello "testdata/$local/easy.go" - ./hello >hello.out - if ! grep -q '^easysub\.Hello' hello.out; then - echo "testdata/$local/easy.go did not generate expected output" - cat hello.out - ok=false - fi - - TEST local imports $2 '(easysub)' - ./testgo build -o hello "testdata/$local/easysub/main.go" - ./hello >hello.out - if ! grep -q '^easysub\.Hello' hello.out; then - echo "testdata/$local/easysub/main.go did not generate expected output" - cat hello.out - ok=false - fi - - TEST local imports $2 '(hard)' - ./testgo build -o hello "testdata/$local/hard.go" - ./hello >hello.out - if ! grep -q '^sub\.Hello' hello.out || ! grep -q '^subsub\.Hello' hello.out ; then - echo "testdata/$local/hard.go did not generate expected output" - cat hello.out - ok=false - fi - - rm -f hello.out hello - - # Test that go install x.go fails. - TEST local imports $2 '(go install should fail)' - if ./testgo install "testdata/$local/easy.go" >/dev/null 2>&1; then - echo "go install testdata/$local/easy.go succeeded" - ok=false - fi -} - -# Test local imports -testlocal local '' - -# Test local imports again, with bad characters in the directory name. -bad='#$%:, &()*;<=>?\^{}' -rm -rf "testdata/$bad" -cp -R testdata/local "testdata/$bad" -testlocal "$bad" 'with bad characters in path' -rm -rf "testdata/$bad" - -TEST error message for syntax error in test go file says FAIL -export GOPATH=$(pwd)/testdata -if ./testgo test syntaxerror 2>testdata/err; then - echo 'go test syntaxerror succeeded' - ok=false -elif ! grep FAIL testdata/err >/dev/null; then - echo 'go test did not say FAIL:' - cat testdata/err - ok=false -fi -rm -f ./testdata/err -unset GOPATH - -TEST wildcards do not look in useless directories -export GOPATH=$(pwd)/testdata -if ./testgo list ... >testdata/err 2>&1; then - echo "go list ... succeeded" - ok=false -elif ! grep badpkg testdata/err >/dev/null; then - echo "go list ... failure does not mention badpkg" - cat testdata/err - ok=false -elif ! ./testgo list m... >testdata/err 2>&1; then - echo "go list m... failed" - ok=false -fi -rm -rf ./testdata/err -unset GOPATH - -# Test tests with relative imports. -TEST relative imports '(go test)' -if ! ./testgo test ./testdata/testimport; then - echo "go test ./testdata/testimport failed" - ok=false -fi - -# Test installation with relative imports. -TEST relative imports '(go test -i)' -if ! ./testgo test -i ./testdata/testimport; then - echo "go test -i ./testdata/testimport failed" - ok=false -fi - -# Test tests with relative imports in packages synthesized -# from Go files named on the command line. -TEST relative imports in command-line package -if ! ./testgo test ./testdata/testimport/*.go; then - echo "go test ./testdata/testimport/*.go failed" - ok=false -fi - -TEST version control error message includes correct directory -export GOPATH=$(pwd)/testdata/shadow/root1 -if ./testgo get -u foo 2>testdata/err; then - echo "go get -u foo succeeded unexpectedly" - ok=false -elif ! grep testdata/shadow/root1/src/foo testdata/err >/dev/null; then - echo "go get -u error does not mention shadow/root1/src/foo:" - cat testdata/err - ok=false -fi -unset GOPATH - -TEST go install fails with no buildable files -export GOPATH=$(pwd)/testdata -export CGO_ENABLED=0 -if ./testgo install cgotest 2>testdata/err; then - echo "go install cgotest succeeded unexpectedly" -elif ! grep 'no buildable Go source files' testdata/err >/dev/null; then - echo "go install cgotest did not report 'no buildable Go source files'" - cat testdata/err - ok=false -fi -unset CGO_ENABLED -unset GOPATH - -# Test that without $GOBIN set, binaries get installed -# into the GOPATH bin directory. -TEST install into GOPATH -rm -rf testdata/bin -if ! GOPATH=$(pwd)/testdata ./testgo install go-cmd-test; then - echo "go install go-cmd-test failed" - ok=false -elif ! test -x testdata/bin/go-cmd-test; then - echo "go install go-cmd-test did not write to testdata/bin/go-cmd-test" - ok=false -fi - -TEST package main_test imports archive not binary -export GOBIN=$(pwd)/testdata/bin -mkdir -p $GOBIN -export GOPATH=$(pwd)/testdata -touch ./testdata/src/main_test/m.go -if ! ./testgo test main_test; then - echo "go test main_test failed without install" - ok=false -elif ! ./testgo install main_test; then - echo "go test main_test failed" - ok=false -elif [ "$(./testgo list -f '{{.Stale}}' main_test)" != false ]; then - echo "after go install, main listed as stale" - ok=false -elif ! ./testgo test main_test; then - echo "go test main_test failed after install" - ok=false -fi -rm -rf $GOBIN -unset GOBIN - -# And with $GOBIN set, binaries get installed to $GOBIN. -TEST install into GOBIN -if ! GOBIN=$(pwd)/testdata/bin1 GOPATH=$(pwd)/testdata ./testgo install go-cmd-test; then - echo "go install go-cmd-test failed" - ok=false -elif ! test -x testdata/bin1/go-cmd-test; then - echo "go install go-cmd-test did not write to testdata/bin1/go-cmd-test" - ok=false -fi - -# Without $GOBIN set, installing a program outside $GOPATH should fail -# (there is nowhere to install it). -TEST install without destination fails -if ./testgo install testdata/src/go-cmd-test/helloworld.go 2>testdata/err; then - echo "go install testdata/src/go-cmd-test/helloworld.go should have failed, did not" - ok=false -elif ! grep 'no install location for .go files listed on command line' testdata/err; then - echo "wrong error:" - cat testdata/err - ok=false -fi -rm -f testdata/err - -# With $GOBIN set, should install there. -TEST install to GOBIN '(command-line package)' -if ! GOBIN=$(pwd)/testdata/bin1 ./testgo install testdata/src/go-cmd-test/helloworld.go; then - echo "go install testdata/src/go-cmd-test/helloworld.go failed" - ok=false -elif ! test -x testdata/bin1/helloworld; then - echo "go install testdata/src/go-cmd-test/helloworld.go did not write testdata/bin1/helloworld" - ok=false -fi - -TEST godoc installs into GOBIN -d=$(mktemp -d -t testgoXXX) -export GOPATH=$d -mkdir $d/gobin -GOBIN=$d/gobin ./testgo get code.google.com/p/go.tools/cmd/godoc -if [ ! -x $d/gobin/godoc ]; then - echo did not install godoc to '$GOBIN' - GOBIN=$d/gobin ./testgo list -f 'Target: {{.Target}}' code.google.com/p/go.tools/cmd/godoc - ok=false -fi - -TEST godoc installs into GOROOT -GOROOT=$(./testgo env GOROOT) -rm -f $GOROOT/bin/godoc -./testgo install code.google.com/p/go.tools/cmd/godoc -if [ ! -x $GOROOT/bin/godoc ]; then - echo did not install godoc to '$GOROOT/bin' - ./testgo list -f 'Target: {{.Target}}' code.google.com/p/go.tools/cmd/godoc - ok=false -fi - -TEST cmd/fix installs into tool -GOOS=$(./testgo env GOOS) -GOARCH=$(./testgo env GOARCH) -rm -f $GOROOT/pkg/tool/${GOOS}_${GOARCH}/fix -./testgo install cmd/fix -if [ ! -x $GOROOT/pkg/tool/${GOOS}_${GOARCH}/fix ]; then - echo 'did not install cmd/fix to $GOROOT/pkg/tool' - GOBIN=$d/gobin ./testgo list -f 'Target: {{.Target}}' cmd/fix - ok=false -fi -rm -f $GOROOT/pkg/tool/${GOOS}_${GOARCH}/fix -GOBIN=$d/gobin ./testgo install cmd/fix -if [ ! -x $GOROOT/pkg/tool/${GOOS}_${GOARCH}/fix ]; then - echo 'did not install cmd/fix to $GOROOT/pkg/tool with $GOBIN set' - GOBIN=$d/gobin ./testgo list -f 'Target: {{.Target}}' cmd/fix - ok=false -fi - -TEST gopath program installs into GOBIN -mkdir $d/src/progname -echo 'package main; func main() {}' >$d/src/progname/p.go -GOBIN=$d/gobin ./testgo install progname -if [ ! -x $d/gobin/progname ]; then - echo 'did not install progname to $GOBIN/progname' - ./testgo list -f 'Target: {{.Target}}' cmd/api - ok=false -fi -rm -f $d/gobin/progname $d/bin/progname - -TEST gopath program installs into GOPATH/bin -./testgo install progname -if [ ! -x $d/bin/progname ]; then - echo 'did not install progname to $GOPATH/bin/progname' - ./testgo list -f 'Target: {{.Target}}' progname - ok=false -fi - -unset GOPATH -rm -rf $d - -# Reject relative paths in GOPATH. -TEST reject relative paths in GOPATH '(command-line package)' -if GOPATH=. ./testgo build testdata/src/go-cmd-test/helloworld.go; then - echo 'GOPATH="." go build should have failed, did not' - ok=false -fi - -TEST reject relative paths in GOPATH -if GOPATH=:$(pwd)/testdata:. ./testgo build go-cmd-test; then - echo 'GOPATH=":$(pwd)/testdata:." go build should have failed, did not' - ok=false -fi - -# issue 4104 -TEST go test with package listed multiple times -if [ $(./testgo test fmt fmt fmt fmt fmt | wc -l) -ne 1 ] ; then - echo 'go test fmt fmt fmt fmt fmt tested the same package multiple times' - ok=false -fi - -# ensure that output of 'go list' is consistent between runs -TEST go list is consistent -./testgo list std > test_std.list -if ! ./testgo list std | cmp -s test_std.list - ; then - echo "go list std ordering is inconsistent" - ok=false -fi -rm -f test_std.list - -# issue 4096. Validate the output of unsuccessful go install foo/quxx -TEST unsuccessful go install should mention missing package -if [ $(./testgo install 'foo/quxx' 2>&1 | grep -c 'cannot find package "foo/quxx" in any of') -ne 1 ] ; then - echo 'go install foo/quxx expected error: .*cannot find package "foo/quxx" in any of' - ok=false -fi -# test GOROOT search failure is reported -TEST GOROOT search failure reporting -if [ $(./testgo install 'foo/quxx' 2>&1 | egrep -c 'foo/quxx \(from \$GOROOT\)$') -ne 1 ] ; then - echo 'go install foo/quxx expected error: .*foo/quxx (from $GOROOT)' - ok=false -fi -# test multiple GOPATH entries are reported separately -TEST multiple GOPATH entries reported separately -if [ $(GOPATH=$(pwd)/testdata/a:$(pwd)/testdata/b ./testgo install 'foo/quxx' 2>&1 | egrep -c 'testdata/./src/foo/quxx') -ne 2 ] ; then - echo 'go install foo/quxx expected error: .*testdata/a/src/foo/quxx (from $GOPATH)\n.*testdata/b/src/foo/quxx' - ok=false -fi -# test (from $GOPATH) annotation is reported for the first GOPATH entry -TEST mention GOPATH in first GOPATH entry -if [ $(GOPATH=$(pwd)/testdata/a:$(pwd)/testdata/b ./testgo install 'foo/quxx' 2>&1 | egrep -c 'testdata/a/src/foo/quxx \(from \$GOPATH\)$') -ne 1 ] ; then - echo 'go install foo/quxx expected error: .*testdata/a/src/foo/quxx (from $GOPATH)' - ok=false -fi -# but not on the second -TEST but not the second entry -if [ $(GOPATH=$(pwd)/testdata/a:$(pwd)/testdata/b ./testgo install 'foo/quxx' 2>&1 | egrep -c 'testdata/b/src/foo/quxx$') -ne 1 ] ; then - echo 'go install foo/quxx expected error: .*testdata/b/src/foo/quxx' - ok=false -fi -# test missing GOPATH is reported -TEST missing GOPATH is reported -if [ $(GOPATH= ./testgo install 'foo/quxx' 2>&1 | egrep -c '\(\$GOPATH not set\)$') -ne 1 ] ; then - echo 'go install foo/quxx expected error: ($GOPATH not set)' - ok=false -fi - -# issue 4186. go get cannot be used to download packages to $GOROOT -# Test that without GOPATH set, go get should fail -TEST without GOPATH, go get fails -d=$(mktemp -d -t testgoXXX) -mkdir -p $d/src/pkg -if GOPATH= GOROOT=$d ./testgo get -d code.google.com/p/go.codereview/cmd/hgpatch ; then - echo 'go get code.google.com/p/go.codereview/cmd/hgpatch should not succeed with $GOPATH unset' - ok=false -fi -rm -rf $d - -# Test that with GOPATH=$GOROOT, go get should fail -TEST with GOPATH=GOROOT, go get fails -d=$(mktemp -d -t testgoXXX) -mkdir -p $d/src/pkg -if GOPATH=$d GOROOT=$d ./testgo get -d code.google.com/p/go.codereview/cmd/hgpatch ; then - echo 'go get code.google.com/p/go.codereview/cmd/hgpatch should not succeed with GOPATH=$GOROOT' - ok=false -fi -rm -rf $d - -TEST ldflags arguments with spaces '(issue 3941)' -d=$(mktemp -d -t testgoXXX) -cat >$d/main.go<hello.out -if ! grep -q '^hello world' hello.out; then - echo "ldflags -X main.extern 'hello world' failed. Output:" - cat hello.out - ok=false -fi -rm -rf $d hello.out - -TEST go test -cpuprofile leaves binary behind -./testgo test -cpuprofile strings.prof strings || ok=false -if [ ! -x strings.test ]; then - echo "go test -cpuprofile did not create strings.test" - ok=false -fi -rm -f strings.prof strings.test - -TEST symlinks do not confuse go list '(issue 4568)' -old=$(pwd) -tmp=$(cd /tmp && pwd -P) -d=$(TMPDIR=$tmp mktemp -d -t testgoXXX) -mkdir -p $d/src -( - ln -s $d $d/src/dir1 - cd $d/src - echo package p >dir1/p.go - export GOPATH=$d - if [ "$($old/testgo list -f '{{.Root}}' dir1)" != "$d" ]; then - echo Confused by symlinks. - echo "Package in current directory $(pwd) should have Root $d" - env|grep WD - $old/testgo list -json . dir1 - touch $d/failed - fi -) -if [ -f $d/failed ]; then - ok=false -fi -rm -rf $d - -TEST 'install with tags (issue 4515)' -d=$(TMPDIR=/var/tmp mktemp -d -t testgoXXX) -mkdir -p $d/src/example/a $d/src/example/b $d/bin -cat >$d/src/example/a/main.go <$d/src/example/b/main.go <$d/src/example/a/a.go <$d/src/example/a/pkg/pkg.go <$d/src/example/a/Pkg/pkg.go <$d/out; then - echo go list example/a should have failed, did not. - ok=false -elif ! grep "case-insensitive import collision" $d/out >/dev/null; then - echo go list example/a did not report import collision. - ok=false -fi -cat >$d/src/example/b/file.go <$d/src/example/b/FILE.go <$d/out; then - echo go list example/b should have failed, did not. - ok=false -elif ! grep "case-insensitive file name collision" $d/out >/dev/null; then - echo go list example/b did not report file name collision. - ok=false -fi - -TEST go get cover -./testgo get code.google.com/p/go.tools/cmd/cover || ok=false - -unset GOPATH -rm -rf $d - -TEST shadowing logic -export GOPATH=$(pwd)/testdata/shadow/root1:$(pwd)/testdata/shadow/root2 - -# The math in root1 is not "math" because the standard math is. -cdir=$(./testgo list -f '({{.ImportPath}}) ({{.ConflictDir}})' ./testdata/shadow/root1/src/math) -if [ "$cdir" != "(_$(pwd)/testdata/shadow/root1/src/math) ($GOROOT/src/pkg/math)" ]; then - echo shadowed math is not shadowed: "$cdir" - ok=false -fi - -# The foo in root1 is "foo". -cdir=$(./testgo list -f '({{.ImportPath}}) ({{.ConflictDir}})' ./testdata/shadow/root1/src/foo) -if [ "$cdir" != "(foo) ()" ]; then - echo unshadowed foo is shadowed: "$cdir" - ok=false -fi - -# The foo in root2 is not "foo" because the foo in root1 got there first. -cdir=$(./testgo list -f '({{.ImportPath}}) ({{.ConflictDir}})' ./testdata/shadow/root2/src/foo) -if [ "$cdir" != "(_$(pwd)/testdata/shadow/root2/src/foo) ($(pwd)/testdata/shadow/root1/src/foo)" ]; then - echo shadowed foo is not shadowed: "$cdir" - ok=false -fi - -# The error for go install should mention the conflicting directory. -err=$(! ./testgo install ./testdata/shadow/root2/src/foo 2>&1) -if [ "$err" != "go install: no install location for $(pwd)/testdata/shadow/root2/src/foo: hidden by $(pwd)/testdata/shadow/root1/src/foo" ]; then - echo wrong shadowed install error: "$err" - ok=false -fi - -# Only succeeds if source order is preserved. -TEST source file name order preserved -./testgo test testdata/example[12]_test.go || ok=false - -# Check that coverage analysis works at all. -# Don't worry about the exact numbers but require not 0.0%. -checkcoverage() { - if grep '[^0-9]0\.0%' testdata/cover.txt >/dev/null; then - echo 'some coverage results are 0.0%' - ok=false - fi - cat testdata/cover.txt - rm -f testdata/cover.txt -} - -TEST coverage runs -./testgo test -short -coverpkg=strings strings regexp >testdata/cover.txt 2>&1 || ok=false -./testgo test -short -cover strings math regexp >>testdata/cover.txt 2>&1 || ok=false -checkcoverage - -# Check that coverage analysis uses set mode. -TEST coverage uses set mode -if ./testgo test -short -cover encoding/binary -coverprofile=testdata/cover.out >testdata/cover.txt 2>&1; then - if ! grep -q 'mode: set' testdata/cover.out; then - ok=false - fi - checkcoverage -else - ok=false -fi -rm -f testdata/cover.out testdata/cover.txt - -TEST coverage uses atomic mode for -race. -if ./testgo test -short -race -cover encoding/binary -coverprofile=testdata/cover.out >testdata/cover.txt 2>&1; then - if ! grep -q 'mode: atomic' testdata/cover.out; then - ok=false - fi - checkcoverage -else - ok=false -fi -rm -f testdata/cover.out - -TEST coverage uses actual setting to override even for -race. -if ./testgo test -short -race -cover encoding/binary -covermode=count -coverprofile=testdata/cover.out >testdata/cover.txt 2>&1; then - if ! grep -q 'mode: count' testdata/cover.out; then - ok=false - fi - checkcoverage -else - ok=false -fi -rm -f testdata/cover.out - -TEST coverage with cgo -d=$(TMPDIR=/var/tmp mktemp -d -t testgoXXX) -./testgo test -short -cover ./testdata/cgocover >testdata/cover.txt 2>&1 || ok=false -checkcoverage - -TEST cgo depends on syscall -rm -rf $GOROOT/pkg/*_race -d=$(TMPDIR=/var/tmp mktemp -d -t testgoXXX) -export GOPATH=$d -mkdir -p $d/src/foo -echo ' -package foo -//#include -import "C" -' >$d/src/foo/foo.go -./testgo build -race foo || ok=false -rm -rf $d -unset GOPATH - -TEST cgo shows full path names -d=$(TMPDIR=/var/tmp mktemp -d -t testgoXXX) -export GOPATH=$d -mkdir -p $d/src/x/y/dirname -echo ' -package foo -import "C" -func f() { -' >$d/src/x/y/dirname/foo.go -if ./testgo build x/y/dirname >$d/err 2>&1; then - echo build succeeded unexpectedly. - ok=false -elif ! grep x/y/dirname $d/err >/dev/null; then - echo error did not use full path. - cat $d/err - ok=false -fi -rm -rf $d -unset GOPATH - -TEST 'cgo handles -Wl,$ORIGIN' -d=$(TMPDIR=/var/tmp mktemp -d -t testgoXXX) -export GOPATH=$d -mkdir -p $d/src/origin -echo ' -package origin -// #cgo !darwin LDFLAGS: -Wl,-rpath -Wl,$ORIGIN -// void f(void) {} -import "C" - -func f() { C.f() } -' >$d/src/origin/origin.go -if ! ./testgo build origin; then - echo build failed - ok=false -fi -rm -rf $d -unset GOPATH - -TEST 'Issue 6480: "go test -c -test.bench=XXX fmt" should not hang' -if ! ./testgo test -c -test.bench=XXX fmt; then - echo build test failed - ok=false -fi -rm -f fmt.test - -TEST 'Issue 7573: cmd/cgo: undefined reference when linking a C-library using gccgo' -d=$(mktemp -d -t testgoXXX) -export GOPATH=$d -mkdir -p $d/src/cgoref -ldflags="-L alibpath -lalib" -echo " -package main -// #cgo LDFLAGS: $ldflags -// void f(void) {} -import \"C\" - -func main() { C.f() } -" >$d/src/cgoref/cgoref.go -go_cmds="$(./testgo build -n -compiler gccgo cgoref 2>&1 1>/dev/null)" -ldflags_count="$(echo "$go_cmds" | egrep -c "^gccgo.*$(echo $ldflags | sed -e 's/-/\\-/g')" || true)" -if [ "$ldflags_count" -lt 1 ]; then - echo "No Go-inline "#cgo LDFLAGS:" (\"$ldflags\") passed to gccgo linking stage." - ok=false -fi -rm -rf $d -unset ldflags_count -unset go_cmds -unset ldflags -unset GOPATH - -TEST list template can use context function -if ! ./testgo list -f "GOARCH: {{context.GOARCH}}"; then - echo unable to use context in list template - ok=false -fi - -TEST 'Issue 7108: cmd/go: "go test" should fail if package does not build' -export GOPATH=$(pwd)/testdata -if ./testgo test notest >/dev/null 2>&1; then - echo 'go test notest succeeded, but should fail' - ok=false -fi -unset GOPATH - -TEST 'Issue 6844: cmd/go: go test -a foo does not rebuild regexp' -if ! ./testgo test -x -a -c testdata/dep_test.go 2>deplist; then - echo "go test -x -a -c testdata/dep_test.go failed" - ok=false -elif ! grep -q regexp deplist; then - echo "go test -x -a -c testdata/dep_test.go did not rebuild regexp" - ok=false -fi -rm -f deplist -rm -f deps.test - -TEST list template can use context function -if ! ./testgo list -f "GOARCH: {{context.GOARCH}}"; then - echo unable to use context in list template - ok=false -fi - -TEST build -i installs dependencies -d=$(TMPDIR=/var/tmp mktemp -d -t testgoXXX) -export GOPATH=$d -mkdir -p $d/src/x/y/foo $d/src/x/y/bar -echo ' -package foo -func F() {} -' >$d/src/x/y/foo/foo.go -echo ' -package bar -import "x/y/foo" -func F() { foo.F() } -' >$d/src/x/y/bar/bar.go -if ! ./testgo build -v -i x/y/bar &> $d/err; then - echo build -i failed - cat $d/err - ok=false -elif ! grep x/y/foo $d/err >/dev/null; then - echo first build -i did not build x/y/foo - cat $d/err - ok=false -fi -if ! ./testgo build -v -i x/y/bar &> $d/err; then - echo second build -i failed - cat $d/err - ok=false -elif grep x/y/foo $d/err >/dev/null; then - echo second build -i built x/y/foo - cat $d/err - ok=false -fi -rm -rf $d -unset GOPATH - -TEST 'go build in test-only directory fails with a good error' -if ./testgo build ./testdata/testonly 2>testdata/err.out; then - echo "go build ./testdata/testonly succeeded, should have failed" - ok=false -elif ! grep 'no buildable Go' testdata/err.out >/dev/null; then - echo "go build ./testdata/testonly produced unexpected error:" - cat testdata/err.out - ok=false -fi -rm -f testdata/err.out - -TEST 'go test detects test-only import cycles' -export GOPATH=$(pwd)/testdata -if ./testgo test -c testcycle/p3 2>testdata/err.out; then - echo "go test testcycle/p3 succeeded, should have failed" - ok=false -elif ! grep 'import cycle not allowed in test' testdata/err.out >/dev/null; then - echo "go test testcycle/p3 produced unexpected error:" - cat testdata/err.out - ok=false -fi -rm -f testdata/err.out -unset GOPATH - -TEST 'go test foo_test.go works' -if ! ./testgo test testdata/standalone_test.go; then - echo "go test testdata/standalone_test.go failed" - ok=false -fi - -TEST 'go test xtestonly works' -export GOPATH=$(pwd)/testdata -./testgo clean -i xtestonly -if ! ./testgo test xtestonly >/dev/null; then - echo "go test xtestonly failed" - ok=false -fi -unset GOPATH - - -# clean up -if $started; then stop; fi -rm -rf testdata/bin testdata/bin1 -rm -f testgo - -if $allok; then - echo PASS -else - echo FAIL - exit 1 -fi diff --git a/libgo/go/cmd/go/test.go b/libgo/go/cmd/go/test.go deleted file mode 100644 index 12990feb43c..00000000000 --- a/libgo/go/cmd/go/test.go +++ /dev/null @@ -1,1608 +0,0 @@ -// Copyright 2011 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package main - -import ( - "bytes" - "errors" - "fmt" - "go/ast" - "go/build" - "go/doc" - "go/parser" - "go/token" - "os" - "os/exec" - "path" - "path/filepath" - "regexp" - "runtime" - "sort" - "strings" - "text/template" - "time" - "unicode" - "unicode/utf8" -) - -// Break init loop. -func init() { - cmdTest.Run = runTest -} - -const testUsage = "test [build/test flags] [packages] [build/test flags & test binary flags]" - -var cmdTest = &Command{ - CustomFlags: true, - UsageLine: testUsage, - Short: "test packages", - Long: ` -'Go test' automates testing the packages named by the import paths. -It prints a summary of the test results in the format: - - ok archive/tar 0.011s - FAIL archive/zip 0.022s - ok compress/gzip 0.033s - ... - -followed by detailed output for each failed package. - -'Go test' recompiles each package along with any files with names matching -the file pattern "*_test.go". -Files whose names begin with "_" (including "_test.go") or "." are ignored. -These additional files can contain test functions, benchmark functions, and -example functions. See 'go help testfunc' for more. -Each listed package causes the execution of a separate test binary. - -Test files that declare a package with the suffix "_test" will be compiled as a -separate package, and then linked and run with the main test binary. - -The go tool will ignore a directory named "testdata", making it available -to hold ancillary data needed by the tests. - -By default, go test needs no arguments. It compiles and tests the package -with source in the current directory, including tests, and runs the tests. - -The package is built in a temporary directory so it does not interfere with the -non-test installation. - -` + strings.TrimSpace(testFlag1) + ` See 'go help testflag' for details. - -For more about build flags, see 'go help build'. -For more about specifying packages, see 'go help packages'. - -See also: go build, go vet. -`, -} - -const testFlag1 = ` -In addition to the build flags, the flags handled by 'go test' itself are: - - -args - Pass the remainder of the command line (everything after -args) - to the test binary, uninterpreted and unchanged. - Because this flag consumes the remainder of the command line, - the package list (if present) must appear before this flag. - - -c - Compile the test binary to pkg.test but do not run it - (where pkg is the last element of the package's import path). - The file name can be changed with the -o flag. - - -exec xprog - Run the test binary using xprog. The behavior is the same as - in 'go run'. See 'go help run' for details. - - -i - Install packages that are dependencies of the test. - Do not run the test. - - -o file - Compile the test binary to the named file. - The test still runs (unless -c or -i is specified). - -The test binary also accepts flags that control execution of the test; these -flags are also accessible by 'go test'. -` - -var helpTestflag = &Command{ - UsageLine: "testflag", - Short: "description of testing flags", - Long: ` -The 'go test' command takes both flags that apply to 'go test' itself -and flags that apply to the resulting test binary. - -Several of the flags control profiling and write an execution profile -suitable for "go tool pprof"; run "go tool pprof -h" for more -information. The --alloc_space, --alloc_objects, and --show_bytes -options of pprof control how the information is presented. - -The following flags are recognized by the 'go test' command and -control the execution of any test: - - ` + strings.TrimSpace(testFlag2) + ` -`, -} - -const testFlag2 = ` - -bench regexp - Run (sub)benchmarks matching a regular expression. - The given regular expression is split into smaller ones by - top-level '/', where each must match the corresponding part of a - benchmark's identifier. - By default, no benchmarks run. To run all benchmarks, - use '-bench .' or '-bench=.'. - - -benchtime t - Run enough iterations of each benchmark to take t, specified - as a time.Duration (for example, -benchtime 1h30s). - The default is 1 second (1s). - - -count n - Run each test and benchmark n times (default 1). - If -cpu is set, run n times for each GOMAXPROCS value. - Examples are always run once. - - -cover - Enable coverage analysis. - - -covermode set,count,atomic - Set the mode for coverage analysis for the package[s] - being tested. The default is "set" unless -race is enabled, - in which case it is "atomic". - The values: - set: bool: does this statement run? - count: int: how many times does this statement run? - atomic: int: count, but correct in multithreaded tests; - significantly more expensive. - Sets -cover. - - -coverpkg pkg1,pkg2,pkg3 - Apply coverage analysis in each test to the given list of packages. - The default is for each test to analyze only the package being tested. - Packages are specified as import paths. - Sets -cover. - - -cpu 1,2,4 - Specify a list of GOMAXPROCS values for which the tests or - benchmarks should be executed. The default is the current value - of GOMAXPROCS. - - -parallel n - Allow parallel execution of test functions that call t.Parallel. - The value of this flag is the maximum number of tests to run - simultaneously; by default, it is set to the value of GOMAXPROCS. - Note that -parallel only applies within a single test binary. - The 'go test' command may run tests for different packages - in parallel as well, according to the setting of the -p flag - (see 'go help build'). - - -run regexp - Run only those tests and examples matching the regular expression. - For tests the regular expression is split into smaller ones by - top-level '/', where each must match the corresponding part of a - test's identifier. - - -short - Tell long-running tests to shorten their run time. - It is off by default but set during all.bash so that installing - the Go tree can run a sanity check but not spend time running - exhaustive tests. - - -timeout t - If a test runs longer than t, panic. - The default is 10 minutes (10m). - - -v - Verbose output: log all tests as they are run. Also print all - text from Log and Logf calls even if the test succeeds. - -The following flags are also recognized by 'go test' and can be used to -profile the tests during execution: - - -benchmem - Print memory allocation statistics for benchmarks. - - -blockprofile block.out - Write a goroutine blocking profile to the specified file - when all tests are complete. - Writes test binary as -c would. - - -blockprofilerate n - Control the detail provided in goroutine blocking profiles by - calling runtime.SetBlockProfileRate with n. - See 'go doc runtime.SetBlockProfileRate'. - The profiler aims to sample, on average, one blocking event every - n nanoseconds the program spends blocked. By default, - if -test.blockprofile is set without this flag, all blocking events - are recorded, equivalent to -test.blockprofilerate=1. - - -coverprofile cover.out - Write a coverage profile to the file after all tests have passed. - Sets -cover. - - -cpuprofile cpu.out - Write a CPU profile to the specified file before exiting. - Writes test binary as -c would. - - -memprofile mem.out - Write a memory profile to the file after all tests have passed. - Writes test binary as -c would. - - -memprofilerate n - Enable more precise (and expensive) memory profiles by setting - runtime.MemProfileRate. See 'go doc runtime.MemProfileRate'. - To profile all memory allocations, use -test.memprofilerate=1 - and pass --alloc_space flag to the pprof tool. - - -mutexprofile mutex.out - Write a mutex contention profile to the specified file - when all tests are complete. - Writes test binary as -c would. - - -mutexprofilefraction n - Sample 1 in n stack traces of goroutines holding a - contended mutex. - - -outputdir directory - Place output files from profiling in the specified directory, - by default the directory in which "go test" is running. - - -trace trace.out - Write an execution trace to the specified file before exiting. - -Each of these flags is also recognized with an optional 'test.' prefix, -as in -test.v. When invoking the generated test binary (the result of -'go test -c') directly, however, the prefix is mandatory. - -The 'go test' command rewrites or removes recognized flags, -as appropriate, both before and after the optional package list, -before invoking the test binary. - -For instance, the command - - go test -v -myflag testdata -cpuprofile=prof.out -x - -will compile the test binary and then run it as - - pkg.test -test.v -myflag testdata -test.cpuprofile=prof.out - -(The -x flag is removed because it applies only to the go command's -execution, not to the test itself.) - -The test flags that generate profiles (other than for coverage) also -leave the test binary in pkg.test for use when analyzing the profiles. - -When 'go test' runs a test binary, it does so from within the -corresponding package's source code directory. Depending on the test, -it may be necessary to do the same when invoking a generated test -binary directly. - -The command-line package list, if present, must appear before any -flag not known to the go test command. Continuing the example above, -the package list would have to appear before -myflag, but could appear -on either side of -v. - -To keep an argument for a test binary from being interpreted as a -known flag or a package name, use -args (see 'go help test') which -passes the remainder of the command line through to the test binary -uninterpreted and unaltered. - -For instance, the command - - go test -v -args -x -v - -will compile the test binary and then run it as - - pkg.test -test.v -x -v - -Similarly, - - go test -args math - -will compile the test binary and then run it as - - pkg.test math - -In the first example, the -x and the second -v are passed through to the -test binary unchanged and with no effect on the go command itself. -In the second example, the argument math is passed through to the test -binary, instead of being interpreted as the package list. -` - -var helpTestfunc = &Command{ - UsageLine: "testfunc", - Short: "description of testing functions", - Long: ` -The 'go test' command expects to find test, benchmark, and example functions -in the "*_test.go" files corresponding to the package under test. - -A test function is one named TestXXX (where XXX is any alphanumeric string -not starting with a lower case letter) and should have the signature, - - func TestXXX(t *testing.T) { ... } - -A benchmark function is one named BenchmarkXXX and should have the signature, - - func BenchmarkXXX(b *testing.B) { ... } - -An example function is similar to a test function but, instead of using -*testing.T to report success or failure, prints output to os.Stdout. -If the last comment in the function starts with "Output:" then the output -is compared exactly against the comment (see examples below). If the last -comment begins with "Unordered output:" then the output is compared to the -comment, however the order of the lines is ignored. An example with no such -comment is compiled but not executed. An example with no text after -"Output:" is compiled, executed, and expected to produce no output. - -Godoc displays the body of ExampleXXX to demonstrate the use -of the function, constant, or variable XXX. An example of a method M with -receiver type T or *T is named ExampleT_M. There may be multiple examples -for a given function, constant, or variable, distinguished by a trailing _xxx, -where xxx is a suffix not beginning with an upper case letter. - -Here is an example of an example: - - func ExamplePrintln() { - Println("The output of\nthis example.") - // Output: The output of - // this example. - } - -Here is another example where the ordering of the output is ignored: - - func ExamplePerm() { - for _, value := range Perm(4) { - fmt.Println(value) - } - - // Unordered output: 4 - // 2 - // 1 - // 3 - // 0 - } - -The entire test file is presented as the example when it contains a single -example function, at least one other function, type, variable, or constant -declaration, and no test or benchmark functions. - -See the documentation of the testing package for more information. -`, -} - -var ( - testC bool // -c flag - testCover bool // -cover flag - testCoverMode string // -covermode flag - testCoverPaths []string // -coverpkg flag - testCoverPkgs []*Package // -coverpkg flag - testO string // -o flag - testProfile bool // some profiling flag - testNeedBinary bool // profile needs to keep binary around - testV bool // -v flag - testTimeout string // -timeout flag - testArgs []string - testBench bool - testStreamOutput bool // show output as it is generated - testShowPass bool // show passing output - - testKillTimeout = 10 * time.Minute -) - -var testMainDeps = map[string]bool{ - // Dependencies for testmain. - "testing": true, - "testing/internal/testdeps": true, - "os": true, -} - -func runTest(cmd *Command, args []string) { - var pkgArgs []string - pkgArgs, testArgs = testFlags(args) - - findExecCmd() // initialize cached result - - instrumentInit() - buildModeInit() - pkgs := packagesForBuild(pkgArgs) - if len(pkgs) == 0 { - fatalf("no packages to test") - } - - if testC && len(pkgs) != 1 { - fatalf("cannot use -c flag with multiple packages") - } - if testO != "" && len(pkgs) != 1 { - fatalf("cannot use -o flag with multiple packages") - } - if testProfile && len(pkgs) != 1 { - fatalf("cannot use test profile flag with multiple packages") - } - - // If a test timeout was given and is parseable, set our kill timeout - // to that timeout plus one minute. This is a backup alarm in case - // the test wedges with a goroutine spinning and its background - // timer does not get a chance to fire. - if dt, err := time.ParseDuration(testTimeout); err == nil && dt > 0 { - testKillTimeout = dt + 1*time.Minute - } - - // show passing test output (after buffering) with -v flag. - // must buffer because tests are running in parallel, and - // otherwise the output will get mixed. - testShowPass = testV - - // stream test output (no buffering) when no package has - // been given on the command line (implicit current directory) - // or when benchmarking. - // Also stream if we're showing output anyway with a - // single package under test or if parallelism is set to 1. - // In these cases, streaming the output produces the same result - // as not streaming, just more immediately. - testStreamOutput = len(pkgArgs) == 0 || testBench || - (testShowPass && (len(pkgs) == 1 || buildP == 1)) - - // For 'go test -i -o x.test', we want to build x.test. Imply -c to make the logic easier. - if buildI && testO != "" { - testC = true - } - - var b builder - b.init() - - if buildI { - buildV = testV - - deps := make(map[string]bool) - for dep := range testMainDeps { - deps[dep] = true - } - - for _, p := range pkgs { - // Dependencies for each test. - for _, path := range p.Imports { - deps[path] = true - } - for _, path := range p.vendored(p.TestImports) { - deps[path] = true - } - for _, path := range p.vendored(p.XTestImports) { - deps[path] = true - } - } - - // translate C to runtime/cgo - if deps["C"] { - delete(deps, "C") - deps["runtime/cgo"] = true - if goos == runtime.GOOS && goarch == runtime.GOARCH && !buildRace && !buildMSan { - deps["cmd/cgo"] = true - } - } - // Ignore pseudo-packages. - delete(deps, "unsafe") - - all := []string{} - for path := range deps { - if !build.IsLocalImport(path) { - all = append(all, path) - } - } - sort.Strings(all) - - a := &action{} - for _, p := range packagesForBuild(all) { - if !reqStdPkgSrc && p.Standard { - continue - } - a.deps = append(a.deps, b.action(modeInstall, modeInstall, p)) - } - b.do(a) - if !testC || a.failed { - return - } - b.init() - } - - var builds, runs, prints []*action - - if testCoverPaths != nil { - // Load packages that were asked about for coverage. - // packagesForBuild exits if the packages cannot be loaded. - testCoverPkgs = packagesForBuild(testCoverPaths) - - // Warn about -coverpkg arguments that are not actually used. - used := make(map[string]bool) - for _, p := range pkgs { - used[p.ImportPath] = true - for _, dep := range p.Deps { - used[dep] = true - } - } - for _, p := range testCoverPkgs { - if !used[p.ImportPath] { - fmt.Fprintf(os.Stderr, "warning: no packages being tested depend on %s\n", p.ImportPath) - } - } - - // Mark all the coverage packages for rebuilding with coverage. - for _, p := range testCoverPkgs { - // There is nothing to cover in package unsafe; it comes from the compiler. - if p.ImportPath == "unsafe" { - continue - } - p.Stale = true // rebuild - p.StaleReason = "rebuild for coverage" - p.fake = true // do not warn about rebuild - p.coverMode = testCoverMode - var coverFiles []string - coverFiles = append(coverFiles, p.GoFiles...) - coverFiles = append(coverFiles, p.CgoFiles...) - coverFiles = append(coverFiles, p.TestGoFiles...) - p.coverVars = declareCoverVars(p.ImportPath, coverFiles...) - } - } - - // Prepare build + run + print actions for all packages being tested. - for _, p := range pkgs { - // sync/atomic import is inserted by the cover tool. See #18486 - if testCover && testCoverMode == "atomic" { - ensureImport(p, "sync/atomic") - } - buildTest, runTest, printTest, err := b.test(p) - if err != nil { - str := err.Error() - if strings.HasPrefix(str, "\n") { - str = str[1:] - } - failed := fmt.Sprintf("FAIL\t%s [setup failed]\n", p.ImportPath) - - if p.ImportPath != "" { - errorf("# %s\n%s\n%s", p.ImportPath, str, failed) - } else { - errorf("%s\n%s", str, failed) - } - continue - } - builds = append(builds, buildTest) - runs = append(runs, runTest) - prints = append(prints, printTest) - } - - // Ultimately the goal is to print the output. - root := &action{deps: prints} - - // Force the printing of results to happen in order, - // one at a time. - for i, a := range prints { - if i > 0 { - a.deps = append(a.deps, prints[i-1]) - } - } - - // Force benchmarks to run in serial. - if !testC && testBench { - // The first run must wait for all builds. - // Later runs must wait for the previous run's print. - for i, run := range runs { - if i == 0 { - run.deps = append(run.deps, builds...) - } else { - run.deps = append(run.deps, prints[i-1]) - } - } - } - - // If we are building any out-of-date packages other - // than those under test, warn. - okBuild := map[*Package]bool{} - for _, p := range pkgs { - okBuild[p] = true - } - warned := false - for _, a := range actionList(root) { - if a.p == nil || okBuild[a.p] { - continue - } - okBuild[a.p] = true // warn at most once - - // Don't warn about packages being rebuilt because of - // things like coverage analysis. - for _, p1 := range a.p.imports { - if p1.fake { - a.p.fake = true - } - } - - if a.f != nil && !okBuild[a.p] && !a.p.fake && !a.p.local { - if !warned { - fmt.Fprintf(os.Stderr, "warning: building out-of-date packages:\n") - warned = true - } - fmt.Fprintf(os.Stderr, "\t%s\n", a.p.ImportPath) - } - } - if warned { - args := strings.Join(pkgArgs, " ") - if args != "" { - args = " " + args - } - extraOpts := "" - if buildRace { - extraOpts = "-race " - } - if buildMSan { - extraOpts = "-msan " - } - fmt.Fprintf(os.Stderr, "installing these packages with 'go test %s-i%s' will speed future tests.\n\n", extraOpts, args) - } - - b.do(root) -} - -// ensures that package p imports the named package. -func ensureImport(p *Package, pkg string) { - for _, d := range p.deps { - if d.Name == pkg { - return - } - } - - a := loadPackage(pkg, &importStack{}) - if a.Error != nil { - fatalf("load %s: %v", pkg, a.Error) - } - computeStale(a) - - p.imports = append(p.imports, a) -} - -func contains(x []string, s string) bool { - for _, t := range x { - if t == s { - return true - } - } - return false -} - -var windowsBadWords = []string{ - "install", - "patch", - "setup", - "update", -} - -func (b *builder) test(p *Package) (buildAction, runAction, printAction *action, err error) { - if len(p.TestGoFiles)+len(p.XTestGoFiles) == 0 { - build := b.action(modeBuild, modeBuild, p) - run := &action{p: p, deps: []*action{build}} - print := &action{f: (*builder).notest, p: p, deps: []*action{run}} - return build, run, print, nil - } - - // Build Package structs describing: - // ptest - package + test files - // pxtest - package of external test files - // pmain - pkg.test binary - var ptest, pxtest, pmain *Package - - var imports, ximports []*Package - var stk importStack - stk.push(p.ImportPath + " (test)") - for i, path := range p.TestImports { - p1 := loadImport(path, p.Dir, p, &stk, p.build.TestImportPos[path], useVendor) - if !reqStdPkgSrc && p1.Standard { - continue - } - if p1.Error != nil { - return nil, nil, nil, p1.Error - } - if len(p1.DepsErrors) > 0 { - err := p1.DepsErrors[0] - err.Pos = "" // show full import stack - return nil, nil, nil, err - } - if contains(p1.Deps, p.ImportPath) || p1.ImportPath == p.ImportPath { - // Same error that loadPackage returns (via reusePackage) in pkg.go. - // Can't change that code, because that code is only for loading the - // non-test copy of a package. - err := &PackageError{ - ImportStack: testImportStack(stk[0], p1, p.ImportPath), - Err: "import cycle not allowed in test", - isImportCycle: true, - } - return nil, nil, nil, err - } - p.TestImports[i] = p1.ImportPath - imports = append(imports, p1) - } - stk.pop() - stk.push(p.ImportPath + "_test") - pxtestNeedsPtest := false - for i, path := range p.XTestImports { - p1 := loadImport(path, p.Dir, p, &stk, p.build.XTestImportPos[path], useVendor) - if !reqStdPkgSrc && p1.Standard { - continue - } - if p1.Error != nil { - return nil, nil, nil, p1.Error - } - if len(p1.DepsErrors) > 0 { - err := p1.DepsErrors[0] - err.Pos = "" // show full import stack - return nil, nil, nil, err - } - if p1.ImportPath == p.ImportPath { - pxtestNeedsPtest = true - } else { - ximports = append(ximports, p1) - } - p.XTestImports[i] = p1.ImportPath - } - stk.pop() - - // Use last element of import path, not package name. - // They differ when package name is "main". - // But if the import path is "command-line-arguments", - // like it is during 'go run', use the package name. - var elem string - if p.ImportPath == "command-line-arguments" { - elem = p.Name - } else { - _, elem = path.Split(p.ImportPath) - } - testBinary := elem + ".test" - - // The ptest package needs to be importable under the - // same import path that p has, but we cannot put it in - // the usual place in the temporary tree, because then - // other tests will see it as the real package. - // Instead we make a _test directory under the import path - // and then repeat the import path there. We tell the - // compiler and linker to look in that _test directory first. - // - // That is, if the package under test is unicode/utf8, - // then the normal place to write the package archive is - // $WORK/unicode/utf8.a, but we write the test package archive to - // $WORK/unicode/utf8/_test/unicode/utf8.a. - // We write the external test package archive to - // $WORK/unicode/utf8/_test/unicode/utf8_test.a. - testDir := filepath.Join(b.work, filepath.FromSlash(p.ImportPath+"/_test")) - ptestObj := buildToolchain.pkgpath(testDir, p) - - // Create the directory for the .a files. - ptestDir, _ := filepath.Split(ptestObj) - if err := b.mkdir(ptestDir); err != nil { - return nil, nil, nil, err - } - - // Should we apply coverage analysis locally, - // only for this package and only for this test? - // Yes, if -cover is on but -coverpkg has not specified - // a list of packages for global coverage. - localCover := testCover && testCoverPaths == nil - - // Test package. - if len(p.TestGoFiles) > 0 || localCover || p.Name == "main" { - ptest = new(Package) - *ptest = *p - ptest.GoFiles = nil - ptest.GoFiles = append(ptest.GoFiles, p.GoFiles...) - ptest.GoFiles = append(ptest.GoFiles, p.TestGoFiles...) - ptest.target = "" - ptest.Imports = stringList(p.Imports, p.TestImports) - ptest.imports = append(append([]*Package{}, p.imports...), imports...) - ptest.pkgdir = testDir - ptest.fake = true - ptest.forceLibrary = true - ptest.Stale = true - ptest.StaleReason = "rebuild for test" - ptest.build = new(build.Package) - *ptest.build = *p.build - m := map[string][]token.Position{} - for k, v := range p.build.ImportPos { - m[k] = append(m[k], v...) - } - for k, v := range p.build.TestImportPos { - m[k] = append(m[k], v...) - } - ptest.build.ImportPos = m - - if localCover { - ptest.coverMode = testCoverMode - var coverFiles []string - coverFiles = append(coverFiles, ptest.GoFiles...) - coverFiles = append(coverFiles, ptest.CgoFiles...) - ptest.coverVars = declareCoverVars(ptest.ImportPath, coverFiles...) - } - } else { - ptest = p - } - - // External test package. - if len(p.XTestGoFiles) > 0 { - pxtest = &Package{ - Name: p.Name + "_test", - ImportPath: p.ImportPath + "_test", - localPrefix: p.localPrefix, - Root: p.Root, - Dir: p.Dir, - GoFiles: p.XTestGoFiles, - Imports: p.XTestImports, - build: &build.Package{ - ImportPos: p.build.XTestImportPos, - }, - imports: ximports, - pkgdir: testDir, - fake: true, - external: true, - Stale: true, - } - if pxtestNeedsPtest { - pxtest.imports = append(pxtest.imports, ptest) - } - } - - // Action for building pkg.test. - pmain = &Package{ - Name: "main", - Dir: testDir, - GoFiles: []string{"_testmain.go"}, - ImportPath: "testmain", - Root: p.Root, - build: &build.Package{Name: "main"}, - pkgdir: testDir, - fake: true, - Stale: true, - omitDWARF: !testC && !testNeedBinary, - } - - // The generated main also imports testing, regexp, and os. - stk.push("testmain") - for dep := range testMainDeps { - if dep == ptest.ImportPath { - pmain.imports = append(pmain.imports, ptest) - } else { - p1 := loadImport(dep, "", nil, &stk, nil, 0) - if !reqStdPkgSrc && p1.Standard { - continue - } - if p1.Error != nil { - return nil, nil, nil, p1.Error - } - pmain.imports = append(pmain.imports, p1) - } - } - - if testCoverPkgs != nil { - // Add imports, but avoid duplicates. - seen := map[*Package]bool{p: true, ptest: true} - for _, p1 := range pmain.imports { - seen[p1] = true - } - for _, p1 := range testCoverPkgs { - if !seen[p1] { - seen[p1] = true - pmain.imports = append(pmain.imports, p1) - } - } - } - - // Do initial scan for metadata needed for writing _testmain.go - // Use that metadata to update the list of imports for package main. - // The list of imports is used by recompileForTest and by the loop - // afterward that gathers t.Cover information. - t, err := loadTestFuncs(ptest) - if err != nil { - return nil, nil, nil, err - } - if len(ptest.GoFiles)+len(ptest.CgoFiles) > 0 { - pmain.imports = append(pmain.imports, ptest) - t.ImportTest = true - } - if pxtest != nil { - pmain.imports = append(pmain.imports, pxtest) - t.ImportXtest = true - } - - if ptest != p && localCover { - // We have made modifications to the package p being tested - // and are rebuilding p (as ptest), writing it to the testDir tree. - // Arrange to rebuild, writing to that same tree, all packages q - // such that the test depends on q, and q depends on p. - // This makes sure that q sees the modifications to p. - // Strictly speaking, the rebuild is only necessary if the - // modifications to p change its export metadata, but - // determining that is a bit tricky, so we rebuild always. - // - // This will cause extra compilation, so for now we only do it - // when testCover is set. The conditions are more general, though, - // and we may find that we need to do it always in the future. - recompileForTest(pmain, p, ptest, testDir) - } - - if buildContext.GOOS == "darwin" { - if buildContext.GOARCH == "arm" || buildContext.GOARCH == "arm64" { - t.IsIOS = true - t.NeedOS = true - } - } - if t.TestMain == nil { - t.NeedOS = true - } - - for _, cp := range pmain.imports { - if len(cp.coverVars) > 0 { - t.Cover = append(t.Cover, coverInfo{cp, cp.coverVars}) - } - } - - if !buildN { - // writeTestmain writes _testmain.go. This must happen after recompileForTest, - // because recompileForTest modifies XXX. - if err := writeTestmain(filepath.Join(testDir, "_testmain.go"), t); err != nil { - return nil, nil, nil, err - } - } - - computeStale(pmain) - - if ptest != p { - a := b.action(modeBuild, modeBuild, ptest) - a.objdir = testDir + string(filepath.Separator) + "_obj_test" + string(filepath.Separator) - a.objpkg = ptestObj - a.target = ptestObj - a.link = false - } - - if pxtest != nil { - a := b.action(modeBuild, modeBuild, pxtest) - a.objdir = testDir + string(filepath.Separator) + "_obj_xtest" + string(filepath.Separator) - a.objpkg = buildToolchain.pkgpath(testDir, pxtest) - a.target = a.objpkg - } - - a := b.action(modeBuild, modeBuild, pmain) - a.objdir = testDir + string(filepath.Separator) - a.objpkg = filepath.Join(testDir, "main.a") - a.target = filepath.Join(testDir, testBinary) + exeSuffix - if goos == "windows" { - // There are many reserved words on Windows that, - // if used in the name of an executable, cause Windows - // to try to ask for extra permissions. - // The word list includes setup, install, update, and patch, - // but it does not appear to be defined anywhere. - // We have run into this trying to run the - // go.codereview/patch tests. - // For package names containing those words, use test.test.exe - // instead of pkgname.test.exe. - // Note that this file name is only used in the Go command's - // temporary directory. If the -c or other flags are - // given, the code below will still use pkgname.test.exe. - // There are two user-visible effects of this change. - // First, you can actually run 'go test' in directories that - // have names that Windows thinks are installer-like, - // without getting a dialog box asking for more permissions. - // Second, in the Windows process listing during go test, - // the test shows up as test.test.exe, not pkgname.test.exe. - // That second one is a drawback, but it seems a small - // price to pay for the test running at all. - // If maintaining the list of bad words is too onerous, - // we could just do this always on Windows. - for _, bad := range windowsBadWords { - if strings.Contains(testBinary, bad) { - a.target = filepath.Join(testDir, "test.test") + exeSuffix - break - } - } - } - buildAction = a - - if testC || testNeedBinary { - // -c or profiling flag: create action to copy binary to ./test.out. - target := filepath.Join(cwd, testBinary+exeSuffix) - if testO != "" { - target = testO - if !filepath.IsAbs(target) { - target = filepath.Join(cwd, target) - } - } - buildAction = &action{ - f: (*builder).install, - deps: []*action{buildAction}, - p: pmain, - target: target, - } - runAction = buildAction // make sure runAction != nil even if not running test - } - if testC { - printAction = &action{p: p, deps: []*action{runAction}} // nop - } else { - // run test - runAction = &action{ - f: (*builder).runTest, - deps: []*action{buildAction}, - p: p, - ignoreFail: true, - } - cleanAction := &action{ - f: (*builder).cleanTest, - deps: []*action{runAction}, - p: p, - } - printAction = &action{ - f: (*builder).printTest, - deps: []*action{cleanAction}, - p: p, - } - } - - return buildAction, runAction, printAction, nil -} - -func testImportStack(top string, p *Package, target string) []string { - stk := []string{top, p.ImportPath} -Search: - for p.ImportPath != target { - for _, p1 := range p.imports { - if p1.ImportPath == target || contains(p1.Deps, target) { - stk = append(stk, p1.ImportPath) - p = p1 - continue Search - } - } - // Can't happen, but in case it does... - stk = append(stk, "") - break - } - return stk -} - -func recompileForTest(pmain, preal, ptest *Package, testDir string) { - // The "test copy" of preal is ptest. - // For each package that depends on preal, make a "test copy" - // that depends on ptest. And so on, up the dependency tree. - testCopy := map[*Package]*Package{preal: ptest} - for _, p := range packageList([]*Package{pmain}) { - // Copy on write. - didSplit := false - split := func() { - if didSplit { - return - } - didSplit = true - if p.pkgdir != testDir { - p1 := new(Package) - testCopy[p] = p1 - *p1 = *p - p1.imports = make([]*Package, len(p.imports)) - copy(p1.imports, p.imports) - p = p1 - p.pkgdir = testDir - p.target = "" - p.fake = true - p.Stale = true - p.StaleReason = "depends on package being tested" - } - } - - // Update p.deps and p.imports to use at test copies. - for i, dep := range p.deps { - if p1 := testCopy[dep]; p1 != nil && p1 != dep { - split() - p.deps[i] = p1 - } - } - for i, imp := range p.imports { - if p1 := testCopy[imp]; p1 != nil && p1 != imp { - split() - p.imports[i] = p1 - } - } - } -} - -var coverIndex = 0 - -// isTestFile reports whether the source file is a set of tests and should therefore -// be excluded from coverage analysis. -func isTestFile(file string) bool { - // We don't cover tests, only the code they test. - return strings.HasSuffix(file, "_test.go") -} - -// declareCoverVars attaches the required cover variables names -// to the files, to be used when annotating the files. -func declareCoverVars(importPath string, files ...string) map[string]*CoverVar { - coverVars := make(map[string]*CoverVar) - for _, file := range files { - if isTestFile(file) { - continue - } - coverVars[file] = &CoverVar{ - File: filepath.Join(importPath, file), - Var: fmt.Sprintf("GoCover_%d", coverIndex), - } - coverIndex++ - } - return coverVars -} - -var noTestsToRun = []byte("\ntesting: warning: no tests to run\n") - -// runTest is the action for running a test binary. -func (b *builder) runTest(a *action) error { - args := stringList(findExecCmd(), a.deps[0].target, testArgs) - a.testOutput = new(bytes.Buffer) - - if buildN || buildX { - b.showcmd("", "%s", strings.Join(args, " ")) - if buildN { - return nil - } - } - - if a.failed { - // We were unable to build the binary. - a.failed = false - fmt.Fprintf(a.testOutput, "FAIL\t%s [build failed]\n", a.p.ImportPath) - setExitStatus(1) - return nil - } - - cmd := exec.Command(args[0], args[1:]...) - cmd.Dir = a.p.Dir - cmd.Env = envForDir(cmd.Dir, origEnv) - var buf bytes.Buffer - if testStreamOutput { - cmd.Stdout = os.Stdout - cmd.Stderr = os.Stderr - } else { - cmd.Stdout = &buf - cmd.Stderr = &buf - } - - // If there are any local SWIG dependencies, we want to load - // the shared library from the build directory. - if a.p.usesSwig() { - env := cmd.Env - found := false - prefix := "LD_LIBRARY_PATH=" - for i, v := range env { - if strings.HasPrefix(v, prefix) { - env[i] = v + ":." - found = true - break - } - } - if !found { - env = append(env, "LD_LIBRARY_PATH=.") - } - cmd.Env = env - } - - t0 := time.Now() - err := cmd.Start() - - // This is a last-ditch deadline to detect and - // stop wedged test binaries, to keep the builders - // running. - if err == nil { - tick := time.NewTimer(testKillTimeout) - startSigHandlers() - done := make(chan error) - go func() { - done <- cmd.Wait() - }() - Outer: - select { - case err = <-done: - // ok - case <-tick.C: - if signalTrace != nil { - // Send a quit signal in the hope that the program will print - // a stack trace and exit. Give it five seconds before resorting - // to Kill. - cmd.Process.Signal(signalTrace) - select { - case err = <-done: - fmt.Fprintf(&buf, "*** Test killed with %v: ran too long (%v).\n", signalTrace, testKillTimeout) - break Outer - case <-time.After(5 * time.Second): - } - } - cmd.Process.Kill() - err = <-done - fmt.Fprintf(&buf, "*** Test killed: ran too long (%v).\n", testKillTimeout) - } - tick.Stop() - } - out := buf.Bytes() - t := fmt.Sprintf("%.3fs", time.Since(t0).Seconds()) - if err == nil { - norun := "" - if testShowPass { - a.testOutput.Write(out) - } - if bytes.HasPrefix(out, noTestsToRun[1:]) || bytes.Contains(out, noTestsToRun) { - norun = " [no tests to run]" - } - fmt.Fprintf(a.testOutput, "ok \t%s\t%s%s%s\n", a.p.ImportPath, t, coveragePercentage(out), norun) - return nil - } - - setExitStatus(1) - if len(out) > 0 { - a.testOutput.Write(out) - // assume printing the test binary's exit status is superfluous - } else { - fmt.Fprintf(a.testOutput, "%s\n", err) - } - fmt.Fprintf(a.testOutput, "FAIL\t%s\t%s\n", a.p.ImportPath, t) - - return nil -} - -// coveragePercentage returns the coverage results (if enabled) for the -// test. It uncovers the data by scanning the output from the test run. -func coveragePercentage(out []byte) string { - if !testCover { - return "" - } - // The string looks like - // test coverage for encoding/binary: 79.9% of statements - // Extract the piece from the percentage to the end of the line. - re := regexp.MustCompile(`coverage: (.*)\n`) - matches := re.FindSubmatch(out) - if matches == nil { - // Probably running "go test -cover" not "go test -cover fmt". - // The coverage output will appear in the output directly. - return "" - } - return fmt.Sprintf("\tcoverage: %s", matches[1]) -} - -// cleanTest is the action for cleaning up after a test. -func (b *builder) cleanTest(a *action) error { - if buildWork { - return nil - } - run := a.deps[0] - testDir := filepath.Join(b.work, filepath.FromSlash(run.p.ImportPath+"/_test")) - os.RemoveAll(testDir) - return nil -} - -// printTest is the action for printing a test result. -func (b *builder) printTest(a *action) error { - clean := a.deps[0] - run := clean.deps[0] - os.Stdout.Write(run.testOutput.Bytes()) - run.testOutput = nil - return nil -} - -// notest is the action for testing a package with no test files. -func (b *builder) notest(a *action) error { - fmt.Printf("? \t%s\t[no test files]\n", a.p.ImportPath) - return nil -} - -// isTestFunc tells whether fn has the type of a testing function. arg -// specifies the parameter type we look for: B, M or T. -func isTestFunc(fn *ast.FuncDecl, arg string) bool { - if fn.Type.Results != nil && len(fn.Type.Results.List) > 0 || - fn.Type.Params.List == nil || - len(fn.Type.Params.List) != 1 || - len(fn.Type.Params.List[0].Names) > 1 { - return false - } - ptr, ok := fn.Type.Params.List[0].Type.(*ast.StarExpr) - if !ok { - return false - } - // We can't easily check that the type is *testing.M - // because we don't know how testing has been imported, - // but at least check that it's *M or *something.M. - // Same applies for B and T. - if name, ok := ptr.X.(*ast.Ident); ok && name.Name == arg { - return true - } - if sel, ok := ptr.X.(*ast.SelectorExpr); ok && sel.Sel.Name == arg { - return true - } - return false -} - -// isTest tells whether name looks like a test (or benchmark, according to prefix). -// It is a Test (say) if there is a character after Test that is not a lower-case letter. -// We don't want TesticularCancer. -func isTest(name, prefix string) bool { - if !strings.HasPrefix(name, prefix) { - return false - } - if len(name) == len(prefix) { // "Test" is ok - return true - } - rune, _ := utf8.DecodeRuneInString(name[len(prefix):]) - return !unicode.IsLower(rune) -} - -type coverInfo struct { - Package *Package - Vars map[string]*CoverVar -} - -// loadTestFuncs returns the testFuncs describing the tests that will be run. -func loadTestFuncs(ptest *Package) (*testFuncs, error) { - t := &testFuncs{ - Package: ptest, - } - for _, file := range ptest.TestGoFiles { - if err := t.load(filepath.Join(ptest.Dir, file), "_test", &t.ImportTest, &t.NeedTest); err != nil { - return nil, err - } - } - for _, file := range ptest.XTestGoFiles { - if err := t.load(filepath.Join(ptest.Dir, file), "_xtest", &t.ImportXtest, &t.NeedXtest); err != nil { - return nil, err - } - } - return t, nil -} - -// writeTestmain writes the _testmain.go file for t to the file named out. -func writeTestmain(out string, t *testFuncs) error { - f, err := os.Create(out) - if err != nil { - return err - } - defer f.Close() - - if err := testmainTmpl.Execute(f, t); err != nil { - return err - } - - return nil -} - -type testFuncs struct { - Tests []testFunc - Benchmarks []testFunc - Examples []testFunc - TestMain *testFunc - Package *Package - ImportTest bool - NeedTest bool - ImportXtest bool - NeedXtest bool - NeedOS bool - IsIOS bool - Cover []coverInfo -} - -func (t *testFuncs) CoverMode() string { - return testCoverMode -} - -func (t *testFuncs) CoverEnabled() bool { - return testCover -} - -// Covered returns a string describing which packages are being tested for coverage. -// If the covered package is the same as the tested package, it returns the empty string. -// Otherwise it is a comma-separated human-readable list of packages beginning with -// " in", ready for use in the coverage message. -func (t *testFuncs) Covered() string { - if testCoverPaths == nil { - return "" - } - return " in " + strings.Join(testCoverPaths, ", ") -} - -// Tested returns the name of the package being tested. -func (t *testFuncs) Tested() string { - return t.Package.Name -} - -type testFunc struct { - Package string // imported package name (_test or _xtest) - Name string // function name - Output string // output, for examples - Unordered bool // output is allowed to be unordered. -} - -var testFileSet = token.NewFileSet() - -func (t *testFuncs) load(filename, pkg string, doImport, seen *bool) error { - f, err := parser.ParseFile(testFileSet, filename, nil, parser.ParseComments) - if err != nil { - return expandScanner(err) - } - for _, d := range f.Decls { - n, ok := d.(*ast.FuncDecl) - if !ok { - continue - } - if n.Recv != nil { - continue - } - name := n.Name.String() - switch { - case name == "TestMain" && isTestFunc(n, "M"): - if t.TestMain != nil { - return errors.New("multiple definitions of TestMain") - } - t.TestMain = &testFunc{pkg, name, "", false} - *doImport, *seen = true, true - case isTest(name, "Test"): - err := checkTestFunc(n, "T") - if err != nil { - return err - } - t.Tests = append(t.Tests, testFunc{pkg, name, "", false}) - *doImport, *seen = true, true - case isTest(name, "Benchmark"): - err := checkTestFunc(n, "B") - if err != nil { - return err - } - t.Benchmarks = append(t.Benchmarks, testFunc{pkg, name, "", false}) - *doImport, *seen = true, true - } - } - ex := doc.Examples(f) - sort.Slice(ex, func(i, j int) bool { return ex[i].Order < ex[j].Order }) - for _, e := range ex { - *doImport = true // import test file whether executed or not - if e.Output == "" && !e.EmptyOutput { - // Don't run examples with no output. - continue - } - t.Examples = append(t.Examples, testFunc{pkg, "Example" + e.Name, e.Output, e.Unordered}) - *seen = true - } - return nil -} - -func checkTestFunc(fn *ast.FuncDecl, arg string) error { - if !isTestFunc(fn, arg) { - name := fn.Name.String() - pos := testFileSet.Position(fn.Pos()) - return fmt.Errorf("%s: wrong signature for %s, must be: func %s(%s *testing.%s)", pos, name, name, strings.ToLower(arg), arg) - } - return nil -} - -var testmainTmpl = template.Must(template.New("main").Parse(` -package main - -import ( -{{if .NeedOS}} - "os" -{{end}} - "testing" - "testing/internal/testdeps" - -{{if .ImportTest}} - {{if .NeedTest}}_test{{else}}_{{end}} {{.Package.ImportPath | printf "%q"}} -{{end}} -{{if .ImportXtest}} - {{if .NeedXtest}}_xtest{{else}}_{{end}} {{.Package.ImportPath | printf "%s_test" | printf "%q"}} -{{end}} -{{range $i, $p := .Cover}} - _cover{{$i}} {{$p.Package.ImportPath | printf "%q"}} -{{end}} - -{{if .IsIOS}} - "os/signal" - _ "runtime/cgo" - "syscall" -{{end}} -) - -var tests = []testing.InternalTest{ -{{range .Tests}} - {"{{.Name}}", {{.Package}}.{{.Name}}}, -{{end}} -} - -var benchmarks = []testing.InternalBenchmark{ -{{range .Benchmarks}} - {"{{.Name}}", {{.Package}}.{{.Name}}}, -{{end}} -} - -var examples = []testing.InternalExample{ -{{range .Examples}} - {"{{.Name}}", {{.Package}}.{{.Name}}, {{.Output | printf "%q"}}, {{.Unordered}}}, -{{end}} -} - -{{if .CoverEnabled}} - -// Only updated by init functions, so no need for atomicity. -var ( - coverCounters = make(map[string][]uint32) - coverBlocks = make(map[string][]testing.CoverBlock) -) - -func init() { - {{range $i, $p := .Cover}} - {{range $file, $cover := $p.Vars}} - coverRegisterFile({{printf "%q" $cover.File}}, _cover{{$i}}.{{$cover.Var}}.Count[:], _cover{{$i}}.{{$cover.Var}}.Pos[:], _cover{{$i}}.{{$cover.Var}}.NumStmt[:]) - {{end}} - {{end}} -} - -func coverRegisterFile(fileName string, counter []uint32, pos []uint32, numStmts []uint16) { - if 3*len(counter) != len(pos) || len(counter) != len(numStmts) { - panic("coverage: mismatched sizes") - } - if coverCounters[fileName] != nil { - // Already registered. - return - } - coverCounters[fileName] = counter - block := make([]testing.CoverBlock, len(counter)) - for i := range counter { - block[i] = testing.CoverBlock{ - Line0: pos[3*i+0], - Col0: uint16(pos[3*i+2]), - Line1: pos[3*i+1], - Col1: uint16(pos[3*i+2]>>16), - Stmts: numStmts[i], - } - } - coverBlocks[fileName] = block -} -{{end}} - -func main() { -{{if .IsIOS}} - // Send a SIGUSR2, which will be intercepted by LLDB to - // tell the test harness that installation was successful. - // See misc/ios/go_darwin_arm_exec.go. - signal.Notify(make(chan os.Signal), syscall.SIGUSR2) - syscall.Kill(0, syscall.SIGUSR2) - signal.Reset(syscall.SIGUSR2) - - // The first argument supplied to an iOS test is an offset - // suffix for the current working directory. - // Process it here, and remove it from os.Args. - const hdr = "cwdSuffix=" - if len(os.Args) < 2 || len(os.Args[1]) <= len(hdr) || os.Args[1][:len(hdr)] != hdr { - panic("iOS test not passed a working directory suffix") - } - suffix := os.Args[1][len(hdr):] - dir, err := os.Getwd() - if err != nil { - panic(err) - } - if err := os.Chdir(dir + "/" + suffix); err != nil { - panic(err) - } - os.Args = append([]string{os.Args[0]}, os.Args[2:]...) -{{end}} - -{{if .CoverEnabled}} - testing.RegisterCover(testing.Cover{ - Mode: {{printf "%q" .CoverMode}}, - Counters: coverCounters, - Blocks: coverBlocks, - CoveredPackages: {{printf "%q" .Covered}}, - }) -{{end}} - m := testing.MainStart(testdeps.TestDeps{}, tests, benchmarks, examples) -{{with .TestMain}} - {{.Package}}.{{.Name}}(m) -{{else}} - os.Exit(m.Run()) -{{end}} -} - -`)) diff --git a/libgo/go/cmd/go/testdata/src/bench/x_test.go b/libgo/go/cmd/go/testdata/src/bench/x_test.go new file mode 100644 index 00000000000..32cabf8a7be --- /dev/null +++ b/libgo/go/cmd/go/testdata/src/bench/x_test.go @@ -0,0 +1,6 @@ +package bench + +import "testing" + +func Benchmark(b *testing.B) { +} diff --git a/libgo/go/cmd/go/testdata/src/cgoasm/p.go b/libgo/go/cmd/go/testdata/src/cgoasm/p.go new file mode 100644 index 00000000000..148b47f6a5b --- /dev/null +++ b/libgo/go/cmd/go/testdata/src/cgoasm/p.go @@ -0,0 +1,8 @@ +package p + +/* +// hi +*/ +import "C" + +func F() {} diff --git a/libgo/go/cmd/go/testdata/src/cgoasm/p.s b/libgo/go/cmd/go/testdata/src/cgoasm/p.s new file mode 100644 index 00000000000..aaade03a437 --- /dev/null +++ b/libgo/go/cmd/go/testdata/src/cgoasm/p.s @@ -0,0 +1,2 @@ +TEXT asm(SB),$0 + RET diff --git a/libgo/go/cmd/go/testdata/src/exclude/empty/x.txt b/libgo/go/cmd/go/testdata/src/exclude/empty/x.txt new file mode 100644 index 00000000000..e69de29bb2d diff --git a/libgo/go/cmd/go/testdata/src/exclude/ignore/_x.go b/libgo/go/cmd/go/testdata/src/exclude/ignore/_x.go new file mode 100644 index 00000000000..823aafd0712 --- /dev/null +++ b/libgo/go/cmd/go/testdata/src/exclude/ignore/_x.go @@ -0,0 +1 @@ +package x diff --git a/libgo/go/cmd/go/testdata/src/exclude/x.go b/libgo/go/cmd/go/testdata/src/exclude/x.go new file mode 100644 index 00000000000..9affd21e8c1 --- /dev/null +++ b/libgo/go/cmd/go/testdata/src/exclude/x.go @@ -0,0 +1,3 @@ +// +build linux,!linux + +package x diff --git a/libgo/go/cmd/go/testdata/src/exclude/x_linux.go b/libgo/go/cmd/go/testdata/src/exclude/x_linux.go new file mode 100644 index 00000000000..41ef6e5d7b1 --- /dev/null +++ b/libgo/go/cmd/go/testdata/src/exclude/x_linux.go @@ -0,0 +1,3 @@ +// +build windows + +package x diff --git a/libgo/go/cmd/go/testdata/src/testlist/bench_test.go b/libgo/go/cmd/go/testdata/src/testlist/bench_test.go new file mode 100644 index 00000000000..22f147b6339 --- /dev/null +++ b/libgo/go/cmd/go/testdata/src/testlist/bench_test.go @@ -0,0 +1,14 @@ +package testlist + +import ( + "fmt" + "testing" +) + +func BenchmarkSimplefunc(b *testing.B) { + b.StopTimer() + b.StartTimer() + for i := 0; i < b.N; i++ { + _ = fmt.Sprint("Test for bench") + } +} diff --git a/libgo/go/cmd/go/testdata/src/testlist/example_test.go b/libgo/go/cmd/go/testdata/src/testlist/example_test.go new file mode 100644 index 00000000000..0298dfde81e --- /dev/null +++ b/libgo/go/cmd/go/testdata/src/testlist/example_test.go @@ -0,0 +1,21 @@ +package testlist + +import ( + "fmt" +) + +func ExampleSimple() { + fmt.Println("Test with Output.") + + // Output: Test with Output. +} + +func ExampleWithEmptyOutput() { + fmt.Println("") + + // Output: +} + +func ExampleNoOutput() { + _ = fmt.Sprint("Test with no output") +} diff --git a/libgo/go/cmd/go/testdata/src/testlist/test_test.go b/libgo/go/cmd/go/testdata/src/testlist/test_test.go new file mode 100644 index 00000000000..bdc09f27c5b --- /dev/null +++ b/libgo/go/cmd/go/testdata/src/testlist/test_test.go @@ -0,0 +1,10 @@ +package testlist + +import ( + "fmt" + "testing" +) + +func TestSimple(t *testing.T) { + _ = fmt.Sprint("Test simple") +} diff --git a/libgo/go/cmd/go/testdata/src/testregexp/x_test.go b/libgo/go/cmd/go/testdata/src/testregexp/x_test.go new file mode 100644 index 00000000000..7573e79e167 --- /dev/null +++ b/libgo/go/cmd/go/testdata/src/testregexp/x_test.go @@ -0,0 +1,17 @@ +package x + +import "testing" + +func TestX(t *testing.T) { + t.Logf("LOG: X running") + t.Run("Y", func(t *testing.T) { + t.Logf("LOG: Y running") + }) +} + +func BenchmarkX(b *testing.B) { + b.Logf("LOG: X running N=%d", b.N) + b.Run("Y", func(b *testing.B) { + b.Logf("LOG: Y running N=%d", b.N) + }) +} diff --git a/libgo/go/cmd/go/testdata/src/testregexp/z_test.go b/libgo/go/cmd/go/testdata/src/testregexp/z_test.go new file mode 100644 index 00000000000..4fd19791544 --- /dev/null +++ b/libgo/go/cmd/go/testdata/src/testregexp/z_test.go @@ -0,0 +1,19 @@ +package x + +import "testing" + +func TestZ(t *testing.T) { + t.Logf("LOG: Z running") +} + +func TestXX(t *testing.T) { + t.Logf("LOG: XX running") +} + +func BenchmarkZ(b *testing.B) { + b.Logf("LOG: Z running N=%d", b.N) +} + +func BenchmarkXX(b *testing.B) { + b.Logf("LOG: XX running N=%d", b.N) +} diff --git a/libgo/go/cmd/go/testflag.go b/libgo/go/cmd/go/testflag.go deleted file mode 100644 index fa53bfcdf09..00000000000 --- a/libgo/go/cmd/go/testflag.go +++ /dev/null @@ -1,294 +0,0 @@ -// Copyright 2011 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package main - -import ( - "flag" - "fmt" - "os" - "strconv" - "strings" -) - -// The flag handling part of go test is large and distracting. -// We can't use the flag package because some of the flags from -// our command line are for us, and some are for 6.out, and -// some are for both. - -// testFlagSpec defines a flag we know about. -type testFlagSpec struct { - name string - boolVar *bool - flagValue flag.Value - passToTest bool // pass to Test - multiOK bool // OK to have multiple instances - present bool // flag has been seen -} - -// testFlagDefn is the set of flags we process. -var testFlagDefn = []*testFlagSpec{ - // local. - {name: "c", boolVar: &testC}, - {name: "i", boolVar: &buildI}, - {name: "o"}, - {name: "cover", boolVar: &testCover}, - {name: "covermode"}, - {name: "coverpkg"}, - {name: "exec"}, - - // passed to 6.out, adding a "test." prefix to the name if necessary: -v becomes -test.v. - {name: "bench", passToTest: true}, - {name: "benchmem", boolVar: new(bool), passToTest: true}, - {name: "benchtime", passToTest: true}, - {name: "count", passToTest: true}, - {name: "coverprofile", passToTest: true}, - {name: "cpu", passToTest: true}, - {name: "cpuprofile", passToTest: true}, - {name: "memprofile", passToTest: true}, - {name: "memprofilerate", passToTest: true}, - {name: "blockprofile", passToTest: true}, - {name: "blockprofilerate", passToTest: true}, - {name: "mutexprofile", passToTest: true}, - {name: "mutexprofilefraction", passToTest: true}, - {name: "outputdir", passToTest: true}, - {name: "parallel", passToTest: true}, - {name: "run", passToTest: true}, - {name: "short", boolVar: new(bool), passToTest: true}, - {name: "timeout", passToTest: true}, - {name: "trace", passToTest: true}, - {name: "v", boolVar: &testV, passToTest: true}, -} - -// add build flags to testFlagDefn -func init() { - var cmd Command - addBuildFlags(&cmd) - cmd.Flag.VisitAll(func(f *flag.Flag) { - if f.Name == "v" { - // test overrides the build -v flag - return - } - testFlagDefn = append(testFlagDefn, &testFlagSpec{ - name: f.Name, - flagValue: f.Value, - }) - }) -} - -// testFlags processes the command line, grabbing -x and -c, rewriting known flags -// to have "test" before them, and reading the command line for the 6.out. -// Unfortunately for us, we need to do our own flag processing because go test -// grabs some flags but otherwise its command line is just a holding place for -// pkg.test's arguments. -// We allow known flags both before and after the package name list, -// to allow both -// go test fmt -custom-flag-for-fmt-test -// go test -x math -func testFlags(args []string) (packageNames, passToTest []string) { - inPkg := false - outputDir := "" - var explicitArgs []string - for i := 0; i < len(args); i++ { - if !strings.HasPrefix(args[i], "-") { - if !inPkg && packageNames == nil { - // First package name we've seen. - inPkg = true - } - if inPkg { - packageNames = append(packageNames, args[i]) - continue - } - } - - if inPkg { - // Found an argument beginning with "-"; end of package list. - inPkg = false - } - - f, value, extraWord := testFlag(args, i) - if f == nil { - // This is a flag we do not know; we must assume - // that any args we see after this might be flag - // arguments, not package names. - inPkg = false - if packageNames == nil { - // make non-nil: we have seen the empty package list - packageNames = []string{} - } - if args[i] == "-args" || args[i] == "--args" { - // -args or --args signals that everything that follows - // should be passed to the test. - explicitArgs = args[i+1:] - break - } - passToTest = append(passToTest, args[i]) - continue - } - if f.flagValue != nil { - if err := f.flagValue.Set(value); err != nil { - fatalf("invalid flag argument for -%s: %v", f.name, err) - } - } else { - // Test-only flags. - // Arguably should be handled by f.flagValue, but aren't. - var err error - switch f.name { - // bool flags. - case "c", "i", "v", "cover": - setBoolFlag(f.boolVar, value) - case "o": - testO = value - testNeedBinary = true - case "exec": - execCmd, err = splitQuotedFields(value) - if err != nil { - fatalf("invalid flag argument for -%s: %v", f.name, err) - } - case "bench": - // record that we saw the flag; don't care about the value - testBench = true - case "timeout": - testTimeout = value - case "blockprofile", "cpuprofile", "memprofile", "mutexprofile": - testProfile = true - testNeedBinary = true - case "trace": - testProfile = true - case "coverpkg": - testCover = true - if value == "" { - testCoverPaths = nil - } else { - testCoverPaths = strings.Split(value, ",") - } - case "coverprofile": - testCover = true - testProfile = true - case "covermode": - switch value { - case "set", "count", "atomic": - testCoverMode = value - default: - fatalf("invalid flag argument for -covermode: %q", value) - } - testCover = true - case "outputdir": - outputDir = value - } - } - if extraWord { - i++ - } - if f.passToTest { - passToTest = append(passToTest, "-test."+f.name+"="+value) - } - } - - if testCoverMode == "" { - testCoverMode = "set" - if buildRace { - // Default coverage mode is atomic when -race is set. - testCoverMode = "atomic" - } - } - - // Tell the test what directory we're running in, so it can write the profiles there. - if testProfile && outputDir == "" { - dir, err := os.Getwd() - if err != nil { - fatalf("error from os.Getwd: %s", err) - } - passToTest = append(passToTest, "-test.outputdir", dir) - } - - passToTest = append(passToTest, explicitArgs...) - return -} - -// testFlag sees if argument i is a known flag and returns its definition, value, and whether it consumed an extra word. -func testFlag(args []string, i int) (f *testFlagSpec, value string, extra bool) { - arg := args[i] - if strings.HasPrefix(arg, "--") { // reduce two minuses to one - arg = arg[1:] - } - switch arg { - case "-?", "-h", "-help": - usage() - } - if arg == "" || arg[0] != '-' { - return - } - name := arg[1:] - // If there's already "test.", drop it for now. - name = strings.TrimPrefix(name, "test.") - equals := strings.Index(name, "=") - if equals >= 0 { - value = name[equals+1:] - name = name[:equals] - } - for _, f = range testFlagDefn { - if name == f.name { - // Booleans are special because they have modes -x, -x=true, -x=false. - if f.boolVar != nil || isBoolFlag(f.flagValue) { - if equals < 0 { // otherwise, it's been set and will be verified in setBoolFlag - value = "true" - } else { - // verify it parses - setBoolFlag(new(bool), value) - } - } else { // Non-booleans must have a value. - extra = equals < 0 - if extra { - if i+1 >= len(args) { - testSyntaxError("missing argument for flag " + f.name) - } - value = args[i+1] - } - } - if f.present && !f.multiOK { - testSyntaxError(f.name + " flag may be set only once") - } - f.present = true - return - } - } - f = nil - return -} - -// isBoolFlag reports whether v is a bool flag. -func isBoolFlag(v flag.Value) bool { - vv, ok := v.(interface { - IsBoolFlag() bool - }) - if ok { - return vv.IsBoolFlag() - } - return false -} - -// setBoolFlag sets the addressed boolean to the value. -func setBoolFlag(flag *bool, value string) { - x, err := strconv.ParseBool(value) - if err != nil { - testSyntaxError("illegal bool flag value " + value) - } - *flag = x -} - -// setIntFlag sets the addressed integer to the value. -func setIntFlag(flag *int, value string) { - x, err := strconv.Atoi(value) - if err != nil { - testSyntaxError("illegal int flag value " + value) - } - *flag = x -} - -func testSyntaxError(msg string) { - fmt.Fprintf(os.Stderr, "go test: %s\n", msg) - fmt.Fprintf(os.Stderr, `run "go help test" or "go help testflag" for more information`+"\n") - os.Exit(2) -} diff --git a/libgo/go/cmd/go/testgo.go b/libgo/go/cmd/go/testgo.go deleted file mode 100644 index e507f34be69..00000000000 --- a/libgo/go/cmd/go/testgo.go +++ /dev/null @@ -1,21 +0,0 @@ -// Copyright 2014 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// This file contains extra hooks for testing the go command. -// It is compiled into the Go binary only when building the -// test copy; it does not get compiled into the standard go -// command, so these testing hooks are not present in the -// go command that everyone uses. - -// +build testgo - -package main - -import "os" - -func init() { - if v := os.Getenv("TESTGO_IS_GO_RELEASE"); v != "" { - isGoRelease = v == "1" - } -} diff --git a/libgo/go/cmd/go/tool.go b/libgo/go/cmd/go/tool.go deleted file mode 100644 index 6e2c68fbf53..00000000000 --- a/libgo/go/cmd/go/tool.go +++ /dev/null @@ -1,173 +0,0 @@ -// Copyright 2011 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package main - -import ( - "fmt" - "go/build" - "os" - "os/exec" - "path/filepath" - "runtime" - "sort" - "strings" -) - -var cmdTool = &Command{ - Run: runTool, - UsageLine: "tool [-n] command [args...]", - Short: "run specified go tool", - Long: ` -Tool runs the go tool command identified by the arguments. -With no arguments it prints the list of known tools. - -The -n flag causes tool to print the command that would be -executed but not execute it. - -For more about each tool command, see 'go tool command -h'. -`, -} - -var ( - toolGOOS = runtime.GOOS - toolGOARCH = runtime.GOARCH - toolIsWindows = toolGOOS == "windows" - toolDir = build.ToolDir - - toolN bool -) - -// List of go tools found in the gccgo tool directory. -// Other binaries could be in the same directory so don't -// show those with the 'go tool' command. - -var gccgoTools = []string{"cgo", "fix", "cover", "godoc", "vet"} - -func init() { - cmdTool.Flag.BoolVar(&toolN, "n", false, "") -} - -const toolWindowsExtension = ".exe" - -func tool(toolName string) string { - toolPath := filepath.Join(toolDir, toolName) - if toolIsWindows { - toolPath += toolWindowsExtension - } - if len(buildToolExec) > 0 { - return toolPath - } - // Give a nice message if there is no tool with that name. - if _, err := os.Stat(toolPath); err != nil { - if isInGoToolsRepo(toolName) { - fmt.Fprintf(os.Stderr, "go tool: no such tool %q; to install:\n\tgo get golang.org/x/tools/cmd/%s\n", toolName, toolName) - } else { - fmt.Fprintf(os.Stderr, "go tool: no such tool %q\n", toolName) - } - setExitStatus(2) - exit() - } - return toolPath -} - -func isInGoToolsRepo(toolName string) bool { - return false -} - -func runTool(cmd *Command, args []string) { - if len(args) == 0 { - listTools() - return - } - toolName := args[0] - // The tool name must be lower-case letters, numbers or underscores. - for _, c := range toolName { - switch { - case 'a' <= c && c <= 'z', '0' <= c && c <= '9', c == '_': - default: - fmt.Fprintf(os.Stderr, "go tool: bad tool name %q\n", toolName) - setExitStatus(2) - return - } - } - toolPath := tool(toolName) - if toolPath == "" { - return - } - if toolN { - cmd := toolPath - if len(args) > 1 { - cmd += " " + strings.Join(args[1:], " ") - } - fmt.Printf("%s\n", cmd) - return - } - args[0] = toolPath // in case the tool wants to re-exec itself, e.g. cmd/dist - toolCmd := &exec.Cmd{ - Path: toolPath, - Args: args, - Stdin: os.Stdin, - Stdout: os.Stdout, - Stderr: os.Stderr, - // Set $GOROOT, mainly for go tool dist. - Env: mergeEnvLists([]string{"GOROOT=" + goroot}, os.Environ()), - } - err := toolCmd.Run() - if err != nil { - // Only print about the exit status if the command - // didn't even run (not an ExitError) or it didn't exit cleanly - // or we're printing command lines too (-x mode). - // Assume if command exited cleanly (even with non-zero status) - // it printed any messages it wanted to print. - if e, ok := err.(*exec.ExitError); !ok || !e.Exited() || buildX { - fmt.Fprintf(os.Stderr, "go tool %s: %s\n", toolName, err) - } - setExitStatus(1) - return - } -} - -// listTools prints a list of the available tools in the tools directory. -func listTools() { - f, err := os.Open(toolDir) - if err != nil { - fmt.Fprintf(os.Stderr, "go tool: no tool directory: %s\n", err) - setExitStatus(2) - return - } - defer f.Close() - names, err := f.Readdirnames(-1) - if err != nil { - fmt.Fprintf(os.Stderr, "go tool: can't read directory: %s\n", err) - setExitStatus(2) - return - } - - sort.Strings(names) - for _, name := range names { - // Unify presentation by going to lower case. - name = strings.ToLower(name) - // If it's windows, don't show the .exe suffix. - if toolIsWindows && strings.HasSuffix(name, toolWindowsExtension) { - name = name[:len(name)-len(toolWindowsExtension)] - } - - // The tool directory used by gccgo will have other binaries - // in additions to go tools. Only display go tools for this list. - - if buildContext.Compiler == "gccgo" { - for _, tool := range gccgoTools { - if tool == name { - fmt.Println(name) - } - } - } else { - - // Not gccgo, list all the tools found in this dir - - fmt.Println(name) - } - } -} diff --git a/libgo/go/cmd/go/vcs.go b/libgo/go/cmd/go/vcs.go deleted file mode 100644 index fcdce220a7c..00000000000 --- a/libgo/go/cmd/go/vcs.go +++ /dev/null @@ -1,1006 +0,0 @@ -// Copyright 2012 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package main - -import ( - "bytes" - "encoding/json" - "errors" - "fmt" - "internal/singleflight" - "log" - "net/url" - "os" - "os/exec" - "path/filepath" - "regexp" - "strings" - "sync" -) - -// A vcsCmd describes how to use a version control system -// like Mercurial, Git, or Subversion. -type vcsCmd struct { - name string - cmd string // name of binary to invoke command - - createCmd []string // commands to download a fresh copy of a repository - downloadCmd []string // commands to download updates into an existing repository - - tagCmd []tagCmd // commands to list tags - tagLookupCmd []tagCmd // commands to lookup tags before running tagSyncCmd - tagSyncCmd []string // commands to sync to specific tag - tagSyncDefault []string // commands to sync to default tag - - scheme []string - pingCmd string - - remoteRepo func(v *vcsCmd, rootDir string) (remoteRepo string, err error) - resolveRepo func(v *vcsCmd, rootDir, remoteRepo string) (realRepo string, err error) -} - -var defaultSecureScheme = map[string]bool{ - "https": true, - "git+ssh": true, - "bzr+ssh": true, - "svn+ssh": true, - "ssh": true, -} - -func (v *vcsCmd) isSecure(repo string) bool { - u, err := url.Parse(repo) - if err != nil { - // If repo is not a URL, it's not secure. - return false - } - return v.isSecureScheme(u.Scheme) -} - -func (v *vcsCmd) isSecureScheme(scheme string) bool { - switch v.cmd { - case "git": - // GIT_ALLOW_PROTOCOL is an environment variable defined by Git. It is a - // colon-separated list of schemes that are allowed to be used with git - // fetch/clone. Any scheme not mentioned will be considered insecure. - if allow := os.Getenv("GIT_ALLOW_PROTOCOL"); allow != "" { - for _, s := range strings.Split(allow, ":") { - if s == scheme { - return true - } - } - return false - } - } - return defaultSecureScheme[scheme] -} - -// A tagCmd describes a command to list available tags -// that can be passed to tagSyncCmd. -type tagCmd struct { - cmd string // command to list tags - pattern string // regexp to extract tags from list -} - -// vcsList lists the known version control systems -var vcsList = []*vcsCmd{ - vcsHg, - vcsGit, - vcsSvn, - vcsBzr, -} - -// vcsByCmd returns the version control system for the given -// command name (hg, git, svn, bzr). -func vcsByCmd(cmd string) *vcsCmd { - for _, vcs := range vcsList { - if vcs.cmd == cmd { - return vcs - } - } - return nil -} - -// vcsHg describes how to use Mercurial. -var vcsHg = &vcsCmd{ - name: "Mercurial", - cmd: "hg", - - createCmd: []string{"clone -U {repo} {dir}"}, - downloadCmd: []string{"pull"}, - - // We allow both tag and branch names as 'tags' - // for selecting a version. This lets people have - // a go.release.r60 branch and a go1 branch - // and make changes in both, without constantly - // editing .hgtags. - tagCmd: []tagCmd{ - {"tags", `^(\S+)`}, - {"branches", `^(\S+)`}, - }, - tagSyncCmd: []string{"update -r {tag}"}, - tagSyncDefault: []string{"update default"}, - - scheme: []string{"https", "http", "ssh"}, - pingCmd: "identify {scheme}://{repo}", - remoteRepo: hgRemoteRepo, -} - -func hgRemoteRepo(vcsHg *vcsCmd, rootDir string) (remoteRepo string, err error) { - out, err := vcsHg.runOutput(rootDir, "paths default") - if err != nil { - return "", err - } - return strings.TrimSpace(string(out)), nil -} - -// vcsGit describes how to use Git. -var vcsGit = &vcsCmd{ - name: "Git", - cmd: "git", - - createCmd: []string{"clone {repo} {dir}", "-go-internal-cd {dir} submodule update --init --recursive"}, - downloadCmd: []string{"pull --ff-only", "submodule update --init --recursive"}, - - tagCmd: []tagCmd{ - // tags/xxx matches a git tag named xxx - // origin/xxx matches a git branch named xxx on the default remote repository - {"show-ref", `(?:tags|origin)/(\S+)$`}, - }, - tagLookupCmd: []tagCmd{ - {"show-ref tags/{tag} origin/{tag}", `((?:tags|origin)/\S+)$`}, - }, - tagSyncCmd: []string{"checkout {tag}", "submodule update --init --recursive"}, - // both createCmd and downloadCmd update the working dir. - // No need to do more here. We used to 'checkout master' - // but that doesn't work if the default branch is not named master. - // DO NOT add 'checkout master' here. - // See golang.org/issue/9032. - tagSyncDefault: []string{"submodule update --init --recursive"}, - - scheme: []string{"git", "https", "http", "git+ssh", "ssh"}, - pingCmd: "ls-remote {scheme}://{repo}", - remoteRepo: gitRemoteRepo, -} - -// scpSyntaxRe matches the SCP-like addresses used by Git to access -// repositories by SSH. -var scpSyntaxRe = regexp.MustCompile(`^([a-zA-Z0-9_]+)@([a-zA-Z0-9._-]+):(.*)$`) - -func gitRemoteRepo(vcsGit *vcsCmd, rootDir string) (remoteRepo string, err error) { - cmd := "config remote.origin.url" - errParse := errors.New("unable to parse output of git " + cmd) - errRemoteOriginNotFound := errors.New("remote origin not found") - outb, err := vcsGit.run1(rootDir, cmd, nil, false) - if err != nil { - // if it doesn't output any message, it means the config argument is correct, - // but the config value itself doesn't exist - if outb != nil && len(outb) == 0 { - return "", errRemoteOriginNotFound - } - return "", err - } - out := strings.TrimSpace(string(outb)) - - var repoURL *url.URL - if m := scpSyntaxRe.FindStringSubmatch(out); m != nil { - // Match SCP-like syntax and convert it to a URL. - // Eg, "git@github.com:user/repo" becomes - // "ssh://git@github.com/user/repo". - repoURL = &url.URL{ - Scheme: "ssh", - User: url.User(m[1]), - Host: m[2], - Path: m[3], - } - } else { - repoURL, err = url.Parse(out) - if err != nil { - return "", err - } - } - - // Iterate over insecure schemes too, because this function simply - // reports the state of the repo. If we can't see insecure schemes then - // we can't report the actual repo URL. - for _, s := range vcsGit.scheme { - if repoURL.Scheme == s { - return repoURL.String(), nil - } - } - return "", errParse -} - -// vcsBzr describes how to use Bazaar. -var vcsBzr = &vcsCmd{ - name: "Bazaar", - cmd: "bzr", - - createCmd: []string{"branch {repo} {dir}"}, - - // Without --overwrite bzr will not pull tags that changed. - // Replace by --overwrite-tags after http://pad.lv/681792 goes in. - downloadCmd: []string{"pull --overwrite"}, - - tagCmd: []tagCmd{{"tags", `^(\S+)`}}, - tagSyncCmd: []string{"update -r {tag}"}, - tagSyncDefault: []string{"update -r revno:-1"}, - - scheme: []string{"https", "http", "bzr", "bzr+ssh"}, - pingCmd: "info {scheme}://{repo}", - remoteRepo: bzrRemoteRepo, - resolveRepo: bzrResolveRepo, -} - -func bzrRemoteRepo(vcsBzr *vcsCmd, rootDir string) (remoteRepo string, err error) { - outb, err := vcsBzr.runOutput(rootDir, "config parent_location") - if err != nil { - return "", err - } - return strings.TrimSpace(string(outb)), nil -} - -func bzrResolveRepo(vcsBzr *vcsCmd, rootDir, remoteRepo string) (realRepo string, err error) { - outb, err := vcsBzr.runOutput(rootDir, "info "+remoteRepo) - if err != nil { - return "", err - } - out := string(outb) - - // Expect: - // ... - // (branch root|repository branch): - // ... - - found := false - for _, prefix := range []string{"\n branch root: ", "\n repository branch: "} { - i := strings.Index(out, prefix) - if i >= 0 { - out = out[i+len(prefix):] - found = true - break - } - } - if !found { - return "", fmt.Errorf("unable to parse output of bzr info") - } - - i := strings.Index(out, "\n") - if i < 0 { - return "", fmt.Errorf("unable to parse output of bzr info") - } - out = out[:i] - return strings.TrimSpace(out), nil -} - -// vcsSvn describes how to use Subversion. -var vcsSvn = &vcsCmd{ - name: "Subversion", - cmd: "svn", - - createCmd: []string{"checkout {repo} {dir}"}, - downloadCmd: []string{"update"}, - - // There is no tag command in subversion. - // The branch information is all in the path names. - - scheme: []string{"https", "http", "svn", "svn+ssh"}, - pingCmd: "info {scheme}://{repo}", - remoteRepo: svnRemoteRepo, -} - -func svnRemoteRepo(vcsSvn *vcsCmd, rootDir string) (remoteRepo string, err error) { - outb, err := vcsSvn.runOutput(rootDir, "info") - if err != nil { - return "", err - } - out := string(outb) - - // Expect: - // ... - // Repository Root: - // ... - - i := strings.Index(out, "\nRepository Root: ") - if i < 0 { - return "", fmt.Errorf("unable to parse output of svn info") - } - out = out[i+len("\nRepository Root: "):] - i = strings.Index(out, "\n") - if i < 0 { - return "", fmt.Errorf("unable to parse output of svn info") - } - out = out[:i] - return strings.TrimSpace(out), nil -} - -func (v *vcsCmd) String() string { - return v.name -} - -// run runs the command line cmd in the given directory. -// keyval is a list of key, value pairs. run expands -// instances of {key} in cmd into value, but only after -// splitting cmd into individual arguments. -// If an error occurs, run prints the command line and the -// command's combined stdout+stderr to standard error. -// Otherwise run discards the command's output. -func (v *vcsCmd) run(dir string, cmd string, keyval ...string) error { - _, err := v.run1(dir, cmd, keyval, true) - return err -} - -// runVerboseOnly is like run but only generates error output to standard error in verbose mode. -func (v *vcsCmd) runVerboseOnly(dir string, cmd string, keyval ...string) error { - _, err := v.run1(dir, cmd, keyval, false) - return err -} - -// runOutput is like run but returns the output of the command. -func (v *vcsCmd) runOutput(dir string, cmd string, keyval ...string) ([]byte, error) { - return v.run1(dir, cmd, keyval, true) -} - -// run1 is the generalized implementation of run and runOutput. -func (v *vcsCmd) run1(dir string, cmdline string, keyval []string, verbose bool) ([]byte, error) { - m := make(map[string]string) - for i := 0; i < len(keyval); i += 2 { - m[keyval[i]] = keyval[i+1] - } - args := strings.Fields(cmdline) - for i, arg := range args { - args[i] = expand(m, arg) - } - - if len(args) >= 2 && args[0] == "-go-internal-cd" { - if filepath.IsAbs(args[1]) { - dir = args[1] - } else { - dir = filepath.Join(dir, args[1]) - } - args = args[2:] - } - - _, err := exec.LookPath(v.cmd) - if err != nil { - fmt.Fprintf(os.Stderr, - "go: missing %s command. See https://golang.org/s/gogetcmd\n", - v.name) - return nil, err - } - - cmd := exec.Command(v.cmd, args...) - cmd.Dir = dir - cmd.Env = envForDir(cmd.Dir, os.Environ()) - if buildX { - fmt.Printf("cd %s\n", dir) - fmt.Printf("%s %s\n", v.cmd, strings.Join(args, " ")) - } - var buf bytes.Buffer - cmd.Stdout = &buf - cmd.Stderr = &buf - err = cmd.Run() - out := buf.Bytes() - if err != nil { - if verbose || buildV { - fmt.Fprintf(os.Stderr, "# cd %s; %s %s\n", dir, v.cmd, strings.Join(args, " ")) - os.Stderr.Write(out) - } - return out, err - } - return out, nil -} - -// ping pings to determine scheme to use. -func (v *vcsCmd) ping(scheme, repo string) error { - return v.runVerboseOnly(".", v.pingCmd, "scheme", scheme, "repo", repo) -} - -// create creates a new copy of repo in dir. -// The parent of dir must exist; dir must not. -func (v *vcsCmd) create(dir, repo string) error { - for _, cmd := range v.createCmd { - if err := v.run(".", cmd, "dir", dir, "repo", repo); err != nil { - return err - } - } - return nil -} - -// download downloads any new changes for the repo in dir. -func (v *vcsCmd) download(dir string) error { - for _, cmd := range v.downloadCmd { - if err := v.run(dir, cmd); err != nil { - return err - } - } - return nil -} - -// tags returns the list of available tags for the repo in dir. -func (v *vcsCmd) tags(dir string) ([]string, error) { - var tags []string - for _, tc := range v.tagCmd { - out, err := v.runOutput(dir, tc.cmd) - if err != nil { - return nil, err - } - re := regexp.MustCompile(`(?m-s)` + tc.pattern) - for _, m := range re.FindAllStringSubmatch(string(out), -1) { - tags = append(tags, m[1]) - } - } - return tags, nil -} - -// tagSync syncs the repo in dir to the named tag, -// which either is a tag returned by tags or is v.tagDefault. -func (v *vcsCmd) tagSync(dir, tag string) error { - if v.tagSyncCmd == nil { - return nil - } - if tag != "" { - for _, tc := range v.tagLookupCmd { - out, err := v.runOutput(dir, tc.cmd, "tag", tag) - if err != nil { - return err - } - re := regexp.MustCompile(`(?m-s)` + tc.pattern) - m := re.FindStringSubmatch(string(out)) - if len(m) > 1 { - tag = m[1] - break - } - } - } - - if tag == "" && v.tagSyncDefault != nil { - for _, cmd := range v.tagSyncDefault { - if err := v.run(dir, cmd); err != nil { - return err - } - } - return nil - } - - for _, cmd := range v.tagSyncCmd { - if err := v.run(dir, cmd, "tag", tag); err != nil { - return err - } - } - return nil -} - -// A vcsPath describes how to convert an import path into a -// version control system and repository name. -type vcsPath struct { - prefix string // prefix this description applies to - re string // pattern for import path - repo string // repository to use (expand with match of re) - vcs string // version control system to use (expand with match of re) - check func(match map[string]string) error // additional checks - ping bool // ping for scheme to use to download repo - - regexp *regexp.Regexp // cached compiled form of re -} - -// vcsFromDir inspects dir and its parents to determine the -// version control system and code repository to use. -// On return, root is the import path -// corresponding to the root of the repository. -func vcsFromDir(dir, srcRoot string) (vcs *vcsCmd, root string, err error) { - // Clean and double-check that dir is in (a subdirectory of) srcRoot. - dir = filepath.Clean(dir) - srcRoot = filepath.Clean(srcRoot) - if len(dir) <= len(srcRoot) || dir[len(srcRoot)] != filepath.Separator { - return nil, "", fmt.Errorf("directory %q is outside source root %q", dir, srcRoot) - } - - origDir := dir - for len(dir) > len(srcRoot) { - for _, vcs := range vcsList { - if _, err := os.Stat(filepath.Join(dir, "."+vcs.cmd)); err == nil { - return vcs, filepath.ToSlash(dir[len(srcRoot)+1:]), nil - } - } - - // Move to parent. - ndir := filepath.Dir(dir) - if len(ndir) >= len(dir) { - // Shouldn't happen, but just in case, stop. - break - } - dir = ndir - } - - return nil, "", fmt.Errorf("directory %q is not using a known version control system", origDir) -} - -// repoRoot represents a version control system, a repo, and a root of -// where to put it on disk. -type repoRoot struct { - vcs *vcsCmd - - // repo is the repository URL, including scheme - repo string - - // root is the import path corresponding to the root of the - // repository - root string - - // isCustom is true for custom import paths (those defined by HTML meta tags) - isCustom bool -} - -var httpPrefixRE = regexp.MustCompile(`^https?:`) - -// securityMode specifies whether a function should make network -// calls using insecure transports (eg, plain text HTTP). -// The zero value is "secure". -type securityMode int - -const ( - secure securityMode = iota - insecure -) - -// repoRootForImportPath analyzes importPath to determine the -// version control system, and code repository to use. -func repoRootForImportPath(importPath string, security securityMode) (*repoRoot, error) { - rr, err := repoRootFromVCSPaths(importPath, "", security, vcsPaths) - if err == errUnknownSite { - // If there are wildcards, look up the thing before the wildcard, - // hoping it applies to the wildcarded parts too. - // This makes 'go get rsc.io/pdf/...' work in a fresh GOPATH. - lookup := strings.TrimSuffix(importPath, "/...") - if i := strings.Index(lookup, "/.../"); i >= 0 { - lookup = lookup[:i] - } - rr, err = repoRootForImportDynamic(lookup, security) - if err != nil { - err = fmt.Errorf("unrecognized import path %q (%v)", importPath, err) - } - } - if err != nil { - rr1, err1 := repoRootFromVCSPaths(importPath, "", security, vcsPathsAfterDynamic) - if err1 == nil { - rr = rr1 - err = nil - } - } - - if err == nil && strings.Contains(importPath, "...") && strings.Contains(rr.root, "...") { - // Do not allow wildcards in the repo root. - rr = nil - err = fmt.Errorf("cannot expand ... in %q", importPath) - } - return rr, err -} - -var errUnknownSite = errors.New("dynamic lookup required to find mapping") - -// repoRootFromVCSPaths attempts to map importPath to a repoRoot -// using the mappings defined in vcsPaths. -// If scheme is non-empty, that scheme is forced. -func repoRootFromVCSPaths(importPath, scheme string, security securityMode, vcsPaths []*vcsPath) (*repoRoot, error) { - // A common error is to use https://packagepath because that's what - // hg and git require. Diagnose this helpfully. - if loc := httpPrefixRE.FindStringIndex(importPath); loc != nil { - // The importPath has been cleaned, so has only one slash. The pattern - // ignores the slashes; the error message puts them back on the RHS at least. - return nil, fmt.Errorf("%q not allowed in import path", importPath[loc[0]:loc[1]]+"//") - } - for _, srv := range vcsPaths { - if !strings.HasPrefix(importPath, srv.prefix) { - continue - } - m := srv.regexp.FindStringSubmatch(importPath) - if m == nil { - if srv.prefix != "" { - return nil, fmt.Errorf("invalid %s import path %q", srv.prefix, importPath) - } - continue - } - - // Build map of named subexpression matches for expand. - match := map[string]string{ - "prefix": srv.prefix, - "import": importPath, - } - for i, name := range srv.regexp.SubexpNames() { - if name != "" && match[name] == "" { - match[name] = m[i] - } - } - if srv.vcs != "" { - match["vcs"] = expand(match, srv.vcs) - } - if srv.repo != "" { - match["repo"] = expand(match, srv.repo) - } - if srv.check != nil { - if err := srv.check(match); err != nil { - return nil, err - } - } - vcs := vcsByCmd(match["vcs"]) - if vcs == nil { - return nil, fmt.Errorf("unknown version control system %q", match["vcs"]) - } - if srv.ping { - if scheme != "" { - match["repo"] = scheme + "://" + match["repo"] - } else { - for _, scheme := range vcs.scheme { - if security == secure && !vcs.isSecureScheme(scheme) { - continue - } - if vcs.ping(scheme, match["repo"]) == nil { - match["repo"] = scheme + "://" + match["repo"] - break - } - } - } - } - rr := &repoRoot{ - vcs: vcs, - repo: match["repo"], - root: match["root"], - } - return rr, nil - } - return nil, errUnknownSite -} - -// repoRootForImportDynamic finds a *repoRoot for a custom domain that's not -// statically known by repoRootForImportPathStatic. -// -// This handles custom import paths like "name.tld/pkg/foo" or just "name.tld". -func repoRootForImportDynamic(importPath string, security securityMode) (*repoRoot, error) { - slash := strings.Index(importPath, "/") - if slash < 0 { - slash = len(importPath) - } - host := importPath[:slash] - if !strings.Contains(host, ".") { - return nil, errors.New("import path does not begin with hostname") - } - urlStr, body, err := httpsOrHTTP(importPath, security) - if err != nil { - msg := "https fetch: %v" - if security == insecure { - msg = "http/" + msg - } - return nil, fmt.Errorf(msg, err) - } - defer body.Close() - imports, err := parseMetaGoImports(body) - if err != nil { - return nil, fmt.Errorf("parsing %s: %v", importPath, err) - } - // Find the matched meta import. - mmi, err := matchGoImport(imports, importPath) - if err != nil { - if _, ok := err.(ImportMismatchError); !ok { - return nil, fmt.Errorf("parse %s: %v", urlStr, err) - } - return nil, fmt.Errorf("parse %s: no go-import meta tags (%s)", urlStr, err) - } - if buildV { - log.Printf("get %q: found meta tag %#v at %s", importPath, mmi, urlStr) - } - // If the import was "uni.edu/bob/project", which said the - // prefix was "uni.edu" and the RepoRoot was "evilroot.com", - // make sure we don't trust Bob and check out evilroot.com to - // "uni.edu" yet (possibly overwriting/preempting another - // non-evil student). Instead, first verify the root and see - // if it matches Bob's claim. - if mmi.Prefix != importPath { - if buildV { - log.Printf("get %q: verifying non-authoritative meta tag", importPath) - } - urlStr0 := urlStr - var imports []metaImport - urlStr, imports, err = metaImportsForPrefix(mmi.Prefix, security) - if err != nil { - return nil, err - } - metaImport2, err := matchGoImport(imports, importPath) - if err != nil || mmi != metaImport2 { - return nil, fmt.Errorf("%s and %s disagree about go-import for %s", urlStr0, urlStr, mmi.Prefix) - } - } - - if !strings.Contains(mmi.RepoRoot, "://") { - return nil, fmt.Errorf("%s: invalid repo root %q; no scheme", urlStr, mmi.RepoRoot) - } - rr := &repoRoot{ - vcs: vcsByCmd(mmi.VCS), - repo: mmi.RepoRoot, - root: mmi.Prefix, - isCustom: true, - } - if rr.vcs == nil { - return nil, fmt.Errorf("%s: unknown vcs %q", urlStr, mmi.VCS) - } - return rr, nil -} - -var fetchGroup singleflight.Group -var ( - fetchCacheMu sync.Mutex - fetchCache = map[string]fetchResult{} // key is metaImportsForPrefix's importPrefix -) - -// metaImportsForPrefix takes a package's root import path as declared in a tag -// and returns its HTML discovery URL and the parsed metaImport lines -// found on the page. -// -// The importPath is of the form "golang.org/x/tools". -// It is an error if no imports are found. -// urlStr will still be valid if err != nil. -// The returned urlStr will be of the form "https://golang.org/x/tools?go-get=1" -func metaImportsForPrefix(importPrefix string, security securityMode) (urlStr string, imports []metaImport, err error) { - setCache := func(res fetchResult) (fetchResult, error) { - fetchCacheMu.Lock() - defer fetchCacheMu.Unlock() - fetchCache[importPrefix] = res - return res, nil - } - - resi, _, _ := fetchGroup.Do(importPrefix, func() (resi interface{}, err error) { - fetchCacheMu.Lock() - if res, ok := fetchCache[importPrefix]; ok { - fetchCacheMu.Unlock() - return res, nil - } - fetchCacheMu.Unlock() - - urlStr, body, err := httpsOrHTTP(importPrefix, security) - if err != nil { - return setCache(fetchResult{urlStr: urlStr, err: fmt.Errorf("fetch %s: %v", urlStr, err)}) - } - imports, err := parseMetaGoImports(body) - if err != nil { - return setCache(fetchResult{urlStr: urlStr, err: fmt.Errorf("parsing %s: %v", urlStr, err)}) - } - if len(imports) == 0 { - err = fmt.Errorf("fetch %s: no go-import meta tag", urlStr) - } - return setCache(fetchResult{urlStr: urlStr, imports: imports, err: err}) - }) - res := resi.(fetchResult) - return res.urlStr, res.imports, res.err -} - -type fetchResult struct { - urlStr string // e.g. "https://foo.com/x/bar?go-get=1" - imports []metaImport - err error -} - -// metaImport represents the parsed tags from HTML files. -type metaImport struct { - Prefix, VCS, RepoRoot string -} - -func splitPathHasPrefix(path, prefix []string) bool { - if len(path) < len(prefix) { - return false - } - for i, p := range prefix { - if path[i] != p { - return false - } - } - return true -} - -// A ImportMismatchError is returned where metaImport/s are present -// but none match our import path. -type ImportMismatchError struct { - importPath string - mismatches []string // the meta imports that were discarded for not matching our importPath -} - -func (m ImportMismatchError) Error() string { - formattedStrings := make([]string, len(m.mismatches)) - for i, pre := range m.mismatches { - formattedStrings[i] = fmt.Sprintf("meta tag %s did not match import path %s", pre, m.importPath) - } - return strings.Join(formattedStrings, ", ") -} - -// matchGoImport returns the metaImport from imports matching importPath. -// An error is returned if there are multiple matches. -// errNoMatch is returned if none match. -func matchGoImport(imports []metaImport, importPath string) (metaImport, error) { - match := -1 - imp := strings.Split(importPath, "/") - - errImportMismatch := ImportMismatchError{importPath: importPath} - for i, im := range imports { - pre := strings.Split(im.Prefix, "/") - - if !splitPathHasPrefix(imp, pre) { - errImportMismatch.mismatches = append(errImportMismatch.mismatches, im.Prefix) - continue - } - - if match != -1 { - return metaImport{}, fmt.Errorf("multiple meta tags match import path %q", importPath) - } - match = i - } - - if match == -1 { - return metaImport{}, errImportMismatch - } - return imports[match], nil -} - -// expand rewrites s to replace {k} with match[k] for each key k in match. -func expand(match map[string]string, s string) string { - for k, v := range match { - s = strings.Replace(s, "{"+k+"}", v, -1) - } - return s -} - -// vcsPaths defines the meaning of import paths referring to -// commonly-used VCS hosting sites (github.com/user/dir) -// and import paths referring to a fully-qualified importPath -// containing a VCS type (foo.com/repo.git/dir) -var vcsPaths = []*vcsPath{ - // Github - { - prefix: "github.com/", - re: `^(?Pgithub\.com/[A-Za-z0-9_.\-]+/[A-Za-z0-9_.\-]+)(/[A-Za-z0-9_.\-]+)*$`, - vcs: "git", - repo: "https://{root}", - check: noVCSSuffix, - }, - - // Bitbucket - { - prefix: "bitbucket.org/", - re: `^(?Pbitbucket\.org/(?P[A-Za-z0-9_.\-]+/[A-Za-z0-9_.\-]+))(/[A-Za-z0-9_.\-]+)*$`, - repo: "https://{root}", - check: bitbucketVCS, - }, - - // IBM DevOps Services (JazzHub) - { - prefix: "hub.jazz.net/git", - re: `^(?Phub.jazz.net/git/[a-z0-9]+/[A-Za-z0-9_.\-]+)(/[A-Za-z0-9_.\-]+)*$`, - vcs: "git", - repo: "https://{root}", - check: noVCSSuffix, - }, - - // Git at Apache - { - prefix: "git.apache.org", - re: `^(?Pgit.apache.org/[a-z0-9_.\-]+\.git)(/[A-Za-z0-9_.\-]+)*$`, - vcs: "git", - repo: "https://{root}", - }, - - // Git at OpenStack - { - prefix: "git.openstack.org", - re: `^(?Pgit\.openstack\.org/[A-Za-z0-9_.\-]+/[A-Za-z0-9_.\-]+)(\.git)?(/[A-Za-z0-9_.\-]+)*$`, - vcs: "git", - repo: "https://{root}", - }, - - // General syntax for any server. - // Must be last. - { - re: `^(?P(?P([a-z0-9.\-]+\.)+[a-z0-9.\-]+(:[0-9]+)?(/~?[A-Za-z0-9_.\-]+)+?)\.(?Pbzr|git|hg|svn))(/~?[A-Za-z0-9_.\-]+)*$`, - ping: true, - }, -} - -// vcsPathsAfterDynamic gives additional vcsPaths entries -// to try after the dynamic HTML check. -// This gives those sites a chance to introduce tags -// as part of a graceful transition away from the hard-coded logic. -var vcsPathsAfterDynamic = []*vcsPath{ - // Launchpad. See golang.org/issue/11436. - { - prefix: "launchpad.net/", - re: `^(?Plaunchpad\.net/((?P[A-Za-z0-9_.\-]+)(?P/[A-Za-z0-9_.\-]+)?|~[A-Za-z0-9_.\-]+/(\+junk|[A-Za-z0-9_.\-]+)/[A-Za-z0-9_.\-]+))(/[A-Za-z0-9_.\-]+)*$`, - vcs: "bzr", - repo: "https://{root}", - check: launchpadVCS, - }, -} - -func init() { - // fill in cached regexps. - // Doing this eagerly discovers invalid regexp syntax - // without having to run a command that needs that regexp. - for _, srv := range vcsPaths { - srv.regexp = regexp.MustCompile(srv.re) - } - for _, srv := range vcsPathsAfterDynamic { - srv.regexp = regexp.MustCompile(srv.re) - } -} - -// noVCSSuffix checks that the repository name does not -// end in .foo for any version control system foo. -// The usual culprit is ".git". -func noVCSSuffix(match map[string]string) error { - repo := match["repo"] - for _, vcs := range vcsList { - if strings.HasSuffix(repo, "."+vcs.cmd) { - return fmt.Errorf("invalid version control suffix in %s path", match["prefix"]) - } - } - return nil -} - -// bitbucketVCS determines the version control system for a -// Bitbucket repository, by using the Bitbucket API. -func bitbucketVCS(match map[string]string) error { - if err := noVCSSuffix(match); err != nil { - return err - } - - var resp struct { - SCM string `json:"scm"` - } - url := expand(match, "https://api.bitbucket.org/1.0/repositories/{bitname}") - data, err := httpGET(url) - if err != nil { - if httpErr, ok := err.(*httpError); ok && httpErr.statusCode == 403 { - // this may be a private repository. If so, attempt to determine which - // VCS it uses. See issue 5375. - root := match["root"] - for _, vcs := range []string{"git", "hg"} { - if vcsByCmd(vcs).ping("https", root) == nil { - resp.SCM = vcs - break - } - } - } - - if resp.SCM == "" { - return err - } - } else { - if err := json.Unmarshal(data, &resp); err != nil { - return fmt.Errorf("decoding %s: %v", url, err) - } - } - - if vcsByCmd(resp.SCM) != nil { - match["vcs"] = resp.SCM - if resp.SCM == "git" { - match["repo"] += ".git" - } - return nil - } - - return fmt.Errorf("unable to detect version control system for bitbucket.org/ path") -} - -// launchpadVCS solves the ambiguity for "lp.net/project/foo". In this case, -// "foo" could be a series name registered in Launchpad with its own branch, -// and it could also be the name of a directory within the main project -// branch one level up. -func launchpadVCS(match map[string]string) error { - if match["project"] == "" || match["series"] == "" { - return nil - } - _, err := httpGET(expand(match, "https://code.launchpad.net/{project}{series}/.bzr/branch-format")) - if err != nil { - match["root"] = expand(match, "launchpad.net/{project}") - match["repo"] = expand(match, "https://{root}") - } - return nil -} diff --git a/libgo/go/cmd/go/vcs_test.go b/libgo/go/cmd/go/vcs_test.go deleted file mode 100644 index c73f5d0e85b..00000000000 --- a/libgo/go/cmd/go/vcs_test.go +++ /dev/null @@ -1,382 +0,0 @@ -// Copyright 2014 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package main - -import ( - "errors" - "internal/testenv" - "io/ioutil" - "os" - "path" - "path/filepath" - "testing" -) - -// Test that RepoRootForImportPath creates the correct RepoRoot for a given importPath. -// TODO(cmang): Add tests for SVN and BZR. -func TestRepoRootForImportPath(t *testing.T) { - testenv.MustHaveExternalNetwork(t) - - tests := []struct { - path string - want *repoRoot - }{ - { - "github.com/golang/groupcache", - &repoRoot{ - vcs: vcsGit, - repo: "https://github.com/golang/groupcache", - }, - }, - // IBM DevOps Services tests - { - "hub.jazz.net/git/user1/pkgname", - &repoRoot{ - vcs: vcsGit, - repo: "https://hub.jazz.net/git/user1/pkgname", - }, - }, - { - "hub.jazz.net/git/user1/pkgname/submodule/submodule/submodule", - &repoRoot{ - vcs: vcsGit, - repo: "https://hub.jazz.net/git/user1/pkgname", - }, - }, - { - "hub.jazz.net", - nil, - }, - { - "hub2.jazz.net", - nil, - }, - { - "hub.jazz.net/someotherprefix", - nil, - }, - { - "hub.jazz.net/someotherprefix/user1/pkgname", - nil, - }, - // Spaces are not valid in user names or package names - { - "hub.jazz.net/git/User 1/pkgname", - nil, - }, - { - "hub.jazz.net/git/user1/pkg name", - nil, - }, - // Dots are not valid in user names - { - "hub.jazz.net/git/user.1/pkgname", - nil, - }, - { - "hub.jazz.net/git/user/pkg.name", - &repoRoot{ - vcs: vcsGit, - repo: "https://hub.jazz.net/git/user/pkg.name", - }, - }, - // User names cannot have uppercase letters - { - "hub.jazz.net/git/USER/pkgname", - nil, - }, - // OpenStack tests - { - "git.openstack.org/openstack/swift", - &repoRoot{ - vcs: vcsGit, - repo: "https://git.openstack.org/openstack/swift", - }, - }, - // Trailing .git is less preferred but included for - // compatibility purposes while the same source needs to - // be compilable on both old and new go - { - "git.openstack.org/openstack/swift.git", - &repoRoot{ - vcs: vcsGit, - repo: "https://git.openstack.org/openstack/swift.git", - }, - }, - { - "git.openstack.org/openstack/swift/go/hummingbird", - &repoRoot{ - vcs: vcsGit, - repo: "https://git.openstack.org/openstack/swift", - }, - }, - { - "git.openstack.org", - nil, - }, - { - "git.openstack.org/openstack", - nil, - }, - // Spaces are not valid in package name - { - "git.apache.org/package name/path/to/lib", - nil, - }, - // Should have ".git" suffix - { - "git.apache.org/package-name/path/to/lib", - nil, - }, - { - "git.apache.org/package-name.git", - &repoRoot{ - vcs: vcsGit, - repo: "https://git.apache.org/package-name.git", - }, - }, - { - "git.apache.org/package-name_2.x.git/path/to/lib", - &repoRoot{ - vcs: vcsGit, - repo: "https://git.apache.org/package-name_2.x.git", - }, - }, - } - - for _, test := range tests { - got, err := repoRootForImportPath(test.path, secure) - want := test.want - - if want == nil { - if err == nil { - t.Errorf("RepoRootForImport(%q): Error expected but not received", test.path) - } - continue - } - if err != nil { - t.Errorf("RepoRootForImport(%q): %v", test.path, err) - continue - } - if got.vcs.name != want.vcs.name || got.repo != want.repo { - t.Errorf("RepoRootForImport(%q) = VCS(%s) Repo(%s), want VCS(%s) Repo(%s)", test.path, got.vcs, got.repo, want.vcs, want.repo) - } - } -} - -// Test that vcsFromDir correctly inspects a given directory and returns the right VCS and root. -func TestFromDir(t *testing.T) { - tempDir, err := ioutil.TempDir("", "vcstest") - if err != nil { - t.Fatal(err) - } - defer os.RemoveAll(tempDir) - - for j, vcs := range vcsList { - dir := filepath.Join(tempDir, "example.com", vcs.name, "."+vcs.cmd) - if j&1 == 0 { - err := os.MkdirAll(dir, 0755) - if err != nil { - t.Fatal(err) - } - } else { - err := os.MkdirAll(filepath.Dir(dir), 0755) - if err != nil { - t.Fatal(err) - } - f, err := os.Create(dir) - if err != nil { - t.Fatal(err) - } - f.Close() - } - - want := repoRoot{ - vcs: vcs, - root: path.Join("example.com", vcs.name), - } - var got repoRoot - got.vcs, got.root, err = vcsFromDir(dir, tempDir) - if err != nil { - t.Errorf("FromDir(%q, %q): %v", dir, tempDir, err) - continue - } - if got.vcs.name != want.vcs.name || got.root != want.root { - t.Errorf("FromDir(%q, %q) = VCS(%s) Root(%s), want VCS(%s) Root(%s)", dir, tempDir, got.vcs, got.root, want.vcs, want.root) - } - } -} - -func TestIsSecure(t *testing.T) { - tests := []struct { - vcs *vcsCmd - url string - secure bool - }{ - {vcsGit, "http://example.com/foo.git", false}, - {vcsGit, "https://example.com/foo.git", true}, - {vcsBzr, "http://example.com/foo.bzr", false}, - {vcsBzr, "https://example.com/foo.bzr", true}, - {vcsSvn, "http://example.com/svn", false}, - {vcsSvn, "https://example.com/svn", true}, - {vcsHg, "http://example.com/foo.hg", false}, - {vcsHg, "https://example.com/foo.hg", true}, - {vcsGit, "ssh://user@example.com/foo.git", true}, - {vcsGit, "user@server:path/to/repo.git", false}, - {vcsGit, "user@server:", false}, - {vcsGit, "server:repo.git", false}, - {vcsGit, "server:path/to/repo.git", false}, - {vcsGit, "example.com:path/to/repo.git", false}, - {vcsGit, "path/that/contains/a:colon/repo.git", false}, - {vcsHg, "ssh://user@example.com/path/to/repo.hg", true}, - } - - for _, test := range tests { - secure := test.vcs.isSecure(test.url) - if secure != test.secure { - t.Errorf("%s isSecure(%q) = %t; want %t", test.vcs, test.url, secure, test.secure) - } - } -} - -func TestIsSecureGitAllowProtocol(t *testing.T) { - tests := []struct { - vcs *vcsCmd - url string - secure bool - }{ - // Same as TestIsSecure to verify same behavior. - {vcsGit, "http://example.com/foo.git", false}, - {vcsGit, "https://example.com/foo.git", true}, - {vcsBzr, "http://example.com/foo.bzr", false}, - {vcsBzr, "https://example.com/foo.bzr", true}, - {vcsSvn, "http://example.com/svn", false}, - {vcsSvn, "https://example.com/svn", true}, - {vcsHg, "http://example.com/foo.hg", false}, - {vcsHg, "https://example.com/foo.hg", true}, - {vcsGit, "user@server:path/to/repo.git", false}, - {vcsGit, "user@server:", false}, - {vcsGit, "server:repo.git", false}, - {vcsGit, "server:path/to/repo.git", false}, - {vcsGit, "example.com:path/to/repo.git", false}, - {vcsGit, "path/that/contains/a:colon/repo.git", false}, - {vcsHg, "ssh://user@example.com/path/to/repo.hg", true}, - // New behavior. - {vcsGit, "ssh://user@example.com/foo.git", false}, - {vcsGit, "foo://example.com/bar.git", true}, - {vcsHg, "foo://example.com/bar.hg", false}, - {vcsSvn, "foo://example.com/svn", false}, - {vcsBzr, "foo://example.com/bar.bzr", false}, - } - - defer os.Unsetenv("GIT_ALLOW_PROTOCOL") - os.Setenv("GIT_ALLOW_PROTOCOL", "https:foo") - for _, test := range tests { - secure := test.vcs.isSecure(test.url) - if secure != test.secure { - t.Errorf("%s isSecure(%q) = %t; want %t", test.vcs, test.url, secure, test.secure) - } - } -} - -func TestMatchGoImport(t *testing.T) { - tests := []struct { - imports []metaImport - path string - mi metaImport - err error - }{ - { - imports: []metaImport{ - {Prefix: "example.com/user/foo", VCS: "git", RepoRoot: "https://example.com/repo/target"}, - }, - path: "example.com/user/foo", - mi: metaImport{Prefix: "example.com/user/foo", VCS: "git", RepoRoot: "https://example.com/repo/target"}, - }, - { - imports: []metaImport{ - {Prefix: "example.com/user/foo", VCS: "git", RepoRoot: "https://example.com/repo/target"}, - }, - path: "example.com/user/foo/", - mi: metaImport{Prefix: "example.com/user/foo", VCS: "git", RepoRoot: "https://example.com/repo/target"}, - }, - { - imports: []metaImport{ - {Prefix: "example.com/user/foo", VCS: "git", RepoRoot: "https://example.com/repo/target"}, - {Prefix: "example.com/user/fooa", VCS: "git", RepoRoot: "https://example.com/repo/target"}, - }, - path: "example.com/user/foo", - mi: metaImport{Prefix: "example.com/user/foo", VCS: "git", RepoRoot: "https://example.com/repo/target"}, - }, - { - imports: []metaImport{ - {Prefix: "example.com/user/foo", VCS: "git", RepoRoot: "https://example.com/repo/target"}, - {Prefix: "example.com/user/fooa", VCS: "git", RepoRoot: "https://example.com/repo/target"}, - }, - path: "example.com/user/fooa", - mi: metaImport{Prefix: "example.com/user/fooa", VCS: "git", RepoRoot: "https://example.com/repo/target"}, - }, - { - imports: []metaImport{ - {Prefix: "example.com/user/foo", VCS: "git", RepoRoot: "https://example.com/repo/target"}, - {Prefix: "example.com/user/foo/bar", VCS: "git", RepoRoot: "https://example.com/repo/target"}, - }, - path: "example.com/user/foo/bar", - err: errors.New("should not be allowed to create nested repo"), - }, - { - imports: []metaImport{ - {Prefix: "example.com/user/foo", VCS: "git", RepoRoot: "https://example.com/repo/target"}, - {Prefix: "example.com/user/foo/bar", VCS: "git", RepoRoot: "https://example.com/repo/target"}, - }, - path: "example.com/user/foo/bar/baz", - err: errors.New("should not be allowed to create nested repo"), - }, - { - imports: []metaImport{ - {Prefix: "example.com/user/foo", VCS: "git", RepoRoot: "https://example.com/repo/target"}, - {Prefix: "example.com/user/foo/bar", VCS: "git", RepoRoot: "https://example.com/repo/target"}, - }, - path: "example.com/user/foo/bar/baz/qux", - err: errors.New("should not be allowed to create nested repo"), - }, - { - imports: []metaImport{ - {Prefix: "example.com/user/foo", VCS: "git", RepoRoot: "https://example.com/repo/target"}, - {Prefix: "example.com/user/foo/bar", VCS: "git", RepoRoot: "https://example.com/repo/target"}, - }, - path: "example.com/user/foo/bar/baz/", - err: errors.New("should not be allowed to create nested repo"), - }, - { - imports: []metaImport{ - {Prefix: "example.com/user/foo", VCS: "git", RepoRoot: "https://example.com/repo/target"}, - {Prefix: "example.com/user/foo/bar", VCS: "git", RepoRoot: "https://example.com/repo/target"}, - }, - path: "example.com", - err: errors.New("pathologically short path"), - }, - { - imports: []metaImport{ - {Prefix: "example.com/user/foo", VCS: "git", RepoRoot: "https://example.com/repo/target"}, - }, - path: "different.example.com/user/foo", - err: errors.New("meta tags do not match import path"), - }, - } - - for _, test := range tests { - mi, err := matchGoImport(test.imports, test.path) - if mi != test.mi { - t.Errorf("unexpected metaImport; got %v, want %v", mi, test.mi) - } - - got := err - want := test.err - if (got == nil) != (want == nil) { - t.Errorf("unexpected error; got %v, want %v", got, want) - } - } -} diff --git a/libgo/go/cmd/go/vendor_test.go b/libgo/go/cmd/go/vendor_test.go index deec02e3413..739ce5a5a45 100644 --- a/libgo/go/cmd/go/vendor_test.go +++ b/libgo/go/cmd/go/vendor_test.go @@ -20,18 +20,18 @@ func TestVendorImports(t *testing.T) { tg := testgo(t) defer tg.cleanup() tg.setenv("GOPATH", filepath.Join(tg.pwd(), "testdata")) - tg.run("list", "-f", "{{.ImportPath}} {{.Imports}}", "vend/...") + tg.run("list", "-f", "{{.ImportPath}} {{.Imports}}", "vend/...", "vend/vendor/...", "vend/x/vendor/...") want := ` vend [vend/vendor/p r] vend/dir1 [] vend/hello [fmt vend/vendor/strings] vend/subdir [vend/vendor/p r] + vend/x [vend/x/vendor/p vend/vendor/q vend/x/vendor/r vend/dir1 vend/vendor/vend/dir1/dir2] + vend/x/invalid [vend/x/invalid/vendor/foo] vend/vendor/p [] vend/vendor/q [] vend/vendor/strings [] vend/vendor/vend/dir1/dir2 [] - vend/x [vend/x/vendor/p vend/vendor/q vend/x/vendor/r vend/dir1 vend/vendor/vend/dir1/dir2] - vend/x/invalid [vend/x/invalid/vendor/foo] vend/x/vendor/p [] vend/x/vendor/p/p [notfound] vend/x/vendor/r [] diff --git a/libgo/go/cmd/go/version.go b/libgo/go/cmd/go/version.go deleted file mode 100644 index 3045f350d7d..00000000000 --- a/libgo/go/cmd/go/version.go +++ /dev/null @@ -1,25 +0,0 @@ -// Copyright 2011 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package main - -import ( - "fmt" - "runtime" -) - -var cmdVersion = &Command{ - Run: runVersion, - UsageLine: "version", - Short: "print Go version", - Long: `Version prints the Go version, as reported by runtime.Version.`, -} - -func runVersion(cmd *Command, args []string) { - if len(args) != 0 { - cmd.Usage() - } - - fmt.Printf("go version %s %s/%s\n", runtime.Version(), runtime.GOOS, runtime.GOARCH) -} diff --git a/libgo/go/cmd/go/vet.go b/libgo/go/cmd/go/vet.go deleted file mode 100644 index 8e296c8572e..00000000000 --- a/libgo/go/cmd/go/vet.go +++ /dev/null @@ -1,52 +0,0 @@ -// Copyright 2011 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package main - -import "path/filepath" - -func init() { - addBuildFlags(cmdVet) -} - -var cmdVet = &Command{ - Run: runVet, - UsageLine: "vet [-n] [-x] [build flags] [packages]", - Short: "run go tool vet on packages", - Long: ` -Vet runs the Go vet command on the packages named by the import paths. - -For more about vet, see 'go doc cmd/vet'. -For more about specifying packages, see 'go help packages'. - -To run the vet tool with specific options, run 'go tool vet'. - -The -n flag prints commands that would be executed. -The -x flag prints commands as they are executed. - -For more about build flags, see 'go help build'. - -See also: go fmt, go fix. - `, -} - -func runVet(cmd *Command, args []string) { - for _, p := range packages(args) { - // Vet expects to be given a set of files all from the same package. - // Run once for package p and once for package p_test. - if len(p.GoFiles)+len(p.CgoFiles)+len(p.TestGoFiles) > 0 { - runVetFiles(p, stringList(p.GoFiles, p.CgoFiles, p.TestGoFiles, p.SFiles)) - } - if len(p.XTestGoFiles) > 0 { - runVetFiles(p, stringList(p.XTestGoFiles)) - } - } -} - -func runVetFiles(p *Package, files []string) { - for i := range files { - files[i] = filepath.Join(p.Dir, files[i]) - } - run(buildToolExec, tool("vet"), relPaths(files)) -} diff --git a/libgo/go/cmd/gofmt/doc.go b/libgo/go/cmd/gofmt/doc.go index 8b22f03f653..e3406655941 100644 --- a/libgo/go/cmd/gofmt/doc.go +++ b/libgo/go/cmd/gofmt/doc.go @@ -4,7 +4,8 @@ /* Gofmt formats Go programs. -It uses tabs (width = 8) for indentation and blanks for alignment. +It uses tabs for indentation and blanks for alignment. +Alignment assumes that an editor is using a fixed-width font. Without an explicit path, it processes the standard input. Given a file, it operates on that file; given a directory, it operates on all .go files in @@ -100,4 +101,4 @@ package main // BUG(rsc): The implementation of -r is a bit slow. // BUG(gri): If -w fails, the restored original file may not have some of the -// original file attributes. +// original file attributes. diff --git a/libgo/go/cmd/gofmt/gofmt.go b/libgo/go/cmd/gofmt/gofmt.go index e1ef0ddb837..d5b7be327a5 100644 --- a/libgo/go/cmd/gofmt/gofmt.go +++ b/libgo/go/cmd/gofmt/gofmt.go @@ -139,11 +139,11 @@ func processFile(filename string, in io.Reader, out io.Writer, stdin bool) error } } if *doDiff { - data, err := diff(src, res) + data, err := diff(src, res, filename) if err != nil { return fmt.Errorf("computing diff: %s", err) } - fmt.Printf("diff %s gofmt/%s\n", filename, filename) + fmt.Printf("diff -u %s %s\n", filepath.ToSlash(filename+".orig"), filepath.ToSlash(filename)) out.Write(data) } } @@ -225,32 +225,76 @@ func gofmtMain() { } } -func diff(b1, b2 []byte) (data []byte, err error) { - f1, err := ioutil.TempFile("", "gofmt") +func writeTempFile(dir, prefix string, data []byte) (string, error) { + file, err := ioutil.TempFile(dir, prefix) + if err != nil { + return "", err + } + _, err = file.Write(data) + if err1 := file.Close(); err == nil { + err = err1 + } + if err != nil { + os.Remove(file.Name()) + return "", err + } + return file.Name(), nil +} + +func diff(b1, b2 []byte, filename string) (data []byte, err error) { + f1, err := writeTempFile("", "gofmt", b1) if err != nil { return } - defer os.Remove(f1.Name()) - defer f1.Close() + defer os.Remove(f1) - f2, err := ioutil.TempFile("", "gofmt") + f2, err := writeTempFile("", "gofmt", b2) if err != nil { return } - defer os.Remove(f2.Name()) - defer f2.Close() + defer os.Remove(f2) - f1.Write(b1) - f2.Write(b2) + cmd := "diff" + if runtime.GOOS == "plan9" { + cmd = "/bin/ape/diff" + } - data, err = exec.Command("diff", "-u", f1.Name(), f2.Name()).CombinedOutput() + data, err = exec.Command(cmd, "-u", f1, f2).CombinedOutput() if len(data) > 0 { // diff exits with a non-zero status when the files don't match. // Ignore that failure as long as we get output. - err = nil + return replaceTempFilename(data, filename) } return +} +// replaceTempFilename replaces temporary filenames in diff with actual one. +// +// --- /tmp/gofmt316145376 2017-02-03 19:13:00.280468375 -0500 +// +++ /tmp/gofmt617882815 2017-02-03 19:13:00.280468375 -0500 +// ... +// -> +// --- path/to/file.go.orig 2017-02-03 19:13:00.280468375 -0500 +// +++ path/to/file.go 2017-02-03 19:13:00.280468375 -0500 +// ... +func replaceTempFilename(diff []byte, filename string) ([]byte, error) { + bs := bytes.SplitN(diff, []byte{'\n'}, 3) + if len(bs) < 3 { + return nil, fmt.Errorf("got unexpected diff for %s", filename) + } + // Preserve timestamps. + var t0, t1 []byte + if i := bytes.LastIndexByte(bs[0], '\t'); i != -1 { + t0 = bs[0][i:] + } + if i := bytes.LastIndexByte(bs[1], '\t'); i != -1 { + t1 = bs[1][i:] + } + // Always print filepath with slash separator. + f := filepath.ToSlash(filename) + bs[0] = []byte(fmt.Sprintf("--- %s%s", f+".orig", t0)) + bs[1] = []byte(fmt.Sprintf("+++ %s%s", f, t1)) + return bytes.Join(bs, []byte{'\n'}), nil } const chmodSupported = runtime.GOOS != "windows" diff --git a/libgo/go/cmd/gofmt/gofmt_test.go b/libgo/go/cmd/gofmt/gofmt_test.go index b7ca9e8d119..16b653b6460 100644 --- a/libgo/go/cmd/gofmt/gofmt_test.go +++ b/libgo/go/cmd/gofmt/gofmt_test.go @@ -9,7 +9,9 @@ import ( "flag" "io/ioutil" "os" + "os/exec" "path/filepath" + "runtime" "strings" "testing" "text/scanner" @@ -110,7 +112,7 @@ func runTest(t *testing.T, in, out string) { } t.Errorf("(gofmt %s) != %s (see %s.gofmt)", in, out, in) - d, err := diff(expected, got) + d, err := diff(expected, got, in) if err == nil { t.Errorf("%s", d) } @@ -184,3 +186,69 @@ func TestBackupFile(t *testing.T) { } t.Logf("Created: %s", name) } + +func TestDiff(t *testing.T) { + if _, err := exec.LookPath("diff"); err != nil { + t.Skipf("skip test on %s: diff command is required", runtime.GOOS) + } + in := []byte("first\nsecond\n") + out := []byte("first\nthird\n") + filename := "difftest.txt" + b, err := diff(in, out, filename) + if err != nil { + t.Fatal(err) + } + + if runtime.GOOS == "windows" { + b = bytes.Replace(b, []byte{'\r', '\n'}, []byte{'\n'}, -1) + } + + bs := bytes.SplitN(b, []byte{'\n'}, 3) + line0, line1 := bs[0], bs[1] + + if prefix := "--- difftest.txt.orig"; !bytes.HasPrefix(line0, []byte(prefix)) { + t.Errorf("diff: first line should start with `%s`\ngot: %s", prefix, line0) + } + + if prefix := "+++ difftest.txt"; !bytes.HasPrefix(line1, []byte(prefix)) { + t.Errorf("diff: second line should start with `%s`\ngot: %s", prefix, line1) + } + + want := `@@ -1,2 +1,2 @@ + first +-second ++third +` + + if got := string(bs[2]); got != want { + t.Errorf("diff: got:\n%s\nwant:\n%s", got, want) + } +} + +func TestReplaceTempFilename(t *testing.T) { + diff := []byte(`--- /tmp/tmpfile1 2017-02-08 00:53:26.175105619 +0900 ++++ /tmp/tmpfile2 2017-02-08 00:53:38.415151275 +0900 +@@ -1,2 +1,2 @@ + first +-second ++third +`) + want := []byte(`--- path/to/file.go.orig 2017-02-08 00:53:26.175105619 +0900 ++++ path/to/file.go 2017-02-08 00:53:38.415151275 +0900 +@@ -1,2 +1,2 @@ + first +-second ++third +`) + // Check path in diff output is always slash regardless of the + // os.PathSeparator (`/` or `\`). + sep := string(os.PathSeparator) + filename := strings.Join([]string{"path", "to", "file.go"}, sep) + got, err := replaceTempFilename(diff, filename) + if err != nil { + t.Fatal(err) + } + if !bytes.Equal(got, want) { + t.Errorf("os.PathSeparator='%s': replacedDiff:\ngot:\n%s\nwant:\n%s", sep, got, want) + } +} diff --git a/libgo/go/cmd/gofmt/rewrite.go b/libgo/go/cmd/gofmt/rewrite.go index 550492bf29a..79b7858a5af 100644 --- a/libgo/go/cmd/gofmt/rewrite.go +++ b/libgo/go/cmd/gofmt/rewrite.go @@ -66,10 +66,10 @@ func rewriteFile(pattern, replace ast.Expr, p *ast.File) *ast.File { if !val.IsValid() { return reflect.Value{} } + val = apply(rewriteVal, val) for k := range m { delete(m, k) } - val = apply(rewriteVal, val) if match(m, pat, val) { val = subst(m, repl, reflect.ValueOf(val.Interface().(ast.Node).Pos())) } diff --git a/libgo/go/cmd/internal/browser/browser.go b/libgo/go/cmd/internal/browser/browser.go index 897086f4717..6867c85d232 100644 --- a/libgo/go/cmd/internal/browser/browser.go +++ b/libgo/go/cmd/internal/browser/browser.go @@ -9,6 +9,7 @@ import ( "os" "os/exec" "runtime" + "time" ) // Commands returns a list of possible commands to use to open a url. @@ -23,7 +24,10 @@ func Commands() [][]string { case "windows": cmds = append(cmds, []string{"cmd", "/c", "start"}) default: - cmds = append(cmds, []string{"xdg-open"}) + if os.Getenv("DISPLAY") != "" { + // xdg-open is only for use in a desktop environment. + cmds = append(cmds, []string{"xdg-open"}) + } } cmds = append(cmds, []string{"chrome"}, @@ -38,9 +42,26 @@ func Commands() [][]string { func Open(url string) bool { for _, args := range Commands() { cmd := exec.Command(args[0], append(args[1:], url)...) - if cmd.Start() == nil { + if cmd.Start() == nil && appearsSuccessful(cmd, 3*time.Second) { return true } } return false } + +// appearsSuccessful reports whether the command appears to have run successfully. +// If the command runs longer than the timeout, it's deemed successful. +// If the command runs within the timeout, it's deemed successful if it exited cleanly. +func appearsSuccessful(cmd *exec.Cmd, timeout time.Duration) bool { + errc := make(chan error, 1) + go func() { + errc <- cmd.Wait() + }() + + select { + case <-time.After(timeout): + return true + case err := <-errc: + return err == nil + } +} diff --git a/libgo/go/cmd/internal/objabi/autotype.go b/libgo/go/cmd/internal/objabi/autotype.go new file mode 100644 index 00000000000..17c42931310 --- /dev/null +++ b/libgo/go/cmd/internal/objabi/autotype.go @@ -0,0 +1,37 @@ +// Derived from Inferno utils/6l/l.h and related files. +// https://bitbucket.org/inferno-os/inferno-os/src/default/utils/6l/l.h +// +// Copyright © 1994-1999 Lucent Technologies Inc. All rights reserved. +// Portions Copyright © 1995-1997 C H Forsyth (forsyth@terzarima.net) +// Portions Copyright © 1997-1999 Vita Nuova Limited +// Portions Copyright © 2000-2007 Vita Nuova Holdings Limited (www.vitanuova.com) +// Portions Copyright © 2004,2006 Bruce Ellis +// Portions Copyright © 2005-2007 C H Forsyth (forsyth@terzarima.net) +// Revisions Copyright © 2000-2007 Lucent Technologies Inc. and others +// Portions Copyright © 2009 The Go Authors. All rights reserved. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +package objabi + +// Auto.name +const ( + A_AUTO = 1 + iota + A_PARAM +) diff --git a/libgo/go/cmd/internal/objabi/doc.go b/libgo/go/cmd/internal/objabi/doc.go new file mode 100644 index 00000000000..7bd5ff63e56 --- /dev/null +++ b/libgo/go/cmd/internal/objabi/doc.go @@ -0,0 +1,120 @@ +// Copyright 2013 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// NOTE: There are *three* independent implementations of this object +// file format in the Go source tree: +// +// - cmd/internal/goobj/read.go (used by cmd/addr2line, cmd/nm, cmd/objdump, cmd/pprof) +// - cmd/internal/obj/objfile.go (used by cmd/asm and cmd/compile) +// - cmd/link/internal/objfile.go (used by cmd/link) +// +// When changing the object file format, remember to change all three. + +// Originally, Go object files were Plan 9 object files, but no longer. +// Now they are more like standard object files, in that each symbol is defined +// by an associated memory image (bytes) and a list of relocations to apply +// during linking. We do not (yet?) use a standard file format, however. +// For now, the format is chosen to be as simple as possible to read and write. +// It may change for reasons of efficiency, or we may even switch to a +// standard file format if there are compelling benefits to doing so. +// See golang.org/s/go13linker for more background. +// +// The file format is: +// +// - magic header: "\x00\x00go19ld" +// - byte 1 - version number +// - sequence of strings giving dependencies (imported packages) +// - empty string (marks end of sequence) +// - sequence of symbol references used by the defined symbols +// - byte 0xff (marks end of sequence) +// - sequence of integer lengths: +// - total data length +// - total number of relocations +// - total number of pcdata +// - total number of automatics +// - total number of funcdata +// - total number of files +// - data, the content of the defined symbols +// - sequence of defined symbols +// - byte 0xff (marks end of sequence) +// - magic footer: "\xff\xffgo19ld" +// +// All integers are stored in a zigzag varint format. +// See golang.org/s/go12symtab for a definition. +// +// Data blocks and strings are both stored as an integer +// followed by that many bytes. +// +// A symbol reference is a string name followed by a version. +// +// A symbol points to other symbols using an index into the symbol +// reference sequence. Index 0 corresponds to a nil symbol pointer. +// In the symbol layout described below "symref index" stands for this +// index. +// +// Each symbol is laid out as the following fields: +// +// - byte 0xfe (sanity check for synchronization) +// - type [byte] +// - name & version [symref index] +// - flags [int] +// 1<<0 dupok +// 1<<1 local +// 1<<2 add to typelink table +// - size [int] +// - gotype [symref index] +// - p [data block] +// - nr [int] +// - r [nr relocations, sorted by off] +// +// If type == STEXT, there are a few more fields: +// +// - args [int] +// - locals [int] +// - nosplit [int] +// - flags [int] +// 1<<0 leaf +// 1<<1 C function +// 1<<2 function may call reflect.Type.Method +// 1<<3 function compiled with -shared +// - nlocal [int] +// - local [nlocal automatics] +// - pcln [pcln table] +// +// Each relocation has the encoding: +// +// - off [int] +// - siz [int] +// - type [int] +// - add [int] +// - sym [symref index] +// +// Each local has the encoding: +// +// - asym [symref index] +// - offset [int] +// - type [int] +// - gotype [symref index] +// +// The pcln table has the encoding: +// +// - pcsp [data block] +// - pcfile [data block] +// - pcline [data block] +// - pcinline [data block] +// - npcdata [int] +// - pcdata [npcdata data blocks] +// - nfuncdata [int] +// - funcdata [nfuncdata symref index] +// - funcdatasym [nfuncdata ints] +// - nfile [int] +// - file [nfile symref index] +// - ninlinedcall [int] +// - inlinedcall [ninlinedcall int symref int symref] +// +// The file layout and meaning of type integers are architecture-independent. +// +// TODO(rsc): The file format is good for a first pass but needs work. +// - There are SymID in the object file that should really just be strings. +package objabi diff --git a/libgo/go/cmd/internal/objabi/flag.go b/libgo/go/cmd/internal/objabi/flag.go new file mode 100644 index 00000000000..e349b413249 --- /dev/null +++ b/libgo/go/cmd/internal/objabi/flag.go @@ -0,0 +1,115 @@ +// Copyright 2015 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package objabi + +import ( + "flag" + "fmt" + "os" + "strconv" +) + +func Flagfn2(string, string, func(string, string)) { panic("flag") } + +func Flagcount(name, usage string, val *int) { + flag.Var((*count)(val), name, usage) +} + +func Flagint32(name, usage string, val *int32) { + flag.Var((*int32Value)(val), name, usage) +} + +func Flagint64(name, usage string, val *int64) { + flag.Int64Var(val, name, *val, usage) +} + +func Flagstr(name, usage string, val *string) { + flag.StringVar(val, name, *val, usage) +} + +func Flagfn0(name, usage string, f func()) { + flag.Var(fn0(f), name, usage) +} + +func Flagfn1(name, usage string, f func(string)) { + flag.Var(fn1(f), name, usage) +} + +func Flagprint(fd int) { + if fd == 1 { + flag.CommandLine.SetOutput(os.Stdout) + } + flag.PrintDefaults() +} + +func Flagparse(usage func()) { + flag.Usage = usage + flag.Parse() +} + +// count is a flag.Value that is like a flag.Bool and a flag.Int. +// If used as -name, it increments the count, but -name=x sets the count. +// Used for verbose flag -v. +type count int + +func (c *count) String() string { + return fmt.Sprint(int(*c)) +} + +func (c *count) Set(s string) error { + switch s { + case "true": + *c++ + case "false": + *c = 0 + default: + n, err := strconv.Atoi(s) + if err != nil { + return fmt.Errorf("invalid count %q", s) + } + *c = count(n) + } + return nil +} + +func (c *count) IsBoolFlag() bool { + return true +} + +type int32Value int32 + +func (i *int32Value) Set(s string) error { + v, err := strconv.ParseInt(s, 0, 64) + *i = int32Value(v) + return err +} + +func (i *int32Value) Get() interface{} { return int32(*i) } + +func (i *int32Value) String() string { return fmt.Sprint(*i) } + +type fn0 func() + +func (f fn0) Set(s string) error { + f() + return nil +} + +func (f fn0) Get() interface{} { return nil } + +func (f fn0) String() string { return "" } + +func (f fn0) IsBoolFlag() bool { + return true +} + +type fn1 func(string) + +func (f fn1) Set(s string) error { + f(s) + return nil +} + +func (f fn1) String() string { return "" } diff --git a/libgo/go/cmd/internal/objabi/funcdata.go b/libgo/go/cmd/internal/objabi/funcdata.go new file mode 100644 index 00000000000..80874edeb0f --- /dev/null +++ b/libgo/go/cmd/internal/objabi/funcdata.go @@ -0,0 +1,25 @@ +// Copyright 2013 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package objabi + +// This file defines the IDs for PCDATA and FUNCDATA instructions +// in Go binaries. +// +// These must agree with ../../../runtime/funcdata.h and +// ../../../runtime/symtab.go. + +const ( + PCDATA_StackMapIndex = 0 + PCDATA_InlTreeIndex = 1 + FUNCDATA_ArgsPointerMaps = 0 + FUNCDATA_LocalsPointerMaps = 1 + FUNCDATA_InlTree = 2 + + // ArgsSizeUnknown is set in Func.argsize to mark all functions + // whose argument size is unknown (C vararg functions, and + // assembly code without an explicit specification). + // This value is generated by the compiler, assembler, or linker. + ArgsSizeUnknown = -0x80000000 +) diff --git a/libgo/go/cmd/internal/objabi/head.go b/libgo/go/cmd/internal/objabi/head.go new file mode 100644 index 00000000000..ff19606cd2b --- /dev/null +++ b/libgo/go/cmd/internal/objabi/head.go @@ -0,0 +1,104 @@ +// Derived from Inferno utils/6l/l.h and related files. +// https://bitbucket.org/inferno-os/inferno-os/src/default/utils/6l/l.h +// +// Copyright © 1994-1999 Lucent Technologies Inc. All rights reserved. +// Portions Copyright © 1995-1997 C H Forsyth (forsyth@terzarima.net) +// Portions Copyright © 1997-1999 Vita Nuova Limited +// Portions Copyright © 2000-2007 Vita Nuova Holdings Limited (www.vitanuova.com) +// Portions Copyright © 2004,2006 Bruce Ellis +// Portions Copyright © 2005-2007 C H Forsyth (forsyth@terzarima.net) +// Revisions Copyright © 2000-2007 Lucent Technologies Inc. and others +// Portions Copyright © 2009 The Go Authors. All rights reserved. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +package objabi + +import "fmt" + +// HeadType is the executable header type. +type HeadType uint8 + +const ( + Hunknown HeadType = iota + Hdarwin + Hdragonfly + Hfreebsd + Hlinux + Hnacl + Hnetbsd + Hopenbsd + Hplan9 + Hsolaris + Hwindows +) + +func (h *HeadType) Set(s string) error { + switch s { + case "darwin": + *h = Hdarwin + case "dragonfly": + *h = Hdragonfly + case "freebsd": + *h = Hfreebsd + case "linux", "android": + *h = Hlinux + case "nacl": + *h = Hnacl + case "netbsd": + *h = Hnetbsd + case "openbsd": + *h = Hopenbsd + case "plan9": + *h = Hplan9 + case "solaris": + *h = Hsolaris + case "windows": + *h = Hwindows + default: + return fmt.Errorf("invalid headtype: %q", s) + } + return nil +} + +func (h *HeadType) String() string { + switch *h { + case Hdarwin: + return "darwin" + case Hdragonfly: + return "dragonfly" + case Hfreebsd: + return "freebsd" + case Hlinux: + return "linux" + case Hnacl: + return "nacl" + case Hnetbsd: + return "netbsd" + case Hopenbsd: + return "openbsd" + case Hplan9: + return "plan9" + case Hsolaris: + return "solaris" + case Hwindows: + return "windows" + } + return fmt.Sprintf("HeadType(%d)", *h) +} diff --git a/libgo/go/cmd/internal/objabi/line.go b/libgo/go/cmd/internal/objabi/line.go new file mode 100644 index 00000000000..ed509b70017 --- /dev/null +++ b/libgo/go/cmd/internal/objabi/line.go @@ -0,0 +1,82 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package objabi + +import ( + "os" + "path/filepath" +) + +// WorkingDir returns the current working directory +// (or "/???" if the directory cannot be identified), +// with "/" as separator. +func WorkingDir() string { + var path string + path, _ = os.Getwd() + if path == "" { + path = "/???" + } + return filepath.ToSlash(path) +} + +// AbsFile returns the absolute filename for file in the given directory. +// It also removes a leading pathPrefix, or else rewrites a leading $GOROOT +// prefix to the literal "$GOROOT". +// If the resulting path is the empty string, the result is "??". +func AbsFile(dir, file, pathPrefix string) string { + abs := file + if dir != "" && !filepath.IsAbs(file) { + abs = filepath.Join(dir, file) + } + + if pathPrefix != "" && hasPathPrefix(abs, pathPrefix) { + if abs == pathPrefix { + abs = "" + } else { + abs = abs[len(pathPrefix)+1:] + } + } else if hasPathPrefix(abs, GOROOT) { + abs = "$GOROOT" + abs[len(GOROOT):] + } + if abs == "" { + abs = "??" + } + + return filepath.Clean(abs) +} + +// Does s have t as a path prefix? +// That is, does s == t or does s begin with t followed by a slash? +// For portability, we allow ASCII case folding, so that hasPathPrefix("a/b/c", "A/B") is true. +// Similarly, we allow slash folding, so that hasPathPrefix("a/b/c", "a\\b") is true. +// We do not allow full Unicode case folding, for fear of causing more confusion +// or harm than good. (For an example of the kinds of things that can go wrong, +// see http://article.gmane.org/gmane.linux.kernel/1853266.) +func hasPathPrefix(s string, t string) bool { + if len(t) > len(s) { + return false + } + var i int + for i = 0; i < len(t); i++ { + cs := int(s[i]) + ct := int(t[i]) + if 'A' <= cs && cs <= 'Z' { + cs += 'a' - 'A' + } + if 'A' <= ct && ct <= 'Z' { + ct += 'a' - 'A' + } + if cs == '\\' { + cs = '/' + } + if ct == '\\' { + ct = '/' + } + if cs != ct { + return false + } + } + return i >= len(s) || s[i] == '/' || s[i] == '\\' +} diff --git a/libgo/go/cmd/internal/objabi/path.go b/libgo/go/cmd/internal/objabi/path.go new file mode 100644 index 00000000000..2a42179a368 --- /dev/null +++ b/libgo/go/cmd/internal/objabi/path.go @@ -0,0 +1,41 @@ +// Copyright 2017 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package objabi + +import "strings" + +// PathToPrefix converts raw string to the prefix that will be used in the +// symbol table. All control characters, space, '%' and '"', as well as +// non-7-bit clean bytes turn into %xx. The period needs escaping only in the +// last segment of the path, and it makes for happier users if we escape that as +// little as possible. +func PathToPrefix(s string) string { + slash := strings.LastIndex(s, "/") + // check for chars that need escaping + n := 0 + for r := 0; r < len(s); r++ { + if c := s[r]; c <= ' ' || (c == '.' && r > slash) || c == '%' || c == '"' || c >= 0x7F { + n++ + } + } + + // quick exit + if n == 0 { + return s + } + + // escape + const hex = "0123456789abcdef" + p := make([]byte, 0, len(s)+2*n) + for r := 0; r < len(s); r++ { + if c := s[r]; c <= ' ' || (c == '.' && r > slash) || c == '%' || c == '"' || c >= 0x7F { + p = append(p, '%', hex[c>>4], hex[c&0xF]) + } else { + p = append(p, c) + } + } + + return string(p) +} diff --git a/libgo/go/cmd/internal/objabi/path_test.go b/libgo/go/cmd/internal/objabi/path_test.go new file mode 100644 index 00000000000..05d7fb436e3 --- /dev/null +++ b/libgo/go/cmd/internal/objabi/path_test.go @@ -0,0 +1,33 @@ +// Copyright 2017 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package objabi + +import "testing" + +func TestPathToPrefix(t *testing.T) { + tests := []struct { + Path string + Expected string + }{{"foo/bar/v1", "foo/bar/v1"}, + {"foo/bar/v.1", "foo/bar/v%2e1"}, + {"f.o.o/b.a.r/v1", "f.o.o/b.a.r/v1"}, + {"f.o.o/b.a.r/v.1", "f.o.o/b.a.r/v%2e1"}, + {"f.o.o/b.a.r/v..1", "f.o.o/b.a.r/v%2e%2e1"}, + {"f.o.o/b.a.r/v..1.", "f.o.o/b.a.r/v%2e%2e1%2e"}, + {"f.o.o/b.a.r/v%1", "f.o.o/b.a.r/v%251"}, + {"runtime", "runtime"}, + {"sync/atomic", "sync/atomic"}, + {"golang.org/x/tools/godoc", "golang.org/x/tools/godoc"}, + {"foo.bar/baz.quux", "foo.bar/baz%2equux"}, + {"", ""}, + {"%foo%bar", "%25foo%25bar"}, + {"\x01\x00\x7F☺", "%01%00%7f%e2%98%ba"}, + } + for _, tc := range tests { + if got := PathToPrefix(tc.Path); got != tc.Expected { + t.Errorf("expected PathToPrefix(%s) = %s, got %s", tc.Path, tc.Expected, got) + } + } +} diff --git a/libgo/go/cmd/internal/objabi/reloctype.go b/libgo/go/cmd/internal/objabi/reloctype.go new file mode 100644 index 00000000000..179f049de7d --- /dev/null +++ b/libgo/go/cmd/internal/objabi/reloctype.go @@ -0,0 +1,200 @@ +// Derived from Inferno utils/6l/l.h and related files. +// https://bitbucket.org/inferno-os/inferno-os/src/default/utils/6l/l.h +// +// Copyright © 1994-1999 Lucent Technologies Inc. All rights reserved. +// Portions Copyright © 1995-1997 C H Forsyth (forsyth@terzarima.net) +// Portions Copyright © 1997-1999 Vita Nuova Limited +// Portions Copyright © 2000-2007 Vita Nuova Holdings Limited (www.vitanuova.com) +// Portions Copyright © 2004,2006 Bruce Ellis +// Portions Copyright © 2005-2007 C H Forsyth (forsyth@terzarima.net) +// Revisions Copyright © 2000-2007 Lucent Technologies Inc. and others +// Portions Copyright © 2009 The Go Authors. All rights reserved. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +package objabi + +type RelocType int32 + +//go:generate stringer -type=RelocType +const ( + R_ADDR RelocType = 1 + iota + // R_ADDRPOWER relocates a pair of "D-form" instructions (instructions with 16-bit + // immediates in the low half of the instruction word), usually addis followed by + // another add or a load, inserting the "high adjusted" 16 bits of the address of + // the referenced symbol into the immediate field of the first instruction and the + // low 16 bits into that of the second instruction. + R_ADDRPOWER + // R_ADDRARM64 relocates an adrp, add pair to compute the address of the + // referenced symbol. + R_ADDRARM64 + // R_ADDRMIPS (only used on mips/mips64) resolves to the low 16 bits of an external + // address, by encoding it into the instruction. + R_ADDRMIPS + // R_ADDROFF resolves to a 32-bit offset from the beginning of the section + // holding the data being relocated to the referenced symbol. + R_ADDROFF + // R_WEAKADDROFF resolves just like R_ADDROFF but is a weak relocation. + // A weak relocation does not make the symbol it refers to reachable, + // and is only honored by the linker if the symbol is in some other way + // reachable. + R_WEAKADDROFF + R_SIZE + R_CALL + R_CALLARM + R_CALLARM64 + R_CALLIND + R_CALLPOWER + // R_CALLMIPS (only used on mips64) resolves to non-PC-relative target address + // of a CALL (JAL) instruction, by encoding the address into the instruction. + R_CALLMIPS + R_CONST + R_PCREL + // R_TLS_LE, used on 386, amd64, and ARM, resolves to the offset of the + // thread-local symbol from the thread local base and is used to implement the + // "local exec" model for tls access (r.Sym is not set on intel platforms but is + // set to a TLS symbol -- runtime.tlsg -- in the linker when externally linking). + R_TLS_LE + // R_TLS_IE, used 386, amd64, and ARM resolves to the PC-relative offset to a GOT + // slot containing the offset from the thread-local symbol from the thread local + // base and is used to implemented the "initial exec" model for tls access (r.Sym + // is not set on intel platforms but is set to a TLS symbol -- runtime.tlsg -- in + // the linker when externally linking). + R_TLS_IE + R_GOTOFF + R_PLT0 + R_PLT1 + R_PLT2 + R_USEFIELD + // R_USETYPE resolves to an *rtype, but no relocation is created. The + // linker uses this as a signal that the pointed-to type information + // should be linked into the final binary, even if there are no other + // direct references. (This is used for types reachable by reflection.) + R_USETYPE + // R_METHODOFF resolves to a 32-bit offset from the beginning of the section + // holding the data being relocated to the referenced symbol. + // It is a variant of R_ADDROFF used when linking from the uncommonType of a + // *rtype, and may be set to zero by the linker if it determines the method + // text is unreachable by the linked program. + R_METHODOFF + R_POWER_TOC + R_GOTPCREL + // R_JMPMIPS (only used on mips64) resolves to non-PC-relative target address + // of a JMP instruction, by encoding the address into the instruction. + // The stack nosplit check ignores this since it is not a function call. + R_JMPMIPS + // R_DWARFREF resolves to the offset of the symbol from its section. + R_DWARFREF + + // Platform dependent relocations. Architectures with fixed width instructions + // have the inherent issue that a 32-bit (or 64-bit!) displacement cannot be + // stuffed into a 32-bit instruction, so an address needs to be spread across + // several instructions, and in turn this requires a sequence of relocations, each + // updating a part of an instruction. This leads to relocation codes that are + // inherently processor specific. + + // Arm64. + + // Set a MOV[NZ] immediate field to bits [15:0] of the offset from the thread + // local base to the thread local variable defined by the referenced (thread + // local) symbol. Error if the offset does not fit into 16 bits. + R_ARM64_TLS_LE + + // Relocates an ADRP; LD64 instruction sequence to load the offset between + // the thread local base and the thread local variable defined by the + // referenced (thread local) symbol from the GOT. + R_ARM64_TLS_IE + + // R_ARM64_GOTPCREL relocates an adrp, ld64 pair to compute the address of the GOT + // slot of the referenced symbol. + R_ARM64_GOTPCREL + + // PPC64. + + // R_POWER_TLS_LE is used to implement the "local exec" model for tls + // access. It resolves to the offset of the thread-local symbol from the + // thread pointer (R13) and inserts this value into the low 16 bits of an + // instruction word. + R_POWER_TLS_LE + + // R_POWER_TLS_IE is used to implement the "initial exec" model for tls access. It + // relocates a D-form, DS-form instruction sequence like R_ADDRPOWER_DS. It + // inserts to the offset of GOT slot for the thread-local symbol from the TOC (the + // GOT slot is filled by the dynamic linker with the offset of the thread-local + // symbol from the thread pointer (R13)). + R_POWER_TLS_IE + + // R_POWER_TLS marks an X-form instruction such as "MOVD 0(R13)(R31*1), g" as + // accessing a particular thread-local symbol. It does not affect code generation + // but is used by the system linker when relaxing "initial exec" model code to + // "local exec" model code. + R_POWER_TLS + + // R_ADDRPOWER_DS is similar to R_ADDRPOWER above, but assumes the second + // instruction is a "DS-form" instruction, which has an immediate field occupying + // bits [15:2] of the instruction word. Bits [15:2] of the address of the + // relocated symbol are inserted into this field; it is an error if the last two + // bits of the address are not 0. + R_ADDRPOWER_DS + + // R_ADDRPOWER_PCREL relocates a D-form, DS-form instruction sequence like + // R_ADDRPOWER_DS but inserts the offset of the GOT slot for the referenced symbol + // from the TOC rather than the symbol's address. + R_ADDRPOWER_GOT + + // R_ADDRPOWER_PCREL relocates two D-form instructions like R_ADDRPOWER, but + // inserts the displacement from the place being relocated to the address of the + // the relocated symbol instead of just its address. + R_ADDRPOWER_PCREL + + // R_ADDRPOWER_TOCREL relocates two D-form instructions like R_ADDRPOWER, but + // inserts the offset from the TOC to the address of the relocated symbol + // rather than the symbol's address. + R_ADDRPOWER_TOCREL + + // R_ADDRPOWER_TOCREL relocates a D-form, DS-form instruction sequence like + // R_ADDRPOWER_DS but inserts the offset from the TOC to the address of the the + // relocated symbol rather than the symbol's address. + R_ADDRPOWER_TOCREL_DS + + // R_PCRELDBL relocates s390x 2-byte aligned PC-relative addresses. + // TODO(mundaym): remove once variants can be serialized - see issue 14218. + R_PCRELDBL + + // R_ADDRMIPSU (only used on mips/mips64) resolves to the sign-adjusted "upper" 16 + // bits (bit 16-31) of an external address, by encoding it into the instruction. + R_ADDRMIPSU + // R_ADDRMIPSTLS (only used on mips64) resolves to the low 16 bits of a TLS + // address (offset from thread pointer), by encoding it into the instruction. + R_ADDRMIPSTLS +) + +// IsDirectJump returns whether r is a relocation for a direct jump. +// A direct jump is a CALL or JMP instruction that takes the target address +// as immediate. The address is embedded into the instruction, possibly +// with limited width. +// An indirect jump is a CALL or JMP instruction that takes the target address +// in register or memory. +func (r RelocType) IsDirectJump() bool { + switch r { + case R_CALL, R_CALLARM, R_CALLARM64, R_CALLPOWER, R_CALLMIPS, R_JMPMIPS: + return true + } + return false +} diff --git a/libgo/go/cmd/internal/objabi/reloctype_string.go b/libgo/go/cmd/internal/objabi/reloctype_string.go new file mode 100644 index 00000000000..182d03f78c1 --- /dev/null +++ b/libgo/go/cmd/internal/objabi/reloctype_string.go @@ -0,0 +1,17 @@ +// Code generated by "stringer -type=RelocType"; DO NOT EDIT. + +package objabi + +import "fmt" + +const _RelocType_name = "R_ADDRR_ADDRPOWERR_ADDRARM64R_ADDRMIPSR_ADDROFFR_WEAKADDROFFR_SIZER_CALLR_CALLARMR_CALLARM64R_CALLINDR_CALLPOWERR_CALLMIPSR_CONSTR_PCRELR_TLS_LER_TLS_IER_GOTOFFR_PLT0R_PLT1R_PLT2R_USEFIELDR_USETYPER_METHODOFFR_POWER_TOCR_GOTPCRELR_JMPMIPSR_DWARFREFR_ARM64_TLS_LER_ARM64_TLS_IER_ARM64_GOTPCRELR_POWER_TLS_LER_POWER_TLS_IER_POWER_TLSR_ADDRPOWER_DSR_ADDRPOWER_GOTR_ADDRPOWER_PCRELR_ADDRPOWER_TOCRELR_ADDRPOWER_TOCREL_DSR_PCRELDBLR_ADDRMIPSUR_ADDRMIPSTLS" + +var _RelocType_index = [...]uint16{0, 6, 17, 28, 38, 47, 60, 66, 72, 81, 92, 101, 112, 122, 129, 136, 144, 152, 160, 166, 172, 178, 188, 197, 208, 219, 229, 238, 248, 262, 276, 292, 306, 320, 331, 345, 360, 377, 395, 416, 426, 437, 450} + +func (i RelocType) String() string { + i -= 1 + if i < 0 || i >= RelocType(len(_RelocType_index)-1) { + return fmt.Sprintf("RelocType(%d)", i+1) + } + return _RelocType_name[_RelocType_index[i]:_RelocType_index[i+1]] +} diff --git a/libgo/go/cmd/internal/objabi/stack.go b/libgo/go/cmd/internal/objabi/stack.go new file mode 100644 index 00000000000..11433932e2d --- /dev/null +++ b/libgo/go/cmd/internal/objabi/stack.go @@ -0,0 +1,20 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package objabi + +// For the linkers. Must match Go definitions. + +const ( + STACKSYSTEM = 0 + StackSystem = STACKSYSTEM + StackBig = 4096 + StackGuard = 880*stackGuardMultiplier + StackSystem + StackSmall = 128 + StackLimit = StackGuard - StackSystem - StackSmall +) + +const ( + StackPreempt = -1314 // 0xfff...fade +) diff --git a/libgo/go/cmd/internal/objabi/symkind.go b/libgo/go/cmd/internal/objabi/symkind.go new file mode 100644 index 00000000000..b037e9e4ed1 --- /dev/null +++ b/libgo/go/cmd/internal/objabi/symkind.go @@ -0,0 +1,60 @@ +// Derived from Inferno utils/6l/l.h and related files. +// https://bitbucket.org/inferno-os/inferno-os/src/default/utils/6l/l.h +// +// Copyright © 1994-1999 Lucent Technologies Inc. All rights reserved. +// Portions Copyright © 1995-1997 C H Forsyth (forsyth@terzarima.net) +// Portions Copyright © 1997-1999 Vita Nuova Limited +// Portions Copyright © 2000-2007 Vita Nuova Holdings Limited (www.vitanuova.com) +// Portions Copyright © 2004,2006 Bruce Ellis +// Portions Copyright © 2005-2007 C H Forsyth (forsyth@terzarima.net) +// Revisions Copyright © 2000-2007 Lucent Technologies Inc. and others +// Portions Copyright © 2009 The Go Authors. All rights reserved. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +package objabi + +// A SymKind describes the kind of memory represented by a symbol. +type SymKind uint8 + +// Defined SymKind values. +// +// TODO(rsc): Give idiomatic Go names. +//go:generate stringer -type=SymKind +const ( + // An otherwise invalid zero value for the type + Sxxx SymKind = iota + // Executable instructions + STEXT + // Read only static data + SRODATA + // Static data that does not contain any pointers + SNOPTRDATA + // Static data + SDATA + // Statically data that is initially all 0s + SBSS + // Statically data that is initially all 0s and does not contain pointers + SNOPTRBSS + // Thread-local data that is initally all 0s + STLSBSS + // Debugging data + SDWARFINFO + SDWARFRANGE +) diff --git a/libgo/go/cmd/internal/objabi/symkind_string.go b/libgo/go/cmd/internal/objabi/symkind_string.go new file mode 100644 index 00000000000..5123dc7097f --- /dev/null +++ b/libgo/go/cmd/internal/objabi/symkind_string.go @@ -0,0 +1,16 @@ +// Code generated by "stringer -type=SymKind"; DO NOT EDIT. + +package objabi + +import "fmt" + +const _SymKind_name = "SxxxSTEXTSRODATASNOPTRDATASDATASBSSSNOPTRBSSSTLSBSSSDWARFINFOSDWARFRANGE" + +var _SymKind_index = [...]uint8{0, 4, 9, 16, 26, 31, 35, 44, 51, 61, 72} + +func (i SymKind) String() string { + if i >= SymKind(len(_SymKind_index)-1) { + return fmt.Sprintf("SymKind(%d)", i) + } + return _SymKind_name[_SymKind_index[i]:_SymKind_index[i+1]] +} diff --git a/libgo/go/cmd/internal/objabi/typekind.go b/libgo/go/cmd/internal/objabi/typekind.go new file mode 100644 index 00000000000..f0e6f472e5b --- /dev/null +++ b/libgo/go/cmd/internal/objabi/typekind.go @@ -0,0 +1,41 @@ +// Copyright 2012 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package objabi + +// Must match runtime and reflect. +// Included by cmd/gc. + +const ( + KindBool = 1 + iota + KindInt + KindInt8 + KindInt16 + KindInt32 + KindInt64 + KindUint + KindUint8 + KindUint16 + KindUint32 + KindUint64 + KindUintptr + KindFloat32 + KindFloat64 + KindComplex64 + KindComplex128 + KindArray + KindChan + KindFunc + KindInterface + KindMap + KindPtr + KindSlice + KindString + KindStruct + KindUnsafePointer + KindDirectIface = 1 << 5 + KindGCProg = 1 << 6 + KindNoPointers = 1 << 7 + KindMask = (1 << 5) - 1 +) diff --git a/libgo/go/cmd/internal/objabi/util.go b/libgo/go/cmd/internal/objabi/util.go new file mode 100644 index 00000000000..1da05021f50 --- /dev/null +++ b/libgo/go/cmd/internal/objabi/util.go @@ -0,0 +1,119 @@ +// Copyright 2015 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package objabi + +import ( + "fmt" + "log" + "os" + "strings" +) + +func envOr(key, value string) string { + if x := os.Getenv(key); x != "" { + return x + } + return value +} + +var ( + GOROOT = envOr("GOROOT", defaultGOROOT) + GOARCH = envOr("GOARCH", defaultGOARCH) + GOOS = envOr("GOOS", defaultGOOS) + GO386 = envOr("GO386", defaultGO386) + GOARM = goarm() + Version = version +) + +func goarm() int { + switch v := envOr("GOARM", defaultGOARM); v { + case "5": + return 5 + case "6": + return 6 + case "7": + return 7 + } + // Fail here, rather than validate at multiple call sites. + log.Fatalf("Invalid GOARM value. Must be 5, 6, or 7.") + panic("unreachable") +} + +func Getgoextlinkenabled() string { + return envOr("GO_EXTLINK_ENABLED", defaultGO_EXTLINK_ENABLED) +} + +func init() { + for _, f := range strings.Split(goexperiment, ",") { + if f != "" { + addexp(f) + } + } +} + +func Framepointer_enabled(goos, goarch string) bool { + return framepointer_enabled != 0 && goarch == "amd64" && goos != "nacl" +} + +func addexp(s string) { + // Could do general integer parsing here, but the runtime copy doesn't yet. + v := 1 + name := s + if len(name) > 2 && name[:2] == "no" { + v = 0 + name = name[2:] + } + for i := 0; i < len(exper); i++ { + if exper[i].name == name { + if exper[i].val != nil { + *exper[i].val = v + } + return + } + } + + fmt.Printf("unknown experiment %s\n", s) + os.Exit(2) +} + +var ( + framepointer_enabled int = 1 + Fieldtrack_enabled int + Preemptibleloops_enabled int + Clobberdead_enabled int +) + +// Toolchain experiments. +// These are controlled by the GOEXPERIMENT environment +// variable recorded when the toolchain is built. +// This list is also known to cmd/gc. +var exper = []struct { + name string + val *int +}{ + {"fieldtrack", &Fieldtrack_enabled}, + {"framepointer", &framepointer_enabled}, + {"preemptibleloops", &Preemptibleloops_enabled}, + {"clobberdead", &Clobberdead_enabled}, +} + +var defaultExpstring = Expstring() + +func DefaultExpstring() string { + return defaultExpstring +} + +func Expstring() string { + buf := "X" + for i := range exper { + if *exper[i].val != 0 { + buf += "," + exper[i].name + } + } + if buf == "X" { + buf += ",none" + } + return "X:" + buf[2:] +} diff --git a/libgo/go/cmd/internal/objabi/zbootstrap.go b/libgo/go/cmd/internal/objabi/zbootstrap.go new file mode 100644 index 00000000000..56450fac618 --- /dev/null +++ b/libgo/go/cmd/internal/objabi/zbootstrap.go @@ -0,0 +1,15 @@ +// auto generated by go tool dist + +package objabi + +import "runtime" + +const defaultGOROOT = `/home/iant/go1.9` +const defaultGO386 = `sse2` +const defaultGOARM = `5` +const defaultGOOS = runtime.GOOS +const defaultGOARCH = runtime.GOARCH +const defaultGO_EXTLINK_ENABLED = `` +const version = `go1.9rc2` +const stackGuardMultiplier = 1 +const goexperiment = `` diff --git a/libgo/go/compress/bzip2/bzip2_test.go b/libgo/go/compress/bzip2/bzip2_test.go index 95fb1895856..a6c3080db3e 100644 --- a/libgo/go/compress/bzip2/bzip2_test.go +++ b/libgo/go/compress/bzip2/bzip2_test.go @@ -204,6 +204,12 @@ func TestMTF(t *testing.T) { } } +var ( + digits = mustLoadFile("testdata/e.txt.bz2") + twain = mustLoadFile("testdata/Mark.Twain-Tom.Sawyer.txt.bz2") + random = mustLoadFile("testdata/random.data.bz2") +) + func benchmarkDecode(b *testing.B, compressed []byte) { // Determine the uncompressed size of testfile. uncompressedSize, err := io.Copy(ioutil.Discard, NewReader(bytes.NewReader(compressed))) @@ -221,18 +227,6 @@ func benchmarkDecode(b *testing.B, compressed []byte) { } } -func BenchmarkDecodeDigits(b *testing.B) { - digits := mustLoadFile("testdata/e.txt.bz2") - b.ResetTimer() - benchmarkDecode(b, digits) -} -func BenchmarkDecodeTwain(b *testing.B) { - twain := mustLoadFile("testdata/Mark.Twain-Tom.Sawyer.txt.bz2") - b.ResetTimer() - benchmarkDecode(b, twain) -} -func BenchmarkDecodeRand(b *testing.B) { - random := mustLoadFile("testdata/random.data.bz2") - b.ResetTimer() - benchmarkDecode(b, random) -} +func BenchmarkDecodeDigits(b *testing.B) { benchmarkDecode(b, digits) } +func BenchmarkDecodeTwain(b *testing.B) { benchmarkDecode(b, twain) } +func BenchmarkDecodeRand(b *testing.B) { benchmarkDecode(b, random) } diff --git a/libgo/go/compress/bzip2/huffman.go b/libgo/go/compress/bzip2/huffman.go index 9d574b9bdef..dbba9a58b5a 100644 --- a/libgo/go/compress/bzip2/huffman.go +++ b/libgo/go/compress/bzip2/huffman.go @@ -108,10 +108,6 @@ func newHuffmanTree(lengths []uint8) (huffmanTree, error) { codes := huffmanCodes(make([]huffmanCode, len(lengths))) for i := len(pairs) - 1; i >= 0; i-- { if length > pairs[i].length { - // If the code length decreases we shift in order to - // zero any bits beyond the end of the code. - length >>= 32 - pairs[i].length - length <<= 32 - pairs[i].length length = pairs[i].length } codes[i].code = code diff --git a/libgo/go/compress/flate/huffman_code.go b/libgo/go/compress/flate/huffman_code.go index bdcbd823b00..891537ed5e6 100644 --- a/libgo/go/compress/flate/huffman_code.go +++ b/libgo/go/compress/flate/huffman_code.go @@ -6,6 +6,7 @@ package flate import ( "math" + "math/bits" "sort" ) @@ -342,3 +343,7 @@ func (s byFreq) Less(i, j int) bool { } func (s byFreq) Swap(i, j int) { s[i], s[j] = s[j], s[i] } + +func reverseBits(number uint16, bitLength byte) uint16 { + return bits.Reverse16(number << (16 - bitLength)) +} diff --git a/libgo/go/compress/flate/inflate.go b/libgo/go/compress/flate/inflate.go index 9a8c4fc455a..faa33cc6e9b 100644 --- a/libgo/go/compress/flate/inflate.go +++ b/libgo/go/compress/flate/inflate.go @@ -10,6 +10,7 @@ package flate import ( "bufio" "io" + mathbits "math/bits" "strconv" "sync" ) @@ -176,7 +177,7 @@ func (h *huffmanDecoder) init(bits []int) bool { link := nextcode[huffmanChunkBits+1] >> 1 h.links = make([][]uint32, huffmanNumChunks-link) for j := uint(link); j < huffmanNumChunks; j++ { - reverse := int(reverseByte[j>>8]) | int(reverseByte[j&0xff])<<8 + reverse := int(mathbits.Reverse16(uint16(j))) reverse >>= uint(16 - huffmanChunkBits) off := j - uint(link) if sanity && h.chunks[reverse] != 0 { @@ -194,7 +195,7 @@ func (h *huffmanDecoder) init(bits []int) bool { code := nextcode[n] nextcode[n]++ chunk := uint32(i<>8]) | int(reverseByte[code&0xff])<<8 + reverse := int(mathbits.Reverse16(uint16(code))) reverse >>= uint(16 - n) if n <= huffmanChunkBits { for off := reverse; off < len(h.chunks); off += 1 << uint(n) { @@ -556,7 +557,7 @@ readLiteral: return } } - dist = int(reverseByte[(f.b&0x1F)<<3]) + dist = int(mathbits.Reverse8(uint8(f.b & 0x1F << 3))) f.b >>= 5 f.nb -= 5 } else { diff --git a/libgo/go/compress/flate/reverse_bits.go b/libgo/go/compress/flate/reverse_bits.go deleted file mode 100644 index 6b222900c1e..00000000000 --- a/libgo/go/compress/flate/reverse_bits.go +++ /dev/null @@ -1,48 +0,0 @@ -// Copyright 2009 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package flate - -var reverseByte = [256]byte{ - 0x00, 0x80, 0x40, 0xc0, 0x20, 0xa0, 0x60, 0xe0, - 0x10, 0x90, 0x50, 0xd0, 0x30, 0xb0, 0x70, 0xf0, - 0x08, 0x88, 0x48, 0xc8, 0x28, 0xa8, 0x68, 0xe8, - 0x18, 0x98, 0x58, 0xd8, 0x38, 0xb8, 0x78, 0xf8, - 0x04, 0x84, 0x44, 0xc4, 0x24, 0xa4, 0x64, 0xe4, - 0x14, 0x94, 0x54, 0xd4, 0x34, 0xb4, 0x74, 0xf4, - 0x0c, 0x8c, 0x4c, 0xcc, 0x2c, 0xac, 0x6c, 0xec, - 0x1c, 0x9c, 0x5c, 0xdc, 0x3c, 0xbc, 0x7c, 0xfc, - 0x02, 0x82, 0x42, 0xc2, 0x22, 0xa2, 0x62, 0xe2, - 0x12, 0x92, 0x52, 0xd2, 0x32, 0xb2, 0x72, 0xf2, - 0x0a, 0x8a, 0x4a, 0xca, 0x2a, 0xaa, 0x6a, 0xea, - 0x1a, 0x9a, 0x5a, 0xda, 0x3a, 0xba, 0x7a, 0xfa, - 0x06, 0x86, 0x46, 0xc6, 0x26, 0xa6, 0x66, 0xe6, - 0x16, 0x96, 0x56, 0xd6, 0x36, 0xb6, 0x76, 0xf6, - 0x0e, 0x8e, 0x4e, 0xce, 0x2e, 0xae, 0x6e, 0xee, - 0x1e, 0x9e, 0x5e, 0xde, 0x3e, 0xbe, 0x7e, 0xfe, - 0x01, 0x81, 0x41, 0xc1, 0x21, 0xa1, 0x61, 0xe1, - 0x11, 0x91, 0x51, 0xd1, 0x31, 0xb1, 0x71, 0xf1, - 0x09, 0x89, 0x49, 0xc9, 0x29, 0xa9, 0x69, 0xe9, - 0x19, 0x99, 0x59, 0xd9, 0x39, 0xb9, 0x79, 0xf9, - 0x05, 0x85, 0x45, 0xc5, 0x25, 0xa5, 0x65, 0xe5, - 0x15, 0x95, 0x55, 0xd5, 0x35, 0xb5, 0x75, 0xf5, - 0x0d, 0x8d, 0x4d, 0xcd, 0x2d, 0xad, 0x6d, 0xed, - 0x1d, 0x9d, 0x5d, 0xdd, 0x3d, 0xbd, 0x7d, 0xfd, - 0x03, 0x83, 0x43, 0xc3, 0x23, 0xa3, 0x63, 0xe3, - 0x13, 0x93, 0x53, 0xd3, 0x33, 0xb3, 0x73, 0xf3, - 0x0b, 0x8b, 0x4b, 0xcb, 0x2b, 0xab, 0x6b, 0xeb, - 0x1b, 0x9b, 0x5b, 0xdb, 0x3b, 0xbb, 0x7b, 0xfb, - 0x07, 0x87, 0x47, 0xc7, 0x27, 0xa7, 0x67, 0xe7, - 0x17, 0x97, 0x57, 0xd7, 0x37, 0xb7, 0x77, 0xf7, - 0x0f, 0x8f, 0x4f, 0xcf, 0x2f, 0xaf, 0x6f, 0xef, - 0x1f, 0x9f, 0x5f, 0xdf, 0x3f, 0xbf, 0x7f, 0xff, -} - -func reverseUint16(v uint16) uint16 { - return uint16(reverseByte[v>>8]) | uint16(reverseByte[v&0xFF])<<8 -} - -func reverseBits(number uint16, bitLength byte) uint16 { - return reverseUint16(number << (16 - bitLength)) -} diff --git a/libgo/go/compress/gzip/gzip.go b/libgo/go/compress/gzip/gzip.go index aafb442a667..0cc44c59e0f 100644 --- a/libgo/go/compress/gzip/gzip.go +++ b/libgo/go/compress/gzip/gzip.go @@ -222,8 +222,9 @@ func (z *Writer) Flush() error { return z.err } -// Close closes the Writer, flushing any unwritten data to the underlying -// io.Writer, but does not close the underlying io.Writer. +// Close closes the Writer by flushing any unwritten data to the underlying +// io.Writer and writing the GZIP footer. +// It does not close the underlying io.Writer. func (z *Writer) Close() error { if z.err != nil { return z.err diff --git a/libgo/go/compress/lzw/reader.go b/libgo/go/compress/lzw/reader.go index 9eef2b2a782..1be52d55e4f 100644 --- a/libgo/go/compress/lzw/reader.go +++ b/libgo/go/compress/lzw/reader.go @@ -57,8 +57,14 @@ type decoder struct { // The next two codes mean clear and EOF. // Other valid codes are in the range [lo, hi] where lo := clear + 2, // with the upper bound incrementing on each code seen. - // overflow is the code at which hi overflows the code width. + // + // overflow is the code at which hi overflows the code width. It always + // equals 1 << width. + // // last is the most recently seen code, or decoderInvalidCode. + // + // An invariant is that + // (hi < overflow) || (hi == overflow && last == decoderInvalidCode) clear, eof, hi, overflow, last uint16 // Each code c in [lo, hi] expands to two or more bytes. For c != hi: @@ -163,7 +169,7 @@ loop: break loop case code <= d.hi: c, i := code, len(d.output)-1 - if code == d.hi { + if code == d.hi && d.last != decoderInvalidCode { // code == hi is a special case which expands to the last expansion // followed by the head of the last expansion. To find the head, we walk // the prefix chain until we find a literal code. @@ -196,6 +202,10 @@ loop: if d.hi >= d.overflow { if d.width == maxWidth { d.last = decoderInvalidCode + // Undo the d.hi++ a few lines above, so that (1) we maintain + // the invariant that d.hi <= d.overflow, and (2) d.hi does not + // eventually overflow a uint16. + d.hi-- } else { d.width++ d.overflow <<= 1 diff --git a/libgo/go/compress/lzw/reader_test.go b/libgo/go/compress/lzw/reader_test.go index 6b9f9a3da70..f8974de28fc 100644 --- a/libgo/go/compress/lzw/reader_test.go +++ b/libgo/go/compress/lzw/reader_test.go @@ -120,6 +120,103 @@ func TestReader(t *testing.T) { } } +type devZero struct{} + +func (devZero) Read(p []byte) (int, error) { + for i := range p { + p[i] = 0 + } + return len(p), nil +} + +func TestHiCodeDoesNotOverflow(t *testing.T) { + r := NewReader(devZero{}, LSB, 8) + d := r.(*decoder) + buf := make([]byte, 1024) + oldHi := uint16(0) + for i := 0; i < 100; i++ { + if _, err := io.ReadFull(r, buf); err != nil { + t.Fatalf("i=%d: %v", i, err) + } + // The hi code should never decrease. + if d.hi < oldHi { + t.Fatalf("i=%d: hi=%d decreased from previous value %d", i, d.hi, oldHi) + } + oldHi = d.hi + } +} + +// TestNoLongerSavingPriorExpansions tests the decoder state when codes other +// than clear codes continue to be seen after decoder.hi and decoder.width +// reach their maximum values (4095 and 12), i.e. after we no longer save prior +// expansions. In particular, it tests seeing the highest possible code, 4095. +func TestNoLongerSavingPriorExpansions(t *testing.T) { + // Iterations is used to calculate how many input bits are needed to get + // the decoder.hi and decoder.width values up to their maximum. + iterations := []struct { + width, n int + }{ + // The final term is 257, not 256, as NewReader initializes d.hi to + // d.clear+1 and the clear code is 256. + {9, 512 - 257}, + {10, 1024 - 512}, + {11, 2048 - 1024}, + {12, 4096 - 2048}, + } + nCodes, nBits := 0, 0 + for _, e := range iterations { + nCodes += e.n + nBits += e.n * e.width + } + if nCodes != 3839 { + t.Fatalf("nCodes: got %v, want %v", nCodes, 3839) + } + if nBits != 43255 { + t.Fatalf("nBits: got %v, want %v", nBits, 43255) + } + + // Construct our input of 43255 zero bits (which gets d.hi and d.width up + // to 4095 and 12), followed by 0xfff (4095) as 12 bits, followed by 0x101 + // (EOF) as 12 bits. + // + // 43255 = 5406*8 + 7, and codes are read in LSB order. The final bytes are + // therefore: + // + // xwwwwwww xxxxxxxx yyyyyxxx zyyyyyyy + // 10000000 11111111 00001111 00001000 + // + // or split out: + // + // .0000000 ........ ........ ........ w = 0x000 + // 1....... 11111111 .....111 ........ x = 0xfff + // ........ ........ 00001... .0001000 y = 0x101 + // + // The 12 'w' bits (not all are shown) form the 3839'th code, with value + // 0x000. Just after decoder.read returns that code, d.hi == 4095 and + // d.last == 0. + // + // The 12 'x' bits form the 3840'th code, with value 0xfff or 4095. Just + // after decoder.read returns that code, d.hi == 4095 and d.last == + // decoderInvalidCode. + // + // The 12 'y' bits form the 3841'st code, with value 0x101, the EOF code. + // + // The 'z' bit is unused. + in := make([]byte, 5406) + in = append(in, 0x80, 0xff, 0x0f, 0x08) + + r := NewReader(bytes.NewReader(in), LSB, 8) + nDecoded, err := io.Copy(ioutil.Discard, r) + if err != nil { + t.Fatalf("Copy: %v", err) + } + // nDecoded should be 3841: 3839 literal codes and then 2 decoded bytes + // from 1 non-literal code. The EOF code contributes 0 decoded bytes. + if nDecoded != int64(nCodes+2) { + t.Fatalf("nDecoded: got %v, want %v", nDecoded, nCodes+2) + } +} + func BenchmarkDecoder(b *testing.B) { buf, err := ioutil.ReadFile("../testdata/e.txt") if err != nil { diff --git a/libgo/go/container/heap/heap.go b/libgo/go/container/heap/heap.go index 7110c513f08..b2c64276c82 100644 --- a/libgo/go/container/heap/heap.go +++ b/libgo/go/container/heap/heap.go @@ -72,8 +72,9 @@ func Remove(h Interface, i int) interface{} { n := h.Len() - 1 if n != i { h.Swap(i, n) - down(h, i, n) - up(h, i) + if !down(h, i, n) { + up(h, i) + } } return h.Pop() } @@ -107,7 +108,7 @@ func down(h Interface, i0, n int) bool { break } j := j1 // left child - if j2 := j1 + 1; j2 < n && !h.Less(j1, j2) { + if j2 := j1 + 1; j2 < n && h.Less(j2, j1) { j = j2 // = 2*i + 2 // right child } if !h.Less(j, i) { diff --git a/libgo/go/context/context.go b/libgo/go/context/context.go index 0aa7c24df9a..892ff27c653 100644 --- a/libgo/go/context/context.go +++ b/libgo/go/context/context.go @@ -96,10 +96,11 @@ type Context interface { // a Done channel for cancelation. Done() <-chan struct{} - // Err returns a non-nil error value after Done is closed. Err returns - // Canceled if the context was canceled or DeadlineExceeded if the - // context's deadline passed. No other values for Err are defined. - // After Done is closed, successive calls to Err return the same value. + // If Done is not yet closed, Err returns nil. + // If Done is closed, Err returns a non-nil error explaining why: + // Canceled if the context was canceled + // or DeadlineExceeded if the context's deadline passed. + // After Err returns a non-nil error, successive calls to Err return the same error. Err() error // Value returns the value associated with this context for key, or nil @@ -234,10 +235,7 @@ func WithCancel(parent Context) (ctx Context, cancel CancelFunc) { // newCancelCtx returns an initialized cancelCtx. func newCancelCtx(parent Context) cancelCtx { - return cancelCtx{ - Context: parent, - done: make(chan struct{}), - } + return cancelCtx{Context: parent} } // propagateCancel arranges for child to be canceled when parent is. @@ -306,20 +304,32 @@ type canceler interface { Done() <-chan struct{} } +// closedchan is a reusable closed channel. +var closedchan = make(chan struct{}) + +func init() { + close(closedchan) +} + // A cancelCtx can be canceled. When canceled, it also cancels any children // that implement canceler. type cancelCtx struct { Context - done chan struct{} // closed by the first cancel call. - - mu sync.Mutex + mu sync.Mutex // protects following fields + done chan struct{} // created lazily, closed by first cancel call children map[canceler]struct{} // set to nil by the first cancel call err error // set to non-nil by the first cancel call } func (c *cancelCtx) Done() <-chan struct{} { - return c.done + c.mu.Lock() + if c.done == nil { + c.done = make(chan struct{}) + } + d := c.done + c.mu.Unlock() + return d } func (c *cancelCtx) Err() error { @@ -344,7 +354,11 @@ func (c *cancelCtx) cancel(removeFromParent bool, err error) { return // already canceled } c.err = err - close(c.done) + if c.done == nil { + c.done = closedchan + } else { + close(c.done) + } for child := range c.children { // NOTE: acquiring the child's lock while holding parent's lock. child.cancel(false, err) diff --git a/libgo/go/context/context_test.go b/libgo/go/context/context_test.go index b5e599fe82f..548476f290d 100644 --- a/libgo/go/context/context_test.go +++ b/libgo/go/context/context_test.go @@ -428,7 +428,7 @@ func XTestAllocs(t testingT, testingShort func() bool, testingAllocsPerRun func( limit := test.limit if runtime.Compiler == "gccgo" { // gccgo does not yet do escape analysis. - // TOOD(iant): Remove this when gccgo does do escape analysis. + // TODO(iant): Remove this when gccgo does do escape analysis. limit = test.gccgoLimit } numRuns := 100 diff --git a/libgo/go/crypto/aes/cipher_generic.go b/libgo/go/crypto/aes/cipher_generic.go index 2c8d299c880..98169bf5fd7 100644 --- a/libgo/go/crypto/aes/cipher_generic.go +++ b/libgo/go/crypto/aes/cipher_generic.go @@ -2,7 +2,7 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -// -build !amd64,!s390x +// -build !amd64,!s390x,!ppc64le package aes diff --git a/libgo/go/crypto/aes/cipher_ppc64le.go b/libgo/go/crypto/aes/cipher_ppc64le.go new file mode 100644 index 00000000000..1d16fe03f04 --- /dev/null +++ b/libgo/go/crypto/aes/cipher_ppc64le.go @@ -0,0 +1,82 @@ +// Copyright 2016 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build ignore + +package aes + +import ( + "crypto/cipher" +) + +// defined in asm_ppc64le.s + +//go:noescape + +func setEncryptKeyAsm(key *byte, keylen int, enc *uint32) int + +//go:noescape + +func setDecryptKeyAsm(key *byte, keylen int, dec *uint32) int + +//go:noescape + +func doEncryptKeyAsm(key *byte, keylen int, dec *uint32) int + +//go:noescape + +func encryptBlockAsm(dst, src *byte, enc *uint32) + +//go:noescape + +func decryptBlockAsm(dst, src *byte, dec *uint32) + +type aesCipherAsm struct { + aesCipher +} + +func newCipher(key []byte) (cipher.Block, error) { + n := 64 // size is fixed for all and round value is stored inside it too + c := aesCipherAsm{aesCipher{make([]uint32, n), make([]uint32, n)}} + k := len(key) + + ret := 0 + ret += setEncryptKeyAsm(&key[0], k*8, &c.enc[0]) + ret += setDecryptKeyAsm(&key[0], k*8, &c.dec[0]) + + if ret > 0 { + return nil, KeySizeError(k) + } + + return &c, nil +} + +func (c *aesCipherAsm) BlockSize() int { return BlockSize } + +func (c *aesCipherAsm) Encrypt(dst, src []byte) { + if len(src) < BlockSize { + panic("crypto/aes: input not full block") + } + if len(dst) < BlockSize { + panic("crypto/aes: output not full block") + } + encryptBlockAsm(&dst[0], &src[0], &c.enc[0]) +} + +func (c *aesCipherAsm) Decrypt(dst, src []byte) { + if len(src) < BlockSize { + panic("crypto/aes: input not full block") + } + if len(dst) < BlockSize { + panic("crypto/aes: output not full block") + } + decryptBlockAsm(&dst[0], &src[0], &c.dec[0]) +} + +// expandKey is used by BenchmarkExpand to ensure that the asm implementation +// of key expansion is used for the benchmark when it is available. +func expandKey(key []byte, enc, dec []uint32) { + setEncryptKeyAsm(&key[0], len(key)*8, &enc[0]) + setDecryptKeyAsm(&key[0], len(key)*8, &dec[0]) +} diff --git a/libgo/go/crypto/crypto.go b/libgo/go/crypto/crypto.go index a80ebd36931..b4d6cdcab4e 100644 --- a/libgo/go/crypto/crypto.go +++ b/libgo/go/crypto/crypto.go @@ -21,40 +21,48 @@ func (h Hash) HashFunc() Hash { } const ( - MD4 Hash = 1 + iota // import golang.org/x/crypto/md4 - MD5 // import crypto/md5 - SHA1 // import crypto/sha1 - SHA224 // import crypto/sha256 - SHA256 // import crypto/sha256 - SHA384 // import crypto/sha512 - SHA512 // import crypto/sha512 - MD5SHA1 // no implementation; MD5+SHA1 used for TLS RSA - RIPEMD160 // import golang.org/x/crypto/ripemd160 - SHA3_224 // import golang.org/x/crypto/sha3 - SHA3_256 // import golang.org/x/crypto/sha3 - SHA3_384 // import golang.org/x/crypto/sha3 - SHA3_512 // import golang.org/x/crypto/sha3 - SHA512_224 // import crypto/sha512 - SHA512_256 // import crypto/sha512 + MD4 Hash = 1 + iota // import golang.org/x/crypto/md4 + MD5 // import crypto/md5 + SHA1 // import crypto/sha1 + SHA224 // import crypto/sha256 + SHA256 // import crypto/sha256 + SHA384 // import crypto/sha512 + SHA512 // import crypto/sha512 + MD5SHA1 // no implementation; MD5+SHA1 used for TLS RSA + RIPEMD160 // import golang.org/x/crypto/ripemd160 + SHA3_224 // import golang.org/x/crypto/sha3 + SHA3_256 // import golang.org/x/crypto/sha3 + SHA3_384 // import golang.org/x/crypto/sha3 + SHA3_512 // import golang.org/x/crypto/sha3 + SHA512_224 // import crypto/sha512 + SHA512_256 // import crypto/sha512 + BLAKE2s_256 // import golang.org/x/crypto/blake2s + BLAKE2b_256 // import golang.org/x/crypto/blake2b + BLAKE2b_384 // import golang.org/x/crypto/blake2b + BLAKE2b_512 // import golang.org/x/crypto/blake2b maxHash ) var digestSizes = []uint8{ - MD4: 16, - MD5: 16, - SHA1: 20, - SHA224: 28, - SHA256: 32, - SHA384: 48, - SHA512: 64, - SHA512_224: 28, - SHA512_256: 32, - SHA3_224: 28, - SHA3_256: 32, - SHA3_384: 48, - SHA3_512: 64, - MD5SHA1: 36, - RIPEMD160: 20, + MD4: 16, + MD5: 16, + SHA1: 20, + SHA224: 28, + SHA256: 32, + SHA384: 48, + SHA512: 64, + SHA512_224: 28, + SHA512_256: 32, + SHA3_224: 28, + SHA3_256: 32, + SHA3_384: 48, + SHA3_512: 64, + MD5SHA1: 36, + RIPEMD160: 20, + BLAKE2s_256: 32, + BLAKE2b_256: 32, + BLAKE2b_384: 48, + BLAKE2b_512: 64, } // Size returns the length, in bytes, of a digest resulting from the given hash diff --git a/libgo/go/crypto/des/block.go b/libgo/go/crypto/des/block.go index 99338d62a6e..21e6d4e82f6 100644 --- a/libgo/go/crypto/des/block.go +++ b/libgo/go/crypto/des/block.go @@ -4,25 +4,29 @@ package des -import ( - "encoding/binary" -) +import "encoding/binary" func cryptBlock(subkeys []uint64, dst, src []byte, decrypt bool) { b := binary.BigEndian.Uint64(src) b = permuteInitialBlock(b) left, right := uint32(b>>32), uint32(b) - var subkey uint64 - for i := 0; i < 16; i++ { - if decrypt { - subkey = subkeys[15-i] - } else { - subkey = subkeys[i] - } + left = (left << 1) | (left >> 31) + right = (right << 1) | (right >> 31) - left, right = right, left^feistel(right, subkey) + if decrypt { + for i := 0; i < 8; i++ { + left, right = feistel(left, right, subkeys[15-2*i], subkeys[15-(2*i+1)]) + } + } else { + for i := 0; i < 8; i++ { + left, right = feistel(left, right, subkeys[2*i], subkeys[2*i+1]) + } } + + left = (left << 31) | (left >> 1) + right = (right << 31) | (right >> 1) + // switch left & right and perform final permutation preOutput := (uint64(right) << 32) | uint64(left) binary.BigEndian.PutUint64(dst, permuteFinalBlock(preOutput)) @@ -39,19 +43,34 @@ func decryptBlock(subkeys []uint64, dst, src []byte) { } // DES Feistel function -func feistel(right uint32, key uint64) (result uint32) { - sBoxLocations := key ^ expandBlock(right) - var sBoxResult uint32 - for i := uint8(0); i < 8; i++ { - sBoxLocation := uint8(sBoxLocations>>42) & 0x3f - sBoxLocations <<= 6 - // row determined by 1st and 6th bit - // column is middle four bits - row := (sBoxLocation & 0x1) | ((sBoxLocation & 0x20) >> 4) - column := (sBoxLocation >> 1) & 0xf - sBoxResult ^= feistelBox[i][16*row+column] - } - return sBoxResult +func feistel(l, r uint32, k0, k1 uint64) (lout, rout uint32) { + var t uint32 + + t = r ^ uint32(k0>>32) + l ^= feistelBox[7][t&0x3f] ^ + feistelBox[5][(t>>8)&0x3f] ^ + feistelBox[3][(t>>16)&0x3f] ^ + feistelBox[1][(t>>24)&0x3f] + + t = ((r << 28) | (r >> 4)) ^ uint32(k0) + l ^= feistelBox[6][(t)&0x3f] ^ + feistelBox[4][(t>>8)&0x3f] ^ + feistelBox[2][(t>>16)&0x3f] ^ + feistelBox[0][(t>>24)&0x3f] + + t = l ^ uint32(k1>>32) + r ^= feistelBox[7][t&0x3f] ^ + feistelBox[5][(t>>8)&0x3f] ^ + feistelBox[3][(t>>16)&0x3f] ^ + feistelBox[1][(t>>24)&0x3f] + + t = ((l << 28) | (l >> 4)) ^ uint32(k1) + r ^= feistelBox[6][(t)&0x3f] ^ + feistelBox[4][(t>>8)&0x3f] ^ + feistelBox[2][(t>>16)&0x3f] ^ + feistelBox[0][(t>>24)&0x3f] + + return l, r } // feistelBox[s][16*i+j] contains the output of permutationFunction @@ -73,27 +92,22 @@ func init() { for j := 0; j < 16; j++ { f := uint64(sBoxes[s][i][j]) << (4 * (7 - uint(s))) f = permuteBlock(f, permutationFunction[:]) - feistelBox[s][16*i+j] = uint32(f) + + // Row is determined by the 1st and 6th bit. + // Column is the middle four bits. + row := uint8(((i & 2) << 4) | i&1) + col := uint8(j << 1) + t := row | col + + // The rotation was performed in the feistel rounds, being factored out and now mixed into the feistelBox. + f = (f << 1) | (f >> 31) + + feistelBox[s][t] = uint32(f) } } } } -// expandBlock expands an input block of 32 bits, -// producing an output block of 48 bits. -func expandBlock(src uint32) (block uint64) { - // rotate the 5 highest bits to the right. - src = (src << 5) | (src >> 27) - for i := 0; i < 8; i++ { - block <<= 6 - // take the 6 bits on the right - block |= uint64(src) & (1<<6 - 1) - // advance by 4 bits. - src = (src << 4) | (src >> 28) - } - return -} - // permuteInitialBlock is equivalent to the permutation defined // by initialPermutation. func permuteInitialBlock(block uint64) uint64 { @@ -218,6 +232,24 @@ func (c *desCipher) generateSubkeys(keyBytes []byte) { // combine halves to form 56-bit input to PC2 pc2Input := uint64(leftRotations[i])<<28 | uint64(rightRotations[i]) // apply PC2 permutation to 7 byte input - c.subkeys[i] = permuteBlock(pc2Input, permutedChoice2[:]) + c.subkeys[i] = unpack(permuteBlock(pc2Input, permutedChoice2[:])) } } + +// Expand 48-bit input to 64-bit, with each 6-bit block padded by extra two bits at the top. +// By doing so, we can have the input blocks (four bits each), and the key blocks (six bits each) well-aligned without +// extra shifts/rotations for alignments. +func unpack(x uint64) uint64 { + var result uint64 + + result = ((x>>(6*1))&0xff)<<(8*0) | + ((x>>(6*3))&0xff)<<(8*1) | + ((x>>(6*5))&0xff)<<(8*2) | + ((x>>(6*7))&0xff)<<(8*3) | + ((x>>(6*0))&0xff)<<(8*4) | + ((x>>(6*2))&0xff)<<(8*5) | + ((x>>(6*4))&0xff)<<(8*6) | + ((x>>(6*6))&0xff)<<(8*7) + + return result +} diff --git a/libgo/go/crypto/des/cipher.go b/libgo/go/crypto/des/cipher.go index 2f929ca7bed..46af5b0f022 100644 --- a/libgo/go/crypto/des/cipher.go +++ b/libgo/go/crypto/des/cipher.go @@ -6,6 +6,7 @@ package des import ( "crypto/cipher" + "encoding/binary" "strconv" ) @@ -61,13 +62,51 @@ func NewTripleDESCipher(key []byte) (cipher.Block, error) { func (c *tripleDESCipher) BlockSize() int { return BlockSize } func (c *tripleDESCipher) Encrypt(dst, src []byte) { - c.cipher1.Encrypt(dst, src) - c.cipher2.Decrypt(dst, dst) - c.cipher3.Encrypt(dst, dst) + b := binary.BigEndian.Uint64(src) + b = permuteInitialBlock(b) + left, right := uint32(b>>32), uint32(b) + + left = (left << 1) | (left >> 31) + right = (right << 1) | (right >> 31) + + for i := 0; i < 8; i++ { + left, right = feistel(left, right, c.cipher1.subkeys[2*i], c.cipher1.subkeys[2*i+1]) + } + for i := 0; i < 8; i++ { + right, left = feistel(right, left, c.cipher2.subkeys[15-2*i], c.cipher2.subkeys[15-(2*i+1)]) + } + for i := 0; i < 8; i++ { + left, right = feistel(left, right, c.cipher3.subkeys[2*i], c.cipher3.subkeys[2*i+1]) + } + + left = (left << 31) | (left >> 1) + right = (right << 31) | (right >> 1) + + preOutput := (uint64(right) << 32) | uint64(left) + binary.BigEndian.PutUint64(dst, permuteFinalBlock(preOutput)) } func (c *tripleDESCipher) Decrypt(dst, src []byte) { - c.cipher3.Decrypt(dst, src) - c.cipher2.Encrypt(dst, dst) - c.cipher1.Decrypt(dst, dst) + b := binary.BigEndian.Uint64(src) + b = permuteInitialBlock(b) + left, right := uint32(b>>32), uint32(b) + + left = (left << 1) | (left >> 31) + right = (right << 1) | (right >> 31) + + for i := 0; i < 8; i++ { + left, right = feistel(left, right, c.cipher3.subkeys[15-2*i], c.cipher3.subkeys[15-(2*i+1)]) + } + for i := 0; i < 8; i++ { + right, left = feistel(right, left, c.cipher2.subkeys[2*i], c.cipher2.subkeys[2*i+1]) + } + for i := 0; i < 8; i++ { + left, right = feistel(left, right, c.cipher1.subkeys[15-2*i], c.cipher1.subkeys[15-(2*i+1)]) + } + + left = (left << 31) | (left >> 1) + right = (right << 31) | (right >> 1) + + preOutput := (uint64(right) << 32) | uint64(left) + binary.BigEndian.PutUint64(dst, permuteFinalBlock(preOutput)) } diff --git a/libgo/go/crypto/des/const.go b/libgo/go/crypto/des/const.go index 2bd485ee80e..a20879d5741 100644 --- a/libgo/go/crypto/des/const.go +++ b/libgo/go/crypto/des/const.go @@ -5,6 +5,9 @@ // Package des implements the Data Encryption Standard (DES) and the // Triple Data Encryption Algorithm (TDEA) as defined // in U.S. Federal Information Processing Standards Publication 46-3. +// +// DES is cryptographically broken and should not be used for secure +// applications. package des // Used to perform an initial permutation of a 64-bit input block. diff --git a/libgo/go/crypto/des/des_test.go b/libgo/go/crypto/des/des_test.go index 2bd525afecc..690a49f5efb 100644 --- a/libgo/go/crypto/des/des_test.go +++ b/libgo/go/crypto/des/des_test.go @@ -1526,17 +1526,6 @@ func TestFinalPermute(t *testing.T) { } } -func TestExpandBlock(t *testing.T) { - for i := uint(0); i < 32; i++ { - bit := uint32(1) << i - got := expandBlock(bit) - want := permuteBlock(uint64(bit), expansionFunction[:]) - if got != want { - t.Errorf("expand(%x) = %x, want %x", bit, got, want) - } - } -} - func BenchmarkEncrypt(b *testing.B) { tt := encryptDESTests[0] c, err := NewCipher(tt.key) @@ -1564,3 +1553,31 @@ func BenchmarkDecrypt(b *testing.B) { c.Decrypt(out, tt.out) } } + +func BenchmarkTDESEncrypt(b *testing.B) { + tt := encryptTripleDESTests[0] + c, err := NewTripleDESCipher(tt.key) + if err != nil { + b.Fatal("NewCipher:", err) + } + out := make([]byte, len(tt.in)) + b.SetBytes(int64(len(out))) + b.ResetTimer() + for i := 0; i < b.N; i++ { + c.Encrypt(out, tt.in) + } +} + +func BenchmarkTDESDecrypt(b *testing.B) { + tt := encryptTripleDESTests[0] + c, err := NewTripleDESCipher(tt.key) + if err != nil { + b.Fatal("NewCipher:", err) + } + out := make([]byte, len(tt.out)) + b.SetBytes(int64(len(out))) + b.ResetTimer() + for i := 0; i < b.N; i++ { + c.Decrypt(out, tt.out) + } +} diff --git a/libgo/go/crypto/dsa/dsa_test.go b/libgo/go/crypto/dsa/dsa_test.go index 8600059f032..7fc246bc2bd 100644 --- a/libgo/go/crypto/dsa/dsa_test.go +++ b/libgo/go/crypto/dsa/dsa_test.go @@ -82,12 +82,17 @@ func fromHex(s string) *big.Int { } func TestSignAndVerify(t *testing.T) { - var priv PrivateKey - priv.P, _ = new(big.Int).SetString("A9B5B793FB4785793D246BAE77E8FF63CA52F442DA763C440259919FE1BC1D6065A9350637A04F75A2F039401D49F08E066C4D275A5A65DA5684BC563C14289D7AB8A67163BFBF79D85972619AD2CFF55AB0EE77A9002B0EF96293BDD0F42685EBB2C66C327079F6C98000FBCB79AACDE1BC6F9D5C7B1A97E3D9D54ED7951FEF", 16) - priv.Q, _ = new(big.Int).SetString("E1D3391245933D68A0714ED34BBCB7A1F422B9C1", 16) - priv.G, _ = new(big.Int).SetString("634364FC25248933D01D1993ECABD0657CC0CB2CEED7ED2E3E8AECDFCDC4A25C3B15E9E3B163ACA2984B5539181F3EFF1A5E8903D71D5B95DA4F27202B77D2C44B430BB53741A8D59A8F86887525C9F2A6A5980A195EAA7F2FF910064301DEF89D3AA213E1FAC7768D89365318E370AF54A112EFBA9246D9158386BA1B4EEFDA", 16) - priv.Y, _ = new(big.Int).SetString("32969E5780CFE1C849A1C276D7AEB4F38A23B591739AA2FE197349AEEBD31366AEE5EB7E6C6DDB7C57D02432B30DB5AA66D9884299FAA72568944E4EEDC92EA3FBC6F39F53412FBCC563208F7C15B737AC8910DBC2D9C9B8C001E72FDC40EB694AB1F06A5A2DBD18D9E36C66F31F566742F11EC0A52E9F7B89355C02FB5D32D2", 16) - priv.X, _ = new(big.Int).SetString("5078D4D29795CBE76D3AACFE48C9AF0BCDBEE91A", 16) + priv := PrivateKey{ + PublicKey: PublicKey{ + Parameters: Parameters{ + P: fromHex("A9B5B793FB4785793D246BAE77E8FF63CA52F442DA763C440259919FE1BC1D6065A9350637A04F75A2F039401D49F08E066C4D275A5A65DA5684BC563C14289D7AB8A67163BFBF79D85972619AD2CFF55AB0EE77A9002B0EF96293BDD0F42685EBB2C66C327079F6C98000FBCB79AACDE1BC6F9D5C7B1A97E3D9D54ED7951FEF"), + Q: fromHex("E1D3391245933D68A0714ED34BBCB7A1F422B9C1"), + G: fromHex("634364FC25248933D01D1993ECABD0657CC0CB2CEED7ED2E3E8AECDFCDC4A25C3B15E9E3B163ACA2984B5539181F3EFF1A5E8903D71D5B95DA4F27202B77D2C44B430BB53741A8D59A8F86887525C9F2A6A5980A195EAA7F2FF910064301DEF89D3AA213E1FAC7768D89365318E370AF54A112EFBA9246D9158386BA1B4EEFDA"), + }, + Y: fromHex("32969E5780CFE1C849A1C276D7AEB4F38A23B591739AA2FE197349AEEBD31366AEE5EB7E6C6DDB7C57D02432B30DB5AA66D9884299FAA72568944E4EEDC92EA3FBC6F39F53412FBCC563208F7C15B737AC8910DBC2D9C9B8C001E72FDC40EB694AB1F06A5A2DBD18D9E36C66F31F566742F11EC0A52E9F7B89355C02FB5D32D2"), + }, + X: fromHex("5078D4D29795CBE76D3AACFE48C9AF0BCDBEE91A"), + } testSignAndVerify(t, 0, &priv) } diff --git a/libgo/go/crypto/md5/md5.go b/libgo/go/crypto/md5/md5.go index ce58d5e713a..b682f008942 100644 --- a/libgo/go/crypto/md5/md5.go +++ b/libgo/go/crypto/md5/md5.go @@ -5,6 +5,9 @@ //go:generate go run gen.go -full -output md5block.go // Package md5 implements the MD5 hash algorithm as defined in RFC 1321. +// +// MD5 is cryptographically broken and should not be used for secure +// applications. package md5 import ( diff --git a/libgo/go/crypto/rand/rand_linux.go b/libgo/go/crypto/rand/rand_linux.go index 472daa7650b..8a4c7572363 100644 --- a/libgo/go/crypto/rand/rand_linux.go +++ b/libgo/go/crypto/rand/rand_linux.go @@ -6,34 +6,20 @@ package rand import ( "internal/syscall/unix" - "sync" ) func init() { altGetRandom = getRandomLinux } -var ( - once sync.Once - useSyscall bool -) - -func pickStrategy() { - // Test whether we should use the system call or /dev/urandom. - // We'll fall back to urandom if: - // - the kernel is too old (before 3.17) - // - the machine has no entropy available (early boot + no hardware - // entropy source?) and we want to avoid blocking later. - var buf [1]byte - n, err := unix.GetRandom(buf[:], unix.GRND_NONBLOCK) - useSyscall = n == 1 && err == nil -} - +// If the kernel is too old (before 3.17) to support the getrandom syscall(), +// unix.GetRandom will immediately return ENOSYS and we will then fall back to +// reading from /dev/urandom in rand_unix.go. unix.GetRandom caches the ENOSYS +// result so we only suffer the syscall overhead once in this case. +// If the kernel supports the getrandom() syscall, unix.GetRandom will block +// until the kernel has sufficient randomness (as we don't use GRND_NONBLOCK). +// In this case, unix.GetRandom will not return an error. func getRandomLinux(p []byte) (ok bool) { - once.Do(pickStrategy) - if !useSyscall { - return false - } n, err := unix.GetRandom(p, 0) return n == len(p) && err == nil } diff --git a/libgo/go/crypto/rand/util.go b/libgo/go/crypto/rand/util.go index 592c57e7638..4dd17112034 100644 --- a/libgo/go/crypto/rand/util.go +++ b/libgo/go/crypto/rand/util.go @@ -107,16 +107,23 @@ func Int(rand io.Reader, max *big.Int) (n *big.Int, err error) { if max.Sign() <= 0 { panic("crypto/rand: argument to Int is <= 0") } - k := (max.BitLen() + 7) / 8 - - // b is the number of bits in the most significant byte of max. - b := uint(max.BitLen() % 8) + n = new(big.Int) + n.Sub(max, n.SetUint64(1)) + // bitLen is the maximum bit length needed to encode a value < max. + bitLen := n.BitLen() + if bitLen == 0 { + // the only valid result is 0 + return + } + // k is the maximum byte length needed to encode a value < max. + k := (bitLen + 7) / 8 + // b is the number of bits in the most significant byte of max-1. + b := uint(bitLen % 8) if b == 0 { b = 8 } bytes := make([]byte, k) - n = new(big.Int) for { _, err = io.ReadFull(rand, bytes) diff --git a/libgo/go/crypto/rand/util_test.go b/libgo/go/crypto/rand/util_test.go index 48a2c3fc0cb..685624e1b3d 100644 --- a/libgo/go/crypto/rand/util_test.go +++ b/libgo/go/crypto/rand/util_test.go @@ -5,7 +5,10 @@ package rand_test import ( + "bytes" "crypto/rand" + "fmt" + "io" "math/big" mathrand "math/rand" "testing" @@ -45,6 +48,56 @@ func TestInt(t *testing.T) { } } +type countingReader struct { + r io.Reader + n int +} + +func (r *countingReader) Read(p []byte) (n int, err error) { + n, err = r.r.Read(p) + r.n += n + return n, err +} + +// Test that Int reads only the necessary number of bytes from the reader for +// max at each bit length +func TestIntReads(t *testing.T) { + for i := 0; i < 32; i++ { + max := int64(1 << uint64(i)) + t.Run(fmt.Sprintf("max=%d", max), func(t *testing.T) { + reader := &countingReader{r: rand.Reader} + + _, err := rand.Int(reader, big.NewInt(max)) + if err != nil { + t.Fatalf("Can't generate random value: %d, %v", max, err) + } + expected := (i + 7) / 8 + if reader.n != expected { + t.Errorf("Int(reader, %d) should read %d bytes, but it read: %d", max, expected, reader.n) + } + }) + } +} + +// Test that Int does not mask out valid return values +func TestIntMask(t *testing.T) { + for max := 1; max <= 256; max++ { + t.Run(fmt.Sprintf("max=%d", max), func(t *testing.T) { + for i := 0; i < max; i++ { + var b bytes.Buffer + b.WriteByte(byte(i)) + n, err := rand.Int(&b, big.NewInt(int64(max))) + if err != nil { + t.Fatalf("Can't generate random value: %d, %v", max, err) + } + if n.Int64() != int64(i) { + t.Errorf("Int(reader, %d) should have returned value of %d, but it returned: %v", max, i, n) + } + } + }) + } +} + func testIntPanics(t *testing.T, b *big.Int) { defer func() { if err := recover(); err == nil { diff --git a/libgo/go/crypto/rc4/rc4.go b/libgo/go/crypto/rc4/rc4.go index bd04aee6955..772af0e7e0d 100644 --- a/libgo/go/crypto/rc4/rc4.go +++ b/libgo/go/crypto/rc4/rc4.go @@ -4,11 +4,11 @@ // Package rc4 implements RC4 encryption, as defined in Bruce Schneier's // Applied Cryptography. +// +// RC4 is cryptographically broken and should not be used for secure +// applications. package rc4 -// BUG(agl): RC4 is in common use but has design weaknesses that make -// it a poor choice for new protocols. - import "strconv" // A Cipher is an instance of RC4 using a particular key. diff --git a/libgo/go/crypto/sha1/sha1.go b/libgo/go/crypto/sha1/sha1.go index fbb2f946132..6b1721470b2 100644 --- a/libgo/go/crypto/sha1/sha1.go +++ b/libgo/go/crypto/sha1/sha1.go @@ -2,7 +2,10 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -// Package sha1 implements the SHA1 hash algorithm as defined in RFC 3174. +// Package sha1 implements the SHA-1 hash algorithm as defined in RFC 3174. +// +// SHA-1 is cryptographically broken and should not be used for secure +// applications. package sha1 import ( @@ -14,10 +17,10 @@ func init() { crypto.RegisterHash(crypto.SHA1, New) } -// The size of a SHA1 checksum in bytes. +// The size of a SHA-1 checksum in bytes. const Size = 20 -// The blocksize of SHA1 in bytes. +// The blocksize of SHA-1 in bytes. const BlockSize = 64 const ( @@ -189,7 +192,7 @@ func (d *digest) constSum() [Size]byte { return digest } -// Sum returns the SHA1 checksum of the data. +// Sum returns the SHA-1 checksum of the data. func Sum(data []byte) [Size]byte { var d digest d.Reset() diff --git a/libgo/go/crypto/sha1/sha1_test.go b/libgo/go/crypto/sha1/sha1_test.go index 3e59a5defe3..faa9916bc06 100644 --- a/libgo/go/crypto/sha1/sha1_test.go +++ b/libgo/go/crypto/sha1/sha1_test.go @@ -2,7 +2,7 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -// SHA1 hash algorithm. See RFC 3174. +// SHA-1 hash algorithm. See RFC 3174. package sha1 diff --git a/libgo/go/crypto/sha1/sha1block.go b/libgo/go/crypto/sha1/sha1block.go index fde3c981c08..1d37544940f 100644 --- a/libgo/go/crypto/sha1/sha1block.go +++ b/libgo/go/crypto/sha1/sha1block.go @@ -11,7 +11,7 @@ const ( _K3 = 0xCA62C1D6 ) -// blockGeneric is a portable, pure Go version of the SHA1 block step. +// blockGeneric is a portable, pure Go version of the SHA-1 block step. // It's used by sha1block_generic.go and tests. func blockGeneric(dig *digest, p []byte) { var w [16]uint32 diff --git a/libgo/go/crypto/sha1/sha1block_amd64.go b/libgo/go/crypto/sha1/sha1block_amd64.go index 8ef8b1faffc..2d7a314d51d 100644 --- a/libgo/go/crypto/sha1/sha1block_amd64.go +++ b/libgo/go/crypto/sha1/sha1block_amd64.go @@ -6,18 +6,18 @@ package sha1 -//go:noescape +import "internal/cpu" +//go:noescape func blockAVX2(dig *digest, p []byte) //go:noescape func blockAMD64(dig *digest, p []byte) -func checkAVX2() bool -var hasAVX2 = checkAVX2() +var useAVX2 = cpu.X86.HasAVX2 && cpu.X86.HasBMI1 && cpu.X86.HasBMI2 func block(dig *digest, p []byte) { - if hasAVX2 && len(p) >= 256 { + if useAVX2 && len(p) >= 256 { // blockAVX2 calculates sha1 for 2 block per iteration // it also interleaves precalculation for next block. // So it may read up-to 192 bytes past end of p diff --git a/libgo/go/crypto/sha1/sha1block_s390x.go b/libgo/go/crypto/sha1/sha1block_s390x.go index 9edcbb0d4b4..55a23599a97 100644 --- a/libgo/go/crypto/sha1/sha1block_s390x.go +++ b/libgo/go/crypto/sha1/sha1block_s390x.go @@ -7,7 +7,7 @@ package sha1 // featureCheck reports whether the CPU supports the -// SHA1 compute intermediate message digest (KIMD) +// SHA-1 compute intermediate message digest (KIMD) // function code. func featureCheck() bool diff --git a/libgo/go/crypto/sha256/sha256block_amd64.go b/libgo/go/crypto/sha256/sha256block_amd64.go new file mode 100644 index 00000000000..b311a125c98 --- /dev/null +++ b/libgo/go/crypto/sha256/sha256block_amd64.go @@ -0,0 +1,11 @@ +// Copyright 2017 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build ignore + +package sha256 + +import "internal/cpu" + +var useAVX2 = cpu.X86.HasAVX2 && cpu.X86.HasBMI2 diff --git a/libgo/go/crypto/sha512/sha512block_amd64.go b/libgo/go/crypto/sha512/sha512block_amd64.go new file mode 100644 index 00000000000..0ce599b8ebc --- /dev/null +++ b/libgo/go/crypto/sha512/sha512block_amd64.go @@ -0,0 +1,26 @@ +// Copyright 2013 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build ignore +// +build amd64 + +package sha512 + +import "internal/cpu" + +//go:noescape +func blockAVX2(dig *digest, p []byte) + +//go:noescape +func blockAMD64(dig *digest, p []byte) + +var useAVX2 = cpu.X86.HasAVX2 && cpu.X86.HasBMI1 && cpu.X86.HasBMI2 + +func block(dig *digest, p []byte) { + if useAVX2 { + blockAVX2(dig, p) + } else { + blockAMD64(dig, p) + } +} diff --git a/libgo/go/crypto/sha512/sha512block_decl.go b/libgo/go/crypto/sha512/sha512block_decl.go index 3859a40e328..3d098c5b17e 100644 --- a/libgo/go/crypto/sha512/sha512block_decl.go +++ b/libgo/go/crypto/sha512/sha512block_decl.go @@ -3,7 +3,7 @@ // license that can be found in the LICENSE file. // +build ignore -// +build amd64 s390x ppc64le +// +build s390x ppc64le package sha512 diff --git a/libgo/go/crypto/tls/common.go b/libgo/go/crypto/tls/common.go index de833a90563..5860838dd25 100644 --- a/libgo/go/crypto/tls/common.go +++ b/libgo/go/crypto/tls/common.go @@ -163,8 +163,8 @@ type ConnectionState struct { HandshakeComplete bool // TLS handshake is complete DidResume bool // connection resumes a previous TLS connection CipherSuite uint16 // cipher suite in use (TLS_RSA_WITH_RC4_128_SHA, ...) - NegotiatedProtocol string // negotiated next protocol (from Config.NextProtos) - NegotiatedProtocolIsMutual bool // negotiated protocol was advertised by server + NegotiatedProtocol string // negotiated next protocol (not guaranteed to be from Config.NextProtos) + NegotiatedProtocolIsMutual bool // negotiated protocol was advertised by server (client side only) ServerName string // server name requested by client, if any (server side only) PeerCertificates []*x509.Certificate // certificate chain presented by remote peer VerifiedChains [][]*x509.Certificate // verified chains built from PeerCertificates @@ -174,9 +174,9 @@ type ConnectionState struct { // TLSUnique contains the "tls-unique" channel binding value (see RFC // 5929, section 3). For resumed sessions this value will be nil // because resumption does not include enough context (see - // https://secure-resumption.com/#channelbindings). This will change in - // future versions of Go once the TLS master-secret fix has been - // standardized and implemented. + // https://mitls.org/pages/attacks/3SHAKE#channelbindings). This will + // change in future versions of Go once the TLS master-secret fix has + // been standardized and implemented. TLSUnique []byte } @@ -206,7 +206,8 @@ type ClientSessionState struct { // ClientSessionCache is a cache of ClientSessionState objects that can be used // by a client to resume a TLS session with a given server. ClientSessionCache // implementations should expect to be called concurrently from different -// goroutines. +// goroutines. Only ticket-based resumption is supported, not SessionID-based +// resumption. type ClientSessionCache interface { // Get searches for a ClientSessionState associated with the given key. // On return, ok is true if one was found. @@ -508,17 +509,13 @@ type Config struct { serverInitOnce sync.Once // guards calling (*Config).serverInit - // mutex protects sessionTicketKeys and originalConfig. + // mutex protects sessionTicketKeys. mutex sync.RWMutex // sessionTicketKeys contains zero or more ticket keys. If the length // is zero, SessionTicketsDisabled must be true. The first key is used // for new tickets and any subsequent keys can be used to decrypt old // tickets. sessionTicketKeys []ticketKey - // originalConfig is set to the Config that was passed to Server if - // this Config is returned by a GetConfigForClient callback. It's used - // by serverInit in order to copy session ticket keys if needed. - originalConfig *Config } // ticketKeyNameLen is the number of bytes of identifier that is prepended to @@ -550,7 +547,7 @@ func ticketKeyFromBytes(b [32]byte) (key ticketKey) { func (c *Config) Clone() *Config { // Running serverInit ensures that it's safe to read // SessionTicketsDisabled. - c.serverInitOnce.Do(c.serverInit) + c.serverInitOnce.Do(func() { c.serverInit(nil) }) var sessionTicketKeys []ticketKey c.mutex.RLock() @@ -584,20 +581,17 @@ func (c *Config) Clone() *Config { Renegotiation: c.Renegotiation, KeyLogWriter: c.KeyLogWriter, sessionTicketKeys: sessionTicketKeys, - // originalConfig is deliberately not duplicated. } } -func (c *Config) serverInit() { +// serverInit is run under c.serverInitOnce to do initialization of c. If c was +// returned by a GetConfigForClient callback then the argument should be the +// Config that was passed to Server, otherwise it should be nil. +func (c *Config) serverInit(originalConfig *Config) { if c.SessionTicketsDisabled || len(c.ticketKeys()) != 0 { return } - var originalConfig *Config - c.mutex.Lock() - originalConfig, c.originalConfig = c.originalConfig, nil - c.mutex.Unlock() - alreadySet := false for _, b := range c.SessionTicketKey { if b != 0 { @@ -947,9 +941,7 @@ func initDefaultCipherSuites() { } varDefaultCipherSuites = make([]uint16, 0, len(cipherSuites)) - for _, topCipher := range topCipherSuites { - varDefaultCipherSuites = append(varDefaultCipherSuites, topCipher) - } + varDefaultCipherSuites = append(varDefaultCipherSuites, topCipherSuites...) NextCipherSuite: for _, suite := range cipherSuites { diff --git a/libgo/go/crypto/tls/conn.go b/libgo/go/crypto/tls/conn.go index 03895a723fa..e6d85aa2639 100644 --- a/libgo/go/crypto/tls/conn.go +++ b/libgo/go/crypto/tls/conn.go @@ -1206,10 +1206,10 @@ func (c *Conn) Close() error { var alertErr error c.handshakeMutex.Lock() - defer c.handshakeMutex.Unlock() if c.handshakeComplete { alertErr = c.closeNotify() } + c.handshakeMutex.Unlock() if err := c.conn.Close(); err != nil { return err diff --git a/libgo/go/crypto/tls/conn_test.go b/libgo/go/crypto/tls/conn_test.go index 5e5c7a2e96e..e27c5414b22 100644 --- a/libgo/go/crypto/tls/conn_test.go +++ b/libgo/go/crypto/tls/conn_test.go @@ -138,7 +138,7 @@ func runDynamicRecordSizingTest(t *testing.T, config *Config) { tlsConn := Client(clientConn, config) if err := tlsConn.Handshake(); err != nil { - t.Errorf("Error from client handshake: %s", err) + t.Errorf("Error from client handshake: %v", err) return } @@ -147,12 +147,12 @@ func runDynamicRecordSizingTest(t *testing.T, config *Config) { var recordSizes []int for { - n, err := clientConn.Read(recordHeader[:]) + n, err := io.ReadFull(clientConn, recordHeader[:]) if err == io.EOF { break } if err != nil || n != len(recordHeader) { - t.Errorf("Error from client read: %s", err) + t.Errorf("io.ReadFull = %d, %v", n, err) return } @@ -161,9 +161,9 @@ func runDynamicRecordSizingTest(t *testing.T, config *Config) { record = make([]byte, length) } - n, err = clientConn.Read(record[:length]) + n, err = io.ReadFull(clientConn, record[:length]) if err != nil || n != length { - t.Errorf("Error from client read: %s", err) + t.Errorf("io.ReadFull = %d, %v", n, err) return } @@ -241,3 +241,34 @@ func TestDynamicRecordSizingWithAEAD(t *testing.T) { config.CipherSuites = []uint16{TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256} runDynamicRecordSizingTest(t, config) } + +// hairpinConn is a net.Conn that makes a “hairpin” call when closed, back into +// the tls.Conn which is calling it. +type hairpinConn struct { + net.Conn + tlsConn *Conn +} + +func (conn *hairpinConn) Close() error { + conn.tlsConn.ConnectionState() + return nil +} + +func TestHairpinInClose(t *testing.T) { + // This tests that the underlying net.Conn can call back into the + // tls.Conn when being closed without deadlocking. + client, server := net.Pipe() + defer server.Close() + defer client.Close() + + conn := &hairpinConn{client, nil} + tlsConn := Server(conn, &Config{ + GetCertificate: func(*ClientHelloInfo) (*Certificate, error) { + panic("unreachable") + }, + }) + conn.tlsConn = tlsConn + + // This call should not deadlock. + tlsConn.Close() +} diff --git a/libgo/go/crypto/tls/generate_cert.go b/libgo/go/crypto/tls/generate_cert.go index 83f9916ff9d..8ee2b5922b3 100644 --- a/libgo/go/crypto/tls/generate_cert.go +++ b/libgo/go/crypto/tls/generate_cert.go @@ -33,7 +33,7 @@ var ( validFor = flag.Duration("duration", 365*24*time.Hour, "Duration that certificate is valid for") isCA = flag.Bool("ca", false, "whether this cert should be its own Certificate Authority") rsaBits = flag.Int("rsa-bits", 2048, "Size of RSA key to generate. Ignored if --ecdsa-curve is set") - ecdsaCurve = flag.String("ecdsa-curve", "", "ECDSA curve to use to generate a key. Valid values are P224, P256, P384, P521") + ecdsaCurve = flag.String("ecdsa-curve", "", "ECDSA curve to use to generate a key. Valid values are P224, P256 (recommended), P384, P521") ) func publicKey(priv interface{}) interface{} { diff --git a/libgo/go/crypto/tls/handshake_client.go b/libgo/go/crypto/tls/handshake_client.go index 6eda18dbfce..a4ca5d34fb8 100644 --- a/libgo/go/crypto/tls/handshake_client.go +++ b/libgo/go/crypto/tls/handshake_client.go @@ -815,7 +815,7 @@ func hostnameInSNI(name string) string { if net.ParseIP(host) != nil { return "" } - if len(name) > 0 && name[len(name)-1] == '.' { + for len(name) > 0 && name[len(name)-1] == '.' { name = name[:len(name)-1] } return name diff --git a/libgo/go/crypto/tls/handshake_messages.go b/libgo/go/crypto/tls/handshake_messages.go index 694bd918d85..0c7581f3e36 100644 --- a/libgo/go/crypto/tls/handshake_messages.go +++ b/libgo/go/crypto/tls/handshake_messages.go @@ -4,7 +4,10 @@ package tls -import "bytes" +import ( + "bytes" + "strings" +) type clientHelloMsg struct { raw []byte @@ -393,6 +396,12 @@ func (m *clientHelloMsg) unmarshal(data []byte) bool { } if nameType == 0 { m.serverName = string(d[:nameLen]) + // An SNI value may not include a + // trailing dot. See + // https://tools.ietf.org/html/rfc6066#section-3. + if strings.HasSuffix(m.serverName, ".") { + return false + } break } d = d[nameLen:] diff --git a/libgo/go/crypto/tls/handshake_messages_test.go b/libgo/go/crypto/tls/handshake_messages_test.go index f1154d4d01d..7add97c32c1 100644 --- a/libgo/go/crypto/tls/handshake_messages_test.go +++ b/libgo/go/crypto/tls/handshake_messages_test.go @@ -8,6 +8,7 @@ import ( "bytes" "math/rand" "reflect" + "strings" "testing" "testing/quick" ) @@ -123,6 +124,9 @@ func (*clientHelloMsg) Generate(rand *rand.Rand, size int) reflect.Value { } if rand.Intn(10) > 5 { m.serverName = randomString(rand.Intn(255), rand) + for strings.HasSuffix(m.serverName, ".") { + m.serverName = m.serverName[:len(m.serverName)-1] + } } m.ocspStapling = rand.Intn(10) > 5 m.supportedPoints = randomBytes(rand.Intn(5)+1, rand) diff --git a/libgo/go/crypto/tls/handshake_server.go b/libgo/go/crypto/tls/handshake_server.go index b786c3083a2..ae328487088 100644 --- a/libgo/go/crypto/tls/handshake_server.go +++ b/libgo/go/crypto/tls/handshake_server.go @@ -40,7 +40,7 @@ type serverHandshakeState struct { func (c *Conn) serverHandshake() error { // If this is the first server handshake, we generate a random key to // encrypt the tickets with. - c.config.serverInitOnce.Do(c.config.serverInit) + c.config.serverInitOnce.Do(func() { c.config.serverInit(nil) }) hs := serverHandshakeState{ c: c, @@ -129,11 +129,7 @@ func (hs *serverHandshakeState) readClientHello() (isResume bool, err error) { c.sendAlert(alertInternalError) return false, err } else if newConfig != nil { - newConfig.mutex.Lock() - newConfig.originalConfig = c.config - newConfig.mutex.Unlock() - - newConfig.serverInitOnce.Do(newConfig.serverInit) + newConfig.serverInitOnce.Do(func() { newConfig.serverInit(c.config) }) c.config = newConfig } } diff --git a/libgo/go/crypto/tls/handshake_server_test.go b/libgo/go/crypto/tls/handshake_server_test.go index bcd3d43ea3c..63845c170d7 100644 --- a/libgo/go/crypto/tls/handshake_server_test.go +++ b/libgo/go/crypto/tls/handshake_server_test.go @@ -137,6 +137,10 @@ func TestNoRC4ByDefault(t *testing.T) { testClientHelloFailure(t, serverConfig, clientHello, "no cipher suite supported by both client and server") } +func TestRejectSNIWithTrailingDot(t *testing.T) { + testClientHelloFailure(t, testConfig, &clientHelloMsg{vers: VersionTLS12, serverName: "foo.com."}, "unexpected message") +} + func TestDontSelectECDSAWithRSAKey(t *testing.T) { // Test that, even when both sides support an ECDSA cipher suite, it // won't be selected if the server's private key doesn't support it. diff --git a/libgo/go/crypto/x509/pkcs1.go b/libgo/go/crypto/x509/pkcs1.go index df20a442049..73bc7623a5a 100644 --- a/libgo/go/crypto/x509/pkcs1.go +++ b/libgo/go/crypto/x509/pkcs1.go @@ -35,6 +35,12 @@ type pkcs1AdditionalRSAPrime struct { Coeff *big.Int } +// pkcs1PublicKey reflects the ASN.1 structure of a PKCS#1 public key. +type pkcs1PublicKey struct { + N *big.Int + E int +} + // ParsePKCS1PrivateKey returns an RSA private key from its ASN.1 PKCS#1 DER encoded form. func ParsePKCS1PrivateKey(der []byte) (*rsa.PrivateKey, error) { var priv pkcs1PrivateKey @@ -113,9 +119,3 @@ func MarshalPKCS1PrivateKey(key *rsa.PrivateKey) []byte { b, _ := asn1.Marshal(priv) return b } - -// rsaPublicKey reflects the ASN.1 structure of a PKCS#1 public key. -type rsaPublicKey struct { - N *big.Int - E int -} diff --git a/libgo/go/crypto/x509/root_bsd.go b/libgo/go/crypto/x509/root_bsd.go index 93172837368..13719338910 100644 --- a/libgo/go/crypto/x509/root_bsd.go +++ b/libgo/go/crypto/x509/root_bsd.go @@ -8,7 +8,8 @@ package x509 // Possible certificate files; stop after finding one. var certFiles = []string{ - "/usr/local/share/certs/ca-root-nss.crt", // FreeBSD/DragonFly + "/usr/local/etc/ssl/cert.pem", // FreeBSD "/etc/ssl/cert.pem", // OpenBSD + "/usr/local/share/certs/ca-root-nss.crt", // DragonFly "/etc/openssl/certs/ca-certificates.crt", // NetBSD } diff --git a/libgo/go/crypto/x509/root_darwin.go b/libgo/go/crypto/x509/root_darwin.go index 66cdb5ea261..bc35a1cf212 100644 --- a/libgo/go/crypto/x509/root_darwin.go +++ b/libgo/go/crypto/x509/root_darwin.go @@ -16,6 +16,7 @@ import ( "io/ioutil" "os" "os/exec" + "os/user" "path/filepath" "strings" "sync" @@ -61,7 +62,26 @@ func execSecurityRoots() (*CertPool, error) { println(fmt.Sprintf("crypto/x509: %d certs have a trust policy", len(hasPolicy))) } - cmd := exec.Command("/usr/bin/security", "find-certificate", "-a", "-p", "/System/Library/Keychains/SystemRootCertificates.keychain") + args := []string{"find-certificate", "-a", "-p", + "/System/Library/Keychains/SystemRootCertificates.keychain", + "/Library/Keychains/System.keychain", + } + + u, err := user.Current() + if err != nil { + if debugExecDarwinRoots { + println(fmt.Sprintf("crypto/x509: get current user: %v", err)) + } + } else { + args = append(args, + filepath.Join(u.HomeDir, "/Library/Keychains/login.keychain"), + + // Fresh installs of Sierra use a slightly different path for the login keychain + filepath.Join(u.HomeDir, "/Library/Keychains/login.keychain-db"), + ) + } + + cmd := exec.Command("/usr/bin/security", args...) data, err := cmd.Output() if err != nil { return nil, err diff --git a/libgo/go/crypto/x509/root_unix.go b/libgo/go/crypto/x509/root_unix.go index c44a5241685..0547460437b 100644 --- a/libgo/go/crypto/x509/root_unix.go +++ b/libgo/go/crypto/x509/root_unix.go @@ -16,28 +16,52 @@ import ( var certDirectories = []string{ "/etc/ssl/certs", // SLES10/SLES11, https://golang.org/issue/12139 "/system/etc/security/cacerts", // Android + "/usr/local/share/certs", // FreeBSD + "/etc/pki/tls/certs", // Fedora/RHEL + "/etc/openssl/certs", // NetBSD "/var/ssl/certs", // AIX } +const ( + // certFileEnv is the environment variable which identifies where to locate + // the SSL certificate file. If set this overrides the system default. + certFileEnv = "SSL_CERT_FILE" + + // certDirEnv is the environment variable which identifies which directory + // to check for SSL certificate files. If set this overrides the system default. + certDirEnv = "SSL_CERT_DIR" +) + func (c *Certificate) systemVerify(opts *VerifyOptions) (chains [][]*Certificate, err error) { return nil, nil } func loadSystemRoots() (*CertPool, error) { roots := NewCertPool() + + files := certFiles + if f := os.Getenv(certFileEnv); f != "" { + files = []string{f} + } + var firstErr error - for _, file := range certFiles { + for _, file := range files { data, err := ioutil.ReadFile(file) if err == nil { roots.AppendCertsFromPEM(data) - return roots, nil + break } if firstErr == nil && !os.IsNotExist(err) { firstErr = err } } - for _, directory := range certDirectories { + dirs := certDirectories + if d := os.Getenv(certDirEnv); d != "" { + dirs = []string{d} + } + + for _, directory := range dirs { fis, err := ioutil.ReadDir(directory) if err != nil { if firstErr == nil && !os.IsNotExist(err) { @@ -57,5 +81,9 @@ func loadSystemRoots() (*CertPool, error) { } } + if len(roots.certs) > 0 { + return roots, nil + } + return nil, firstErr } diff --git a/libgo/go/crypto/x509/root_unix_test.go b/libgo/go/crypto/x509/root_unix_test.go new file mode 100644 index 00000000000..03f935d4e82 --- /dev/null +++ b/libgo/go/crypto/x509/root_unix_test.go @@ -0,0 +1,127 @@ +// Copyright 2017 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build dragonfly freebsd linux netbsd openbsd solaris + +package x509 + +import ( + "fmt" + "os" + "testing" +) + +const ( + testDir = "testdata" + testDirCN = "test-dir" + testFile = "test-file.crt" + testFileCN = "test-file" + testMissing = "missing" +) + +func TestEnvVars(t *testing.T) { + testCases := []struct { + name string + fileEnv string + dirEnv string + files []string + dirs []string + cns []string + }{ + { + // Environment variables override the default locations preventing fall through. + name: "override-defaults", + fileEnv: testMissing, + dirEnv: testMissing, + files: []string{testFile}, + dirs: []string{testDir}, + cns: nil, + }, + { + // File environment overrides default file locations. + name: "file", + fileEnv: testFile, + dirEnv: "", + files: nil, + dirs: nil, + cns: []string{testFileCN}, + }, + { + // Directory environment overrides default directory locations. + name: "dir", + fileEnv: "", + dirEnv: testDir, + files: nil, + dirs: nil, + cns: []string{testDirCN}, + }, + { + // File & directory environment overrides both default locations. + name: "file+dir", + fileEnv: testFile, + dirEnv: testDir, + files: nil, + dirs: nil, + cns: []string{testFileCN, testDirCN}, + }, + { + // Environment variable empty / unset uses default locations. + name: "empty-fall-through", + fileEnv: "", + dirEnv: "", + files: []string{testFile}, + dirs: []string{testDir}, + cns: []string{testFileCN, testDirCN}, + }, + } + + // Save old settings so we can restore before the test ends. + origCertFiles, origCertDirectories := certFiles, certDirectories + origFile, origDir := os.Getenv(certFileEnv), os.Getenv(certDirEnv) + defer func() { + certFiles = origCertFiles + certDirectories = origCertDirectories + os.Setenv(certFileEnv, origFile) + os.Setenv(certDirEnv, origDir) + }() + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + if err := os.Setenv(certFileEnv, tc.fileEnv); err != nil { + t.Fatalf("setenv %q failed: %v", certFileEnv, err) + } + if err := os.Setenv(certDirEnv, tc.dirEnv); err != nil { + t.Fatalf("setenv %q failed: %v", certDirEnv, err) + } + + certFiles, certDirectories = tc.files, tc.dirs + + r, err := loadSystemRoots() + if err != nil { + t.Fatal("unexpected failure:", err) + } + + if r == nil { + if tc.cns == nil { + // Expected nil + return + } + t.Fatal("nil roots") + } + + // Verify that the returned certs match, otherwise report where the mismatch is. + for i, cn := range tc.cns { + if i >= len(r.certs) { + t.Errorf("missing cert %v @ %v", cn, i) + } else if r.certs[i].Subject.CommonName != cn { + fmt.Printf("%#v\n", r.certs[0].Subject) + t.Errorf("unexpected cert common name %q, want %q", r.certs[i].Subject.CommonName, cn) + } + } + if len(r.certs) > len(tc.cns) { + t.Errorf("got %v certs, which is more than %v wanted", len(r.certs), len(tc.cns)) + } + }) + } +} diff --git a/libgo/go/crypto/x509/test-file.crt b/libgo/go/crypto/x509/test-file.crt new file mode 100644 index 00000000000..caa83b9f824 --- /dev/null +++ b/libgo/go/crypto/x509/test-file.crt @@ -0,0 +1,32 @@ +-----BEGIN CERTIFICATE----- +MIIFbTCCA1WgAwIBAgIJAN338vEmMtLsMA0GCSqGSIb3DQEBCwUAME0xCzAJBgNV +BAYTAlVLMRMwEQYDVQQIDApUZXN0LVN0YXRlMRUwEwYDVQQKDAxHb2xhbmcgVGVz +dHMxEjAQBgNVBAMMCXRlc3QtZmlsZTAeFw0xNzAyMDEyMzUyMDhaFw0yNzAxMzAy +MzUyMDhaME0xCzAJBgNVBAYTAlVLMRMwEQYDVQQIDApUZXN0LVN0YXRlMRUwEwYD +VQQKDAxHb2xhbmcgVGVzdHMxEjAQBgNVBAMMCXRlc3QtZmlsZTCCAiIwDQYJKoZI +hvcNAQEBBQADggIPADCCAgoCggIBAPMGiLjdiffQo3Xc8oUe7wsDhSaAJFOhO6Qs +i0xYrYl7jmCuz9rGD2fdgk5cLqGazKuQ6fIFzHXFU2BKs4CWXt9KO0KFEhfvZeuW +jG5d7C1ZUiuKOrPqjKVu8SZtFPc7y7Ke7msXzY+Z2LLyiJJ93LCMq4+cTSGNXVlI +KqUxhxeoD5/QkUPyQy/ilu3GMYfx/YORhDP6Edcuskfj8wRh1UxBejP8YPMvI6St +cE2GkxoEGqDWnQ/61F18te6WI3MD29tnKXOkXVhnSC+yvRLljotW2/tAhHKBG4tj +iQWT5Ri4Wrw2tXxPKRLsVWc7e1/hdxhnuvYpXkWNhKsm002jzkFXlzfEwPd8nZdw +5aT6gPUBN2AAzdoqZI7E200i0orEF7WaSoMfjU1tbHvExp3vyAPOfJ5PS2MQ6W03 +Zsy5dTVH+OBH++rkRzQCFcnIv/OIhya5XZ9KX9nFPgBEP7Xq2A+IjH7B6VN/S/bv +8lhp2V+SQvlew9GttKC4hKuPsl5o7+CMbcqcNUdxm9gGkN8epGEKCuix97bpNlxN +fHZxHE5+8GMzPXMkCD56y5TNKR6ut7JGHMPtGl5lPCLqzG/HzYyFgxsDfDUu2B0A +GKj0lGpnLfGqwhs2/s3jpY7+pcvVQxEpvVTId5byDxu1ujP4HjO/VTQ2P72rE8Ft +C6J2Av0tAgMBAAGjUDBOMB0GA1UdDgQWBBTLT/RbyfBB/Pa07oBnaM+QSJPO9TAf +BgNVHSMEGDAWgBTLT/RbyfBB/Pa07oBnaM+QSJPO9TAMBgNVHRMEBTADAQH/MA0G +CSqGSIb3DQEBCwUAA4ICAQB3sCntCcQwhMgRPPyvOCMyTcQ/Iv+cpfxz2Ck14nlx +AkEAH2CH0ov5GWTt07/ur3aa5x+SAKi0J3wTD1cdiw4U/6Uin6jWGKKxvoo4IaeK +SbM8w/6eKx6UbmHx7PA/eRABY9tTlpdPCVgw7/o3WDr03QM+IAtatzvaCPPczake +pbdLwmBZB/v8V+6jUajy6jOgdSH0PyffGnt7MWgDETmNC6p/Xigp5eh+C8Fb4NGT +xgHES5PBC+sruWp4u22bJGDKTvYNdZHsnw/CaKQWNsQqwisxa3/8N5v+PCff/pxl +r05pE3PdHn9JrCl4iWdVlgtiI9BoPtQyDfa/OEFaScE8KYR8LxaAgdgp3zYncWls +BpwQ6Y/A2wIkhlD9eEp5Ib2hz7isXOs9UwjdriKqrBXqcIAE5M+YIk3+KAQKxAtd +4YsK3CSJ010uphr12YKqlScj4vuKFjuOtd5RyyMIxUG3lrrhAu2AzCeKCLdVgA8+ +75FrYMApUdvcjp4uzbBoED4XRQlx9kdFHVbYgmE/+yddBYJM8u4YlgAL0hW2/D8p +z9JWIfxVmjJnBnXaKGBuiUyZ864A3PJndP6EMMo7TzS2CDnfCYuJjvI0KvDjFNmc +rQA04+qfMSEz3nmKhbbZu4eYLzlADhfH8tT4GMtXf71WLA5AUHGf2Y4+HIHTsmHG +vQ== +-----END CERTIFICATE----- diff --git a/libgo/go/crypto/x509/testdata/test-dir.crt b/libgo/go/crypto/x509/testdata/test-dir.crt new file mode 100644 index 00000000000..b7fc9c51861 --- /dev/null +++ b/libgo/go/crypto/x509/testdata/test-dir.crt @@ -0,0 +1,31 @@ +-----BEGIN CERTIFICATE----- +MIIFazCCA1OgAwIBAgIJAL8a/lsnspOqMA0GCSqGSIb3DQEBCwUAMEwxCzAJBgNV +BAYTAlVLMRMwEQYDVQQIDApUZXN0LVN0YXRlMRUwEwYDVQQKDAxHb2xhbmcgVGVz +dHMxETAPBgNVBAMMCHRlc3QtZGlyMB4XDTE3MDIwMTIzNTAyN1oXDTI3MDEzMDIz +NTAyN1owTDELMAkGA1UEBhMCVUsxEzARBgNVBAgMClRlc3QtU3RhdGUxFTATBgNV +BAoMDEdvbGFuZyBUZXN0czERMA8GA1UEAwwIdGVzdC1kaXIwggIiMA0GCSqGSIb3 +DQEBAQUAA4ICDwAwggIKAoICAQDzBoi43Yn30KN13PKFHu8LA4UmgCRToTukLItM +WK2Je45grs/axg9n3YJOXC6hmsyrkOnyBcx1xVNgSrOAll7fSjtChRIX72Xrloxu +XewtWVIrijqz6oylbvEmbRT3O8uynu5rF82Pmdiy8oiSfdywjKuPnE0hjV1ZSCql +MYcXqA+f0JFD8kMv4pbtxjGH8f2DkYQz+hHXLrJH4/MEYdVMQXoz/GDzLyOkrXBN +hpMaBBqg1p0P+tRdfLXuliNzA9vbZylzpF1YZ0gvsr0S5Y6LVtv7QIRygRuLY4kF +k+UYuFq8NrV8TykS7FVnO3tf4XcYZ7r2KV5FjYSrJtNNo85BV5c3xMD3fJ2XcOWk ++oD1ATdgAM3aKmSOxNtNItKKxBe1mkqDH41NbWx7xMad78gDznyeT0tjEOltN2bM +uXU1R/jgR/vq5Ec0AhXJyL/ziIcmuV2fSl/ZxT4ARD+16tgPiIx+welTf0v27/JY +adlfkkL5XsPRrbSguISrj7JeaO/gjG3KnDVHcZvYBpDfHqRhCgrosfe26TZcTXx2 +cRxOfvBjMz1zJAg+esuUzSkerreyRhzD7RpeZTwi6sxvx82MhYMbA3w1LtgdABio +9JRqZy3xqsIbNv7N46WO/qXL1UMRKb1UyHeW8g8btboz+B4zv1U0Nj+9qxPBbQui +dgL9LQIDAQABo1AwTjAdBgNVHQ4EFgQUy0/0W8nwQfz2tO6AZ2jPkEiTzvUwHwYD +VR0jBBgwFoAUy0/0W8nwQfz2tO6AZ2jPkEiTzvUwDAYDVR0TBAUwAwEB/zANBgkq +hkiG9w0BAQsFAAOCAgEAvEVnUYsIOt87rggmLPqEueynkuQ+562M8EDHSQl82zbe +xDCxeg3DvPgKb+RvaUdt1362z/szK10SoeMgx6+EQLoV9LiVqXwNqeYfixrhrdw3 +ppAhYYhymdkbUQCEMHypmXP1vPhAz4o8Bs+eES1M+zO6ErBiD7SqkmBElT+GixJC +6epC9ZQFs+dw3lPlbiZSsGE85sqc3VAs0/JgpL/pb1/Eg4s0FUhZD2C2uWdSyZGc +g0/v3aXJCp4j/9VoNhI1WXz3M45nysZIL5OQgXymLqJElQa1pZ3Wa4i/nidvT4AT +Xlxc/qijM8set/nOqp7hVd5J0uG6qdwLRILUddZ6OpXd7ZNi1EXg+Bpc7ehzGsDt +3UFGzYXDjxYnK2frQfjLS8stOQIqSrGthW6x0fdkVx0y8BByvd5J6+JmZl4UZfzA +m99VxXSt4B9x6BvnY7ktzcFDOjtuLc4B/7yg9fv1eQuStA4cHGGAttsCg1X/Kx8W +PvkkeH0UWDZ9vhH9K36703z89da6MWF+bz92B0+4HoOmlVaXRkvblsNaynJnL0LC +Ayry7QBxuh5cMnDdRwJB3AVJIiJ1GVpb7aGvBOnx+s2lwRv9HWtghb+cbwwktx1M +JHyBf3GZNSWTpKY7cD8V+NnBv3UuioOVVo+XAU4LF/bYUjdRpxWADJizNtZrtFo= +-----END CERTIFICATE----- diff --git a/libgo/go/crypto/x509/verify.go b/libgo/go/crypto/x509/verify.go index 29345a1755c..2b4f39d62ea 100644 --- a/libgo/go/crypto/x509/verify.go +++ b/libgo/go/crypto/x509/verify.go @@ -87,7 +87,7 @@ func (h HostnameError) Error() string { valid += san.String() } } else { - if len(c.DNSNames) > 0 { + if c.hasSANExtension() { valid = strings.Join(c.DNSNames, ", ") } else { valid = c.Subject.CommonName @@ -166,7 +166,7 @@ const ( func matchNameConstraint(domain, constraint string) bool { // The meaning of zero length constraints is not specified, but this - // code follows NSS and accepts them as valid for everything. + // code follows NSS and accepts them as matching everything. if len(constraint) == 0 { return true } @@ -220,6 +220,12 @@ func (c *Certificate) isValid(certType int, currentChain []*Certificate, opts *V } } + for _, constraint := range c.ExcludedDNSDomains { + if matchNameConstraint(opts.DNSName, constraint) { + return CertificateInvalidError{c, CANotAuthorizedForThisName} + } + } + // KeyUsage status flags are ignored. From Engineering Security, Peter // Gutmann: A European government CA marked its signing certificates as // being valid for encryption only, but no-one noticed. Another @@ -482,7 +488,7 @@ func (c *Certificate) VerifyHostname(h string) error { lowered := toLowerCaseASCII(h) - if len(c.DNSNames) > 0 { + if c.hasSANExtension() { for _, match := range c.DNSNames { if matchHostnames(toLowerCaseASCII(match), lowered) { return nil diff --git a/libgo/go/crypto/x509/verify_test.go b/libgo/go/crypto/x509/verify_test.go index 15c40914444..335c477d0da 100644 --- a/libgo/go/crypto/x509/verify_test.go +++ b/libgo/go/crypto/x509/verify_test.go @@ -263,6 +263,39 @@ var verifyTests = []verifyTest{ errorCallback: expectSubjectIssuerMismatcthError, }, + { + // An X.509 v1 certificate should not be accepted as an + // intermediate. + leaf: x509v1TestLeaf, + intermediates: []string{x509v1TestIntermediate}, + roots: []string{x509v1TestRoot}, + currentTime: 1481753183, + systemSkip: true, + + errorCallback: expectNotAuthorizedError, + }, + { + // If any SAN extension is present (even one without any DNS + // names), the CN should be ignored. + leaf: ignoreCNWithSANLeaf, + dnsName: "foo.example.com", + roots: []string{ignoreCNWithSANRoot}, + currentTime: 1486684488, + systemSkip: true, + + errorCallback: expectHostnameError, + }, + { + // Test that excluded names are respected. + leaf: excludedNamesLeaf, + dnsName: "bender.local", + intermediates: []string{excludedNamesIntermediate}, + roots: []string{excludedNamesRoot}, + currentTime: 1486684488, + systemSkip: true, + + errorCallback: expectNameConstraintsError, + }, } func expectHostnameError(t *testing.T, i int, err error) (ok bool) { @@ -330,6 +363,22 @@ func expectSubjectIssuerMismatcthError(t *testing.T, i int, err error) (ok bool) return true } +func expectNameConstraintsError(t *testing.T, i int, err error) (ok bool) { + if inval, ok := err.(CertificateInvalidError); !ok || inval.Reason != CANotAuthorizedForThisName { + t.Errorf("#%d: error was not a CANotAuthorizedForThisName: %s", i, err) + return false + } + return true +} + +func expectNotAuthorizedError(t *testing.T, i int, err error) (ok bool) { + if inval, ok := err.(CertificateInvalidError); !ok || inval.Reason != NotAuthorizedToSign { + t.Errorf("#%d: error was not a NotAuthorizedToSign: %s", i, err) + return false + } + return true +} + func certificateFromPEM(pemBytes string) (*Certificate, error) { block, _ := pem.Decode([]byte(pemBytes)) if block == nil { @@ -1269,6 +1318,174 @@ vEA4PCJprUYmoj3q9MKSSRYDlq5kIbl87mSRR4GqtAwJKxIasvOvULOxziQ= -----END CERTIFICATE----- ` +const x509v1TestRoot = ` +-----BEGIN CERTIFICATE----- +MIICIDCCAYmgAwIBAgIIAj5CwoHlWuYwDQYJKoZIhvcNAQELBQAwIzEPMA0GA1UE +ChMGR29sYW5nMRAwDgYDVQQDEwdSb290IENBMB4XDTE1MDEwMTAwMDAwMFoXDTI1 +MDEwMTAwMDAwMFowIzEPMA0GA1UEChMGR29sYW5nMRAwDgYDVQQDEwdSb290IENB +MIGfMA0GCSqGSIb3DQEBAQUAA4GNADCBiQKBgQDpDn8RDOZa5oaDcPZRBy4CeBH1 +siSSOO4mYgLHlPE+oXdqwI/VImi2XeJM2uCFETXCknJJjYG0iJdrt/yyRFvZTQZw ++QzGj+mz36NqhGxDWb6dstB2m8PX+plZw7jl81MDvUnWs8yiQ/6twgu5AbhWKZQD +JKcNKCEpqa6UW0r5nwIDAQABo10wWzAOBgNVHQ8BAf8EBAMCAgQwHQYDVR0lBBYw +FAYIKwYBBQUHAwEGCCsGAQUFBwMCMA8GA1UdEwEB/wQFMAMBAf8wGQYDVR0OBBIE +EEA31wH7QC+4HH5UBCeMWQEwDQYJKoZIhvcNAQELBQADgYEAcIwqeNUpQr9cOcYm +YjpGpYkQ6b248xijCK7zI+lOeWN89zfSXn1AvfsC9pSdTMeDklWktbF/Ad0IN8Md +h2NtN34ard0hEfHc8qW8mkXdsysVmq6cPvFYaHz+dBtkHuHDoy8YQnC0zdN/WyYB +/1JmacUUofl+HusHuLkDxmadogI= +-----END CERTIFICATE-----` + +const x509v1TestIntermediate = ` +-----BEGIN CERTIFICATE----- +MIIByjCCATMCCQCCdEMsT8ykqTANBgkqhkiG9w0BAQsFADAjMQ8wDQYDVQQKEwZH +b2xhbmcxEDAOBgNVBAMTB1Jvb3QgQ0EwHhcNMTUwMTAxMDAwMDAwWhcNMjUwMTAx +MDAwMDAwWjAwMQ8wDQYDVQQKEwZHb2xhbmcxHTAbBgNVBAMTFFguNTA5djEgaW50 +ZXJtZWRpYXRlMIGfMA0GCSqGSIb3DQEBAQUAA4GNADCBiQKBgQDJ2QyniAOT+5YL +jeinEBJr3NsC/Q2QJ/VKmgvp+xRxuKTHJiVmxVijmp0vWg8AWfkmuE4p3hXQbbqM +k5yxrk1n60ONhim2L4VXriEvCE7X2OXhTmBls5Ufr7aqIgPMikwjScCXwz8E8qI8 +UxyAhnjeJwMYBU8TuwBImSd4LBHoQQIDAQABMA0GCSqGSIb3DQEBCwUAA4GBAIab +DRG6FbF9kL9jb/TDHkbVBk+sl/Pxi4/XjuFyIALlARgAkeZcPmL5tNW1ImHkwsHR +zWE77kJDibzd141u21ZbLsKvEdUJXjla43bdyMmEqf5VGpC3D4sFt3QVH7lGeRur +x5Wlq1u3YDL/j6s1nU2dQ3ySB/oP7J+vQ9V4QeM+ +-----END CERTIFICATE-----` + +const x509v1TestLeaf = ` +-----BEGIN CERTIFICATE----- +MIICMzCCAZygAwIBAgIJAPo99mqJJrpJMA0GCSqGSIb3DQEBCwUAMDAxDzANBgNV +BAoTBkdvbGFuZzEdMBsGA1UEAxMUWC41MDl2MSBpbnRlcm1lZGlhdGUwHhcNMTUw +MTAxMDAwMDAwWhcNMjUwMTAxMDAwMDAwWjArMQ8wDQYDVQQKEwZHb2xhbmcxGDAW +BgNVBAMTD2Zvby5leGFtcGxlLmNvbTCBnzANBgkqhkiG9w0BAQEFAAOBjQAwgYkC +gYEApUh60Z+a5/oKJxG//Dn8CihSo2CJHNIIO3zEJZ1EeNSMZCynaIR6D3IPZEIR ++RG2oGt+f5EEukAPYxwasp6VeZEezoQWJ+97nPCT6DpwLlWp3i2MF8piK2R9vxkG +Z5n0+HzYk1VM8epIrZFUXSMGTX8w1y041PX/yYLxbdEifdcCAwEAAaNaMFgwDgYD +VR0PAQH/BAQDAgWgMB0GA1UdJQQWMBQGCCsGAQUFBwMBBggrBgEFBQcDAjAMBgNV +HRMBAf8EAjAAMBkGA1UdDgQSBBBFozXe0SnzAmjy+1U6M/cvMA0GCSqGSIb3DQEB +CwUAA4GBADYzYUvaToO/ucBskPdqXV16AaakIhhSENswYVSl97/sODaxsjishKq9 +5R7siu+JnIFotA7IbBe633p75xEnLN88X626N/XRFG9iScLzpj0o0PWXBUiB+fxL +/jt8qszOXCv2vYdUTPNuPqufXLWMoirpuXrr1liJDmedCcAHepY/ +-----END CERTIFICATE-----` + +const ignoreCNWithSANRoot = ` +-----BEGIN CERTIFICATE----- +MIIDPzCCAiegAwIBAgIIJkzCwkNrPHMwDQYJKoZIhvcNAQELBQAwMDEQMA4GA1UE +ChMHVEVTVElORzEcMBoGA1UEAxMTKipUZXN0aW5nKiogUm9vdCBDQTAeFw0xNTAx +MDEwMDAwMDBaFw0yNTAxMDEwMDAwMDBaMDAxEDAOBgNVBAoTB1RFU1RJTkcxHDAa +BgNVBAMTEyoqVGVzdGluZyoqIFJvb3QgQ0EwggEiMA0GCSqGSIb3DQEBAQUAA4IB +DwAwggEKAoIBAQC4YAf5YqlXGcikvbMWtVrNICt+V/NNWljwfvSKdg4Inm7k6BwW +P6y4Y+n4qSYIWNU4iRkdpajufzctxQCO6ty13iw3qVktzcC5XBIiS6ymiRhhDgnY +VQqyakVGw9MxrPwdRZVlssUv3Hmy6tU+v5Ok31SLY5z3wKgYWvSyYs0b8bKNU8kf +2FmSHnBN16lxGdjhe3ji58F/zFMr0ds+HakrLIvVdFcQFAnQopM8FTHpoWNNzGU3 +KaiO0jBbMFkd6uVjVnuRJ+xjuiqi/NWwiwQA+CEr9HKzGkxOF8nAsHamdmO1wW+w +OsCrC0qWQ/f5NTOVATTJe0vj88OMTvo3071VAgMBAAGjXTBbMA4GA1UdDwEB/wQE +AwICpDAdBgNVHSUEFjAUBggrBgEFBQcDAQYIKwYBBQUHAwIwDwYDVR0TAQH/BAUw +AwEB/zAZBgNVHQ4EEgQQQDfXAftAL7gcflQEJ4xZATANBgkqhkiG9w0BAQsFAAOC +AQEAGOn3XjxHyHbXLKrRmpwV447B7iNBXR5VlhwOgt1kWaHDL2+8f/9/h0HMkB6j +fC+/yyuYVqYuOeavqMGVrh33D2ODuTQcFlOx5lXukP46j3j+Lm0jjZ1qNX7vlP8I +VlUXERhbelkw8O4oikakwIY9GE8syuSgYf+VeBW/lvuAZQrdnPfabxe05Tre6RXy +nJHMB1q07YHpbwIkcV/lfCE9pig2nPXTLwYZz9cl46Ul5RCpPUi+IKURo3x8y0FU +aSLjI/Ya0zwUARMmyZ3RRGCyhIarPb20mKSaMf1/Nb23pS3k1QgmZhk5pAnXYsWu +BJ6bvwEAasFiLGP6Zbdmxb2hIA== +-----END CERTIFICATE-----` + +const ignoreCNWithSANLeaf = ` +-----BEGIN CERTIFICATE----- +MIIDaTCCAlGgAwIBAgIJAONakvRTxgJhMA0GCSqGSIb3DQEBCwUAMDAxEDAOBgNV +BAoTB1RFU1RJTkcxHDAaBgNVBAMTEyoqVGVzdGluZyoqIFJvb3QgQ0EwHhcNMTUw +MTAxMDAwMDAwWhcNMjUwMTAxMDAwMDAwWjAsMRAwDgYDVQQKEwdURVNUSU5HMRgw +FgYDVQQDEw9mb28uZXhhbXBsZS5jb20wggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAw +ggEKAoIBAQDBqskp89V/JMIBBqcauKSOVLcMyIE/t0jgSWVrsI4sksBTabLsfMdS +ui2n+dHQ1dRBuw3o4g4fPrWwS3nMnV3pZUHEn2TPi5N1xkjTaxObXgKIY2GKmFP3 +rJ9vYqHT6mT4K93kCHoRcmJWWySc7S3JAOhTcdB4G+tIdQJN63E+XRYQQfNrn5HZ +hxQoOzaguHFx+ZGSD4Ntk6BSZz5NfjqCYqYxe+iCpTpEEYhIpi8joSPSmkTMTxBW +S1W2gXbYNQ9KjNkGM6FnQsUJrSPMrWs4v3UB/U88N5LkZeF41SqD9ySFGwbGajFV +nyzj12+4K4D8BLhlOc0Eo/F/8GwOwvmxAgMBAAGjgYkwgYYwDgYDVR0PAQH/BAQD +AgWgMB0GA1UdJQQWMBQGCCsGAQUFBwMBBggrBgEFBQcDAjAMBgNVHRMBAf8EAjAA +MBkGA1UdDgQSBBCjeab27q+5pV43jBGANOJ1MBsGA1UdIwQUMBKAEEA31wH7QC+4 +HH5UBCeMWQEwDwYDVR0RBAgwBocEfwAAATANBgkqhkiG9w0BAQsFAAOCAQEAGZfZ +ErTVxxpIg64s22mQpXSk/72THVQsfsKHzlXmztM0CJzH8ccoN67ZqKxJCfdiE/FI +Emb6BVV4cGPeIKpcxaM2dwX/Y+Y0JaxpQJvqLxs+EByRL0gPP3shgg86WWCjYLxv +AgOn862d/JXGDrC9vIlQ/DDQcyL5g0JV5UjG2G9TUigbnrXxBw7BoWK6wmoSaHnR +sZKEHSs3RUJvm7qqpA9Yfzm9jg+i9j32zh1xFacghAOmFRFXa9eCVeigZ/KK2mEY +j2kBQyvnyKsXHLAKUoUOpd6t/1PHrfXnGj+HmzZNloJ/BZ1kiWb4eLvMljoLGkZn +xZbqP3Krgjj4XNaXjg== +-----END CERTIFICATE-----` + +const excludedNamesLeaf = ` +-----BEGIN CERTIFICATE----- +MIID4DCCAsigAwIBAgIHDUSFtJknhzANBgkqhkiG9w0BAQsFADCBnjELMAkGA1UE +BhMCVVMxEzARBgNVBAgMCkNhbGlmb3JuaWExEjAQBgNVBAcMCUxvcyBHYXRvczEU +MBIGA1UECgwLTmV0ZmxpeCBJbmMxLTArBgNVBAsMJFBsYXRmb3JtIFNlY3VyaXR5 +ICgzNzM0NTE1NTYyODA2Mzk3KTEhMB8GA1UEAwwYSW50ZXJtZWRpYXRlIENBIGZv +ciAzMzkyMB4XDTE3MDIwODIxMTUwNFoXDTE4MDIwODIwMjQ1OFowgZAxCzAJBgNV +BAYTAlVTMRMwEQYDVQQIDApDYWxpZm9ybmlhMRIwEAYDVQQHDAlMb3MgR2F0b3Mx +FDASBgNVBAoMC05ldGZsaXggSW5jMS0wKwYDVQQLDCRQbGF0Zm9ybSBTZWN1cml0 +eSAoMzczNDUxNTc0ODUwMjY5NikxEzARBgNVBAMMCjE3Mi4xNi4wLjEwggEiMA0G +CSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQCZ0oP1bMv6bOeqcKbzinnGpNOpenhA +zdFFsgea62znWsH3Wg4+1Md8uPCqlaQIsaJQKZHc50eKD3bg0Io7c6kxHkBQr1b8 +Q7cGeK3CjdqG3NwS/aizzrLKOwL693hFwwy7JY7GGCvogbhyQRKn6iV0U9zMm7bu +/9pQVV/wx8u01u2uAlLttjyQ5LJkxo5t8cATFVqxdN5J9eY//VSDiTwXnlpQITBP +/Ow+zYuZ3kFlzH3CtCOhOEvNG3Ar1NvP3Icq35PlHV+Eki4otnKfixwByoiGpqCB +UEIY04VrZJjwBxk08y/3jY2B3VLYGgi+rryyCxIqkB7UpSNPMMWSG4UpAgMBAAGj +LzAtMAwGA1UdEwEB/wQCMAAwHQYDVR0RBBYwFIIMYmVuZGVyLmxvY2FshwSsEAAB +MA0GCSqGSIb3DQEBCwUAA4IBAQCLW3JO8L7LKByjzj2RciPjCGH5XF87Wd20gYLq +sNKcFwCIeyZhnQy5aZ164a5G9AIk2HLvH6HevBFPhA9Ivmyv/wYEfnPd1VcFkpgP +hDt8MCFJ8eSjCyKdtZh1MPMLrLVymmJV+Rc9JUUYM9TIeERkpl0rskcO1YGewkYt +qKlWE+0S16+pzsWvKn831uylqwIb8ANBPsCX4aM4muFBHavSWAHgRO+P+yXVw8Q+ +VQDnMHUe5PbZd1/+1KKVs1K/CkBCtoHNHp1d/JT+2zUQJphwja9CcgfFdVhSnHL4 +oEEOFtqVMIuQfR2isi08qW/JGOHc4sFoLYB8hvdaxKWSE19A +-----END CERTIFICATE----- +` + +const excludedNamesIntermediate = ` +-----BEGIN CERTIFICATE----- +MIIDzTCCArWgAwIBAgIHDUSFqYeczDANBgkqhkiG9w0BAQsFADCBmTELMAkGA1UE +BhMCVVMxEzARBgNVBAgMCkNhbGlmb3JuaWExEjAQBgNVBAcMCUxvcyBHYXRvczEU +MBIGA1UECgwLTmV0ZmxpeCBJbmMxLTArBgNVBAsMJFBsYXRmb3JtIFNlY3VyaXR5 +ICgzNzM0NTE1NDc5MDY0NjAyKTEcMBoGA1UEAwwTTG9jYWwgUm9vdCBmb3IgMzM5 +MjAeFw0xNzAyMDgyMTE1MDRaFw0xODAyMDgyMDI0NThaMIGeMQswCQYDVQQGEwJV +UzETMBEGA1UECAwKQ2FsaWZvcm5pYTESMBAGA1UEBwwJTG9zIEdhdG9zMRQwEgYD +VQQKDAtOZXRmbGl4IEluYzEtMCsGA1UECwwkUGxhdGZvcm0gU2VjdXJpdHkgKDM3 +MzQ1MTU1NjI4MDYzOTcpMSEwHwYDVQQDDBhJbnRlcm1lZGlhdGUgQ0EgZm9yIDMz +OTIwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQCOyEs6tJ/t9emQTvlx +3FS7uJSou5rKkuqVxZdIuYQ+B2ZviBYUnMRT9bXDB0nsVdKZdp0hdchdiwNXDG/I +CiWu48jkcv/BdynVyayOT+0pOJSYLaPYpzBx1Pb9M5651ct9GSbj6Tz0ChVonoIE +1AIZ0kkebucZRRFHd0xbAKVRKyUzPN6HJ7WfgyauUp7RmlC35wTmrmARrFohQLlL +7oICy+hIQePMy9x1LSFTbPxZ5AUUXVC3eUACU3vLClF/Xs8XGHebZpUXCdMQjOGS +nq1eFguFHR1poSB8uSmmLqm4vqUH9CDhEgiBAC8yekJ8//kZQ7lUEqZj3YxVbk+Y +E4H5AgMBAAGjEzARMA8GA1UdEwEB/wQFMAMBAf8wDQYJKoZIhvcNAQELBQADggEB +ADxrnmNX5gWChgX9K5fYwhFDj5ofxZXAKVQk+WjmkwMcmCx3dtWSm++Wdksj/ZlA +V1cLW3ohWv1/OAZuOlw7sLf98aJpX+UUmIYYQxDubq+4/q7VA7HzEf2k/i/oN1NI +JgtrhpPcZ/LMO6k7DYx0qlfYq8pTSfd6MI4LnWKgLc+JSPJJjmvspgio2ZFcnYr7 +A264BwLo6v1Mos1o1JUvFFcp4GANlw0XFiWh7JXYRl8WmS5DoouUC+aNJ3lmyF6z +LbIjZCSfgZnk/LK1KU1j91FI2bc2ULYZvAC1PAg8/zvIgxn6YM2Q7ZsdEgWw0FpS +zMBX1/lk4wkFckeUIlkD55Y= +-----END CERTIFICATE-----` + +const excludedNamesRoot = ` +-----BEGIN CERTIFICATE----- +MIIEGTCCAwGgAwIBAgIHDUSFpInn/zANBgkqhkiG9w0BAQsFADCBozELMAkGA1UE +BhMCVVMxEzARBgNVBAgMCkNhbGlmb3JuaWExEjAQBgNVBAcMCUxvcyBHYXRvczEU +MBIGA1UECgwLTmV0ZmxpeCBJbmMxLTArBgNVBAsMJFBsYXRmb3JtIFNlY3VyaXR5 +ICgzNzMxNTA5NDM3NDYyNDg1KTEmMCQGA1UEAwwdTmFtZSBDb25zdHJhaW50cyBU +ZXN0IFJvb3QgQ0EwHhcNMTcwMjA4MjExNTA0WhcNMTgwMjA4MjAyNDU4WjCBmTEL +MAkGA1UEBhMCVVMxEzARBgNVBAgMCkNhbGlmb3JuaWExEjAQBgNVBAcMCUxvcyBH +YXRvczEUMBIGA1UECgwLTmV0ZmxpeCBJbmMxLTArBgNVBAsMJFBsYXRmb3JtIFNl +Y3VyaXR5ICgzNzM0NTE1NDc5MDY0NjAyKTEcMBoGA1UEAwwTTG9jYWwgUm9vdCBm +b3IgMzM5MjCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAJymcnX29ekc +7+MLyr8QuAzoHWznmGdDd2sITwWRjM89/21cdlHCGKSpULUNdFp9HDLWvYECtxt+ +8TuzKiQz7qAerzGUT1zI5McIjHy0e/i4xIkfiBiNeTCuB/N9QRbZlcfM80ErkaA4 +gCAFK8qZAcWkHIl6e+KaQFMPLKk9kckgAnVDHEJe8oLNCogCJ15558b65g05p9eb +5Lg+E98hoPRTQaDwlz3CZPfTTA2EiEZInSi8qzodFCbTpJUVTbiVUH/JtVjlibbb +smdcx5PORK+8ZJkhLEh54AjaWOX4tB/7Tkk8stg2VBmrIARt/j4UVj7cTrIWU3bV +m8TwHJG+YgsCAwEAAaNaMFgwDwYDVR0TAQH/BAUwAwEB/zBFBgNVHR4EPjA8oBww +CocICgEAAP//AAAwDoIMYmVuZGVyLmxvY2FsoRwwCocICgEAAP//AAAwDoIMYmVu +ZGVyLmxvY2FsMA0GCSqGSIb3DQEBCwUAA4IBAQAMjbheffPxtSKSv9NySW+8qmHs +n7Mb5GGyCFu+cMZSoSaabstbml+zHEFJvWz6/1E95K4F8jKhAcu/CwDf4IZrSD2+ +Hee0DolVSQhZpnHgPyj7ZATz48e3aJaQPUlhCEOh0wwF4Y0N4FV0t7R6woLylYRZ +yU1yRHUqUYpN0DWFpsPbBqgM6uUAVO2ayBFhPgWUaqkmSbZ/Nq7isGvknaTmcIwT +6mOAFN0qFb4RGzfGJW7x6z7KCULS7qVDp6fU3tRoScHFEgRubks6jzQ1W5ooSm4o ++NQCZDd5eFeU8PpNX7rgaYE4GPq+EEmLVCBYmdctr8QVdqJ//8Xu3+1phjDy +-----END CERTIFICATE-----` + var unknownAuthorityErrorTests = []struct { cert string expected string @@ -1294,7 +1511,7 @@ func TestUnknownAuthorityError(t *testing.T) { hintCert: c, } actual := uae.Error() - if strings.Compare(actual, tt.expected) != 0 { + if actual != tt.expected { t.Errorf("#%d: UnknownAuthorityError.Error() response invalid actual: %s expected: %s", i, actual, tt.expected) } } diff --git a/libgo/go/crypto/x509/x509.go b/libgo/go/crypto/x509/x509.go index 949ce018561..fdc7c5307bd 100644 --- a/libgo/go/crypto/x509/x509.go +++ b/libgo/go/crypto/x509/x509.go @@ -3,6 +3,10 @@ // license that can be found in the LICENSE file. // Package x509 parses X.509-encoded keys and certificates. +// +// On UNIX systems the environment variables SSL_CERT_FILE and SSL_CERT_DIR +// can be used to override the system default locations for the SSL certificate +// file and SSL certificate files directory, respectively. package x509 import ( @@ -59,7 +63,7 @@ func ParsePKIXPublicKey(derBytes []byte) (pub interface{}, err error) { func marshalPublicKey(pub interface{}) (publicKeyBytes []byte, publicKeyAlgorithm pkix.AlgorithmIdentifier, err error) { switch pub := pub.(type) { case *rsa.PublicKey: - publicKeyBytes, err = asn1.Marshal(rsaPublicKey{ + publicKeyBytes, err = asn1.Marshal(pkcs1PublicKey{ N: pub.N, E: pub.E, }) @@ -69,9 +73,7 @@ func marshalPublicKey(pub interface{}) (publicKeyBytes []byte, publicKeyAlgorith publicKeyAlgorithm.Algorithm = oidPublicKeyRSA // This is a NULL parameters value which is required by // https://tools.ietf.org/html/rfc3279#section-2.3.1. - publicKeyAlgorithm.Parameters = asn1.RawValue{ - Tag: 5, - } + publicKeyAlgorithm.Parameters = asn1.NullRawValue case *ecdsa.PublicKey: publicKeyBytes = elliptic.Marshal(pub.Curve, pub.X, pub.Y) oid, ok := oidFromNamedCurve(pub.Curve) @@ -355,10 +357,8 @@ func rsaPSSParameters(hashFunc crypto.Hash) asn1.RawValue { params := pssParameters{ Hash: pkix.AlgorithmIdentifier{ - Algorithm: hashOID, - Parameters: asn1.RawValue{ - Tag: 5, /* ASN.1 NULL */ - }, + Algorithm: hashOID, + Parameters: asn1.NullRawValue, }, MGF: pkix.AlgorithmIdentifier{ Algorithm: oidMGF1, @@ -368,10 +368,8 @@ func rsaPSSParameters(hashFunc crypto.Hash) asn1.RawValue { } mgf1Params := pkix.AlgorithmIdentifier{ - Algorithm: hashOID, - Parameters: asn1.RawValue{ - Tag: 5, /* ASN.1 NULL */ - }, + Algorithm: hashOID, + Parameters: asn1.NullRawValue, } var err error @@ -418,11 +416,10 @@ func getSignatureAlgorithmFromAI(ai pkix.AlgorithmIdentifier) SignatureAlgorithm // https://tools.ietf.org/html/rfc3447#section-8.1), that the // salt length matches the hash length, and that the trailer // field has the default value. - asn1NULL := []byte{0x05, 0x00} - if !bytes.Equal(params.Hash.Parameters.FullBytes, asn1NULL) || + if !bytes.Equal(params.Hash.Parameters.FullBytes, asn1.NullBytes) || !params.MGF.Algorithm.Equal(oidMGF1) || !mgf1HashFunc.Algorithm.Equal(params.Hash.Algorithm) || - !bytes.Equal(mgf1HashFunc.Parameters.FullBytes, asn1NULL) || + !bytes.Equal(mgf1HashFunc.Parameters.FullBytes, asn1.NullBytes) || params.TrailerField != 1 { return UnknownSignatureAlgorithm } @@ -668,13 +665,28 @@ type Certificate struct { ExtKeyUsage []ExtKeyUsage // Sequence of extended key usages. UnknownExtKeyUsage []asn1.ObjectIdentifier // Encountered extended key usages unknown to this package. - BasicConstraintsValid bool // if true then the next two fields are valid. + // BasicConstraintsValid indicates whether IsCA, MaxPathLen, + // and MaxPathLenZero are valid. + BasicConstraintsValid bool IsCA bool - MaxPathLen int - // MaxPathLenZero indicates that BasicConstraintsValid==true and - // MaxPathLen==0 should be interpreted as an actual maximum path length - // of zero. Otherwise, that combination is interpreted as MaxPathLen - // not being set. + + // MaxPathLen and MaxPathLenZero indicate the presence and + // value of the BasicConstraints' "pathLenConstraint". + // + // When parsing a certificate, a positive non-zero MaxPathLen + // means that the field was specified, -1 means it was unset, + // and MaxPathLenZero being true mean that the field was + // explicitly set to zero. The case of MaxPathLen==0 with MaxPathLenZero==false + // should be treated equivalent to -1 (unset). + // + // When generating a certificate, an unset pathLenConstraint + // can be requested with either MaxPathLen == -1 or using the + // zero value for both MaxPathLen and MaxPathLenZero. + MaxPathLen int + // MaxPathLenZero indicates that BasicConstraintsValid==true + // and MaxPathLen==0 should be interpreted as an actual + // maximum path length of zero. Otherwise, that combination is + // interpreted as MaxPathLen not being set. MaxPathLenZero bool SubjectKeyId []byte @@ -692,6 +704,7 @@ type Certificate struct { // Name constraints PermittedDNSDomainsCritical bool // if true then the name constraints are marked critical. PermittedDNSDomains []string + ExcludedDNSDomains []string // CRL Distribution Points CRLDistributionPoints []string @@ -723,6 +736,10 @@ func (c *Certificate) Equal(other *Certificate) bool { return bytes.Equal(c.Raw, other.Raw) } +func (c *Certificate) hasSANExtension() bool { + return oidInExtensions(oidExtensionSubjectAltName, c.Extensions) +} + // Entrust have a broken root certificate (CN=Entrust.net Certification // Authority (2048)) which isn't marked as a CA certificate and is thus invalid // according to PKIX. @@ -924,20 +941,17 @@ type distributionPointName struct { RelativeName pkix.RDNSequence `asn1:"optional,tag:1"` } -// asn1Null is the ASN.1 encoding of a NULL value. -var asn1Null = []byte{5, 0} - func parsePublicKey(algo PublicKeyAlgorithm, keyData *publicKeyInfo) (interface{}, error) { asn1Data := keyData.PublicKey.RightAlign() switch algo { case RSA: // RSA public keys must have a NULL in the parameters // (https://tools.ietf.org/html/rfc3279#section-2.3.1). - if !bytes.Equal(keyData.Algorithm.Parameters.FullBytes, asn1Null) { + if !bytes.Equal(keyData.Algorithm.Parameters.FullBytes, asn1.NullBytes) { return nil, errors.New("x509: RSA key missing NULL parameters") } - p := new(rsaPublicKey) + p := new(pkcs1PublicKey) rest, err := asn1.Unmarshal(asn1Data, p) if err != nil { return nil, err @@ -1150,7 +1164,7 @@ func parseCertificate(in *certificate) (*Certificate, error) { out.IsCA = constraints.IsCA out.MaxPathLen = constraints.MaxPathLen out.MaxPathLenZero = out.MaxPathLen == 0 - + // TODO: map out.MaxPathLen to 0 if it has the -1 default value? (Issue 19285) case 17: out.DNSNames, out.EmailAddresses, out.IPAddresses, err = parseSANExtension(e.Value) if err != nil { @@ -1185,19 +1199,27 @@ func parseCertificate(in *certificate) (*Certificate, error) { return nil, errors.New("x509: trailing data after X.509 NameConstraints") } - if len(constraints.Excluded) > 0 && e.Critical { - return out, UnhandledCriticalExtension{} - } - - for _, subtree := range constraints.Permitted { - if len(subtree.Name) == 0 { - if e.Critical { - return out, UnhandledCriticalExtension{} + getDNSNames := func(subtrees []generalSubtree, isCritical bool) (dnsNames []string, err error) { + for _, subtree := range subtrees { + if len(subtree.Name) == 0 { + if isCritical { + return nil, UnhandledCriticalExtension{} + } + continue } - continue + dnsNames = append(dnsNames, subtree.Name) } - out.PermittedDNSDomains = append(out.PermittedDNSDomains, subtree.Name) + + return dnsNames, nil + } + + if out.PermittedDNSDomains, err = getDNSNames(constraints.Permitted, e.Critical); err != nil { + return out, err + } + if out.ExcludedDNSDomains, err = getDNSNames(constraints.Excluded, e.Critical); err != nil { + return out, err } + out.PermittedDNSDomainsCritical = e.Critical case 31: // RFC 5280, 4.2.1.13 @@ -1451,7 +1473,7 @@ func marshalSANs(dnsNames, emailAddresses []string, ipAddresses []net.IP) (derBy return asn1.Marshal(rawValues) } -func buildExtensions(template *Certificate) (ret []pkix.Extension, err error) { +func buildExtensions(template *Certificate, authorityKeyId []byte) (ret []pkix.Extension, err error) { ret = make([]pkix.Extension, 10 /* maximum number of elements. */) n := 0 @@ -1525,9 +1547,9 @@ func buildExtensions(template *Certificate) (ret []pkix.Extension, err error) { n++ } - if len(template.AuthorityKeyId) > 0 && !oidInExtensions(oidExtensionAuthorityKeyId, template.ExtraExtensions) { + if len(authorityKeyId) > 0 && !oidInExtensions(oidExtensionAuthorityKeyId, template.ExtraExtensions) { ret[n].Id = oidExtensionAuthorityKeyId - ret[n].Value, err = asn1.Marshal(authKeyId{template.AuthorityKeyId}) + ret[n].Value, err = asn1.Marshal(authKeyId{authorityKeyId}) if err != nil { return } @@ -1581,16 +1603,22 @@ func buildExtensions(template *Certificate) (ret []pkix.Extension, err error) { n++ } - if len(template.PermittedDNSDomains) > 0 && + if (len(template.PermittedDNSDomains) > 0 || len(template.ExcludedDNSDomains) > 0) && !oidInExtensions(oidExtensionNameConstraints, template.ExtraExtensions) { ret[n].Id = oidExtensionNameConstraints ret[n].Critical = template.PermittedDNSDomainsCritical var out nameConstraints + out.Permitted = make([]generalSubtree, len(template.PermittedDNSDomains)) for i, permitted := range template.PermittedDNSDomains { out.Permitted[i] = generalSubtree{Name: permitted} } + out.Excluded = make([]generalSubtree, len(template.ExcludedDNSDomains)) + for i, excluded := range template.ExcludedDNSDomains { + out.Excluded[i] = generalSubtree{Name: excluded} + } + ret[n].Value, err = asn1.Marshal(out) if err != nil { return @@ -1646,9 +1674,7 @@ func signingParamsForPublicKey(pub interface{}, requestedSigAlgo SignatureAlgori pubType = RSA hashFunc = crypto.SHA256 sigAlgo.Algorithm = oidSignatureSHA256WithRSA - sigAlgo.Parameters = asn1.RawValue{ - Tag: 5, - } + sigAlgo.Parameters = asn1.NullRawValue case *ecdsa.PublicKey: pubType = ECDSA @@ -1706,11 +1732,12 @@ func signingParamsForPublicKey(pub interface{}, requestedSigAlgo SignatureAlgori return } -// CreateCertificate creates a new certificate based on a template. The -// following members of template are used: SerialNumber, Subject, NotBefore, -// NotAfter, KeyUsage, ExtKeyUsage, UnknownExtKeyUsage, BasicConstraintsValid, -// IsCA, MaxPathLen, SubjectKeyId, DNSNames, PermittedDNSDomainsCritical, -// PermittedDNSDomains, SignatureAlgorithm. +// CreateCertificate creates a new certificate based on a template. +// The following members of template are used: AuthorityKeyId, +// BasicConstraintsValid, DNSNames, ExcludedDNSDomains, ExtKeyUsage, +// IsCA, KeyUsage, MaxPathLen, MaxPathLenZero, NotAfter, NotBefore, +// PermittedDNSDomains, PermittedDNSDomainsCritical, SerialNumber, +// SignatureAlgorithm, Subject, SubjectKeyId, and UnknownExtKeyUsage. // // The certificate is signed by parent. If parent is equal to template then the // certificate is self-signed. The parameter pub is the public key of the @@ -1720,6 +1747,10 @@ func signingParamsForPublicKey(pub interface{}, requestedSigAlgo SignatureAlgori // // All keys types that are implemented via crypto.Signer are supported (This // includes *rsa.PublicKey and *ecdsa.PublicKey.) +// +// The AuthorityKeyId will be taken from the SubjectKeyId of parent, if any, +// unless the resulting certificate is self-signed. Otherwise the value from +// template will be used. func CreateCertificate(rand io.Reader, template, parent *Certificate, pub, priv interface{}) (cert []byte, err error) { key, ok := priv.(crypto.Signer) if !ok { @@ -1750,11 +1781,12 @@ func CreateCertificate(rand io.Reader, template, parent *Certificate, pub, priv return } + authorityKeyId := template.AuthorityKeyId if !bytes.Equal(asn1Issuer, asn1Subject) && len(parent.SubjectKeyId) > 0 { - template.AuthorityKeyId = parent.SubjectKeyId + authorityKeyId = parent.SubjectKeyId } - extensions, err := buildExtensions(template) + extensions, err := buildExtensions(template, authorityKeyId) if err != nil { return } @@ -2025,10 +2057,10 @@ func parseCSRExtensions(rawAttributes []asn1.RawValue) ([]pkix.Extension, error) return ret, nil } -// CreateCertificateRequest creates a new certificate request based on a template. -// The following members of template are used: Subject, Attributes, -// SignatureAlgorithm, Extensions, DNSNames, EmailAddresses, and IPAddresses. -// The private key is the private key of the signer. +// CreateCertificateRequest creates a new certificate request based on a +// template. The following members of template are used: Attributes, DNSNames, +// EmailAddresses, ExtraExtensions, IPAddresses, SignatureAlgorithm, and +// Subject. The private key is the private key of the signer. // // The returned slice is the certificate request in DER encoding. // diff --git a/libgo/go/crypto/x509/x509_test.go b/libgo/go/crypto/x509/x509_test.go index b085dad90f0..2d1acf93bf5 100644 --- a/libgo/go/crypto/x509/x509_test.go +++ b/libgo/go/crypto/x509/x509_test.go @@ -405,6 +405,7 @@ func TestCreateSelfSignedCertificate(t *testing.T) { PolicyIdentifiers: []asn1.ObjectIdentifier{[]int{1, 2, 3}}, PermittedDNSDomains: []string{".example.com", "example.com"}, + ExcludedDNSDomains: []string{"bar.example.com"}, CRLDistributionPoints: []string{"http://crl1.example.com/ca1.crl", "http://crl2.example.com/ca1.crl"}, @@ -442,6 +443,10 @@ func TestCreateSelfSignedCertificate(t *testing.T) { t.Errorf("%s: failed to parse name constraints: %#v", test.name, cert.PermittedDNSDomains) } + if len(cert.ExcludedDNSDomains) != 1 || cert.ExcludedDNSDomains[0] != "bar.example.com" { + t.Errorf("%s: failed to parse name constraint exclusions: %#v", test.name, cert.ExcludedDNSDomains) + } + if cert.Subject.CommonName != commonName { t.Errorf("%s: subject wasn't correctly copied from the template. Got %s, want %s", test.name, cert.Subject.CommonName, commonName) } diff --git a/libgo/go/database/sql/convert.go b/libgo/go/database/sql/convert.go index ea2f377810e..4983181fe75 100644 --- a/libgo/go/database/sql/convert.go +++ b/libgo/go/database/sql/convert.go @@ -12,6 +12,7 @@ import ( "fmt" "reflect" "strconv" + "sync" "time" "unicode" "unicode/utf8" @@ -37,86 +38,180 @@ func validateNamedValueName(name string) error { return fmt.Errorf("name %q does not begin with a letter", name) } +func driverNumInput(ds *driverStmt) int { + ds.Lock() + defer ds.Unlock() // in case NumInput panics + return ds.si.NumInput() +} + +// ccChecker wraps the driver.ColumnConverter and allows it to be used +// as if it were a NamedValueChecker. If the driver ColumnConverter +// is not present then the NamedValueChecker will return driver.ErrSkip. +type ccChecker struct { + sync.Locker + cci driver.ColumnConverter + want int +} + +func (c ccChecker) CheckNamedValue(nv *driver.NamedValue) error { + if c.cci == nil { + return driver.ErrSkip + } + // The column converter shouldn't be called on any index + // it isn't expecting. The final error will be thrown + // in the argument converter loop. + index := nv.Ordinal - 1 + if c.want <= index { + return nil + } + + // First, see if the value itself knows how to convert + // itself to a driver type. For example, a NullString + // struct changing into a string or nil. + if vr, ok := nv.Value.(driver.Valuer); ok { + sv, err := callValuerValue(vr) + if err != nil { + return err + } + if !driver.IsValue(sv) { + return fmt.Errorf("non-subset type %T returned from Value", sv) + } + nv.Value = sv + } + + // Second, ask the column to sanity check itself. For + // example, drivers might use this to make sure that + // an int64 values being inserted into a 16-bit + // integer field is in range (before getting + // truncated), or that a nil can't go into a NOT NULL + // column before going across the network to get the + // same error. + var err error + arg := nv.Value + c.Lock() + nv.Value, err = c.cci.ColumnConverter(index).ConvertValue(arg) + c.Unlock() + if err != nil { + return err + } + if !driver.IsValue(nv.Value) { + return fmt.Errorf("driver ColumnConverter error converted %T to unsupported type %T", arg, nv.Value) + } + return nil +} + +// defaultCheckNamedValue wraps the default ColumnConverter to have the same +// function signature as the CheckNamedValue in the driver.NamedValueChecker +// interface. +func defaultCheckNamedValue(nv *driver.NamedValue) (err error) { + nv.Value, err = driver.DefaultParameterConverter.ConvertValue(nv.Value) + return err +} + // driverArgs converts arguments from callers of Stmt.Exec and // Stmt.Query into driver Values. // // The statement ds may be nil, if no statement is available. -func driverArgs(ds *driverStmt, args []interface{}) ([]driver.NamedValue, error) { +func driverArgs(ci driver.Conn, ds *driverStmt, args []interface{}) ([]driver.NamedValue, error) { nvargs := make([]driver.NamedValue, len(args)) + + // -1 means the driver doesn't know how to count the number of + // placeholders, so we won't sanity check input here and instead let the + // driver deal with errors. + want := -1 + var si driver.Stmt + var cc ccChecker if ds != nil { si = ds.si + want = driverNumInput(ds) + cc.Locker = ds.Locker + cc.want = want } - cc, ok := si.(driver.ColumnConverter) - // Normal path, for a driver.Stmt that is not a ColumnConverter. + // Check all types of interfaces from the start. + // Drivers may opt to use the NamedValueChecker for special + // argument types, then return driver.ErrSkip to pass it along + // to the column converter. + nvc, ok := si.(driver.NamedValueChecker) if !ok { - for n, arg := range args { - var err error - nv := &nvargs[n] - nv.Ordinal = n + 1 - if np, ok := arg.(NamedArg); ok { - if err := validateNamedValueName(np.Name); err != nil { - return nil, err - } - arg = np.Value - nvargs[n].Name = np.Name - } - nv.Value, err = driver.DefaultParameterConverter.ConvertValue(arg) - - if err != nil { - return nil, fmt.Errorf("sql: converting Exec argument %s type: %v", describeNamedValue(nv), err) - } - } - return nvargs, nil + nvc, ok = ci.(driver.NamedValueChecker) + } + cci, ok := si.(driver.ColumnConverter) + if ok { + cc.cci = cci } - // Let the Stmt convert its own arguments. - for n, arg := range args { + // Loop through all the arguments, checking each one. + // If no error is returned simply increment the index + // and continue. However if driver.ErrRemoveArgument + // is returned the argument is not included in the query + // argument list. + var err error + var n int + for _, arg := range args { nv := &nvargs[n] - nv.Ordinal = n + 1 if np, ok := arg.(NamedArg); ok { - if err := validateNamedValueName(np.Name); err != nil { + if err = validateNamedValueName(np.Name); err != nil { return nil, err } arg = np.Value nv.Name = np.Name } - // First, see if the value itself knows how to convert - // itself to a driver type. For example, a NullString - // struct changing into a string or nil. - if vr, ok := arg.(driver.Valuer); ok { - sv, err := callValuerValue(vr) - if err != nil { - return nil, fmt.Errorf("sql: argument %s from Value: %v", describeNamedValue(nv), err) - } - if !driver.IsValue(sv) { - return nil, fmt.Errorf("sql: argument %s: non-subset type %T returned from Value", describeNamedValue(nv), sv) - } - arg = sv + nv.Ordinal = n + 1 + nv.Value = arg + + // Checking sequence has four routes: + // A: 1. Default + // B: 1. NamedValueChecker 2. Column Converter 3. Default + // C: 1. NamedValueChecker 3. Default + // D: 1. Column Converter 2. Default + // + // The only time a Column Converter is called is first + // or after NamedValueConverter. If first it is handled before + // the nextCheck label. Thus for repeats tries only when the + // NamedValueConverter is selected should the Column Converter + // be used in the retry. + checker := defaultCheckNamedValue + nextCC := false + switch { + case nvc != nil: + nextCC = cci != nil + checker = nvc.CheckNamedValue + case cci != nil: + checker = cc.CheckNamedValue } - // Second, ask the column to sanity check itself. For - // example, drivers might use this to make sure that - // an int64 values being inserted into a 16-bit - // integer field is in range (before getting - // truncated), or that a nil can't go into a NOT NULL - // column before going across the network to get the - // same error. - var err error - ds.Lock() - nv.Value, err = cc.ColumnConverter(n).ConvertValue(arg) - ds.Unlock() - if err != nil { + nextCheck: + err = checker(nv) + switch err { + case nil: + n++ + continue + case driver.ErrRemoveArgument: + nvargs = nvargs[:len(nvargs)-1] + continue + case driver.ErrSkip: + if nextCC { + nextCC = false + checker = cc.CheckNamedValue + } else { + checker = defaultCheckNamedValue + } + goto nextCheck + default: return nil, fmt.Errorf("sql: converting argument %s type: %v", describeNamedValue(nv), err) } - if !driver.IsValue(nv.Value) { - return nil, fmt.Errorf("sql: for argument %s, driver ColumnConverter error converted %T to unsupported type %T", - describeNamedValue(nv), arg, nv.Value) - } + } + + // Check the length of arguments after convertion to allow for omitted + // arguments. + if want != -1 && len(nvargs) != want { + return nil, fmt.Errorf("sql: expected %d arguments, got %d", want, len(nvargs)) } return nvargs, nil + } // convertAssign copies to dest the value in src, converting it if possible. @@ -270,6 +365,11 @@ func convertAssign(dest, src interface{}) error { return nil } + // The following conversions use a string value as an intermediate representation + // to convert between various numeric types. + // + // This also allows scanning into user defined types such as "type Int int64". + // For symmetry, also check for string destination types. switch dv.Kind() { case reflect.Ptr: if src == nil { @@ -306,6 +406,15 @@ func convertAssign(dest, src interface{}) error { } dv.SetFloat(f64) return nil + case reflect.String: + switch v := src.(type) { + case string: + dv.SetString(v) + return nil + case []byte: + dv.SetString(string(v)) + return nil + } } return fmt.Errorf("unsupported Scan, storing driver.Value type %T into type %T", src, dest) diff --git a/libgo/go/database/sql/convert_test.go b/libgo/go/database/sql/convert_test.go index 4dfab1f6bef..cfe52d7f548 100644 --- a/libgo/go/database/sql/convert_test.go +++ b/libgo/go/database/sql/convert_test.go @@ -10,6 +10,7 @@ import ( "reflect" "runtime" "strings" + "sync" "testing" "time" ) @@ -17,9 +18,11 @@ import ( var someTime = time.Unix(123, 0) var answer int64 = 42 -type userDefined float64 - -type userDefinedSlice []int +type ( + userDefined float64 + userDefinedSlice []int + userDefinedString string +) type conversionTest struct { s, d interface{} // source and destination @@ -39,6 +42,7 @@ type conversionTest struct { wantptr *int64 // if non-nil, *d's pointed value must be equal to *wantptr wantnil bool // if true, *d must be *int64(nil) wantusrdef userDefined + wantusrstr userDefinedString } // Target variables for scanning into. @@ -171,6 +175,7 @@ var conversionTests = []conversionTest{ {s: int64(123), d: new(userDefined), wantusrdef: 123}, {s: "1.5", d: new(userDefined), wantusrdef: 1.5}, {s: []byte{1, 2, 3}, d: new(userDefinedSlice), wanterr: `unsupported Scan, storing driver.Value type []uint8 into type *sql.userDefinedSlice`}, + {s: "str", d: new(userDefinedString), wantusrstr: "str"}, // Other errors {s: complex(1, 2), d: &scanstr, wanterr: `unsupported Scan, storing driver.Value type complex128 into type *string`}, @@ -260,6 +265,9 @@ func TestConversions(t *testing.T) { if ct.wantusrdef != 0 && ct.wantusrdef != *ct.d.(*userDefined) { errf("want userDefined %f, got %f", ct.wantusrdef, *ct.d.(*userDefined)) } + if len(ct.wantusrstr) != 0 && ct.wantusrstr != *ct.d.(*userDefinedString) { + errf("want userDefined %q, got %q", ct.wantusrstr, *ct.d.(*userDefinedString)) + } } } @@ -461,8 +469,8 @@ func TestDriverArgs(t *testing.T) { }, } for i, tt := range tests { - ds := new(driverStmt) - got, err := driverArgs(ds, tt.args) + ds := &driverStmt{Locker: &sync.Mutex{}, si: stubDriverStmt{nil}} + got, err := driverArgs(nil, ds, tt.args) if err != nil { t.Errorf("test[%d]: %v", i, err) continue diff --git a/libgo/go/database/sql/driver/driver.go b/libgo/go/database/sql/driver/driver.go index d66196fd481..0262ca24ba2 100644 --- a/libgo/go/database/sql/driver/driver.go +++ b/libgo/go/database/sql/driver/driver.go @@ -262,9 +262,39 @@ type StmtQueryContext interface { QueryContext(ctx context.Context, args []NamedValue) (Rows, error) } +// ErrRemoveArgument may be returned from NamedValueChecker to instruct the +// sql package to not pass the argument to the driver query interface. +// Return when accepting query specific options or structures that aren't +// SQL query arguments. +var ErrRemoveArgument = errors.New("driver: remove argument from query") + +// NamedValueChecker may be optionally implemented by Conn or Stmt. It provides +// the driver more control to handle Go and database types beyond the default +// Values types allowed. +// +// The sql package checks for value checkers in the following order, +// stopping at the first found match: Stmt.NamedValueChecker, Conn.NamedValueChecker, +// Stmt.ColumnConverter, DefaultParameterConverter. +// +// If CheckNamedValue returns ErrRemoveArgument, the NamedValue will not be included in +// the final query arguments. This may be used to pass special options to +// the query itself. +// +// If ErrSkip is returned the column converter error checking +// path is used for the argument. Drivers may wish to return ErrSkip after +// they have exhausted their own special cases. +type NamedValueChecker interface { + // CheckNamedValue is called before passing arguments to the driver + // and is called in place of any ColumnConverter. CheckNamedValue must do type + // validation and conversion as appropriate for the driver. + CheckNamedValue(*NamedValue) error +} + // ColumnConverter may be optionally implemented by Stmt if the // statement is aware of its own columns' types and can convert from // any type to a driver Value. +// +// Deprecated: Drivers should implement NamedValueChecker. type ColumnConverter interface { // ColumnConverter returns a ValueConverter for the provided // column index. If the type of a specific column isn't known diff --git a/libgo/go/database/sql/fakedb_test.go b/libgo/go/database/sql/fakedb_test.go index 4b15f5bec7b..4dcd096ca4d 100644 --- a/libgo/go/database/sql/fakedb_test.go +++ b/libgo/go/database/sql/fakedb_test.go @@ -58,9 +58,10 @@ type fakeDriver struct { type fakeDB struct { name string - mu sync.Mutex - tables map[string]*table - badConn bool + mu sync.Mutex + tables map[string]*table + badConn bool + allowAny bool } type table struct { @@ -83,11 +84,20 @@ type row struct { cols []interface{} // must be same size as its table colname + coltype } +type memToucher interface { + // touchMem reads & writes some memory, to help find data races. + touchMem() +} + type fakeConn struct { db *fakeDB // where to return ourselves to currTx *fakeTx + // Every operation writes to line to enable the race detector + // check for data races. + line int64 + // Stats for tests: mu sync.Mutex stmtsMade int @@ -99,6 +109,10 @@ type fakeConn struct { stickyBad bool } +func (c *fakeConn) touchMem() { + c.line++ +} + func (c *fakeConn) incrStat(v *int) { c.mu.Lock() *v++ @@ -116,6 +130,7 @@ type boundCol struct { } type fakeStmt struct { + memToucher c *fakeConn q string // just for debugging @@ -298,6 +313,7 @@ func (c *fakeConn) Begin() (driver.Tx, error) { if c.currTx != nil { return nil, errors.New("already in a transaction") } + c.touchMem() c.currTx = &fakeTx{c: c} return c.currTx, nil } @@ -339,6 +355,7 @@ func (c *fakeConn) Close() (err error) { drv.mu.Unlock() } }() + c.touchMem() if c.currTx != nil { return errors.New("can't close fakeConn; in a Transaction") } @@ -352,12 +369,14 @@ func (c *fakeConn) Close() (err error) { return nil } -func checkSubsetTypes(args []driver.NamedValue) error { +func checkSubsetTypes(allowAny bool, args []driver.NamedValue) error { for _, arg := range args { switch arg.Value.(type) { case int64, float64, bool, nil, []byte, string, time.Time: default: - return fmt.Errorf("fakedb_test: invalid argument ordinal %[1]d: %[2]v, type %[2]T", arg.Ordinal, arg.Value) + if !allowAny { + return fmt.Errorf("fakedb_test: invalid argument ordinal %[1]d: %[2]v, type %[2]T", arg.Ordinal, arg.Value) + } } } return nil @@ -373,7 +392,7 @@ func (c *fakeConn) ExecContext(ctx context.Context, query string, args []driver. // just to check that all the args are of the proper types. // ErrSkip is returned so the caller acts as if we didn't // implement this at all. - err := checkSubsetTypes(args) + err := checkSubsetTypes(c.db.allowAny, args) if err != nil { return nil, err } @@ -390,7 +409,7 @@ func (c *fakeConn) QueryContext(ctx context.Context, query string, args []driver // just to check that all the args are of the proper types. // ErrSkip is returned so the caller acts as if we didn't // implement this at all. - err := checkSubsetTypes(args) + err := checkSubsetTypes(c.db.allowAny, args) if err != nil { return nil, err } @@ -524,13 +543,14 @@ func (c *fakeConn) PrepareContext(ctx context.Context, query string) (driver.Stm return nil, driver.ErrBadConn } + c.touchMem() var firstStmt, prev *fakeStmt for _, query := range strings.Split(query, ";") { parts := strings.Split(query, "|") if len(parts) < 1 { return nil, errf("empty query") } - stmt := &fakeStmt{q: query, c: c} + stmt := &fakeStmt{q: query, c: c, memToucher: c} if firstStmt == nil { firstStmt = stmt } @@ -612,6 +632,7 @@ func (s *fakeStmt) Close() error { if s.c.db == nil { panic("in fakeStmt.Close, conn's db is nil (already closed)") } + s.touchMem() if !s.closed { s.c.incrStat(&s.c.stmtsClosed) s.closed = true @@ -642,10 +663,11 @@ func (s *fakeStmt) ExecContext(ctx context.Context, args []driver.NamedValue) (d return nil, driver.ErrBadConn } - err := checkSubsetTypes(args) + err := checkSubsetTypes(s.c.db.allowAny, args) if err != nil { return nil, err } + s.touchMem() if s.wait > 0 { time.Sleep(s.wait) @@ -753,11 +775,12 @@ func (s *fakeStmt) QueryContext(ctx context.Context, args []driver.NamedValue) ( return nil, driver.ErrBadConn } - err := checkSubsetTypes(args) + err := checkSubsetTypes(s.c.db.allowAny, args) if err != nil { return nil, err } + s.touchMem() db := s.c.db if len(args) != s.placeholders { panic("error in pkg db; should only get here if size is correct") @@ -853,11 +876,12 @@ func (s *fakeStmt) QueryContext(ctx context.Context, args []driver.NamedValue) ( } cursor := &rowsCursor{ - posRow: -1, - rows: setMRows, - cols: setColumns, - colType: setColType, - errPos: -1, + parentMem: s.c, + posRow: -1, + rows: setMRows, + cols: setColumns, + colType: setColType, + errPos: -1, } return cursor, nil } @@ -877,6 +901,7 @@ func (tx *fakeTx) Commit() error { if hookCommitBadConn != nil && hookCommitBadConn() { return driver.ErrBadConn } + tx.c.touchMem() return nil } @@ -888,16 +913,18 @@ func (tx *fakeTx) Rollback() error { if hookRollbackBadConn != nil && hookRollbackBadConn() { return driver.ErrBadConn } + tx.c.touchMem() return nil } type rowsCursor struct { - cols [][]string - colType [][]string - posSet int - posRow int - rows [][]*row - closed bool + parentMem memToucher + cols [][]string + colType [][]string + posSet int + posRow int + rows [][]*row + closed bool // errPos and err are for making Next return early with error. errPos int @@ -907,6 +934,16 @@ type rowsCursor struct { // the original slice's first byte address. we clone them // just so we're able to corrupt them on close. bytesClone map[*byte][]byte + + // Every operation writes to line to enable the race detector + // check for data races. + // This is separate from the fakeConn.line to allow for drivers that + // can start multiple queries on the same transaction at the same time. + line int64 +} + +func (rc *rowsCursor) touchMem() { + rc.line++ } func (rc *rowsCursor) Close() error { @@ -915,6 +952,8 @@ func (rc *rowsCursor) Close() error { bs[0] = 255 // first byte corrupted } } + rc.touchMem() + rc.parentMem.touchMem() rc.closed = true return nil } @@ -937,6 +976,7 @@ func (rc *rowsCursor) Next(dest []driver.Value) error { if rc.closed { return errors.New("fakedb: cursor is closed") } + rc.touchMem() rc.posRow++ if rc.posRow == rc.errPos { return rc.err @@ -970,10 +1010,12 @@ func (rc *rowsCursor) Next(dest []driver.Value) error { } func (rc *rowsCursor) HasNextResultSet() bool { + rc.touchMem() return rc.posSet < len(rc.rows)-1 } func (rc *rowsCursor) NextResultSet() error { + rc.touchMem() if rc.HasNextResultSet() { rc.posSet++ rc.posRow = -1 @@ -1004,6 +1046,12 @@ func (fakeDriverString) ConvertValue(v interface{}) (driver.Value, error) { return fmt.Sprintf("%v", v), nil } +type anyTypeConverter struct{} + +func (anyTypeConverter) ConvertValue(v interface{}) (driver.Value, error) { + return v, nil +} + func converterForType(typ string) driver.ValueConverter { switch typ { case "bool": @@ -1030,6 +1078,8 @@ func converterForType(typ string) driver.ValueConverter { return driver.Null{Converter: driver.DefaultParameterConverter} case "datetime": return driver.DefaultParameterConverter + case "any": + return anyTypeConverter{} } panic("invalid fakedb column type of " + typ) } @@ -1056,6 +1106,8 @@ func colTypeToReflectType(typ string) reflect.Type { return reflect.TypeOf(NullFloat64{}) case "datetime": return reflect.TypeOf(time.Time{}) + case "any": + return reflect.TypeOf(new(interface{})).Elem() } panic("invalid fakedb column type of " + typ) } diff --git a/libgo/go/database/sql/sql.go b/libgo/go/database/sql/sql.go index f8a884446e4..c609fe4cc43 100644 --- a/libgo/go/database/sql/sql.go +++ b/libgo/go/database/sql/sql.go @@ -278,6 +278,27 @@ type Scanner interface { Scan(src interface{}) error } +// Out may be used to retrieve OUTPUT value parameters from stored procedures. +// +// Not all drivers and databases support OUTPUT value parameters. +// +// Example usage: +// +// var outArg string +// _, err := db.ExecContext(ctx, "ProcName", sql.Named("Arg1", Out{Dest: &outArg})) +type Out struct { + _Named_Fields_Required struct{} + + // Dest is a pointer to the value that will be set to the result of the + // stored procedure's OUTPUT parameter. + Dest interface{} + + // In is whether the parameter is an INOUT parameter. If so, the input value to the stored + // procedure is the dereferenced value of Dest's pointer, which is then replaced with + // the output value. + In bool +} + // ErrNoRows is returned by Scan when QueryRow doesn't return a // row. In such a case, QueryRow returns a placeholder *Row value that // defers this error until a Scan. @@ -372,11 +393,19 @@ func (dc *driverConn) expired(timeout time.Duration) bool { return dc.createdAt.Add(timeout).Before(nowFunc()) } -func (dc *driverConn) prepareLocked(ctx context.Context, query string) (*driverStmt, error) { +// prepareLocked prepares the query on dc. When cg == nil the dc must keep track of +// the prepared statements in a pool. +func (dc *driverConn) prepareLocked(ctx context.Context, cg stmtConnGrabber, query string) (*driverStmt, error) { si, err := ctxDriverPrepare(ctx, dc.ci, query) if err != nil { return nil, err } + ds := &driverStmt{Locker: dc, si: si} + + // No need to manage open statements if there is a single connection grabber. + if cg != nil { + return ds, nil + } // Track each driverConn's open statements, so we can close them // before closing the conn. @@ -385,9 +414,7 @@ func (dc *driverConn) prepareLocked(ctx context.Context, query string) (*driverS if dc.openStmt == nil { dc.openStmt = make(map[*driverStmt]bool) } - ds := &driverStmt{Locker: dc, si: si} dc.openStmt[ds] = true - return ds, nil } @@ -583,6 +610,17 @@ func Open(driverName, dataSourceName string) (*DB, error) { return db, nil } +func (db *DB) pingDC(ctx context.Context, dc *driverConn, release func(error)) error { + var err error + if pinger, ok := dc.ci.(driver.Pinger); ok { + withLock(dc, func() { + err = pinger.Ping(ctx) + }) + } + release(err) + return err +} + // PingContext verifies a connection to the database is still alive, // establishing a connection if necessary. func (db *DB) PingContext(ctx context.Context) error { @@ -602,11 +640,7 @@ func (db *DB) PingContext(ctx context.Context) error { return err } - if pinger, ok := dc.ci.(driver.Pinger); ok { - err = pinger.Ping(ctx) - } - db.putConn(dc, err) - return err + return db.pingDC(ctx, dc, dc.releaseConn) } // Ping verifies a connection to the database is still alive, @@ -975,9 +1009,9 @@ func (db *DB) conn(ctx context.Context, strategy connReuseStrategy) (*driverConn db: db, createdAt: nowFunc(), ci: ci, + inUse: true, } db.addDepLocked(dc, dc) - dc.inUse = true db.mu.Unlock() return dc, nil } @@ -1137,22 +1171,39 @@ func (db *DB) prepare(ctx context.Context, query string, strategy connReuseStrat if err != nil { return nil, err } + return db.prepareDC(ctx, dc, dc.releaseConn, nil, query) +} + +// prepareDC prepares a query on the driverConn and calls release before +// returning. When cg == nil it implies that a connection pool is used, and +// when cg != nil only a single driver connection is used. +func (db *DB) prepareDC(ctx context.Context, dc *driverConn, release func(error), cg stmtConnGrabber, query string) (*Stmt, error) { var ds *driverStmt + var err error + defer func() { + release(err) + }() withLock(dc, func() { - ds, err = dc.prepareLocked(ctx, query) + ds, err = dc.prepareLocked(ctx, cg, query) }) if err != nil { - db.putConn(dc, err) return nil, err } stmt := &Stmt{ - db: db, - query: query, - css: []connStmt{{dc, ds}}, - lastNumClosed: atomic.LoadUint64(&db.numClosed), + db: db, + query: query, + cg: cg, + cgds: ds, + } + + // When cg == nil this statement will need to keep track of various + // connections they are prepared on and record the stmt dependency on + // the DB. + if cg == nil { + stmt.css = []connStmt{{dc, ds}} + stmt.lastNumClosed = atomic.LoadUint64(&db.numClosed) + db.addDep(stmt, stmt) } - db.addDep(stmt, stmt) - db.putConn(dc, nil) return stmt, nil } @@ -1179,18 +1230,21 @@ func (db *DB) Exec(query string, args ...interface{}) (Result, error) { return db.ExecContext(context.Background(), query, args...) } -func (db *DB) exec(ctx context.Context, query string, args []interface{}, strategy connReuseStrategy) (res Result, err error) { +func (db *DB) exec(ctx context.Context, query string, args []interface{}, strategy connReuseStrategy) (Result, error) { dc, err := db.conn(ctx, strategy) if err != nil { return nil, err } + return db.execDC(ctx, dc, dc.releaseConn, query, args) +} + +func (db *DB) execDC(ctx context.Context, dc *driverConn, release func(error), query string, args []interface{}) (res Result, err error) { defer func() { - db.putConn(dc, err) + release(err) }() - if execer, ok := dc.ci.(driver.Execer); ok { var dargs []driver.NamedValue - dargs, err = driverArgs(nil, args) + dargs, err = driverArgs(dc.ci, nil, args) if err != nil { return nil, err } @@ -1215,7 +1269,7 @@ func (db *DB) exec(ctx context.Context, query string, args []interface{}, strate } ds := &driverStmt{Locker: dc, si: si} defer ds.Close() - return resultFromStatement(ctx, ds, args...) + return resultFromStatement(ctx, dc.ci, ds, args...) } // QueryContext executes a query that returns rows, typically a SELECT. @@ -1242,19 +1296,21 @@ func (db *DB) Query(query string, args ...interface{}) (*Rows, error) { } func (db *DB) query(ctx context.Context, query string, args []interface{}, strategy connReuseStrategy) (*Rows, error) { - ci, err := db.conn(ctx, strategy) + dc, err := db.conn(ctx, strategy) if err != nil { return nil, err } - return db.queryConn(ctx, ci, ci.releaseConn, query, args) + return db.queryDC(ctx, nil, dc, dc.releaseConn, query, args) } -// queryConn executes a query on the given connection. +// queryDC executes a query on the given connection. // The connection gets released by the releaseConn function. -func (db *DB) queryConn(ctx context.Context, dc *driverConn, releaseConn func(error), query string, args []interface{}) (*Rows, error) { +// The ctx context is from a query method and the txctx context is from an +// optional transaction context. +func (db *DB) queryDC(ctx, txctx context.Context, dc *driverConn, releaseConn func(error), query string, args []interface{}) (*Rows, error) { if queryer, ok := dc.ci.(driver.Queryer); ok { - dargs, err := driverArgs(nil, args) + dargs, err := driverArgs(dc.ci, nil, args) if err != nil { releaseConn(err) return nil, err @@ -1275,7 +1331,7 @@ func (db *DB) queryConn(ctx context.Context, dc *driverConn, releaseConn func(er releaseConn: releaseConn, rowsi: rowsi, } - rows.initContextClose(ctx) + rows.initContextClose(ctx, txctx) return rows, nil } } @@ -1291,7 +1347,7 @@ func (db *DB) queryConn(ctx context.Context, dc *driverConn, releaseConn func(er } ds := &driverStmt{Locker: dc, si: si} - rowsi, err := rowsiFromStatement(ctx, ds, args...) + rowsi, err := rowsiFromStatement(ctx, dc.ci, ds, args...) if err != nil { ds.Close() releaseConn(err) @@ -1306,13 +1362,16 @@ func (db *DB) queryConn(ctx context.Context, dc *driverConn, releaseConn func(er rowsi: rowsi, closeStmt: ds, } - rows.initContextClose(ctx) + rows.initContextClose(ctx, txctx) return rows, nil } // QueryRowContext executes a query that is expected to return at most one row. // QueryRowContext always returns a non-nil value. Errors are deferred until // Row's Scan method is called. +// If the query selects no rows, the *Row's Scan will return ErrNoRows. +// Otherwise, the *Row's Scan scans the first selected row and discards +// the rest. func (db *DB) QueryRowContext(ctx context.Context, query string, args ...interface{}) *Row { rows, err := db.QueryContext(ctx, query, args...) return &Row{rows: rows, err: err} @@ -1321,6 +1380,9 @@ func (db *DB) QueryRowContext(ctx context.Context, query string, args ...interfa // QueryRow executes a query that is expected to return at most one row. // QueryRow always returns a non-nil value. Errors are deferred until // Row's Scan method is called. +// If the query selects no rows, the *Row's Scan will return ErrNoRows. +// Otherwise, the *Row's Scan scans the first selected row and discards +// the rest. func (db *DB) QueryRow(query string, args ...interface{}) *Row { return db.QueryRowContext(context.Background(), query, args...) } @@ -1361,12 +1423,17 @@ func (db *DB) begin(ctx context.Context, opts *TxOptions, strategy connReuseStra if err != nil { return nil, err } + return db.beginDC(ctx, dc, dc.releaseConn, opts) +} + +// beginDC starts a transaction. The provided dc must be valid and ready to use. +func (db *DB) beginDC(ctx context.Context, dc *driverConn, release func(error), opts *TxOptions) (tx *Tx, err error) { var txi driver.Tx withLock(dc, func() { txi, err = ctxDriverBegin(ctx, opts, dc.ci) }) if err != nil { - db.putConn(dc, err) + release(err) return nil, err } @@ -1374,11 +1441,12 @@ func (db *DB) begin(ctx context.Context, opts *TxOptions, strategy connReuseStra // The cancel function in Tx will be called after done is set to true. ctx, cancel := context.WithCancel(ctx) tx = &Tx{ - db: db, - dc: dc, - txi: txi, - cancel: cancel, - ctx: ctx, + db: db, + dc: dc, + releaseConn: release, + txi: txi, + cancel: cancel, + ctx: ctx, } go tx.awaitDone() return tx, nil @@ -1389,6 +1457,189 @@ func (db *DB) Driver() driver.Driver { return db.driver } +// ErrConnDone is returned by any operation that is performed on a connection +// that has already been committed or rolled back. +var ErrConnDone = errors.New("database/sql: connection is already closed") + +// Conn returns a single connection by either opening a new connection +// or returning an existing connection from the connection pool. Conn will +// block until either a connection is returned or ctx is canceled. +// Queries run on the same Conn will be run in the same database session. +// +// Every Conn must be returned to the database pool after use by +// calling Conn.Close. +func (db *DB) Conn(ctx context.Context) (*Conn, error) { + var dc *driverConn + var err error + for i := 0; i < maxBadConnRetries; i++ { + dc, err = db.conn(ctx, cachedOrNewConn) + if err != driver.ErrBadConn { + break + } + } + if err == driver.ErrBadConn { + dc, err = db.conn(ctx, cachedOrNewConn) + } + if err != nil { + return nil, err + } + + conn := &Conn{ + db: db, + dc: dc, + } + return conn, nil +} + +type releaseConn func(error) + +// Conn represents a single database session rather a pool of database +// sessions. Prefer running queries from DB unless there is a specific +// need for a continuous single database session. +// +// A Conn must call Close to return the connection to the database pool +// and may do so concurrently with a running query. +// +// After a call to Close, all operations on the +// connection fail with ErrConnDone. +type Conn struct { + db *DB + + // closemu prevents the connection from closing while there + // is an active query. It is held for read during queries + // and exclusively during close. + closemu sync.RWMutex + + // dc is owned until close, at which point + // it's returned to the connection pool. + dc *driverConn + + // done transitions from 0 to 1 exactly once, on close. + // Once done, all operations fail with ErrConnDone. + // Use atomic operations on value when checking value. + done int32 +} + +func (c *Conn) grabConn(context.Context) (*driverConn, releaseConn, error) { + if atomic.LoadInt32(&c.done) != 0 { + return nil, nil, ErrConnDone + } + c.closemu.RLock() + return c.dc, c.closemuRUnlockCondReleaseConn, nil +} + +// PingContext verifies the connection to the database is still alive. +func (c *Conn) PingContext(ctx context.Context) error { + dc, release, err := c.grabConn(ctx) + if err != nil { + return err + } + return c.db.pingDC(ctx, dc, release) +} + +// ExecContext executes a query without returning any rows. +// The args are for any placeholder parameters in the query. +func (c *Conn) ExecContext(ctx context.Context, query string, args ...interface{}) (Result, error) { + dc, release, err := c.grabConn(ctx) + if err != nil { + return nil, err + } + return c.db.execDC(ctx, dc, release, query, args) +} + +// QueryContext executes a query that returns rows, typically a SELECT. +// The args are for any placeholder parameters in the query. +func (c *Conn) QueryContext(ctx context.Context, query string, args ...interface{}) (*Rows, error) { + dc, release, err := c.grabConn(ctx) + if err != nil { + return nil, err + } + return c.db.queryDC(ctx, nil, dc, release, query, args) +} + +// QueryRowContext executes a query that is expected to return at most one row. +// QueryRowContext always returns a non-nil value. Errors are deferred until +// Row's Scan method is called. +// If the query selects no rows, the *Row's Scan will return ErrNoRows. +// Otherwise, the *Row's Scan scans the first selected row and discards +// the rest. +func (c *Conn) QueryRowContext(ctx context.Context, query string, args ...interface{}) *Row { + rows, err := c.QueryContext(ctx, query, args...) + return &Row{rows: rows, err: err} +} + +// PrepareContext creates a prepared statement for later queries or executions. +// Multiple queries or executions may be run concurrently from the +// returned statement. +// The caller must call the statement's Close method +// when the statement is no longer needed. +// +// The provided context is used for the preparation of the statement, not for the +// execution of the statement. +func (c *Conn) PrepareContext(ctx context.Context, query string) (*Stmt, error) { + dc, release, err := c.grabConn(ctx) + if err != nil { + return nil, err + } + return c.db.prepareDC(ctx, dc, release, c, query) +} + +// BeginTx starts a transaction. +// +// The provided context is used until the transaction is committed or rolled back. +// If the context is canceled, the sql package will roll back +// the transaction. Tx.Commit will return an error if the context provided to +// BeginTx is canceled. +// +// The provided TxOptions is optional and may be nil if defaults should be used. +// If a non-default isolation level is used that the driver doesn't support, +// an error will be returned. +func (c *Conn) BeginTx(ctx context.Context, opts *TxOptions) (*Tx, error) { + dc, release, err := c.grabConn(ctx) + if err != nil { + return nil, err + } + return c.db.beginDC(ctx, dc, release, opts) +} + +// closemuRUnlockCondReleaseConn read unlocks closemu +// as the sql operation is done with the dc. +func (c *Conn) closemuRUnlockCondReleaseConn(err error) { + c.closemu.RUnlock() + if err == driver.ErrBadConn { + c.close(err) + } +} + +func (c *Conn) txCtx() context.Context { + return nil +} + +func (c *Conn) close(err error) error { + if !atomic.CompareAndSwapInt32(&c.done, 0, 1) { + return ErrConnDone + } + + // Lock around releasing the driver connection + // to ensure all queries have been stopped before doing so. + c.closemu.Lock() + defer c.closemu.Unlock() + + c.dc.releaseConn(err) + c.dc = nil + c.db = nil + return err +} + +// Close returns the connection to the connection pool. +// All operations after a Close will return with ErrConnDone. +// Close is safe to call concurrently with other operations and will +// block until all other operations finish. It may be useful to first +// cancel any used context and then call close directly after. +func (c *Conn) Close() error { + return c.close(nil) +} + // Tx is an in-progress database transaction. // // A transaction must end with a call to Commit or Rollback. @@ -1412,6 +1663,10 @@ type Tx struct { dc *driverConn txi driver.Tx + // releaseConn is called once the Tx is closed to release + // any held driverConn back to the pool. + releaseConn func(error) + // done transitions from 0 to 1 exactly once, on Commit // or Rollback. once done, all operations fail with // ErrTxDone. @@ -1425,7 +1680,7 @@ type Tx struct { v []*Stmt } - // cancel is called after done transitions from false to true. + // cancel is called after done transitions from 0 to 1. cancel func() // ctx lives for the life of the transaction. @@ -1457,11 +1712,12 @@ var ErrTxDone = errors.New("sql: Transaction has already been committed or rolle // close returns the connection to the pool and // must only be called by Tx.rollback or Tx.Commit. func (tx *Tx) close(err error) { + tx.cancel() + tx.closemu.Lock() defer tx.closemu.Unlock() - tx.db.putConn(tx.dc, err) - tx.cancel() + tx.releaseConn(err) tx.dc = nil tx.txi = nil } @@ -1470,19 +1726,36 @@ func (tx *Tx) close(err error) { // a successful call to (*Tx).grabConn. For tests. var hookTxGrabConn func() -func (tx *Tx) grabConn(ctx context.Context) (*driverConn, error) { +func (tx *Tx) grabConn(ctx context.Context) (*driverConn, releaseConn, error) { select { default: case <-ctx.Done(): - return nil, ctx.Err() + return nil, nil, ctx.Err() } + + // closeme.RLock must come before the check for isDone to prevent the Tx from + // closing while a query is executing. + tx.closemu.RLock() if tx.isDone() { - return nil, ErrTxDone + tx.closemu.RUnlock() + return nil, nil, ErrTxDone } if hookTxGrabConn != nil { // test hook hookTxGrabConn() } - return tx.dc, nil + return tx.dc, tx.closemuRUnlockRelease, nil +} + +func (tx *Tx) txCtx() context.Context { + return tx.ctx +} + +// closemuRUnlockRelease is used as a func(error) method value in +// ExecContext and QueryContext. Unlocking in the releaseConn keeps +// the driver conn from being returned to the connection pool until +// the Rows has been closed. +func (tx *Tx) closemuRUnlockRelease(error) { + tx.closemu.RUnlock() } // Closes all Stmts prepared for this transaction. @@ -1540,7 +1813,7 @@ func (tx *Tx) Rollback() error { return tx.rollback(false) } -// Prepare creates a prepared statement for use within a transaction. +// PrepareContext creates a prepared statement for use within a transaction. // // The returned statement operates within the transaction and will be closed // when the transaction has been committed or rolled back. @@ -1551,44 +1824,15 @@ func (tx *Tx) Rollback() error { // for the execution of the returned statement. The returned statement // will run in the transaction context. func (tx *Tx) PrepareContext(ctx context.Context, query string) (*Stmt, error) { - tx.closemu.RLock() - defer tx.closemu.RUnlock() - - // TODO(bradfitz): We could be more efficient here and either - // provide a method to take an existing Stmt (created on - // perhaps a different Conn), and re-create it on this Conn if - // necessary. Or, better: keep a map in DB of query string to - // Stmts, and have Stmt.Execute do the right thing and - // re-prepare if the Conn in use doesn't have that prepared - // statement. But we'll want to avoid caching the statement - // in the case where we only call conn.Prepare implicitly - // (such as in db.Exec or tx.Exec), but the caller package - // can't be holding a reference to the returned statement. - // Perhaps just looking at the reference count (by noting - // Stmt.Close) would be enough. We might also want a finalizer - // on Stmt to drop the reference count. - dc, err := tx.grabConn(ctx) + dc, release, err := tx.grabConn(ctx) if err != nil { return nil, err } - var si driver.Stmt - withLock(dc, func() { - si, err = ctxDriverPrepare(ctx, dc.ci, query) - }) + stmt, err := tx.db.prepareDC(ctx, dc, release, tx, query) if err != nil { return nil, err } - - stmt := &Stmt{ - db: tx.db, - tx: tx, - txds: &driverStmt{ - Locker: dc, - si: si, - }, - query: query, - } tx.stmts.Lock() tx.stmts.v = append(tx.stmts.v, stmt) tx.stmts.Unlock() @@ -1618,34 +1862,67 @@ func (tx *Tx) Prepare(query string) (*Stmt, error) { // The returned statement operates within the transaction and will be closed // when the transaction has been committed or rolled back. func (tx *Tx) StmtContext(ctx context.Context, stmt *Stmt) *Stmt { - tx.closemu.RLock() - defer tx.closemu.RUnlock() - - // TODO(bradfitz): optimize this. Currently this re-prepares - // each time. This is fine for now to illustrate the API but - // we should really cache already-prepared statements - // per-Conn. See also the big comment in Tx.Prepare. + dc, release, err := tx.grabConn(ctx) + if err != nil { + return &Stmt{stickyErr: err} + } + defer release(nil) if tx.db != stmt.db { return &Stmt{stickyErr: errors.New("sql: Tx.Stmt: statement from different database used")} } - dc, err := tx.grabConn(ctx) - if err != nil { - return &Stmt{stickyErr: err} - } var si driver.Stmt - withLock(dc, func() { - si, err = ctxDriverPrepare(ctx, dc.ci, stmt.query) - }) + var parentStmt *Stmt + stmt.mu.Lock() + if stmt.closed || stmt.cg != nil { + // If the statement has been closed or already belongs to a + // transaction, we can't reuse it in this connection. + // Since tx.StmtContext should never need to be called with a + // Stmt already belonging to tx, we ignore this edge case and + // re-prepare the statement in this case. No need to add + // code-complexity for this. + stmt.mu.Unlock() + withLock(dc, func() { + si, err = ctxDriverPrepare(ctx, dc.ci, stmt.query) + }) + if err != nil { + return &Stmt{stickyErr: err} + } + } else { + stmt.removeClosedStmtLocked() + // See if the statement has already been prepared on this connection, + // and reuse it if possible. + for _, v := range stmt.css { + if v.dc == dc { + si = v.ds.si + break + } + } + + stmt.mu.Unlock() + + if si == nil { + cs, err := stmt.prepareOnConnLocked(ctx, dc) + if err != nil { + return &Stmt{stickyErr: err} + } + si = cs.si + } + parentStmt = stmt + } + txs := &Stmt{ db: tx.db, - tx: tx, - txds: &driverStmt{ + cg: tx, + cgds: &driverStmt{ Locker: dc, si: si, }, - query: stmt.query, - stickyErr: err, + parentStmt: parentStmt, + query: stmt.query, + } + if parentStmt != nil { + tx.db.addDep(parentStmt, txs) } tx.stmts.Lock() tx.stmts.v = append(tx.stmts.v, txs) @@ -1672,42 +1949,11 @@ func (tx *Tx) Stmt(stmt *Stmt) *Stmt { // ExecContext executes a query that doesn't return rows. // For example: an INSERT and UPDATE. func (tx *Tx) ExecContext(ctx context.Context, query string, args ...interface{}) (Result, error) { - tx.closemu.RLock() - defer tx.closemu.RUnlock() - - dc, err := tx.grabConn(ctx) + dc, release, err := tx.grabConn(ctx) if err != nil { return nil, err } - - if execer, ok := dc.ci.(driver.Execer); ok { - dargs, err := driverArgs(nil, args) - if err != nil { - return nil, err - } - var resi driver.Result - withLock(dc, func() { - resi, err = ctxDriverExec(ctx, execer, query, dargs) - }) - if err == nil { - return driverResult{dc, resi}, nil - } - if err != driver.ErrSkip { - return nil, err - } - } - - var si driver.Stmt - withLock(dc, func() { - si, err = ctxDriverPrepare(ctx, dc.ci, query) - }) - if err != nil { - return nil, err - } - ds := &driverStmt{Locker: dc, si: si} - defer ds.Close() - - return resultFromStatement(ctx, ds, args...) + return tx.db.execDC(ctx, dc, release, query, args) } // Exec executes a query that doesn't return rows. @@ -1718,15 +1964,12 @@ func (tx *Tx) Exec(query string, args ...interface{}) (Result, error) { // QueryContext executes a query that returns rows, typically a SELECT. func (tx *Tx) QueryContext(ctx context.Context, query string, args ...interface{}) (*Rows, error) { - tx.closemu.RLock() - defer tx.closemu.RUnlock() - - dc, err := tx.grabConn(ctx) + dc, release, err := tx.grabConn(ctx) if err != nil { return nil, err } - releaseConn := func(error) {} - return tx.db.queryConn(ctx, dc, releaseConn, query, args) + + return tx.db.queryDC(ctx, tx.ctx, dc, release, query, args) } // Query executes a query that returns rows, typically a SELECT. @@ -1737,6 +1980,9 @@ func (tx *Tx) Query(query string, args ...interface{}) (*Rows, error) { // QueryRowContext executes a query that is expected to return at most one row. // QueryRowContext always returns a non-nil value. Errors are deferred until // Row's Scan method is called. +// If the query selects no rows, the *Row's Scan will return ErrNoRows. +// Otherwise, the *Row's Scan scans the first selected row and discards +// the rest. func (tx *Tx) QueryRowContext(ctx context.Context, query string, args ...interface{}) *Row { rows, err := tx.QueryContext(ctx, query, args...) return &Row{rows: rows, err: err} @@ -1745,6 +1991,9 @@ func (tx *Tx) QueryRowContext(ctx context.Context, query string, args ...interfa // QueryRow executes a query that is expected to return at most one row. // QueryRow always returns a non-nil value. Errors are deferred until // Row's Scan method is called. +// If the query selects no rows, the *Row's Scan will return ErrNoRows. +// Otherwise, the *Row's Scan scans the first selected row and discards +// the rest. func (tx *Tx) QueryRow(query string, args ...interface{}) *Row { return tx.QueryRowContext(context.Background(), query, args...) } @@ -1755,6 +2004,24 @@ type connStmt struct { ds *driverStmt } +// stmtConnGrabber represents a Tx or Conn that will return the underlying +// driverConn and release function. +type stmtConnGrabber interface { + // grabConn returns the driverConn and the associated release function + // that must be called when the operation completes. + grabConn(context.Context) (*driverConn, releaseConn, error) + + // txCtx returns the transaction context if available. + // The returned context should be selected on along with + // any query context when awaiting a cancel. + txCtx() context.Context +} + +var ( + _ stmtConnGrabber = &Tx{} + _ stmtConnGrabber = &Conn{} +) + // Stmt is a prepared statement. // A Stmt is safe for concurrent use by multiple goroutines. type Stmt struct { @@ -1765,17 +2032,29 @@ type Stmt struct { closemu sync.RWMutex // held exclusively during close, for read otherwise. - // If in a transaction, else both nil: - tx *Tx - txds *driverStmt + // If Stmt is prepared on a Tx or Conn then cg is present and will + // only ever grab a connection from cg. + // If cg is nil then the Stmt must grab an arbitrary connection + // from db and determine if it must prepare the stmt again by + // inspecting css. + cg stmtConnGrabber + cgds *driverStmt + + // parentStmt is set when a transaction-specific statement + // is requested from an identical statement prepared on the same + // conn. parentStmt is used to track the dependency of this statement + // on its originating ("parent") statement so that parentStmt may + // be closed by the user without them having to know whether or not + // any transactions are still using it. + parentStmt *Stmt mu sync.Mutex // protects the rest of the fields closed bool // css is a list of underlying driver statement interfaces // that are valid on particular connections. This is only - // used if tx == nil and one is found that has idle - // connections. If tx != nil, txsi is always used. + // used if cg == nil and one is found that has idle + // connections. If cg != nil, cgds is always used. css []connStmt // lastNumClosed is copied from db.numClosed when Stmt is created @@ -1790,8 +2069,12 @@ func (s *Stmt) ExecContext(ctx context.Context, args ...interface{}) (Result, er defer s.closemu.RUnlock() var res Result - for i := 0; i < maxBadConnRetries; i++ { - _, releaseConn, ds, err := s.connStmt(ctx) + strategy := cachedOrNewConn + for i := 0; i < maxBadConnRetries+1; i++ { + if i == maxBadConnRetries { + strategy = alwaysNewConn + } + dc, releaseConn, ds, err := s.connStmt(ctx, strategy) if err != nil { if err == driver.ErrBadConn { continue @@ -1799,7 +2082,7 @@ func (s *Stmt) ExecContext(ctx context.Context, args ...interface{}) (Result, er return nil, err } - res, err = resultFromStatement(ctx, ds, args...) + res, err = resultFromStatement(ctx, dc.ci, ds, args...) releaseConn(err) if err != driver.ErrBadConn { return res, err @@ -1814,23 +2097,8 @@ func (s *Stmt) Exec(args ...interface{}) (Result, error) { return s.ExecContext(context.Background(), args...) } -func driverNumInput(ds *driverStmt) int { - ds.Lock() - defer ds.Unlock() // in case NumInput panics - return ds.si.NumInput() -} - -func resultFromStatement(ctx context.Context, ds *driverStmt, args ...interface{}) (Result, error) { - want := driverNumInput(ds) - - // -1 means the driver doesn't know how to count the number of - // placeholders, so we won't sanity check input here and instead let the - // driver deal with errors. - if want != -1 && len(args) != want { - return nil, fmt.Errorf("sql: expected %d arguments, got %d", want, len(args)) - } - - dargs, err := driverArgs(ds, args) +func resultFromStatement(ctx context.Context, ci driver.Conn, ds *driverStmt, args ...interface{}) (Result, error) { + dargs, err := driverArgs(ci, ds, args) if err != nil { return nil, err } @@ -1874,7 +2142,7 @@ func (s *Stmt) removeClosedStmtLocked() { // connStmt returns a free driver connection on which to execute the // statement, a function to call to release the connection, and a // statement bound to that connection. -func (s *Stmt) connStmt(ctx context.Context) (ci *driverConn, releaseConn func(error), ds *driverStmt, err error) { +func (s *Stmt) connStmt(ctx context.Context, strategy connReuseStrategy) (dc *driverConn, releaseConn func(error), ds *driverStmt, err error) { if err = s.stickyErr; err != nil { return } @@ -1885,22 +2153,21 @@ func (s *Stmt) connStmt(ctx context.Context) (ci *driverConn, releaseConn func(e return } - // In a transaction, we always use the connection that the - // transaction was created on. - if s.tx != nil { + // In a transaction or connection, we always use the connection that the + // the stmt was created on. + if s.cg != nil { s.mu.Unlock() - ci, err = s.tx.grabConn(ctx) // blocks, waiting for the connection. + dc, releaseConn, err = s.cg.grabConn(ctx) // blocks, waiting for the connection. if err != nil { return } - releaseConn = func(error) {} - return ci, releaseConn, s.txds, nil + return dc, releaseConn, s.cgds, nil } s.removeClosedStmtLocked() s.mu.Unlock() - dc, err := s.db.conn(ctx, cachedOrNewConn) + dc, err = s.db.conn(ctx, strategy) if err != nil { return nil, nil, nil, err } @@ -1916,18 +2183,28 @@ func (s *Stmt) connStmt(ctx context.Context) (ci *driverConn, releaseConn func(e // No luck; we need to prepare the statement on this connection withLock(dc, func() { - ds, err = dc.prepareLocked(ctx, s.query) + ds, err = s.prepareOnConnLocked(ctx, dc) }) if err != nil { - s.db.putConn(dc, err) + dc.releaseConn(err) return nil, nil, nil, err } + + return dc, dc.releaseConn, ds, nil +} + +// prepareOnConnLocked prepares the query in Stmt s on dc and adds it to the list of +// open connStmt on the statement. It assumes the caller is holding the lock on dc. +func (s *Stmt) prepareOnConnLocked(ctx context.Context, dc *driverConn) (*driverStmt, error) { + si, err := dc.prepareLocked(ctx, s.cg, s.query) + if err != nil { + return nil, err + } + cs := connStmt{dc, si} s.mu.Lock() - cs := connStmt{dc, ds} s.css = append(s.css, cs) s.mu.Unlock() - - return dc, dc.releaseConn, ds, nil + return cs.ds, nil } // QueryContext executes a prepared query statement with the given arguments @@ -1937,8 +2214,12 @@ func (s *Stmt) QueryContext(ctx context.Context, args ...interface{}) (*Rows, er defer s.closemu.RUnlock() var rowsi driver.Rows - for i := 0; i < maxBadConnRetries; i++ { - dc, releaseConn, ds, err := s.connStmt(ctx) + strategy := cachedOrNewConn + for i := 0; i < maxBadConnRetries+1; i++ { + if i == maxBadConnRetries { + strategy = alwaysNewConn + } + dc, releaseConn, ds, err := s.connStmt(ctx, strategy) if err != nil { if err == driver.ErrBadConn { continue @@ -1946,7 +2227,7 @@ func (s *Stmt) QueryContext(ctx context.Context, args ...interface{}) (*Rows, er return nil, err } - rowsi, err = rowsiFromStatement(ctx, ds, args...) + rowsi, err = rowsiFromStatement(ctx, dc.ci, ds, args...) if err == nil { // Note: ownership of ci passes to the *Rows, to be freed // with releaseConn. @@ -1955,12 +2236,21 @@ func (s *Stmt) QueryContext(ctx context.Context, args ...interface{}) (*Rows, er rowsi: rowsi, // releaseConn set below } + // addDep must be added before initContextClose or it could attempt + // to removeDep before it has been added. s.db.addDep(s, rows) + + // releaseConn must be set before initContextClose or it could + // release the connection before it is set. rows.releaseConn = func(err error) { releaseConn(err) s.db.removeDep(s, rows) } - rows.initContextClose(ctx) + var txctx context.Context + if s.cg != nil { + txctx = s.cg.txCtx() + } + rows.initContextClose(ctx, txctx) return rows, nil } @@ -1978,7 +2268,7 @@ func (s *Stmt) Query(args ...interface{}) (*Rows, error) { return s.QueryContext(context.Background(), args...) } -func rowsiFromStatement(ctx context.Context, ds *driverStmt, args ...interface{}) (driver.Rows, error) { +func rowsiFromStatement(ctx context.Context, ci driver.Conn, ds *driverStmt, args ...interface{}) (driver.Rows, error) { var want int withLock(ds, func() { want = ds.si.NumInput() @@ -1991,7 +2281,7 @@ func rowsiFromStatement(ctx context.Context, ds *driverStmt, args ...interface{} return nil, fmt.Errorf("sql: statement expects %d inputs; got %d", want, len(args)) } - dargs, err := driverArgs(ds, args) + dargs, err := driverArgs(ci, ds, args) if err != nil { return nil, err } @@ -2054,13 +2344,21 @@ func (s *Stmt) Close() error { return nil } s.closed = true + txds := s.cgds + s.cgds = nil + s.mu.Unlock() - if s.tx != nil { - return s.txds.Close() + if s.cg == nil { + return s.db.removeDep(s, s) } - return s.db.removeDep(s, s) + if s.parentStmt != nil { + // If parentStmt is set, we must not close s.txds since it's stored + // in the css array of the parentStmt. + return s.db.removeDep(s.parentStmt, s) + } + return txds.Close() } func (s *Stmt) finalClose() error { @@ -2107,18 +2405,28 @@ type Rows struct { lasterr error // non-nil only if closed is true // lastcols is only used in Scan, Next, and NextResultSet which are expected - // not not be called concurrently. + // not to be called concurrently. lastcols []driver.Value } -func (rs *Rows) initContextClose(ctx context.Context) { +func (rs *Rows) initContextClose(ctx, txctx context.Context) { ctx, rs.cancel = context.WithCancel(ctx) - go rs.awaitDone(ctx) + go rs.awaitDone(ctx, txctx) } -// awaitDone blocks until the rows are closed or the context canceled. -func (rs *Rows) awaitDone(ctx context.Context) { - <-ctx.Done() +// awaitDone blocks until either ctx or txctx is canceled. The ctx is provided +// from the query context and is canceled when the query Rows is closed. +// If the query was issued in a transaction, the transaction's context +// is also provided in txctx to ensure Rows is closed if the Tx is closed. +func (rs *Rows) awaitDone(ctx, txctx context.Context) { + var txctxDone <-chan struct{} + if txctx != nil { + txctxDone = txctx.Done() + } + select { + case <-ctx.Done(): + case <-txctxDone: + } rs.close(ctx.Err()) } @@ -2407,7 +2715,7 @@ func (rs *Rows) Scan(dest ...interface{}) error { } // rowsCloseHook returns a function so tests may install the -// hook throug a test only mutex. +// hook through a test only mutex. var rowsCloseHook = func() func(*Rows, *error) { return nil } // Close closes the Rows, preventing further enumeration. If Next is called @@ -2431,7 +2739,9 @@ func (rs *Rows) close(err error) error { rs.lasterr = err } - err = rs.rowsi.Close() + withLock(rs.dc, func() { + err = rs.rowsi.Close() + }) if fn := rowsCloseHook(); fn != nil { fn(rs, &err) } diff --git a/libgo/go/database/sql/sql_test.go b/libgo/go/database/sql/sql_test.go index 381aafc86b7..c935eb43480 100644 --- a/libgo/go/database/sql/sql_test.go +++ b/libgo/go/database/sql/sql_test.go @@ -139,6 +139,7 @@ func closeDB(t testing.TB, db *DB) { t.Errorf("Error closing fakeConn: %v", err) } }) + db.mu.Lock() for i, dc := range db.freeConn { if n := len(dc.openStmt); n > 0 { // Just a sanity check. This is legal in @@ -149,6 +150,8 @@ func closeDB(t testing.TB, db *DB) { t.Errorf("while closing db, freeConn %d/%d had %d open stmts; want 0", i, len(db.freeConn), n) } } + db.mu.Unlock() + err := db.Close() if err != nil { t.Fatalf("error closing DB: %v", err) @@ -322,7 +325,7 @@ func TestQueryContext(t *testing.T) { select { case <-ctx.Done(): if err := ctx.Err(); err != context.Canceled { - t.Fatalf("context err = %v; want context.Canceled", ctx.Err()) + t.Fatalf("context err = %v; want context.Canceled", err) } default: t.Fatalf("context err = nil; want context.Canceled") @@ -413,7 +416,7 @@ func TestTxContextWait(t *testing.T) { db := newTestDB(t, "people") defer closeDB(t, db) - ctx, cancel := context.WithTimeout(context.Background(), time.Millisecond*15) + ctx, cancel := context.WithTimeout(context.Background(), 15*time.Millisecond) defer cancel() tx, err := db.BeginTx(ctx, nil) @@ -590,13 +593,13 @@ func TestPoolExhaustOnCancel(t *testing.T) { saturate.Wait() // Now cancel the request while it is waiting. - ctx, cancel := context.WithTimeout(context.Background(), time.Second*2) + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) defer cancel() for i := 0; i < max; i++ { ctxReq, cancelReq := context.WithCancel(ctx) go func() { - time.Sleep(time.Millisecond * 100) + time.Sleep(100 * time.Millisecond) cancelReq() }() err := db.PingContext(ctxReq) @@ -874,7 +877,7 @@ func TestStatementClose(t *testing.T) { msg string }{ {&Stmt{stickyErr: want}, "stickyErr not propagated"}, - {&Stmt{tx: &Tx{}, txds: &driverStmt{Locker: &sync.Mutex{}, si: stubDriverStmt{want}}}, "driverStmt.Close() error not propagated"}, + {&Stmt{cg: &Tx{}, cgds: &driverStmt{Locker: &sync.Mutex{}, si: stubDriverStmt{want}}}, "driverStmt.Close() error not propagated"}, } for _, test := range tests { if err := test.stmt.Close(); err != want { @@ -1024,6 +1027,196 @@ func TestTxStmt(t *testing.T) { } } +func TestTxStmtPreparedOnce(t *testing.T) { + db := newTestDB(t, "") + defer closeDB(t, db) + exec(t, db, "CREATE|t1|name=string,age=int32") + + prepares0 := numPrepares(t, db) + + // db.Prepare increments numPrepares. + stmt, err := db.Prepare("INSERT|t1|name=?,age=?") + if err != nil { + t.Fatalf("Stmt, err = %v, %v", stmt, err) + } + defer stmt.Close() + + tx, err := db.Begin() + if err != nil { + t.Fatalf("Begin = %v", err) + } + + txs1 := tx.Stmt(stmt) + txs2 := tx.Stmt(stmt) + + _, err = txs1.Exec("Go", 7) + if err != nil { + t.Fatalf("Exec = %v", err) + } + txs1.Close() + + _, err = txs2.Exec("Gopher", 8) + if err != nil { + t.Fatalf("Exec = %v", err) + } + txs2.Close() + + err = tx.Commit() + if err != nil { + t.Fatalf("Commit = %v", err) + } + + if prepares := numPrepares(t, db) - prepares0; prepares != 1 { + t.Errorf("executed %d Prepare statements; want 1", prepares) + } +} + +func TestTxStmtClosedRePrepares(t *testing.T) { + db := newTestDB(t, "") + defer closeDB(t, db) + exec(t, db, "CREATE|t1|name=string,age=int32") + + prepares0 := numPrepares(t, db) + + // db.Prepare increments numPrepares. + stmt, err := db.Prepare("INSERT|t1|name=?,age=?") + if err != nil { + t.Fatalf("Stmt, err = %v, %v", stmt, err) + } + tx, err := db.Begin() + if err != nil { + t.Fatalf("Begin = %v", err) + } + err = stmt.Close() + if err != nil { + t.Fatalf("stmt.Close() = %v", err) + } + // tx.Stmt increments numPrepares because stmt is closed. + txs := tx.Stmt(stmt) + if txs.stickyErr != nil { + t.Fatal(txs.stickyErr) + } + if txs.parentStmt != nil { + t.Fatal("expected nil parentStmt") + } + _, err = txs.Exec(`Eric`, 82) + if err != nil { + t.Fatalf("txs.Exec = %v", err) + } + + err = txs.Close() + if err != nil { + t.Fatalf("txs.Close = %v", err) + } + + tx.Rollback() + + if prepares := numPrepares(t, db) - prepares0; prepares != 2 { + t.Errorf("executed %d Prepare statements; want 2", prepares) + } +} + +func TestParentStmtOutlivesTxStmt(t *testing.T) { + db := newTestDB(t, "") + defer closeDB(t, db) + exec(t, db, "CREATE|t1|name=string,age=int32") + + // Make sure everything happens on the same connection. + db.SetMaxOpenConns(1) + + prepares0 := numPrepares(t, db) + + // db.Prepare increments numPrepares. + stmt, err := db.Prepare("INSERT|t1|name=?,age=?") + if err != nil { + t.Fatalf("Stmt, err = %v, %v", stmt, err) + } + defer stmt.Close() + tx, err := db.Begin() + if err != nil { + t.Fatalf("Begin = %v", err) + } + txs := tx.Stmt(stmt) + if len(stmt.css) != 1 { + t.Fatalf("len(stmt.css) = %v; want 1", len(stmt.css)) + } + err = txs.Close() + if err != nil { + t.Fatalf("txs.Close() = %v", err) + } + err = tx.Rollback() + if err != nil { + t.Fatalf("tx.Rollback() = %v", err) + } + // txs must not be valid. + _, err = txs.Exec("Suzan", 30) + if err == nil { + t.Fatalf("txs.Exec(), expected err") + } + // Stmt must still be valid. + _, err = stmt.Exec("Janina", 25) + if err != nil { + t.Fatalf("stmt.Exec() = %v", err) + } + + if prepares := numPrepares(t, db) - prepares0; prepares != 1 { + t.Errorf("executed %d Prepare statements; want 1", prepares) + } +} + +// Test that tx.Stmt called with a statement already +// associated with tx as argument re-prepares the same +// statement again. +func TestTxStmtFromTxStmtRePrepares(t *testing.T) { + db := newTestDB(t, "") + defer closeDB(t, db) + exec(t, db, "CREATE|t1|name=string,age=int32") + prepares0 := numPrepares(t, db) + // db.Prepare increments numPrepares. + stmt, err := db.Prepare("INSERT|t1|name=?,age=?") + if err != nil { + t.Fatalf("Stmt, err = %v, %v", stmt, err) + } + defer stmt.Close() + + tx, err := db.Begin() + if err != nil { + t.Fatalf("Begin = %v", err) + } + txs1 := tx.Stmt(stmt) + + // tx.Stmt(txs1) increments numPrepares because txs1 already + // belongs to a transaction (albeit the same transaction). + txs2 := tx.Stmt(txs1) + if txs2.stickyErr != nil { + t.Fatal(txs2.stickyErr) + } + if txs2.parentStmt != nil { + t.Fatal("expected nil parentStmt") + } + _, err = txs2.Exec(`Eric`, 82) + if err != nil { + t.Fatal(err) + } + + err = txs1.Close() + if err != nil { + t.Fatalf("txs1.Close = %v", err) + } + err = txs2.Close() + if err != nil { + t.Fatalf("txs1.Close = %v", err) + } + err = tx.Rollback() + if err != nil { + t.Fatalf("tx.Rollback = %v", err) + } + + if prepares := numPrepares(t, db) - prepares0; prepares != 2 { + t.Errorf("executed %d Prepare statements; want 2", prepares) + } +} + // Issue: https://golang.org/issue/2784 // This test didn't fail before because we got lucky with the fakedb driver. // It was failing, and now not, in github.com/bradfitz/go-sql-test @@ -1108,6 +1301,69 @@ func TestTxErrBadConn(t *testing.T) { } } +func TestConnQuery(t *testing.T) { + db := newTestDB(t, "people") + defer closeDB(t, db) + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + conn, err := db.Conn(ctx) + if err != nil { + t.Fatal(err) + } + defer conn.Close() + + var name string + err = conn.QueryRowContext(ctx, "SELECT|people|name|age=?", 3).Scan(&name) + if err != nil { + t.Fatal(err) + } + if name != "Chris" { + t.Fatalf("unexpected result, got %q want Chris", name) + } + + err = conn.PingContext(ctx) + if err != nil { + t.Fatal(err) + } +} + +func TestConnTx(t *testing.T) { + db := newTestDB(t, "people") + defer closeDB(t, db) + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + conn, err := db.Conn(ctx) + if err != nil { + t.Fatal(err) + } + defer conn.Close() + + tx, err := conn.BeginTx(ctx, nil) + if err != nil { + t.Fatal(err) + } + insertName, insertAge := "Nancy", 33 + _, err = tx.ExecContext(ctx, "INSERT|people|name=?,age=?,photo=APHOTO", insertName, insertAge) + if err != nil { + t.Fatal(err) + } + err = tx.Commit() + if err != nil { + t.Fatal(err) + } + + var selectName string + err = conn.QueryRowContext(ctx, "SELECT|people|name|age=?", insertAge).Scan(&selectName) + if err != nil { + t.Fatal(err) + } + if selectName != insertName { + t.Fatalf("got %q want %q", selectName, insertName) + } +} + // Tests fix for issue 2542, that we release a lock when querying on // a closed connection. func TestIssue2542Deadlock(t *testing.T) { @@ -1831,8 +2087,8 @@ func TestConnMaxLifetime(t *testing.T) { } // Expire first conn - offset = time.Second * 11 - db.SetConnMaxLifetime(time.Second * 10) + offset = 11 * time.Second + db.SetConnMaxLifetime(10 * time.Second) if err != nil { t.Fatal(err) } @@ -2078,9 +2334,13 @@ func TestStmtCloseOrder(t *testing.T) { // Test cases where there's more than maxBadConnRetries bad connections in the // pool (issue 8834) func TestManyErrBadConn(t *testing.T) { - manyErrBadConnSetup := func() *DB { + manyErrBadConnSetup := func(first ...func(db *DB)) *DB { db := newTestDB(t, "people") + for _, f := range first { + f(db) + } + nconn := maxBadConnRetries + 1 db.SetMaxIdleConns(nconn) db.SetMaxOpenConns(nconn) @@ -2148,6 +2408,128 @@ func TestManyErrBadConn(t *testing.T) { if err = stmt.Close(); err != nil { t.Fatal(err) } + + // Stmt.Exec + db = manyErrBadConnSetup(func(db *DB) { + stmt, err = db.Prepare("INSERT|people|name=Julia,age=19") + if err != nil { + t.Fatal(err) + } + }) + defer closeDB(t, db) + _, err = stmt.Exec() + if err != nil { + t.Fatal(err) + } + if err = stmt.Close(); err != nil { + t.Fatal(err) + } + + // Stmt.Query + db = manyErrBadConnSetup(func(db *DB) { + stmt, err = db.Prepare("SELECT|people|age,name|") + if err != nil { + t.Fatal(err) + } + }) + defer closeDB(t, db) + rows, err = stmt.Query() + if err != nil { + t.Fatal(err) + } + if err = rows.Close(); err != nil { + t.Fatal(err) + } + if err = stmt.Close(); err != nil { + t.Fatal(err) + } + + // Conn + db = manyErrBadConnSetup() + defer closeDB(t, db) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + conn, err := db.Conn(ctx) + if err != nil { + t.Fatal(err) + } + err = conn.Close() + if err != nil { + t.Fatal(err) + } + + // Ping + db = manyErrBadConnSetup() + defer closeDB(t, db) + err = db.PingContext(ctx) + if err != nil { + t.Fatal(err) + } +} + +// TestIssue20575 ensures the Rows from query does not block +// closing a transaction. Ensure Rows is closed while closing a trasaction. +func TestIssue20575(t *testing.T) { + db := newTestDB(t, "people") + defer closeDB(t, db) + + tx, err := db.Begin() + if err != nil { + t.Fatal(err) + } + ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second) + defer cancel() + _, err = tx.QueryContext(ctx, "SELECT|people|age,name|") + if err != nil { + t.Fatal(err) + } + // Do not close Rows from QueryContext. + err = tx.Rollback() + if err != nil { + t.Fatal(err) + } + select { + default: + case <-ctx.Done(): + t.Fatal("timeout: failed to rollback query without closing rows:", ctx.Err()) + } +} + +// TestIssue20622 tests closing the transaction before rows is closed, requires +// the race detector to fail. +func TestIssue20622(t *testing.T) { + db := newTestDB(t, "people") + defer closeDB(t, db) + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + tx, err := db.BeginTx(ctx, nil) + if err != nil { + t.Fatal(err) + } + + rows, err := tx.Query("SELECT|people|age,name|") + if err != nil { + t.Fatal(err) + } + + count := 0 + for rows.Next() { + count++ + var age int + var name string + if err := rows.Scan(&age, &name); err != nil { + t.Fatal("scan failed", err) + } + + if count == 1 { + cancel() + } + time.Sleep(100 * time.Millisecond) + } + rows.Close() + tx.Commit() } // golang.org/issue/5718 @@ -2751,7 +3133,7 @@ func TestIssue18429(t *testing.T) { if err != nil { return } - // This is expected to give a cancel error many, but not all the time. + // This is expected to give a cancel error most, but not all the time. // Test failure will happen with a panic or other race condition being // reported. rows, _ := tx.QueryContext(ctx, "WAIT|"+qwait+"|SELECT|people|name|") @@ -2766,6 +3148,46 @@ func TestIssue18429(t *testing.T) { wg.Wait() } +// TestIssue20160 attempts to test a short context life on a stmt Query. +func TestIssue20160(t *testing.T) { + db := newTestDB(t, "people") + defer closeDB(t, db) + + ctx := context.Background() + sem := make(chan bool, 20) + var wg sync.WaitGroup + + const milliWait = 30 + + stmt, err := db.PrepareContext(ctx, "SELECT|people|name|") + if err != nil { + t.Fatal(err) + } + defer stmt.Close() + + for i := 0; i < 100; i++ { + sem <- true + wg.Add(1) + go func() { + defer func() { + <-sem + wg.Done() + }() + ctx, cancel := context.WithTimeout(ctx, time.Duration(rand.Intn(milliWait))*time.Millisecond) + defer cancel() + + // This is expected to give a cancel error most, but not all the time. + // Test failure will happen with a panic or other race condition being + // reported. + rows, _ := stmt.QueryContext(ctx) + if rows != nil { + rows.Close() + } + }() + } + wg.Wait() +} + // TestIssue18719 closes the context right before use. The sql.driverConn // will nil out the ci on close in a lock, but if another process uses it right after // it will panic with on the nil ref. @@ -2788,7 +3210,7 @@ func TestIssue18719(t *testing.T) { // Wait for the context to cancel and tx to rollback. for tx.isDone() == false { - time.Sleep(time.Millisecond * 3) + time.Sleep(3 * time.Millisecond) } } defer func() { hookTxGrabConn = nil }() @@ -2807,19 +3229,64 @@ func TestIssue18719(t *testing.T) { // canceled context. cancel() - waitForRowsClose(t, rows, 5*time.Second) +} + +func TestIssue20647(t *testing.T) { + db := newTestDB(t, "people") + defer closeDB(t, db) + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + conn, err := db.Conn(ctx) + if err != nil { + t.Fatal(err) + } + defer conn.Close() + + stmt, err := conn.PrepareContext(ctx, "SELECT|people|name|") + if err != nil { + t.Fatal(err) + } + defer stmt.Close() + + rows1, err := stmt.QueryContext(ctx) + if err != nil { + t.Fatal("rows1", err) + } + defer rows1.Close() + + rows2, err := stmt.QueryContext(ctx) + if err != nil { + t.Fatal("rows2", err) + } + defer rows2.Close() + + if rows1.dc != rows2.dc { + t.Fatal("stmt prepared on Conn does not use same connection") + } } func TestConcurrency(t *testing.T) { - doConcurrentTest(t, new(concurrentDBQueryTest)) - doConcurrentTest(t, new(concurrentDBExecTest)) - doConcurrentTest(t, new(concurrentStmtQueryTest)) - doConcurrentTest(t, new(concurrentStmtExecTest)) - doConcurrentTest(t, new(concurrentTxQueryTest)) - doConcurrentTest(t, new(concurrentTxExecTest)) - doConcurrentTest(t, new(concurrentTxStmtQueryTest)) - doConcurrentTest(t, new(concurrentTxStmtExecTest)) - doConcurrentTest(t, new(concurrentRandomTest)) + list := []struct { + name string + ct concurrentTest + }{ + {"Query", new(concurrentDBQueryTest)}, + {"Exec", new(concurrentDBExecTest)}, + {"StmtQuery", new(concurrentStmtQueryTest)}, + {"StmtExec", new(concurrentStmtExecTest)}, + {"TxQuery", new(concurrentTxQueryTest)}, + {"TxExec", new(concurrentTxExecTest)}, + {"TxStmtQuery", new(concurrentTxStmtQueryTest)}, + {"TxStmtExec", new(concurrentTxStmtExecTest)}, + {"Random", new(concurrentRandomTest)}, + } + for _, item := range list { + t.Run(item.name, func(t *testing.T) { + doConcurrentTest(t, item.ct) + }) + } } func TestConnectionLeak(t *testing.T) { @@ -2874,6 +3341,131 @@ func TestConnectionLeak(t *testing.T) { wg.Wait() } +type nvcDriver struct { + fakeDriver + skipNamedValueCheck bool +} + +func (d *nvcDriver) Open(dsn string) (driver.Conn, error) { + c, err := d.fakeDriver.Open(dsn) + fc := c.(*fakeConn) + fc.db.allowAny = true + return &nvcConn{fc, d.skipNamedValueCheck}, err +} + +type nvcConn struct { + *fakeConn + skipNamedValueCheck bool +} + +type decimal struct { + value int +} + +type doNotInclude struct{} + +var _ driver.NamedValueChecker = &nvcConn{} + +func (c *nvcConn) CheckNamedValue(nv *driver.NamedValue) error { + if c.skipNamedValueCheck { + return driver.ErrSkip + } + switch v := nv.Value.(type) { + default: + return driver.ErrSkip + case Out: + switch ov := v.Dest.(type) { + default: + return errors.New("unkown NameValueCheck OUTPUT type") + case *string: + *ov = "from-server" + nv.Value = "OUT:*string" + } + return nil + case decimal, []int64: + return nil + case doNotInclude: + return driver.ErrRemoveArgument + } +} + +func TestNamedValueChecker(t *testing.T) { + Register("NamedValueCheck", &nvcDriver{}) + db, err := Open("NamedValueCheck", "") + if err != nil { + t.Fatal(err) + } + defer db.Close() + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + _, err = db.ExecContext(ctx, "WIPE") + if err != nil { + t.Fatal("exec wipe", err) + } + + _, err = db.ExecContext(ctx, "CREATE|keys|dec1=any,str1=string,out1=string,array1=any") + if err != nil { + t.Fatal("exec create", err) + } + + o1 := "" + _, err = db.ExecContext(ctx, "INSERT|keys|dec1=?A,str1=?,out1=?O1,array1=?", Named("A", decimal{123}), "hello", Named("O1", Out{Dest: &o1}), []int64{42, 128, 707}, doNotInclude{}) + if err != nil { + t.Fatal("exec insert", err) + } + var ( + str1 string + dec1 decimal + arr1 []int64 + ) + err = db.QueryRowContext(ctx, "SELECT|keys|dec1,str1,array1|").Scan(&dec1, &str1, &arr1) + if err != nil { + t.Fatal("select", err) + } + + list := []struct{ got, want interface{} }{ + {o1, "from-server"}, + {dec1, decimal{123}}, + {str1, "hello"}, + {arr1, []int64{42, 128, 707}}, + } + + for index, item := range list { + if !reflect.DeepEqual(item.got, item.want) { + t.Errorf("got %#v wanted %#v for index %d", item.got, item.want, index) + } + } +} + +func TestNamedValueCheckerSkip(t *testing.T) { + Register("NamedValueCheckSkip", &nvcDriver{skipNamedValueCheck: true}) + db, err := Open("NamedValueCheckSkip", "") + if err != nil { + t.Fatal(err) + } + defer db.Close() + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + _, err = db.ExecContext(ctx, "WIPE") + if err != nil { + t.Fatal("exec wipe", err) + } + + _, err = db.ExecContext(ctx, "CREATE|keys|dec1=any") + if err != nil { + t.Fatal("exec create", err) + } + + _, err = db.ExecContext(ctx, "INSERT|keys|dec1=?A", Named("A", decimal{123})) + if err == nil { + t.Fatalf("expected error with bad argument, got %v", err) + } +} + // badConn implements a bad driver.Conn, for TestBadDriver. // The Exec method panics. type badConn struct{} @@ -2965,6 +3557,24 @@ func TestPing(t *testing.T) { } } +// Issue 18101. +func TestTypedString(t *testing.T) { + db := newTestDB(t, "people") + defer closeDB(t, db) + + type Str string + var scanned Str + + err := db.QueryRow("SELECT|people|name|name=?", "Alice").Scan(&scanned) + if err != nil { + t.Fatal(err) + } + expected := Str("Alice") + if scanned != expected { + t.Errorf("expected %+v, got %+v", expected, scanned) + } +} + func BenchmarkConcurrentDBExec(b *testing.B) { b.ReportAllocs() ct := new(concurrentDBExecTest) diff --git a/libgo/go/debug/dwarf/export_test.go b/libgo/go/debug/dwarf/export_test.go new file mode 100644 index 00000000000..b8a25ff531d --- /dev/null +++ b/libgo/go/debug/dwarf/export_test.go @@ -0,0 +1,7 @@ +// Copyright 2017 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package dwarf + +var PathJoin = pathJoin diff --git a/libgo/go/debug/dwarf/line.go b/libgo/go/debug/dwarf/line.go index ed82feef92b..4e6e1429d93 100644 --- a/libgo/go/debug/dwarf/line.go +++ b/libgo/go/debug/dwarf/line.go @@ -9,6 +9,7 @@ import ( "fmt" "io" "path" + "strings" ) // A LineReader reads a sequence of LineEntry structures from a DWARF @@ -247,10 +248,10 @@ func (r *LineReader) readHeader() error { if len(directory) == 0 { break } - if !path.IsAbs(directory) { + if !pathIsAbs(directory) { // Relative paths are implicitly relative to // the compilation directory. - directory = path.Join(r.directories[0], directory) + directory = pathJoin(r.directories[0], directory) } r.directories = append(r.directories, directory) } @@ -283,11 +284,11 @@ func (r *LineReader) readFileEntry() (bool, error) { } off := r.buf.off dirIndex := int(r.buf.uint()) - if !path.IsAbs(name) { + if !pathIsAbs(name) { if dirIndex >= len(r.directories) { return false, DecodeError{"line", off, "directory index too large"} } - name = path.Join(r.directories[dirIndex], name) + name = pathJoin(r.directories[dirIndex], name) } mtime := r.buf.uint() length := int(r.buf.uint()) @@ -588,3 +589,68 @@ func (r *LineReader) SeekPC(pc uint64, entry *LineEntry) error { *entry = next } } + +// pathIsAbs returns whether path is an absolute path (or "full path +// name" in DWARF parlance). This is in "whatever form makes sense for +// the host system", so this accepts both UNIX-style and DOS-style +// absolute paths. We avoid the filepath package because we want this +// to behave the same regardless of our host system and because we +// don't know what system the paths came from. +func pathIsAbs(path string) bool { + _, path = splitDrive(path) + return len(path) > 0 && (path[0] == '/' || path[0] == '\\') +} + +// pathJoin joins dirname and filename. filename must be relative. +// DWARF paths can be UNIX-style or DOS-style, so this handles both. +func pathJoin(dirname, filename string) string { + if len(dirname) == 0 { + return filename + } + // dirname should be absolute, which means we can determine + // whether it's a DOS path reasonably reliably by looking for + // a drive letter or UNC path. + drive, dirname := splitDrive(dirname) + if drive == "" { + // UNIX-style path. + return path.Join(dirname, filename) + } + // DOS-style path. + drive2, filename := splitDrive(filename) + if drive2 != "" { + if strings.ToLower(drive) != strings.ToLower(drive2) { + // Different drives. There's not much we can + // do here, so just ignore the directory. + return drive2 + filename + } + // Drives are the same. Ignore drive on filename. + } + if !(strings.HasSuffix(dirname, "/") || strings.HasSuffix(dirname, `\`)) && dirname != "" { + dirname += `\` + } + return drive + dirname + filename +} + +// splitDrive splits the DOS drive letter or UNC share point from +// path, if any. path == drive + rest +func splitDrive(path string) (drive, rest string) { + if len(path) >= 2 && path[1] == ':' { + if c := path[0]; 'a' <= c && c <= 'z' || 'A' <= c && c <= 'Z' { + return path[:2], path[2:] + } + } + if len(path) > 3 && (path[0] == '\\' || path[0] == '/') && (path[1] == '\\' || path[1] == '/') { + // Normalize the path so we can search for just \ below. + npath := strings.Replace(path, "/", `\`, -1) + // Get the host part, which must be non-empty. + slash1 := strings.IndexByte(npath[2:], '\\') + 2 + if slash1 > 2 { + // Get the mount-point part, which must be non-empty. + slash2 := strings.IndexByte(npath[slash1+1:], '\\') + slash1 + 1 + if slash2 > slash1 { + return path[:slash2], path[slash2:] + } + } + } + return "", path +} diff --git a/libgo/go/debug/dwarf/line_test.go b/libgo/go/debug/dwarf/line_test.go index cc363f54785..11a254464a2 100644 --- a/libgo/go/debug/dwarf/line_test.go +++ b/libgo/go/debug/dwarf/line_test.go @@ -7,6 +7,7 @@ package dwarf_test import ( . "debug/dwarf" "io" + "strings" "testing" ) @@ -46,6 +47,46 @@ func TestLineELFGCC(t *testing.T) { testLineTable(t, want, elfData(t, "testdata/line-gcc.elf")) } +func TestLineGCCWindows(t *testing.T) { + // Generated by: + // > gcc --version + // gcc (tdm64-1) 4.9.2 + // > gcc -g -o line-gcc-win.bin line1.c C:\workdir\go\src\debug\dwarf\testdata\line2.c + + toWindows := func(lf *LineFile) *LineFile { + lf2 := *lf + lf2.Name = strings.Replace(lf2.Name, "/home/austin/go.dev/", "C:\\workdir\\go\\", -1) + lf2.Name = strings.Replace(lf2.Name, "/", "\\", -1) + return &lf2 + } + file1C := toWindows(file1C) + file1H := toWindows(file1H) + file2C := toWindows(file2C) + + // Line table based on objdump --dwarf=rawline,decodedline + want := []LineEntry{ + {Address: 0x401530, File: file1H, Line: 2, IsStmt: true}, + {Address: 0x401538, File: file1H, Line: 5, IsStmt: true}, + {Address: 0x401541, File: file1H, Line: 6, IsStmt: true, Discriminator: 3}, + {Address: 0x40154b, File: file1H, Line: 5, IsStmt: true, Discriminator: 3}, + {Address: 0x40154f, File: file1H, Line: 5, IsStmt: false, Discriminator: 1}, + {Address: 0x401555, File: file1H, Line: 7, IsStmt: true}, + {Address: 0x40155b, File: file1C, Line: 6, IsStmt: true}, + {Address: 0x401563, File: file1C, Line: 6, IsStmt: true}, + {Address: 0x401568, File: file1C, Line: 7, IsStmt: true}, + {Address: 0x40156d, File: file1C, Line: 8, IsStmt: true}, + {Address: 0x401572, File: file1C, Line: 9, IsStmt: true}, + {Address: 0x401578, EndSequence: true}, + + {Address: 0x401580, File: file2C, Line: 4, IsStmt: true}, + {Address: 0x401588, File: file2C, Line: 5, IsStmt: true}, + {Address: 0x401595, File: file2C, Line: 6, IsStmt: true}, + {Address: 0x40159b, EndSequence: true}, + } + + testLineTable(t, want, peData(t, "testdata/line-gcc-win.bin")) +} + func TestLineELFClang(t *testing.T) { // Generated by: // # clang --version | head -n1 @@ -183,6 +224,11 @@ func testLineTable(t *testing.T, want []LineEntry, d *Data) { } t.Fatal("lr.Next:", err) } + // Ignore sources from the Windows build environment. + if strings.HasPrefix(line.File.Name, "C:\\crossdev\\") || + strings.HasPrefix(line.File.Name, "C:/crossdev/") { + continue + } got = append(got, line) } } @@ -227,3 +273,42 @@ func dumpLines(t *testing.T, lines []LineEntry) { t.Logf(" %+v File:%+v", l, l.File) } } + +type joinTest struct { + dirname, filename string + path string +} + +var joinTests = []joinTest{ + {"a", "b", "a/b"}, + {"a", "", "a"}, + {"", "b", "b"}, + {"/a", "b", "/a/b"}, + {"/a/", "b", "/a/b"}, + + {`C:\Windows\`, `System32`, `C:\Windows\System32`}, + {`C:\Windows\`, ``, `C:\Windows\`}, + {`C:\`, `Windows`, `C:\Windows`}, + {`C:\Windows\`, `C:System32`, `C:\Windows\System32`}, + {`C:\Windows`, `a/b`, `C:\Windows\a/b`}, + {`\\host\share\`, `foo`, `\\host\share\foo`}, + {`\\host\share\`, `foo\bar`, `\\host\share\foo\bar`}, + {`//host/share/`, `foo/bar`, `//host/share/foo/bar`}, + + // The following are "best effort". We shouldn't see relative + // base directories in DWARF, but these test that pathJoin + // doesn't fail miserably if it sees one. + {`C:`, `a`, `C:a`}, + {`C:`, `a\b`, `C:a\b`}, + {`C:.`, `a`, `C:.\a`}, + {`C:a`, `b`, `C:a\b`}, +} + +func TestPathJoin(t *testing.T) { + for _, test := range joinTests { + got := PathJoin(test.dirname, test.filename) + if test.path != got { + t.Errorf("pathJoin(%q, %q) = %q, want %q", test.dirname, test.filename, got, test.path) + } + } +} diff --git a/libgo/go/debug/dwarf/testdata/line-gcc-win.bin b/libgo/go/debug/dwarf/testdata/line-gcc-win.bin new file mode 100644 index 00000000000..583ad44dd07 Binary files /dev/null and b/libgo/go/debug/dwarf/testdata/line-gcc-win.bin differ diff --git a/libgo/go/debug/dwarf/type_test.go b/libgo/go/debug/dwarf/type_test.go index 0283466f088..6c06731ea18 100644 --- a/libgo/go/debug/dwarf/type_test.go +++ b/libgo/go/debug/dwarf/type_test.go @@ -8,6 +8,7 @@ import ( . "debug/dwarf" "debug/elf" "debug/macho" + "debug/pe" "testing" ) @@ -67,6 +68,19 @@ func machoData(t *testing.T, name string) *Data { return d } +func peData(t *testing.T, name string) *Data { + f, err := pe.Open(name) + if err != nil { + t.Fatal(err) + } + + d, err := f.DWARF() + if err != nil { + t.Fatal(err) + } + return d +} + func TestTypedefsELF(t *testing.T) { testTypedefs(t, elfData(t, "testdata/typedef.elf"), "elf") } func TestTypedefsMachO(t *testing.T) { diff --git a/libgo/go/debug/pe/file_cgo_test.go b/libgo/go/debug/pe/file_cgo_test.go new file mode 100644 index 00000000000..739671d73f1 --- /dev/null +++ b/libgo/go/debug/pe/file_cgo_test.go @@ -0,0 +1,31 @@ +// Copyright 2017 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build cgo + +package pe + +import ( + "os/exec" + "testing" +) + +func testCgoDWARF(t *testing.T, linktype int) { + if _, err := exec.LookPath("gcc"); err != nil { + t.Skip("skipping test: gcc is missing") + } + testDWARF(t, linktype) +} + +func TestDefaultLinkerDWARF(t *testing.T) { + testCgoDWARF(t, linkCgoDefault) +} + +func TestInternalLinkerDWARF(t *testing.T) { + testCgoDWARF(t, linkCgoInternal) +} + +func TestExternalLinkerDWARF(t *testing.T) { + testCgoDWARF(t, linkCgoExternal) +} diff --git a/libgo/go/debug/pe/file_test.go b/libgo/go/debug/pe/file_test.go index 5a740c87050..8645d676b75 100644 --- a/libgo/go/debug/pe/file_test.go +++ b/libgo/go/debug/pe/file_test.go @@ -12,8 +12,11 @@ import ( "os/exec" "path/filepath" "reflect" + "regexp" "runtime" + "strconv" "testing" + "text/template" ) type fileTest struct { @@ -288,28 +291,70 @@ func TestOpenFailure(t *testing.T) { } } -func TestDWARF(t *testing.T) { +const ( + linkNoCgo = iota + linkCgoDefault + linkCgoInternal + linkCgoExternal +) + +func testDWARF(t *testing.T, linktype int) { if runtime.GOOS != "windows" { t.Skip("skipping windows only test") } + testenv.MustHaveGoRun(t) tmpdir, err := ioutil.TempDir("", "TestDWARF") if err != nil { - t.Fatal("TempDir failed: ", err) + t.Fatal(err) } defer os.RemoveAll(tmpdir) - prog := ` -package main -func main() { -} -` src := filepath.Join(tmpdir, "a.go") + file, err := os.Create(src) + if err != nil { + t.Fatal(err) + } + err = template.Must(template.New("main").Parse(testprog)).Execute(file, linktype != linkNoCgo) + if err != nil { + if err := file.Close(); err != nil { + t.Error(err) + } + t.Fatal(err) + } + if err := file.Close(); err != nil { + t.Fatal(err) + } + exe := filepath.Join(tmpdir, "a.exe") - err = ioutil.WriteFile(src, []byte(prog), 0644) - output, err := exec.Command(testenv.GoToolPath(t), "build", "-o", exe, src).CombinedOutput() + args := []string{"build", "-o", exe} + switch linktype { + case linkNoCgo: + case linkCgoDefault: + case linkCgoInternal: + args = append(args, "-ldflags", "-linkmode=internal") + case linkCgoExternal: + args = append(args, "-ldflags", "-linkmode=external") + default: + t.Fatalf("invalid linktype parameter of %v", linktype) + } + args = append(args, src) + out, err := exec.Command(testenv.GoToolPath(t), args...).CombinedOutput() + if err != nil { + t.Fatalf("building test executable for linktype %d failed: %s %s", linktype, err, out) + } + out, err = exec.Command(exe).CombinedOutput() if err != nil { - t.Fatalf("building test executable failed: %s %s", err, output) + t.Fatalf("running test executable failed: %s %s", err, out) + } + + matches := regexp.MustCompile("main=(.*)\n").FindStringSubmatch(string(out)) + if len(matches) < 2 { + t.Fatalf("unexpected program output: %s", out) + } + wantaddr, err := strconv.ParseUint(matches[1], 0, 64) + if err != nil { + t.Fatalf("unexpected main address %q: %s", matches[1], err) } f, err := Open(exe) @@ -318,6 +363,16 @@ func main() { } defer f.Close() + var foundDebugGDBScriptsSection bool + for _, sect := range f.Sections { + if sect.Name == ".debug_gdb_scripts" { + foundDebugGDBScriptsSection = true + } + } + if !foundDebugGDBScriptsSection { + t.Error(".debug_gdb_scripts section is not found") + } + d, err := f.DWARF() if err != nil { t.Fatal(err) @@ -334,8 +389,8 @@ func main() { break } if e.Tag == dwarf.TagSubprogram { - for _, f := range e.Field { - if f.Attr == dwarf.AttrName && e.Val(dwarf.AttrName) == "main.main" { + if name, ok := e.Val(dwarf.AttrName).(string); ok && name == "main.main" { + if addr, ok := e.Val(dwarf.AttrLowpc).(uint64); ok && addr == wantaddr { return } } @@ -415,3 +470,65 @@ main(void) } } } + +func TestDWARF(t *testing.T) { + testDWARF(t, linkNoCgo) +} + +const testprog = ` +package main + +import "fmt" +{{if .}}import "C" +{{end}} + +func main() { + fmt.Printf("main=%p\n", main) +} +` + +func TestBuildingWindowsGUI(t *testing.T) { + testenv.MustHaveGoBuild(t) + + if runtime.GOOS != "windows" { + t.Skip("skipping windows only test") + } + tmpdir, err := ioutil.TempDir("", "TestBuildingWindowsGUI") + if err != nil { + t.Fatal(err) + } + defer os.RemoveAll(tmpdir) + + src := filepath.Join(tmpdir, "a.go") + err = ioutil.WriteFile(src, []byte(`package main; func main() {}`), 0644) + if err != nil { + t.Fatal(err) + } + exe := filepath.Join(tmpdir, "a.exe") + cmd := exec.Command(testenv.GoToolPath(t), "build", "-ldflags", "-H=windowsgui", "-o", exe, src) + out, err := cmd.CombinedOutput() + if err != nil { + t.Fatalf("building test executable failed: %s %s", err, out) + } + + f, err := Open(exe) + if err != nil { + t.Fatal(err) + } + defer f.Close() + + const _IMAGE_SUBSYSTEM_WINDOWS_GUI = 2 + + switch oh := f.OptionalHeader.(type) { + case *OptionalHeader32: + if oh.Subsystem != _IMAGE_SUBSYSTEM_WINDOWS_GUI { + t.Errorf("unexpected Subsystem value: have %d, but want %d", oh.Subsystem, _IMAGE_SUBSYSTEM_WINDOWS_GUI) + } + case *OptionalHeader64: + if oh.Subsystem != _IMAGE_SUBSYSTEM_WINDOWS_GUI { + t.Errorf("unexpected Subsystem value: have %d, but want %d", oh.Subsystem, _IMAGE_SUBSYSTEM_WINDOWS_GUI) + } + default: + t.Fatalf("unexpected OptionalHeader type: have %T, but want *pe.OptionalHeader32 or *pe.OptionalHeader64", oh) + } +} diff --git a/libgo/go/encoding/ascii85/ascii85_test.go b/libgo/go/encoding/ascii85/ascii85_test.go index aad199b4fad..1a3a87a596d 100644 --- a/libgo/go/encoding/ascii85/ascii85_test.go +++ b/libgo/go/encoding/ascii85/ascii85_test.go @@ -16,6 +16,18 @@ type testpair struct { decoded, encoded string } +var bigtest = testpair{ + "Man is distinguished, not only by his reason, but by this singular passion from " + + "other animals, which is a lust of the mind, that by a perseverance of delight in " + + "the continued and indefatigable generation of knowledge, exceeds the short " + + "vehemence of any carnal pleasure.", + "9jqo^BlbD-BleB1DJ+*+F(f,q/0JhKFCj@.4Gp$d7F!,L7@<6@)/0JDEF@3BB/F*&OCAfu2/AKY\n" + + "i(DIb:@FD,*)+C]U=@3BN#EcYf8ATD3s@q?d$AftVqCh[NqF-FD5W8ARlolDIa\n" + + "l(DIduD.RTpAKYo'+CT/5+Cei#DII?(E,9)oF*2M7/c\n", +} + var pairs = []testpair{ // Encode returns 0 when len(src) is 0 { @@ -23,17 +35,7 @@ var pairs = []testpair{ "", }, // Wikipedia example - { - "Man is distinguished, not only by his reason, but by this singular passion from " + - "other animals, which is a lust of the mind, that by a perseverance of delight in " + - "the continued and indefatigable generation of knowledge, exceeds the short " + - "vehemence of any carnal pleasure.", - "9jqo^BlbD-BleB1DJ+*+F(f,q/0JhKFCj@.4Gp$d7F!,L7@<6@)/0JDEF@3BB/F*&OCAfu2/AKY\n" + - "i(DIb:@FD,*)+C]U=@3BN#EcYf8ATD3s@q?d$AftVqCh[NqF-FD5W8ARlolDIa\n" + - "l(DIduD.RTpAKYo'+CT/5+Cei#DII?(E,9)oF*2M7/c\n", - }, + bigtest, // Special case when shortening !!!!! to z. { "\000\000\000\000", @@ -41,9 +43,8 @@ var pairs = []testpair{ }, } -var bigtest = pairs[len(pairs)-1] - func testEqual(t *testing.T, msg string, args ...interface{}) bool { + t.Helper() if args[len(args)-2] != args[len(args)-1] { t.Errorf(msg, args...) return false @@ -134,11 +135,15 @@ func TestDecoderBuffering(t *testing.T) { decoder := NewDecoder(strings.NewReader(bigtest.encoded)) buf := make([]byte, len(bigtest.decoded)+12) var total int - for total = 0; total < len(bigtest.decoded); { - n, err := decoder.Read(buf[total : total+bs]) - testEqual(t, "Read from %q at pos %d = %d, %v, want _, %v", bigtest.encoded, total, n, err, error(nil)) + var n int + var err error + for total = 0; total < len(bigtest.decoded) && err == nil; { + n, err = decoder.Read(buf[total : total+bs]) total += n } + if err != nil && err != io.EOF { + t.Errorf("Read from %q at pos %d = %d, unexpected error %v", bigtest.encoded, total, n, err) + } testEqual(t, "Decoding/%d of %q = %q, want %q", bs, bigtest.encoded, string(buf[0:total]), bigtest.decoded) } } diff --git a/libgo/go/encoding/asn1/asn1.go b/libgo/go/encoding/asn1/asn1.go index 044f74ab469..b8e2770596e 100644 --- a/libgo/go/encoding/asn1/asn1.go +++ b/libgo/go/encoding/asn1/asn1.go @@ -22,6 +22,7 @@ package asn1 import ( "errors" "fmt" + "math" "math/big" "reflect" "strconv" @@ -206,6 +207,14 @@ func parseBitString(bytes []byte) (ret BitString, err error) { return } +// NULL + +// NullRawValue is a RawValue with its Tag set to the ASN.1 NULL type tag (5). +var NullRawValue = RawValue{Tag: TagNull} + +// NullBytes contains bytes representing the DER-encoded ASN.1 NULL type. +var NullBytes = []byte{TagNull, 0} + // OBJECT IDENTIFIER // An ObjectIdentifier represents an ASN.1 OBJECT IDENTIFIER. @@ -293,16 +302,24 @@ type Flag bool // given byte slice. It returns the value and the new offset. func parseBase128Int(bytes []byte, initOffset int) (ret, offset int, err error) { offset = initOffset + var ret64 int64 for shifted := 0; offset < len(bytes); shifted++ { - if shifted == 4 { + // 5 * 7 bits per byte == 35 bits of data + // Thus the representation is either non-minimal or too large for an int32 + if shifted == 5 { err = StructuralError{"base 128 integer too large"} return } - ret <<= 7 + ret64 <<= 7 b := bytes[offset] - ret |= int(b & 0x7f) + ret64 |= int64(b & 0x7f) offset++ if b&0x80 == 0 { + ret = int(ret64) + // Ensure that the returned value fits in an int on all platforms + if ret64 > math.MaxInt32 { + err = StructuralError{"base 128 integer too large"} + } return } } @@ -975,12 +992,12 @@ func setDefaultValue(v reflect.Value, params fieldParameters) (ok bool) { // // The following tags on struct fields have special meaning to Unmarshal: // -// application specifies that a APPLICATION tag is used -// default:x sets the default value for optional integer fields (only used if optional is also present) -// explicit specifies that an additional, explicit tag wraps the implicit one -// optional marks the field as ASN.1 OPTIONAL -// set causes a SET, rather than a SEQUENCE type to be expected -// tag:x specifies the ASN.1 tag number; implies ASN.1 CONTEXT SPECIFIC +// application specifies that a APPLICATION tag is used +// default:x sets the default value for optional integer fields (only used if optional is also present) +// explicit specifies that an additional, explicit tag wraps the implicit one +// optional marks the field as ASN.1 OPTIONAL +// set causes a SET, rather than a SEQUENCE type to be expected +// tag:x specifies the ASN.1 tag number; implies ASN.1 CONTEXT SPECIFIC // // If the type of the first field of a structure is RawContent then the raw // ASN1 contents of the struct will be stored in it. diff --git a/libgo/go/encoding/asn1/asn1_test.go b/libgo/go/encoding/asn1/asn1_test.go index 9976656df89..c9eda4069dc 100644 --- a/libgo/go/encoding/asn1/asn1_test.go +++ b/libgo/go/encoding/asn1/asn1_test.go @@ -7,6 +7,7 @@ package asn1 import ( "bytes" "fmt" + "math" "math/big" "reflect" "strings" @@ -386,6 +387,8 @@ var tagAndLengthData = []tagAndLengthTest{ {[]byte{0xa0, 0x81, 0x7f}, false, tagAndLength{}}, // Tag numbers which would overflow int32 are rejected. (The value below is 2^31.) {[]byte{0x1f, 0x88, 0x80, 0x80, 0x80, 0x00, 0x00}, false, tagAndLength{}}, + // Tag numbers that fit in an int32 are valid. (The value below is 2^31 - 1.) + {[]byte{0x1f, 0x87, 0xFF, 0xFF, 0xFF, 0x7F, 0x00}, true, tagAndLength{tag: math.MaxInt32}}, // Long tag number form may not be used for tags that fit in short form. {[]byte{0x1f, 0x1e, 0x00}, false, tagAndLength{}}, } @@ -476,6 +479,7 @@ var unmarshalTestData = []struct { out interface{} }{ {[]byte{0x02, 0x01, 0x42}, newInt(0x42)}, + {[]byte{0x05, 0x00}, &RawValue{0, 5, false, []byte{}, []byte{0x05, 0x00}}}, {[]byte{0x30, 0x08, 0x06, 0x06, 0x2a, 0x86, 0x48, 0x86, 0xf7, 0x0d}, &TestObjectIdentifierStruct{[]int{1, 2, 840, 113549}}}, {[]byte{0x03, 0x04, 0x06, 0x6e, 0x5d, 0xc0}, &BitString{[]byte{110, 93, 192}, 18}}, {[]byte{0x30, 0x09, 0x02, 0x01, 0x01, 0x02, 0x01, 0x02, 0x02, 0x01, 0x03}, &[]int{1, 2, 3}}, @@ -1004,3 +1008,28 @@ func TestUnexportedStructField(t *testing.T) { t.Errorf("got %v, want %v", err, want) } } + +func TestNull(t *testing.T) { + marshaled, err := Marshal(NullRawValue) + if err != nil { + t.Fatal(err) + } + if !bytes.Equal(NullBytes, marshaled) { + t.Errorf("Expected Marshal of NullRawValue to yeild %x, got %x", NullBytes, marshaled) + } + + unmarshaled := RawValue{} + if _, err := Unmarshal(NullBytes, &unmarshaled); err != nil { + t.Fatal(err) + } + + unmarshaled.FullBytes = NullRawValue.FullBytes + if len(unmarshaled.Bytes) == 0 { + // DeepEqual considers a nil slice and an empty slice to be different. + unmarshaled.Bytes = NullRawValue.Bytes + } + + if !reflect.DeepEqual(NullRawValue, unmarshaled) { + t.Errorf("Expected Unmarshal of NullBytes to yield %v, got %v", NullRawValue, unmarshaled) + } +} diff --git a/libgo/go/encoding/asn1/common.go b/libgo/go/encoding/asn1/common.go index 06951808277..cd93b27ecb8 100644 --- a/libgo/go/encoding/asn1/common.go +++ b/libgo/go/encoding/asn1/common.go @@ -24,6 +24,7 @@ const ( TagInteger = 2 TagBitString = 3 TagOctetString = 4 + TagNull = 5 TagOID = 6 TagEnum = 10 TagUTF8String = 12 diff --git a/libgo/go/encoding/asn1/marshal.go b/libgo/go/encoding/asn1/marshal.go index 225fd0849c6..fdadb3996ef 100644 --- a/libgo/go/encoding/asn1/marshal.go +++ b/libgo/go/encoding/asn1/marshal.go @@ -643,10 +643,12 @@ func makeField(v reflect.Value, params fieldParameters) (e encoder, err error) { // In addition to the struct tags recognised by Unmarshal, the following can be // used: // -// ia5: causes strings to be marshaled as ASN.1, IA5 strings -// omitempty: causes empty slices to be skipped -// printable: causes strings to be marshaled as ASN.1, PrintableString strings. -// utf8: causes strings to be marshaled as ASN.1, UTF8 strings +// ia5: causes strings to be marshaled as ASN.1, IA5String values +// omitempty: causes empty slices to be skipped +// printable: causes strings to be marshaled as ASN.1, PrintableString values +// utf8: causes strings to be marshaled as ASN.1, UTF8String values +// utc: causes time.Time to be marshaled as ASN.1, UTCTime values +// generalized: causes time.Time to be marshaled as ASN.1, GeneralizedTime values func Marshal(val interface{}) ([]byte, error) { e, err := makeField(reflect.ValueOf(val), fieldParameters{}) if err != nil { diff --git a/libgo/go/encoding/base32/base32.go b/libgo/go/encoding/base32/base32.go index c193e65e1bc..bf341b54f3d 100644 --- a/libgo/go/encoding/base32/base32.go +++ b/libgo/go/encoding/base32/base32.go @@ -23,8 +23,14 @@ import ( type Encoding struct { encode string decodeMap [256]byte + padChar rune } +const ( + StdPadding rune = '=' // Standard padding character + NoPadding rune = -1 // No padding +) + const encodeStd = "ABCDEFGHIJKLMNOPQRSTUVWXYZ234567" const encodeHex = "0123456789ABCDEFGHIJKLMNOPQRSTUV" @@ -33,6 +39,8 @@ const encodeHex = "0123456789ABCDEFGHIJKLMNOPQRSTUV" func NewEncoding(encoder string) *Encoding { e := new(Encoding) e.encode = encoder + e.padChar = StdPadding + for i := 0; i < len(e.decodeMap); i++ { e.decodeMap[i] = 0xFF } @@ -57,6 +65,26 @@ var removeNewlinesMapper = func(r rune) rune { return r } +// WithPadding creates a new encoding identical to enc except +// with a specified padding character, or NoPadding to disable padding. +// The padding character must not be '\r' or '\n', must not +// be contained in the encoding's alphabet and must be a rune equal or +// below '\xff'. +func (enc Encoding) WithPadding(padding rune) *Encoding { + if padding == '\r' || padding == '\n' || padding > 0xff { + panic("invalid padding") + } + + for i := 0; i < len(enc.encode); i++ { + if rune(enc.encode[i]) == padding { + panic("padding contained in alphabet") + } + } + + enc.padChar = padding + return &enc +} + /* * Encoder */ @@ -73,60 +101,63 @@ func (enc *Encoding) Encode(dst, src []byte) { } for len(src) > 0 { - var b0, b1, b2, b3, b4, b5, b6, b7 byte + var b [8]byte // Unpack 8x 5-bit source blocks into a 5 byte // destination quantum switch len(src) { default: - b7 = src[4] & 0x1F - b6 = src[4] >> 5 + b[7] = src[4] & 0x1F + b[6] = src[4] >> 5 fallthrough case 4: - b6 |= (src[3] << 3) & 0x1F - b5 = (src[3] >> 2) & 0x1F - b4 = src[3] >> 7 + b[6] |= (src[3] << 3) & 0x1F + b[5] = (src[3] >> 2) & 0x1F + b[4] = src[3] >> 7 fallthrough case 3: - b4 |= (src[2] << 1) & 0x1F - b3 = (src[2] >> 4) & 0x1F + b[4] |= (src[2] << 1) & 0x1F + b[3] = (src[2] >> 4) & 0x1F fallthrough case 2: - b3 |= (src[1] << 4) & 0x1F - b2 = (src[1] >> 1) & 0x1F - b1 = (src[1] >> 6) & 0x1F + b[3] |= (src[1] << 4) & 0x1F + b[2] = (src[1] >> 1) & 0x1F + b[1] = (src[1] >> 6) & 0x1F fallthrough case 1: - b1 |= (src[0] << 2) & 0x1F - b0 = src[0] >> 3 + b[1] |= (src[0] << 2) & 0x1F + b[0] = src[0] >> 3 } // Encode 5-bit blocks using the base32 alphabet - dst[0] = enc.encode[b0] - dst[1] = enc.encode[b1] - dst[2] = enc.encode[b2] - dst[3] = enc.encode[b3] - dst[4] = enc.encode[b4] - dst[5] = enc.encode[b5] - dst[6] = enc.encode[b6] - dst[7] = enc.encode[b7] + for i := 0; i < 8; i++ { + if len(dst) > i { + dst[i] = enc.encode[b[i]] + } + } // Pad the final quantum if len(src) < 5 { - dst[7] = '=' + if enc.padChar == NoPadding { + break + } + + dst[7] = byte(enc.padChar) if len(src) < 4 { - dst[6] = '=' - dst[5] = '=' + dst[6] = byte(enc.padChar) + dst[5] = byte(enc.padChar) if len(src) < 3 { - dst[4] = '=' + dst[4] = byte(enc.padChar) if len(src) < 2 { - dst[3] = '=' - dst[2] = '=' + dst[3] = byte(enc.padChar) + dst[2] = byte(enc.padChar) } } } + break } + src = src[5:] dst = dst[8:] } @@ -219,7 +250,12 @@ func NewEncoder(enc *Encoding, w io.Writer) io.WriteCloser { // EncodedLen returns the length in bytes of the base32 encoding // of an input buffer of length n. -func (enc *Encoding) EncodedLen(n int) int { return (n + 4) / 5 * 8 } +func (enc *Encoding) EncodedLen(n int) int { + if enc.padChar == NoPadding { + return (n*8 + 4) / 5 + } + return (n + 4) / 5 * 8 +} /* * Decoder @@ -243,19 +279,28 @@ func (enc *Encoding) decode(dst, src []byte) (n int, end bool, err error) { dlen := 8 for j := 0; j < 8; { - if len(src) == 0 { + + // We have reached the end and are missing padding + if len(src) == 0 && enc.padChar != NoPadding { return n, false, CorruptInputError(olen - len(src) - j) } + + // We have reached the end and are not expecing any padding + if len(src) == 0 && enc.padChar == NoPadding { + dlen, end = j, true + break + } + in := src[0] src = src[1:] - if in == '=' && j >= 2 && len(src) < 8 { + if in == byte(enc.padChar) && j >= 2 && len(src) < 8 { // We've reached the end and there's padding if len(src)+j < 8-1 { // not enough padding return n, false, CorruptInputError(olen) } for k := 0; k < 8-1-j; k++ { - if len(src) > k && src[k] != '=' { + if len(src) > k && src[k] != byte(enc.padChar) { // incorrect padding return n, false, CorruptInputError(olen - len(src) + k - 1) } @@ -296,7 +341,11 @@ func (enc *Encoding) decode(dst, src []byte) (n int, end bool, err error) { case 2: dst[0] = dbuf[0]<<3 | dbuf[1]>>2 } - dst = dst[5:] + + if !end { + dst = dst[5:] + } + switch dlen { case 2: n += 1 @@ -343,18 +392,33 @@ type decoder struct { outbuf [1024 / 8 * 5]byte } -func (d *decoder) Read(p []byte) (n int, err error) { - if d.err != nil { - return 0, d.err +func readEncodedData(r io.Reader, buf []byte, min int) (n int, err error) { + for n < min && err == nil { + var nn int + nn, err = r.Read(buf[n:]) + n += nn } + if n < min && n > 0 && err == io.EOF { + err = io.ErrUnexpectedEOF + } + return +} +func (d *decoder) Read(p []byte) (n int, err error) { // Use leftover decoded output from last read. if len(d.out) > 0 { n = copy(p, d.out) d.out = d.out[n:] + if len(d.out) == 0 { + return n, d.err + } return n, nil } + if d.err != nil { + return 0, d.err + } + // Read a chunk. nn := len(p) / 5 * 8 if nn < 8 { @@ -363,7 +427,8 @@ func (d *decoder) Read(p []byte) (n int, err error) { if nn > len(d.buf) { nn = len(d.buf) } - nn, d.err = io.ReadAtLeast(d.r, d.buf[d.nbuf:nn], 8-d.nbuf) + + nn, d.err = readEncodedData(d.r, d.buf[d.nbuf:nn], 8-d.nbuf) d.nbuf += nn if d.nbuf < 8 { return 0, d.err @@ -373,21 +438,30 @@ func (d *decoder) Read(p []byte) (n int, err error) { nr := d.nbuf / 8 * 8 nw := d.nbuf / 8 * 5 if nw > len(p) { - nw, d.end, d.err = d.enc.decode(d.outbuf[0:], d.buf[0:nr]) + nw, d.end, err = d.enc.decode(d.outbuf[0:], d.buf[0:nr]) d.out = d.outbuf[0:nw] n = copy(p, d.out) d.out = d.out[n:] } else { - n, d.end, d.err = d.enc.decode(p, d.buf[0:nr]) + n, d.end, err = d.enc.decode(p, d.buf[0:nr]) } d.nbuf -= nr for i := 0; i < d.nbuf; i++ { d.buf[i] = d.buf[i+nr] } - if d.err == nil { + if err != nil && (d.err == nil || d.err == io.EOF) { d.err = err } + + if len(d.out) > 0 { + // We cannot return all the decoded bytes to the caller in this + // invocation of Read, so we return a nil error to ensure that Read + // will be called again. The error stored in d.err, if any, will be + // returned with the last set of decoded bytes. + return n, nil + } + return n, d.err } @@ -407,7 +481,7 @@ func (r *newlineFilteringReader) Read(p []byte) (int, error) { offset++ } } - if offset > 0 { + if err != nil || offset > 0 { return offset, err } // Previous buffer entirely whitespace, read again @@ -423,4 +497,10 @@ func NewDecoder(enc *Encoding, r io.Reader) io.Reader { // DecodedLen returns the maximum length in bytes of the decoded data // corresponding to n bytes of base32-encoded data. -func (enc *Encoding) DecodedLen(n int) int { return n / 8 * 5 } +func (enc *Encoding) DecodedLen(n int) int { + if enc.padChar == NoPadding { + return n * 5 / 8 + } + + return n / 8 * 5 +} diff --git a/libgo/go/encoding/base32/base32_test.go b/libgo/go/encoding/base32/base32_test.go index 66a48a3f6f7..56b229d15a6 100644 --- a/libgo/go/encoding/base32/base32_test.go +++ b/libgo/go/encoding/base32/base32_test.go @@ -6,6 +6,7 @@ package base32 import ( "bytes" + "errors" "io" "io/ioutil" "strings" @@ -43,6 +44,7 @@ var bigtest = testpair{ } func testEqual(t *testing.T, msg string, args ...interface{}) bool { + t.Helper() if args[len(args)-2] != args[len(args)-1] { t.Errorf(msg, args...) return false @@ -123,16 +125,174 @@ func TestDecoder(t *testing.T) { } } +type badReader struct { + data []byte + errs []error + called int + limit int +} + +// Populates p with data, returns a count of the bytes written and an +// error. The error returned is taken from badReader.errs, with each +// invocation of Read returning the next error in this slice, or io.EOF, +// if all errors from the slice have already been returned. The +// number of bytes returned is determined by the size of the input buffer +// the test passes to decoder.Read and will be a multiple of 8, unless +// badReader.limit is non zero. +func (b *badReader) Read(p []byte) (int, error) { + lim := len(p) + if b.limit != 0 && b.limit < lim { + lim = b.limit + } + if len(b.data) < lim { + lim = len(b.data) + } + for i := range p[:lim] { + p[i] = b.data[i] + } + b.data = b.data[lim:] + err := io.EOF + if b.called < len(b.errs) { + err = b.errs[b.called] + } + b.called++ + return lim, err +} + +// TestIssue20044 tests that decoder.Read behaves correctly when the caller +// supplied reader returns an error. +func TestIssue20044(t *testing.T) { + badErr := errors.New("bad reader error") + testCases := []struct { + r badReader + res string + err error + dbuflen int + }{ + // Check valid input data accompanied by an error is processed and the error is propagated. + {r: badReader{data: []byte("MY======"), errs: []error{badErr}}, + res: "f", err: badErr}, + // Check a read error accompanied by input data consisting of newlines only is propagated. + {r: badReader{data: []byte("\n\n\n\n\n\n\n\n"), errs: []error{badErr, nil}}, + res: "", err: badErr}, + // Reader will be called twice. The first time it will return 8 newline characters. The + // second time valid base32 encoded data and an error. The data should be decoded + // correctly and the error should be propagated. + {r: badReader{data: []byte("\n\n\n\n\n\n\n\nMY======"), errs: []error{nil, badErr}}, + res: "f", err: badErr, dbuflen: 8}, + // Reader returns invalid input data (too short) and an error. Verify the reader + // error is returned. + {r: badReader{data: []byte("MY====="), errs: []error{badErr}}, + res: "", err: badErr}, + // Reader returns invalid input data (too short) but no error. Verify io.ErrUnexpectedEOF + // is returned. + {r: badReader{data: []byte("MY====="), errs: []error{nil}}, + res: "", err: io.ErrUnexpectedEOF}, + // Reader returns invalid input data and an error. Verify the reader and not the + // decoder error is returned. + {r: badReader{data: []byte("Ma======"), errs: []error{badErr}}, + res: "", err: badErr}, + // Reader returns valid data and io.EOF. Check data is decoded and io.EOF is propagated. + {r: badReader{data: []byte("MZXW6YTB"), errs: []error{io.EOF}}, + res: "fooba", err: io.EOF}, + // Check errors are properly reported when decoder.Read is called multiple times. + // decoder.Read will be called 8 times, badReader.Read will be called twice, returning + // valid data both times but an error on the second call. + {r: badReader{data: []byte("NRSWC43VOJSS4==="), errs: []error{nil, badErr}}, + res: "leasure.", err: badErr, dbuflen: 1}, + // Check io.EOF is properly reported when decoder.Read is called multiple times. + // decoder.Read will be called 8 times, badReader.Read will be called twice, returning + // valid data both times but io.EOF on the second call. + {r: badReader{data: []byte("NRSWC43VOJSS4==="), errs: []error{nil, io.EOF}}, + res: "leasure.", err: io.EOF, dbuflen: 1}, + // The following two test cases check that errors are propagated correctly when more than + // 8 bytes are read at a time. + {r: badReader{data: []byte("NRSWC43VOJSS4==="), errs: []error{io.EOF}}, + res: "leasure.", err: io.EOF, dbuflen: 11}, + {r: badReader{data: []byte("NRSWC43VOJSS4==="), errs: []error{badErr}}, + res: "leasure.", err: badErr, dbuflen: 11}, + // Check that errors are correctly propagated when the reader returns valid bytes in + // groups that are not divisible by 8. The first read will return 11 bytes and no + // error. The second will return 7 and an error. The data should be decoded correctly + // and the error should be propagated. + {r: badReader{data: []byte("NRSWC43VOJSS4==="), errs: []error{nil, badErr}, limit: 11}, + res: "leasure.", err: badErr}, + } + + for _, tc := range testCases { + input := tc.r.data + decoder := NewDecoder(StdEncoding, &tc.r) + var dbuflen int + if tc.dbuflen > 0 { + dbuflen = tc.dbuflen + } else { + dbuflen = StdEncoding.DecodedLen(len(input)) + } + dbuf := make([]byte, dbuflen) + var err error + var res []byte + for err == nil { + var n int + n, err = decoder.Read(dbuf) + if n > 0 { + res = append(res, dbuf[:n]...) + } + } + + testEqual(t, "Decoding of %q = %q, want %q", string(input), string(res), tc.res) + testEqual(t, "Decoding of %q err = %v, expected %v", string(input), err, tc.err) + } +} + +// TestDecoderError verifies decode errors are propagated when there are no read +// errors. +func TestDecoderError(t *testing.T) { + for _, readErr := range []error{io.EOF, nil} { + input := "MZXW6YTb" + dbuf := make([]byte, StdEncoding.DecodedLen(len(input))) + br := badReader{data: []byte(input), errs: []error{readErr}} + decoder := NewDecoder(StdEncoding, &br) + n, err := decoder.Read(dbuf) + testEqual(t, "Read after EOF, n = %d, expected %d", n, 0) + if _, ok := err.(CorruptInputError); !ok { + t.Errorf("Corrupt input error expected. Found %T", err) + } + } +} + +// TestReaderEOF ensures decoder.Read behaves correctly when input data is +// exhausted. +func TestReaderEOF(t *testing.T) { + for _, readErr := range []error{io.EOF, nil} { + input := "MZXW6YTB" + br := badReader{data: []byte(input), errs: []error{nil, readErr}} + decoder := NewDecoder(StdEncoding, &br) + dbuf := make([]byte, StdEncoding.DecodedLen(len(input))) + n, err := decoder.Read(dbuf) + testEqual(t, "Decoding of %q err = %v, expected %v", string(input), err, error(nil)) + n, err = decoder.Read(dbuf) + testEqual(t, "Read after EOF, n = %d, expected %d", n, 0) + testEqual(t, "Read after EOF, err = %v, expected %v", err, io.EOF) + n, err = decoder.Read(dbuf) + testEqual(t, "Read after EOF, n = %d, expected %d", n, 0) + testEqual(t, "Read after EOF, err = %v, expected %v", err, io.EOF) + } +} + func TestDecoderBuffering(t *testing.T) { for bs := 1; bs <= 12; bs++ { decoder := NewDecoder(StdEncoding, strings.NewReader(bigtest.encoded)) buf := make([]byte, len(bigtest.decoded)+12) var total int - for total = 0; total < len(bigtest.decoded); { - n, err := decoder.Read(buf[total : total+bs]) - testEqual(t, "Read from %q at pos %d = %d, %v, want _, %v", bigtest.encoded, total, n, err, error(nil)) + var n int + var err error + for total = 0; total < len(bigtest.decoded) && err == nil; { + n, err = decoder.Read(buf[total : total+bs]) total += n } + if err != nil && err != io.EOF { + t.Errorf("Read from %q at pos %d = %d, unexpected error %v", bigtest.encoded, total, n, err) + } testEqual(t, "Decoding/%d of %q = %q, want %q", bs, bigtest.encoded, string(buf[0:total]), bigtest.decoded) } } @@ -300,3 +460,121 @@ func BenchmarkDecodeString(b *testing.B) { StdEncoding.DecodeString(data) } } + +func TestWithCustomPadding(t *testing.T) { + for _, testcase := range pairs { + defaultPadding := StdEncoding.EncodeToString([]byte(testcase.decoded)) + customPadding := StdEncoding.WithPadding('@').EncodeToString([]byte(testcase.decoded)) + expected := strings.Replace(defaultPadding, "=", "@", -1) + + if expected != customPadding { + t.Errorf("Expected custom %s, got %s", expected, customPadding) + } + if testcase.encoded != defaultPadding { + t.Errorf("Expected %s, got %s", testcase.encoded, defaultPadding) + } + } +} + +func TestWithoutPadding(t *testing.T) { + for _, testcase := range pairs { + defaultPadding := StdEncoding.EncodeToString([]byte(testcase.decoded)) + customPadding := StdEncoding.WithPadding(NoPadding).EncodeToString([]byte(testcase.decoded)) + expected := strings.TrimRight(defaultPadding, "=") + + if expected != customPadding { + t.Errorf("Expected custom %s, got %s", expected, customPadding) + } + if testcase.encoded != defaultPadding { + t.Errorf("Expected %s, got %s", testcase.encoded, defaultPadding) + } + } +} + +func TestDecodeWithPadding(t *testing.T) { + encodings := []*Encoding{ + StdEncoding, + StdEncoding.WithPadding('-'), + StdEncoding.WithPadding(NoPadding), + } + + for i, enc := range encodings { + for _, pair := range pairs { + + input := pair.decoded + encoded := enc.EncodeToString([]byte(input)) + + decoded, err := enc.DecodeString(encoded) + if err != nil { + t.Errorf("DecodeString Error for encoding %d (%q): %v", i, input, err) + } + + if input != string(decoded) { + t.Errorf("Unexpected result for encoding %d: got %q; want %q", i, decoded, input) + } + } + } +} + +func TestDecodeWithWrongPadding(t *testing.T) { + encoded := StdEncoding.EncodeToString([]byte("foobar")) + + _, err := StdEncoding.WithPadding('-').DecodeString(encoded) + if err == nil { + t.Error("expected error") + } + + _, err = StdEncoding.WithPadding(NoPadding).DecodeString(encoded) + if err == nil { + t.Error("expected error") + } +} + +func TestEncodedDecodedLen(t *testing.T) { + type test struct { + in int + wantEnc int + wantDec int + } + data := bytes.Repeat([]byte("x"), 100) + for _, test := range []struct { + name string + enc *Encoding + cases []test + }{ + {"StdEncoding", StdEncoding, []test{ + {0, 0, 0}, + {1, 8, 5}, + {5, 8, 5}, + {6, 16, 10}, + {10, 16, 10}, + }}, + {"NoPadding", StdEncoding.WithPadding(NoPadding), []test{ + {0, 0, 0}, + {1, 2, 1}, + {2, 4, 2}, + {5, 8, 5}, + {6, 10, 6}, + {7, 12, 7}, + {10, 16, 10}, + {11, 18, 11}, + }}, + } { + t.Run(test.name, func(t *testing.T) { + for _, tc := range test.cases { + encLen := test.enc.EncodedLen(tc.in) + decLen := test.enc.DecodedLen(encLen) + enc := test.enc.EncodeToString(data[:tc.in]) + if len(enc) != encLen { + t.Fatalf("EncodedLen(%d) = %d but encoded to %q (%d)", tc.in, encLen, enc, len(enc)) + } + if encLen != tc.wantEnc { + t.Fatalf("EncodedLen(%d) = %d; want %d", tc.in, encLen, tc.wantEnc) + } + if decLen != tc.wantDec { + t.Fatalf("DecodedLen(%d) = %d; want %d", encLen, decLen, tc.wantDec) + } + } + }) + } +} diff --git a/libgo/go/encoding/base64/base64.go b/libgo/go/encoding/base64/base64.go index d2efad4518b..b208f9e4d8d 100644 --- a/libgo/go/encoding/base64/base64.go +++ b/libgo/go/encoding/base64/base64.go @@ -35,13 +35,19 @@ const encodeStd = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz012345678 const encodeURL = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-_" // NewEncoding returns a new padded Encoding defined by the given alphabet, -// which must be a 64-byte string. +// which must be a 64-byte string that does not contain the padding character +// or CR / LF ('\r', '\n'). // The resulting Encoding uses the default padding character ('='), // which may be changed or disabled via WithPadding. func NewEncoding(encoder string) *Encoding { if len(encoder) != 64 { panic("encoding alphabet is not 64-bytes long") } + for i := 0; i < len(encoder); i++ { + if encoder[i] == '\n' || encoder[i] == '\r' { + panic("encoding alphabet contains newline character") + } + } e := new(Encoding) e.padChar = StdPadding @@ -58,7 +64,20 @@ func NewEncoding(encoder string) *Encoding { // WithPadding creates a new encoding identical to enc except // with a specified padding character, or NoPadding to disable padding. +// The padding character must not be '\r' or '\n', must not +// be contained in the encoding's alphabet and must be a rune equal or +// below '\xff'. func (enc Encoding) WithPadding(padding rune) *Encoding { + if padding == '\r' || padding == '\n' || padding > 0xff { + panic("invalid padding") + } + + for i := 0; i < len(enc.encode); i++ { + if rune(enc.encode[i]) == padding { + panic("padding contained in alphabet") + } + } + enc.padChar = padding return &enc } @@ -256,19 +275,17 @@ func (e CorruptInputError) Error() string { func (enc *Encoding) decode(dst, src []byte) (n int, end bool, err error) { si := 0 - // skip over newlines - for si < len(src) && (src[si] == '\n' || src[si] == '\r') { - si++ - } - for si < len(src) && !end { // Decode quantum using the base64 alphabet var dbuf [4]byte dinc, dlen := 3, 4 - for j := range dbuf { + for j := 0; j < len(dbuf); j++ { if len(src) == si { - if enc.padChar != NoPadding || j < 2 { + switch { + case j == 0: + return n, false, nil + case j == 1, enc.padChar != NoPadding: return n, false, CorruptInputError(si - j) } dinc, dlen, end = j-1, j, true @@ -277,11 +294,17 @@ func (enc *Encoding) decode(dst, src []byte) (n int, end bool, err error) { in := src[si] si++ - // skip over newlines - for si < len(src) && (src[si] == '\n' || src[si] == '\r') { - si++ + + out := enc.decodeMap[in] + if out != 0xFF { + dbuf[j] = out + continue } + if in == '\n' || in == '\r' { + j-- + continue + } if rune(in) == enc.padChar { // We've reached the end and there's padding switch j { @@ -290,6 +313,10 @@ func (enc *Encoding) decode(dst, src []byte) (n int, end bool, err error) { return n, false, CorruptInputError(si - 1) case 2: // "==" is expected, the first "=" is already consumed. + // skip over newlines + for si < len(src) && (src[si] == '\n' || src[si] == '\r') { + si++ + } if si == len(src) { // not enough padding return n, false, CorruptInputError(len(src)) @@ -300,10 +327,10 @@ func (enc *Encoding) decode(dst, src []byte) (n int, end bool, err error) { } si++ - // skip over newlines - for si < len(src) && (src[si] == '\n' || src[si] == '\r') { - si++ - } + } + // skip over newlines + for si < len(src) && (src[si] == '\n' || src[si] == '\r') { + si++ } if si < len(src) { // trailing garbage @@ -312,10 +339,7 @@ func (enc *Encoding) decode(dst, src []byte) (n int, end bool, err error) { dinc, dlen, end = 3, j, true break } - dbuf[j] = enc.decodeMap[in] - if dbuf[j] == 0xFF { - return n, false, CorruptInputError(si - 1) - } + return n, false, CorruptInputError(si - 1) } // Convert 4x 6bit source bytes into 3 bytes diff --git a/libgo/go/encoding/base64/base64_test.go b/libgo/go/encoding/base64/base64_test.go index e2e1d59f3c0..05011fbdf38 100644 --- a/libgo/go/encoding/base64/base64_test.go +++ b/libgo/go/encoding/base64/base64_test.go @@ -7,6 +7,7 @@ package base64 import ( "bytes" "errors" + "fmt" "io" "io/ioutil" "reflect" @@ -63,7 +64,7 @@ func rawRef(ref string) string { } // Both URL and unpadding conversions -func rawUrlRef(ref string) string { +func rawURLRef(ref string) string { return rawRef(urlRef(ref)) } @@ -83,12 +84,12 @@ var encodingTests = []encodingTest{ {StdEncoding, stdRef}, {URLEncoding, urlRef}, {RawStdEncoding, rawRef}, - {RawURLEncoding, rawUrlRef}, + {RawURLEncoding, rawURLRef}, {funnyEncoding, funnyRef}, {StdEncoding.Strict(), stdRef}, {URLEncoding.Strict(), urlRef}, {RawStdEncoding.Strict(), rawRef}, - {RawURLEncoding.Strict(), rawUrlRef}, + {RawURLEncoding.Strict(), rawURLRef}, {funnyEncoding.Strict(), funnyRef}, } @@ -98,6 +99,7 @@ var bigtest = testpair{ } func testEqual(t *testing.T, msg string, args ...interface{}) bool { + t.Helper() if args[len(args)-2] != args[len(args)-1] { t.Errorf(msg, args...) return false @@ -187,11 +189,15 @@ func TestDecoderBuffering(t *testing.T) { decoder := NewDecoder(StdEncoding, strings.NewReader(bigtest.encoded)) buf := make([]byte, len(bigtest.decoded)+12) var total int - for total = 0; total < len(bigtest.decoded); { - n, err := decoder.Read(buf[total : total+bs]) - testEqual(t, "Read from %q at pos %d = %d, %v, want _, %v", bigtest.encoded, total, n, err, error(nil)) + var n int + var err error + for total = 0; total < len(bigtest.decoded) && err == nil; { + n, err = decoder.Read(buf[total : total+bs]) total += n } + if err != nil && err != io.EOF { + t.Errorf("Read from %q at pos %d = %d, unexpected error %v", bigtest.encoded, total, n, err) + } testEqual(t, "Decoding/%d of %q = %q, want %q", bs, bigtest.encoded, string(buf[0:total]), bigtest.decoded) } } @@ -202,6 +208,9 @@ func TestDecodeCorrupt(t *testing.T) { offset int // -1 means no corruption. }{ {"", -1}, + {"\n", -1}, + {"AAA=\n", -1}, + {"AAAA\n", -1}, {"!!!!", 0}, {"====", 0}, {"x===", 1}, @@ -220,6 +229,8 @@ func TestDecodeCorrupt(t *testing.T) { {"AAAA", -1}, {"AAAAAA=", 7}, {"YWJjZA=====", 8}, + {"A!\n", 1}, + {"A=\n", 1}, } for _, tc := range testCases { dbuf := make([]byte, StdEncoding.DecodedLen(len(tc.input))) @@ -466,10 +477,19 @@ func BenchmarkEncodeToString(b *testing.B) { } func BenchmarkDecodeString(b *testing.B) { - data := StdEncoding.EncodeToString(make([]byte, 8192)) - b.SetBytes(int64(len(data))) - for i := 0; i < b.N; i++ { - StdEncoding.DecodeString(data) + sizes := []int{2, 4, 8, 64, 8192} + benchFunc := func(b *testing.B, benchSize int) { + data := StdEncoding.EncodeToString(make([]byte, benchSize)) + b.SetBytes(int64(len(data))) + b.ResetTimer() + for i := 0; i < b.N; i++ { + StdEncoding.DecodeString(data) + } + } + for _, size := range sizes { + b.Run(fmt.Sprintf("%d", size), func(b *testing.B) { + benchFunc(b, size) + }) } } diff --git a/libgo/go/encoding/binary/binary.go b/libgo/go/encoding/binary/binary.go index 38342545966..2d01a3cb287 100644 --- a/libgo/go/encoding/binary/binary.go +++ b/libgo/go/encoding/binary/binary.go @@ -152,7 +152,8 @@ func (bigEndian) GoString() string { return "binary.BigEndian" } // When reading into structs, the field data for fields with // blank (_) field names is skipped; i.e., blank field names // may be used for padding. -// When reading into a struct, all non-blank fields must be exported. +// When reading into a struct, all non-blank fields must be exported +// or Read may panic. // // The error is EOF only if no bytes were read. // If an EOF happens after reading some but not all the bytes, diff --git a/libgo/go/encoding/binary/binary_test.go b/libgo/go/encoding/binary/binary_test.go index fc7f2765efb..0547bee4370 100644 --- a/libgo/go/encoding/binary/binary_test.go +++ b/libgo/go/encoding/binary/binary_test.go @@ -500,3 +500,27 @@ func BenchmarkWriteSlice1000Int32s(b *testing.B) { } b.StopTimer() } + +func BenchmarkPutUint16(b *testing.B) { + buf := [2]byte{} + b.SetBytes(2) + for i := 0; i < b.N; i++ { + BigEndian.PutUint16(buf[:], uint16(i)) + } +} + +func BenchmarkPutUint32(b *testing.B) { + buf := [4]byte{} + b.SetBytes(4) + for i := 0; i < b.N; i++ { + BigEndian.PutUint32(buf[:], uint32(i)) + } +} + +func BenchmarkPutUint64(b *testing.B) { + buf := [8]byte{} + b.SetBytes(8) + for i := 0; i < b.N; i++ { + BigEndian.PutUint64(buf[:], uint64(i)) + } +} diff --git a/libgo/go/encoding/binary/varint.go b/libgo/go/encoding/binary/varint.go index d7a75f99b16..bcb8ac9a459 100644 --- a/libgo/go/encoding/binary/varint.go +++ b/libgo/go/encoding/binary/varint.go @@ -53,9 +53,9 @@ func PutUvarint(buf []byte, x uint64) int { // number of bytes read (> 0). If an error occurred, the value is 0 // and the number of bytes n is <= 0 meaning: // -// n == 0: buf too small -// n < 0: value larger than 64 bits (overflow) -// and -n is the number of bytes read +// n == 0: buf too small +// n < 0: value larger than 64 bits (overflow) +// and -n is the number of bytes read // func Uvarint(buf []byte) (uint64, int) { var x uint64 @@ -87,9 +87,9 @@ func PutVarint(buf []byte, x int64) int { // number of bytes read (> 0). If an error occurred, the value is 0 // and the number of bytes n is <= 0 with the following meaning: // -// n == 0: buf too small -// n < 0: value larger than 64 bits (overflow) -// and -n is the number of bytes read +// n == 0: buf too small +// n < 0: value larger than 64 bits (overflow) +// and -n is the number of bytes read // func Varint(buf []byte) (int64, int) { ux, n := Uvarint(buf) // ok to continue in presence of error diff --git a/libgo/go/encoding/csv/reader.go b/libgo/go/encoding/csv/reader.go index c8c4ca77583..a3497c84f96 100644 --- a/libgo/go/encoding/csv/reader.go +++ b/libgo/go/encoding/csv/reader.go @@ -110,6 +110,10 @@ type Reader struct { // If TrimLeadingSpace is true, leading white space in a field is ignored. // This is done even if the field delimiter, Comma, is white space. TrimLeadingSpace bool + // ReuseRecord controls whether calls to Read may return a slice sharing + // the backing array of the previous call's returned slice for performance. + // By default, each call to Read returns newly allocated memory owned by the caller. + ReuseRecord bool line int column int @@ -122,6 +126,9 @@ type Reader struct { // Indexes of fields inside lineBuffer // The i'th field starts at offset fieldIndexes[i] in lineBuffer. fieldIndexes []int + + // only used when ReuseRecord == true + lastRecord []string } // NewReader returns a new Reader that reads from r. @@ -147,26 +154,17 @@ func (r *Reader) error(err error) error { // Except for that case, Read always returns either a non-nil // record or a non-nil error, but not both. // If there is no data left to be read, Read returns nil, io.EOF. +// If ReuseRecord is true, the returned slice may be shared +// between multiple calls to Read. func (r *Reader) Read() (record []string, err error) { - for { - record, err = r.parseRecord() - if record != nil { - break - } - if err != nil { - return nil, err - } + if r.ReuseRecord { + record, err = r.readRecord(r.lastRecord) + r.lastRecord = record + } else { + record, err = r.readRecord(nil) } - if r.FieldsPerRecord > 0 { - if len(record) != r.FieldsPerRecord { - r.column = 0 // report at start of record - return record, r.error(ErrFieldCount) - } - } else if r.FieldsPerRecord == 0 { - r.FieldsPerRecord = len(record) - } - return record, nil + return record, err } // ReadAll reads all the remaining records from r. @@ -176,7 +174,7 @@ func (r *Reader) Read() (record []string, err error) { // reported. func (r *Reader) ReadAll() (records [][]string, err error) { for { - record, err := r.Read() + record, err := r.readRecord(nil) if err == io.EOF { return records, nil } @@ -187,6 +185,31 @@ func (r *Reader) ReadAll() (records [][]string, err error) { } } +// readRecord reads and parses a single csv record from r. +// Unlike parseRecord, readRecord handles FieldsPerRecord. +// If dst has enough capacity it will be used for the returned record. +func (r *Reader) readRecord(dst []string) (record []string, err error) { + for { + record, err = r.parseRecord(dst) + if record != nil { + break + } + if err != nil { + return nil, err + } + } + + if r.FieldsPerRecord > 0 { + if len(record) != r.FieldsPerRecord { + r.column = 0 // report at start of record + return record, r.error(ErrFieldCount) + } + } else if r.FieldsPerRecord == 0 { + r.FieldsPerRecord = len(record) + } + return record, nil +} + // readRune reads one rune from r, folding \r\n to \n and keeping track // of how far into the line we have read. r.column will point to the start // of this rune, not the end of this rune. @@ -223,7 +246,8 @@ func (r *Reader) skip(delim rune) error { } // parseRecord reads and parses a single csv record from r. -func (r *Reader) parseRecord() (fields []string, err error) { +// If dst has enough capacity it will be used for the returned fields. +func (r *Reader) parseRecord(dst []string) (fields []string, err error) { // Each record starts on a new line. We increment our line // number (lines start at 1, not 0) and set column to -1 // so as we increment in readRune it points to the character we read. @@ -275,7 +299,12 @@ func (r *Reader) parseRecord() (fields []string, err error) { // minimal and a tradeoff for better performance through the combined // allocations. line := r.lineBuffer.String() - fields = make([]string, fieldCount) + + if cap(dst) >= fieldCount { + fields = dst[:fieldCount] + } else { + fields = make([]string, fieldCount) + } for i, idx := range r.fieldIndexes { if i == fieldCount-1 { diff --git a/libgo/go/encoding/csv/reader_test.go b/libgo/go/encoding/csv/reader_test.go index 7b3aca4c5f3..5ab1b61256c 100644 --- a/libgo/go/encoding/csv/reader_test.go +++ b/libgo/go/encoding/csv/reader_test.go @@ -24,6 +24,7 @@ var readTests = []struct { LazyQuotes bool TrailingComma bool TrimLeadingSpace bool + ReuseRecord bool Error string Line int // Expected error line if != 0 @@ -260,6 +261,15 @@ x,,, {"c", "d", "e"}, }, }, + { + Name: "ReadAllReuseRecord", + ReuseRecord: true, + Input: "a,b\nc,d", + Output: [][]string{ + {"a", "b"}, + {"c", "d"}, + }, + }, } func TestRead(t *testing.T) { @@ -274,6 +284,7 @@ func TestRead(t *testing.T) { r.LazyQuotes = tt.LazyQuotes r.TrailingComma = tt.TrailingComma r.TrimLeadingSpace = tt.TrimLeadingSpace + r.ReuseRecord = tt.ReuseRecord if tt.Comma != 0 { r.Comma = tt.Comma } @@ -369,3 +380,23 @@ xxxxxxxxxxxxxxxxxxxxxxxx,yyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyy,zzzzzzzzzzzzzzzzzzzzzz xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx,yyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyy,zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz,wwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwww,vvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvv `, 3)) } + +func BenchmarkReadReuseRecord(b *testing.B) { + benchmarkRead(b, func(r *Reader) { r.ReuseRecord = true }, benchmarkCSVData) +} + +func BenchmarkReadReuseRecordWithFieldsPerRecord(b *testing.B) { + benchmarkRead(b, func(r *Reader) { r.ReuseRecord = true; r.FieldsPerRecord = 4 }, benchmarkCSVData) +} + +func BenchmarkReadReuseRecordWithoutFieldsPerRecord(b *testing.B) { + benchmarkRead(b, func(r *Reader) { r.ReuseRecord = true; r.FieldsPerRecord = -1 }, benchmarkCSVData) +} + +func BenchmarkReadReuseRecordLargeFields(b *testing.B) { + benchmarkRead(b, func(r *Reader) { r.ReuseRecord = true }, strings.Repeat(`xxxxxxxxxxxxxxxx,yyyyyyyyyyyyyyyy,zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz,wwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwww,vvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvv +xxxxxxxxxxxxxxxxxxxxxxxx,yyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyy,zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz,wwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwww,vvvv +,,zzzz,wwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwww,vvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvv +xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx,yyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyy,zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz,wwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwww,vvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvv +`, 3)) +} diff --git a/libgo/go/encoding/gob/codec_test.go b/libgo/go/encoding/gob/codec_test.go index d4002cbccab..eb9f306bcf5 100644 --- a/libgo/go/encoding/gob/codec_test.go +++ b/libgo/go/encoding/gob/codec_test.go @@ -47,7 +47,6 @@ func testError(t *testing.T) { if e := recover(); e != nil { t.Error(e.(gobError).err) // Will re-panic if not one of our errors, such as a runtime error. } - return } func newDecBuffer(data []byte) *decBuffer { @@ -321,7 +320,7 @@ func TestScalarEncInstructions(t *testing.T) { } } -func execDec(typ string, instr *decInstr, state *decoderState, t *testing.T, value reflect.Value) { +func execDec(instr *decInstr, state *decoderState, t *testing.T, value reflect.Value) { defer testError(t) v := int(state.decodeUint()) if v+state.fieldnum != 6 { @@ -348,7 +347,7 @@ func TestScalarDecInstructions(t *testing.T) { var data bool instr := &decInstr{decBool, 6, nil, ovfl} state := newDecodeStateFromData(boolResult) - execDec("bool", instr, state, t, reflect.ValueOf(&data)) + execDec(instr, state, t, reflect.ValueOf(&data)) if data != true { t.Errorf("bool a = %v not true", data) } @@ -358,7 +357,7 @@ func TestScalarDecInstructions(t *testing.T) { var data int instr := &decInstr{decOpTable[reflect.Int], 6, nil, ovfl} state := newDecodeStateFromData(signedResult) - execDec("int", instr, state, t, reflect.ValueOf(&data)) + execDec(instr, state, t, reflect.ValueOf(&data)) if data != 17 { t.Errorf("int a = %v not 17", data) } @@ -369,7 +368,7 @@ func TestScalarDecInstructions(t *testing.T) { var data uint instr := &decInstr{decOpTable[reflect.Uint], 6, nil, ovfl} state := newDecodeStateFromData(unsignedResult) - execDec("uint", instr, state, t, reflect.ValueOf(&data)) + execDec(instr, state, t, reflect.ValueOf(&data)) if data != 17 { t.Errorf("uint a = %v not 17", data) } @@ -380,7 +379,7 @@ func TestScalarDecInstructions(t *testing.T) { var data int8 instr := &decInstr{decInt8, 6, nil, ovfl} state := newDecodeStateFromData(signedResult) - execDec("int8", instr, state, t, reflect.ValueOf(&data)) + execDec(instr, state, t, reflect.ValueOf(&data)) if data != 17 { t.Errorf("int8 a = %v not 17", data) } @@ -391,7 +390,7 @@ func TestScalarDecInstructions(t *testing.T) { var data uint8 instr := &decInstr{decUint8, 6, nil, ovfl} state := newDecodeStateFromData(unsignedResult) - execDec("uint8", instr, state, t, reflect.ValueOf(&data)) + execDec(instr, state, t, reflect.ValueOf(&data)) if data != 17 { t.Errorf("uint8 a = %v not 17", data) } @@ -402,7 +401,7 @@ func TestScalarDecInstructions(t *testing.T) { var data int16 instr := &decInstr{decInt16, 6, nil, ovfl} state := newDecodeStateFromData(signedResult) - execDec("int16", instr, state, t, reflect.ValueOf(&data)) + execDec(instr, state, t, reflect.ValueOf(&data)) if data != 17 { t.Errorf("int16 a = %v not 17", data) } @@ -413,7 +412,7 @@ func TestScalarDecInstructions(t *testing.T) { var data uint16 instr := &decInstr{decUint16, 6, nil, ovfl} state := newDecodeStateFromData(unsignedResult) - execDec("uint16", instr, state, t, reflect.ValueOf(&data)) + execDec(instr, state, t, reflect.ValueOf(&data)) if data != 17 { t.Errorf("uint16 a = %v not 17", data) } @@ -424,7 +423,7 @@ func TestScalarDecInstructions(t *testing.T) { var data int32 instr := &decInstr{decInt32, 6, nil, ovfl} state := newDecodeStateFromData(signedResult) - execDec("int32", instr, state, t, reflect.ValueOf(&data)) + execDec(instr, state, t, reflect.ValueOf(&data)) if data != 17 { t.Errorf("int32 a = %v not 17", data) } @@ -435,7 +434,7 @@ func TestScalarDecInstructions(t *testing.T) { var data uint32 instr := &decInstr{decUint32, 6, nil, ovfl} state := newDecodeStateFromData(unsignedResult) - execDec("uint32", instr, state, t, reflect.ValueOf(&data)) + execDec(instr, state, t, reflect.ValueOf(&data)) if data != 17 { t.Errorf("uint32 a = %v not 17", data) } @@ -446,7 +445,7 @@ func TestScalarDecInstructions(t *testing.T) { var data uintptr instr := &decInstr{decOpTable[reflect.Uintptr], 6, nil, ovfl} state := newDecodeStateFromData(unsignedResult) - execDec("uintptr", instr, state, t, reflect.ValueOf(&data)) + execDec(instr, state, t, reflect.ValueOf(&data)) if data != 17 { t.Errorf("uintptr a = %v not 17", data) } @@ -457,7 +456,7 @@ func TestScalarDecInstructions(t *testing.T) { var data int64 instr := &decInstr{decInt64, 6, nil, ovfl} state := newDecodeStateFromData(signedResult) - execDec("int64", instr, state, t, reflect.ValueOf(&data)) + execDec(instr, state, t, reflect.ValueOf(&data)) if data != 17 { t.Errorf("int64 a = %v not 17", data) } @@ -468,7 +467,7 @@ func TestScalarDecInstructions(t *testing.T) { var data uint64 instr := &decInstr{decUint64, 6, nil, ovfl} state := newDecodeStateFromData(unsignedResult) - execDec("uint64", instr, state, t, reflect.ValueOf(&data)) + execDec(instr, state, t, reflect.ValueOf(&data)) if data != 17 { t.Errorf("uint64 a = %v not 17", data) } @@ -479,7 +478,7 @@ func TestScalarDecInstructions(t *testing.T) { var data float32 instr := &decInstr{decFloat32, 6, nil, ovfl} state := newDecodeStateFromData(floatResult) - execDec("float32", instr, state, t, reflect.ValueOf(&data)) + execDec(instr, state, t, reflect.ValueOf(&data)) if data != 17 { t.Errorf("float32 a = %v not 17", data) } @@ -490,7 +489,7 @@ func TestScalarDecInstructions(t *testing.T) { var data float64 instr := &decInstr{decFloat64, 6, nil, ovfl} state := newDecodeStateFromData(floatResult) - execDec("float64", instr, state, t, reflect.ValueOf(&data)) + execDec(instr, state, t, reflect.ValueOf(&data)) if data != 17 { t.Errorf("float64 a = %v not 17", data) } @@ -501,7 +500,7 @@ func TestScalarDecInstructions(t *testing.T) { var data complex64 instr := &decInstr{decOpTable[reflect.Complex64], 6, nil, ovfl} state := newDecodeStateFromData(complexResult) - execDec("complex", instr, state, t, reflect.ValueOf(&data)) + execDec(instr, state, t, reflect.ValueOf(&data)) if data != 17+19i { t.Errorf("complex a = %v not 17+19i", data) } @@ -512,7 +511,7 @@ func TestScalarDecInstructions(t *testing.T) { var data complex128 instr := &decInstr{decOpTable[reflect.Complex128], 6, nil, ovfl} state := newDecodeStateFromData(complexResult) - execDec("complex", instr, state, t, reflect.ValueOf(&data)) + execDec(instr, state, t, reflect.ValueOf(&data)) if data != 17+19i { t.Errorf("complex a = %v not 17+19i", data) } @@ -523,7 +522,7 @@ func TestScalarDecInstructions(t *testing.T) { var data []byte instr := &decInstr{decUint8Slice, 6, nil, ovfl} state := newDecodeStateFromData(bytesResult) - execDec("bytes", instr, state, t, reflect.ValueOf(&data)) + execDec(instr, state, t, reflect.ValueOf(&data)) if string(data) != "hello" { t.Errorf(`bytes a = %q not "hello"`, string(data)) } @@ -534,7 +533,7 @@ func TestScalarDecInstructions(t *testing.T) { var data string instr := &decInstr{decString, 6, nil, ovfl} state := newDecodeStateFromData(bytesResult) - execDec("bytes", instr, state, t, reflect.ValueOf(&data)) + execDec(instr, state, t, reflect.ValueOf(&data)) if data != "hello" { t.Errorf(`bytes a = %q not "hello"`, data) } @@ -545,11 +544,18 @@ func TestEndToEnd(t *testing.T) { type T2 struct { T string } - s1 := "string1" - s2 := "string2" + type T3 struct { + X float64 + Z *int + } type T1 struct { A, B, C int M map[string]*float64 + M2 map[int]T3 + Mstring map[string]string + Mintptr map[int]*int + Mcomp map[complex128]complex128 + Marr map[[2]string][2]*float64 EmptyMap map[string]int // to check that we receive a non-nil map. N *[3]float64 Strs *[2]string @@ -561,11 +567,35 @@ func TestEndToEnd(t *testing.T) { } pi := 3.14159 e := 2.71828 + two := 2.0 + meaning := 42 + fingers := 5 + s1 := "string1" + s2 := "string2" + var comp1 complex128 = complex(1.0, 1.0) + var comp2 complex128 = complex(1.0, 1.0) + var arr1 [2]string + arr1[0] = s1 + arr1[1] = s2 + var arr2 [2]string + arr2[0] = s2 + arr2[1] = s1 + var floatArr1 [2]*float64 + floatArr1[0] = &pi + floatArr1[1] = &e + var floatArr2 [2]*float64 + floatArr2[0] = &e + floatArr2[1] = &two t1 := &T1{ A: 17, B: 18, C: -5, M: map[string]*float64{"pi": &pi, "e": &e}, + M2: map[int]T3{4: T3{X: pi, Z: &meaning}, 10: T3{X: e, Z: &fingers}}, + Mstring: map[string]string{"pi": "3.14", "e": "2.71"}, + Mintptr: map[int]*int{meaning: &fingers, fingers: &meaning}, + Mcomp: map[complex128]complex128{comp1: comp2, comp2: comp1}, + Marr: map[[2]string][2]*float64{arr1: floatArr1, arr2: floatArr2}, EmptyMap: make(map[string]int), N: &[3]float64{1.5, 2.5, 3.5}, Strs: &[2]string{s1, s2}, diff --git a/libgo/go/encoding/gob/decode.go b/libgo/go/encoding/gob/decode.go index 9645dc57906..8dece42e908 100644 --- a/libgo/go/encoding/gob/decode.go +++ b/libgo/go/encoding/gob/decode.go @@ -11,6 +11,7 @@ import ( "errors" "io" "math" + "math/bits" "reflect" ) @@ -313,12 +314,7 @@ func decUint64(i *decInstr, state *decoderState, value reflect.Value) { // (for example) transmit more compactly. This routine does the // unswizzling. func float64FromBits(u uint64) float64 { - var v uint64 - for i := 0; i < 8; i++ { - v <<= 8 - v |= u & 0xFF - u >>= 8 - } + v := bits.ReverseBytes64(u) return math.Float64frombits(v) } @@ -430,7 +426,7 @@ type decEngine struct { // decodeSingle decodes a top-level value that is not a struct and stores it in value. // Such values are preceded by a zero, making them have the memory layout of a // struct field (although with an illegal field number). -func (dec *Decoder) decodeSingle(engine *decEngine, ut *userTypeInfo, value reflect.Value) { +func (dec *Decoder) decodeSingle(engine *decEngine, value reflect.Value) { state := dec.newDecoderState(&dec.buf) defer dec.freeDecoderState(state) state.fieldnum = singletonField @@ -446,7 +442,7 @@ func (dec *Decoder) decodeSingle(engine *decEngine, ut *userTypeInfo, value refl // differ from ut.indir, which was computed when the engine was built. // This state cannot arise for decodeSingle, which is called directly // from the user's value, not from the innards of an engine. -func (dec *Decoder) decodeStruct(engine *decEngine, ut *userTypeInfo, value reflect.Value) { +func (dec *Decoder) decodeStruct(engine *decEngine, value reflect.Value) { state := dec.newDecoderState(&dec.buf) defer dec.freeDecoderState(state) state.fieldnum = -1 @@ -538,7 +534,7 @@ func (dec *Decoder) decodeArrayHelper(state *decoderState, value reflect.Value, // decodeArray decodes an array and stores it in value. // The length is an unsigned integer preceding the elements. Even though the length is redundant // (it's part of the type), it's a useful check and is included in the encoding. -func (dec *Decoder) decodeArray(atyp reflect.Type, state *decoderState, value reflect.Value, elemOp decOp, length int, ovfl error, helper decHelper) { +func (dec *Decoder) decodeArray(state *decoderState, value reflect.Value, elemOp decOp, length int, ovfl error, helper decHelper) { if n := state.decodeUint(); n != uint64(length) { errorf("length mismatch in decodeArray") } @@ -546,12 +542,12 @@ func (dec *Decoder) decodeArray(atyp reflect.Type, state *decoderState, value re } // decodeIntoValue is a helper for map decoding. -func decodeIntoValue(state *decoderState, op decOp, isPtr bool, value reflect.Value, ovfl error) reflect.Value { - instr := &decInstr{op, 0, nil, ovfl} +func decodeIntoValue(state *decoderState, op decOp, isPtr bool, value reflect.Value, instr *decInstr) reflect.Value { v := value if isPtr { v = decAlloc(value) } + op(instr, state, v) return value } @@ -561,17 +557,24 @@ func decodeIntoValue(state *decoderState, op decOp, isPtr bool, value reflect.Va // Because the internals of maps are not visible to us, we must // use reflection rather than pointer magic. func (dec *Decoder) decodeMap(mtyp reflect.Type, state *decoderState, value reflect.Value, keyOp, elemOp decOp, ovfl error) { + n := int(state.decodeUint()) if value.IsNil() { - // Allocate map. - value.Set(reflect.MakeMap(mtyp)) + value.Set(reflect.MakeMapWithSize(mtyp, n)) } - n := int(state.decodeUint()) keyIsPtr := mtyp.Key().Kind() == reflect.Ptr elemIsPtr := mtyp.Elem().Kind() == reflect.Ptr + keyInstr := &decInstr{keyOp, 0, nil, ovfl} + elemInstr := &decInstr{elemOp, 0, nil, ovfl} + keyP := reflect.New(mtyp.Key()) + keyZ := reflect.Zero(mtyp.Key()) + elemP := reflect.New(mtyp.Elem()) + elemZ := reflect.Zero(mtyp.Elem()) for i := 0; i < n; i++ { - key := decodeIntoValue(state, keyOp, keyIsPtr, allocValue(mtyp.Key()), ovfl) - elem := decodeIntoValue(state, elemOp, elemIsPtr, allocValue(mtyp.Elem()), ovfl) + key := decodeIntoValue(state, keyOp, keyIsPtr, keyP.Elem(), keyInstr) + elem := decodeIntoValue(state, elemOp, elemIsPtr, elemP.Elem(), elemInstr) value.SetMapIndex(key, elem) + keyP.Elem().Set(keyZ) + elemP.Elem().Set(elemZ) } } @@ -657,12 +660,12 @@ func (dec *Decoder) decodeInterface(ityp reflect.Type, state *decoderState, valu errorf("name too long (%d bytes): %.20q...", len(name), name) } // The concrete type must be registered. - registerLock.RLock() - typ, ok := nameToConcreteType[string(name)] - registerLock.RUnlock() + typi, ok := nameToConcreteType.Load(string(name)) if !ok { errorf("name not registered for interface: %q", name) } + typ := typi.(reflect.Type) + // Read the type id of the concrete value. concreteId := dec.decodeTypeSequence(true) if concreteId < 0 { @@ -813,7 +816,7 @@ func (dec *Decoder) decOpFor(wireId typeId, rt reflect.Type, name string, inProg ovfl := overflow(name) helper := decArrayHelper[t.Elem().Kind()] op = func(i *decInstr, state *decoderState, value reflect.Value) { - state.dec.decodeArray(t, state, value, *elemOp, t.Len(), ovfl, helper) + state.dec.decodeArray(state, value, *elemOp, t.Len(), ovfl, helper) } case reflect.Map: @@ -854,7 +857,7 @@ func (dec *Decoder) decOpFor(wireId typeId, rt reflect.Type, name string, inProg } op = func(i *decInstr, state *decoderState, value reflect.Value) { // indirect through enginePtr to delay evaluation for recursive structs. - dec.decodeStruct(*enginePtr, ut, value) + dec.decodeStruct(*enginePtr, value) } case reflect.Interface: op = func(i *decInstr, state *decoderState, value reflect.Value) { @@ -1197,9 +1200,9 @@ func (dec *Decoder) decodeValue(wireId typeId, value reflect.Value) { name := base.Name() errorf("type mismatch: no fields matched compiling decoder for %s", name) } - dec.decodeStruct(engine, ut, value) + dec.decodeStruct(engine, value) } else { - dec.decodeSingle(engine, ut, value) + dec.decodeSingle(engine, value) } } diff --git a/libgo/go/encoding/gob/decoder.go b/libgo/go/encoding/gob/decoder.go index c1829417733..8e0b1dd3750 100644 --- a/libgo/go/encoding/gob/decoder.go +++ b/libgo/go/encoding/gob/decoder.go @@ -19,6 +19,10 @@ const tooBig = 1 << 30 // A Decoder manages the receipt of type and data information read from the // remote side of a connection. +// +// The Decoder does only basic sanity checking on decoded input sizes, +// and its limits are not configurable. Take caution when decoding gob data +// from untrusted sources. type Decoder struct { mutex sync.Mutex // each item must be received atomically r io.Reader // source of the data diff --git a/libgo/go/encoding/gob/doc.go b/libgo/go/encoding/gob/doc.go index 1536574fed2..db734ecc1ec 100644 --- a/libgo/go/encoding/gob/doc.go +++ b/libgo/go/encoding/gob/doc.go @@ -4,7 +4,7 @@ /* Package gob manages streams of gobs - binary values exchanged between an -Encoder (transmitter) and a Decoder (receiver). A typical use is transporting +Encoder (transmitter) and a Decoder (receiver). A typical use is transporting arguments and results of remote procedure calls (RPCs) such as those provided by package "net/rpc". @@ -14,28 +14,28 @@ amortizing the cost of compilation. Basics -A stream of gobs is self-describing. Each data item in the stream is preceded by +A stream of gobs is self-describing. Each data item in the stream is preceded by a specification of its type, expressed in terms of a small set of predefined -types. Pointers are not transmitted, but the things they point to are +types. Pointers are not transmitted, but the things they point to are transmitted; that is, the values are flattened. Nil pointers are not permitted, as they have no value. Recursive types work fine, but -recursive values (data with cycles) are problematic. This may change. +recursive values (data with cycles) are problematic. This may change. To use gobs, create an Encoder and present it with a series of data items as -values or addresses that can be dereferenced to values. The Encoder makes sure -all type information is sent before it is needed. At the receive side, a +values or addresses that can be dereferenced to values. The Encoder makes sure +all type information is sent before it is needed. At the receive side, a Decoder retrieves values from the encoded stream and unpacks them into local variables. Types and Values -The source and destination values/types need not correspond exactly. For structs, +The source and destination values/types need not correspond exactly. For structs, fields (identified by name) that are in the source but absent from the receiving -variable will be ignored. Fields that are in the receiving variable but missing -from the transmitted type or value will be ignored in the destination. If a field +variable will be ignored. Fields that are in the receiving variable but missing +from the transmitted type or value will be ignored in the destination. If a field with the same name is present in both, their types must be compatible. Both the receiver and transmitter will do all necessary indirection and dereferencing to -convert between gobs and actual Go values. For instance, a gob type that is +convert between gobs and actual Go values. For instance, a gob type that is schematically, struct { A, B int } @@ -63,8 +63,8 @@ Attempting to receive into these types will draw a decode error: struct { C, D int } // no field names in common Integers are transmitted two ways: arbitrary precision signed integers or -arbitrary precision unsigned integers. There is no int8, int16 etc. -discrimination in the gob format; there are only signed and unsigned integers. As +arbitrary precision unsigned integers. There is no int8, int16 etc. +discrimination in the gob format; there are only signed and unsigned integers. As described below, the transmitter sends the value in a variable-length encoding; the receiver accepts the value and stores it in the destination variable. Floating-point numbers are always sent using IEEE-754 64-bit precision (see @@ -72,7 +72,7 @@ below). Signed integers may be received into any signed integer variable: int, int16, etc.; unsigned integers may be received into any unsigned integer variable; and floating -point values may be received into any floating point variable. However, +point values may be received into any floating point variable. However, the destination variable must be able to represent the value or the decode operation will fail. @@ -106,17 +106,17 @@ Encoding Details This section documents the encoding, details that are not important for most users. Details are presented bottom-up. -An unsigned integer is sent one of two ways. If it is less than 128, it is sent -as a byte with that value. Otherwise it is sent as a minimal-length big-endian +An unsigned integer is sent one of two ways. If it is less than 128, it is sent +as a byte with that value. Otherwise it is sent as a minimal-length big-endian (high byte first) byte stream holding the value, preceded by one byte holding the -byte count, negated. Thus 0 is transmitted as (00), 7 is transmitted as (07) and +byte count, negated. Thus 0 is transmitted as (00), 7 is transmitted as (07) and 256 is transmitted as (FE 01 00). A boolean is encoded within an unsigned integer: 0 for false, 1 for true. -A signed integer, i, is encoded within an unsigned integer, u. Within u, bits 1 +A signed integer, i, is encoded within an unsigned integer, u. Within u, bits 1 upward contain the value; bit 0 says whether they should be complemented upon -receipt. The encode algorithm looks like this: +receipt. The encode algorithm looks like this: var u uint if i < 0 { @@ -127,14 +127,14 @@ receipt. The encode algorithm looks like this: encodeUnsigned(u) The low bit is therefore analogous to a sign bit, but making it the complement bit -instead guarantees that the largest negative integer is not a special case. For +instead guarantees that the largest negative integer is not a special case. For example, -129=^128=(^256>>1) encodes as (FE 01 01). Floating-point numbers are always sent as a representation of a float64 value. -That value is converted to a uint64 using math.Float64bits. The uint64 is then -byte-reversed and sent as a regular unsigned integer. The byte-reversal means the -exponent and high-precision part of the mantissa go first. Since the low bits are -often zero, this can save encoding bytes. For instance, 17.0 is encoded in only +That value is converted to a uint64 using math.Float64bits. The uint64 is then +byte-reversed and sent as a regular unsigned integer. The byte-reversal means the +exponent and high-precision part of the mantissa go first. Since the low bits are +often zero, this can save encoding bytes. For instance, 17.0 is encoded in only three bytes (FE 31 40). Strings and slices of bytes are sent as an unsigned count followed by that many @@ -151,33 +151,39 @@ is nil and not at the top level. In slices and arrays, as well as maps, all elements, even zero-valued elements, are transmitted, even if all the elements are zero. -Structs are sent as a sequence of (field number, field value) pairs. The field -value is sent using the standard gob encoding for its type, recursively. If a +Structs are sent as a sequence of (field number, field value) pairs. The field +value is sent using the standard gob encoding for its type, recursively. If a field has the zero value for its type (except for arrays; see above), it is omitted -from the transmission. The field number is defined by the type of the encoded +from the transmission. The field number is defined by the type of the encoded struct: the first field of the encoded type is field 0, the second is field 1, -etc. When encoding a value, the field numbers are delta encoded for efficiency +etc. When encoding a value, the field numbers are delta encoded for efficiency and the fields are always sent in order of increasing field number; the deltas are -therefore unsigned. The initialization for the delta encoding sets the field +therefore unsigned. The initialization for the delta encoding sets the field number to -1, so an unsigned integer field 0 with value 7 is transmitted as unsigned -delta = 1, unsigned value = 7 or (01 07). Finally, after all the fields have been -sent a terminating mark denotes the end of the struct. That mark is a delta=0 +delta = 1, unsigned value = 7 or (01 07). Finally, after all the fields have been +sent a terminating mark denotes the end of the struct. That mark is a delta=0 value, which has representation (00). Interface types are not checked for compatibility; all interface types are treated, for transmission, as members of a single "interface" type, analogous to -int or []byte - in effect they're all treated as interface{}. Interface values +int or []byte - in effect they're all treated as interface{}. Interface values are transmitted as a string identifying the concrete type being sent (a name that must be pre-defined by calling Register), followed by a byte count of the length of the following data (so the value can be skipped if it cannot be stored), followed by the usual encoding of concrete (dynamic) value stored in -the interface value. (A nil interface value is identified by the empty string +the interface value. (A nil interface value is identified by the empty string and transmits no value.) Upon receipt, the decoder verifies that the unpacked concrete item satisfies the interface of the receiving variable. -The representation of types is described below. When a type is defined on a given +If a value is passed to Encode and the type is not a struct (or pointer to struct, +etc.), for simplicity of processing it is represented as a struct of one field. +The only visible effect of this is to encode a zero byte after the value, just as +after the last field of an encoded struct, so that the decode algorithm knows when +the top-level value is complete. + +The representation of types is described below. When a type is defined on a given connection between an Encoder and Decoder, it is assigned a signed integer type -id. When Encoder.Encode(v) is called, it makes sure there is an id assigned for +id. When Encoder.Encode(v) is called, it makes sure there is an id assigned for the type of v and all its elements and then it sends the pair (typeid, encoded-v) where typeid is the type id of the encoded type of v and encoded-v is the gob encoding of the value v. @@ -223,7 +229,7 @@ If there are nested type ids, the types for all inner type ids must be defined before the top-level type id is used to describe an encoded-v. For simplicity in setup, the connection is defined to understand these types a -priori, as well as the basic gob types int, uint, etc. Their ids are: +priori, as well as the basic gob types int, uint, etc. Their ids are: bool 1 int 2 @@ -244,7 +250,7 @@ priori, as well as the basic gob types int, uint, etc. Their ids are: MapType 23 Finally, each message created by a call to Encode is preceded by an encoded -unsigned integer count of the number of bytes remaining in the message. After +unsigned integer count of the number of bytes remaining in the message. After the initial type name, interface values are wrapped the same way; in effect, the interface value acts like a recursive invocation of Encode. @@ -256,7 +262,7 @@ where * signifies zero or more repetitions and the type id of a value must be predefined or be defined before the value in the stream. Compatibility: Any future changes to the package will endeavor to maintain -compatibility with streams encoded using previous versions. That is, any released +compatibility with streams encoded using previous versions. That is, any released version of this package should be able to decode data written with any previously released version, subject to issues such as security fixes. See the Go compatibility document for background: https://golang.org/doc/go1compat @@ -315,7 +321,7 @@ StructValue: */ /* -For implementers and the curious, here is an encoded example. Given +For implementers and the curious, here is an encoded example. Given type Point struct {X, Y int} and the value p := Point{22, 33} @@ -326,14 +332,14 @@ the bytes transmitted that encode p will be: They are determined as follows. Since this is the first transmission of type Point, the type descriptor -for Point itself must be sent before the value. This is the first type +for Point itself must be sent before the value. This is the first type we've sent on this Encoder, so it has type id 65 (0 through 64 are reserved). 1f // This item (a type descriptor) is 31 bytes long. ff 81 // The negative of the id for the type we're defining, -65. // This is one byte (indicated by FF = -1) followed by - // ^-65<<1 | 1. The low 1 bit signals to complement the + // ^-65<<1 | 1. The low 1 bit signals to complement the // rest upon receipt. // Now we send a type descriptor, which is itself a struct (wireType). @@ -370,7 +376,7 @@ reserved). 00 // end of wireType.structType structure 00 // end of wireType structure -Now we can send the Point value. Again the field number resets to -1: +Now we can send the Point value. Again the field number resets to -1: 07 // this value is 7 bytes long ff 82 // the type number, 65 (1 byte (-FF) followed by 65<<1) @@ -387,7 +393,7 @@ output will be just: 07 ff 82 01 2c 01 42 00 A single non-struct value at top level is transmitted like a field with -delta tag 0. For instance, a signed integer with value 3 presented as +delta tag 0. For instance, a signed integer with value 3 presented as the argument to Encode will emit: 03 04 00 06 diff --git a/libgo/go/encoding/gob/encode.go b/libgo/go/encoding/gob/encode.go index 50cd6adb463..5371e7245f4 100644 --- a/libgo/go/encoding/gob/encode.go +++ b/libgo/go/encoding/gob/encode.go @@ -8,7 +8,9 @@ package gob import ( "encoding" + "encoding/binary" "math" + "math/bits" "reflect" "sync" ) @@ -107,14 +109,12 @@ func (state *encoderState) encodeUint(x uint64) { state.b.WriteByte(uint8(x)) return } - i := uint64Size - for x > 0 { - state.buf[i] = uint8(x) - x >>= 8 - i-- - } - state.buf[i] = uint8(i - uint64Size) // = loop count, negated - state.b.Write(state.buf[i : uint64Size+1]) + + binary.BigEndian.PutUint64(state.buf[1:], x) + bc := bits.LeadingZeros64(x) >> 3 // 8 - bytelen(x) + state.buf[bc] = uint8(bc - uint64Size) // and then we subtract 8 to get -bytelen(x) + + state.b.Write(state.buf[bc : uint64Size+1]) } // encodeInt writes an encoded signed integer to state.w. @@ -209,13 +209,7 @@ func encUint(i *encInstr, state *encoderState, v reflect.Value) { // swizzling. func floatBits(f float64) uint64 { u := math.Float64bits(f) - var v uint64 - for i := 0; i < 8; i++ { - v <<= 8 - v |= u & 0xFF - u >>= 8 - } - return v + return bits.ReverseBytes64(u) } // encFloat encodes the floating point value (float32 float64) referenced by v. @@ -404,12 +398,12 @@ func (enc *Encoder) encodeInterface(b *encBuffer, iv reflect.Value) { } ut := userType(iv.Elem().Type()) - registerLock.RLock() - name, ok := concreteTypeToName[ut.base] - registerLock.RUnlock() + namei, ok := concreteTypeToName.Load(ut.base) if !ok { errorf("type not registered for interface: %s", ut.base) } + name := namei.(string) + // Send the name. state.encodeUint(uint64(len(name))) state.b.WriteString(name) diff --git a/libgo/go/encoding/gob/encoder_test.go b/libgo/go/encoding/gob/encoder_test.go index 9256848b50e..a1ca252ccd1 100644 --- a/libgo/go/encoding/gob/encoder_test.go +++ b/libgo/go/encoding/gob/encoder_test.go @@ -55,6 +55,71 @@ func TestBasicEncoderDecoder(t *testing.T) { } } +func TestEncodeIntSlice(t *testing.T) { + + s8 := []int8{1, 5, 12, 22, 35, 51, 70, 92, 117} + s16 := []int16{145, 176, 210, 247, 287, 330, 376, 425, 477} + s32 := []int32{532, 590, 651, 715, 782, 852, 925, 1001, 1080} + s64 := []int64{1162, 1247, 1335, 1426, 1520, 1617, 1717, 1820, 1926} + + t.Run("int8", func(t *testing.T) { + var sink bytes.Buffer + enc := NewEncoder(&sink) + enc.Encode(s8) + + dec := NewDecoder(&sink) + res := make([]int8, 9) + dec.Decode(&res) + + if !reflect.DeepEqual(s8, res) { + t.Fatalf("EncodeIntSlice: expected %v, got %v", s8, res) + } + }) + + t.Run("int16", func(t *testing.T) { + var sink bytes.Buffer + enc := NewEncoder(&sink) + enc.Encode(s16) + + dec := NewDecoder(&sink) + res := make([]int16, 9) + dec.Decode(&res) + + if !reflect.DeepEqual(s16, res) { + t.Fatalf("EncodeIntSlice: expected %v, got %v", s16, res) + } + }) + + t.Run("int32", func(t *testing.T) { + var sink bytes.Buffer + enc := NewEncoder(&sink) + enc.Encode(s32) + + dec := NewDecoder(&sink) + res := make([]int32, 9) + dec.Decode(&res) + + if !reflect.DeepEqual(s32, res) { + t.Fatalf("EncodeIntSlice: expected %v, got %v", s32, res) + } + }) + + t.Run("int64", func(t *testing.T) { + var sink bytes.Buffer + enc := NewEncoder(&sink) + enc.Encode(s64) + + dec := NewDecoder(&sink) + res := make([]int64, 9) + dec.Decode(&res) + + if !reflect.DeepEqual(s64, res) { + t.Fatalf("EncodeIntSlice: expected %v, got %v", s64, res) + } + }) + +} + type ET0 struct { A int B string diff --git a/libgo/go/encoding/gob/error.go b/libgo/go/encoding/gob/error.go index 8b5265c278f..949333bc037 100644 --- a/libgo/go/encoding/gob/error.go +++ b/libgo/go/encoding/gob/error.go @@ -39,5 +39,4 @@ func catchError(err *error) { } *err = ge.err } - return } diff --git a/libgo/go/encoding/gob/gobencdec_test.go b/libgo/go/encoding/gob/gobencdec_test.go index ecc91eef1f8..41a06b26c87 100644 --- a/libgo/go/encoding/gob/gobencdec_test.go +++ b/libgo/go/encoding/gob/gobencdec_test.go @@ -746,7 +746,7 @@ func (i *isZeroBugInterface) GobDecode(data []byte) error { } func TestGobEncodeIsZero(t *testing.T) { - x := isZeroBug{time.Now(), "hello", -55, isZeroBugArray{1, 2}, isZeroBugInterface{}} + x := isZeroBug{time.Unix(1e9, 0), "hello", -55, isZeroBugArray{1, 2}, isZeroBugInterface{}} b := new(bytes.Buffer) enc := NewEncoder(b) err := enc.Encode(x) diff --git a/libgo/go/encoding/gob/timing_test.go b/libgo/go/encoding/gob/timing_test.go index 424b7e6ea8e..3478bd247ed 100644 --- a/libgo/go/encoding/gob/timing_test.go +++ b/libgo/go/encoding/gob/timing_test.go @@ -8,6 +8,7 @@ import ( "bytes" "io" "os" + "reflect" "runtime" "testing" ) @@ -132,89 +133,60 @@ func TestCountDecodeMallocs(t *testing.T) { } } +func benchmarkEncodeSlice(b *testing.B, a interface{}) { + b.ResetTimer() + b.RunParallel(func(pb *testing.PB) { + var buf bytes.Buffer + enc := NewEncoder(&buf) + + for pb.Next() { + buf.Reset() + err := enc.Encode(a) + if err != nil { + b.Fatal(err) + } + } + }) +} + func BenchmarkEncodeComplex128Slice(b *testing.B) { - var buf bytes.Buffer - enc := NewEncoder(&buf) a := make([]complex128, 1000) for i := range a { a[i] = 1.2 + 3.4i } - b.ResetTimer() - for i := 0; i < b.N; i++ { - buf.Reset() - err := enc.Encode(a) - if err != nil { - b.Fatal(err) - } - } + benchmarkEncodeSlice(b, a) } func BenchmarkEncodeFloat64Slice(b *testing.B) { - var buf bytes.Buffer - enc := NewEncoder(&buf) a := make([]float64, 1000) for i := range a { a[i] = 1.23e4 } - b.ResetTimer() - for i := 0; i < b.N; i++ { - buf.Reset() - err := enc.Encode(a) - if err != nil { - b.Fatal(err) - } - } + benchmarkEncodeSlice(b, a) } func BenchmarkEncodeInt32Slice(b *testing.B) { - var buf bytes.Buffer - enc := NewEncoder(&buf) a := make([]int32, 1000) for i := range a { - a[i] = 1234 - } - b.ResetTimer() - for i := 0; i < b.N; i++ { - buf.Reset() - err := enc.Encode(a) - if err != nil { - b.Fatal(err) - } + a[i] = int32(i * 100) } + benchmarkEncodeSlice(b, a) } func BenchmarkEncodeStringSlice(b *testing.B) { - var buf bytes.Buffer - enc := NewEncoder(&buf) a := make([]string, 1000) for i := range a { a[i] = "now is the time" } - b.ResetTimer() - for i := 0; i < b.N; i++ { - buf.Reset() - err := enc.Encode(a) - if err != nil { - b.Fatal(err) - } - } + benchmarkEncodeSlice(b, a) } func BenchmarkEncodeInterfaceSlice(b *testing.B) { - var buf bytes.Buffer - enc := NewEncoder(&buf) a := make([]interface{}, 1000) for i := range a { a[i] = "now is the time" } - b.ResetTimer() - for i := 0; i < b.N; i++ { - buf.Reset() - err := enc.Encode(a) - if err != nil { - b.Fatal(err) - } - } + benchmarkEncodeSlice(b, a) } // benchmarkBuf is a read buffer we can reset @@ -245,120 +217,96 @@ func (b *benchmarkBuf) reset() { b.offset = 0 } -func BenchmarkDecodeComplex128Slice(b *testing.B) { +func benchmarkDecodeSlice(b *testing.B, a interface{}) { var buf bytes.Buffer enc := NewEncoder(&buf) - a := make([]complex128, 1000) - for i := range a { - a[i] = 1.2 + 3.4i - } err := enc.Encode(a) if err != nil { b.Fatal(err) } - x := make([]complex128, 1000) - bbuf := benchmarkBuf{data: buf.Bytes()} + + ra := reflect.ValueOf(a) + rt := ra.Type() b.ResetTimer() - for i := 0; i < b.N; i++ { - bbuf.reset() - dec := NewDecoder(&bbuf) - err := dec.Decode(&x) - if err != nil { - b.Fatal(i, err) + + b.RunParallel(func(pb *testing.PB) { + // TODO(#19025): Move per-thread allocation before ResetTimer. + rp := reflect.New(rt) + rp.Elem().Set(reflect.MakeSlice(rt, ra.Len(), ra.Cap())) + p := rp.Interface() + + bbuf := benchmarkBuf{data: buf.Bytes()} + + for pb.Next() { + bbuf.reset() + dec := NewDecoder(&bbuf) + err := dec.Decode(p) + if err != nil { + b.Fatal(err) + } } + }) +} + +func BenchmarkDecodeComplex128Slice(b *testing.B) { + a := make([]complex128, 1000) + for i := range a { + a[i] = 1.2 + 3.4i } + benchmarkDecodeSlice(b, a) } func BenchmarkDecodeFloat64Slice(b *testing.B) { - var buf bytes.Buffer - enc := NewEncoder(&buf) a := make([]float64, 1000) for i := range a { a[i] = 1.23e4 } - err := enc.Encode(a) - if err != nil { - b.Fatal(err) - } - x := make([]float64, 1000) - bbuf := benchmarkBuf{data: buf.Bytes()} - b.ResetTimer() - for i := 0; i < b.N; i++ { - bbuf.reset() - dec := NewDecoder(&bbuf) - err := dec.Decode(&x) - if err != nil { - b.Fatal(i, err) - } - } + benchmarkDecodeSlice(b, a) } func BenchmarkDecodeInt32Slice(b *testing.B) { - var buf bytes.Buffer - enc := NewEncoder(&buf) a := make([]int32, 1000) for i := range a { a[i] = 1234 } - err := enc.Encode(a) - if err != nil { - b.Fatal(err) - } - x := make([]int32, 1000) - bbuf := benchmarkBuf{data: buf.Bytes()} - b.ResetTimer() - for i := 0; i < b.N; i++ { - bbuf.reset() - dec := NewDecoder(&bbuf) - err := dec.Decode(&x) - if err != nil { - b.Fatal(i, err) - } - } + benchmarkDecodeSlice(b, a) } func BenchmarkDecodeStringSlice(b *testing.B) { - var buf bytes.Buffer - enc := NewEncoder(&buf) a := make([]string, 1000) for i := range a { a[i] = "now is the time" } - err := enc.Encode(a) - if err != nil { - b.Fatal(err) - } - x := make([]string, 1000) - bbuf := benchmarkBuf{data: buf.Bytes()} - b.ResetTimer() - for i := 0; i < b.N; i++ { - bbuf.reset() - dec := NewDecoder(&bbuf) - err := dec.Decode(&x) - if err != nil { - b.Fatal(i, err) - } - } + benchmarkDecodeSlice(b, a) } func BenchmarkDecodeInterfaceSlice(b *testing.B) { - var buf bytes.Buffer - enc := NewEncoder(&buf) a := make([]interface{}, 1000) for i := range a { a[i] = "now is the time" } - err := enc.Encode(a) + benchmarkDecodeSlice(b, a) +} + +func BenchmarkDecodeMap(b *testing.B) { + count := 1000 + m := make(map[int]int, count) + for i := 0; i < count; i++ { + m[i] = i + } + var buf bytes.Buffer + enc := NewEncoder(&buf) + err := enc.Encode(m) if err != nil { b.Fatal(err) } - x := make([]interface{}, 1000) bbuf := benchmarkBuf{data: buf.Bytes()} b.ResetTimer() for i := 0; i < b.N; i++ { + var rm map[int]int bbuf.reset() dec := NewDecoder(&bbuf) - err := dec.Decode(&x) + err := dec.Decode(&rm) if err != nil { b.Fatal(i, err) } diff --git a/libgo/go/encoding/gob/type.go b/libgo/go/encoding/gob/type.go index c27f7e9707e..31c0ef7af15 100644 --- a/libgo/go/encoding/gob/type.go +++ b/libgo/go/encoding/gob/type.go @@ -36,31 +36,21 @@ const ( xText // encoding.TextMarshaler or encoding.TextUnmarshaler ) -var ( - // Protected by an RWMutex because we read it a lot and write - // it only when we see a new type, typically when compiling. - userTypeLock sync.RWMutex - userTypeCache = make(map[reflect.Type]*userTypeInfo) -) +var userTypeCache sync.Map // map[reflect.Type]*userTypeInfo // validType returns, and saves, the information associated with user-provided type rt. // If the user type is not valid, err will be non-nil. To be used when the error handler // is not set up. -func validUserType(rt reflect.Type) (ut *userTypeInfo, err error) { - userTypeLock.RLock() - ut = userTypeCache[rt] - userTypeLock.RUnlock() - if ut != nil { - return - } - // Now set the value under the write lock. - userTypeLock.Lock() - defer userTypeLock.Unlock() - if ut = userTypeCache[rt]; ut != nil { - // Lost the race; not a problem. - return +func validUserType(rt reflect.Type) (*userTypeInfo, error) { + if ui, ok := userTypeCache.Load(rt); ok { + return ui.(*userTypeInfo), nil } - ut = new(userTypeInfo) + + // Construct a new userTypeInfo and atomically add it to the userTypeCache. + // If we lose the race, we'll waste a little CPU and create a little garbage + // but return the existing value anyway. + + ut := new(userTypeInfo) ut.base = rt ut.user = rt // A type that is just a cycle of pointers (such as type T *T) cannot @@ -108,8 +98,8 @@ func validUserType(rt reflect.Type) (ut *userTypeInfo, err error) { // ut.externalDec, ut.decIndir = xText, indir // } - userTypeCache[rt] = ut - return + ui, _ := userTypeCache.LoadOrStore(rt, ut) + return ui.(*userTypeInfo), nil } var ( @@ -808,9 +798,8 @@ type GobDecoder interface { } var ( - registerLock sync.RWMutex - nameToConcreteType = make(map[string]reflect.Type) - concreteTypeToName = make(map[reflect.Type]string) + nameToConcreteType sync.Map // map[string]reflect.Type + concreteTypeToName sync.Map // map[reflect.Type]string ) // RegisterName is like Register but uses the provided name rather than the @@ -820,21 +809,22 @@ func RegisterName(name string, value interface{}) { // reserved for nil panic("attempt to register empty name") } - registerLock.Lock() - defer registerLock.Unlock() + ut := userType(reflect.TypeOf(value)) + // Check for incompatible duplicates. The name must refer to the // same user type, and vice versa. - if t, ok := nameToConcreteType[name]; ok && t != ut.user { + + // Store the name and type provided by the user.... + if t, dup := nameToConcreteType.LoadOrStore(name, reflect.TypeOf(value)); dup && t != ut.user { panic(fmt.Sprintf("gob: registering duplicate types for %q: %s != %s", name, t, ut.user)) } - if n, ok := concreteTypeToName[ut.base]; ok && n != name { + + // but the flattened type in the type table, since that's what decode needs. + if n, dup := concreteTypeToName.LoadOrStore(ut.base, name); dup && n != name { + nameToConcreteType.Delete(name) panic(fmt.Sprintf("gob: registering duplicate names for %s: %q != %q", ut.user, n, name)) } - // Store the name and type provided by the user.... - nameToConcreteType[name] = reflect.TypeOf(value) - // but the flattened type in the type table, since that's what decode needs. - concreteTypeToName[ut.base] = name } // Register records a type, identified by a value for that type, under its diff --git a/libgo/go/encoding/gob/type_test.go b/libgo/go/encoding/gob/type_test.go index e230d22d431..14f25d8ac4c 100644 --- a/libgo/go/encoding/gob/type_test.go +++ b/libgo/go/encoding/gob/type_test.go @@ -178,9 +178,7 @@ func TestRegistrationNaming(t *testing.T) { Register(tc.t) tct := reflect.TypeOf(tc.t) - registerLock.RLock() - ct := nameToConcreteType[tc.name] - registerLock.RUnlock() + ct, _ := nameToConcreteType.Load(tc.name) if ct != tct { t.Errorf("nameToConcreteType[%q] = %v, want %v", tc.name, ct, tct) } @@ -188,7 +186,7 @@ func TestRegistrationNaming(t *testing.T) { if tct.Kind() == reflect.Ptr { tct = tct.Elem() } - if n := concreteTypeToName[tct]; n != tc.name { + if n, _ := concreteTypeToName.Load(tct); n != tc.name { t.Errorf("concreteTypeToName[%v] got %v, want %v", tct, n, tc.name) } } diff --git a/libgo/go/encoding/hex/hex.go b/libgo/go/encoding/hex/hex.go index b43c1c4b456..2768f1bac69 100644 --- a/libgo/go/encoding/hex/hex.go +++ b/libgo/go/encoding/hex/hex.go @@ -12,10 +12,7 @@ import ( "io" ) -var hextable = [16]byte{ - '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', - 'a', 'b', 'c', 'd', 'e', 'f', -} +const hextable = "0123456789abcdef" // EncodedLen returns the length of an encoding of n source bytes. // Specifically, it returns n * 2. diff --git a/libgo/go/encoding/json/bench_test.go b/libgo/go/encoding/json/bench_test.go index cd7380b1efb..85d7ae043b8 100644 --- a/libgo/go/encoding/json/bench_test.go +++ b/libgo/go/encoding/json/bench_test.go @@ -82,12 +82,14 @@ func BenchmarkCodeEncoder(b *testing.B) { codeInit() b.StartTimer() } - enc := NewEncoder(ioutil.Discard) - for i := 0; i < b.N; i++ { - if err := enc.Encode(&codeStruct); err != nil { - b.Fatal("Encode:", err) + b.RunParallel(func(pb *testing.PB) { + enc := NewEncoder(ioutil.Discard) + for pb.Next() { + if err := enc.Encode(&codeStruct); err != nil { + b.Fatal("Encode:", err) + } } - } + }) b.SetBytes(int64(len(codeJSON))) } @@ -97,11 +99,13 @@ func BenchmarkCodeMarshal(b *testing.B) { codeInit() b.StartTimer() } - for i := 0; i < b.N; i++ { - if _, err := Marshal(&codeStruct); err != nil { - b.Fatal("Marshal:", err) + b.RunParallel(func(pb *testing.PB) { + for pb.Next() { + if _, err := Marshal(&codeStruct); err != nil { + b.Fatal("Marshal:", err) + } } - } + }) b.SetBytes(int64(len(codeJSON))) } @@ -111,19 +115,21 @@ func BenchmarkCodeDecoder(b *testing.B) { codeInit() b.StartTimer() } - var buf bytes.Buffer - dec := NewDecoder(&buf) - var r codeResponse - for i := 0; i < b.N; i++ { - buf.Write(codeJSON) - // hide EOF - buf.WriteByte('\n') - buf.WriteByte('\n') - buf.WriteByte('\n') - if err := dec.Decode(&r); err != nil { - b.Fatal("Decode:", err) + b.RunParallel(func(pb *testing.PB) { + var buf bytes.Buffer + dec := NewDecoder(&buf) + var r codeResponse + for pb.Next() { + buf.Write(codeJSON) + // hide EOF + buf.WriteByte('\n') + buf.WriteByte('\n') + buf.WriteByte('\n') + if err := dec.Decode(&r); err != nil { + b.Fatal("Decode:", err) + } } - } + }) b.SetBytes(int64(len(codeJSON))) } @@ -155,12 +161,14 @@ func BenchmarkCodeUnmarshal(b *testing.B) { codeInit() b.StartTimer() } - for i := 0; i < b.N; i++ { - var r codeResponse - if err := Unmarshal(codeJSON, &r); err != nil { - b.Fatal("Unmarshal:", err) + b.RunParallel(func(pb *testing.PB) { + for pb.Next() { + var r codeResponse + if err := Unmarshal(codeJSON, &r); err != nil { + b.Fatal("Unmarshal:", err) + } } - } + }) b.SetBytes(int64(len(codeJSON))) } @@ -170,54 +178,75 @@ func BenchmarkCodeUnmarshalReuse(b *testing.B) { codeInit() b.StartTimer() } - var r codeResponse - for i := 0; i < b.N; i++ { - if err := Unmarshal(codeJSON, &r); err != nil { - b.Fatal("Unmarshal:", err) + b.RunParallel(func(pb *testing.PB) { + var r codeResponse + for pb.Next() { + if err := Unmarshal(codeJSON, &r); err != nil { + b.Fatal("Unmarshal:", err) + } } - } + }) + // TODO(bcmills): Is there a missing b.SetBytes here? } func BenchmarkUnmarshalString(b *testing.B) { data := []byte(`"hello, world"`) - var s string - - for i := 0; i < b.N; i++ { - if err := Unmarshal(data, &s); err != nil { - b.Fatal("Unmarshal:", err) + b.RunParallel(func(pb *testing.PB) { + var s string + for pb.Next() { + if err := Unmarshal(data, &s); err != nil { + b.Fatal("Unmarshal:", err) + } } - } + }) } func BenchmarkUnmarshalFloat64(b *testing.B) { - var f float64 data := []byte(`3.14`) - - for i := 0; i < b.N; i++ { - if err := Unmarshal(data, &f); err != nil { - b.Fatal("Unmarshal:", err) + b.RunParallel(func(pb *testing.PB) { + var f float64 + for pb.Next() { + if err := Unmarshal(data, &f); err != nil { + b.Fatal("Unmarshal:", err) + } } - } + }) } func BenchmarkUnmarshalInt64(b *testing.B) { - var x int64 data := []byte(`3`) - - for i := 0; i < b.N; i++ { - if err := Unmarshal(data, &x); err != nil { - b.Fatal("Unmarshal:", err) + b.RunParallel(func(pb *testing.PB) { + var x int64 + for pb.Next() { + if err := Unmarshal(data, &x); err != nil { + b.Fatal("Unmarshal:", err) + } } - } + }) } func BenchmarkIssue10335(b *testing.B) { b.ReportAllocs() - var s struct{} j := []byte(`{"a":{ }}`) - for n := 0; n < b.N; n++ { - if err := Unmarshal(j, &s); err != nil { - b.Fatal(err) + b.RunParallel(func(pb *testing.PB) { + var s struct{} + for pb.Next() { + if err := Unmarshal(j, &s); err != nil { + b.Fatal(err) + } } - } + }) +} + +func BenchmarkUnmapped(b *testing.B) { + b.ReportAllocs() + j := []byte(`{"s": "hello", "y": 2, "o": {"x": 0}, "a": [1, 99, {"x": 1}]}`) + b.RunParallel(func(pb *testing.PB) { + var s struct{} + for pb.Next() { + if err := Unmarshal(j, &s); err != nil { + b.Fatal(err) + } + } + }) } diff --git a/libgo/go/encoding/json/decode.go b/libgo/go/encoding/json/decode.go index 77fc4607dba..420a07e1549 100644 --- a/libgo/go/encoding/json/decode.go +++ b/libgo/go/encoding/json/decode.go @@ -22,7 +22,8 @@ import ( ) // Unmarshal parses the JSON-encoded data and stores the result -// in the value pointed to by v. +// in the value pointed to by v. If v is nil or not a pointer, +// Unmarshal returns an InvalidUnmarshalError. // // Unmarshal uses the inverse of the encodings that // Marshal uses, allocating maps, slices, and pointers as necessary, @@ -78,7 +79,9 @@ import ( // or if a JSON number overflows the target type, Unmarshal // skips that field and completes the unmarshaling as best it can. // If no more serious errors are encountered, Unmarshal returns -// an UnmarshalTypeError describing the earliest such error. +// an UnmarshalTypeError describing the earliest such error. In any +// case, it's not guaranteed that all the remaining fields following +// the problematic one will be unmarshaled into the target object. // // The JSON null value unmarshals into an interface, map, pointer, or slice // by setting that Go value to nil. Because null is often used in JSON to mean diff --git a/libgo/go/encoding/json/encode.go b/libgo/go/encoding/json/encode.go index 8f21ddaed94..0371f0a24dd 100644 --- a/libgo/go/encoding/json/encode.go +++ b/libgo/go/encoding/json/encode.go @@ -332,10 +332,7 @@ type encOpts struct { type encoderFunc func(e *encodeState, v reflect.Value, opts encOpts) -var encoderCache struct { - sync.RWMutex - m map[reflect.Type]encoderFunc -} +var encoderCache sync.Map // map[reflect.Type]encoderFunc func valueEncoder(v reflect.Value) encoderFunc { if !v.IsValid() { @@ -345,36 +342,31 @@ func valueEncoder(v reflect.Value) encoderFunc { } func typeEncoder(t reflect.Type) encoderFunc { - encoderCache.RLock() - f := encoderCache.m[t] - encoderCache.RUnlock() - if f != nil { - return f + if fi, ok := encoderCache.Load(t); ok { + return fi.(encoderFunc) } // To deal with recursive types, populate the map with an // indirect func before we build it. This type waits on the // real func (f) to be ready and then calls it. This indirect // func is only used for recursive types. - encoderCache.Lock() - if encoderCache.m == nil { - encoderCache.m = make(map[reflect.Type]encoderFunc) - } - var wg sync.WaitGroup + var ( + wg sync.WaitGroup + f encoderFunc + ) wg.Add(1) - encoderCache.m[t] = func(e *encodeState, v reflect.Value, opts encOpts) { + fi, loaded := encoderCache.LoadOrStore(t, encoderFunc(func(e *encodeState, v reflect.Value, opts encOpts) { wg.Wait() f(e, v, opts) + })) + if loaded { + return fi.(encoderFunc) } - encoderCache.Unlock() - // Compute fields without lock. - // Might duplicate effort but won't hold other computations back. + // Compute the real encoder and replace the indirect func with it. f = newTypeEncoder(t, true) wg.Done() - encoderCache.Lock() - encoderCache.m[t] = f - encoderCache.Unlock() + encoderCache.Store(t, f) return f } @@ -1101,7 +1093,22 @@ func typeFields(t reflect.Type) []field { // Scan f.typ for fields to include. for i := 0; i < f.typ.NumField(); i++ { sf := f.typ.Field(i) - if sf.PkgPath != "" && !sf.Anonymous { // unexported + if sf.Anonymous { + t := sf.Type + if t.Kind() == reflect.Ptr { + t = t.Elem() + } + // If embedded, StructField.PkgPath is not a reliable + // indicator of whether the field is exported. + // See https://golang.org/issue/21122 + if !isExported(t.Name()) && t.Kind() != reflect.Struct { + // Ignore embedded fields of unexported non-struct types. + // Do not ignore embedded fields of unexported struct types + // since they may have exported fields. + continue + } + } else if sf.PkgPath != "" { + // Ignore unexported non-embedded fields. continue } tag := sf.Tag.Get("json") @@ -1219,6 +1226,12 @@ func typeFields(t reflect.Type) []field { return fields } +// isExported reports whether the identifier is exported. +func isExported(id string) bool { + r, _ := utf8.DecodeRuneInString(id) + return unicode.IsUpper(r) +} + // dominantField looks through the fields, all of which are known to // have the same name, to find the single field that dominates the // others using Go's embedding rules, modified by the presence of diff --git a/libgo/go/encoding/json/encode_test.go b/libgo/go/encoding/json/encode_test.go index 6d574cfc47b..3fda6a0c719 100644 --- a/libgo/go/encoding/json/encode_test.go +++ b/libgo/go/encoding/json/encode_test.go @@ -253,23 +253,167 @@ func TestMarshalerEscaping(t *testing.T) { } } -type IntType int - -type MyStruct struct { - IntType -} - -func TestAnonymousNonstruct(t *testing.T) { - var i IntType = 11 - a := MyStruct{i} - const want = `{"IntType":11}` +func TestAnonymousFields(t *testing.T) { + tests := []struct { + label string // Test name + makeInput func() interface{} // Function to create input value + want string // Expected JSON output + }{{ + // Both S1 and S2 have a field named X. From the perspective of S, + // it is ambiguous which one X refers to. + // This should not serialize either field. + label: "AmbiguousField", + makeInput: func() interface{} { + type ( + S1 struct{ x, X int } + S2 struct{ x, X int } + S struct { + S1 + S2 + } + ) + return S{S1{1, 2}, S2{3, 4}} + }, + want: `{}`, + }, { + label: "DominantField", + // Both S1 and S2 have a field named X, but since S has an X field as + // well, it takes precedence over S1.X and S2.X. + makeInput: func() interface{} { + type ( + S1 struct{ x, X int } + S2 struct{ x, X int } + S struct { + S1 + S2 + x, X int + } + ) + return S{S1{1, 2}, S2{3, 4}, 5, 6} + }, + want: `{"X":6}`, + }, { + // Unexported embedded field of non-struct type should not be serialized. + label: "UnexportedEmbeddedInt", + makeInput: func() interface{} { + type ( + myInt int + S struct{ myInt } + ) + return S{5} + }, + want: `{}`, + }, { + // Exported embedded field of non-struct type should be serialized. + label: "ExportedEmbeddedInt", + makeInput: func() interface{} { + type ( + MyInt int + S struct{ MyInt } + ) + return S{5} + }, + want: `{"MyInt":5}`, + }, { + // Unexported embedded field of pointer to non-struct type + // should not be serialized. + label: "UnexportedEmbeddedIntPointer", + makeInput: func() interface{} { + type ( + myInt int + S struct{ *myInt } + ) + s := S{new(myInt)} + *s.myInt = 5 + return s + }, + want: `{}`, + }, { + // Exported embedded field of pointer to non-struct type + // should be serialized. + label: "ExportedEmbeddedIntPointer", + makeInput: func() interface{} { + type ( + MyInt int + S struct{ *MyInt } + ) + s := S{new(MyInt)} + *s.MyInt = 5 + return s + }, + want: `{"MyInt":5}`, + }, { + // Exported fields of embedded structs should have their + // exported fields be serialized regardless of whether the struct types + // themselves are exported. + label: "EmbeddedStruct", + makeInput: func() interface{} { + type ( + s1 struct{ x, X int } + S2 struct{ y, Y int } + S struct { + s1 + S2 + } + ) + return S{s1{1, 2}, S2{3, 4}} + }, + want: `{"X":2,"Y":4}`, + }, { + // Exported fields of pointers to embedded structs should have their + // exported fields be serialized regardless of whether the struct types + // themselves are exported. + label: "EmbeddedStructPointer", + makeInput: func() interface{} { + type ( + s1 struct{ x, X int } + S2 struct{ y, Y int } + S struct { + *s1 + *S2 + } + ) + return S{&s1{1, 2}, &S2{3, 4}} + }, + want: `{"X":2,"Y":4}`, + }, { + // Exported fields on embedded unexported structs at multiple levels + // of nesting should still be serialized. + label: "NestedStructAndInts", + makeInput: func() interface{} { + type ( + MyInt1 int + MyInt2 int + myInt int + s2 struct { + MyInt2 + myInt + } + s1 struct { + MyInt1 + myInt + s2 + } + S struct { + s1 + myInt + } + ) + return S{s1{1, 2, s2{3, 4}}, 6} + }, + want: `{"MyInt1":1,"MyInt2":3}`, + }} - b, err := Marshal(a) - if err != nil { - t.Fatalf("Marshal: %v", err) - } - if got := string(b); got != want { - t.Errorf("got %q, want %q", got, want) + for _, tt := range tests { + t.Run(tt.label, func(t *testing.T) { + b, err := Marshal(tt.makeInput()) + if err != nil { + t.Fatalf("Marshal() = %v, want nil error", err) + } + if string(b) != tt.want { + t.Fatalf("Marshal() = %q, want %q", b, tt.want) + } + }) } } diff --git a/libgo/go/encoding/json/scanner.go b/libgo/go/encoding/json/scanner.go index a6d8706c731..ae34418d1da 100644 --- a/libgo/go/encoding/json/scanner.go +++ b/libgo/go/encoding/json/scanner.go @@ -15,6 +15,11 @@ package json import "strconv" +// Valid reports whether data is a valid JSON encoding. +func Valid(data []byte) bool { + return checkValid(data, &scanner{}) == nil +} + // checkValid verifies that data is valid JSON-encoded data. // scan is passed in for use by checkValid to avoid an allocation. func checkValid(data []byte, scan *scanner) error { diff --git a/libgo/go/encoding/json/scanner_test.go b/libgo/go/encoding/json/scanner_test.go index c5c1be31f1e..0d4518a632b 100644 --- a/libgo/go/encoding/json/scanner_test.go +++ b/libgo/go/encoding/json/scanner_test.go @@ -12,6 +12,26 @@ import ( "testing" ) +var validTests = []struct { + data string + ok bool +}{ + {`foo`, false}, + {`}{`, false}, + {`{]`, false}, + {`{}`, true}, + {`{"foo":"bar"}`, true}, + {`{"foo":"bar","bar":{"baz":["qux"]}}`, true}, +} + +func TestValid(t *testing.T) { + for _, tt := range validTests { + if ok := Valid([]byte(tt.data)); ok != tt.ok { + t.Errorf("Valid(%#q) = %v, want %v", tt.data, ok, tt.ok) + } + } +} + // Tests of simple examples. type example struct { diff --git a/libgo/go/encoding/json/stream_test.go b/libgo/go/encoding/json/stream_test.go index 84edeb187c2..d0b3ffbce93 100644 --- a/libgo/go/encoding/json/stream_test.go +++ b/libgo/go/encoding/json/stream_test.go @@ -268,11 +268,13 @@ func BenchmarkEncoderEncode(b *testing.B) { X, Y string } v := &T{"foo", "bar"} - for i := 0; i < b.N; i++ { - if err := NewEncoder(ioutil.Discard).Encode(v); err != nil { - b.Fatal(err) + b.RunParallel(func(pb *testing.PB) { + for pb.Next() { + if err := NewEncoder(ioutil.Discard).Encode(v); err != nil { + b.Fatal(err) + } } - } + }) } type tokenStreamCase struct { diff --git a/libgo/go/encoding/pem/pem.go b/libgo/go/encoding/pem/pem.go index fbf49997d5e..5e1ab90cffc 100644 --- a/libgo/go/encoding/pem/pem.go +++ b/libgo/go/encoding/pem/pem.go @@ -135,20 +135,26 @@ func Decode(data []byte) (p *Block, rest []byte) { return decodeError(data, rest) } - // After the "-----" of the ending line should be the same type and a - // final five dashes. + // After the "-----" of the ending line, there should be the same type + // and then a final five dashes. endTrailer := rest[endTrailerIndex:] endTrailerLen := len(typeLine) + len(pemEndOfLine) if len(endTrailer) < endTrailerLen { return decodeError(data, rest) } + restOfEndLine := endTrailer[endTrailerLen:] endTrailer = endTrailer[:endTrailerLen] if !bytes.HasPrefix(endTrailer, typeLine) || !bytes.HasSuffix(endTrailer, pemEndOfLine) { return decodeError(data, rest) } + // The line must end with only whitespace. + if s, _ := getLine(restOfEndLine); len(s) != 0 { + return decodeError(data, rest) + } + base64Data := removeWhitespace(rest[:endIndex]) p.Bytes = make([]byte, base64.StdEncoding.DecodedLen(len(base64Data))) n, err := base64.StdEncoding.Decode(p.Bytes, base64Data) diff --git a/libgo/go/encoding/pem/pem_test.go b/libgo/go/encoding/pem/pem_test.go index 6321dec3820..1a1250a52f4 100644 --- a/libgo/go/encoding/pem/pem_test.go +++ b/libgo/go/encoding/pem/pem_test.go @@ -83,6 +83,16 @@ const pemTooFewEndingDashes = ` dGVzdA== -----END FOO----` +const pemTooManyEndingDashes = ` +-----BEGIN FOO----- +dGVzdA== +-----END FOO------` + +const pemTrailingNonWhitespace = ` +-----BEGIN FOO----- +dGVzdA== +-----END FOO----- .` + const pemWrongEndingType = ` -----BEGIN FOO----- dGVzdA== @@ -101,6 +111,14 @@ var badPEMTests = []struct { "too few trailing dashes", pemTooFewEndingDashes, }, + { + "too many trailing dashes", + pemTooManyEndingDashes, + }, + { + "trailing non-whitespace", + pemTrailingNonWhitespace, + }, { "incorrect ending type", pemWrongEndingType, @@ -188,10 +206,20 @@ func TestLineBreaker(t *testing.T) { } func TestFuzz(t *testing.T) { + // PEM is a text-based format. Assume header fields with leading/trailing spaces + // or embedded newlines will not round trip correctly and don't need to be tested. + isBad := func(s string) bool { + return strings.ContainsAny(s, "\r\n") || strings.TrimSpace(s) != s + } + testRoundtrip := func(block Block) bool { - for key := range block.Headers { - if strings.Contains(key, ":") { - // Keys with colons cannot be encoded. + if isBad(block.Type) { + return true + } + for key, val := range block.Headers { + // Reject bad key/val. + // Also, keys with colons cannot be encoded, because : is the key: val separator. + if isBad(key) || isBad(val) || strings.Contains(key, ":") { return true } } diff --git a/libgo/go/encoding/xml/marshal_test.go b/libgo/go/encoding/xml/marshal_test.go index 0126146d332..674c6b5b3f4 100644 --- a/libgo/go/encoding/xml/marshal_test.go +++ b/libgo/go/encoding/xml/marshal_test.go @@ -1652,28 +1652,31 @@ func TestMarshal(t *testing.T) { if test.UnmarshalOnly { continue } - data, err := Marshal(test.Value) - if err != nil { - if test.MarshalError == "" { - t.Errorf("#%d: marshal(%#v): %s", idx, test.Value, err) - continue + + t.Run(fmt.Sprintf("%d", idx), func(t *testing.T) { + data, err := Marshal(test.Value) + if err != nil { + if test.MarshalError == "" { + t.Errorf("marshal(%#v): %s", test.Value, err) + return + } + if !strings.Contains(err.Error(), test.MarshalError) { + t.Errorf("marshal(%#v): %s, want %q", test.Value, err, test.MarshalError) + } + return } - if !strings.Contains(err.Error(), test.MarshalError) { - t.Errorf("#%d: marshal(%#v): %s, want %q", idx, test.Value, err, test.MarshalError) + if test.MarshalError != "" { + t.Errorf("Marshal succeeded, want error %q", test.MarshalError) + return } - continue - } - if test.MarshalError != "" { - t.Errorf("#%d: Marshal succeeded, want error %q", idx, test.MarshalError) - continue - } - if got, want := string(data), test.ExpectXML; got != want { - if strings.Contains(want, "\n") { - t.Errorf("#%d: marshal(%#v):\nHAVE:\n%s\nWANT:\n%s", idx, test.Value, got, want) - } else { - t.Errorf("#%d: marshal(%#v):\nhave %#q\nwant %#q", idx, test.Value, got, want) + if got, want := string(data), test.ExpectXML; got != want { + if strings.Contains(want, "\n") { + t.Errorf("marshal(%#v):\nHAVE:\n%s\nWANT:\n%s", test.Value, got, want) + } else { + t.Errorf("marshal(%#v):\nhave %#q\nwant %#q", test.Value, got, want) + } } - } + }) } } @@ -1781,27 +1784,29 @@ func TestUnmarshal(t *testing.T) { dest := reflect.New(vt.Elem()).Interface() err := Unmarshal([]byte(test.ExpectXML), dest) - switch fix := dest.(type) { - case *Feed: - fix.Author.InnerXML = "" - for i := range fix.Entry { - fix.Entry[i].Author.InnerXML = "" + t.Run(fmt.Sprintf("%d", i), func(t *testing.T) { + switch fix := dest.(type) { + case *Feed: + fix.Author.InnerXML = "" + for i := range fix.Entry { + fix.Entry[i].Author.InnerXML = "" + } } - } - if err != nil { - if test.UnmarshalError == "" { - t.Errorf("#%d: unmarshal(%#v): %s", i, test.ExpectXML, err) - continue + if err != nil { + if test.UnmarshalError == "" { + t.Errorf("unmarshal(%#v): %s", test.ExpectXML, err) + return + } + if !strings.Contains(err.Error(), test.UnmarshalError) { + t.Errorf("unmarshal(%#v): %s, want %q", test.ExpectXML, err, test.UnmarshalError) + } + return } - if !strings.Contains(err.Error(), test.UnmarshalError) { - t.Errorf("#%d: unmarshal(%#v): %s, want %q", i, test.ExpectXML, err, test.UnmarshalError) + if got, want := dest, test.Value; !reflect.DeepEqual(got, want) { + t.Errorf("unmarshal(%q):\nhave %#v\nwant %#v", test.ExpectXML, got, want) } - continue - } - if got, want := dest, test.Value; !reflect.DeepEqual(got, want) { - t.Errorf("#%d: unmarshal(%q):\nhave %#v\nwant %#v", i, test.ExpectXML, got, want) - } + }) } } @@ -1896,17 +1901,21 @@ func TestMarshalFlush(t *testing.T) { func BenchmarkMarshal(b *testing.B) { b.ReportAllocs() - for i := 0; i < b.N; i++ { - Marshal(atomValue) - } + b.RunParallel(func(pb *testing.PB) { + for pb.Next() { + Marshal(atomValue) + } + }) } func BenchmarkUnmarshal(b *testing.B) { b.ReportAllocs() xml := []byte(atomXml) - for i := 0; i < b.N; i++ { - Unmarshal(xml, &Feed{}) - } + b.RunParallel(func(pb *testing.PB) { + for pb.Next() { + Unmarshal(xml, &Feed{}) + } + }) } // golang.org/issue/6556 @@ -2428,10 +2437,7 @@ func TestIssue16158(t *testing.T) { err := Unmarshal([]byte(data), &struct { B byte `xml:"b,attr,omitempty"` }{}) - - // For Go 1.8.1 we've restored the old "no errors reported" behavior. - // We'll try again in Go 1.9 to report errors. - if err != nil { - t.Errorf("Unmarshal: expected nil, got error") + if err == nil { + t.Errorf("Unmarshal: expected error, got nil") } } diff --git a/libgo/go/encoding/xml/read.go b/libgo/go/encoding/xml/read.go index 799b57e9d15..000d9fbd0ef 100644 --- a/libgo/go/encoding/xml/read.go +++ b/libgo/go/encoding/xml/read.go @@ -120,6 +120,9 @@ import ( // Unmarshal maps an XML element to a pointer by setting the pointer // to a freshly allocated value and then mapping the element to that value. // +// A missing element or empty attribute value will be unmarshaled as a zero value. +// If the field is a slice, a zero value will be appended to the field. Otherwise, the +// field will be set to its zero value. func Unmarshal(data []byte, v interface{}) error { return NewDecoder(bytes.NewReader(data)).Decode(v) } @@ -211,7 +214,7 @@ func (p *Decoder) unmarshalInterface(val Unmarshaler, start *StartElement) error // unmarshalTextInterface unmarshals a single XML element into val. // The chardata contained in the element (but not its children) // is passed to the text unmarshaler. -func (p *Decoder) unmarshalTextInterface(val encoding.TextUnmarshaler, start *StartElement) error { +func (p *Decoder) unmarshalTextInterface(val encoding.TextUnmarshaler) error { var buf []byte depth := 1 for depth > 0 { @@ -285,8 +288,7 @@ func (p *Decoder) unmarshalAttr(val reflect.Value, attr Attr) error { return nil } - copyValue(val, []byte(attr.Value)) - return nil + return copyValue(val, []byte(attr.Value)) } var ( @@ -342,13 +344,13 @@ func (p *Decoder) unmarshal(val reflect.Value, start *StartElement) error { } if val.CanInterface() && val.Type().Implements(textUnmarshalerType) { - return p.unmarshalTextInterface(val.Interface().(encoding.TextUnmarshaler), start) + return p.unmarshalTextInterface(val.Interface().(encoding.TextUnmarshaler)) } if val.CanAddr() { pv := val.Addr() if pv.CanInterface() && pv.Type().Implements(textUnmarshalerType) { - return p.unmarshalTextInterface(pv.Interface().(encoding.TextUnmarshaler), start) + return p.unmarshalTextInterface(pv.Interface().(encoding.TextUnmarshaler)) } } @@ -608,24 +610,40 @@ func copyValue(dst reflect.Value, src []byte) (err error) { default: return errors.New("cannot unmarshal into " + dst0.Type().String()) case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: + if len(src) == 0 { + dst.SetInt(0) + return nil + } itmp, err := strconv.ParseInt(string(src), 10, dst.Type().Bits()) if err != nil { return err } dst.SetInt(itmp) case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr: + if len(src) == 0 { + dst.SetUint(0) + return nil + } utmp, err := strconv.ParseUint(string(src), 10, dst.Type().Bits()) if err != nil { return err } dst.SetUint(utmp) case reflect.Float32, reflect.Float64: + if len(src) == 0 { + dst.SetFloat(0) + return nil + } ftmp, err := strconv.ParseFloat(string(src), dst.Type().Bits()) if err != nil { return err } dst.SetFloat(ftmp) case reflect.Bool: + if len(src) == 0 { + dst.SetBool(false) + return nil + } value, err := strconv.ParseBool(strings.TrimSpace(string(src))) if err != nil { return err diff --git a/libgo/go/encoding/xml/read_test.go b/libgo/go/encoding/xml/read_test.go index 273c303d16d..a1eb5161877 100644 --- a/libgo/go/encoding/xml/read_test.go +++ b/libgo/go/encoding/xml/read_test.go @@ -752,3 +752,159 @@ func TestInvalidInnerXMLType(t *testing.T) { t.Errorf("NotInnerXML = %v, want nil", v.NotInnerXML) } } + +type Child struct { + G struct { + I int + } +} + +type ChildToEmbed struct { + X bool +} + +type Parent struct { + I int + IPtr *int + Is []int + IPtrs []*int + F float32 + FPtr *float32 + Fs []float32 + FPtrs []*float32 + B bool + BPtr *bool + Bs []bool + BPtrs []*bool + Bytes []byte + BytesPtr *[]byte + S string + SPtr *string + Ss []string + SPtrs []*string + MyI MyInt + Child Child + Children []Child + ChildPtr *Child + ChildToEmbed +} + +const ( + emptyXML = ` + + + + + + + + + + + + + + + + + + + + + + + + + +` +) + +// github.com/golang/go/issues/13417 +func TestUnmarshalEmptyValues(t *testing.T) { + // Test first with a zero-valued dst. + v := new(Parent) + if err := Unmarshal([]byte(emptyXML), v); err != nil { + t.Fatalf("zero: Unmarshal failed: got %v", err) + } + + zBytes, zInt, zStr, zFloat, zBool := []byte{}, 0, "", float32(0), false + want := &Parent{ + IPtr: &zInt, + Is: []int{zInt}, + IPtrs: []*int{&zInt}, + FPtr: &zFloat, + Fs: []float32{zFloat}, + FPtrs: []*float32{&zFloat}, + BPtr: &zBool, + Bs: []bool{zBool}, + BPtrs: []*bool{&zBool}, + Bytes: []byte{}, + BytesPtr: &zBytes, + SPtr: &zStr, + Ss: []string{zStr}, + SPtrs: []*string{&zStr}, + Children: []Child{{}}, + ChildPtr: new(Child), + ChildToEmbed: ChildToEmbed{}, + } + if !reflect.DeepEqual(v, want) { + t.Fatalf("zero: Unmarshal:\nhave: %#+v\nwant: %#+v", v, want) + } + + // Test with a pre-populated dst. + // Multiple addressable copies, as pointer-to fields will replace value during unmarshal. + vBytes0, vInt0, vStr0, vFloat0, vBool0 := []byte("x"), 1, "x", float32(1), true + vBytes1, vInt1, vStr1, vFloat1, vBool1 := []byte("x"), 1, "x", float32(1), true + vInt2, vStr2, vFloat2, vBool2 := 1, "x", float32(1), true + v = &Parent{ + I: vInt0, + IPtr: &vInt1, + Is: []int{vInt0}, + IPtrs: []*int{&vInt2}, + F: vFloat0, + FPtr: &vFloat1, + Fs: []float32{vFloat0}, + FPtrs: []*float32{&vFloat2}, + B: vBool0, + BPtr: &vBool1, + Bs: []bool{vBool0}, + BPtrs: []*bool{&vBool2}, + Bytes: vBytes0, + BytesPtr: &vBytes1, + S: vStr0, + SPtr: &vStr1, + Ss: []string{vStr0}, + SPtrs: []*string{&vStr2}, + MyI: MyInt(vInt0), + Child: Child{G: struct{ I int }{I: vInt0}}, + Children: []Child{{G: struct{ I int }{I: vInt0}}}, + ChildPtr: &Child{G: struct{ I int }{I: vInt0}}, + ChildToEmbed: ChildToEmbed{X: vBool0}, + } + if err := Unmarshal([]byte(emptyXML), v); err != nil { + t.Fatalf("populated: Unmarshal failed: got %v", err) + } + + want = &Parent{ + IPtr: &zInt, + Is: []int{vInt0, zInt}, + IPtrs: []*int{&vInt0, &zInt}, + FPtr: &zFloat, + Fs: []float32{vFloat0, zFloat}, + FPtrs: []*float32{&vFloat0, &zFloat}, + BPtr: &zBool, + Bs: []bool{vBool0, zBool}, + BPtrs: []*bool{&vBool0, &zBool}, + Bytes: []byte{}, + BytesPtr: &zBytes, + SPtr: &zStr, + Ss: []string{vStr0, zStr}, + SPtrs: []*string{&vStr0, &zStr}, + Child: Child{G: struct{ I int }{I: vInt0}}, // I should == zInt0? (zero value) + Children: []Child{{G: struct{ I int }{I: vInt0}}, {}}, + ChildPtr: &Child{G: struct{ I int }{I: vInt0}}, // I should == zInt0? (zero value) + } + if !reflect.DeepEqual(v, want) { + t.Fatalf("populated: Unmarshal:\nhave: %#+v\nwant: %#+v", v, want) + } +} diff --git a/libgo/go/encoding/xml/typeinfo.go b/libgo/go/encoding/xml/typeinfo.go index 6623c78308e..751caa97aa1 100644 --- a/libgo/go/encoding/xml/typeinfo.go +++ b/libgo/go/encoding/xml/typeinfo.go @@ -42,21 +42,18 @@ const ( fMode = fElement | fAttr | fCDATA | fCharData | fInnerXml | fComment | fAny ) -var tinfoMap = make(map[reflect.Type]*typeInfo) -var tinfoLock sync.RWMutex +var tinfoMap sync.Map // map[reflect.Type]*typeInfo var nameType = reflect.TypeOf(Name{}) // getTypeInfo returns the typeInfo structure with details necessary // for marshaling and unmarshaling typ. func getTypeInfo(typ reflect.Type) (*typeInfo, error) { - tinfoLock.RLock() - tinfo, ok := tinfoMap[typ] - tinfoLock.RUnlock() - if ok { - return tinfo, nil + if ti, ok := tinfoMap.Load(typ); ok { + return ti.(*typeInfo), nil } - tinfo = &typeInfo{} + + tinfo := &typeInfo{} if typ.Kind() == reflect.Struct && typ != nameType { n := typ.NumField() for i := 0; i < n; i++ { @@ -105,10 +102,9 @@ func getTypeInfo(typ reflect.Type) (*typeInfo, error) { } } } - tinfoLock.Lock() - tinfoMap[typ] = tinfo - tinfoLock.Unlock() - return tinfo, nil + + ti, _ := tinfoMap.LoadOrStore(typ, tinfo) + return ti.(*typeInfo), nil } // structFieldInfo builds and returns a fieldInfo for f. diff --git a/libgo/go/encoding/xml/xml_test.go b/libgo/go/encoding/xml/xml_test.go index f43a5e7eebc..dad6ed98c1b 100644 --- a/libgo/go/encoding/xml/xml_test.go +++ b/libgo/go/encoding/xml/xml_test.go @@ -797,37 +797,3 @@ func TestIssue12417(t *testing.T) { } } } - -func TestIssue19333(t *testing.T) { - type X struct { - XMLName Name `xml:"X"` - A int `xml:",attr"` - C int - } - - var tests = []struct { - input string - ok bool - }{ - {``, true}, - {``, true}, - {``, true}, - {``, true}, - {``, false}, - {``, false}, - {`bad`, false}, - } - - for _, tt := range tests { - err := Unmarshal([]byte(tt.input), new(X)) - if tt.ok { - if err != nil { - t.Errorf("%s: unexpected error: %v", tt.input, err) - } - } else { - if err == nil { - t.Errorf("%s: unexpected success", tt.input) - } - } - } -} diff --git a/libgo/go/expvar/expvar.go b/libgo/go/expvar/expvar.go index 7339fa00b79..64dae70c627 100644 --- a/libgo/go/expvar/expvar.go +++ b/libgo/go/expvar/expvar.go @@ -99,9 +99,9 @@ func (v *Float) Set(value float64) { // Map is a string-to-Var map variable that satisfies the Var interface. type Map struct { - mu sync.RWMutex - m map[string]Var - keys []string // sorted + m sync.Map // map[string]Var + keysMu sync.RWMutex + keys []string // sorted } // KeyValue represents a single entry in a Map. @@ -111,12 +111,10 @@ type KeyValue struct { } func (v *Map) String() string { - v.mu.RLock() - defer v.mu.RUnlock() var b bytes.Buffer fmt.Fprintf(&b, "{") first := true - v.doLocked(func(kv KeyValue) { + v.Do(func(kv KeyValue) { if !first { fmt.Fprintf(&b, ", ") } @@ -127,79 +125,66 @@ func (v *Map) String() string { return b.String() } -func (v *Map) Init() *Map { - v.m = make(map[string]Var) - return v -} +func (v *Map) Init() *Map { return v } // updateKeys updates the sorted list of keys in v.keys. -// must be called with v.mu held. -func (v *Map) updateKeys() { - if len(v.m) == len(v.keys) { - // No new key. - return - } - v.keys = v.keys[:0] - for k := range v.m { - v.keys = append(v.keys, k) - } +func (v *Map) addKey(key string) { + v.keysMu.Lock() + defer v.keysMu.Unlock() + v.keys = append(v.keys, key) sort.Strings(v.keys) } func (v *Map) Get(key string) Var { - v.mu.RLock() - defer v.mu.RUnlock() - return v.m[key] + i, _ := v.m.Load(key) + av, _ := i.(Var) + return av } func (v *Map) Set(key string, av Var) { - v.mu.Lock() - defer v.mu.Unlock() - v.m[key] = av - v.updateKeys() + // Before we store the value, check to see whether the key is new. Try a Load + // before LoadOrStore: LoadOrStore causes the key interface to escape even on + // the Load path. + if _, ok := v.m.Load(key); !ok { + if _, dup := v.m.LoadOrStore(key, av); !dup { + v.addKey(key) + return + } + } + + v.m.Store(key, av) } +// Add adds delta to the *Int value stored under the given map key. func (v *Map) Add(key string, delta int64) { - v.mu.RLock() - av, ok := v.m[key] - v.mu.RUnlock() + i, ok := v.m.Load(key) if !ok { - // check again under the write lock - v.mu.Lock() - av, ok = v.m[key] - if !ok { - av = new(Int) - v.m[key] = av - v.updateKeys() + var dup bool + i, dup = v.m.LoadOrStore(key, new(Int)) + if !dup { + v.addKey(key) } - v.mu.Unlock() } // Add to Int; ignore otherwise. - if iv, ok := av.(*Int); ok { + if iv, ok := i.(*Int); ok { iv.Add(delta) } } // AddFloat adds delta to the *Float value stored under the given map key. func (v *Map) AddFloat(key string, delta float64) { - v.mu.RLock() - av, ok := v.m[key] - v.mu.RUnlock() + i, ok := v.m.Load(key) if !ok { - // check again under the write lock - v.mu.Lock() - av, ok = v.m[key] - if !ok { - av = new(Float) - v.m[key] = av - v.updateKeys() + var dup bool + i, dup = v.m.LoadOrStore(key, new(Float)) + if !dup { + v.addKey(key) } - v.mu.Unlock() } // Add to Float; ignore otherwise. - if iv, ok := av.(*Float); ok { + if iv, ok := i.(*Float); ok { iv.Add(delta) } } @@ -208,45 +193,34 @@ func (v *Map) AddFloat(key string, delta float64) { // The map is locked during the iteration, // but existing entries may be concurrently updated. func (v *Map) Do(f func(KeyValue)) { - v.mu.RLock() - defer v.mu.RUnlock() - v.doLocked(f) -} - -// doLocked calls f for each entry in the map. -// v.mu must be held for reads. -func (v *Map) doLocked(f func(KeyValue)) { + v.keysMu.RLock() + defer v.keysMu.RUnlock() for _, k := range v.keys { - f(KeyValue{k, v.m[k]}) + i, _ := v.m.Load(k) + f(KeyValue{k, i.(Var)}) } } // String is a string variable, and satisfies the Var interface. type String struct { - mu sync.RWMutex - s string + s atomic.Value // string } func (v *String) Value() string { - v.mu.RLock() - defer v.mu.RUnlock() - return v.s + p, _ := v.s.Load().(string) + return p } // String implements the Val interface. To get the unquoted string // use Value. func (v *String) String() string { - v.mu.RLock() - s := v.s - v.mu.RUnlock() + s := v.Value() b, _ := json.Marshal(s) return string(b) } func (v *String) Set(value string) { - v.mu.Lock() - defer v.mu.Unlock() - v.s = value + v.s.Store(value) } // Func implements Var by calling the function @@ -264,21 +238,20 @@ func (f Func) String() string { // All published variables. var ( - mutex sync.RWMutex - vars = make(map[string]Var) - varKeys []string // sorted + vars sync.Map // map[string]Var + varKeysMu sync.RWMutex + varKeys []string // sorted ) // Publish declares a named exported variable. This should be called from a // package's init function when it creates its Vars. If the name is already // registered then this will log.Panic. func Publish(name string, v Var) { - mutex.Lock() - defer mutex.Unlock() - if _, existing := vars[name]; existing { + if _, dup := vars.LoadOrStore(name, v); dup { log.Panicln("Reuse of exported var name:", name) } - vars[name] = v + varKeysMu.Lock() + defer varKeysMu.Unlock() varKeys = append(varKeys, name) sort.Strings(varKeys) } @@ -286,9 +259,9 @@ func Publish(name string, v Var) { // Get retrieves a named exported variable. It returns nil if the name has // not been registered. func Get(name string) Var { - mutex.RLock() - defer mutex.RUnlock() - return vars[name] + i, _ := vars.Load(name) + v, _ := i.(Var) + return v } // Convenience functions for creating new exported variables. @@ -321,10 +294,11 @@ func NewString(name string) *String { // The global variable map is locked during the iteration, // but existing entries may be concurrently updated. func Do(f func(KeyValue)) { - mutex.RLock() - defer mutex.RUnlock() + varKeysMu.RLock() + defer varKeysMu.RUnlock() for _, k := range varKeys { - f(KeyValue{k, vars[k]}) + val, _ := vars.Load(k) + f(KeyValue{k, val.(Var)}) } } diff --git a/libgo/go/expvar/expvar_test.go b/libgo/go/expvar/expvar_test.go index 0efa8643c0b..7014063d4f7 100644 --- a/libgo/go/expvar/expvar_test.go +++ b/libgo/go/expvar/expvar_test.go @@ -7,21 +7,25 @@ package expvar import ( "bytes" "encoding/json" + "fmt" "net" "net/http/httptest" "reflect" "runtime" "strconv" "sync" + "sync/atomic" "testing" ) // RemoveAll removes all exported variables. // This is for tests only. func RemoveAll() { - mutex.Lock() - defer mutex.Unlock() - vars = make(map[string]Var) + varKeysMu.Lock() + defer varKeysMu.Unlock() + for _, k := range varKeys { + vars.Delete(k) + } varKeys = nil } @@ -36,8 +40,8 @@ func TestNil(t *testing.T) { func TestInt(t *testing.T) { RemoveAll() reqs := NewInt("requests") - if reqs.i != 0 { - t.Errorf("reqs.i = %v, want 0", reqs.i) + if i := reqs.Value(); i != 0 { + t.Errorf("reqs.Value() = %v, want 0", i) } if reqs != Get("requests").(*Int) { t.Errorf("Get() failed.") @@ -45,8 +49,8 @@ func TestInt(t *testing.T) { reqs.Add(1) reqs.Add(3) - if reqs.i != 4 { - t.Errorf("reqs.i = %v, want 4", reqs.i) + if i := reqs.Value(); i != 4 { + t.Errorf("reqs.Value() = %v, want 4", i) } if s := reqs.String(); s != "4" { @@ -54,12 +58,8 @@ func TestInt(t *testing.T) { } reqs.Set(-2) - if reqs.i != -2 { - t.Errorf("reqs.i = %v, want -2", reqs.i) - } - - if v, want := reqs.Value(), int64(-2); v != want { - t.Errorf("reqs.Value() = %q, want %q", v, want) + if i := reqs.Value(); i != -2 { + t.Errorf("reqs.Value() = %v, want -2", i) } } @@ -132,27 +132,22 @@ func BenchmarkFloatSet(b *testing.B) { func TestString(t *testing.T) { RemoveAll() name := NewString("my-name") - if name.s != "" { - t.Errorf("name.s = %q, want \"\"", name.s) + if s := name.Value(); s != "" { + t.Errorf(`NewString("my-name").Value() = %q, want ""`, s) } name.Set("Mike") - if name.s != "Mike" { - t.Errorf("name.s = %q, want \"Mike\"", name.s) - } - if s, want := name.String(), `"Mike"`; s != want { - t.Errorf("from %q, name.String() = %q, want %q", name.s, s, want) + t.Errorf(`after name.Set("Mike"), name.String() = %q, want %q`, s, want) } - if s, want := name.Value(), "Mike"; s != want { - t.Errorf("from %q, name.Value() = %q, want %q", name.s, s, want) + t.Errorf(`after name.Set("Mike"), name.Value() = %q, want %q`, s, want) } // Make sure we produce safe JSON output. - name.Set(`<`) + name.Set("<") if s, want := name.String(), "\"\\u003c\""; s != want { - t.Errorf("from %q, name.String() = %q, want %q", name.s, s, want) + t.Errorf(`after name.Set("<"), name.String() = %q, want %q`, s, want) } } @@ -174,13 +169,13 @@ func TestMapCounter(t *testing.T) { colors.Add("red", 2) colors.Add("blue", 4) colors.AddFloat(`green "midori"`, 4.125) - if x := colors.m["red"].(*Int).i; x != 3 { + if x := colors.Get("red").(*Int).Value(); x != 3 { t.Errorf("colors.m[\"red\"] = %v, want 3", x) } - if x := colors.m["blue"].(*Int).i; x != 4 { + if x := colors.Get("blue").(*Int).Value(); x != 4 { t.Errorf("colors.m[\"blue\"] = %v, want 4", x) } - if x := colors.m[`green "midori"`].(*Float).Value(); x != 4.125 { + if x := colors.Get(`green "midori"`).(*Float).Value(); x != 4.125 { t.Errorf("colors.m[`green \"midori\"] = %v, want 4.125", x) } @@ -218,24 +213,117 @@ func BenchmarkMapSet(b *testing.B) { }) } -func BenchmarkMapAddSame(b *testing.B) { - for i := 0; i < b.N; i++ { - m := new(Map).Init() - m.Add("red", 1) - m.Add("red", 1) - m.Add("red", 1) - m.Add("red", 1) +func BenchmarkMapSetDifferent(b *testing.B) { + procKeys := make([][]string, runtime.GOMAXPROCS(0)) + for i := range procKeys { + keys := make([]string, 4) + for j := range keys { + keys[j] = fmt.Sprint(i, j) + } + procKeys[i] = keys } + + m := new(Map).Init() + v := new(Int) + b.ResetTimer() + + var n int32 + b.RunParallel(func(pb *testing.PB) { + i := int(atomic.AddInt32(&n, 1)-1) % len(procKeys) + keys := procKeys[i] + + for pb.Next() { + for _, k := range keys { + m.Set(k, v) + } + } + }) +} + +func BenchmarkMapSetString(b *testing.B) { + m := new(Map).Init() + + v := new(String) + v.Set("Hello, !") + + b.RunParallel(func(pb *testing.PB) { + for pb.Next() { + m.Set("red", v) + } + }) +} + +func BenchmarkMapAddSame(b *testing.B) { + b.RunParallel(func(pb *testing.PB) { + for pb.Next() { + m := new(Map).Init() + m.Add("red", 1) + m.Add("red", 1) + m.Add("red", 1) + m.Add("red", 1) + } + }) } func BenchmarkMapAddDifferent(b *testing.B) { - for i := 0; i < b.N; i++ { - m := new(Map).Init() - m.Add("red", 1) - m.Add("blue", 1) - m.Add("green", 1) - m.Add("yellow", 1) + procKeys := make([][]string, runtime.GOMAXPROCS(0)) + for i := range procKeys { + keys := make([]string, 4) + for j := range keys { + keys[j] = fmt.Sprint(i, j) + } + procKeys[i] = keys + } + + b.ResetTimer() + + var n int32 + b.RunParallel(func(pb *testing.PB) { + i := int(atomic.AddInt32(&n, 1)-1) % len(procKeys) + keys := procKeys[i] + + for pb.Next() { + m := new(Map).Init() + for _, k := range keys { + m.Add(k, 1) + } + } + }) +} + +func BenchmarkMapAddSameSteadyState(b *testing.B) { + m := new(Map).Init() + b.RunParallel(func(pb *testing.PB) { + for pb.Next() { + m.Add("red", 1) + } + }) +} + +func BenchmarkMapAddDifferentSteadyState(b *testing.B) { + procKeys := make([][]string, runtime.GOMAXPROCS(0)) + for i := range procKeys { + keys := make([]string, 4) + for j := range keys { + keys[j] = fmt.Sprint(i, j) + } + procKeys[i] = keys } + + m := new(Map).Init() + b.ResetTimer() + + var n int32 + b.RunParallel(func(pb *testing.PB) { + i := int(atomic.AddInt32(&n, 1)-1) % len(procKeys) + keys := procKeys[i] + + for pb.Next() { + for _, k := range keys { + m.Add(k, 1) + } + } + }) } func TestFunc(t *testing.T) { diff --git a/libgo/go/flag/flag.go b/libgo/go/flag/flag.go index bbbc55a2799..b166c5753ad 100644 --- a/libgo/go/flag/flag.go +++ b/libgo/go/flag/flag.go @@ -114,7 +114,7 @@ func newIntValue(val int, p *int) *intValue { } func (i *intValue) Set(s string) error { - v, err := strconv.ParseInt(s, 0, 64) + v, err := strconv.ParseInt(s, 0, strconv.IntSize) *i = intValue(v) return err } @@ -150,7 +150,7 @@ func newUintValue(val uint, p *uint) *uintValue { } func (i *uintValue) Set(s string) error { - v, err := strconv.ParseUint(s, 0, 64) + v, err := strconv.ParseUint(s, 0, strconv.IntSize) *i = uintValue(v) return err } diff --git a/libgo/go/flag/flag_test.go b/libgo/go/flag/flag_test.go index e2319ec94c8..02da2c77506 100644 --- a/libgo/go/flag/flag_test.go +++ b/libgo/go/flag/flag_test.go @@ -10,6 +10,7 @@ import ( "fmt" "os" "sort" + "strconv" "strings" "testing" "time" @@ -415,3 +416,19 @@ func TestPrintDefaults(t *testing.T) { t.Errorf("got %q want %q\n", got, defaultOutput) } } + +// Issue 19230: validate range of Int and Uint flag values. +func TestIntFlagOverflow(t *testing.T) { + if strconv.IntSize != 32 { + return + } + ResetForTesting(nil) + Int("i", 0, "") + Uint("u", 0, "") + if err := Set("i", "2147483648"); err == nil { + t.Error("unexpected success setting Int") + } + if err := Set("u", "4294967296"); err == nil { + t.Error("unexpected success setting Uint") + } +} diff --git a/libgo/go/fmt/doc.go b/libgo/go/fmt/doc.go index a2faecb36e1..014ba06948d 100644 --- a/libgo/go/fmt/doc.go +++ b/libgo/go/fmt/doc.go @@ -38,7 +38,7 @@ %E scientific notation, e.g. -1.234456E+78 %f decimal point but no exponent, e.g. 123.456 %F synonym for %f - %g %e for large exponents, %f otherwise + %g %e for large exponents, %f otherwise. Precision is discussed below. %G %E for large exponents, %F otherwise String and slice of bytes (treated equivalently with these verbs): %s the uninterpreted bytes of the string or slice @@ -94,7 +94,7 @@ precision sets the number of places after the decimal, if appropriate, except that for %g/%G precision sets the total number of significant digits. For example, given 12.345 the format %6.3f prints 12.345 while - %.3g prints 12.3. The default precision for %e and %f is 6; for %g it + %.3g prints 12.3. The default precision for %e, %f and %#g is 6; for %g it is the smallest number of digits necessary to identify the value uniquely. For complex numbers, the width and precision apply to the two @@ -109,6 +109,8 @@ 0X for hex (%#X); suppress 0x for %p (%#p); for %q, print a raw (backquoted) string if strconv.CanBackquote returns true; + always print a decimal point for %e, %E, %f, %F, %g and %G; + do not remove trailing zeros for %g and %G; write e.g. U+0078 'x' if the character is printable for %U (%#U). ' ' (space) leave a space for elided sign in numbers (% d); put spaces between bytes printing strings or slices in hex (% x, % X) @@ -190,9 +192,9 @@ For example, fmt.Sprintf("%[2]d %[1]d\n", 11, 22) will yield "22 11", while - fmt.Sprintf("%[3]*.[2]*[1]f", 12.0, 2, 6), + fmt.Sprintf("%[3]*.[2]*[1]f", 12.0, 2, 6) equivalent to - fmt.Sprintf("%6.2f", 12.0), + fmt.Sprintf("%6.2f", 12.0) will yield " 12.00". Because an explicit index affects subsequent verbs, this notation can be used to print the same values multiple times by resetting the index for the first argument to be repeated: diff --git a/libgo/go/fmt/fmt_test.go b/libgo/go/fmt/fmt_test.go index ce00456c7ba..9e6fcfa7016 100644 --- a/libgo/go/fmt/fmt_test.go +++ b/libgo/go/fmt/fmt_test.go @@ -416,6 +416,32 @@ var fmtTests = []struct { {"% .3g", 1.0, " 1"}, {"%b", float32(1.0), "8388608p-23"}, {"%b", 1.0, "4503599627370496p-52"}, + // Test sharp flag used with floats. + {"%#g", 1e-323, "1.00000e-323"}, + {"%#g", -1.0, "-1.00000"}, + {"%#g", 1.1, "1.10000"}, + {"%#g", 123456.0, "123456."}, + {"%#g", 1234567.0, "1.234567e+06"}, + {"%#g", 1230000.0, "1.23000e+06"}, + {"%#g", 1000000.0, "1.00000e+06"}, + {"%#.0f", 1.0, "1."}, + {"%#.0e", 1.0, "1.e+00"}, + {"%#.0g", 1.0, "1."}, + {"%#.0g", 1100000.0, "1.e+06"}, + {"%#.4f", 1.0, "1.0000"}, + {"%#.4e", 1.0, "1.0000e+00"}, + {"%#.4g", 1.0, "1.000"}, + {"%#.4g", 100000.0, "1.000e+05"}, + {"%#.0f", 123.0, "123."}, + {"%#.0e", 123.0, "1.e+02"}, + {"%#.0g", 123.0, "1.e+02"}, + {"%#.4f", 123.0, "123.0000"}, + {"%#.4e", 123.0, "1.2300e+02"}, + {"%#.4g", 123.0, "123.0"}, + {"%#.4g", 123000.0, "1.230e+05"}, + {"%#9.4g", 1.0, " 1.000"}, + // The sharp flag has no effect for binary float format. + {"%#b", 1.0, "4503599627370496p-52"}, // Precision has no effect for binary float format. {"%.4b", float32(1.0), "8388608p-23"}, {"%.4b", -1.0, "-4503599627370496p-52"}, @@ -466,8 +492,24 @@ var fmtTests = []struct { {"% .3E", -1 - 2i, "(-1.000E+00-2.000E+00i)"}, {"%+.3g", 1 + 2i, "(+1+2i)"}, {"%+.3g", complex64(1 + 2i), "(+1+2i)"}, + {"%#g", 1 + 2i, "(1.00000+2.00000i)"}, + {"%#g", 123456 + 789012i, "(123456.+789012.i)"}, + {"%#g", 1e-10i, "(0.00000+1.00000e-10i)"}, + {"%#g", -1e10 - 1.11e100i, "(-1.00000e+10-1.11000e+100i)"}, + {"%#.0f", 1.23 + 1.0i, "(1.+1.i)"}, + {"%#.0e", 1.23 + 1.0i, "(1.e+00+1.e+00i)"}, + {"%#.0g", 1.23 + 1.0i, "(1.+1.i)"}, + {"%#.0g", 0 + 100000i, "(0.+1.e+05i)"}, + {"%#.0g", 1230000 + 0i, "(1.e+06+0.i)"}, + {"%#.4f", 1 + 1.23i, "(1.0000+1.2300i)"}, + {"%#.4e", 123 + 1i, "(1.2300e+02+1.0000e+00i)"}, + {"%#.4g", 123 + 1.23i, "(123.0+1.230i)"}, + {"%#12.5g", 0 + 100000i, "( 0.0000 +1.0000e+05i)"}, + {"%#12.5g", 1230000 - 0i, "( 1.2300e+06 +0.0000i)"}, {"%b", 1 + 2i, "(4503599627370496p-52+4503599627370496p-51i)"}, {"%b", complex64(1 + 2i), "(8388608p-23+8388608p-22i)"}, + // The sharp flag has no effect for binary complex format. + {"%#b", 1 + 2i, "(4503599627370496p-52+4503599627370496p-51i)"}, // Precision has no effect for binary complex format. {"%.4b", 1 + 2i, "(4503599627370496p-52+4503599627370496p-51i)"}, {"%.4b", complex64(1 + 2i), "(8388608p-23+8388608p-22i)"}, diff --git a/libgo/go/fmt/format.go b/libgo/go/fmt/format.go index f77048338ae..d4b92f81214 100644 --- a/libgo/go/fmt/format.go +++ b/libgo/go/fmt/format.go @@ -480,6 +480,46 @@ func (f *fmt) fmt_float(v float64, size int, verb rune, prec int) { f.zero = oldZero return } + // The sharp flag forces printing a decimal point for non-binary formats + // and retains trailing zeros, which we may need to restore. + if f.sharp && verb != 'b' { + digits := 0 + switch verb { + case 'v', 'g', 'G': + digits = prec + // If no precision is set explicitly use a precision of 6. + if digits == -1 { + digits = 6 + } + } + + // Buffer pre-allocated with enough room for + // exponent notations of the form "e+123". + var tailBuf [5]byte + tail := tailBuf[:0] + + hasDecimalPoint := false + // Starting from i = 1 to skip sign at num[0]. + for i := 1; i < len(num); i++ { + switch num[i] { + case '.': + hasDecimalPoint = true + case 'e', 'E': + tail = append(tail, num[i:]...) + num = num[:i] + default: + digits-- + } + } + if !hasDecimalPoint { + num = append(num, '.') + } + for digits > 0 { + num = append(num, '0') + digits-- + } + num = append(num, tail...) + } // We want a sign if asked for and if the sign is not positive. if f.plus || num[0] != '+' { // If we're zero padding to the left we want the sign before the leading zeros. diff --git a/libgo/go/fmt/print.go b/libgo/go/fmt/print.go index a7ef2e5ac21..2bd88f95a2b 100644 --- a/libgo/go/fmt/print.go +++ b/libgo/go/fmt/print.go @@ -684,8 +684,6 @@ func (p *pp) printArg(arg interface{}, verb rune) { } } -var byteType = reflect.TypeOf(byte(0)) - // printValue is similar to printArg but starts with a reflect value, not an interface{} value. // It does not handle 'p' and 'T' verbs because these should have been already handled by printArg. func (p *pp) printValue(value reflect.Value, verb rune, depth int) { diff --git a/libgo/go/go/ast/ast.go b/libgo/go/go/ast/ast.go index a197b5a5bfc..9ab7b1e6256 100644 --- a/libgo/go/go/ast/ast.go +++ b/libgo/go/go/ast/ast.go @@ -848,6 +848,7 @@ type ( TypeSpec struct { Doc *CommentGroup // associated documentation; or nil Name *Ident // type name + Assign token.Pos // position of '=', if any Type Expr // *Ident, *ParenExpr, *SelectorExpr, *StarExpr, or any of the *XxxTypes Comment *CommentGroup // line comments; or nil } @@ -926,7 +927,7 @@ type ( Recv *FieldList // receiver (methods); or nil (functions) Name *Ident // function/method name Type *FuncType // function signature: parameters, results, and position of "func" keyword - Body *BlockStmt // function body; or nil (forward declaration) + Body *BlockStmt // function body; or nil for external (non-Go) function } ) @@ -966,6 +967,19 @@ func (*FuncDecl) declNode() {} // appearance, including the comments that are pointed to from other nodes // via Doc and Comment fields. // +// For correct printing of source code containing comments (using packages +// go/format and go/printer), special care must be taken to update comments +// when a File's syntax tree is modified: For printing, comments are interspersed +// between tokens based on their position. If syntax tree nodes are +// removed or moved, relevant comments in their vicinity must also be removed +// (from the File.Comments list) or moved accordingly (by updating their +// positions). A CommentMap may be used to facilitate some of these operations. +// +// Whether and how a comment is associated with a node depends on the +// interpretation of the syntax tree by the manipulating program: Except for Doc +// and Comment comments directly associated with nodes, the remaining comments +// are "free-floating" (see also issues #18593, #20744). +// type File struct { Doc *CommentGroup // associated documentation; or nil Package token.Pos // position of "package" keyword diff --git a/libgo/go/go/build/build.go b/libgo/go/go/build/build.go index cce5d482ada..5aa32d0ae44 100644 --- a/libgo/go/go/build/build.go +++ b/libgo/go/go/build/build.go @@ -155,6 +155,7 @@ func (ctxt *Context) hasSubdir(root, dir string) (rel string, ok bool) { return hasSubdir(rootSym, dirSym) } +// hasSubdir reports if dir is within root by performing lexical analysis only. func hasSubdir(root, dir string) (rel string, ok bool) { const sep = string(filepath.Separator) root = filepath.Clean(root) @@ -290,7 +291,8 @@ func defaultContext() Context { // in all releases >= Go 1.x. Code that requires Go 1.x or later should // say "+build go1.x", and code that should only be built before Go 1.x // (perhaps it is the stub to use in that case) should say "+build !go1.x". - c.ReleaseTags = []string{"go1.1", "go1.2", "go1.3", "go1.4", "go1.5", "go1.6", "go1.7", "go1.8"} + // NOTE: If you add to this list, also update the doc comment in doc.go. + c.ReleaseTags = []string{"go1.1", "go1.2", "go1.3", "go1.4", "go1.5", "go1.6", "go1.7", "go1.8", "go1.9"} env := os.Getenv("CGO_ENABLED") // No defaultCGO_ENABLED in gccgo. @@ -529,6 +531,7 @@ func (ctxt *Context) Import(path string, srcDir string, mode ImportMode) (*Packa if !ctxt.isAbsPath(path) { p.Dir = ctxt.joinPath(srcDir, path) } + // p.Dir directory may or may not exist. Gather partial information first, check if it exists later. // Determine canonical import path, if any. // Exclude results where the import path would include /testdata/. inTestdata := func(sub string) bool { @@ -683,6 +686,16 @@ Found: } } + // If it's a local import path, by the time we get here, we still haven't checked + // that p.Dir directory exists. This is the right time to do that check. + // We can't do it earlier, because we want to gather partial information for the + // non-nil *Package returned when an error occurs. + // We need to do this before we return early on FindOnly flag. + if IsLocalImport(path) && !ctxt.isDir(p.Dir) { + // package was not found + return p, fmt.Errorf("cannot find package %q in:\n\t%s", path, p.Dir) + } + if mode&FindOnly != 0 { return p, pkgerr } @@ -718,7 +731,7 @@ Found: p.InvalidGoFiles = append(p.InvalidGoFiles, name) } - match, data, filename, err := ctxt.matchFile(p.Dir, name, true, allTags, &p.BinaryOnly) + match, data, filename, err := ctxt.matchFile(p.Dir, name, allTags, &p.BinaryOnly) if err != nil { badFile(err) continue @@ -1032,19 +1045,19 @@ func parseWord(data []byte) (word, rest []byte) { // MatchFile considers the name of the file and may use ctxt.OpenFile to // read some or all of the file's content. func (ctxt *Context) MatchFile(dir, name string) (match bool, err error) { - match, _, _, err = ctxt.matchFile(dir, name, false, nil, nil) + match, _, _, err = ctxt.matchFile(dir, name, nil, nil) return } // matchFile determines whether the file with the given name in the given directory // should be included in the package being constructed. // It returns the data read from the file. -// If returnImports is true and name denotes a Go program, matchFile reads -// until the end of the imports (and returns that data) even though it only -// considers text until the first non-comment. +// If name denotes a Go program, matchFile reads until the end of the +// imports (and returns that data) even though it only considers text +// until the first non-comment. // If allTags is non-nil, matchFile records any encountered build tag // by setting allTags[tag] = true. -func (ctxt *Context) matchFile(dir, name string, returnImports bool, allTags map[string]bool, binaryOnly *bool) (match bool, data []byte, filename string, err error) { +func (ctxt *Context) matchFile(dir, name string, allTags map[string]bool, binaryOnly *bool) (match bool, data []byte, filename string, err error) { if strings.HasPrefix(name, "_") || strings.HasPrefix(name, ".") { return @@ -1270,6 +1283,12 @@ func (ctxt *Context) saveCgo(filename string, di *Package, cg *ast.CommentGroup) args[i] = arg } + switch verb { + case "CFLAGS", "CPPFLAGS", "CXXFLAGS", "FFLAGS", "LDFLAGS": + // Change relative paths to absolute. + ctxt.makePathsAbsolute(args, di.Dir) + } + switch verb { case "CFLAGS": di.CgoCFLAGS = append(di.CgoCFLAGS, args...) @@ -1298,40 +1317,63 @@ func expandSrcDir(str string, srcdir string) (string, bool) { // to "/" before starting (eg: on windows). srcdir = filepath.ToSlash(srcdir) - // Spaces are tolerated in ${SRCDIR}, but not anywhere else. chunks := strings.Split(str, "${SRCDIR}") if len(chunks) < 2 { - return str, safeCgoName(str, false) + return str, safeCgoName(str) } ok := true for _, chunk := range chunks { - ok = ok && (chunk == "" || safeCgoName(chunk, false)) + ok = ok && (chunk == "" || safeCgoName(chunk)) } - ok = ok && (srcdir == "" || safeCgoName(srcdir, true)) + ok = ok && (srcdir == "" || safeCgoName(srcdir)) res := strings.Join(chunks, srcdir) return res, ok && res != "" } +// makePathsAbsolute looks for compiler options that take paths and +// makes them absolute. We do this because through the 1.8 release we +// ran the compiler in the package directory, so any relative -I or -L +// options would be relative to that directory. In 1.9 we changed to +// running the compiler in the build directory, to get consistent +// build results (issue #19964). To keep builds working, we change any +// relative -I or -L options to be absolute. +// +// Using filepath.IsAbs and filepath.Join here means the results will be +// different on different systems, but that's OK: -I and -L options are +// inherently system-dependent. +func (ctxt *Context) makePathsAbsolute(args []string, srcDir string) { + nextPath := false + for i, arg := range args { + if nextPath { + if !filepath.IsAbs(arg) { + args[i] = filepath.Join(srcDir, arg) + } + nextPath = false + } else if strings.HasPrefix(arg, "-I") || strings.HasPrefix(arg, "-L") { + if len(arg) == 2 { + nextPath = true + } else { + if !filepath.IsAbs(arg[2:]) { + args[i] = arg[:2] + filepath.Join(srcDir, arg[2:]) + } + } + } + } +} + // NOTE: $ is not safe for the shell, but it is allowed here because of linker options like -Wl,$ORIGIN. // We never pass these arguments to a shell (just to programs we construct argv for), so this should be okay. // See golang.org/issue/6038. // The @ is for OS X. See golang.org/issue/13720. // The % is for Jenkins. See golang.org/issue/16959. -const safeString = "+-.,/0123456789=ABCDEFGHIJKLMNOPQRSTUVWXYZ_abcdefghijklmnopqrstuvwxyz:$@%" -const safeSpaces = " " +const safeString = "+-.,/0123456789=ABCDEFGHIJKLMNOPQRSTUVWXYZ_abcdefghijklmnopqrstuvwxyz:$@% " -var safeBytes = []byte(safeSpaces + safeString) - -func safeCgoName(s string, spaces bool) bool { +func safeCgoName(s string) bool { if s == "" { return false } - safe := safeBytes - if !spaces { - safe = safe[len(safeSpaces):] - } for i := 0; i < len(s); i++ { - if c := s[i]; c < utf8.RuneSelf && bytes.IndexByte(safe, c) < 0 { + if c := s[i]; c < utf8.RuneSelf && strings.IndexByte(safeString, c) < 0 { return false } } diff --git a/libgo/go/go/build/build_test.go b/libgo/go/go/build/build_test.go index 2e2ff84fb12..38e2e8ad881 100644 --- a/libgo/go/go/build/build_test.go +++ b/libgo/go/go/build/build_test.go @@ -287,9 +287,11 @@ func TestShellSafety(t *testing.T) { {"-I${SRCDIR}/../include", "/projects/src/issue 11868", "-I/projects/src/issue 11868/../include", true}, {"-I${SRCDIR}", "wtf$@%", "-Iwtf$@%", true}, {"-X${SRCDIR}/1,${SRCDIR}/2", "/projects/src/issue 11868", "-X/projects/src/issue 11868/1,/projects/src/issue 11868/2", true}, - {"-I/tmp -I/tmp", "/tmp2", "-I/tmp -I/tmp", false}, + {"-I/tmp -I/tmp", "/tmp2", "-I/tmp -I/tmp", true}, {"-I/tmp", "/tmp/[0]", "-I/tmp", true}, {"-I${SRCDIR}/dir", "/tmp/[0]", "-I/tmp/[0]/dir", false}, + {"-I${SRCDIR}/dir", "/tmp/go go", "-I/tmp/go go/dir", true}, + {"-I${SRCDIR}/dir dir", "/tmp/go", "-I/tmp/go/dir dir", true}, } for _, test := range tests { output, ok := expandSrcDir(test.input, test.srcdir) @@ -302,6 +304,40 @@ func TestShellSafety(t *testing.T) { } } +// Want to get a "cannot find package" error when directory for package does not exist. +// There should be valid partial information in the returned non-nil *Package. +func TestImportDirNotExist(t *testing.T) { + testenv.MustHaveGoBuild(t) // really must just have source + ctxt := Default + ctxt.GOPATH = "" + + tests := []struct { + label string + path, srcDir string + mode ImportMode + }{ + {"Import(full, 0)", "go/build/doesnotexist", "", 0}, + {"Import(local, 0)", "./doesnotexist", filepath.Join(ctxt.GOROOT, "src/go/build"), 0}, + {"Import(full, FindOnly)", "go/build/doesnotexist", "", FindOnly}, + {"Import(local, FindOnly)", "./doesnotexist", filepath.Join(ctxt.GOROOT, "src/go/build"), FindOnly}, + } + for _, test := range tests { + p, err := ctxt.Import(test.path, test.srcDir, test.mode) + if err == nil || !strings.HasPrefix(err.Error(), "cannot find package") { + t.Errorf(`%s got error: %q, want "cannot find package" error`, test.label, err) + } + // If an error occurs, build.Import is documented to return + // a non-nil *Package containing partial information. + if p == nil { + t.Fatalf(`%s got nil p, want non-nil *Package`, test.label) + } + // Verify partial information in p. + if p.ImportPath != "go/build/doesnotexist" { + t.Errorf(`%s got p.ImportPath: %q, want "go/build/doesnotexist"`, test.label, p.ImportPath) + } + } +} + func TestImportVendor(t *testing.T) { testenv.MustHaveGoBuild(t) // really must just have source ctxt := Default diff --git a/libgo/go/go/build/deps_test.go b/libgo/go/go/build/deps_test.go index 147eaf6aba3..87abfba9219 100644 --- a/libgo/go/go/build/deps_test.go +++ b/libgo/go/go/build/deps_test.go @@ -43,6 +43,7 @@ var pkgDeps = map[string][]string{ "sync": {"internal/race", "runtime", "sync/atomic", "unsafe"}, "sync/atomic": {"unsafe"}, "unsafe": {}, + "internal/cpu": {"runtime"}, "L0": { "errors", @@ -52,11 +53,13 @@ var pkgDeps = map[string][]string{ "sync", "sync/atomic", "unsafe", + "internal/cpu", }, // L1 adds simple functions and strings processing, // but not Unicode tables. - "math": {"unsafe"}, + "math": {"internal/cpu", "unsafe"}, + "math/bits": {}, "math/cmplx": {"math"}, "math/rand": {"L0", "math"}, "strconv": {"L0", "unicode/utf8", "math"}, @@ -66,6 +69,7 @@ var pkgDeps = map[string][]string{ "L1": { "L0", "math", + "math/bits", "math/cmplx", "math/rand", "sort", @@ -150,7 +154,8 @@ var pkgDeps = map[string][]string{ "syscall", }, - "os": {"L1", "os", "syscall", "time", "internal/syscall/windows"}, + "internal/poll": {"L0", "internal/race", "syscall", "time", "unicode/utf16", "unicode/utf8"}, + "os": {"L1", "os", "syscall", "time", "internal/poll", "internal/syscall/windows"}, "path/filepath": {"L2", "os", "syscall"}, "io/ioutil": {"L2", "os", "path/filepath", "time"}, "os/exec": {"L2", "os", "context", "path/filepath", "syscall"}, @@ -171,17 +176,16 @@ var pkgDeps = map[string][]string{ "log": {"L1", "os", "fmt", "time"}, // Packages used by testing must be low-level (L2+fmt). - "regexp": {"L2", "regexp/syntax"}, - "regexp/syntax": {"L2"}, - "runtime/debug": {"L2", "fmt", "io/ioutil", "os", "time"}, - "runtime/pprof/internal/protopprof": {"L2", "fmt", "internal/pprof/profile", "os", "time"}, - "runtime/pprof": {"L2", "fmt", "internal/pprof/profile", "os", "runtime/pprof/internal/protopprof", "text/tabwriter", "time"}, - "runtime/trace": {"L0"}, - "text/tabwriter": {"L2"}, + "regexp": {"L2", "regexp/syntax"}, + "regexp/syntax": {"L2"}, + "runtime/debug": {"L2", "fmt", "io/ioutil", "os", "time"}, + "runtime/pprof": {"L2", "compress/gzip", "context", "encoding/binary", "fmt", "io/ioutil", "os", "text/tabwriter", "time"}, + "runtime/trace": {"L0"}, + "text/tabwriter": {"L2"}, "testing": {"L2", "flag", "fmt", "internal/race", "os", "runtime/debug", "runtime/pprof", "runtime/trace", "time"}, "testing/iotest": {"L2", "log"}, - "testing/quick": {"L2", "flag", "fmt", "reflect"}, + "testing/quick": {"L2", "flag", "fmt", "reflect", "time"}, "internal/testenv": {"L2", "OS", "flag", "testing", "syscall"}, // L4 is defined as L3+fmt+log+time, because in general once @@ -215,59 +219,60 @@ var pkgDeps = map[string][]string{ // Go type checking. "go/constant": {"L4", "go/token", "math/big"}, - "go/importer": {"L4", "go/internal/gcimporter", "go/internal/gccgoimporter", "go/types"}, + "go/importer": {"L4", "go/build", "go/internal/gccgoimporter", "go/internal/gcimporter", "go/internal/srcimporter", "go/token", "go/types"}, "go/internal/gcimporter": {"L4", "OS", "go/build", "go/constant", "go/token", "go/types", "text/scanner"}, "go/internal/gccgoimporter": {"L4", "OS", "debug/elf", "go/constant", "go/token", "go/types", "text/scanner"}, + "go/internal/srcimporter": {"L4", "fmt", "go/ast", "go/build", "go/parser", "go/token", "go/types", "path/filepath"}, "go/types": {"L4", "GOPARSER", "container/heap", "go/constant"}, // One of a kind. - "archive/tar": {"L4", "OS", "syscall"}, - "archive/zip": {"L4", "OS", "compress/flate"}, - "container/heap": {"sort"}, - "compress/bzip2": {"L4"}, - "compress/flate": {"L4"}, - "compress/gzip": {"L4", "compress/flate"}, - "compress/lzw": {"L4"}, - "compress/zlib": {"L4", "compress/flate"}, - "context": {"errors", "fmt", "reflect", "sync", "time"}, - "database/sql": {"L4", "container/list", "context", "database/sql/driver", "database/sql/internal"}, - "database/sql/driver": {"L4", "context", "time", "database/sql/internal"}, - "debug/dwarf": {"L4"}, - "debug/elf": {"L4", "OS", "debug/dwarf", "compress/zlib"}, - "debug/gosym": {"L4"}, - "debug/macho": {"L4", "OS", "debug/dwarf"}, - "debug/pe": {"L4", "OS", "debug/dwarf"}, - "debug/plan9obj": {"L4", "OS"}, - "encoding": {"L4"}, - "encoding/ascii85": {"L4"}, - "encoding/asn1": {"L4", "math/big"}, - "encoding/csv": {"L4"}, - "encoding/gob": {"L4", "OS", "encoding"}, - "encoding/hex": {"L4"}, - "encoding/json": {"L4", "encoding"}, - "encoding/pem": {"L4"}, - "encoding/xml": {"L4", "encoding"}, - "flag": {"L4", "OS"}, - "go/build": {"L4", "OS", "GOPARSER"}, - "html": {"L4"}, - "image/draw": {"L4", "image/internal/imageutil"}, - "image/gif": {"L4", "compress/lzw", "image/color/palette", "image/draw"}, - "image/internal/imageutil": {"L4"}, - "image/jpeg": {"L4", "image/internal/imageutil"}, - "image/png": {"L4", "compress/zlib"}, - "index/suffixarray": {"L4", "regexp"}, - "internal/singleflight": {"sync"}, - "internal/trace": {"L4", "OS"}, - "internal/pprof/profile": {"L4", "OS", "compress/gzip", "regexp"}, - "math/big": {"L4"}, - "mime": {"L4", "OS", "syscall", "internal/syscall/windows/registry"}, - "mime/quotedprintable": {"L4"}, - "net/internal/socktest": {"L4", "OS", "syscall"}, - "net/url": {"L4"}, - "plugin": {"L0", "OS", "CGO"}, - "testing/internal/testdeps": {"L4", "runtime/pprof", "regexp"}, - "text/scanner": {"L4", "OS"}, - "text/template/parse": {"L4"}, + "archive/tar": {"L4", "OS", "syscall"}, + "archive/zip": {"L4", "OS", "compress/flate"}, + "container/heap": {"sort"}, + "compress/bzip2": {"L4"}, + "compress/flate": {"L4"}, + "compress/gzip": {"L4", "compress/flate"}, + "compress/lzw": {"L4"}, + "compress/zlib": {"L4", "compress/flate"}, + "context": {"errors", "fmt", "reflect", "sync", "time"}, + "database/sql": {"L4", "container/list", "context", "database/sql/driver", "database/sql/internal"}, + "database/sql/driver": {"L4", "context", "time", "database/sql/internal"}, + "debug/dwarf": {"L4"}, + "debug/elf": {"L4", "OS", "debug/dwarf", "compress/zlib"}, + "debug/gosym": {"L4"}, + "debug/macho": {"L4", "OS", "debug/dwarf"}, + "debug/pe": {"L4", "OS", "debug/dwarf"}, + "debug/plan9obj": {"L4", "OS"}, + "encoding": {"L4"}, + "encoding/ascii85": {"L4"}, + "encoding/asn1": {"L4", "math/big"}, + "encoding/csv": {"L4"}, + "encoding/gob": {"L4", "OS", "encoding"}, + "encoding/hex": {"L4"}, + "encoding/json": {"L4", "encoding"}, + "encoding/pem": {"L4"}, + "encoding/xml": {"L4", "encoding"}, + "flag": {"L4", "OS"}, + "go/build": {"L4", "OS", "GOPARSER"}, + "html": {"L4"}, + "image/draw": {"L4", "image/internal/imageutil"}, + "image/gif": {"L4", "compress/lzw", "image/color/palette", "image/draw"}, + "image/internal/imageutil": {"L4"}, + "image/jpeg": {"L4", "image/internal/imageutil"}, + "image/png": {"L4", "compress/zlib"}, + "index/suffixarray": {"L4", "regexp"}, + "internal/singleflight": {"sync"}, + "internal/trace": {"L4", "OS"}, + "math/big": {"L4"}, + "mime": {"L4", "OS", "syscall", "internal/syscall/windows/registry"}, + "mime/quotedprintable": {"L4"}, + "net/internal/socktest": {"L4", "OS", "syscall"}, + "net/url": {"L4"}, + "plugin": {"L0", "OS", "CGO"}, + "runtime/pprof/internal/profile": {"L4", "OS", "compress/gzip", "regexp"}, + "testing/internal/testdeps": {"L4", "runtime/pprof", "regexp"}, + "text/scanner": {"L4", "OS"}, + "text/template/parse": {"L4"}, "html/template": { "L4", "OS", "encoding/json", "html", "text/template", @@ -299,8 +304,8 @@ var pkgDeps = map[string][]string{ // do networking portably, it must have a small dependency set: just L0+basic os. "net": { "L0", "CGO", - "context", "math/rand", "os", "sort", "syscall", "time", - "internal/nettrace", + "context", "math/rand", "os", "reflect", "sort", "syscall", "time", + "internal/nettrace", "internal/poll", "internal/syscall/windows", "internal/singleflight", "internal/race", "golang_org/x/net/lif", "golang_org/x/net/route", }, @@ -372,7 +377,7 @@ var pkgDeps = map[string][]string{ }, "crypto/x509": { "L4", "CRYPTO-MATH", "OS", "CGO", - "crypto/x509/pkix", "encoding/pem", "encoding/hex", "net", "syscall", + "crypto/x509/pkix", "encoding/pem", "encoding/hex", "net", "os/user", "syscall", }, "crypto/x509/pkix": {"L4", "CRYPTO-MATH"}, @@ -391,6 +396,7 @@ var pkgDeps = map[string][]string{ "golang_org/x/net/http2/hpack", "golang_org/x/net/idna", "golang_org/x/net/lex/httplex", + "golang_org/x/net/proxy", "golang_org/x/text/unicode/norm", "golang_org/x/text/width", "internal/nettrace", @@ -406,8 +412,8 @@ var pkgDeps = map[string][]string{ "expvar": {"L4", "OS", "encoding/json", "net/http"}, "net/http/cgi": {"L4", "NET", "OS", "crypto/tls", "net/http", "regexp"}, "net/http/cookiejar": {"L4", "NET", "net/http"}, - "net/http/fcgi": {"L4", "NET", "OS", "net/http", "net/http/cgi"}, - "net/http/httptest": {"L4", "NET", "OS", "crypto/tls", "flag", "net/http", "net/http/internal"}, + "net/http/fcgi": {"L4", "NET", "OS", "context", "net/http", "net/http/cgi"}, + "net/http/httptest": {"L4", "NET", "OS", "crypto/tls", "flag", "net/http", "net/http/internal", "crypto/x509"}, "net/http/httputil": {"L4", "NET", "OS", "context", "net/http", "net/http/internal"}, "net/http/pprof": {"L4", "OS", "html/template", "net/http", "runtime/pprof", "runtime/trace"}, "net/rpc": {"L4", "NET", "encoding/gob", "html/template", "net/http"}, diff --git a/libgo/go/go/build/doc.go b/libgo/go/go/build/doc.go index 979d0477dfb..422e1a5ffd2 100644 --- a/libgo/go/go/build/doc.go +++ b/libgo/go/go/build/doc.go @@ -105,6 +105,7 @@ // - "go1.6", from Go version 1.6 onward // - "go1.7", from Go version 1.7 onward // - "go1.8", from Go version 1.8 onward +// - "go1.9", from Go version 1.9 onward // - any additional words listed in ctxt.BuildTags // // If a file's name, after stripping the extension and a possible _test suffix, diff --git a/libgo/go/go/constant/value.go b/libgo/go/go/constant/value.go index 7c32473c616..5474e73e24b 100644 --- a/libgo/go/go/constant/value.go +++ b/libgo/go/go/constant/value.go @@ -205,13 +205,8 @@ func rtof(x ratVal) floatVal { func vtoc(x Value) complexVal { return complexVal{x, int64Val(0)} } -var ( - minInt64 = big.NewInt(-1 << 63) - maxInt64 = big.NewInt(1<<63 - 1) -) - func makeInt(x *big.Int) Value { - if minInt64.Cmp(x) <= 0 && x.Cmp(maxInt64) <= 0 { + if x.IsInt64() { return int64Val(x.Int64()) } return intVal{x} @@ -252,6 +247,13 @@ func makeFloatFromLiteral(lit string) Value { if f, ok := newFloat().SetString(lit); ok { if smallRat(f) { // ok to use rationals + if f.Sign() == 0 { + // Issue 20228: If the float underflowed to zero, parse just "0". + // Otherwise, lit might contain a value with a large negative exponent, + // such as -6e-1886451601. As a float, that will underflow to 0, + // but it'll take forever to parse as a Rat. + lit = "0" + } r, _ := newRat().SetString(lit) return ratVal{r} } @@ -413,7 +415,7 @@ func Uint64Val(x Value) (uint64, bool) { case int64Val: return uint64(x), x >= 0 case intVal: - return x.val.Uint64(), x.val.Sign() >= 0 && x.val.BitLen() <= 64 + return x.val.Uint64(), x.val.IsUint64() case unknownVal: return 0, false default: diff --git a/libgo/go/go/constant/value_test.go b/libgo/go/go/constant/value_test.go index 8a8a08eaaa1..954a0e05db3 100644 --- a/libgo/go/go/constant/value_test.go +++ b/libgo/go/go/constant/value_test.go @@ -244,7 +244,8 @@ var stringTests = []struct { {"1e9999", "1e+9999", "0x.f8d4a9da224650a8cb2959e10d985ad92adbd44c62917e608b1f24c0e1b76b6f61edffeb15c135a4b601637315f7662f325f82325422b244286a07663c9415d2p+33216"}, {"1e-9999", "1e-9999", "0x.83b01ba6d8c0425eec1b21e96f7742d63c2653ed0a024cf8a2f9686df578d7b07d7a83d84df6a2ec70a921d1f6cd5574893a7eda4d28ee719e13a5dce2700759p-33215"}, {"2.71828182845904523536028747135266249775724709369995957496696763", "2.71828", "271828182845904523536028747135266249775724709369995957496696763/100000000000000000000000000000000000000000000000000000000000000"}, - {"0e9999999999", "0", "0"}, // issue #16176 + {"0e9999999999", "0", "0"}, // issue #16176 + {"-6e-1886451601", "0", "0"}, // issue #20228 // Complex {"0i", "(0 + 0i)", "(0 + 0i)"}, diff --git a/libgo/go/go/doc/comment.go b/libgo/go/go/doc/comment.go index 15e034b6df8..4228e8cd9c5 100644 --- a/libgo/go/go/doc/comment.go +++ b/libgo/go/go/doc/comment.go @@ -48,12 +48,19 @@ const ( identRx = `[\pL_][\pL_0-9]*` // Regexp for URLs - protocol = `https?|ftp|file|gopher|mailto|news|nntp|telnet|wais|prospero` - hostPart = `[a-zA-Z0-9_@\-]+` - filePart = `[a-zA-Z0-9_?%#~&/\-+=()]+` // parentheses may not be matching; see pairedParensPrefixLen - urlRx = `(` + protocol + `)://` + // http:// - hostPart + `([.:]` + hostPart + `)*/?` + // //www.google.com:8080/ - filePart + `([:.,;]` + filePart + `)*` + // Match parens, and check in pairedParensPrefixLen for balance - see #5043 + // Match .,:;?! within path, but not at end - see #18139, #16565 + // This excludes some rare yet valid urls ending in common punctuation + // in order to allow sentences ending in URLs. + + // protocol (required) e.g. http + protoPart = `(https?|ftp|file|gopher|mailto|nntp)` + // host (required) e.g. www.example.com or [::1]:8080 + hostPart = `([a-zA-Z0-9_@\-.\[\]:]+)` + // path+query+fragment (optional) e.g. /path/index.html?q=foo#bar + pathPart = `([.,:;?!]*[a-zA-Z0-9$'()*+&#=@~_/\-\[\]%])*` + + urlRx = protoPart + `://` + hostPart + pathPart ) var matchRx = regexp.MustCompile(`(` + urlRx + `)|(` + identRx + `)`) diff --git a/libgo/go/go/doc/comment_test.go b/libgo/go/go/doc/comment_test.go index 76dfbeac796..0523ab899ee 100644 --- a/libgo/go/go/doc/comment_test.go +++ b/libgo/go/go/doc/comment_test.go @@ -150,6 +150,12 @@ func TestToText(t *testing.T) { var emphasizeTests = []struct { in, out string }{ + {"http://[::1]:8080/foo.txt", `http://[::1]:8080/foo.txt`}, + {"before (https://www.google.com) after", `before (https://www.google.com) after`}, + {"before https://www.google.com:30/x/y/z:b::c. After", `before https://www.google.com:30/x/y/z:b::c. After`}, + {"http://www.google.com/path/:;!-/?query=%34b#093124", `http://www.google.com/path/:;!-/?query=%34b#093124`}, + {"http://www.google.com/path/:;!-/?query=%34bar#093124", `http://www.google.com/path/:;!-/?query=%34bar#093124`}, + {"http://www.google.com/index.html! After", `http://www.google.com/index.html! After`}, {"http://www.google.com/", `http://www.google.com/`}, {"https://www.google.com/", `https://www.google.com/`}, {"http://www.google.com/path.", `http://www.google.com/path.`}, diff --git a/libgo/go/go/doc/doc_test.go b/libgo/go/go/doc/doc_test.go index 82e63100d47..ad8ba5378f3 100644 --- a/libgo/go/go/doc/doc_test.go +++ b/libgo/go/go/doc/doc_test.go @@ -25,7 +25,7 @@ var files = flag.String("files", "", "consider only Go test files matching this const dataDir = "testdata" -var templateTxt *template.Template +var templateTxt = readTemplate("template.txt") func readTemplate(filename string) *template.Template { t := template.New(filename) @@ -96,9 +96,6 @@ func test(t *testing.T, mode Mode) { if err != nil { t.Fatal(err) } - if templateTxt == nil { - templateTxt = readTemplate("template.txt") - } // test packages for _, pkg := range pkgs { diff --git a/libgo/go/go/doc/exports.go b/libgo/go/go/doc/exports.go index 4a12b1e55f8..da9ea1f027d 100644 --- a/libgo/go/go/doc/exports.go +++ b/libgo/go/go/doc/exports.go @@ -150,7 +150,7 @@ func (r *reader) filterType(parent *namedType, typ ast.Expr) { } } -func (r *reader) filterSpec(spec ast.Spec, tok token.Token) bool { +func (r *reader) filterSpec(spec ast.Spec) bool { switch s := spec.(type) { case *ast.ImportSpec: // always keep imports so we can collect them @@ -215,7 +215,7 @@ func (r *reader) filterSpecList(list []ast.Spec, tok token.Token) []ast.Spec { j := 0 for _, s := range list { - if r.filterSpec(s, tok) { + if r.filterSpec(s) { list[j] = s j++ } diff --git a/libgo/go/go/format/internal.go b/libgo/go/go/format/internal.go index b8b470da8bb..4918681de57 100644 --- a/libgo/go/go/format/internal.go +++ b/libgo/go/go/format/internal.go @@ -37,14 +37,14 @@ func parse(fset *token.FileSet, filename string, src []byte, fragmentOk bool) ( // If this is a declaration list, make it a source file // by inserting a package clause. - // Insert using a ;, not a newline, so that the line numbers + // Insert using a ';', not a newline, so that the line numbers // in psrc match the ones in src. psrc := append([]byte("package p;"), src...) file, err = parser.ParseFile(fset, filename, psrc, parserMode) if err == nil { sourceAdj = func(src []byte, indent int) []byte { // Remove the package clause. - // Gofmt has turned the ; into a \n. + // Gofmt has turned the ';' into a '\n'. src = src[indent+len("package p\n"):] return bytes.TrimSpace(src) } @@ -60,7 +60,7 @@ func parse(fset *token.FileSet, filename string, src []byte, fragmentOk bool) ( // If this is a statement list, make it a source file // by inserting a package clause and turning the list // into a function body. This handles expressions too. - // Insert using a ;, not a newline, so that the line numbers + // Insert using a ';', not a newline, so that the line numbers // in fsrc match the ones in src. Add an extra '\n' before the '}' // to make sure comments are flushed before the '}'. fsrc := append(append([]byte("package p; func _() {"), src...), '\n', '\n', '}') @@ -72,7 +72,7 @@ func parse(fset *token.FileSet, filename string, src []byte, fragmentOk bool) ( indent = 0 } // Remove the wrapping. - // Gofmt has turned the ; into a \n\n. + // Gofmt has turned the ';' into a '\n'. // There will be two non-blank lines with indent, hence 2*indent. src = src[2*indent+len("package p\n\nfunc _() {"):] // Remove only the "}\n" suffix: remaining whitespaces will be trimmed anyway diff --git a/libgo/go/go/importer/importer.go b/libgo/go/go/importer/importer.go index f655bc1e924..fab65181cd8 100644 --- a/libgo/go/go/importer/importer.go +++ b/libgo/go/go/importer/importer.go @@ -6,8 +6,11 @@ package importer import ( + "go/build" "go/internal/gccgoimporter" "go/internal/gcimporter" + "go/internal/srcimporter" + "go/token" "go/types" "io" "runtime" @@ -17,22 +20,30 @@ import ( // a given import path, or an error if no matching package is found. type Lookup func(path string) (io.ReadCloser, error) -// For returns an Importer for the given compiler and lookup interface, -// or nil. Supported compilers are "gc", and "gccgo". If lookup is nil, -// the default package lookup mechanism for the given compiler is used. +// For returns an Importer for importing from installed packages +// for the compilers "gc" and "gccgo", or for importing directly +// from the source if the compiler argument is "source". In this +// latter case, importing may fail under circumstances where the +// exported API is not entirely defined in pure Go source code +// (if the package API depends on cgo-defined entities, the type +// checker won't have access to those). +// +// If lookup is nil, the default package lookup mechanism for the +// given compiler is used. +// // BUG(issue13847): For does not support non-nil lookup functions. func For(compiler string, lookup Lookup) types.Importer { switch compiler { case "gc": if lookup != nil { - panic("gc importer for custom import path lookup not yet implemented") + panic("gc importer for custom import path lookup not supported (issue #13847).") } return make(gcimports) case "gccgo": if lookup != nil { - panic("gccgo importer for custom import path lookup not yet implemented") + panic("gccgo importer for custom import path lookup not supported (issue #13847).") } var inst gccgoimporter.GccgoInstallation @@ -43,6 +54,13 @@ func For(compiler string, lookup Lookup) types.Importer { packages: make(map[string]*types.Package), importer: inst.GetImporter(nil, nil), } + + case "source": + if lookup != nil { + panic("source importer for custom import path lookup not supported (issue #13847).") + } + + return srcimporter.New(&build.Default, token.NewFileSet(), make(map[string]*types.Package)) } // compiler not supported @@ -55,7 +73,7 @@ func Default() types.Importer { return For(runtime.Compiler, nil) } -// gc support +// gc importer type gcimports map[string]*types.Package @@ -70,7 +88,7 @@ func (m gcimports) ImportFrom(path, srcDir string, mode types.ImportMode) (*type return gcimporter.Import(m, path, srcDir) } -// gccgo support +// gccgo importer type gccgoimports struct { packages map[string]*types.Package diff --git a/libgo/go/go/internal/gccgoimporter/importer_test.go b/libgo/go/go/internal/gccgoimporter/importer_test.go index 2b454701beb..4fca828bf60 100644 --- a/libgo/go/go/internal/gccgoimporter/importer_test.go +++ b/libgo/go/go/internal/gccgoimporter/importer_test.go @@ -101,6 +101,7 @@ var importerTests = [...]importerTest{ {pkgpath: "unicode", name: "IsUpper", want: "func IsUpper(r rune) bool"}, {pkgpath: "unicode", name: "MaxRune", want: "const MaxRune untyped rune", wantval: "1114111"}, {pkgpath: "imports", wantinits: []string{"imports..import", "fmt..import", "math..import"}}, + {pkgpath: "alias", name: "IntAlias2", want: "type IntAlias2 = Int"}, } func TestGoxImporter(t *testing.T) { diff --git a/libgo/go/go/internal/gccgoimporter/parser.go b/libgo/go/go/internal/gccgoimporter/parser.go index 3b97c96d433..8a1ad5ff07f 100644 --- a/libgo/go/go/internal/gccgoimporter/parser.go +++ b/libgo/go/go/internal/gccgoimporter/parser.go @@ -370,27 +370,41 @@ func (p *parser) parseConst(pkg *types.Package) *types.Const { return types.NewConst(token.NoPos, pkg, name, typ, val) } -// TypeName = ExportedName . -func (p *parser) parseTypeName() *types.TypeName { +// NamedType = TypeName [ "=" ] Type { Method } . +// TypeName = ExportedName . +// Method = "func" "(" Param ")" Name ParamList ResultList ";" . +func (p *parser) parseNamedType(n int) types.Type { pkg, name := p.parseExportedName() scope := pkg.Scope() - if obj := scope.Lookup(name); obj != nil { - return obj.(*types.TypeName) + + if p.tok == '=' { + // type alias + p.next() + typ := p.parseType(pkg) + if obj := scope.Lookup(name); obj != nil { + typ = obj.Type() // use previously imported type + if typ == nil { + p.errorf("%v (type alias) used in cycle", obj) + } + } else { + obj = types.NewTypeName(token.NoPos, pkg, name, typ) + scope.Insert(obj) + } + p.typeMap[n] = typ + return typ } - obj := types.NewTypeName(token.NoPos, pkg, name, nil) - // a named type may be referred to before the underlying type - // is known - set it up - types.NewNamed(obj, nil, nil) - scope.Insert(obj) - return obj -} -// NamedType = TypeName Type { Method } . -// Method = "func" "(" Param ")" Name ParamList ResultList ";" . -func (p *parser) parseNamedType(n int) types.Type { - obj := p.parseTypeName() + // named type + obj := scope.Lookup(name) + if obj == nil { + // a named type may be referred to before the underlying type + // is known - set it up + tname := types.NewTypeName(token.NoPos, pkg, name, nil) + types.NewNamed(tname, nil, nil) + scope.Insert(tname) + obj = tname + } - pkg := obj.Pkg() typ := obj.Type() p.typeMap[n] = typ @@ -409,8 +423,8 @@ func (p *parser) parseNamedType(n int) types.Type { nt.SetUnderlying(underlying.Underlying()) } + // collect associated methods for p.tok == scanner.Ident { - // collect associated methods p.expectKeyword("func") p.expect('(') receiver, _ := p.parseParam(pkg) @@ -725,7 +739,7 @@ func (p *parser) discardDirectiveWhileParsingTypes(pkg *types.Package) { case ';': return case '<': - p.parseType(p.pkg) + p.parseType(pkg) case scanner.EOF: p.error("unexpected EOF") default: diff --git a/libgo/go/go/internal/gcimporter/bimport.go b/libgo/go/go/internal/gcimporter/bimport.go index a8f349052ac..2045f5517b1 100644 --- a/libgo/go/go/internal/gcimporter/bimport.go +++ b/libgo/go/go/internal/gcimporter/bimport.go @@ -19,16 +19,18 @@ import ( ) type importer struct { - imports map[string]*types.Package - data []byte - path string - buf []byte // for reading strings - version int // export format version + imports map[string]*types.Package + data []byte + importpath string + buf []byte // for reading strings + version int // export format version // object lists - strList []string // in order of appearance - pkgList []*types.Package // in order of appearance - typList []types.Type // in order of appearance + strList []string // in order of appearance + pathList []string // in order of appearance + pkgList []*types.Package // in order of appearance + typList []types.Type // in order of appearance + interfaceList []*types.Interface // for delayed completion only trackAllTypes bool // position encoding @@ -47,24 +49,26 @@ type importer struct { // and returns the number of bytes consumed and a reference to the package. // If the export data version is not recognized or the format is otherwise // compromised, an error is returned. -func BImportData(fset *token.FileSet, imports map[string]*types.Package, data []byte, path string) (_ int, _ *types.Package, err error) { +func BImportData(fset *token.FileSet, imports map[string]*types.Package, data []byte, path string) (_ int, pkg *types.Package, err error) { // catch panics and return them as errors defer func() { if e := recover(); e != nil { // The package (filename) causing the problem is added to this // error by a wrapper in the caller (Import in gcimporter.go). + // Return a (possibly nil or incomplete) package unchanged (see #16088). err = fmt.Errorf("cannot import, possibly version skew (%v) - reinstall package", e) } }() p := importer{ - imports: imports, - data: data, - path: path, - version: -1, // unknown version - strList: []string{""}, // empty string is mapped to 0 - fset: fset, - files: make(map[string]*token.File), + imports: imports, + data: data, + importpath: path, + version: -1, // unknown version + strList: []string{""}, // empty string is mapped to 0 + pathList: []string{""}, // empty string is mapped to 0 + fset: fset, + files: make(map[string]*token.File), } // read version info @@ -98,10 +102,10 @@ func BImportData(fset *token.FileSet, imports map[string]*types.Package, data [] // read version specific flags - extend as necessary switch p.version { - // case 4: + // case 6: // ... // fallthrough - case 3, 2, 1: + case 5, 4, 3, 2, 1: p.debugFormat = p.rawStringln(p.rawByte()) == "debug" p.trackAllTypes = p.int() != 0 p.posInfoFormat = p.int() != 0 @@ -117,7 +121,7 @@ func BImportData(fset *token.FileSet, imports map[string]*types.Package, data [] p.typList = append(p.typList, predeclared...) // read package data - pkg := p.pkg() + pkg = p.pkg() // read objects of phase 1 only (see cmd/compiler/internal/gc/bexport.go) objcount := 0 @@ -138,15 +142,9 @@ func BImportData(fset *token.FileSet, imports map[string]*types.Package, data [] // ignore compiler-specific import data // complete interfaces - for _, typ := range p.typList { - // If we only record named types (!p.trackAllTypes), - // we must check the underlying types here. If we - // track all types, the Underlying() method call is - // not needed. - // TODO(gri) Remove if p.trackAllTypes is gone. - if it, ok := typ.Underlying().(*types.Interface); ok { - it.Complete() - } + // TODO(gri) re-investigate if we still need to do this in a delayed fashion + for _, typ := range p.interfaceList { + typ.Complete() } // record all referenced packages as imports @@ -173,12 +171,17 @@ func (p *importer) pkg() *types.Package { // otherwise, i is the package tag (< 0) if i != packageTag { - errorf("unexpected package tag %d", i) + errorf("unexpected package tag %d version %d", i, p.version) } // read package data name := p.string() - path := p.string() + var path string + if p.version >= 5 { + path = p.path() + } else { + path = p.string() + } // we should never see an empty package name if name == "" { @@ -193,7 +196,7 @@ func (p *importer) pkg() *types.Package { // if the package was imported before, use that one; otherwise create a new one if path == "" { - path = p.path + path = p.importpath } pkg := p.imports[path] if pkg == nil { @@ -208,7 +211,6 @@ func (p *importer) pkg() *types.Package { } // objTag returns the tag value for each object kind. -// obj must not be a *types.Alias. func objTag(obj types.Object) int { switch obj.(type) { case *types.Const: @@ -219,7 +221,6 @@ func objTag(obj types.Object) int { return varTag case *types.Func: return funcTag - // Aliases are not exported multiple times, thus we should not see them here. default: errorf("unexpected object: %v (%T)", obj, obj) // panics panic("unreachable") @@ -237,14 +238,14 @@ func (p *importer) declare(obj types.Object) { pkg := obj.Pkg() if alt := pkg.Scope().Insert(obj); alt != nil { // This can only trigger if we import a (non-type) object a second time. - // Excluding aliases, this cannot happen because 1) we only import a package + // Excluding type aliases, this cannot happen because 1) we only import a package // once; and b) we ignore compiler-specific export data which may contain // functions whose inlined function bodies refer to other functions that // were already imported. - // However, aliases require reexporting the original object, so we need + // However, type aliases require reexporting the original type, so we need // to allow it (see also the comment in cmd/compile/internal/gc/bimport.go, // method importer.obj, switch case importing functions). - // Note that the original itself cannot be an alias. + // TODO(gri) review/update this comment once the gc compiler handles type aliases. if !sameObj(obj, alt) { errorf("inconsistent import:\n\t%v\npreviously imported as:\n\t%v\n", obj, alt) } @@ -260,6 +261,13 @@ func (p *importer) obj(tag int) { val := p.value() p.declare(types.NewConst(pos, pkg, name, typ, val)) + case aliasTag: + // TODO(gri) verify type alias hookup is correct + pos := p.pos() + pkg, name := p.qualifiedName() + typ := p.typ(nil) + p.declare(types.NewTypeName(pos, pkg, name, typ)) + case typeTag: p.typ(nil) @@ -277,24 +285,13 @@ func (p *importer) obj(tag int) { sig := types.NewSignature(nil, params, result, isddd) p.declare(types.NewFunc(pos, pkg, name, sig)) - case aliasTag: - pos := p.pos() - name := p.string() - var orig types.Object - if pkg, name := p.qualifiedName(); pkg != nil { - orig = pkg.Scope().Lookup(name) - } - // Alias-related code. Keep for now. - _ = pos - _ = name - _ = orig - // p.declare(types.NewAlias(pos, p.pkgList[0], name, orig)) - default: errorf("unexpected object tag %d", tag) } } +const deltaNewFile = -64 // see cmd/compile/internal/gc/bexport.go + func (p *importer) pos() token.Pos { if !p.posInfoFormat { return token.NoPos @@ -302,15 +299,26 @@ func (p *importer) pos() token.Pos { file := p.prevFile line := p.prevLine - if delta := p.int(); delta != 0 { - // line changed - line += delta - } else if n := p.int(); n >= 0 { - // file changed - file = p.prevFile[:n] + p.string() - p.prevFile = file - line = p.int() + delta := p.int() + line += delta + if p.version >= 5 { + if delta == deltaNewFile { + if n := p.int(); n >= 0 { + // file changed + file = p.path() + line = n + } + } + } else { + if delta == 0 { + if n := p.int(); n >= 0 { + // file changed + file = p.prevFile[:n] + p.string() + line = p.int() + } + } } + p.prevFile = file p.prevLine = line // Synthesize a token.Pos @@ -349,9 +357,7 @@ var ( func (p *importer) qualifiedName() (pkg *types.Package, name string) { name = p.string() - if name != "" { - pkg = p.pkg() - } + pkg = p.pkg() return } @@ -501,12 +507,14 @@ func (p *importer) typ(parent *types.Package) types.Type { p.record(nil) } - // no embedded interfaces with gc compiler - if p.int() != 0 { - errorf("unexpected embedded interface") + var embeddeds []*types.Named + for n := p.int(); n > 0; n-- { + p.pos() + embeddeds = append(embeddeds, p.typ(parent).(*types.Named)) } - t := types.NewInterface(p.methodList(parent), nil) + t := types.NewInterface(p.methodList(parent), embeddeds) + p.interfaceList = append(p.interfaceList, t) if p.trackAllTypes { p.typList[n] = t } @@ -556,17 +564,17 @@ func (p *importer) fieldList(parent *types.Package) (fields []*types.Var, tags [ fields = make([]*types.Var, n) tags = make([]string, n) for i := range fields { - fields[i] = p.field(parent) - tags[i] = p.string() + fields[i], tags[i] = p.field(parent) } } return } -func (p *importer) field(parent *types.Package) *types.Var { +func (p *importer) field(parent *types.Package) (*types.Var, string) { pos := p.pos() - pkg, name := p.fieldName(parent) + pkg, name, alias := p.fieldName(parent) typ := p.typ(parent) + tag := p.string() anonymous := false if name == "" { @@ -578,12 +586,15 @@ func (p *importer) field(parent *types.Package) *types.Var { case *types.Named: name = typ.Obj().Name() default: - errorf("anonymous field expected") + errorf("named base type expected") } anonymous = true + } else if alias { + // anonymous field: we have an explicit name because it's an alias + anonymous = true } - return types.NewField(pos, pkg, name, typ, anonymous) + return types.NewField(pos, pkg, name, typ, anonymous), tag } func (p *importer) methodList(parent *types.Package) (methods []*types.Func) { @@ -598,31 +609,42 @@ func (p *importer) methodList(parent *types.Package) (methods []*types.Func) { func (p *importer) method(parent *types.Package) *types.Func { pos := p.pos() - pkg, name := p.fieldName(parent) + pkg, name, _ := p.fieldName(parent) params, isddd := p.paramList() result, _ := p.paramList() sig := types.NewSignature(nil, params, result, isddd) return types.NewFunc(pos, pkg, name, sig) } -func (p *importer) fieldName(parent *types.Package) (*types.Package, string) { - name := p.string() - pkg := parent +func (p *importer) fieldName(parent *types.Package) (pkg *types.Package, name string, alias bool) { + name = p.string() + pkg = parent if pkg == nil { // use the imported package instead pkg = p.pkgList[0] } if p.version == 0 && name == "_" { // version 0 didn't export a package for _ fields - return pkg, name - } - if name != "" && !exported(name) { - if name == "?" { - name = "" - } + return + } + switch name { + case "": + // 1) field name matches base type name and is exported: nothing to do + case "?": + // 2) field name matches base type name and is not exported: need package + name = "" pkg = p.pkg() + case "@": + // 3) field name doesn't match type name (alias) + name = p.string() + alias = true + fallthrough + default: + if !exported(name) { + pkg = p.pkg() + } } - return pkg, name + return } func (p *importer) paramList() (*types.Tuple, bool) { @@ -774,6 +796,26 @@ func (p *importer) int64() int64 { return p.rawInt64() } +func (p *importer) path() string { + if p.debugFormat { + p.marker('p') + } + // if the path was seen before, i is its index (>= 0) + // (the empty string is at index 0) + i := p.rawInt64() + if i >= 0 { + return p.pathList[i] + } + // otherwise, i is the negative path length (< 0) + a := make([]string, -i) + for n := range a { + a[n] = p.string() + } + s := strings.Join(a, "/") + p.pathList = append(p.pathList, s) + return s +} + func (p *importer) string() string { if p.debugFormat { p.marker('s') @@ -893,7 +935,7 @@ const ( nilTag // only used by gc (appears in exported inlined function bodies) unknownTag // not used by gc (only appears in packages with errors) - // Aliases + // Type aliases aliasTag ) @@ -917,7 +959,7 @@ var predeclared = []types.Type{ types.Typ[types.Complex128], types.Typ[types.String], - // aliases + // basic type aliases types.Universe.Lookup("byte").Type(), types.Universe.Lookup("rune").Type(), diff --git a/libgo/go/go/internal/gcimporter/gcimporter.go b/libgo/go/go/internal/gcimporter/gcimporter.go index f99f0f8befe..f3f90f25911 100644 --- a/libgo/go/go/internal/gcimporter/gcimporter.go +++ b/libgo/go/go/internal/gcimporter/gcimporter.go @@ -43,6 +43,7 @@ func FindPkg(path, srcDir string) (filename, id string) { } bp, _ := build.Import(path, srcDir, build.FindOnly|build.AllowBinary) if bp.PkgObj == "" { + id = path // make sure we have an id to print in error message return } noext = strings.TrimSuffix(bp.PkgObj, ".a") @@ -89,7 +90,7 @@ func Import(packages map[string]*types.Package, path, srcDir string) (pkg *types if path == "unsafe" { return types.Unsafe, nil } - err = fmt.Errorf("can't find import: %s", id) + err = fmt.Errorf("can't find import: %q", id) return } diff --git a/libgo/go/go/internal/gcimporter/gcimporter_test.go b/libgo/go/go/internal/gcimporter/gcimporter_test.go index a0697faeb68..c34f07c4c35 100644 --- a/libgo/go/go/internal/gcimporter/gcimporter_test.go +++ b/libgo/go/go/internal/gcimporter/gcimporter_test.go @@ -92,7 +92,6 @@ func TestImportTestdata(t *testing.T) { // This package only handles gc export data. if runtime.Compiler != "gc" { t.Skipf("gc-built packages not available (compiler = %s)", runtime.Compiler) - return } if outFn := compile(t, "testdata", "exports.go"); outFn != "" { @@ -124,7 +123,6 @@ func TestVersionHandling(t *testing.T) { // This package only handles gc export data. if runtime.Compiler != "gc" { t.Skipf("gc-built packages not available (compiler = %s)", runtime.Compiler) - return } const dir = "./testdata/versions" @@ -188,7 +186,6 @@ func TestImportStdLib(t *testing.T) { // This package only handles gc export data. if runtime.Compiler != "gc" { t.Skipf("gc-built packages not available (compiler = %s)", runtime.Compiler) - return } dt := maxTime @@ -205,7 +202,7 @@ var importedObjectTests = []struct { }{ {"math.Pi", "const Pi untyped float"}, {"io.Reader", "type Reader interface{Read(p []byte) (n int, err error)}"}, - {"io.ReadWriter", "type ReadWriter interface{Read(p []byte) (n int, err error); Write(p []byte) (n int, err error)}"}, + {"io.ReadWriter", "type ReadWriter interface{Reader; Writer}"}, {"math.Sin", "func Sin(x float64) float64"}, // TODO(gri) add more tests } @@ -216,7 +213,6 @@ func TestImportedTypes(t *testing.T) { // This package only handles gc export data. if runtime.Compiler != "gc" { t.Skipf("gc-built packages not available (compiler = %s)", runtime.Compiler) - return } for _, test := range importedObjectTests { @@ -252,13 +248,9 @@ func TestIssue5815(t *testing.T) { // This package only handles gc export data. if runtime.Compiler != "gc" { t.Skipf("gc-built packages not available (compiler = %s)", runtime.Compiler) - return } - pkg, err := Import(make(map[string]*types.Package), "strings", ".") - if err != nil { - t.Fatal(err) - } + pkg := importPkg(t, "strings") scope := pkg.Scope() for _, name := range scope.Names() { @@ -285,7 +277,6 @@ func TestCorrectMethodPackage(t *testing.T) { // This package only handles gc export data. if runtime.Compiler != "gc" { t.Skipf("gc-built packages not available (compiler = %s)", runtime.Compiler) - return } imports := make(map[string]*types.Package) @@ -309,7 +300,6 @@ func TestIssue13566(t *testing.T) { // This package only handles gc export data. if runtime.Compiler != "gc" { t.Skipf("gc-built packages not available (compiler = %s)", runtime.Compiler) - return } // On windows, we have to set the -D option for the compiler to avoid having a drive @@ -326,10 +316,7 @@ func TestIssue13566(t *testing.T) { } // import must succeed (test for issue at hand) - pkg, err := Import(make(map[string]*types.Package), "./testdata/b", ".") - if err != nil { - t.Fatal(err) - } + pkg := importPkg(t, "./testdata/b") // make sure all indirectly imported packages have names for _, imp := range pkg.Imports() { @@ -345,7 +332,6 @@ func TestIssue13898(t *testing.T) { // This package only handles gc export data. if runtime.Compiler != "gc" { t.Skipf("gc-built packages not available (compiler = %s)", runtime.Compiler) - return } // import go/internal/gcimporter which imports go/types partially @@ -368,10 +354,7 @@ func TestIssue13898(t *testing.T) { } // look for go/types.Object type - obj := goTypesPkg.Scope().Lookup("Object") - if obj == nil { - t.Fatal("go/types.Object not found") - } + obj := lookupObj(t, goTypesPkg.Scope(), "Object") typ, ok := obj.Type().(*types.Named) if !ok { t.Fatalf("go/types.Object type is %v; wanted named type", typ) @@ -395,7 +378,6 @@ func TestIssue15517(t *testing.T) { // This package only handles gc export data. if runtime.Compiler != "gc" { t.Skipf("gc-built packages not available (compiler = %s)", runtime.Compiler) - return } // On windows, we have to set the -D option for the compiler to avoid having a drive @@ -434,7 +416,6 @@ func TestIssue15920(t *testing.T) { // This package only handles gc export data. if runtime.Compiler != "gc" { t.Skipf("gc-built packages not available (compiler = %s)", runtime.Compiler) - return } // On windows, we have to set the -D option for the compiler to avoid having a drive @@ -447,8 +428,47 @@ func TestIssue15920(t *testing.T) { defer os.Remove(f) } - imports := make(map[string]*types.Package) - if _, err := Import(imports, "./testdata/issue15920", "."); err != nil { + importPkg(t, "./testdata/issue15920") +} + +func TestIssue20046(t *testing.T) { + skipSpecialPlatforms(t) + + // This package only handles gc export data. + if runtime.Compiler != "gc" { + t.Skipf("gc-built packages not available (compiler = %s)", runtime.Compiler) + } + + // On windows, we have to set the -D option for the compiler to avoid having a drive + // letter and an illegal ':' in the import path - just skip it (see also issue #3483). + if runtime.GOOS == "windows" { + t.Skip("avoid dealing with relative paths/drive letters on windows") + } + + if f := compile(t, "testdata", "issue20046.go"); f != "" { + defer os.Remove(f) + } + + // "./issue20046".V.M must exist + pkg := importPkg(t, "./testdata/issue20046") + obj := lookupObj(t, pkg.Scope(), "V") + if m, index, indirect := types.LookupFieldOrMethod(obj.Type(), false, nil, "M"); m == nil { + t.Fatalf("V.M not found (index = %v, indirect = %v)", index, indirect) + } +} + +func importPkg(t *testing.T, path string) *types.Package { + pkg, err := Import(make(map[string]*types.Package), path, ".") + if err != nil { t.Fatal(err) } + return pkg +} + +func lookupObj(t *testing.T, scope *types.Scope, name string) types.Object { + if obj := scope.Lookup(name); obj != nil { + return obj + } + t.Fatalf("%s not found", name) + return nil } diff --git a/libgo/go/go/internal/gcimporter/testdata/issue20046.go b/libgo/go/go/internal/gcimporter/testdata/issue20046.go new file mode 100644 index 00000000000..c63ee821c95 --- /dev/null +++ b/libgo/go/go/internal/gcimporter/testdata/issue20046.go @@ -0,0 +1,9 @@ +// Copyright 2017 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package p + +var V interface { + M() +} diff --git a/libgo/go/go/internal/srcimporter/srcimporter.go b/libgo/go/go/internal/srcimporter/srcimporter.go new file mode 100644 index 00000000000..50cf361dbb0 --- /dev/null +++ b/libgo/go/go/internal/srcimporter/srcimporter.go @@ -0,0 +1,211 @@ +// Copyright 2017 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package srcimporter implements importing directly +// from source files rather than installed packages. +package srcimporter // import "go/internal/srcimporter" + +import ( + "fmt" + "go/ast" + "go/build" + "go/parser" + "go/token" + "go/types" + "path/filepath" + "sync" +) + +// An Importer provides the context for importing packages from source code. +type Importer struct { + ctxt *build.Context + fset *token.FileSet + sizes types.Sizes + packages map[string]*types.Package +} + +// NewImporter returns a new Importer for the given context, file set, and map +// of packages. The context is used to resolve import paths to package paths, +// and identifying the files belonging to the package. If the context provides +// non-nil file system functions, they are used instead of the regular package +// os functions. The file set is used to track position information of package +// files; and imported packages are added to the packages map. +func New(ctxt *build.Context, fset *token.FileSet, packages map[string]*types.Package) *Importer { + return &Importer{ + ctxt: ctxt, + fset: fset, + sizes: types.SizesFor(ctxt.Compiler, ctxt.GOARCH), // uses go/types default if GOARCH not found + packages: packages, + } +} + +// Importing is a sentinel taking the place in Importer.packages +// for a package that is in the process of being imported. +var importing types.Package + +// Import(path) is a shortcut for ImportFrom(path, "", 0). +func (p *Importer) Import(path string) (*types.Package, error) { + return p.ImportFrom(path, "", 0) +} + +// ImportFrom imports the package with the given import path resolved from the given srcDir, +// adds the new package to the set of packages maintained by the importer, and returns the +// package. Package path resolution and file system operations are controlled by the context +// maintained with the importer. The import mode must be zero but is otherwise ignored. +// Packages that are not comprised entirely of pure Go files may fail to import because the +// type checker may not be able to determine all exported entities (e.g. due to cgo dependencies). +func (p *Importer) ImportFrom(path, srcDir string, mode types.ImportMode) (*types.Package, error) { + if mode != 0 { + panic("non-zero import mode") + } + + // determine package path (do vendor resolution) + var bp *build.Package + var err error + switch { + default: + if abs, err := p.absPath(srcDir); err == nil { // see issue #14282 + srcDir = abs + } + bp, err = p.ctxt.Import(path, srcDir, build.FindOnly) + + case build.IsLocalImport(path): + // "./x" -> "srcDir/x" + bp, err = p.ctxt.ImportDir(filepath.Join(srcDir, path), build.FindOnly) + + case p.isAbsPath(path): + return nil, fmt.Errorf("invalid absolute import path %q", path) + } + if err != nil { + return nil, err // err may be *build.NoGoError - return as is + } + + // package unsafe is known to the type checker + if bp.ImportPath == "unsafe" { + return types.Unsafe, nil + } + + // no need to re-import if the package was imported completely before + pkg := p.packages[bp.ImportPath] + if pkg != nil { + if pkg == &importing { + return nil, fmt.Errorf("import cycle through package %q", bp.ImportPath) + } + if !pkg.Complete() { + // Package exists but is not complete - we cannot handle this + // at the moment since the source importer replaces the package + // wholesale rather than augmenting it (see #19337 for details). + // Return incomplete package with error (see #16088). + return pkg, fmt.Errorf("reimported partially imported package %q", bp.ImportPath) + } + return pkg, nil + } + + p.packages[bp.ImportPath] = &importing + defer func() { + // clean up in case of error + // TODO(gri) Eventually we may want to leave a (possibly empty) + // package in the map in all cases (and use that package to + // identify cycles). See also issue 16088. + if p.packages[bp.ImportPath] == &importing { + p.packages[bp.ImportPath] = nil + } + }() + + // collect package files + bp, err = p.ctxt.ImportDir(bp.Dir, 0) + if err != nil { + return nil, err // err may be *build.NoGoError - return as is + } + var filenames []string + filenames = append(filenames, bp.GoFiles...) + filenames = append(filenames, bp.CgoFiles...) + + files, err := p.parseFiles(bp.Dir, filenames) + if err != nil { + return nil, err + } + + // type-check package files + conf := types.Config{ + IgnoreFuncBodies: true, + FakeImportC: true, + Importer: p, + Sizes: p.sizes, + } + pkg, err = conf.Check(bp.ImportPath, p.fset, files, nil) + if err != nil { + // Type-checking stops after the first error (types.Config.Error is not set), + // so the returned package is very likely incomplete. Don't return it since + // we don't know its condition: It's very likely unsafe to use and it's also + // not added to p.packages which may cause further problems (issue #20837). + return nil, fmt.Errorf("type-checking package %q failed (%v)", bp.ImportPath, err) + } + + p.packages[bp.ImportPath] = pkg + return pkg, nil +} + +func (p *Importer) parseFiles(dir string, filenames []string) ([]*ast.File, error) { + open := p.ctxt.OpenFile // possibly nil + + files := make([]*ast.File, len(filenames)) + errors := make([]error, len(filenames)) + + var wg sync.WaitGroup + wg.Add(len(filenames)) + for i, filename := range filenames { + go func(i int, filepath string) { + defer wg.Done() + if open != nil { + src, err := open(filepath) + if err != nil { + errors[i] = fmt.Errorf("opening package file %s failed (%v)", filepath, err) + return + } + files[i], errors[i] = parser.ParseFile(p.fset, filepath, src, 0) + src.Close() // ignore Close error - parsing may have succeeded which is all we need + } else { + // Special-case when ctxt doesn't provide a custom OpenFile and use the + // parser's file reading mechanism directly. This appears to be quite a + // bit faster than opening the file and providing an io.ReaderCloser in + // both cases. + // TODO(gri) investigate performance difference (issue #19281) + files[i], errors[i] = parser.ParseFile(p.fset, filepath, nil, 0) + } + }(i, p.joinPath(dir, filename)) + } + wg.Wait() + + // if there are errors, return the first one for deterministic results + for _, err := range errors { + if err != nil { + return nil, err + } + } + + return files, nil +} + +// context-controlled file system operations + +func (p *Importer) absPath(path string) (string, error) { + // TODO(gri) This should be using p.ctxt.AbsPath which doesn't + // exist but probably should. See also issue #14282. + return filepath.Abs(path) +} + +func (p *Importer) isAbsPath(path string) bool { + if f := p.ctxt.IsAbsPath; f != nil { + return f(path) + } + return filepath.IsAbs(path) +} + +func (p *Importer) joinPath(elem ...string) string { + if f := p.ctxt.JoinPath; f != nil { + return f(elem...) + } + return filepath.Join(elem...) +} diff --git a/libgo/go/go/internal/srcimporter/srcimporter_test.go b/libgo/go/go/internal/srcimporter/srcimporter_test.go new file mode 100644 index 00000000000..79921b5e785 --- /dev/null +++ b/libgo/go/go/internal/srcimporter/srcimporter_test.go @@ -0,0 +1,150 @@ +// Copyright 2017 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package srcimporter + +import ( + "go/build" + "go/token" + "go/types" + "internal/testenv" + "io/ioutil" + "path/filepath" + "runtime" + "strings" + "testing" + "time" +) + +const maxTime = 2 * time.Second + +var importer = New(&build.Default, token.NewFileSet(), make(map[string]*types.Package)) + +func doImport(t *testing.T, path, srcDir string) { + t0 := time.Now() + if _, err := importer.ImportFrom(path, srcDir, 0); err != nil { + // don't report an error if there's no buildable Go files + if _, nogo := err.(*build.NoGoError); !nogo { + t.Errorf("import %q failed (%v)", path, err) + } + return + } + t.Logf("import %q: %v", path, time.Since(t0)) +} + +// walkDir imports the all the packages with the given path +// prefix recursively. It returns the number of packages +// imported and whether importing was aborted because time +// has passed endTime. +func walkDir(t *testing.T, path string, endTime time.Time) (int, bool) { + if time.Now().After(endTime) { + t.Log("testing time used up") + return 0, true + } + + // ignore fake packages and testdata directories + if path == "builtin" || path == "unsafe" || strings.HasSuffix(path, "testdata") { + return 0, false + } + + list, err := ioutil.ReadDir(filepath.Join(runtime.GOROOT(), "src", path)) + if err != nil { + t.Fatalf("walkDir %s failed (%v)", path, err) + } + + nimports := 0 + hasGoFiles := false + for _, f := range list { + if f.IsDir() { + n, abort := walkDir(t, filepath.Join(path, f.Name()), endTime) + nimports += n + if abort { + return nimports, true + } + } else if strings.HasSuffix(f.Name(), ".go") { + hasGoFiles = true + } + } + + if hasGoFiles { + doImport(t, path, "") + nimports++ + } + + return nimports, false +} + +func TestImportStdLib(t *testing.T) { + if !testenv.HasSrc() { + t.Skip("no source code available") + } + + dt := maxTime + if testing.Short() && testenv.Builder() == "" { + dt = 500 * time.Millisecond + } + nimports, _ := walkDir(t, "", time.Now().Add(dt)) // installed packages + t.Logf("tested %d imports", nimports) +} + +var importedObjectTests = []struct { + name string + want string +}{ + {"flag.Bool", "func Bool(name string, value bool, usage string) *bool"}, + {"io.Reader", "type Reader interface{Read(p []byte) (n int, err error)}"}, + {"io.ReadWriter", "type ReadWriter interface{Reader; Writer}"}, // go/types.gcCompatibilityMode is off => interface not flattened + {"math.Pi", "const Pi untyped float"}, + {"math.Sin", "func Sin(x float64) float64"}, + {"math/big.Int", "type Int struct{neg bool; abs nat}"}, + {"golang_org/x/text/unicode/norm.MaxSegmentSize", "const MaxSegmentSize untyped int"}, +} + +func TestImportedTypes(t *testing.T) { + if !testenv.HasSrc() { + t.Skip("no source code available") + } + + for _, test := range importedObjectTests { + s := strings.Split(test.name, ".") + if len(s) != 2 { + t.Fatal("invalid test data format") + } + importPath := s[0] + objName := s[1] + + pkg, err := importer.ImportFrom(importPath, ".", 0) + if err != nil { + t.Error(err) + continue + } + + obj := pkg.Scope().Lookup(objName) + if obj == nil { + t.Errorf("%s: object not found", test.name) + continue + } + + got := types.ObjectString(obj, types.RelativeTo(pkg)) + if got != test.want { + t.Errorf("%s: got %q; want %q", test.name, got, test.want) + } + } +} + +func TestReimport(t *testing.T) { + if !testenv.HasSrc() { + t.Skip("no source code available") + } + + // Reimporting a partially imported (incomplete) package is not supported (see issue #19337). + // Make sure we recognize the situation and report an error. + + mathPkg := types.NewPackage("math", "math") // incomplete package + importer := New(&build.Default, token.NewFileSet(), map[string]*types.Package{mathPkg.Path(): mathPkg}) + _, err := importer.ImportFrom("math", ".", 0) + if err == nil || !strings.HasPrefix(err.Error(), "reimport") { + t.Errorf("got %v; want reimport error", err) + } +} diff --git a/libgo/go/go/parser/error_test.go b/libgo/go/go/parser/error_test.go index 1a08d5a6b11..ef91e1ea60e 100644 --- a/libgo/go/go/parser/error_test.go +++ b/libgo/go/go/parser/error_test.go @@ -66,7 +66,7 @@ var errRx = regexp.MustCompile(`^/\* *ERROR *(HERE)? *"([^"]*)" *\*/$`) // expectedErrors collects the regular expressions of ERROR comments found // in files and returns them as a map of error positions to error messages. // -func expectedErrors(t *testing.T, fset *token.FileSet, filename string, src []byte) map[token.Pos]string { +func expectedErrors(fset *token.FileSet, filename string, src []byte) map[token.Pos]string { errors := make(map[token.Pos]string) var s scanner.Scanner @@ -161,7 +161,7 @@ func checkErrors(t *testing.T, filename string, input interface{}) { // we are expecting the following errors // (collect these after parsing a file so that it is found in the file set) - expected := expectedErrors(t, fset, filename, src) + expected := expectedErrors(fset, filename, src) // verify errors returned by the parser compareErrors(t, fset, expected, found) diff --git a/libgo/go/go/parser/parser.go b/libgo/go/go/parser/parser.go index d3ef7db31ea..2b58724521f 100644 --- a/libgo/go/go/parser/parser.go +++ b/libgo/go/go/parser/parser.go @@ -327,7 +327,7 @@ func (p *parser) next() { // The comment is on same line as the previous token; it // cannot be a lead comment but may be a line comment. comment, endline = p.consumeCommentGroup(0) - if p.file.Line(p.pos) != endline { + if p.file.Line(p.pos) != endline || p.tok == token.EOF { // The next token is on a different line, thus // the last comment group is a line comment. p.lineComment = comment @@ -1707,8 +1707,8 @@ func (p *parser) parseSimpleStmt(mode int) (ast.Stmt, bool) { } // The label declaration typically starts at x[0].Pos(), but the label // declaration may be erroneous due to a token after that position (and - // before the ':'). If SpuriousErrors is not set, the (only) error re- - // ported for the line is the illegal label error instead of the token + // before the ':'). If SpuriousErrors is not set, the (only) error + // reported for the line is the illegal label error instead of the token // before the ':' that caused the problem. Thus, use the (latest) colon // position for error reporting. p.error(colon, "illegal label declaration") @@ -2327,7 +2327,10 @@ func (p *parser) parseTypeSpec(doc *ast.CommentGroup, _ token.Token, _ int) ast. // (Global identifiers are resolved in a separate phase after parsing.) spec := &ast.TypeSpec{Doc: doc, Name: ident} p.declare(spec, nil, p.topScope, ast.Typ, ident) - + if p.tok == token.ASSIGN { + spec.Assign = p.pos + p.next() + } spec.Type = p.parseType() p.expectSemi() // call before accessing p.linecomment spec.Comment = p.lineComment diff --git a/libgo/go/go/parser/parser_test.go b/libgo/go/go/parser/parser_test.go index c7bb36d789d..fb35a88ba1a 100644 --- a/libgo/go/go/parser/parser_test.go +++ b/libgo/go/go/parser/parser_test.go @@ -531,3 +531,18 @@ func TestIncompleteSelection(t *testing.T) { } } } + +func TestLastLineComment(t *testing.T) { + const src = `package main +type x int // comment +` + fset := token.NewFileSet() + f, err := ParseFile(fset, "", src, ParseComments) + if err != nil { + t.Fatal(err) + } + comment := f.Decls[0].(*ast.GenDecl).Specs[0].(*ast.TypeSpec).Comment.List[0].Text + if comment != "// comment" { + t.Errorf("got %q, want %q", comment, "// comment") + } +} diff --git a/libgo/go/go/parser/performance_test.go b/libgo/go/go/parser/performance_test.go index b2e1c11e9d5..f2732c0e2b1 100644 --- a/libgo/go/go/parser/performance_test.go +++ b/libgo/go/go/parser/performance_test.go @@ -10,12 +10,17 @@ import ( "testing" ) -func BenchmarkParse(b *testing.B) { - src, err := ioutil.ReadFile("parser.go") +var src = readFile("parser.go") + +func readFile(filename string) []byte { + data, err := ioutil.ReadFile(filename) if err != nil { - b.Fatal(err) + panic(err) } - b.ResetTimer() + return data +} + +func BenchmarkParse(b *testing.B) { b.SetBytes(int64(len(src))) for i := 0; i < b.N; i++ { if _, err := ParseFile(token.NewFileSet(), "", src, ParseComments); err != nil { diff --git a/libgo/go/go/parser/short_test.go b/libgo/go/go/parser/short_test.go index cdd343ea3c1..6f8ef6b0f77 100644 --- a/libgo/go/go/parser/short_test.go +++ b/libgo/go/go/parser/short_test.go @@ -46,6 +46,8 @@ var valids = []string{ `package p; const (x = 0; y; z)`, // issue 9639 `package p; var _ = map[P]int{P{}:0, {}:1}`, `package p; var _ = map[*P]int{&P{}:0, {}:1}`, + `package p; type T = int`, + `package p; type (T = p.T; _ = struct{}; x = *T)`, } func TestValid(t *testing.T) { diff --git a/libgo/go/go/printer/nodes.go b/libgo/go/go/printer/nodes.go index ea432860a8e..4eaadeb448f 100644 --- a/libgo/go/go/printer/nodes.go +++ b/libgo/go/go/printer/nodes.go @@ -887,8 +887,6 @@ func (p *printer) expr1(expr ast.Expr, prec1, depth int) { default: panic("unreachable") } - - return } func (p *printer) possibleSelectorExpr(expr ast.Expr, prec1, depth int) bool { @@ -1268,8 +1266,6 @@ func (p *printer) stmt(stmt ast.Stmt, nextIsRBrace bool) { default: panic("unreachable") } - - return } // ---------------------------------------------------------------------------- @@ -1447,6 +1443,9 @@ func (p *printer) spec(spec ast.Spec, n int, doIndent bool) { } else { p.print(vtab) } + if s.Assign.IsValid() { + p.print(token.ASSIGN, blank) + } p.expr(s.Type) p.setComment(s.Comment) @@ -1533,6 +1532,16 @@ func (p *printer) nodeSize(n ast.Node, maxSize int) (size int) { return } +// numLines returns the number of lines spanned by node n in the original source. +func (p *printer) numLines(n ast.Node) int { + if from := n.Pos(); from.IsValid() { + if to := n.End(); to.IsValid() { + return p.lineFor(to) - p.lineFor(from) + 1 + } + } + return infinity +} + // bodySize is like nodeSize but it is specialized for *ast.BlockStmt's. func (p *printer) bodySize(b *ast.BlockStmt, maxSize int) int { pos1 := b.Pos() @@ -1669,7 +1678,9 @@ func (p *printer) declList(list []ast.Decl) { if prev != tok || getDoc(d) != nil { min = 2 } - p.linebreak(p.lineFor(d.Pos()), min, ignore, false) + // start a new section if the next declaration is a function + // that spans multiple lines (see also issue #19544) + p.linebreak(p.lineFor(d.Pos()), min, ignore, tok == token.FUNC && p.numLines(d) > 1) } p.decl(d) } diff --git a/libgo/go/go/printer/printer.go b/libgo/go/go/printer/printer.go index be61dad590e..dbb4bbd90cf 100644 --- a/libgo/go/go/printer/printer.go +++ b/libgo/go/go/printer/printer.go @@ -69,7 +69,7 @@ type printer struct { // The out position differs from the pos position when the result // formatting differs from the source formatting (in the amount of // white space). If there's a difference and SourcePos is set in - // ConfigMode, //line comments are used in the output to restore + // ConfigMode, //line directives are used in the output to restore // original source positions for a reader. pos token.Position // current position in AST (source) space out token.Position // current position in output space @@ -203,19 +203,20 @@ func (p *printer) lineFor(pos token.Pos) int { return p.cachedLine } -// atLineBegin emits a //line comment if necessary and prints indentation. -func (p *printer) atLineBegin(pos token.Position) { - // write a //line comment if necessary - if p.Config.Mode&SourcePos != 0 && pos.IsValid() && (p.out.Line != pos.Line || p.out.Filename != pos.Filename) { +// writeLineDirective writes a //line directive if necessary. +func (p *printer) writeLineDirective(pos token.Position) { + if pos.IsValid() && (p.out.Line != pos.Line || p.out.Filename != pos.Filename) { p.output = append(p.output, tabwriter.Escape) // protect '\n' in //line from tabwriter interpretation p.output = append(p.output, fmt.Sprintf("//line %s:%d\n", pos.Filename, pos.Line)...) p.output = append(p.output, tabwriter.Escape) - // p.out must match the //line comment + // p.out must match the //line directive p.out.Filename = pos.Filename p.out.Line = pos.Line } +} - // write indentation +// writeIndent writes indentation. +func (p *printer) writeIndent() { // use "hard" htabs - indentation columns // must not be discarded by the tabwriter n := p.Config.Indent + p.indent // include base indentation @@ -230,9 +231,11 @@ func (p *printer) atLineBegin(pos token.Position) { } // writeByte writes ch n times to p.output and updates p.pos. +// Only used to write formatting (white space) characters. func (p *printer) writeByte(ch byte, n int) { if p.out.Column == 1 { - p.atLineBegin(p.pos) + // no need to write line directives before white space + p.writeIndent() } for i := 0; i < n; i++ { @@ -265,13 +268,16 @@ func (p *printer) writeByte(ch byte, n int) { // func (p *printer) writeString(pos token.Position, s string, isLit bool) { if p.out.Column == 1 { - p.atLineBegin(pos) + if p.Config.Mode&SourcePos != 0 { + p.writeLineDirective(pos) + } + p.writeIndent() } if pos.IsValid() { // update p.pos (if pos is invalid, continue with existing p.pos) // Note: Must do this after handling line beginnings because - // atLineBegin updates p.pos if there's indentation, but p.pos + // writeIndent updates p.pos if there's indentation, but p.pos // is the position of s. p.pos = pos } @@ -325,7 +331,7 @@ func (p *printer) writeString(pos token.Position, s string, isLit bool) { // after all pending comments, prev is the previous comment in // a group of comments (or nil), and tok is the next token. // -func (p *printer) writeCommentPrefix(pos, next token.Position, prev, comment *ast.Comment, tok token.Token) { +func (p *printer) writeCommentPrefix(pos, next token.Position, prev *ast.Comment, tok token.Token) { if len(p.output) == 0 { // the comment is the first item to be printed - don't write any whitespace return @@ -733,7 +739,7 @@ func (p *printer) intersperseComments(next token.Position, tok token.Token) (wro var last *ast.Comment for p.commentBefore(next) { for _, c := range p.comment.List { - p.writeCommentPrefix(p.posFor(c.Pos()), next, last, c, tok) + p.writeCommentPrefix(p.posFor(c.Pos()), next, last, tok) p.writeComment(c) last = c } @@ -1037,6 +1043,28 @@ func getDoc(n ast.Node) *ast.CommentGroup { return nil } +func getLastComment(n ast.Node) *ast.CommentGroup { + switch n := n.(type) { + case *ast.Field: + return n.Comment + case *ast.ImportSpec: + return n.Comment + case *ast.ValueSpec: + return n.Comment + case *ast.TypeSpec: + return n.Comment + case *ast.GenDecl: + if len(n.Specs) > 0 { + return getLastComment(n.Specs[len(n.Specs)-1]) + } + case *ast.File: + if len(n.Comments) > 0 { + return n.Comments[len(n.Comments)-1] + } + } + return nil +} + func (p *printer) printNode(node interface{}) error { // unpack *CommentedNode, if any var comments []*ast.CommentGroup @@ -1060,6 +1088,11 @@ func (p *printer) printNode(node interface{}) error { if doc := getDoc(n); doc != nil { beg = doc.Pos() } + if com := getLastComment(n); com != nil { + if e := com.End(); e > end { + end = e + } + } // token.Pos values are global offsets, we can // compare them directly i := 0 @@ -1237,7 +1270,7 @@ const ( RawFormat Mode = 1 << iota // do not use a tabwriter; if set, UseSpaces is ignored TabIndent // use tabs for indentation independent of UseSpaces UseSpaces // use spaces instead of tabs for alignment - SourcePos // emit //line comments to preserve original source positions + SourcePos // emit //line directives to preserve original source positions ) // A Config node controls the output of Fprint. @@ -1315,7 +1348,7 @@ func (cfg *Config) Fprint(output io.Writer, fset *token.FileSet, node interface{ // Fprint "pretty-prints" an AST node to output. // It calls Config.Fprint with default settings. -// Note that gofmt uses tabs for indentation but spaces for alignent; +// Note that gofmt uses tabs for indentation but spaces for alignment; // use format.Node (package go/format) for output that matches gofmt. // func Fprint(output io.Writer, fset *token.FileSet, node interface{}) error { diff --git a/libgo/go/go/printer/printer_test.go b/libgo/go/go/printer/printer_test.go index 0badbfba697..5984d2c4d24 100644 --- a/libgo/go/go/printer/printer_test.go +++ b/libgo/go/go/printer/printer_test.go @@ -363,7 +363,7 @@ func identCount(f *ast.File) int { return n } -// Verify that the SourcePos mode emits correct //line comments +// Verify that the SourcePos mode emits correct //line directives // by testing that position information for matching identifiers // is maintained. func TestSourcePos(t *testing.T) { @@ -394,7 +394,7 @@ func (t *t) foo(a, b, c int) int { } // parse pretty printed original - // (//line comments must be interpreted even w/o parser.ParseComments set) + // (//line directives must be interpreted even w/o parser.ParseComments set) f2, err := parser.ParseFile(fset, "", buf.Bytes(), 0) if err != nil { t.Fatalf("%s\n%s", err, buf.Bytes()) @@ -434,6 +434,53 @@ func (t *t) foo(a, b, c int) int { } } +// Verify that the SourcePos mode doesn't emit unnecessary //line directives +// before empty lines. +func TestIssue5945(t *testing.T) { + const orig = ` +package p // line 2 +func f() {} // line 3 + +var x, y, z int + + +func g() { // line 8 +} +` + + const want = `//line src.go:2 +package p + +//line src.go:3 +func f() {} + +var x, y, z int + +//line src.go:8 +func g() { +} +` + + // parse original + f1, err := parser.ParseFile(fset, "src.go", orig, 0) + if err != nil { + t.Fatal(err) + } + + // pretty-print original + var buf bytes.Buffer + err = (&Config{Mode: UseSpaces | SourcePos, Tabwidth: 8}).Fprint(&buf, fset, f1) + if err != nil { + t.Fatal(err) + } + got := buf.String() + + // compare original with desired output + if got != want { + t.Errorf("got:\n%s\nwant:\n%s\n", got, want) + } +} + var decls = []string{ `import "fmt"`, "const pi = 3.1415\nconst e = 2.71828\n\nvar x = pi", @@ -612,3 +659,54 @@ func _() {} t.Error(err) } } + +func TestCommentedNode(t *testing.T) { + const ( + input = `package main + +func foo() { + // comment inside func +} + +// leading comment +type bar int // comment2 + +` + + foo = `func foo() { + // comment inside func +}` + + bar = `// leading comment +type bar int // comment2 +` + ) + + fset := token.NewFileSet() + f, err := parser.ParseFile(fset, "input.go", input, parser.ParseComments) + if err != nil { + t.Fatal(err) + } + + var buf bytes.Buffer + + err = Fprint(&buf, fset, &CommentedNode{Node: f.Decls[0], Comments: f.Comments}) + if err != nil { + t.Fatal(err) + } + + if buf.String() != foo { + t.Errorf("got %q, want %q", buf.String(), foo) + } + + buf.Reset() + + err = Fprint(&buf, fset, &CommentedNode{Node: f.Decls[1], Comments: f.Comments}) + if err != nil { + t.Fatal(err) + } + + if buf.String() != bar { + t.Errorf("got %q, want %q", buf.String(), bar) + } +} diff --git a/libgo/go/go/printer/testdata/declarations.golden b/libgo/go/go/printer/testdata/declarations.golden index 82f5e0f9147..bebc0eaa637 100644 --- a/libgo/go/go/printer/testdata/declarations.golden +++ b/libgo/go/go/printer/testdata/declarations.golden @@ -778,6 +778,12 @@ func _() { /* multi-line func because block is on multiple lines */ } +// test case for issue #19544 +func _() {} +func _longer_name_() { // this comment must not force the {} from above to alignment + // multiple lines +} + // ellipsis parameters func _(...int) func _(...*int) @@ -985,3 +991,18 @@ func _(struct { x int y int }) // no extra comma between } and ) + +// alias declarations + +type c0 struct{} +type c1 = C +type c2 = struct{ x int } +type c3 = p.C +type ( + s struct{} + a = A + b = A + c = foo + d = interface{} + ddd = p.Foo +) diff --git a/libgo/go/go/printer/testdata/declarations.input b/libgo/go/go/printer/testdata/declarations.input index a0a3783b846..a858051ef01 100644 --- a/libgo/go/go/printer/testdata/declarations.input +++ b/libgo/go/go/printer/testdata/declarations.input @@ -795,6 +795,11 @@ func _() { /* multi-line function because of "long-ish" comment - much more comm func _() { /* multi-line func because block is on multiple lines */ } +// test case for issue #19544 +func _() {} +func _longer_name_() { // this comment must not force the {} from above to alignment + // multiple lines +} // ellipsis parameters func _(...int) @@ -999,3 +1004,18 @@ func _(struct { x int y int }) // no extra comma between } and ) + +// alias declarations + +type c0 struct{} +type c1 = C +type c2 = struct{ x int} +type c3 = p.C +type ( + s struct{} + a = A + b = A + c = foo + d = interface{} + ddd = p.Foo +) \ No newline at end of file diff --git a/libgo/go/go/token/position.go b/libgo/go/go/token/position.go index d4171d80e0c..88d74168a1b 100644 --- a/libgo/go/go/token/position.go +++ b/libgo/go/go/token/position.go @@ -71,7 +71,7 @@ func (pos Position) String() string { type Pos int // The zero value for Pos is NoPos; there is no file and line information -// associated with it, and NoPos().IsValid() is false. NoPos is always +// associated with it, and NoPos.IsValid() is false. NoPos is always // smaller than any other Pos value. The corresponding Position value // for NoPos is the zero value for Position. // @@ -94,7 +94,8 @@ type File struct { base int // Pos value range for this file is [base...base+size] size int // file size as provided to AddFile - // lines and infos are protected by set.mutex + // lines and infos are protected by mutex + mutex sync.Mutex lines []int // lines contains the offset of the first character for each line (the first entry is always 0) infos []lineInfo } @@ -116,9 +117,9 @@ func (f *File) Size() int { // LineCount returns the number of lines in file f. func (f *File) LineCount() int { - f.set.mutex.RLock() + f.mutex.Lock() n := len(f.lines) - f.set.mutex.RUnlock() + f.mutex.Unlock() return n } @@ -127,11 +128,11 @@ func (f *File) LineCount() int { // and smaller than the file size; otherwise the line offset is ignored. // func (f *File) AddLine(offset int) { - f.set.mutex.Lock() + f.mutex.Lock() if i := len(f.lines); (i == 0 || f.lines[i-1] < offset) && offset < f.size { f.lines = append(f.lines, offset) } - f.set.mutex.Unlock() + f.mutex.Unlock() } // MergeLine merges a line with the following line. It is akin to replacing @@ -143,8 +144,8 @@ func (f *File) MergeLine(line int) { if line <= 0 { panic("illegal line number (line numbering starts at 1)") } - f.set.mutex.Lock() - defer f.set.mutex.Unlock() + f.mutex.Lock() + defer f.mutex.Unlock() if line >= len(f.lines) { panic("illegal line number") } @@ -176,9 +177,9 @@ func (f *File) SetLines(lines []int) bool { } // set lines table - f.set.mutex.Lock() + f.mutex.Lock() f.lines = lines - f.set.mutex.Unlock() + f.mutex.Unlock() return true } @@ -198,9 +199,9 @@ func (f *File) SetLinesForContent(content []byte) { } // set lines table - f.set.mutex.Lock() + f.mutex.Lock() f.lines = lines - f.set.mutex.Unlock() + f.mutex.Unlock() } // A lineInfo object describes alternative file and line number @@ -222,11 +223,11 @@ type lineInfo struct { // information for //line filename:line comments in source files. // func (f *File) AddLineInfo(offset int, filename string, line int) { - f.set.mutex.Lock() + f.mutex.Lock() if i := len(f.infos); i == 0 || f.infos[i-1].Offset < offset && offset < f.size { f.infos = append(f.infos, lineInfo{offset, filename, line}) } - f.set.mutex.Unlock() + f.mutex.Unlock() } // Pos returns the Pos value for the given file offset; @@ -267,6 +268,8 @@ func searchLineInfos(a []lineInfo, x int) int { // possibly adjusted by //line comments; otherwise those comments are ignored. // func (f *File) unpack(offset int, adjusted bool) (filename string, line, column int) { + f.mutex.Lock() + defer f.mutex.Unlock() filename = f.name if i := searchInts(f.lines, offset); i >= 0 { line, column = i+1, offset-f.lines[i]+1 @@ -371,7 +374,7 @@ func (s *FileSet) AddFile(filename string, base, size int) *File { panic("illegal base or size") } // base >= s.base && size >= 0 - f := &File{s, filename, base, size, []int{0}, nil} + f := &File{set: s, name: filename, base: base, size: size, lines: []int{0}} base += size + 1 // +1 because EOF also has a position if base < 0 { panic("token.Pos offset overflow (> 2G of source code in file set)") @@ -446,9 +449,7 @@ func (s *FileSet) File(p Pos) (f *File) { func (s *FileSet) PositionFor(p Pos, adjusted bool) (pos Position) { if p != NoPos { if f := s.file(p); f != nil { - s.mutex.RLock() - pos = f.position(p, adjusted) - s.mutex.RUnlock() + return f.position(p, adjusted) } } return diff --git a/libgo/go/go/token/serialize.go b/libgo/go/go/token/serialize.go index 4adc8f9e334..d0ea34517a4 100644 --- a/libgo/go/go/token/serialize.go +++ b/libgo/go/go/token/serialize.go @@ -30,7 +30,14 @@ func (s *FileSet) Read(decode func(interface{}) error) error { files := make([]*File, len(ss.Files)) for i := 0; i < len(ss.Files); i++ { f := &ss.Files[i] - files[i] = &File{s, f.Name, f.Base, f.Size, f.Lines, f.Infos} + files[i] = &File{ + set: s, + name: f.Name, + base: f.Base, + size: f.Size, + lines: f.Lines, + infos: f.Infos, + } } s.files = files s.last = nil @@ -47,7 +54,15 @@ func (s *FileSet) Write(encode func(interface{}) error) error { ss.Base = s.base files := make([]serializedFile, len(s.files)) for i, f := range s.files { - files[i] = serializedFile{f.name, f.base, f.size, f.lines, f.infos} + f.mutex.Lock() + files[i] = serializedFile{ + Name: f.name, + Base: f.base, + Size: f.size, + Lines: append([]int(nil), f.lines...), + Infos: append([]lineInfo(nil), f.infos...), + } + f.mutex.Unlock() } ss.Files = files s.mutex.Unlock() diff --git a/libgo/go/go/types/api.go b/libgo/go/go/types/api.go index 5b911cb96cc..11e76867a1f 100644 --- a/libgo/go/go/types/api.go +++ b/libgo/go/go/types/api.go @@ -57,10 +57,9 @@ func (err Error) Error() string { // vendored packages. See https://golang.org/s/go15vendor. // If possible, external implementations should implement ImporterFrom. type Importer interface { - // Import returns the imported package for the given import - // path, or an error if the package couldn't be imported. - // Two calls to Import with the same path return the same - // package. + // Import returns the imported package for the given import path. + // The semantics is like for ImporterFrom.ImportFrom except that + // dir and mode are ignored (since they are not present). Import(path string) (*Package, error) } @@ -79,12 +78,15 @@ type ImporterFrom interface { Importer // ImportFrom returns the imported package for the given import - // path when imported by the package in srcDir, or an error - // if the package couldn't be imported. The mode value must - // be 0; it is reserved for future use. - // Two calls to ImportFrom with the same path and srcDir return - // the same package. - ImportFrom(path, srcDir string, mode ImportMode) (*Package, error) + // path when imported by a package file located in dir. + // If the import failed, besides returning an error, ImportFrom + // is encouraged to cache and return a package anyway, if one + // was created. This will reduce package inconsistencies and + // follow-on type checker errors due to the missing package. + // The mode value must be 0; it is reserved for future use. + // Two calls to ImportFrom with the same path and dir must + // return the same package. + ImportFrom(path, dir string, mode ImportMode) (*Package, error) } // A Config specifies the configuration for type checking. @@ -99,7 +101,7 @@ type Config struct { // identifiers referring to package C (which won't find an object). // This feature is intended for the standard library cmd/api tool. // - // Caution: Effects may be unpredictable due to follow-up errors. + // Caution: Effects may be unpredictable due to follow-on errors. // Do not use casually! FakeImportC bool @@ -121,7 +123,7 @@ type Config struct { Importer Importer // If Sizes != nil, it provides the sizing functions for package unsafe. - // Otherwise &StdSizes{WordSize: 8, MaxAlign: 8} is used instead. + // Otherwise SizesFor("gc", "amd64") is used instead. Sizes Sizes // If DisableUnusedImportCheck is set, packages are not checked @@ -243,7 +245,7 @@ func (info *Info) TypeOf(e ast.Expr) Type { // Precondition: the Uses and Defs maps are populated. // func (info *Info) ObjectOf(id *ast.Ident) Object { - if obj, _ := info.Defs[id]; obj != nil { + if obj := info.Defs[id]; obj != nil { return obj } return info.Uses[id] diff --git a/libgo/go/go/types/api_test.go b/libgo/go/go/types/api_test.go index 818a51993ed..1b175796410 100644 --- a/libgo/go/go/types/api_test.go +++ b/libgo/go/go/types/api_test.go @@ -1298,154 +1298,73 @@ func f(x int) { y := x; print(y) } } } -// Alias-related code. Keep for now. -/* -func TestAliases(t *testing.T) { - testenv.MustHaveGoBuild(t) - - const src = ` -package b - -import ( - "./testdata/alias" - a "./testdata/alias" - "math" -) - -const ( - c1 = alias.Pi1 - c2 => a.Pi1 - c3 => a.Pi2 - c4 => math.Pi -) - -var ( - v1 => alias.Default - v2 => a.Default - v3 = f1 -) - -type ( - t1 => alias.Context - t2 => a.Context -) - -func f1 => alias.Sin -func f2 => a.Sin - -func _() { - assert(c1 == alias.Pi1 && c2 == a.Pi1 && c3 == a.Pi2 && c4 == math.Pi) - assert(c2 == c2 && c2 == c3 && c3 == c4) - v1 = v2 // must be assignable - var _ *t1 = new(t2) // must be assignable - var _ t2 = alias.Default - f1(1) // must be callable - f2(1) - _ = alias.Sin(1) - _ = a.Sin(1) -} -` - - if out := compile(t, "testdata", "alias.go"); out != "" { - defer os.Remove(out) - } - - DefPredeclaredTestFuncs() // declare assert built-in for testing - mustTypecheck(t, "Aliases", src, nil) -} - -func compile(t *testing.T, dirname, filename string) string { - cmd := exec.Command(testenv.GoToolPath(t), "tool", "compile", filename) - cmd.Dir = dirname - out, err := cmd.CombinedOutput() - if err != nil { - t.Logf("%s", out) - t.Fatalf("go tool compile %s failed: %s", filename, err) - } - // filename should end with ".go" - return filepath.Join(dirname, filename[:len(filename)-2]+"o") -} - -func TestAliasDefUses(t *testing.T) { +// TestFailedImport tests that we don't get follow-on errors +// elsewhere in a package due to failing to import a package. +func TestFailedImport(t *testing.T) { testenv.MustHaveGoBuild(t) const src = ` package p -import( - "go/build" - "go/types" -) +import "foo" // should only see an error here -// Defs -const Invalid => types.Invalid -type Struct => types.Struct -var Default => build.Default -func Implements => types.Implements - -// Uses -const _ = Invalid -var _ types.Struct = Struct{} // types must be identical -var _ build.Context = Default -var _ = Implements(nil, nil) +const c = foo.C +type T = foo.T +var v T = c +func f(x T) T { return foo.F(x) } ` - - info := Info{ - Defs: make(map[*ast.Ident]Object), - Uses: make(map[*ast.Ident]Object), - } - mustTypecheck(t, "TestAliasDefUses", src, &info) - - // verify Defs - defs := map[string]string{ - "Invalid": "types.Invalid", - "Struct": "types.Struct", - "Default": "build.Default", - "Implements": "types.Implements", + fset := token.NewFileSet() + f, err := parser.ParseFile(fset, "src", src, 0) + if err != nil { + t.Fatal(err) } - - for ident, obj := range info.Defs { - if alias, ok := obj.(*Alias); ok { - if want := defs[ident.Name]; want != "" { - orig := alias.Orig() - if got := orig.Pkg().Name() + "." + orig.Name(); got != want { - t.Errorf("%v: got %v, want %v", ident, got, want) + files := []*ast.File{f} + + // type-check using all possible importers + for _, compiler := range []string{"gc", "gccgo", "source"} { + errcount := 0 + conf := Config{ + Error: func(err error) { + // we should only see the import error + if errcount > 0 || !strings.Contains(err.Error(), "could not import foo") { + t.Errorf("for %s importer, got unexpected error: %v", compiler, err) } - delete(defs, ident.Name) // mark as found - } else { - t.Errorf("unexpected alias def of %v", ident) - } + errcount++ + }, + Importer: importer.For(compiler, nil), } - } - if len(defs) != 0 { - t.Errorf("missing aliases: %v", defs) - } + info := &Info{ + Uses: make(map[*ast.Ident]Object), + } + pkg, _ := conf.Check("p", fset, files, info) + if pkg == nil { + t.Errorf("for %s importer, type-checking failed to return a package", compiler) + continue + } - // verify Uses - uses := map[string]string{ - "Invalid": "types.Invalid", - "Struct": "types.Struct", - "Default": "build.Default", - "Implements": "types.Implements", - } + imports := pkg.Imports() + if len(imports) != 1 { + t.Errorf("for %s importer, got %d imports, want 1", compiler, len(imports)) + continue + } + imp := imports[0] + if imp.Name() != "foo" { + t.Errorf(`for %s importer, got %q, want "foo"`, compiler, imp.Name()) + continue + } - for ident, obj := range info.Uses { - if alias, ok := obj.(*Alias); ok { - if want := uses[ident.Name]; want != "" { - orig := alias.Orig() - if got := orig.Pkg().Name() + "." + orig.Name(); got != want { - t.Errorf("%v: got %v, want %v", ident, got, want) + // verify that all uses of foo refer to the imported package foo (imp) + for ident, obj := range info.Uses { + if ident.Name == "foo" { + if obj, ok := obj.(*PkgName); ok { + if obj.Imported() != imp { + t.Errorf("%s resolved to %v; want %v", ident, obj.Imported(), imp) + } + } else { + t.Errorf("%s resolved to %v; want package name", ident, obj) } - delete(uses, ident.Name) // mark as found - } else { - t.Errorf("unexpected alias use of %v", ident) } } } - - if len(uses) != 0 { - t.Errorf("missing aliases: %v", defs) - } } -*/ diff --git a/libgo/go/go/types/assignments.go b/libgo/go/go/types/assignments.go index 18f893d4781..e5ea071e864 100644 --- a/libgo/go/go/types/assignments.go +++ b/libgo/go/go/types/assignments.go @@ -219,7 +219,7 @@ func (check *Checker) initVars(lhs []*Var, rhs []ast.Expr, returnPos token.Pos) check.errorf(returnPos, "wrong number of return values (want %d, got %d)", l, r) return } - check.errorf(rhs[0].Pos(), "assignment count mismatch (%d vs %d)", l, r) + check.errorf(rhs[0].Pos(), "cannot initialize %d variables with %d values", l, r) return } @@ -253,7 +253,7 @@ func (check *Checker) assignVars(lhs, rhs []ast.Expr) { } if l != r { check.useGetter(get, r) - check.errorf(rhs[0].Pos(), "assignment count mismatch (%d vs %d)", l, r) + check.errorf(rhs[0].Pos(), "cannot assign %d values to %d variables", r, l) return } diff --git a/libgo/go/go/types/call.go b/libgo/go/go/types/call.go index 8e5c5371f22..ffd96297775 100644 --- a/libgo/go/go/types/call.go +++ b/libgo/go/go/types/call.go @@ -93,7 +93,9 @@ func (check *Checker) call(x *operand, e *ast.CallExpr) exprKind { func (check *Checker) use(arg ...ast.Expr) { var x operand for _, e := range arg { - check.rawExpr(&x, e, nil) + if e != nil { // be safe + check.rawExpr(&x, e, nil) + } } } @@ -250,7 +252,7 @@ func (check *Checker) argument(fun ast.Expr, sig *Signature, i int, x *operand, check.errorf(ellipsis, "can only use ... with matching parameter") return } - if _, ok := x.typ.Underlying().(*Slice); !ok { + if _, ok := x.typ.Underlying().(*Slice); !ok && x.typ != Typ[UntypedNil] { // see issue #18268 check.errorf(x.pos(), "cannot use %s as parameter of type %s", x, typ) return } @@ -275,8 +277,6 @@ func (check *Checker) selector(x *operand, e *ast.SelectorExpr) { // so we don't need a "package" mode for operands: package names // can only appear in qualified identifiers which are mapped to // selector expressions. - // (see also decl.go: checker.aliasDecl) - // TODO(gri) factor this code out and share with checker.aliasDecl if ident, ok := e.X.(*ast.Ident); ok { _, obj := check.scope.LookupParent(ident.Name, check.pos) if pname, _ := obj.(*PkgName); pname != nil { @@ -296,12 +296,6 @@ func (check *Checker) selector(x *operand, e *ast.SelectorExpr) { // ok to continue } check.recordUse(e.Sel, exp) - exp = original(exp) - - // avoid further errors if the imported object is an alias that's broken - if exp == nil { - goto Error - } // Simplified version of the code for *ast.Idents: // - imported objects are always fully initialized diff --git a/libgo/go/go/types/check.go b/libgo/go/go/types/check.go index 28e94f1940e..26db5769b98 100644 --- a/libgo/go/go/types/check.go +++ b/libgo/go/go/types/check.go @@ -57,6 +57,16 @@ type context struct { hasCallOrRecv bool // set if an expression contains a function call or channel receive operation } +// An importKey identifies an imported package by import path and source directory +// (directory containing the file containing the import). In practice, the directory +// may always be the same, or may not matter. Given an (import path, directory), an +// importer must always return the same package (but given two different import paths, +// an importer may still return the same package by mapping them to the same package +// paths). +type importKey struct { + path, dir string +} + // A Checker maintains the state of the type checker. // It must be created with NewChecker. type Checker struct { @@ -66,7 +76,8 @@ type Checker struct { fset *token.FileSet pkg *Package *Info - objMap map[Object]*declInfo // maps package-level object to declaration info + objMap map[Object]*declInfo // maps package-level object to declaration info + impMap map[importKey]*Package // maps (import path, source directory) to (complete or fake) package // information collected during type-checking of a set of package files // (initialized by Files, valid only for the duration of check.Files; @@ -162,6 +173,7 @@ func NewChecker(conf *Config, fset *token.FileSet, pkg *Package, info *Info) *Ch pkg: pkg, Info: info, objMap: make(map[Object]*declInfo), + impMap: make(map[importKey]*Package), } } diff --git a/libgo/go/go/types/check_test.go b/libgo/go/go/types/check_test.go index f8445752695..24b3365717b 100644 --- a/libgo/go/go/types/check_test.go +++ b/libgo/go/go/types/check_test.go @@ -68,11 +68,11 @@ var tests = [][]string{ {"testdata/decls1.src"}, {"testdata/decls2a.src", "testdata/decls2b.src"}, {"testdata/decls3.src"}, + {"testdata/decls4.src"}, {"testdata/const0.src"}, {"testdata/const1.src"}, {"testdata/constdecl.src"}, {"testdata/vardecl.src"}, - //{"testdata/aliasdecl.src"}, {"testdata/expr0.src"}, {"testdata/expr1.src"}, {"testdata/expr2.src"}, diff --git a/libgo/go/go/types/decl.go b/libgo/go/go/types/decl.go index dced7a6d6dc..7428f8f9950 100644 --- a/libgo/go/go/types/decl.go +++ b/libgo/go/go/types/decl.go @@ -81,14 +81,10 @@ func (check *Checker) objDecl(obj Object, def *Named, path []*TypeName) { check.varDecl(obj, d.lhs, d.typ, d.init) case *TypeName: // invalid recursive types are detected via path - check.typeDecl(obj, d.typ, def, path) + check.typeDecl(obj, d.typ, def, path, d.alias) case *Func: // functions may be recursive - no need to track dependencies check.funcDecl(obj, d) - // Alias-related code. Keep for now. - // case *Alias: - // // aliases cannot be recursive - no need to track dependencies - // check.aliasDecl(obj, d) default: unreachable() } @@ -219,33 +215,42 @@ func (n *Named) setUnderlying(typ Type) { } } -func (check *Checker) typeDecl(obj *TypeName, typ ast.Expr, def *Named, path []*TypeName) { +func (check *Checker) typeDecl(obj *TypeName, typ ast.Expr, def *Named, path []*TypeName, alias bool) { assert(obj.typ == nil) // type declarations cannot use iota assert(check.iota == nil) - named := &Named{obj: obj} - def.setUnderlying(named) - obj.typ = named // make sure recursive type declarations terminate - - // determine underlying type of named - check.typExpr(typ, named, append(path, obj)) - - // The underlying type of named may be itself a named type that is - // incomplete: - // - // type ( - // A B - // B *C - // C A - // ) - // - // The type of C is the (named) type of A which is incomplete, - // and which has as its underlying type the named type B. - // Determine the (final, unnamed) underlying type by resolving - // any forward chain (they always end in an unnamed type). - named.underlying = underlying(named.underlying) + if alias { + + obj.typ = Typ[Invalid] + obj.typ = check.typExpr(typ, nil, append(path, obj)) + + } else { + + named := &Named{obj: obj} + def.setUnderlying(named) + obj.typ = named // make sure recursive type declarations terminate + + // determine underlying type of named + check.typExpr(typ, named, append(path, obj)) + + // The underlying type of named may be itself a named type that is + // incomplete: + // + // type ( + // A B + // B *C + // C A + // ) + // + // The type of C is the (named) type of A which is incomplete, + // and which has as its underlying type the named type B. + // Determine the (final, unnamed) underlying type by resolving + // any forward chain (they always end in an unnamed type). + named.underlying = underlying(named.underlying) + + } // check and add associated methods // TODO(gri) It's easy to create pathological cases where the @@ -268,21 +273,23 @@ func (check *Checker) addMethodDecls(obj *TypeName) { // spec: "If the base type is a struct type, the non-blank method // and field names must be distinct." - base := obj.typ.(*Named) - if t, _ := base.underlying.(*Struct); t != nil { - for _, fld := range t.fields { - if fld.name != "_" { - assert(mset.insert(fld) == nil) + base, _ := obj.typ.(*Named) // nil if receiver base type is type alias + if base != nil { + if t, _ := base.underlying.(*Struct); t != nil { + for _, fld := range t.fields { + if fld.name != "_" { + assert(mset.insert(fld) == nil) + } } } - } - // Checker.Files may be called multiple times; additional package files - // may add methods to already type-checked types. Add pre-existing methods - // so that we can detect redeclarations. - for _, m := range base.methods { - assert(m.name != "_") - assert(mset.insert(m) == nil) + // Checker.Files may be called multiple times; additional package files + // may add methods to already type-checked types. Add pre-existing methods + // so that we can detect redeclarations. + for _, m := range base.methods { + assert(m.name != "_") + assert(mset.insert(m) == nil) + } } // type-check methods @@ -295,7 +302,7 @@ func (check *Checker) addMethodDecls(obj *TypeName) { case *Var: check.errorf(m.pos, "field and method with the same name %s", m.name) case *Func: - check.errorf(m.pos, "method %s already declared for %s", m.name, base) + check.errorf(m.pos, "method %s already declared for %s", m.name, obj) default: unreachable() } @@ -303,9 +310,12 @@ func (check *Checker) addMethodDecls(obj *TypeName) { continue } } + + // type-check check.objDecl(m, nil, nil) + // methods with blank _ names cannot be found - don't keep them - if m.name != "_" { + if base != nil && m.name != "_" { base.methods = append(base.methods, m) } } @@ -333,106 +343,6 @@ func (check *Checker) funcDecl(obj *Func, decl *declInfo) { } } -// original returns the original Object if obj is an Alias; -// otherwise it returns obj. The result is never an Alias, -// but it may be nil. -func original(obj Object) Object { - // an alias stands for the original object; use that one instead - if alias, _ := obj.(*disabledAlias); alias != nil { - obj = alias.orig - // aliases always refer to non-alias originals - if _, ok := obj.(*disabledAlias); ok { - panic("original is an alias") - } - } - return obj -} - -func (check *Checker) aliasDecl(obj *disabledAlias, decl *declInfo) { - assert(obj.typ == nil) - - // alias declarations cannot use iota - assert(check.iota == nil) - - // assume alias is invalid to start with - obj.typ = Typ[Invalid] - - // rhs must be package-qualified identifer pkg.sel (see also call.go: checker.selector) - // TODO(gri) factor this code out and share with checker.selector - rhs := decl.init - var pkg *Package - var sel *ast.Ident - if sexpr, ok := rhs.(*ast.SelectorExpr); ok { - if ident, ok := sexpr.X.(*ast.Ident); ok { - _, obj := check.scope.LookupParent(ident.Name, check.pos) - if pname, _ := obj.(*PkgName); pname != nil { - assert(pname.pkg == check.pkg) - check.recordUse(ident, pname) - pname.used = true - pkg = pname.imported - sel = sexpr.Sel - } - } - } - if pkg == nil { - check.errorf(rhs.Pos(), "invalid alias: %v is not a package-qualified identifier", rhs) - return - } - - // qualified identifier must denote an exported object - orig := pkg.scope.Lookup(sel.Name) - if orig == nil || !orig.Exported() { - if !pkg.fake { - check.errorf(rhs.Pos(), "%s is not exported by package %s", sel.Name, pkg.name) - } - return - } - check.recordUse(sel, orig) - orig = original(orig) - - // avoid further errors if the imported object is an alias that's broken - if orig == nil { - return - } - - // An alias declaration must not refer to package unsafe. - if orig.Pkg() == Unsafe { - check.errorf(rhs.Pos(), "invalid alias: %s refers to package unsafe (%v)", obj.Name(), orig) - return - } - - // The original must be of the same kind as the alias declaration. - var why string - switch obj.kind { - case token.CONST: - if _, ok := orig.(*Const); !ok { - why = "constant" - } - case token.TYPE: - if _, ok := orig.(*TypeName); !ok { - why = "type" - } - case token.VAR: - if _, ok := orig.(*Var); !ok { - why = "variable" - } - case token.FUNC: - if _, ok := orig.(*Func); !ok { - why = "function" - } - default: - unreachable() - } - if why != "" { - check.errorf(rhs.Pos(), "invalid alias: %v is not a %s", orig, why) - return - } - - // alias is valid - obj.typ = orig.Type() - obj.orig = orig -} - func (check *Checker) declStmt(decl ast.Decl) { pkg := check.pkg @@ -540,7 +450,7 @@ func (check *Checker) declStmt(decl ast.Decl) { // the innermost containing block." scopePos := s.Name.Pos() check.declare(check.scope, s.Name, obj, scopePos) - check.typeDecl(obj, s.Type, nil, nil) + check.typeDecl(obj, s.Type, nil, nil, s.Assign.IsValid()) default: check.invalidAST(s.Pos(), "const, type, or var declaration expected") diff --git a/libgo/go/go/types/example_test.go b/libgo/go/go/types/example_test.go index 9daad8765a8..a0c9c217ff9 100644 --- a/libgo/go/go/types/example_test.go +++ b/libgo/go/go/types/example_test.go @@ -239,10 +239,10 @@ func fib(x int) int { // type S string: // defined at fib.go:4:6 // used at 6:23 - // type int int: + // type int: // defined at - // used at 8:12, 8:17 - // type string string: + // type string: // defined at - // used at 4:8 // var b S: diff --git a/libgo/go/go/types/expr.go b/libgo/go/go/types/expr.go index f76da17fe53..461f0a525ba 100644 --- a/libgo/go/go/types/expr.go +++ b/libgo/go/go/types/expr.go @@ -633,13 +633,13 @@ func (check *Checker) shift(x, y *operand, e *ast.BinaryExpr, op token.Token) { } // spec: "The right operand in a shift expression must have unsigned - // integer type or be an untyped constant that can be converted to - // unsigned integer type." + // integer type or be an untyped constant representable by a value of + // type uint." switch { case isUnsigned(y.typ): // nothing to do case isUntyped(y.typ): - check.convertUntyped(y, Typ[UntypedInt]) + check.convertUntyped(y, Typ[Uint]) if y.mode == invalid { x.mode = invalid return @@ -800,10 +800,24 @@ func (check *Checker) binary(x *operand, e *ast.BinaryExpr, lhs, rhs ast.Expr, o return } - if (op == token.QUO || op == token.REM) && (x.mode == constant_ || isInteger(x.typ)) && y.mode == constant_ && constant.Sign(y.val) == 0 { - check.invalidOp(y.pos(), "division by zero") - x.mode = invalid - return + if op == token.QUO || op == token.REM { + // check for zero divisor + if (x.mode == constant_ || isInteger(x.typ)) && y.mode == constant_ && constant.Sign(y.val) == 0 { + check.invalidOp(y.pos(), "division by zero") + x.mode = invalid + return + } + + // check for divisor underflow in complex division (see issue 20227) + if x.mode == constant_ && y.mode == constant_ && isComplex(x.typ) { + re, im := constant.Real(y.val), constant.Imag(y.val) + re2, im2 := constant.BinaryOp(re, token.MUL, re), constant.BinaryOp(im, token.MUL, im) + if constant.Sign(re2) == 0 && constant.Sign(im2) == 0 { + check.invalidOp(y.pos(), "division by zero") + x.mode = invalid + return + } + } } if x.mode == constant_ && y.mode == constant_ { @@ -1111,6 +1125,16 @@ func (check *Checker) exprInternal(x *operand, e ast.Expr, hint Type) exprKind { } case *Array: + // Prevent crash if the array referred to is not yet set up. + // This is a stop-gap solution; a better approach would use the mechanism of + // Checker.ident (typexpr.go) using a path of types. But that would require + // passing the path everywhere (all expression-checking methods, not just + // type expression checking), and we're not set up for that (quite possibly + // an indication that cycle detection needs to be rethought). Was issue #18643. + if utyp.elem == nil { + check.error(e.Pos(), "illegal cycle in type declaration") + goto Error + } n := check.indexedElts(e.Elts, utyp.elem, utyp.len) // If we have an "open" [...]T array, set the length now that we know it // and record the type for [...] (usually done by check.typExpr which is @@ -1121,9 +1145,21 @@ func (check *Checker) exprInternal(x *operand, e ast.Expr, hint Type) exprKind { } case *Slice: + // Prevent crash if the slice referred to is not yet set up. + // See analogous comment for *Array. + if utyp.elem == nil { + check.error(e.Pos(), "illegal cycle in type declaration") + goto Error + } check.indexedElts(e.Elts, utyp.elem, -1) case *Map: + // Prevent crash if the map referred to is not yet set up. + // See analogous comment for *Array. + if utyp.key == nil || utyp.elem == nil { + check.error(e.Pos(), "illegal cycle in type declaration") + goto Error + } visited := make(map[interface{}][]Type, len(e.Elts)) for _, e := range e.Elts { kv, _ := e.(*ast.KeyValueExpr) @@ -1161,6 +1197,17 @@ func (check *Checker) exprInternal(x *operand, e ast.Expr, hint Type) exprKind { } default: + // when "using" all elements unpack KeyValueExpr + // explicitly because check.use doesn't accept them + for _, e := range e.Elts { + if kv, _ := e.(*ast.KeyValueExpr); kv != nil { + // Ideally, we should also "use" kv.Key but we can't know + // if it's an externally defined struct key or not. Going + // forward anyway can lead to other errors. Give up instead. + e = kv.Value + } + check.use(e) + } // if utyp is invalid, an error was reported before if utyp != Typ[Invalid] { check.errorf(e.Pos(), "invalid composite literal type %s", typ) @@ -1182,6 +1229,7 @@ func (check *Checker) exprInternal(x *operand, e ast.Expr, hint Type) exprKind { case *ast.IndexExpr: check.expr(x, e.X) if x.mode == invalid { + check.use(e.Index) goto Error } @@ -1251,6 +1299,7 @@ func (check *Checker) exprInternal(x *operand, e ast.Expr, hint Type) exprKind { case *ast.SliceExpr: check.expr(x, e.X) if x.mode == invalid { + check.use(e.Low, e.High, e.Max) goto Error } diff --git a/libgo/go/go/types/gotype.go b/libgo/go/go/types/gotype.go index 0a36c080839..196fc9bbd60 100644 --- a/libgo/go/go/types/gotype.go +++ b/libgo/go/go/types/gotype.go @@ -7,40 +7,46 @@ // Build this command explicitly: go build gotype.go /* -The gotype command does syntactic and semantic analysis of Go files -and packages like the front-end of a Go compiler. Errors are reported -if the analysis fails; otherwise gotype is quiet (unless -v is set). +The gotype command, like the front-end of a Go compiler, parses and +type-checks a single Go package. Errors are reported if the analysis +fails; otherwise gotype is quiet (unless -v is set). Without a list of paths, gotype reads from standard input, which must provide a single Go source file defining a complete package. -If a single path is specified that is a directory, gotype checks -the Go files in that directory; they must all belong to the same -package. +With a single directory argument, gotype checks the Go files in +that directory, comprising a single package. Use -t to include the +(in-package) _test.go files. Use -x to type check only external +test files. -Otherwise, each path must be the filename of Go file belonging to -the same package. +Otherwise, each path must be the filename of a Go file belonging +to the same package. + +Imports are processed by importing directly from the source of +imported packages (default), or by importing from compiled and +installed packages (by setting -c to the respective compiler). + +The -c flag must be set to a compiler ("gc", "gccgo") when type- +checking packages containing imports with relative import paths +(import "./mypkg") because the source importer cannot know which +files to include for such packages. Usage: gotype [flags] [path...] The flags are: - -a - use all (incl. _test.go) files when processing a directory + -t + include local test files in a directory (ignored if -x is provided) + -x + consider only external test files in a directory -e report all errors (not just the first 10) -v verbose mode -c - compiler used to compile packages (gc or gccgo); default: gc - (gotype based on Go1.5 and up only) - -gccgo - use gccimporter instead of gcimporter - (gotype based on Go1.4 and before only) - -Debugging flags: - -seq - parse sequentially, rather than in parallel + compiler used for installed packages (gc, gccgo, or source); default: source + +Flags controlling additional output: -ast print AST (forces -seq) -trace @@ -54,13 +60,14 @@ To check the files a.go, b.go, and c.go: gotype a.go b.go c.go -To check an entire package in the directory dir and print the processed files: +To check an entire package including (in-package) tests in the directory dir and print the processed files: - gotype -v dir + gotype -t -v dir -To check an entire package including tests in the local directory: +To check the external test package (if any) in the current directory, based on installed packages compiled with +cmd/compile: - gotype -a . + gotype -c=gc -x . To verify the output of a pipe: @@ -82,18 +89,19 @@ import ( "io/ioutil" "os" "path/filepath" + "sync" "time" ) var ( // main operation modes - allFiles = flag.Bool("a", false, "use all (incl. _test.go) files when processing a directory") - allErrors = flag.Bool("e", false, "report all errors (not just the first 10)") - verbose = flag.Bool("v", false, "verbose mode") - gccgo = flag.Bool("gccgo", false, "use gccgoimporter instead of gcimporter") + testFiles = flag.Bool("t", false, "include in-package test files in a directory") + xtestFiles = flag.Bool("x", false, "consider only external test files in a directory") + allErrors = flag.Bool("e", false, "report all errors, not just the first 10") + verbose = flag.Bool("v", false, "verbose mode") + compiler = flag.String("c", "source", "compiler used for installed packages (gc, gccgo, or source)") - // debugging support - sequential = flag.Bool("seq", false, "parse sequentially, rather than in parallel") + // additional output control printAST = flag.Bool("ast", false, "print AST (forces -seq)") printTrace = flag.Bool("trace", false, "print parse trace (forces -seq)") parseComments = flag.Bool("comments", false, "parse comments (ignored unless -ast or -trace is provided)") @@ -102,36 +110,55 @@ var ( var ( fset = token.NewFileSet() errorCount = 0 + sequential = false parserMode parser.Mode - sizes types.Sizes ) func initParserMode() { if *allErrors { parserMode |= parser.AllErrors } + if *printAST { + sequential = true + } if *printTrace { parserMode |= parser.Trace + sequential = true } if *parseComments && (*printAST || *printTrace) { parserMode |= parser.ParseComments } } -func initSizes() { - wordSize := 8 - maxAlign := 8 - switch build.Default.GOARCH { - case "386", "arm": - wordSize = 4 - maxAlign = 4 - // add more cases as needed - } - sizes = &types.StdSizes{WordSize: int64(wordSize), MaxAlign: int64(maxAlign)} -} +const usageString = `usage: gotype [flags] [path ...] + +The gotype command, like the front-end of a Go compiler, parses and +type-checks a single Go package. Errors are reported if the analysis +fails; otherwise gotype is quiet (unless -v is set). + +Without a list of paths, gotype reads from standard input, which +must provide a single Go source file defining a complete package. + +With a single directory argument, gotype checks the Go files in +that directory, comprising a single package. Use -t to include the +(in-package) _test.go files. Use -x to type check only external +test files. + +Otherwise, each path must be the filename of a Go file belonging +to the same package. + +Imports are processed by importing directly from the source of +imported packages (default), or by importing from compiled and +installed packages (by setting -c to the respective compiler). + +The -c flag must be set to a compiler ("gc", "gccgo") when type- +checking packages containing imports with relative import paths +(import "./mypkg") because the source importer cannot know which +files to include for such packages. +` func usage() { - fmt.Fprintln(os.Stderr, "usage: gotype [flags] [path ...]") + fmt.Fprintln(os.Stderr, usageString) flag.PrintDefaults() os.Exit(2) } @@ -165,60 +192,49 @@ func parseStdin() (*ast.File, error) { return parse("", src) } -func parseFiles(filenames []string) ([]*ast.File, error) { +func parseFiles(dir string, filenames []string) ([]*ast.File, error) { files := make([]*ast.File, len(filenames)) + errors := make([]error, len(filenames)) - if *sequential { - for i, filename := range filenames { - var err error - files[i], err = parse(filename, nil) - if err != nil { - return nil, err // leave unfinished goroutines hanging - } - } - } else { - type parseResult struct { - file *ast.File - err error - } - - out := make(chan parseResult) - for _, filename := range filenames { - go func(filename string) { - file, err := parse(filename, nil) - out <- parseResult{file, err} - }(filename) + var wg sync.WaitGroup + for i, filename := range filenames { + wg.Add(1) + go func(i int, filepath string) { + defer wg.Done() + files[i], errors[i] = parse(filepath, nil) + }(i, filepath.Join(dir, filename)) + if sequential { + wg.Wait() } + } + wg.Wait() - for i := range filenames { - res := <-out - if res.err != nil { - return nil, res.err // leave unfinished goroutines hanging - } - files[i] = res.file + // if there are errors, return the first one for deterministic results + for _, err := range errors { + if err != nil { + return nil, err } } return files, nil } -func parseDir(dirname string) ([]*ast.File, error) { +func parseDir(dir string) ([]*ast.File, error) { ctxt := build.Default - pkginfo, err := ctxt.ImportDir(dirname, 0) + pkginfo, err := ctxt.ImportDir(dir, 0) if _, nogo := err.(*build.NoGoError); err != nil && !nogo { return nil, err } - filenames := append(pkginfo.GoFiles, pkginfo.CgoFiles...) - if *allFiles { - filenames = append(filenames, pkginfo.TestGoFiles...) - } - // complete file names - for i, filename := range filenames { - filenames[i] = filepath.Join(dirname, filename) + if *xtestFiles { + return parseFiles(dir, pkginfo.XTestGoFiles) } - return parseFiles(filenames) + filenames := append(pkginfo.GoFiles, pkginfo.CgoFiles...) + if *testFiles { + filenames = append(filenames, pkginfo.TestGoFiles...) + } + return parseFiles(dir, filenames) } func getPkgFiles(args []string) ([]*ast.File, error) { @@ -244,15 +260,13 @@ func getPkgFiles(args []string) ([]*ast.File, error) { } // list of files - return parseFiles(args) + return parseFiles("", args) } func checkPkgFiles(files []*ast.File) { - compiler := "gc" - if *gccgo { - compiler = "gccgo" - } type bailout struct{} + + // if checkPkgFiles is called multiple times, set up conf only once conf := types.Config{ FakeImportC: true, Error: func(err error) { @@ -261,8 +275,8 @@ func checkPkgFiles(files []*ast.File) { } report(err) }, - Importer: importer.For(compiler, nil), - Sizes: sizes, + Importer: importer.For(*compiler, nil), + Sizes: types.SizesFor(build.Default.Compiler, build.Default.GOARCH), } defer func() { @@ -297,11 +311,7 @@ func printStats(d time.Duration) { func main() { flag.Usage = usage flag.Parse() - if *printAST || *printTrace { - *sequential = true - } initParserMode() - initSizes() start := time.Now() diff --git a/libgo/go/go/types/hilbert_test.go b/libgo/go/go/types/hilbert_test.go index 07d5a635e0c..500fd27ff19 100644 --- a/libgo/go/go/types/hilbert_test.go +++ b/libgo/go/go/types/hilbert_test.go @@ -52,8 +52,7 @@ func TestHilbert(t *testing.T) { func program(n int, out string) []byte { var g gen - g.p(`// WARNING: GENERATED FILE - DO NOT MODIFY MANUALLY! -// (To generate, in go/types directory: go test -run=Hilbert -H=%d -out=%q) + g.p(`// Code generated by: go test -run=Hilbert -H=%d -out=%q. DO NOT EDIT. // This program tests arbitrary precision constant arithmetic // by generating the constant elements of a Hilbert matrix H, diff --git a/libgo/go/go/types/lookup.go b/libgo/go/go/types/lookup.go index 3caca5519b6..ee8202d9e42 100644 --- a/libgo/go/go/types/lookup.go +++ b/libgo/go/go/types/lookup.go @@ -67,24 +67,22 @@ func lookupFieldOrMethod(T Type, addressable bool, pkg *Package, name string) (o } typ, isPtr := deref(T) - named, _ := typ.(*Named) // *typ where typ is an interface has no methods. - if isPtr { - utyp := typ - if named != nil { - utyp = named.underlying - } - if _, ok := utyp.(*Interface); ok { - return - } + if isPtr && IsInterface(typ) { + return } // Start with typ as single entry at shallowest depth. - // If typ is not a named type, insert a nil type instead. - current := []embeddedType{{named, nil, isPtr, false}} - - // named types that we have seen already, allocated lazily + current := []embeddedType{{typ, nil, isPtr, false}} + + // Named types that we have seen already, allocated lazily. + // Used to avoid endless searches in case of recursive types. + // Since only Named types can be used for recursive types, we + // only need to track those. + // (If we ever allow type aliases to construct recursive types, + // we must use type identity rather than pointer equality for + // the map key comparison, as we do in consolidateMultiples.) var seen map[*Named]bool // search current depth @@ -93,11 +91,12 @@ func lookupFieldOrMethod(T Type, addressable bool, pkg *Package, name string) (o // look for (pkg, name) in all types at current depth for _, e := range current { - // The very first time only, e.typ may be nil. - // In this case, we don't have a named type and - // we simply continue with the underlying type. - if e.typ != nil { - if seen[e.typ] { + typ := e.typ + + // If we have a named type, we may have associated methods. + // Look for those first. + if named, _ := typ.(*Named); named != nil { + if seen[named] { // We have seen this type before, at a more shallow depth // (note that multiples of this type at the current depth // were consolidated before). The type at that depth shadows @@ -108,10 +107,10 @@ func lookupFieldOrMethod(T Type, addressable bool, pkg *Package, name string) (o if seen == nil { seen = make(map[*Named]bool) } - seen[e.typ] = true + seen[named] = true // look for a matching attached method - if i, m := lookupMethod(e.typ.methods, pkg, name); m != nil { + if i, m := lookupMethod(named.methods, pkg, name); m != nil { // potential match assert(m.typ != nil) index = concat(e.index, i) @@ -124,7 +123,7 @@ func lookupFieldOrMethod(T Type, addressable bool, pkg *Package, name string) (o } // continue with underlying type - typ = e.typ.underlying + typ = named.underlying } switch t := typ.(type) { @@ -147,16 +146,15 @@ func lookupFieldOrMethod(T Type, addressable bool, pkg *Package, name string) (o // we have a name collision on the same depth; in either // case we don't need to look further). // Embedded fields are always of the form T or *T where - // T is a named type. If e.typ appeared multiple times at + // T is a type name. If e.typ appeared multiple times at // this depth, f.typ appears multiple times at the next // depth. if obj == nil && f.anonymous { - // Ignore embedded basic types - only user-defined - // named types can have methods or struct fields. typ, isPtr := deref(f.typ) - if t, _ := typ.(*Named); t != nil { - next = append(next, embeddedType{t, concat(e.index, i), e.indirect || isPtr, e.multiples}) - } + // TODO(gri) optimization: ignore types that can't + // have fields or methods (only Named, Struct, and + // Interface types need to be considered). + next = append(next, embeddedType{typ, concat(e.index, i), e.indirect || isPtr, e.multiples}) } } @@ -193,12 +191,12 @@ func lookupFieldOrMethod(T Type, addressable bool, pkg *Package, name string) (o return nil, nil, false // not found } -// embeddedType represents an embedded named type +// embeddedType represents an embedded type type embeddedType struct { - typ *Named // nil means use the outer typ variable instead - index []int // embedded field indices, starting with index at depth 0 - indirect bool // if set, there was a pointer indirection on the path to this field - multiples bool // if set, typ appears multiple times at this depth + typ Type + index []int // embedded field indices, starting with index at depth 0 + indirect bool // if set, there was a pointer indirection on the path to this field + multiples bool // if set, typ appears multiple times at this depth } // consolidateMultiples collects multiple list entries with the same type @@ -209,10 +207,10 @@ func consolidateMultiples(list []embeddedType) []embeddedType { return list // at most one entry - nothing to do } - n := 0 // number of entries w/ unique type - prev := make(map[*Named]int) // index at which type was previously seen + n := 0 // number of entries w/ unique type + prev := make(map[Type]int) // index at which type was previously seen for _, e := range list { - if i, found := prev[e.typ]; found { + if i, found := lookupType(prev, e.typ); found { list[i].multiples = true // ignore this entry } else { @@ -224,6 +222,21 @@ func consolidateMultiples(list []embeddedType) []embeddedType { return list[:n] } +func lookupType(m map[Type]int, typ Type) (int, bool) { + // fast path: maybe the types are equal + if i, found := m[typ]; found { + return i, true + } + + for t, i := range m { + if Identical(t, typ) { + return i, true + } + } + + return 0, false +} + // MissingMethod returns (nil, false) if V implements T, otherwise it // returns a missing method required by T and whether it is missing or // just has the wrong type. diff --git a/libgo/go/go/types/methodset.go b/libgo/go/go/types/methodset.go index b27f2dac34a..4f791d9d51c 100644 --- a/libgo/go/go/types/methodset.go +++ b/libgo/go/go/types/methodset.go @@ -72,24 +72,22 @@ func NewMethodSet(T Type) *MethodSet { var base methodSet typ, isPtr := deref(T) - named, _ := typ.(*Named) // *typ where typ is an interface has no methods. - if isPtr { - utyp := typ - if named != nil { - utyp = named.underlying - } - if _, ok := utyp.(*Interface); ok { - return &emptyMethodSet - } + if isPtr && IsInterface(typ) { + return &emptyMethodSet } // Start with typ as single entry at shallowest depth. - // If typ is not a named type, insert a nil type instead. - current := []embeddedType{{named, nil, isPtr, false}} - - // named types that we have seen already, allocated lazily + current := []embeddedType{{typ, nil, isPtr, false}} + + // Named types that we have seen already, allocated lazily. + // Used to avoid endless searches in case of recursive types. + // Since only Named types can be used for recursive types, we + // only need to track those. + // (If we ever allow type aliases to construct recursive types, + // we must use type identity rather than pointer equality for + // the map key comparison, as we do in consolidateMultiples.) var seen map[*Named]bool // collect methods at current depth @@ -101,11 +99,12 @@ func NewMethodSet(T Type) *MethodSet { var mset methodSet for _, e := range current { - // The very first time only, e.typ may be nil. - // In this case, we don't have a named type and - // we simply continue with the underlying type. - if e.typ != nil { - if seen[e.typ] { + typ := e.typ + + // If we have a named type, we may have associated methods. + // Look for those first. + if named, _ := typ.(*Named); named != nil { + if seen[named] { // We have seen this type before, at a more shallow depth // (note that multiples of this type at the current depth // were consolidated before). The type at that depth shadows @@ -116,12 +115,12 @@ func NewMethodSet(T Type) *MethodSet { if seen == nil { seen = make(map[*Named]bool) } - seen[e.typ] = true + seen[named] = true - mset = mset.add(e.typ.methods, e.index, e.indirect, e.multiples) + mset = mset.add(named.methods, e.index, e.indirect, e.multiples) // continue with underlying type - typ = e.typ.underlying + typ = named.underlying } switch t := typ.(type) { @@ -130,16 +129,15 @@ func NewMethodSet(T Type) *MethodSet { fset = fset.add(f, e.multiples) // Embedded fields are always of the form T or *T where - // T is a named type. If typ appeared multiple times at + // T is a type name. If typ appeared multiple times at // this depth, f.Type appears multiple times at the next // depth. if f.anonymous { - // Ignore embedded basic types - only user-defined - // named types can have methods or struct fields. typ, isPtr := deref(f.typ) - if t, _ := typ.(*Named); t != nil { - next = append(next, embeddedType{t, concat(e.index, i), e.indirect || isPtr, e.multiples}) - } + // TODO(gri) optimization: ignore types that can't + // have fields or methods (only Named, Struct, and + // Interface types need to be considered). + next = append(next, embeddedType{typ, concat(e.index, i), e.indirect || isPtr, e.multiples}) } } diff --git a/libgo/go/go/types/object.go b/libgo/go/go/types/object.go index 6c0c5c4a244..f0bcd6766b5 100644 --- a/libgo/go/go/types/object.go +++ b/libgo/go/go/types/object.go @@ -25,7 +25,7 @@ type Object interface { Name() string // package local object name Type() Type // object type Exported() bool // reports whether the name starts with a capital letter - Id() string // object id (see Id below) + Id() string // object name if exported, qualified name if not exported (see func Id) // String returns a human-readable string of the object. String() string @@ -64,15 +64,10 @@ func Id(pkg *Package, name string) string { // inside a package and outside a package - which breaks some // tests) path := "_" - // TODO(gri): shouldn't !ast.IsExported(name) => pkg != nil be an precondition? - // if pkg == nil { - // panic("nil package in lookup of unexported name") - // } - if pkg != nil { + // pkg is nil for objects in Universe scope and possibly types + // introduced via Eval (see also comment in object.sameId) + if pkg != nil && pkg.path != "" { path = pkg.path - if path == "" { - path = "_" - } } return path + "." + name } @@ -154,7 +149,7 @@ func NewConst(pos token.Pos, pkg *Package, name string, typ Type, val constant.V func (obj *Const) Val() constant.Value { return obj.val } func (*Const) isDependency() {} // a constant may be a dependency of an initialization expression -// A TypeName represents a declared type. +// A TypeName represents a name for a (named or alias) type. type TypeName struct { object } @@ -163,6 +158,30 @@ func NewTypeName(pos token.Pos, pkg *Package, name string, typ Type) *TypeName { return &TypeName{object{nil, pos, pkg, name, typ, 0, token.NoPos}} } +// IsAlias reports whether obj is an alias name for a type. +func (obj *TypeName) IsAlias() bool { + switch t := obj.typ.(type) { + case nil: + return false + case *Basic: + // unsafe.Pointer is not an alias. + if obj.pkg == Unsafe { + return false + } + // Any user-defined type name for a basic type is an alias for a + // basic type (because basic types are pre-declared in the Universe + // scope, outside any package scope), and so is any type name with + // a different name than the name of the basic type it refers to. + // Additionally, we need to look for "byte" and "rune" because they + // are aliases but have the same names (for better error messages). + return obj.pkg != nil || t.name != obj.name || t == universeByte || t == universeRune + case *Named: + return obj != t.obj + default: + return true + } +} + // A Variable represents a declared variable (including function parameters and results, and struct fields). type Var struct { object @@ -215,28 +234,6 @@ func (obj *Func) FullName() string { func (obj *Func) Scope() *Scope { return obj.typ.(*Signature).scope } func (*Func) isDependency() {} // a function may be a dependency of an initialization expression -// An Alias represents a declared alias. -type disabledAlias struct { - object - orig Object // aliased constant, type, variable, or function; never an alias - kind token.Token // token.CONST, token.TYPE, token.VAR, or token.FUNC (only needed during resolve phase) -} - -func disabledNewAlias(pos token.Pos, pkg *Package, name string, orig Object) *disabledAlias { - var typ Type = Typ[Invalid] - if orig != nil { - typ = orig.Type() - } - // No need to set a valid Alias.kind - that field is only used during identifier - // resolution (1st type-checker pass). We could store the field outside but it's - // easier to keep it here. - return &disabledAlias{object{nil, pos, pkg, name, typ, 0, token.NoPos}, orig, token.ILLEGAL} -} - -// Orig returns the aliased object, or nil if there was an error. -// The returned object is never an Alias. -func (obj *disabledAlias) disabledOrig() Object { return obj.orig } - // A Label represents a declared label. type Label struct { object @@ -264,7 +261,9 @@ type Nil struct { } func writeObject(buf *bytes.Buffer, obj Object, qf Qualifier) { + var tname *TypeName typ := obj.Type() + switch obj := obj.(type) { case *PkgName: fmt.Fprintf(buf, "package %s", obj.Name()) @@ -277,8 +276,8 @@ func writeObject(buf *bytes.Buffer, obj Object, qf Qualifier) { buf.WriteString("const") case *TypeName: + tname = obj buf.WriteString("type") - typ = typ.Underlying() case *Var: if obj.isField { @@ -295,10 +294,6 @@ func writeObject(buf *bytes.Buffer, obj Object, qf Qualifier) { } return - // Alias-related code. Keep for now. - // case *Alias: - // buf.WriteString("alias") - case *Label: buf.WriteString("label") typ = nil @@ -322,10 +317,27 @@ func writeObject(buf *bytes.Buffer, obj Object, qf Qualifier) { writePackage(buf, obj.Pkg(), qf) } buf.WriteString(obj.Name()) - if typ != nil { - buf.WriteByte(' ') - WriteType(buf, typ, qf) + + if typ == nil { + return } + + if tname != nil { + // We have a type object: Don't print anything more for + // basic types since there's no more information (names + // are the same; see also comment in TypeName.IsAlias). + if _, ok := typ.(*Basic); ok { + return + } + if tname.IsAlias() { + buf.WriteString(" =") + } else { + typ = typ.Underlying() + } + } + + buf.WriteByte(' ') + WriteType(buf, typ, qf) } func writePackage(buf *bytes.Buffer, pkg *Package, qf Qualifier) { @@ -353,15 +365,14 @@ func ObjectString(obj Object, qf Qualifier) string { return buf.String() } -func (obj *PkgName) String() string { return ObjectString(obj, nil) } -func (obj *Const) String() string { return ObjectString(obj, nil) } -func (obj *TypeName) String() string { return ObjectString(obj, nil) } -func (obj *Var) String() string { return ObjectString(obj, nil) } -func (obj *Func) String() string { return ObjectString(obj, nil) } -func (obj *disabledAlias) String() string { return ObjectString(obj, nil) } -func (obj *Label) String() string { return ObjectString(obj, nil) } -func (obj *Builtin) String() string { return ObjectString(obj, nil) } -func (obj *Nil) String() string { return ObjectString(obj, nil) } +func (obj *PkgName) String() string { return ObjectString(obj, nil) } +func (obj *Const) String() string { return ObjectString(obj, nil) } +func (obj *TypeName) String() string { return ObjectString(obj, nil) } +func (obj *Var) String() string { return ObjectString(obj, nil) } +func (obj *Func) String() string { return ObjectString(obj, nil) } +func (obj *Label) String() string { return ObjectString(obj, nil) } +func (obj *Builtin) String() string { return ObjectString(obj, nil) } +func (obj *Nil) String() string { return ObjectString(obj, nil) } func writeFuncName(buf *bytes.Buffer, f *Func, qf Qualifier) { if f.typ != nil { diff --git a/libgo/go/go/types/object_test.go b/libgo/go/go/types/object_test.go new file mode 100644 index 00000000000..b0acdd91bac --- /dev/null +++ b/libgo/go/go/types/object_test.go @@ -0,0 +1,44 @@ +// Copyright 2016 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package types + +import "testing" + +func TestIsAlias(t *testing.T) { + check := func(obj *TypeName, want bool) { + if got := obj.IsAlias(); got != want { + t.Errorf("%v: got IsAlias = %v; want %v", obj, got, want) + } + } + + // predeclared types + check(Unsafe.Scope().Lookup("Pointer").(*TypeName), false) + for _, name := range Universe.Names() { + if obj, _ := Universe.Lookup(name).(*TypeName); obj != nil { + check(obj, name == "byte" || name == "rune") + } + } + + // various other types + pkg := NewPackage("p", "p") + t1 := NewTypeName(0, pkg, "t1", nil) + n1 := NewNamed(t1, new(Struct), nil) + for _, test := range []struct { + name *TypeName + alias bool + }{ + {NewTypeName(0, nil, "t0", nil), false}, // no type yet + {NewTypeName(0, pkg, "t0", nil), false}, // no type yet + {t1, false}, // type name refers to named type and vice versa + {NewTypeName(0, nil, "t2", new(Interface)), true}, // type name refers to unnamed type + {NewTypeName(0, pkg, "t3", n1), true}, // type name refers to named type with different type name + {NewTypeName(0, nil, "t4", Typ[Int32]), true}, // type name refers to basic type with different name + {NewTypeName(0, nil, "int32", Typ[Int32]), false}, // type name refers to basic type with same name + {NewTypeName(0, pkg, "int32", Typ[Int32]), true}, // type name is declared in user-defined package (outside Universe) + {NewTypeName(0, nil, "rune", Typ[Rune]), true}, // type name refers to basic type rune which is an alias already + } { + check(test.name, test.alias) + } +} diff --git a/libgo/go/go/types/operand.go b/libgo/go/go/types/operand.go index b2f16b64d80..07247bd6f58 100644 --- a/libgo/go/go/types/operand.go +++ b/libgo/go/go/types/operand.go @@ -122,13 +122,15 @@ func operandString(x *operand, qf Qualifier) string { case invalid, novalue, builtin, typexpr: // no type default: - // has type - if isUntyped(x.typ) { - buf.WriteString(x.typ.(*Basic).name) - buf.WriteByte(' ') - break + // should have a type, but be cautious (don't crash during printing) + if x.typ != nil { + if isUntyped(x.typ) { + buf.WriteString(x.typ.(*Basic).name) + buf.WriteByte(' ') + break + } + hasType = true } - hasType = true } // diff --git a/libgo/go/go/types/package.go b/libgo/go/go/types/package.go index a588ee73dce..cd202a0ed9f 100644 --- a/libgo/go/go/types/package.go +++ b/libgo/go/go/types/package.go @@ -19,13 +19,9 @@ type Package struct { fake bool // scope lookup errors are silently dropped if package is fake (internal use only) } -// NewPackage returns a new Package for the given package path and name; -// the name must not be the blank identifier. +// NewPackage returns a new Package for the given package path and name. // The package is not complete and contains no explicit imports. func NewPackage(path, name string) *Package { - if name == "_" { - panic("invalid package name _") - } scope := NewScope(Universe, token.NoPos, token.NoPos, fmt.Sprintf("package %q", path)) return &Package{path: path, name: name, scope: scope} } @@ -52,7 +48,7 @@ func (pkg *Package) Complete() bool { return pkg.complete } func (pkg *Package) MarkComplete() { pkg.complete = true } // Imports returns the list of packages directly imported by -// pkg; the list is in source order. Package unsafe is excluded. +// pkg; the list is in source order. // // If pkg was loaded from export data, Imports includes packages that // provide package-level objects referenced by pkg. This may be more or diff --git a/libgo/go/go/types/predicates.go b/libgo/go/go/types/predicates.go index 21fd81e3c26..c3b87dd9cd4 100644 --- a/libgo/go/go/types/predicates.go +++ b/libgo/go/go/types/predicates.go @@ -139,7 +139,7 @@ func identical(x, y Type, cmpTags bool, p *ifacePair) bool { case *Basic: // Basic types are singletons except for the rune and byte // aliases, thus we cannot solely rely on the x == y check - // above. + // above. See also comment in TypeName.IsAlias. if y, ok := y.(*Basic); ok { return x.kind == y.kind } diff --git a/libgo/go/go/types/resolver.go b/libgo/go/go/types/resolver.go index 046e147456d..05603b3442d 100644 --- a/libgo/go/go/types/resolver.go +++ b/libgo/go/go/types/resolver.go @@ -14,13 +14,14 @@ import ( "unicode" ) -// A declInfo describes a package-level const, type, var, func, or alias declaration. +// A declInfo describes a package-level const, type, var, or func declaration. type declInfo struct { file *Scope // scope of file containing this declaration lhs []*Var // lhs of n:1 variable declarations, or nil typ ast.Expr // type, or nil init ast.Expr // init/orig expression, or nil fdecl *ast.FuncDecl // func declaration, or nil + alias bool // type alias declaration // The deps field tracks initialization expression dependencies. // As a special (overloaded) case, it also tracks dependencies of @@ -124,6 +125,73 @@ func (check *Checker) filename(fileNo int) string { return fmt.Sprintf("file[%d]", fileNo) } +func (check *Checker) importPackage(pos token.Pos, path, dir string) *Package { + // If we already have a package for the given (path, dir) + // pair, use it instead of doing a full import. + // Checker.impMap only caches packages that are marked Complete + // or fake (dummy packages for failed imports). Incomplete but + // non-fake packages do require an import to complete them. + key := importKey{path, dir} + imp := check.impMap[key] + if imp != nil { + return imp + } + + // no package yet => import it + if path == "C" && check.conf.FakeImportC { + imp = NewPackage("C", "C") + imp.fake = true + } else { + // ordinary import + var err error + if importer := check.conf.Importer; importer == nil { + err = fmt.Errorf("Config.Importer not installed") + } else if importerFrom, ok := importer.(ImporterFrom); ok { + imp, err = importerFrom.ImportFrom(path, dir, 0) + if imp == nil && err == nil { + err = fmt.Errorf("Config.Importer.ImportFrom(%s, %s, 0) returned nil but no error", path, dir) + } + } else { + imp, err = importer.Import(path) + if imp == nil && err == nil { + err = fmt.Errorf("Config.Importer.Import(%s) returned nil but no error", path) + } + } + // make sure we have a valid package name + // (errors here can only happen through manipulation of packages after creation) + if err == nil && imp != nil && (imp.name == "_" || imp.name == "") { + err = fmt.Errorf("invalid package name: %q", imp.name) + imp = nil // create fake package below + } + if err != nil { + check.errorf(pos, "could not import %s (%s)", path, err) + if imp == nil { + // create a new fake package + // come up with a sensible package name (heuristic) + name := path + if i := len(name); i > 0 && name[i-1] == '/' { + name = name[:i-1] + } + if i := strings.LastIndex(name, "/"); i >= 0 { + name = name[i+1:] + } + imp = NewPackage(path, name) + } + // continue to use the package as best as we can + imp.fake = true // avoid follow-up lookup failures + } + } + + // package should be complete or marked fake, but be cautious + if imp.complete || imp.fake { + check.impMap[key] = imp + return imp + } + + // something went wrong (importer may have returned incomplete package without error) + return nil +} + // collectObjects collects all file and package objects and inserts them // into their respective scopes. It also performs imports and associates // methods with receiver base type names. @@ -133,25 +201,14 @@ func (check *Checker) collectObjects() { // pkgImports is the set of packages already imported by any package file seen // so far. Used to avoid duplicate entries in pkg.imports. Allocate and populate // it (pkg.imports may not be empty if we are checking test files incrementally). + // Note that pkgImports is keyed by package (and thus package path), not by an + // importKey value. Two different importKey values may map to the same package + // which is why we cannot use the check.impMap here. var pkgImports = make(map[*Package]bool) for _, imp := range pkg.imports { pkgImports[imp] = true } - // srcDir is the directory used by the Importer to look up packages. - // The typechecker itself doesn't need this information so it is not - // explicitly provided. Instead, we extract it from position info of - // the source files as needed. - // This is the only place where the type-checker (just the importer) - // needs to know the actual source location of a file. - // TODO(gri) can we come up with a better API instead? - var srcDir string - if len(check.files) > 0 { - // FileName may be "" (typically for tests) in which case - // we get "." as the srcDir which is what we would want. - srcDir = dir(check.fset.Position(check.files[0].Name.Pos()).Filename) - } - for fileNo, file := range check.files { // The package identifier denotes the current package, // but there is no corresponding package object. @@ -167,6 +224,11 @@ func (check *Checker) collectObjects() { fileScope := NewScope(check.pkg.scope, pos, end, check.filename(fileNo)) check.recordScope(file, fileScope) + // determine file directory, necessary to resolve imports + // FileName may be "" (typically for tests) in which case + // we get "." as the directory which is what we would want. + fileDir := dir(check.fset.Position(file.Name.Pos()).Filename) + for _, decl := range file.Decls { switch d := decl.(type) { case *ast.BadDecl: @@ -178,35 +240,15 @@ func (check *Checker) collectObjects() { switch s := spec.(type) { case *ast.ImportSpec: // import package - var imp *Package path, err := validatedImportPath(s.Path.Value) if err != nil { check.errorf(s.Path.Pos(), "invalid import path (%s)", err) continue } - if path == "C" && check.conf.FakeImportC { - // TODO(gri) shouldn't create a new one each time - imp = NewPackage("C", "C") - imp.fake = true - } else { - // ordinary import - if importer := check.conf.Importer; importer == nil { - err = fmt.Errorf("Config.Importer not installed") - } else if importerFrom, ok := importer.(ImporterFrom); ok { - imp, err = importerFrom.ImportFrom(path, srcDir, 0) - if imp == nil && err == nil { - err = fmt.Errorf("Config.Importer.ImportFrom(%s, %s, 0) returned nil but no error", path, pkg.path) - } - } else { - imp, err = importer.Import(path) - if imp == nil && err == nil { - err = fmt.Errorf("Config.Importer.Import(%s) returned nil but no error", path) - } - } - if err != nil { - check.errorf(s.Path.Pos(), "could not import %s (%s)", path, err) - continue - } + + imp := check.importPackage(s.Path.Pos(), path, fileDir) + if imp == nil { + continue } // add package to list of explicit imports @@ -214,9 +256,7 @@ func (check *Checker) collectObjects() { // for clients; it is not needed for type-checking) if !pkgImports[imp] { pkgImports[imp] = true - if imp != Unsafe { - pkg.imports = append(pkg.imports, imp) - } + pkg.imports = append(pkg.imports, imp) } // local name overrides imported package name @@ -274,13 +314,6 @@ func (check *Checker) collectObjects() { check.declare(fileScope, nil, obj, token.NoPos) } - // Alias-related code. Keep for now. - // case *ast.AliasSpec: - // obj := NewAlias(s.Name.Pos(), pkg, s.Name.Name, nil) - // obj.typ = nil // unresolved - // obj.kind = d.Tok - // check.declarePkgObj(s.Name, obj, &declInfo{file: fileScope, init: s.Orig}) - case *ast.ValueSpec: switch d.Tok { case token.CONST: @@ -347,7 +380,7 @@ func (check *Checker) collectObjects() { case *ast.TypeSpec: obj := NewTypeName(s.Name.Pos(), pkg, s.Name.Name, nil) - check.declarePkgObj(s.Name, obj, &declInfo{file: fileScope, typ: s.Type}) + check.declarePkgObj(s.Name, obj, &declInfo{file: fileScope, typ: s.Type, alias: s.Assign.IsValid()}) default: check.invalidAST(s.Pos(), "unknown ast.Spec node %T", s) diff --git a/libgo/go/go/types/sizes.go b/libgo/go/go/types/sizes.go index 3bbe5aee40a..0821a61359d 100644 --- a/libgo/go/go/types/sizes.go +++ b/libgo/go/go/types/sizes.go @@ -153,8 +153,43 @@ func (s *StdSizes) Sizeof(T Type) int64 { return s.WordSize // catch-all } +// common architecture word sizes and alignments +var gcArchSizes = map[string]*StdSizes{ + "386": {4, 4}, + "arm": {4, 4}, + "arm64": {8, 8}, + "amd64": {8, 8}, + "amd64p32": {4, 8}, + "mips": {4, 4}, + "mipsle": {4, 4}, + "mips64": {8, 8}, + "mips64le": {8, 8}, + "ppc64": {8, 8}, + "ppc64le": {8, 8}, + "s390x": {8, 8}, + // When adding more architectures here, + // update the doc string of SizesFor below. +} + +// SizesFor returns the Sizes used by a compiler for an architecture. +// The result is nil if a compiler/architecture pair is not known. +// +// Supported architectures for compiler "gc": +// "386", "arm", "arm64", "amd64", "amd64p32", "mips", "mipsle", +// "mips64", "mips64le", "ppc64", "ppc64le", "s390x". +func SizesFor(compiler, arch string) Sizes { + if compiler != "gc" { + return nil + } + s, ok := gcArchSizes[arch] + if !ok { + return nil + } + return s +} + // stdSizes is used if Config.Sizes == nil. -var stdSizes = StdSizes{8, 8} +var stdSizes = SizesFor("gc", "amd64") func (conf *Config) alignof(T Type) int64 { if s := conf.Sizes; s != nil { diff --git a/libgo/go/go/types/stdlib_test.go b/libgo/go/go/types/stdlib_test.go index 8fc51d4b17d..56c48b7547e 100644 --- a/libgo/go/go/types/stdlib_test.go +++ b/libgo/go/go/types/stdlib_test.go @@ -101,12 +101,23 @@ func testTestDir(t *testing.T, path string, ignore ...string) { // get per-file instructions expectErrors := false filename := filepath.Join(path, f.Name()) - if cmd := firstComment(filename); cmd != "" { - switch cmd { + if comment := firstComment(filename); comment != "" { + fields := strings.Fields(comment) + switch fields[0] { case "skip", "compiledir": continue // ignore this file case "errorcheck": expectErrors = true + for _, arg := range fields[1:] { + if arg == "-0" || arg == "-+" || arg == "-std" { + // Marked explicitly as not expected errors (-0), + // or marked as compiling runtime/stdlib, which is only done + // to trigger runtime/stdlib-only error output. + // In both cases, the code should typecheck. + expectErrors = false + break + } + } } } @@ -163,6 +174,9 @@ func TestStdFixed(t *testing.T) { "issue15002.go", // uses Mmap; testTestDir should consult build tags "issue16369.go", // go/types handles this correctly - not an issue "issue18459.go", // go/types doesn't check validity of //go:xxx directives + "issue18882.go", // go/types doesn't check validity of //go:xxx directives + "issue20232.go", // go/types handles larger constants than gc + "issue20529.go", // go/types does not have constraints on stack size ) } diff --git a/libgo/go/go/types/testdata/builtins.src b/libgo/go/go/types/testdata/builtins.src index 7fb7b58a48d..ecdba51553e 100644 --- a/libgo/go/go/types/testdata/builtins.src +++ b/libgo/go/go/types/testdata/builtins.src @@ -19,6 +19,7 @@ func append1() { _ = append(nil /* ERROR not a slice */ , s) _ = append(x /* ERROR not a slice */ , s) _ = append(s) + _ = append(s, nil...) append /* ERROR not used */ (s) _ = append(s, b) diff --git a/libgo/go/go/types/testdata/cycles.src b/libgo/go/go/types/testdata/cycles.src index 621d83c9450..b4bd5d8b157 100644 --- a/libgo/go/go/types/testdata/cycles.src +++ b/libgo/go/go/types/testdata/cycles.src @@ -140,4 +140,13 @@ func (*T12) m() {} type ( P3 *T13 T13 /* ERROR cycle */ T13 -) \ No newline at end of file +) + +// test cases for issue 18643 +// (type cycle detection when non-type expressions are involved) +type ( + T14 [len(T14 /* ERROR cycle */ {})]int + T15 [][len(T15 /* ERROR cycle */ {})]int + T16 map[[len(T16 /* ERROR cycle */ {1:2})]int]int + T17 map[int][len(T17 /* ERROR cycle */ {1:2})]int +) diff --git a/libgo/go/go/types/testdata/decls1.src b/libgo/go/go/types/testdata/decls1.src index cb162f7aa7b..1ef28067645 100644 --- a/libgo/go/go/types/testdata/decls1.src +++ b/libgo/go/go/types/testdata/decls1.src @@ -78,7 +78,7 @@ var ( u2 = iface.([]int) u3 = iface.(a /* ERROR "not a type" */ ) u4, ok = iface.(int) - u5, ok2, ok3 = iface /* ERROR "assignment count mismatch" */ .(int) + u5, ok2, ok3 = iface /* ERROR "cannot initialize" */ .(int) ) // Constant expression initializations diff --git a/libgo/go/go/types/testdata/expr3.src b/libgo/go/go/types/testdata/expr3.src index ab1a9f684b8..95d5c09be09 100644 --- a/libgo/go/go/types/testdata/expr3.src +++ b/libgo/go/go/types/testdata/expr3.src @@ -207,7 +207,7 @@ func struct_literals() { _ = time.Time{} _ = time.Time{sec /* ERROR "unknown field" */ : 0} _ = time.Time{ - 0 /* ERROR implicit assignment to unexported field sec in time.Time literal */, + 0 /* ERROR implicit assignment to unexported field wall in time.Time literal */, 0 /* ERROR implicit assignment */ , nil /* ERROR implicit assignment */ , } diff --git a/libgo/go/go/types/testdata/issues.src b/libgo/go/go/types/testdata/issues.src index 6579aa3b117..8729555e175 100644 --- a/libgo/go/go/types/testdata/issues.src +++ b/libgo/go/go/types/testdata/issues.src @@ -98,8 +98,8 @@ func issue10979() { // issue11347 // These should not crash. var a1, b1 /* ERROR cycle */ , c1 /* ERROR cycle */ b1 = 0 > 0<<""[""[c1]]>c1 -var a2, b2 /* ERROR cycle */ = 0 /* ERROR mismatch */ /* ERROR mismatch */ > 0<<""[b2] -var a3, b3 /* ERROR cycle */ = int /* ERROR mismatch */ /* ERROR mismatch */ (1<<""[b3]) +var a2, b2 /* ERROR cycle */ = 0 /* ERROR cannot initialize */ /* ERROR cannot initialize */ > 0<<""[b2] +var a3, b3 /* ERROR cycle */ = int /* ERROR cannot initialize */ /* ERROR cannot initialize */ (1<<""[b3]) // issue10260 // Check that error messages explain reason for interface assignment failures. @@ -186,3 +186,24 @@ func issue15755() { _ = u _ = v } + +// Test that we don't get "declared but not used" +// errors in the context of invalid/C objects. +func issue20358() { + var F C /* ERROR "undeclared" */ .F + var A C /* ERROR "undeclared" */ .A + var S C /* ERROR "undeclared" */ .S + type T C /* ERROR "undeclared" */ .T + type P C /* ERROR "undeclared" */ .P + + // these variables must be "used" even though + // the LHS expressions/types below in which + // context they are used are unknown/invalid + var f, a, s1, s2, s3, t, p int + + _ = F(f) + _ = A[a] + _ = S[s1:s2:s3] + _ = T{t} + _ = P{f: p} +} diff --git a/libgo/go/go/types/testdata/shifts.src b/libgo/go/go/types/testdata/shifts.src index 099c9ecc7cd..dc029fc647a 100644 --- a/libgo/go/go/types/testdata/shifts.src +++ b/libgo/go/go/types/testdata/shifts.src @@ -10,7 +10,7 @@ func shifts0() { s = 10 _ = 0<<0 _ = 1<> 1.1 /* ERROR "must be unsigned integer" */ // example from issue 11325 - _ = 0 >> 1.1 /* ERROR "must be unsigned integer" */ - _ = 0 << 1.1 /* ERROR "must be unsigned integer" */ + var _ = 0 >> 1.1 /* ERROR "truncated to uint" */ // example from issue 11325 + _ = 0 >> 1.1 /* ERROR "truncated to uint" */ + _ = 0 << 1.1 /* ERROR "truncated to uint" */ _ = 0 >> 1. - _ = 1 >> 1.1 /* ERROR "must be unsigned integer" */ + _ = 1 >> 1.1 /* ERROR "truncated to uint" */ _ = 1 >> 1. _ = 1. >> 1 _ = 1. >> 1. diff --git a/libgo/go/go/types/testdata/stmt0.src b/libgo/go/go/types/testdata/stmt0.src index 87f08e4314f..446997ac098 100644 --- a/libgo/go/go/types/testdata/stmt0.src +++ b/libgo/go/go/types/testdata/stmt0.src @@ -15,19 +15,19 @@ func assignments0() (int, int) { f3 := func() (int, int, int) { return 1, 2, 3 } a, b, c = 1, 2, 3 - a, b, c = 1 /* ERROR "assignment count mismatch" */ , 2 - a, b, c = 1 /* ERROR "assignment count mismatch" */ , 2, 3, 4 + a, b, c = 1 /* ERROR "cannot assign [1-9]+ values to [1-9]+ variables" */ , 2 + a, b, c = 1 /* ERROR "cannot assign [1-9]+ values to [1-9]+ variables" */ , 2, 3, 4 _, _, _ = a, b, c a = f0 /* ERROR "used as value" */ () a = f1() - a = f2 /* ERROR "assignment count mismatch" */ () + a = f2 /* ERROR "cannot assign [1-9]+ values to [1-9]+ variables" */ () a, b = f2() - a, b, c = f2 /* ERROR "assignment count mismatch" */ () + a, b, c = f2 /* ERROR "cannot assign [1-9]+ values to [1-9]+ variables" */ () a, b, c = f3() - a, b = f3 /* ERROR "assignment count mismatch" */ () + a, b = f3 /* ERROR "cannot assign [1-9]+ values to [1-9]+ variables" */ () - a, b, c = <- /* ERROR "assignment count mismatch" */ ch + a, b, c = <- /* ERROR "cannot assign [1-9]+ values to [1-9]+ variables" */ ch return /* ERROR "wrong number of return values" */ return /* ERROR "wrong number of return values" */ 1 @@ -43,7 +43,7 @@ func assignments1() { c = s /* ERROR "cannot use .* in assignment" */ s = b /* ERROR "cannot use .* in assignment" */ - v0, v1, v2 := 1 /* ERROR "mismatch" */ , 2, 3, 4 + v0, v1, v2 := 1 /* ERROR "cannot initialize" */ , 2, 3, 4 _, _, _ = v0, v1, v2 b = true @@ -108,7 +108,7 @@ func assignments2() { s, b = m["foo"] _, d = m["bar"] m["foo"] = nil - m["foo"] = nil /* ERROR assignment count mismatch */ , false + m["foo"] = nil /* ERROR cannot assign [1-9]+ values to [1-9]+ variables */ , false _ = append(m["foo"]) _ = append(m["foo"], true) @@ -116,12 +116,12 @@ func assignments2() { _, b = <-c _, d = <-c <- /* ERROR cannot assign */ c = 0 - <-c = 0 /* ERROR assignment count mismatch */ , false + <-c = 0 /* ERROR cannot assign [1-9]+ values to [1-9]+ variables */ , false var x interface{} _, b = x.(int) x /* ERROR cannot assign */ .(int) = 0 - x.(int) = 0 /* ERROR assignment count mismatch */ , false + x.(int) = 0 /* ERROR cannot assign [1-9]+ values to [1-9]+ variables */ , false assignments2 /* ERROR used as value */ () = nil int /* ERROR not an expression */ = 0 diff --git a/libgo/go/go/types/testdata/vardecl.src b/libgo/go/go/types/testdata/vardecl.src index 00825371f2c..35f44e6c48b 100644 --- a/libgo/go/go/types/testdata/vardecl.src +++ b/libgo/go/go/types/testdata/vardecl.src @@ -28,39 +28,39 @@ var _ = f /* ERROR "used as value" */ () // Identifier and expression arity must match. var _, _ = 1, 2 var _ = 1, 2 /* ERROR "extra init expr 2" */ -var _, _ = 1 /* ERROR "assignment count mismatch" */ +var _, _ = 1 /* ERROR "cannot initialize [0-9]+ variables with [0-9]+ values" */ var _, _, _ /* ERROR "missing init expr for _" */ = 1, 2 var _ = g /* ERROR "2-valued g" */ () var _, _ = g() -var _, _, _ = g /* ERROR "assignment count mismatch" */ () +var _, _, _ = g /* ERROR "cannot initialize [0-9]+ variables with [0-9]+ values" */ () var _ = m["foo"] var _, _ = m["foo"] -var _, _, _ = m /* ERROR "assignment count mismatch" */ ["foo"] +var _, _, _ = m /* ERROR "cannot initialize [0-9]+ variables with [0-9]+ values" */ ["foo"] var _, _ int = 1, 2 var _ int = 1, 2 /* ERROR "extra init expr 2" */ -var _, _ int = 1 /* ERROR "assignment count mismatch" */ +var _, _ int = 1 /* ERROR "cannot initialize [0-9]+ variables with [0-9]+ values" */ var _, _, _ /* ERROR "missing init expr for _" */ int = 1, 2 var ( _, _ = 1, 2 _ = 1, 2 /* ERROR "extra init expr 2" */ - _, _ = 1 /* ERROR "assignment count mismatch" */ + _, _ = 1 /* ERROR "cannot initialize [0-9]+ variables with [0-9]+ values" */ _, _, _ /* ERROR "missing init expr for _" */ = 1, 2 _ = g /* ERROR "2-valued g" */ () _, _ = g() - _, _, _ = g /* ERROR "assignment count mismatch" */ () + _, _, _ = g /* ERROR "cannot initialize [0-9]+ variables with [0-9]+ values" */ () _ = m["foo"] _, _ = m["foo"] - _, _, _ = m /* ERROR "assignment count mismatch" */ ["foo"] + _, _, _ = m /* ERROR "cannot initialize [0-9]+ variables with [0-9]+ values" */ ["foo"] _, _ int = 1, 2 _ int = 1, 2 /* ERROR "extra init expr 2" */ - _, _ int = 1 /* ERROR "assignment count mismatch" */ + _, _ int = 1 /* ERROR "cannot initialize [0-9]+ variables with [0-9]+ values" */ _, _, _ /* ERROR "missing init expr for _" */ int = 1, 2 ) @@ -155,7 +155,7 @@ func (r T) _(a, b, c int) (u, v, w int) { func _() { var a, b, c int var x, y int - x, y = a /* ERROR assignment count mismatch */ , b, c + x, y = a /* ERROR cannot assign [0-9]+ values to [0-9]+ variables */ , b, c _ = x _ = y } diff --git a/libgo/go/go/types/type.go b/libgo/go/go/types/type.go index 01adee8a3ef..a0a12383953 100644 --- a/libgo/go/go/types/type.go +++ b/libgo/go/go/types/type.go @@ -394,7 +394,7 @@ func NewNamed(obj *TypeName, underlying Type, methods []*Func) *Named { return typ } -// TypeName returns the type name for the named type t. +// Obj returns the type name for the named type t. func (t *Named) Obj() *TypeName { return t.obj } // NumMethods returns the number of explicit methods whose receiver is named type t. diff --git a/libgo/go/go/types/typestring.go b/libgo/go/go/types/typestring.go index 47378e744c0..0f8a7adc247 100644 --- a/libgo/go/go/types/typestring.go +++ b/libgo/go/go/types/typestring.go @@ -56,6 +56,7 @@ func RelativeTo(pkg *Package) Qualifier { // This flag is exported in the x/tools/go/types package. We don't // need it at the moment in the std repo and so we don't export it // anymore. We should eventually try to remove it altogether. +// TODO(gri) remove this var gcCompatibilityMode bool // TypeString returns the string representation of typ. diff --git a/libgo/go/go/types/typestring_test.go b/libgo/go/go/types/typestring_test.go index 52aa6b67e04..7f7c1064b0a 100644 --- a/libgo/go/go/types/typestring_test.go +++ b/libgo/go/go/types/typestring_test.go @@ -17,8 +17,7 @@ import ( const filename = "" -func makePkg(t *testing.T, src string) (*Package, error) { - t.Skip("skipping for gccgo--no importer") +func makePkg(src string) (*Package, error) { fset := token.NewFileSet() file, err := parser.ParseFile(fset, filename, src, parser.DeclarationErrors) if err != nil { @@ -127,7 +126,7 @@ func TestTypeString(t *testing.T) { for _, test := range tests { src := `package p; import "io"; type _ io.Writer; type T ` + test.src - pkg, err := makePkg(t, src) + pkg, err := makePkg(src) if err != nil { t.Errorf("%s: %s", src, err) continue diff --git a/libgo/go/go/types/typexpr.go b/libgo/go/go/types/typexpr.go index ecc0a7da021..5f1587bf0f5 100644 --- a/libgo/go/go/types/typexpr.go +++ b/libgo/go/go/types/typexpr.go @@ -45,17 +45,6 @@ func (check *Checker) ident(x *operand, e *ast.Ident, def *Named, path []*TypeNa delete(check.unusedDotImports[scope], pkg) } - // Alias-related code. Keep for now. - // An alias stands for the original object; use that one instead. - // TODO(gri) We should be able to factor out the Typ[Invalid] test. - // if alias, _ := obj.(*Alias); alias != nil { - // obj = original(obj) - // if obj == nil || typ == Typ[Invalid] { - // return - // } - // assert(typ == obj.Type()) - // } - switch obj := obj.(type) { case *PkgName: check.errorf(e.Pos(), "use of package %s not in selector", obj.name) @@ -634,7 +623,7 @@ func (check *Checker) structType(styp *Struct, e *ast.StructType, path []*TypeNa // current field typ and tag var typ Type var tag string - add := func(field *ast.Field, ident *ast.Ident, anonymous bool, pos token.Pos) { + add := func(ident *ast.Ident, anonymous bool, pos token.Pos) { if tag != "" && tags == nil { tags = make([]string, len(fields)) } @@ -657,51 +646,45 @@ func (check *Checker) structType(styp *Struct, e *ast.StructType, path []*TypeNa if len(f.Names) > 0 { // named fields for _, name := range f.Names { - add(f, name, false, name.Pos()) + add(name, false, name.Pos()) } } else { // anonymous field - name := anonymousFieldIdent(f.Type) + // spec: "An embedded type must be specified as a type name T or as a pointer + // to a non-interface type name *T, and T itself may not be a pointer type." pos := f.Type.Pos() + name := anonymousFieldIdent(f.Type) + if name == nil { + check.invalidAST(pos, "anonymous field type %s has no name", f.Type) + continue + } t, isPtr := deref(typ) - switch t := t.(type) { + // Because we have a name, typ must be of the form T or *T, where T is the name + // of a (named or alias) type, and t (= deref(typ)) must be the type of T. + switch t := t.Underlying().(type) { case *Basic: if t == Typ[Invalid] { // error was reported before continue } + // unsafe.Pointer is treated like a regular pointer if t.kind == UnsafePointer { check.errorf(pos, "anonymous field type cannot be unsafe.Pointer") continue } - add(f, name, true, pos) - - case *Named: - // spec: "An embedded type must be specified as a type name - // T or as a pointer to a non-interface type name *T, and T - // itself may not be a pointer type." - switch u := t.underlying.(type) { - case *Basic: - // unsafe.Pointer is treated like a regular pointer - if u.kind == UnsafePointer { - check.errorf(pos, "anonymous field type cannot be unsafe.Pointer") - continue - } - case *Pointer: - check.errorf(pos, "anonymous field type cannot be a pointer") + + case *Pointer: + check.errorf(pos, "anonymous field type cannot be a pointer") + continue + + case *Interface: + if isPtr { + check.errorf(pos, "anonymous field type cannot be a pointer to an interface") continue - case *Interface: - if isPtr { - check.errorf(pos, "anonymous field type cannot be a pointer to an interface") - continue - } } - add(f, name, true, pos) - - default: - check.invalidAST(pos, "anonymous field type %s must be named", typ) } + add(name, true, pos) } } @@ -714,7 +697,10 @@ func anonymousFieldIdent(e ast.Expr) *ast.Ident { case *ast.Ident: return e case *ast.StarExpr: - return anonymousFieldIdent(e.X) + // *T is valid, but **T is not + if _, ok := e.X.(*ast.StarExpr); !ok { + return anonymousFieldIdent(e.X) + } case *ast.SelectorExpr: return e.Sel } diff --git a/libgo/go/golang_org/x/crypto/chacha20poly1305/chacha20poly1305.go b/libgo/go/golang_org/x/crypto/chacha20poly1305/chacha20poly1305.go index eb6739a109f..3f0dcb9d8cf 100644 --- a/libgo/go/golang_org/x/crypto/chacha20poly1305/chacha20poly1305.go +++ b/libgo/go/golang_org/x/crypto/chacha20poly1305/chacha20poly1305.go @@ -3,7 +3,7 @@ // license that can be found in the LICENSE file. // Package chacha20poly1305 implements the ChaCha20-Poly1305 AEAD as specified in RFC 7539. -package chacha20poly1305 +package chacha20poly1305 // import "golang.org/x/crypto/chacha20poly1305" import ( "crypto/cipher" diff --git a/libgo/go/golang_org/x/crypto/chacha20poly1305/chacha20poly1305_amd64.go b/libgo/go/golang_org/x/crypto/chacha20poly1305/chacha20poly1305_amd64.go index 47550332126..1e523b9951b 100644 --- a/libgo/go/golang_org/x/crypto/chacha20poly1305/chacha20poly1305_amd64.go +++ b/libgo/go/golang_org/x/crypto/chacha20poly1305/chacha20poly1305_amd64.go @@ -14,13 +14,60 @@ func chacha20Poly1305Open(dst []byte, key []uint32, src, ad []byte) bool //go:noescape func chacha20Poly1305Seal(dst []byte, key []uint32, src, ad []byte) -//go:noescape -func haveSSSE3() bool +// cpuid is implemented in chacha20poly1305_amd64.s. +func cpuid(eaxArg, ecxArg uint32) (eax, ebx, ecx, edx uint32) + +// xgetbv with ecx = 0 is implemented in chacha20poly1305_amd64.s. +func xgetbv() (eax, edx uint32) -var canUseASM bool +var ( + useASM bool + useAVX2 bool +) func init() { - canUseASM = haveSSSE3() + detectCpuFeatures() +} + +// detectCpuFeatures is used to detect if cpu instructions +// used by the functions implemented in assembler in +// chacha20poly1305_amd64.s are supported. +func detectCpuFeatures() { + maxId, _, _, _ := cpuid(0, 0) + if maxId < 1 { + return + } + + _, _, ecx1, _ := cpuid(1, 0) + + haveSSSE3 := isSet(9, ecx1) + useASM = haveSSSE3 + + haveOSXSAVE := isSet(27, ecx1) + + osSupportsAVX := false + // For XGETBV, OSXSAVE bit is required and sufficient. + if haveOSXSAVE { + eax, _ := xgetbv() + // Check if XMM and YMM registers have OS support. + osSupportsAVX = isSet(1, eax) && isSet(2, eax) + } + haveAVX := isSet(28, ecx1) && osSupportsAVX + + if maxId < 7 { + return + } + + _, ebx7, _, _ := cpuid(7, 0) + haveAVX2 := isSet(5, ebx7) && haveAVX + haveBMI2 := isSet(8, ebx7) + + useAVX2 = haveAVX2 && haveBMI2 +} + +// isSet checks if bit at bitpos is set in value. +func isSet(bitpos uint, value uint32) bool { + return value&(1<> 2) & 0x3ffff03) + r2 = uint64((binary.LittleEndian.Uint32(key[6:]) >> 4) & 0x3ffc0ff) + r3 = uint64((binary.LittleEndian.Uint32(key[9:]) >> 6) & 0x3f03fff) + r4 = uint64((binary.LittleEndian.Uint32(key[12:]) >> 8) & 0x00fffff) + + R1, R2, R3, R4 := r1*5, r2*5, r3*5, r4*5 + + for len(msg) >= TagSize { + // h += msg + h0 += binary.LittleEndian.Uint32(msg[0:]) & 0x3ffffff + h1 += (binary.LittleEndian.Uint32(msg[3:]) >> 2) & 0x3ffffff + h2 += (binary.LittleEndian.Uint32(msg[6:]) >> 4) & 0x3ffffff + h3 += (binary.LittleEndian.Uint32(msg[9:]) >> 6) & 0x3ffffff + h4 += (binary.LittleEndian.Uint32(msg[12:]) >> 8) | (1 << 24) + + // h *= r + d0 := (uint64(h0) * r0) + (uint64(h1) * R4) + (uint64(h2) * R3) + (uint64(h3) * R2) + (uint64(h4) * R1) + d1 := (d0 >> 26) + (uint64(h0) * r1) + (uint64(h1) * r0) + (uint64(h2) * R4) + (uint64(h3) * R3) + (uint64(h4) * R2) + d2 := (d1 >> 26) + (uint64(h0) * r2) + (uint64(h1) * r1) + (uint64(h2) * r0) + (uint64(h3) * R4) + (uint64(h4) * R3) + d3 := (d2 >> 26) + (uint64(h0) * r3) + (uint64(h1) * r2) + (uint64(h2) * r1) + (uint64(h3) * r0) + (uint64(h4) * R4) + d4 := (d3 >> 26) + (uint64(h0) * r4) + (uint64(h1) * r3) + (uint64(h2) * r2) + (uint64(h3) * r1) + (uint64(h4) * r0) + + // h %= p + h0 = uint32(d0) & 0x3ffffff + h1 = uint32(d1) & 0x3ffffff + h2 = uint32(d2) & 0x3ffffff + h3 = uint32(d3) & 0x3ffffff + h4 = uint32(d4) & 0x3ffffff + + h0 += uint32(d4>>26) * 5 + h1 += h0 >> 26 + h0 = h0 & 0x3ffffff + + msg = msg[TagSize:] } -multiplyaddatleast16bytes: - - m2 = 2279 - m20 = uint32(m[p+8]) - y7 = h7 + alpha130 - - m2 <<= 51 - m3 = 2343 - m21 = uint32(m[p+9]) - y6 = h6 + alpha130 - - m3 <<= 51 - m0 = 2151 - m22 = uint32(m[p+10]) - y1 = h1 + alpha32 - - m0 <<= 51 - m1 = 2215 - m23 = uint32(m[p+11]) - y0 = h0 + alpha32 - - m1 <<= 51 - m30 = uint32(m[p+12]) - y7 -= alpha130 - - m21 <<= 8 - m2 += int64(m20) - m31 = uint32(m[p+13]) - y6 -= alpha130 - - m22 <<= 16 - m2 += int64(m21) - m32 = uint32(m[p+14]) - y1 -= alpha32 - - m23 <<= 24 - m2 += int64(m22) - m33 = uint64(m[p+15]) - y0 -= alpha32 - - m2 += int64(m23) - m00 = uint32(m[p+0]) - y5 = h5 + alpha96 - - m31 <<= 8 - m3 += int64(m30) - m01 = uint32(m[p+1]) - y4 = h4 + alpha96 - - m32 <<= 16 - m02 = uint32(m[p+2]) - x7 = h7 - y7 - y7 *= scale - - m33 += 256 - m03 = uint32(m[p+3]) - x6 = h6 - y6 - y6 *= scale - - m33 <<= 24 - m3 += int64(m31) - m10 = uint32(m[p+4]) - x1 = h1 - y1 - - m01 <<= 8 - m3 += int64(m32) - m11 = uint32(m[p+5]) - x0 = h0 - y0 - - m3 += int64(m33) - m0 += int64(m00) - m12 = uint32(m[p+6]) - y5 -= alpha96 - - m02 <<= 16 - m0 += int64(m01) - m13 = uint32(m[p+7]) - y4 -= alpha96 - - m03 <<= 24 - m0 += int64(m02) - d2 = m2 - x1 += y7 - - m0 += int64(m03) - d3 = m3 - x0 += y6 - - m11 <<= 8 - m1 += int64(m10) - d0 = m0 - x7 += y5 - - m12 <<= 16 - m1 += int64(m11) - x6 += y4 - - m13 <<= 24 - m1 += int64(m12) - y3 = h3 + alpha64 - - m1 += int64(m13) - d1 = m1 - y2 = h2 + alpha64 - - x0 += x1 - - x6 += x7 - - y3 -= alpha64 - r3low = r3low_stack - - y2 -= alpha64 - r0low = r0low_stack - - x5 = h5 - y5 - r3lowx0 = r3low * x0 - r3high = r3high_stack - - x4 = h4 - y4 - r0lowx6 = r0low * x6 - r0high = r0high_stack - - x3 = h3 - y3 - r3highx0 = r3high * x0 - sr1low = sr1low_stack - - x2 = h2 - y2 - r0highx6 = r0high * x6 - sr1high = sr1high_stack - - x5 += y3 - r0lowx0 = r0low * x0 - r1low = r1low_stack - - h6 = r3lowx0 + r0lowx6 - sr1lowx6 = sr1low * x6 - r1high = r1high_stack - - x4 += y2 - r0highx0 = r0high * x0 - sr2low = sr2low_stack - - h7 = r3highx0 + r0highx6 - sr1highx6 = sr1high * x6 - sr2high = sr2high_stack - - x3 += y1 - r1lowx0 = r1low * x0 - r2low = r2low_stack - - h0 = r0lowx0 + sr1lowx6 - sr2lowx6 = sr2low * x6 - r2high = r2high_stack - - x2 += y0 - r1highx0 = r1high * x0 - sr3low = sr3low_stack - - h1 = r0highx0 + sr1highx6 - sr2highx6 = sr2high * x6 - sr3high = sr3high_stack - - x4 += x5 - r2lowx0 = r2low * x0 - z2 = math.Float64frombits(uint64(d2)) - - h2 = r1lowx0 + sr2lowx6 - sr3lowx6 = sr3low * x6 - - x2 += x3 - r2highx0 = r2high * x0 - z3 = math.Float64frombits(uint64(d3)) - - h3 = r1highx0 + sr2highx6 - sr3highx6 = sr3high * x6 - - r1highx4 = r1high * x4 - z2 -= alpha64 - - h4 = r2lowx0 + sr3lowx6 - r1lowx4 = r1low * x4 - - r0highx4 = r0high * x4 - z3 -= alpha96 - - h5 = r2highx0 + sr3highx6 - r0lowx4 = r0low * x4 - - h7 += r1highx4 - sr3highx4 = sr3high * x4 - - h6 += r1lowx4 - sr3lowx4 = sr3low * x4 - - h5 += r0highx4 - sr2highx4 = sr2high * x4 - - h4 += r0lowx4 - sr2lowx4 = sr2low * x4 - - h3 += sr3highx4 - r0lowx2 = r0low * x2 - - h2 += sr3lowx4 - r0highx2 = r0high * x2 - - h1 += sr2highx4 - r1lowx2 = r1low * x2 - - h0 += sr2lowx4 - r1highx2 = r1high * x2 - - h2 += r0lowx2 - r2lowx2 = r2low * x2 - - h3 += r0highx2 - r2highx2 = r2high * x2 - - h4 += r1lowx2 - sr3lowx2 = sr3low * x2 - - h5 += r1highx2 - sr3highx2 = sr3high * x2 - - p += 16 - l -= 16 - h6 += r2lowx2 - - h7 += r2highx2 - - z1 = math.Float64frombits(uint64(d1)) - h0 += sr3lowx2 - - z0 = math.Float64frombits(uint64(d0)) - h1 += sr3highx2 - - z1 -= alpha32 - - z0 -= alpha0 - - h5 += z3 - - h3 += z2 - - h1 += z1 - - h0 += z0 - - if l >= 16 { - goto multiplyaddatleast16bytes - } - -multiplyaddatmost15bytes: - - y7 = h7 + alpha130 - - y6 = h6 + alpha130 - - y1 = h1 + alpha32 - - y0 = h0 + alpha32 - - y7 -= alpha130 - - y6 -= alpha130 - - y1 -= alpha32 - - y0 -= alpha32 - - y5 = h5 + alpha96 - - y4 = h4 + alpha96 - - x7 = h7 - y7 - y7 *= scale - - x6 = h6 - y6 - y6 *= scale - - x1 = h1 - y1 - - x0 = h0 - y0 - - y5 -= alpha96 - - y4 -= alpha96 - - x1 += y7 - - x0 += y6 - - x7 += y5 - - x6 += y4 - - y3 = h3 + alpha64 - - y2 = h2 + alpha64 - - x0 += x1 - - x6 += x7 - - y3 -= alpha64 - r3low = r3low_stack - - y2 -= alpha64 - r0low = r0low_stack - - x5 = h5 - y5 - r3lowx0 = r3low * x0 - r3high = r3high_stack - - x4 = h4 - y4 - r0lowx6 = r0low * x6 - r0high = r0high_stack - - x3 = h3 - y3 - r3highx0 = r3high * x0 - sr1low = sr1low_stack - - x2 = h2 - y2 - r0highx6 = r0high * x6 - sr1high = sr1high_stack - - x5 += y3 - r0lowx0 = r0low * x0 - r1low = r1low_stack - - h6 = r3lowx0 + r0lowx6 - sr1lowx6 = sr1low * x6 - r1high = r1high_stack - - x4 += y2 - r0highx0 = r0high * x0 - sr2low = sr2low_stack - - h7 = r3highx0 + r0highx6 - sr1highx6 = sr1high * x6 - sr2high = sr2high_stack - - x3 += y1 - r1lowx0 = r1low * x0 - r2low = r2low_stack - - h0 = r0lowx0 + sr1lowx6 - sr2lowx6 = sr2low * x6 - r2high = r2high_stack - - x2 += y0 - r1highx0 = r1high * x0 - sr3low = sr3low_stack - - h1 = r0highx0 + sr1highx6 - sr2highx6 = sr2high * x6 - sr3high = sr3high_stack - - x4 += x5 - r2lowx0 = r2low * x0 - - h2 = r1lowx0 + sr2lowx6 - sr3lowx6 = sr3low * x6 - - x2 += x3 - r2highx0 = r2high * x0 - - h3 = r1highx0 + sr2highx6 - sr3highx6 = sr3high * x6 - - r1highx4 = r1high * x4 - - h4 = r2lowx0 + sr3lowx6 - r1lowx4 = r1low * x4 - - r0highx4 = r0high * x4 - - h5 = r2highx0 + sr3highx6 - r0lowx4 = r0low * x4 - - h7 += r1highx4 - sr3highx4 = sr3high * x4 - - h6 += r1lowx4 - sr3lowx4 = sr3low * x4 - - h5 += r0highx4 - sr2highx4 = sr2high * x4 - - h4 += r0lowx4 - sr2lowx4 = sr2low * x4 - - h3 += sr3highx4 - r0lowx2 = r0low * x2 - - h2 += sr3lowx4 - r0highx2 = r0high * x2 - - h1 += sr2highx4 - r1lowx2 = r1low * x2 - - h0 += sr2lowx4 - r1highx2 = r1high * x2 - - h2 += r0lowx2 - r2lowx2 = r2low * x2 - - h3 += r0highx2 - r2highx2 = r2high * x2 - - h4 += r1lowx2 - sr3lowx2 = sr3low * x2 - - h5 += r1highx2 - sr3highx2 = sr3high * x2 - - h6 += r2lowx2 - - h7 += r2highx2 - - h0 += sr3lowx2 - - h1 += sr3highx2 - -addatmost15bytes: - - if l == 0 { - goto nomorebytes + if len(msg) > 0 { + var block [TagSize]byte + off := copy(block[:], msg) + block[off] = 0x01 + + // h += msg + h0 += binary.LittleEndian.Uint32(block[0:]) & 0x3ffffff + h1 += (binary.LittleEndian.Uint32(block[3:]) >> 2) & 0x3ffffff + h2 += (binary.LittleEndian.Uint32(block[6:]) >> 4) & 0x3ffffff + h3 += (binary.LittleEndian.Uint32(block[9:]) >> 6) & 0x3ffffff + h4 += (binary.LittleEndian.Uint32(block[12:]) >> 8) + + // h *= r + d0 := (uint64(h0) * r0) + (uint64(h1) * R4) + (uint64(h2) * R3) + (uint64(h3) * R2) + (uint64(h4) * R1) + d1 := (d0 >> 26) + (uint64(h0) * r1) + (uint64(h1) * r0) + (uint64(h2) * R4) + (uint64(h3) * R3) + (uint64(h4) * R2) + d2 := (d1 >> 26) + (uint64(h0) * r2) + (uint64(h1) * r1) + (uint64(h2) * r0) + (uint64(h3) * R4) + (uint64(h4) * R3) + d3 := (d2 >> 26) + (uint64(h0) * r3) + (uint64(h1) * r2) + (uint64(h2) * r1) + (uint64(h3) * r0) + (uint64(h4) * R4) + d4 := (d3 >> 26) + (uint64(h0) * r4) + (uint64(h1) * r3) + (uint64(h2) * r2) + (uint64(h3) * r1) + (uint64(h4) * r0) + + // h %= p + h0 = uint32(d0) & 0x3ffffff + h1 = uint32(d1) & 0x3ffffff + h2 = uint32(d2) & 0x3ffffff + h3 = uint32(d3) & 0x3ffffff + h4 = uint32(d4) & 0x3ffffff + + h0 += uint32(d4>>26) * 5 + h1 += h0 >> 26 + h0 = h0 & 0x3ffffff } - lbelow2 = l - 2 - - lbelow3 = l - 3 - - lbelow2 >>= 31 - lbelow4 = l - 4 - - m00 = uint32(m[p+0]) - lbelow3 >>= 31 - p += lbelow2 - - m01 = uint32(m[p+1]) - lbelow4 >>= 31 - p += lbelow3 - - m02 = uint32(m[p+2]) - p += lbelow4 - m0 = 2151 - - m03 = uint32(m[p+3]) - m0 <<= 51 - m1 = 2215 - - m0 += int64(m00) - m01 &^= uint32(lbelow2) - - m02 &^= uint32(lbelow3) - m01 -= uint32(lbelow2) - - m01 <<= 8 - m03 &^= uint32(lbelow4) - - m0 += int64(m01) - lbelow2 -= lbelow3 - - m02 += uint32(lbelow2) - lbelow3 -= lbelow4 - - m02 <<= 16 - m03 += uint32(lbelow3) - - m03 <<= 24 - m0 += int64(m02) - - m0 += int64(m03) - lbelow5 = l - 5 - - lbelow6 = l - 6 - lbelow7 = l - 7 - - lbelow5 >>= 31 - lbelow8 = l - 8 - - lbelow6 >>= 31 - p += lbelow5 - - m10 = uint32(m[p+4]) - lbelow7 >>= 31 - p += lbelow6 - - m11 = uint32(m[p+5]) - lbelow8 >>= 31 - p += lbelow7 - - m12 = uint32(m[p+6]) - m1 <<= 51 - p += lbelow8 - - m13 = uint32(m[p+7]) - m10 &^= uint32(lbelow5) - lbelow4 -= lbelow5 - - m10 += uint32(lbelow4) - lbelow5 -= lbelow6 - - m11 &^= uint32(lbelow6) - m11 += uint32(lbelow5) - - m11 <<= 8 - m1 += int64(m10) - - m1 += int64(m11) - m12 &^= uint32(lbelow7) - - lbelow6 -= lbelow7 - m13 &^= uint32(lbelow8) - - m12 += uint32(lbelow6) - lbelow7 -= lbelow8 - - m12 <<= 16 - m13 += uint32(lbelow7) - - m13 <<= 24 - m1 += int64(m12) - - m1 += int64(m13) - m2 = 2279 - - lbelow9 = l - 9 - m3 = 2343 - - lbelow10 = l - 10 - lbelow11 = l - 11 - - lbelow9 >>= 31 - lbelow12 = l - 12 - - lbelow10 >>= 31 - p += lbelow9 - - m20 = uint32(m[p+8]) - lbelow11 >>= 31 - p += lbelow10 - - m21 = uint32(m[p+9]) - lbelow12 >>= 31 - p += lbelow11 - - m22 = uint32(m[p+10]) - m2 <<= 51 - p += lbelow12 - - m23 = uint32(m[p+11]) - m20 &^= uint32(lbelow9) - lbelow8 -= lbelow9 - - m20 += uint32(lbelow8) - lbelow9 -= lbelow10 - - m21 &^= uint32(lbelow10) - m21 += uint32(lbelow9) - - m21 <<= 8 - m2 += int64(m20) - - m2 += int64(m21) - m22 &^= uint32(lbelow11) - - lbelow10 -= lbelow11 - m23 &^= uint32(lbelow12) - - m22 += uint32(lbelow10) - lbelow11 -= lbelow12 - - m22 <<= 16 - m23 += uint32(lbelow11) - - m23 <<= 24 - m2 += int64(m22) - - m3 <<= 51 - lbelow13 = l - 13 - - lbelow13 >>= 31 - lbelow14 = l - 14 - - lbelow14 >>= 31 - p += lbelow13 - lbelow15 = l - 15 - - m30 = uint32(m[p+12]) - lbelow15 >>= 31 - p += lbelow14 - - m31 = uint32(m[p+13]) - p += lbelow15 - m2 += int64(m23) - - m32 = uint32(m[p+14]) - m30 &^= uint32(lbelow13) - lbelow12 -= lbelow13 - - m30 += uint32(lbelow12) - lbelow13 -= lbelow14 - - m3 += int64(m30) - m31 &^= uint32(lbelow14) - - m31 += uint32(lbelow13) - m32 &^= uint32(lbelow15) - - m31 <<= 8 - lbelow14 -= lbelow15 - - m3 += int64(m31) - m32 += uint32(lbelow14) - d0 = m0 - - m32 <<= 16 - m33 = uint64(lbelow15 + 1) - d1 = m1 - - m33 <<= 24 - m3 += int64(m32) - d2 = m2 - - m3 += int64(m33) - d3 = m3 - - z3 = math.Float64frombits(uint64(d3)) - - z2 = math.Float64frombits(uint64(d2)) - - z1 = math.Float64frombits(uint64(d1)) - - z0 = math.Float64frombits(uint64(d0)) - - z3 -= alpha96 - - z2 -= alpha64 - - z1 -= alpha32 - - z0 -= alpha0 - - h5 += z3 - - h3 += z2 - - h1 += z1 - - h0 += z0 - - y7 = h7 + alpha130 - - y6 = h6 + alpha130 - - y1 = h1 + alpha32 - - y0 = h0 + alpha32 - - y7 -= alpha130 - - y6 -= alpha130 - - y1 -= alpha32 - - y0 -= alpha32 - - y5 = h5 + alpha96 - - y4 = h4 + alpha96 - - x7 = h7 - y7 - y7 *= scale - - x6 = h6 - y6 - y6 *= scale - - x1 = h1 - y1 - - x0 = h0 - y0 - - y5 -= alpha96 - - y4 -= alpha96 - - x1 += y7 - - x0 += y6 - - x7 += y5 - - x6 += y4 - - y3 = h3 + alpha64 - - y2 = h2 + alpha64 - - x0 += x1 - - x6 += x7 - - y3 -= alpha64 - r3low = r3low_stack - - y2 -= alpha64 - r0low = r0low_stack - - x5 = h5 - y5 - r3lowx0 = r3low * x0 - r3high = r3high_stack - - x4 = h4 - y4 - r0lowx6 = r0low * x6 - r0high = r0high_stack - - x3 = h3 - y3 - r3highx0 = r3high * x0 - sr1low = sr1low_stack - - x2 = h2 - y2 - r0highx6 = r0high * x6 - sr1high = sr1high_stack - - x5 += y3 - r0lowx0 = r0low * x0 - r1low = r1low_stack - - h6 = r3lowx0 + r0lowx6 - sr1lowx6 = sr1low * x6 - r1high = r1high_stack - - x4 += y2 - r0highx0 = r0high * x0 - sr2low = sr2low_stack - - h7 = r3highx0 + r0highx6 - sr1highx6 = sr1high * x6 - sr2high = sr2high_stack - - x3 += y1 - r1lowx0 = r1low * x0 - r2low = r2low_stack - - h0 = r0lowx0 + sr1lowx6 - sr2lowx6 = sr2low * x6 - r2high = r2high_stack - - x2 += y0 - r1highx0 = r1high * x0 - sr3low = sr3low_stack - - h1 = r0highx0 + sr1highx6 - sr2highx6 = sr2high * x6 - sr3high = sr3high_stack - - x4 += x5 - r2lowx0 = r2low * x0 - - h2 = r1lowx0 + sr2lowx6 - sr3lowx6 = sr3low * x6 - - x2 += x3 - r2highx0 = r2high * x0 - - h3 = r1highx0 + sr2highx6 - sr3highx6 = sr3high * x6 - - r1highx4 = r1high * x4 - - h4 = r2lowx0 + sr3lowx6 - r1lowx4 = r1low * x4 - - r0highx4 = r0high * x4 - - h5 = r2highx0 + sr3highx6 - r0lowx4 = r0low * x4 - - h7 += r1highx4 - sr3highx4 = sr3high * x4 - - h6 += r1lowx4 - sr3lowx4 = sr3low * x4 - - h5 += r0highx4 - sr2highx4 = sr2high * x4 - - h4 += r0lowx4 - sr2lowx4 = sr2low * x4 - - h3 += sr3highx4 - r0lowx2 = r0low * x2 - - h2 += sr3lowx4 - r0highx2 = r0high * x2 - - h1 += sr2highx4 - r1lowx2 = r1low * x2 - - h0 += sr2lowx4 - r1highx2 = r1high * x2 - - h2 += r0lowx2 - r2lowx2 = r2low * x2 - - h3 += r0highx2 - r2highx2 = r2high * x2 - - h4 += r1lowx2 - sr3lowx2 = sr3low * x2 - - h5 += r1highx2 - sr3highx2 = sr3high * x2 - - h6 += r2lowx2 - - h7 += r2highx2 - - h0 += sr3lowx2 - - h1 += sr3highx2 - -nomorebytes: - - y7 = h7 + alpha130 - - y0 = h0 + alpha32 - - y1 = h1 + alpha32 - - y2 = h2 + alpha64 - - y7 -= alpha130 - - y3 = h3 + alpha64 - - y4 = h4 + alpha96 - - y5 = h5 + alpha96 - - x7 = h7 - y7 - y7 *= scale - - y0 -= alpha32 - - y1 -= alpha32 - - y2 -= alpha64 - - h6 += x7 - - y3 -= alpha64 - - y4 -= alpha96 - - y5 -= alpha96 - - y6 = h6 + alpha130 - - x0 = h0 - y0 - - x1 = h1 - y1 - - x2 = h2 - y2 - - y6 -= alpha130 - - x0 += y7 - - x3 = h3 - y3 - - x4 = h4 - y4 - - x5 = h5 - y5 - - x6 = h6 - y6 - - y6 *= scale - - x2 += y0 - - x3 += y1 - - x4 += y2 - - x0 += y6 - - x5 += y3 - - x6 += y4 - - x2 += x3 - - x0 += x1 - - x4 += x5 - - x6 += y5 - - x2 += offset1 - d1 = int64(math.Float64bits(x2)) - - x0 += offset0 - d0 = int64(math.Float64bits(x0)) - - x4 += offset2 - d2 = int64(math.Float64bits(x4)) - - x6 += offset3 - d3 = int64(math.Float64bits(x6)) - - f0 = uint64(d0) - - f1 = uint64(d1) - bits32 = math.MaxUint64 - - f2 = uint64(d2) - bits32 >>= 32 - - f3 = uint64(d3) - f = f0 >> 32 - - f0 &= bits32 - f &= 255 - - f1 += f - g0 = f0 + 5 - - g = g0 >> 32 - g0 &= bits32 - - f = f1 >> 32 - f1 &= bits32 - - f &= 255 - g1 = f1 + g - - g = g1 >> 32 - f2 += f - - f = f2 >> 32 - g1 &= bits32 - - f2 &= bits32 - f &= 255 - - f3 += f - g2 = f2 + g - - g = g2 >> 32 - g2 &= bits32 - - f4 = f3 >> 32 - f3 &= bits32 - - f4 &= 255 - g3 = f3 + g - - g = g3 >> 32 - g3 &= bits32 - - g4 = f4 + g - - g4 = g4 - 4 - s00 = uint32(s[0]) - - f = uint64(int64(g4) >> 63) - s01 = uint32(s[1]) - - f0 &= f - g0 &^= f - s02 = uint32(s[2]) - - f1 &= f - f0 |= g0 - s03 = uint32(s[3]) - - g1 &^= f - f2 &= f - s10 = uint32(s[4]) - - f3 &= f - g2 &^= f - s11 = uint32(s[5]) - - g3 &^= f - f1 |= g1 - s12 = uint32(s[6]) - - f2 |= g2 - f3 |= g3 - s13 = uint32(s[7]) - - s01 <<= 8 - f0 += uint64(s00) - s20 = uint32(s[8]) - - s02 <<= 16 - f0 += uint64(s01) - s21 = uint32(s[9]) - - s03 <<= 24 - f0 += uint64(s02) - s22 = uint32(s[10]) - - s11 <<= 8 - f1 += uint64(s10) - s23 = uint32(s[11]) - - s12 <<= 16 - f1 += uint64(s11) - s30 = uint32(s[12]) - - s13 <<= 24 - f1 += uint64(s12) - s31 = uint32(s[13]) - - f0 += uint64(s03) - f1 += uint64(s13) - s32 = uint32(s[14]) - - s21 <<= 8 - f2 += uint64(s20) - s33 = uint32(s[15]) - - s22 <<= 16 - f2 += uint64(s21) - - s23 <<= 24 - f2 += uint64(s22) - - s31 <<= 8 - f3 += uint64(s30) - - s32 <<= 16 - f3 += uint64(s31) - - s33 <<= 24 - f3 += uint64(s32) - - f2 += uint64(s23) - f3 += uint64(s33) - - out[0] = byte(f0) - f0 >>= 8 - out[1] = byte(f0) - f0 >>= 8 - out[2] = byte(f0) - f0 >>= 8 - out[3] = byte(f0) - f0 >>= 8 - f1 += f0 - - out[4] = byte(f1) - f1 >>= 8 - out[5] = byte(f1) - f1 >>= 8 - out[6] = byte(f1) - f1 >>= 8 - out[7] = byte(f1) - f1 >>= 8 - f2 += f1 - - out[8] = byte(f2) - f2 >>= 8 - out[9] = byte(f2) - f2 >>= 8 - out[10] = byte(f2) - f2 >>= 8 - out[11] = byte(f2) - f2 >>= 8 - f3 += f2 - - out[12] = byte(f3) - f3 >>= 8 - out[13] = byte(f3) - f3 >>= 8 - out[14] = byte(f3) - f3 >>= 8 - out[15] = byte(f3) + // h %= p reduction + h2 += h1 >> 26 + h1 &= 0x3ffffff + h3 += h2 >> 26 + h2 &= 0x3ffffff + h4 += h3 >> 26 + h3 &= 0x3ffffff + h0 += 5 * (h4 >> 26) + h4 &= 0x3ffffff + h1 += h0 >> 26 + h0 &= 0x3ffffff + + // h - p + t0 := h0 + 5 + t1 := h1 + (t0 >> 26) + t2 := h2 + (t1 >> 26) + t3 := h3 + (t2 >> 26) + t4 := h4 + (t3 >> 26) - (1 << 26) + t0 &= 0x3ffffff + t1 &= 0x3ffffff + t2 &= 0x3ffffff + t3 &= 0x3ffffff + + // select h if h < p else h - p + t_mask := (t4 >> 31) - 1 + h_mask := ^t_mask + h0 = (h0 & h_mask) | (t0 & t_mask) + h1 = (h1 & h_mask) | (t1 & t_mask) + h2 = (h2 & h_mask) | (t2 & t_mask) + h3 = (h3 & h_mask) | (t3 & t_mask) + h4 = (h4 & h_mask) | (t4 & t_mask) + + // h %= 2^128 + h0 |= h1 << 26 + h1 = ((h1 >> 6) | (h2 << 20)) + h2 = ((h2 >> 12) | (h3 << 14)) + h3 = ((h3 >> 18) | (h4 << 8)) + + // s: the s part of the key + // tag = (h + s) % (2^128) + t := uint64(h0) + uint64(binary.LittleEndian.Uint32(key[16:])) + h0 = uint32(t) + t = uint64(h1) + uint64(binary.LittleEndian.Uint32(key[20:])) + (t >> 32) + h1 = uint32(t) + t = uint64(h2) + uint64(binary.LittleEndian.Uint32(key[24:])) + (t >> 32) + h2 = uint32(t) + t = uint64(h3) + uint64(binary.LittleEndian.Uint32(key[28:])) + (t >> 32) + h3 = uint32(t) + + binary.LittleEndian.PutUint32(out[0:], h0) + binary.LittleEndian.PutUint32(out[4:], h1) + binary.LittleEndian.PutUint32(out[8:], h2) + binary.LittleEndian.PutUint32(out[12:], h3) } diff --git a/libgo/go/golang_org/x/net/http2/hpack/encode.go b/libgo/go/golang_org/x/net/http2/hpack/encode.go index f9bb0339848..54726c2a3c5 100644 --- a/libgo/go/golang_org/x/net/http2/hpack/encode.go +++ b/libgo/go/golang_org/x/net/http2/hpack/encode.go @@ -39,13 +39,14 @@ func NewEncoder(w io.Writer) *Encoder { tableSizeUpdate: false, w: w, } + e.dynTab.table.init() e.dynTab.setMaxSize(initialHeaderTableSize) return e } // WriteField encodes f into a single Write to e's underlying Writer. // This function may also produce bytes for "Header Table Size Update" -// if necessary. If produced, it is done before encoding f. +// if necessary. If produced, it is done before encoding f. func (e *Encoder) WriteField(f HeaderField) error { e.buf = e.buf[:0] @@ -88,29 +89,17 @@ func (e *Encoder) WriteField(f HeaderField) error { // only name matches, i points to that index and nameValueMatch // becomes false. func (e *Encoder) searchTable(f HeaderField) (i uint64, nameValueMatch bool) { - for idx, hf := range staticTable { - if !constantTimeStringCompare(hf.Name, f.Name) { - continue - } - if i == 0 { - i = uint64(idx + 1) - } - if f.Sensitive { - continue - } - if !constantTimeStringCompare(hf.Value, f.Value) { - continue - } - i = uint64(idx + 1) - nameValueMatch = true - return + i, nameValueMatch = staticTable.search(f) + if nameValueMatch { + return i, true } - j, nameValueMatch := e.dynTab.search(f) + j, nameValueMatch := e.dynTab.table.search(f) if nameValueMatch || (i == 0 && j != 0) { - i = j + uint64(len(staticTable)) + return j + uint64(staticTable.len()), nameValueMatch } - return + + return i, false } // SetMaxDynamicTableSize changes the dynamic header table size to v. diff --git a/libgo/go/golang_org/x/net/http2/hpack/encode_test.go b/libgo/go/golang_org/x/net/http2/hpack/encode_test.go index 92286f3bad1..05f12db9cdc 100644 --- a/libgo/go/golang_org/x/net/http2/hpack/encode_test.go +++ b/libgo/go/golang_org/x/net/http2/hpack/encode_test.go @@ -7,6 +7,8 @@ package hpack import ( "bytes" "encoding/hex" + "fmt" + "math/rand" "reflect" "strings" "testing" @@ -101,17 +103,20 @@ func TestEncoderSearchTable(t *testing.T) { wantMatch bool }{ // Name and Value match - {pair("foo", "bar"), uint64(len(staticTable) + 3), true}, - {pair("blake", "miz"), uint64(len(staticTable) + 2), true}, + {pair("foo", "bar"), uint64(staticTable.len()) + 3, true}, + {pair("blake", "miz"), uint64(staticTable.len()) + 2, true}, {pair(":method", "GET"), 2, true}, - // Only name match because Sensitive == true - {HeaderField{":method", "GET", true}, 2, false}, + // Only name match because Sensitive == true. This is allowed to match + // any ":method" entry. The current implementation uses the last entry + // added in newStaticTable. + {HeaderField{":method", "GET", true}, 3, false}, // Only Name matches - {pair("foo", "..."), uint64(len(staticTable) + 3), false}, - {pair("blake", "..."), uint64(len(staticTable) + 2), false}, - {pair(":method", "..."), 2, false}, + {pair("foo", "..."), uint64(staticTable.len()) + 3, false}, + {pair("blake", "..."), uint64(staticTable.len()) + 2, false}, + // As before, this is allowed to match any ":method" entry. + {pair(":method", "..."), 3, false}, // None match {pair("foo-", "bar"), 0, false}, @@ -328,3 +333,54 @@ func TestEncoderSetMaxDynamicTableSizeLimit(t *testing.T) { func removeSpace(s string) string { return strings.Replace(s, " ", "", -1) } + +func BenchmarkEncoderSearchTable(b *testing.B) { + e := NewEncoder(nil) + + // A sample of possible header fields. + // This is not based on any actual data from HTTP/2 traces. + var possible []HeaderField + for _, f := range staticTable.ents { + if f.Value == "" { + possible = append(possible, f) + continue + } + // Generate 5 random values, except for cookie and set-cookie, + // which we know can have many values in practice. + num := 5 + if f.Name == "cookie" || f.Name == "set-cookie" { + num = 25 + } + for i := 0; i < num; i++ { + f.Value = fmt.Sprintf("%s-%d", f.Name, i) + possible = append(possible, f) + } + } + for k := 0; k < 10; k++ { + f := HeaderField{ + Name: fmt.Sprintf("x-header-%d", k), + Sensitive: rand.Int()%2 == 0, + } + for i := 0; i < 5; i++ { + f.Value = fmt.Sprintf("%s-%d", f.Name, i) + possible = append(possible, f) + } + } + + // Add a random sample to the dynamic table. This very loosely simulates + // a history of 100 requests with 20 header fields per request. + for r := 0; r < 100*20; r++ { + f := possible[rand.Int31n(int32(len(possible)))] + // Skip if this is in the staticTable verbatim. + if _, has := staticTable.search(f); !has { + e.dynTab.add(f) + } + } + + b.ResetTimer() + for n := 0; n < b.N; n++ { + for _, f := range possible { + e.searchTable(f) + } + } +} diff --git a/libgo/go/golang_org/x/net/http2/hpack/hpack.go b/libgo/go/golang_org/x/net/http2/hpack/hpack.go index 8aa197ad678..176644acdac 100644 --- a/libgo/go/golang_org/x/net/http2/hpack/hpack.go +++ b/libgo/go/golang_org/x/net/http2/hpack/hpack.go @@ -57,11 +57,11 @@ func (hf HeaderField) String() string { return fmt.Sprintf("header field %q = %q%s", hf.Name, hf.Value, suffix) } -// Size returns the size of an entry per RFC 7540 section 5.2. +// Size returns the size of an entry per RFC 7541 section 4.1. func (hf HeaderField) Size() uint32 { // http://http2.github.io/http2-spec/compression.html#rfc.section.4.1 // "The size of the dynamic table is the sum of the size of - // its entries. The size of an entry is the sum of its name's + // its entries. The size of an entry is the sum of its name's // length in octets (as defined in Section 5.2), its value's // length in octets (see Section 5.2), plus 32. The size of // an entry is calculated using the length of the name and @@ -102,6 +102,7 @@ func NewDecoder(maxDynamicTableSize uint32, emitFunc func(f HeaderField)) *Decod emit: emitFunc, emitEnabled: true, } + d.dynTab.table.init() d.dynTab.allowedMaxSize = maxDynamicTableSize d.dynTab.setMaxSize(maxDynamicTableSize) return d @@ -154,12 +155,9 @@ func (d *Decoder) SetAllowedMaxDynamicTableSize(v uint32) { } type dynamicTable struct { - // ents is the FIFO described at // http://http2.github.io/http2-spec/compression.html#rfc.section.2.3.2 - // The newest (low index) is append at the end, and items are - // evicted from the front. - ents []HeaderField - size uint32 + table headerFieldTable + size uint32 // in bytes maxSize uint32 // current maxSize allowedMaxSize uint32 // maxSize may go up to this, inclusive } @@ -169,95 +167,45 @@ func (dt *dynamicTable) setMaxSize(v uint32) { dt.evict() } -// TODO: change dynamicTable to be a struct with a slice and a size int field, -// per http://http2.github.io/http2-spec/compression.html#rfc.section.4.1: -// -// -// Then make add increment the size. maybe the max size should move from Decoder to -// dynamicTable and add should return an ok bool if there was enough space. -// -// Later we'll need a remove operation on dynamicTable. - func (dt *dynamicTable) add(f HeaderField) { - dt.ents = append(dt.ents, f) + dt.table.addEntry(f) dt.size += f.Size() dt.evict() } -// If we're too big, evict old stuff (front of the slice) +// If we're too big, evict old stuff. func (dt *dynamicTable) evict() { - base := dt.ents // keep base pointer of slice - for dt.size > dt.maxSize { - dt.size -= dt.ents[0].Size() - dt.ents = dt.ents[1:] - } - - // Shift slice contents down if we evicted things. - if len(dt.ents) != len(base) { - copy(base, dt.ents) - dt.ents = base[:len(dt.ents)] + var n int + for dt.size > dt.maxSize && n < dt.table.len() { + dt.size -= dt.table.ents[n].Size() + n++ } -} - -// constantTimeStringCompare compares string a and b in a constant -// time manner. -func constantTimeStringCompare(a, b string) bool { - if len(a) != len(b) { - return false - } - - c := byte(0) - - for i := 0; i < len(a); i++ { - c |= a[i] ^ b[i] - } - - return c == 0 -} - -// Search searches f in the table. The return value i is 0 if there is -// no name match. If there is name match or name/value match, i is the -// index of that entry (1-based). If both name and value match, -// nameValueMatch becomes true. -func (dt *dynamicTable) search(f HeaderField) (i uint64, nameValueMatch bool) { - l := len(dt.ents) - for j := l - 1; j >= 0; j-- { - ent := dt.ents[j] - if !constantTimeStringCompare(ent.Name, f.Name) { - continue - } - if i == 0 { - i = uint64(l - j) - } - if f.Sensitive { - continue - } - if !constantTimeStringCompare(ent.Value, f.Value) { - continue - } - i = uint64(l - j) - nameValueMatch = true - return - } - return + dt.table.evictOldest(n) } func (d *Decoder) maxTableIndex() int { - return len(d.dynTab.ents) + len(staticTable) + // This should never overflow. RFC 7540 Section 6.5.2 limits the size of + // the dynamic table to 2^32 bytes, where each entry will occupy more than + // one byte. Further, the staticTable has a fixed, small length. + return d.dynTab.table.len() + staticTable.len() } func (d *Decoder) at(i uint64) (hf HeaderField, ok bool) { - if i < 1 { + // See Section 2.3.3. + if i == 0 { return } + if i <= uint64(staticTable.len()) { + return staticTable.ents[i-1], true + } if i > uint64(d.maxTableIndex()) { return } - if i <= uint64(len(staticTable)) { - return staticTable[i-1], true - } - dents := d.dynTab.ents - return dents[len(dents)-(int(i)-len(staticTable))], true + // In the dynamic table, newer entries have lower indices. + // However, dt.ents[0] is the oldest entry. Hence, dt.ents is + // the reversed dynamic table. + dt := d.dynTab.table + return dt.ents[dt.len()-(int(i)-staticTable.len())], true } // Decode decodes an entire block. @@ -307,7 +255,7 @@ func (d *Decoder) Write(p []byte) (n int, err error) { err = d.parseHeaderFieldRepr() if err == errNeedMore { // Extra paranoia, making sure saveBuf won't - // get too large. All the varint and string + // get too large. All the varint and string // reading code earlier should already catch // overlong things and return ErrStringLength, // but keep this as a last resort. diff --git a/libgo/go/golang_org/x/net/http2/hpack/hpack_test.go b/libgo/go/golang_org/x/net/http2/hpack/hpack_test.go index 4c7b17bfb14..bc7f4767822 100644 --- a/libgo/go/golang_org/x/net/http2/hpack/hpack_test.go +++ b/libgo/go/golang_org/x/net/http2/hpack/hpack_test.go @@ -5,117 +5,16 @@ package hpack import ( - "bufio" "bytes" "encoding/hex" "fmt" "math/rand" "reflect" - "regexp" - "strconv" "strings" "testing" "time" ) -func TestStaticTable(t *testing.T) { - fromSpec := ` - +-------+-----------------------------+---------------+ - | 1 | :authority | | - | 2 | :method | GET | - | 3 | :method | POST | - | 4 | :path | / | - | 5 | :path | /index.html | - | 6 | :scheme | http | - | 7 | :scheme | https | - | 8 | :status | 200 | - | 9 | :status | 204 | - | 10 | :status | 206 | - | 11 | :status | 304 | - | 12 | :status | 400 | - | 13 | :status | 404 | - | 14 | :status | 500 | - | 15 | accept-charset | | - | 16 | accept-encoding | gzip, deflate | - | 17 | accept-language | | - | 18 | accept-ranges | | - | 19 | accept | | - | 20 | access-control-allow-origin | | - | 21 | age | | - | 22 | allow | | - | 23 | authorization | | - | 24 | cache-control | | - | 25 | content-disposition | | - | 26 | content-encoding | | - | 27 | content-language | | - | 28 | content-length | | - | 29 | content-location | | - | 30 | content-range | | - | 31 | content-type | | - | 32 | cookie | | - | 33 | date | | - | 34 | etag | | - | 35 | expect | | - | 36 | expires | | - | 37 | from | | - | 38 | host | | - | 39 | if-match | | - | 40 | if-modified-since | | - | 41 | if-none-match | | - | 42 | if-range | | - | 43 | if-unmodified-since | | - | 44 | last-modified | | - | 45 | link | | - | 46 | location | | - | 47 | max-forwards | | - | 48 | proxy-authenticate | | - | 49 | proxy-authorization | | - | 50 | range | | - | 51 | referer | | - | 52 | refresh | | - | 53 | retry-after | | - | 54 | server | | - | 55 | set-cookie | | - | 56 | strict-transport-security | | - | 57 | transfer-encoding | | - | 58 | user-agent | | - | 59 | vary | | - | 60 | via | | - | 61 | www-authenticate | | - +-------+-----------------------------+---------------+ -` - bs := bufio.NewScanner(strings.NewReader(fromSpec)) - re := regexp.MustCompile(`\| (\d+)\s+\| (\S+)\s*\| (\S(.*\S)?)?\s+\|`) - for bs.Scan() { - l := bs.Text() - if !strings.Contains(l, "|") { - continue - } - m := re.FindStringSubmatch(l) - if m == nil { - continue - } - i, err := strconv.Atoi(m[1]) - if err != nil { - t.Errorf("Bogus integer on line %q", l) - continue - } - if i < 1 || i > len(staticTable) { - t.Errorf("Bogus index %d on line %q", i, l) - continue - } - if got, want := staticTable[i-1].Name, m[2]; got != want { - t.Errorf("header index %d name = %q; want %q", i, got, want) - } - if got, want := staticTable[i-1].Value, m[3]; got != want { - t.Errorf("header index %d value = %q; want %q", i, got, want) - } - } - if err := bs.Err(); err != nil { - t.Error(err) - } -} - func (d *Decoder) mustAt(idx int) HeaderField { if hf, ok := d.at(uint64(idx)); !ok { panic(fmt.Sprintf("bogus index %d", idx)) @@ -132,10 +31,10 @@ func TestDynamicTableAt(t *testing.T) { } d.dynTab.add(pair("foo", "bar")) d.dynTab.add(pair("blake", "miz")) - if got, want := at(len(staticTable)+1), (pair("blake", "miz")); got != want { + if got, want := at(staticTable.len()+1), (pair("blake", "miz")); got != want { t.Errorf("at(dyn 1) = %v; want %v", got, want) } - if got, want := at(len(staticTable)+2), (pair("foo", "bar")); got != want { + if got, want := at(staticTable.len()+2), (pair("foo", "bar")); got != want { t.Errorf("at(dyn 2) = %v; want %v", got, want) } if got, want := at(3), (pair(":method", "POST")); got != want { @@ -143,41 +42,6 @@ func TestDynamicTableAt(t *testing.T) { } } -func TestDynamicTableSearch(t *testing.T) { - dt := dynamicTable{} - dt.setMaxSize(4096) - - dt.add(pair("foo", "bar")) - dt.add(pair("blake", "miz")) - dt.add(pair(":method", "GET")) - - tests := []struct { - hf HeaderField - wantI uint64 - wantMatch bool - }{ - // Name and Value match - {pair("foo", "bar"), 3, true}, - {pair(":method", "GET"), 1, true}, - - // Only name match because of Sensitive == true - {HeaderField{"blake", "miz", true}, 2, false}, - - // Only Name matches - {pair("foo", "..."), 3, false}, - {pair("blake", "..."), 2, false}, - {pair(":method", "..."), 1, false}, - - // None match - {pair("foo-", "bar"), 0, false}, - } - for _, tt := range tests { - if gotI, gotMatch := dt.search(tt.hf); gotI != tt.wantI || gotMatch != tt.wantMatch { - t.Errorf("d.search(%+v) = %v, %v; want %v, %v", tt.hf, gotI, gotMatch, tt.wantI, tt.wantMatch) - } - } -} - func TestDynamicTableSizeEvict(t *testing.T) { d := NewDecoder(4096, nil) if want := uint32(0); d.dynTab.size != want { @@ -196,7 +60,7 @@ func TestDynamicTableSizeEvict(t *testing.T) { if want := uint32(6 + 32); d.dynTab.size != want { t.Fatalf("after setMaxSize, size = %d; want %d", d.dynTab.size, want) } - if got, want := d.mustAt(len(staticTable)+1), (pair("foo", "bar")); got != want { + if got, want := d.mustAt(staticTable.len()+1), (pair("foo", "bar")); got != want { t.Errorf("at(dyn 1) = %v; want %v", got, want) } add(pair("long", strings.Repeat("x", 500))) @@ -255,9 +119,9 @@ func TestDecoderDecode(t *testing.T) { } func (dt *dynamicTable) reverseCopy() (hf []HeaderField) { - hf = make([]HeaderField, len(dt.ents)) + hf = make([]HeaderField, len(dt.table.ents)) for i := range hf { - hf[i] = dt.ents[len(dt.ents)-1-i] + hf[i] = dt.table.ents[len(dt.table.ents)-1-i] } return } @@ -784,6 +648,10 @@ func TestHuffmanFuzzCrash(t *testing.T) { } } +func pair(name, value string) HeaderField { + return HeaderField{Name: name, Value: value} +} + func dehex(s string) []byte { s = strings.Replace(s, " ", "", -1) s = strings.Replace(s, "\n", "", -1) diff --git a/libgo/go/golang_org/x/net/http2/hpack/tables.go b/libgo/go/golang_org/x/net/http2/hpack/tables.go index b9283a02330..8bd975d388d 100644 --- a/libgo/go/golang_org/x/net/http2/hpack/tables.go +++ b/libgo/go/golang_org/x/net/http2/hpack/tables.go @@ -4,73 +4,200 @@ package hpack -func pair(name, value string) HeaderField { - return HeaderField{Name: name, Value: value} +import ( + "fmt" +) + +// headerFieldTable implements a list of HeaderFields. +// This is used to implement the static and dynamic tables. +type headerFieldTable struct { + // For static tables, entries are never evicted. + // + // For dynamic tables, entries are evicted from ents[0] and added to the end. + // Each entry has a unique id that starts at one and increments for each + // entry that is added. This unique id is stable across evictions, meaning + // it can be used as a pointer to a specific entry. As in hpack, unique ids + // are 1-based. The unique id for ents[k] is k + evictCount + 1. + // + // Zero is not a valid unique id. + // + // evictCount should not overflow in any remotely practical situation. In + // practice, we will have one dynamic table per HTTP/2 connection. If we + // assume a very powerful server that handles 1M QPS per connection and each + // request adds (then evicts) 100 entries from the table, it would still take + // 2M years for evictCount to overflow. + ents []HeaderField + evictCount uint64 + + // byName maps a HeaderField name to the unique id of the newest entry with + // the same name. See above for a definition of "unique id". + byName map[string]uint64 + + // byNameValue maps a HeaderField name/value pair to the unique id of the newest + // entry with the same name and value. See above for a definition of "unique id". + byNameValue map[pairNameValue]uint64 +} + +type pairNameValue struct { + name, value string +} + +func (t *headerFieldTable) init() { + t.byName = make(map[string]uint64) + t.byNameValue = make(map[pairNameValue]uint64) +} + +// len reports the number of entries in the table. +func (t *headerFieldTable) len() int { + return len(t.ents) +} + +// addEntry adds a new entry. +func (t *headerFieldTable) addEntry(f HeaderField) { + id := uint64(t.len()) + t.evictCount + 1 + t.byName[f.Name] = id + t.byNameValue[pairNameValue{f.Name, f.Value}] = id + t.ents = append(t.ents, f) +} + +// evictOldest evicts the n oldest entries in the table. +func (t *headerFieldTable) evictOldest(n int) { + if n > t.len() { + panic(fmt.Sprintf("evictOldest(%v) on table with %v entries", n, t.len())) + } + for k := 0; k < n; k++ { + f := t.ents[k] + id := t.evictCount + uint64(k) + 1 + if t.byName[f.Name] == id { + delete(t.byName, f.Name) + } + if p := (pairNameValue{f.Name, f.Value}); t.byNameValue[p] == id { + delete(t.byNameValue, p) + } + } + copy(t.ents, t.ents[n:]) + for k := t.len() - n; k < t.len(); k++ { + t.ents[k] = HeaderField{} // so strings can be garbage collected + } + t.ents = t.ents[:t.len()-n] + if t.evictCount+uint64(n) < t.evictCount { + panic("evictCount overflow") + } + t.evictCount += uint64(n) +} + +// search finds f in the table. If there is no match, i is 0. +// If both name and value match, i is the matched index and nameValueMatch +// becomes true. If only name matches, i points to that index and +// nameValueMatch becomes false. +// +// The returned index is a 1-based HPACK index. For dynamic tables, HPACK says +// that index 1 should be the newest entry, but t.ents[0] is the oldest entry, +// meaning t.ents is reversed for dynamic tables. Hence, when t is a dynamic +// table, the return value i actually refers to the entry t.ents[t.len()-i]. +// +// All tables are assumed to be a dynamic tables except for the global +// staticTable pointer. +// +// See Section 2.3.3. +func (t *headerFieldTable) search(f HeaderField) (i uint64, nameValueMatch bool) { + if !f.Sensitive { + if id := t.byNameValue[pairNameValue{f.Name, f.Value}]; id != 0 { + return t.idToIndex(id), true + } + } + if id := t.byName[f.Name]; id != 0 { + return t.idToIndex(id), false + } + return 0, false +} + +// idToIndex converts a unique id to an HPACK index. +// See Section 2.3.3. +func (t *headerFieldTable) idToIndex(id uint64) uint64 { + if id <= t.evictCount { + panic(fmt.Sprintf("id (%v) <= evictCount (%v)", id, t.evictCount)) + } + k := id - t.evictCount - 1 // convert id to an index t.ents[k] + if t != staticTable { + return uint64(t.len()) - k // dynamic table + } + return k + 1 } // http://tools.ietf.org/html/draft-ietf-httpbis-header-compression-07#appendix-B -var staticTable = [...]HeaderField{ - pair(":authority", ""), // index 1 (1-based) - pair(":method", "GET"), - pair(":method", "POST"), - pair(":path", "/"), - pair(":path", "/index.html"), - pair(":scheme", "http"), - pair(":scheme", "https"), - pair(":status", "200"), - pair(":status", "204"), - pair(":status", "206"), - pair(":status", "304"), - pair(":status", "400"), - pair(":status", "404"), - pair(":status", "500"), - pair("accept-charset", ""), - pair("accept-encoding", "gzip, deflate"), - pair("accept-language", ""), - pair("accept-ranges", ""), - pair("accept", ""), - pair("access-control-allow-origin", ""), - pair("age", ""), - pair("allow", ""), - pair("authorization", ""), - pair("cache-control", ""), - pair("content-disposition", ""), - pair("content-encoding", ""), - pair("content-language", ""), - pair("content-length", ""), - pair("content-location", ""), - pair("content-range", ""), - pair("content-type", ""), - pair("cookie", ""), - pair("date", ""), - pair("etag", ""), - pair("expect", ""), - pair("expires", ""), - pair("from", ""), - pair("host", ""), - pair("if-match", ""), - pair("if-modified-since", ""), - pair("if-none-match", ""), - pair("if-range", ""), - pair("if-unmodified-since", ""), - pair("last-modified", ""), - pair("link", ""), - pair("location", ""), - pair("max-forwards", ""), - pair("proxy-authenticate", ""), - pair("proxy-authorization", ""), - pair("range", ""), - pair("referer", ""), - pair("refresh", ""), - pair("retry-after", ""), - pair("server", ""), - pair("set-cookie", ""), - pair("strict-transport-security", ""), - pair("transfer-encoding", ""), - pair("user-agent", ""), - pair("vary", ""), - pair("via", ""), - pair("www-authenticate", ""), +var staticTable = newStaticTable() +var staticTableEntries = [...]HeaderField{ + HeaderField{Name: ":authority"}, + HeaderField{Name: ":method", Value: "GET"}, + HeaderField{Name: ":method", Value: "POST"}, + HeaderField{Name: ":path", Value: "/"}, + HeaderField{Name: ":path", Value: "/index.html"}, + HeaderField{Name: ":scheme", Value: "http"}, + HeaderField{Name: ":scheme", Value: "https"}, + HeaderField{Name: ":status", Value: "200"}, + HeaderField{Name: ":status", Value: "204"}, + HeaderField{Name: ":status", Value: "206"}, + HeaderField{Name: ":status", Value: "304"}, + HeaderField{Name: ":status", Value: "400"}, + HeaderField{Name: ":status", Value: "404"}, + HeaderField{Name: ":status", Value: "500"}, + HeaderField{Name: "accept-charset"}, + HeaderField{Name: "accept-encoding", Value: "gzip, deflate"}, + HeaderField{Name: "accept-language"}, + HeaderField{Name: "accept-ranges"}, + HeaderField{Name: "accept"}, + HeaderField{Name: "access-control-allow-origin"}, + HeaderField{Name: "age"}, + HeaderField{Name: "allow"}, + HeaderField{Name: "authorization"}, + HeaderField{Name: "cache-control"}, + HeaderField{Name: "content-disposition"}, + HeaderField{Name: "content-encoding"}, + HeaderField{Name: "content-language"}, + HeaderField{Name: "content-length"}, + HeaderField{Name: "content-location"}, + HeaderField{Name: "content-range"}, + HeaderField{Name: "content-type"}, + HeaderField{Name: "cookie"}, + HeaderField{Name: "date"}, + HeaderField{Name: "etag"}, + HeaderField{Name: "expect"}, + HeaderField{Name: "expires"}, + HeaderField{Name: "from"}, + HeaderField{Name: "host"}, + HeaderField{Name: "if-match"}, + HeaderField{Name: "if-modified-since"}, + HeaderField{Name: "if-none-match"}, + HeaderField{Name: "if-range"}, + HeaderField{Name: "if-unmodified-since"}, + HeaderField{Name: "last-modified"}, + HeaderField{Name: "link"}, + HeaderField{Name: "location"}, + HeaderField{Name: "max-forwards"}, + HeaderField{Name: "proxy-authenticate"}, + HeaderField{Name: "proxy-authorization"}, + HeaderField{Name: "range"}, + HeaderField{Name: "referer"}, + HeaderField{Name: "refresh"}, + HeaderField{Name: "retry-after"}, + HeaderField{Name: "server"}, + HeaderField{Name: "set-cookie"}, + HeaderField{Name: "strict-transport-security"}, + HeaderField{Name: "transfer-encoding"}, + HeaderField{Name: "user-agent"}, + HeaderField{Name: "vary"}, + HeaderField{Name: "via"}, + HeaderField{Name: "www-authenticate"}, +} + +func newStaticTable() *headerFieldTable { + t := &headerFieldTable{} + t.init() + for _, e := range staticTableEntries[:] { + t.addEntry(e) + } + return t } var huffmanCodes = [256]uint32{ diff --git a/libgo/go/golang_org/x/net/http2/hpack/tables_test.go b/libgo/go/golang_org/x/net/http2/hpack/tables_test.go new file mode 100644 index 00000000000..d963f363544 --- /dev/null +++ b/libgo/go/golang_org/x/net/http2/hpack/tables_test.go @@ -0,0 +1,214 @@ +// Copyright 2017 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package hpack + +import ( + "bufio" + "regexp" + "strconv" + "strings" + "testing" +) + +func TestHeaderFieldTable(t *testing.T) { + table := &headerFieldTable{} + table.init() + table.addEntry(pair("key1", "value1-1")) + table.addEntry(pair("key2", "value2-1")) + table.addEntry(pair("key1", "value1-2")) + table.addEntry(pair("key3", "value3-1")) + table.addEntry(pair("key4", "value4-1")) + table.addEntry(pair("key2", "value2-2")) + + // Tests will be run twice: once before evicting anything, and + // again after evicting the three oldest entries. + tests := []struct { + f HeaderField + beforeWantStaticI uint64 + beforeWantMatch bool + afterWantStaticI uint64 + afterWantMatch bool + }{ + {HeaderField{"key1", "value1-1", false}, 1, true, 0, false}, + {HeaderField{"key1", "value1-2", false}, 3, true, 0, false}, + {HeaderField{"key1", "value1-3", false}, 3, false, 0, false}, + {HeaderField{"key2", "value2-1", false}, 2, true, 3, false}, + {HeaderField{"key2", "value2-2", false}, 6, true, 3, true}, + {HeaderField{"key2", "value2-3", false}, 6, false, 3, false}, + {HeaderField{"key4", "value4-1", false}, 5, true, 2, true}, + // Name match only, because sensitive. + {HeaderField{"key4", "value4-1", true}, 5, false, 2, false}, + // Key not found. + {HeaderField{"key5", "value5-x", false}, 0, false, 0, false}, + } + + staticToDynamic := func(i uint64) uint64 { + if i == 0 { + return 0 + } + return uint64(table.len()) - i + 1 // dynamic is the reversed table + } + + searchStatic := func(f HeaderField) (uint64, bool) { + old := staticTable + staticTable = table + defer func() { staticTable = old }() + return staticTable.search(f) + } + + searchDynamic := func(f HeaderField) (uint64, bool) { + return table.search(f) + } + + for _, test := range tests { + gotI, gotMatch := searchStatic(test.f) + if wantI, wantMatch := test.beforeWantStaticI, test.beforeWantMatch; gotI != wantI || gotMatch != wantMatch { + t.Errorf("before evictions: searchStatic(%+v)=%v,%v want %v,%v", test.f, gotI, gotMatch, wantI, wantMatch) + } + gotI, gotMatch = searchDynamic(test.f) + wantDynamicI := staticToDynamic(test.beforeWantStaticI) + if wantI, wantMatch := wantDynamicI, test.beforeWantMatch; gotI != wantI || gotMatch != wantMatch { + t.Errorf("before evictions: searchDynamic(%+v)=%v,%v want %v,%v", test.f, gotI, gotMatch, wantI, wantMatch) + } + } + + table.evictOldest(3) + + for _, test := range tests { + gotI, gotMatch := searchStatic(test.f) + if wantI, wantMatch := test.afterWantStaticI, test.afterWantMatch; gotI != wantI || gotMatch != wantMatch { + t.Errorf("after evictions: searchStatic(%+v)=%v,%v want %v,%v", test.f, gotI, gotMatch, wantI, wantMatch) + } + gotI, gotMatch = searchDynamic(test.f) + wantDynamicI := staticToDynamic(test.afterWantStaticI) + if wantI, wantMatch := wantDynamicI, test.afterWantMatch; gotI != wantI || gotMatch != wantMatch { + t.Errorf("after evictions: searchDynamic(%+v)=%v,%v want %v,%v", test.f, gotI, gotMatch, wantI, wantMatch) + } + } +} + +func TestHeaderFieldTable_LookupMapEviction(t *testing.T) { + table := &headerFieldTable{} + table.init() + table.addEntry(pair("key1", "value1-1")) + table.addEntry(pair("key2", "value2-1")) + table.addEntry(pair("key1", "value1-2")) + table.addEntry(pair("key3", "value3-1")) + table.addEntry(pair("key4", "value4-1")) + table.addEntry(pair("key2", "value2-2")) + + // evict all pairs + table.evictOldest(table.len()) + + if l := table.len(); l > 0 { + t.Errorf("table.len() = %d, want 0", l) + } + + if l := len(table.byName); l > 0 { + t.Errorf("len(table.byName) = %d, want 0", l) + } + + if l := len(table.byNameValue); l > 0 { + t.Errorf("len(table.byNameValue) = %d, want 0", l) + } +} + +func TestStaticTable(t *testing.T) { + fromSpec := ` + +-------+-----------------------------+---------------+ + | 1 | :authority | | + | 2 | :method | GET | + | 3 | :method | POST | + | 4 | :path | / | + | 5 | :path | /index.html | + | 6 | :scheme | http | + | 7 | :scheme | https | + | 8 | :status | 200 | + | 9 | :status | 204 | + | 10 | :status | 206 | + | 11 | :status | 304 | + | 12 | :status | 400 | + | 13 | :status | 404 | + | 14 | :status | 500 | + | 15 | accept-charset | | + | 16 | accept-encoding | gzip, deflate | + | 17 | accept-language | | + | 18 | accept-ranges | | + | 19 | accept | | + | 20 | access-control-allow-origin | | + | 21 | age | | + | 22 | allow | | + | 23 | authorization | | + | 24 | cache-control | | + | 25 | content-disposition | | + | 26 | content-encoding | | + | 27 | content-language | | + | 28 | content-length | | + | 29 | content-location | | + | 30 | content-range | | + | 31 | content-type | | + | 32 | cookie | | + | 33 | date | | + | 34 | etag | | + | 35 | expect | | + | 36 | expires | | + | 37 | from | | + | 38 | host | | + | 39 | if-match | | + | 40 | if-modified-since | | + | 41 | if-none-match | | + | 42 | if-range | | + | 43 | if-unmodified-since | | + | 44 | last-modified | | + | 45 | link | | + | 46 | location | | + | 47 | max-forwards | | + | 48 | proxy-authenticate | | + | 49 | proxy-authorization | | + | 50 | range | | + | 51 | referer | | + | 52 | refresh | | + | 53 | retry-after | | + | 54 | server | | + | 55 | set-cookie | | + | 56 | strict-transport-security | | + | 57 | transfer-encoding | | + | 58 | user-agent | | + | 59 | vary | | + | 60 | via | | + | 61 | www-authenticate | | + +-------+-----------------------------+---------------+ +` + bs := bufio.NewScanner(strings.NewReader(fromSpec)) + re := regexp.MustCompile(`\| (\d+)\s+\| (\S+)\s*\| (\S(.*\S)?)?\s+\|`) + for bs.Scan() { + l := bs.Text() + if !strings.Contains(l, "|") { + continue + } + m := re.FindStringSubmatch(l) + if m == nil { + continue + } + i, err := strconv.Atoi(m[1]) + if err != nil { + t.Errorf("Bogus integer on line %q", l) + continue + } + if i < 1 || i > staticTable.len() { + t.Errorf("Bogus index %d on line %q", i, l) + continue + } + if got, want := staticTable.ents[i-1].Name, m[2]; got != want { + t.Errorf("header index %d name = %q; want %q", i, got, want) + } + if got, want := staticTable.ents[i-1].Value, m[3]; got != want { + t.Errorf("header index %d value = %q; want %q", i, got, want) + } + } + if err := bs.Err(); err != nil { + t.Error(err) + } +} diff --git a/libgo/go/golang_org/x/net/idna/idna.go b/libgo/go/golang_org/x/net/idna/idna.go index 3daa8979e1d..e8307f92c52 100644 --- a/libgo/go/golang_org/x/net/idna/idna.go +++ b/libgo/go/golang_org/x/net/idna/idna.go @@ -1,61 +1,661 @@ -// Copyright 2012 The Go Authors. All rights reserved. +// Code generated by running "go run gen.go -core" in golang.org/x/text. DO NOT EDIT. + +// Copyright 2016 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -// Package idna implements IDNA2008 (Internationalized Domain Names for -// Applications), defined in RFC 5890, RFC 5891, RFC 5892, RFC 5893 and -// RFC 5894. -package idna // import "golang.org/x/net/idna" +// Package idna implements IDNA2008 using the compatibility processing +// defined by UTS (Unicode Technical Standard) #46, which defines a standard to +// deal with the transition from IDNA2003. +// +// IDNA2008 (Internationalized Domain Names for Applications), is defined in RFC +// 5890, RFC 5891, RFC 5892, RFC 5893 and RFC 5894. +// UTS #46 is defined in http://www.unicode.org/reports/tr46. +// See http://unicode.org/cldr/utility/idna.jsp for a visualization of the +// differences between these two standards. +package idna // import "golang_org/x/text/internal/export/idna" import ( + "fmt" "strings" "unicode/utf8" -) -// TODO(nigeltao): specify when errors occur. For example, is ToASCII(".") or -// ToASCII("foo\x00") an error? See also http://www.unicode.org/faq/idn.html#11 + "golang_org/x/text/secure/bidirule" + "golang_org/x/text/unicode/norm" +) -// acePrefix is the ASCII Compatible Encoding prefix. -const acePrefix = "xn--" +// NOTE: Unlike common practice in Go APIs, the functions will return a +// sanitized domain name in case of errors. Browsers sometimes use a partially +// evaluated string as lookup. +// TODO: the current error handling is, in my opinion, the least opinionated. +// Other strategies are also viable, though: +// Option 1) Return an empty string in case of error, but allow the user to +// specify explicitly which errors to ignore. +// Option 2) Return the partially evaluated string if it is itself a valid +// string, otherwise return the empty string in case of error. +// Option 3) Option 1 and 2. +// Option 4) Always return an empty string for now and implement Option 1 as +// needed, and document that the return string may not be empty in case of +// error in the future. +// I think Option 1 is best, but it is quite opinionated. -// ToASCII converts a domain or domain label to its ASCII form. For example, -// ToASCII("bücher.example.com") is "xn--bcher-kva.example.com", and -// ToASCII("golang") is "golang". +// ToASCII is a wrapper for Punycode.ToASCII. func ToASCII(s string) (string, error) { - if ascii(s) { - return s, nil - } - labels := strings.Split(s, ".") - for i, label := range labels { - if !ascii(label) { - a, err := encode(acePrefix, label) - if err != nil { - return "", err - } - labels[i] = a + return Punycode.process(s, true) +} + +// ToUnicode is a wrapper for Punycode.ToUnicode. +func ToUnicode(s string) (string, error) { + return Punycode.process(s, false) +} + +// An Option configures a Profile at creation time. +type Option func(*options) + +// Transitional sets a Profile to use the Transitional mapping as defined in UTS +// #46. This will cause, for example, "ß" to be mapped to "ss". Using the +// transitional mapping provides a compromise between IDNA2003 and IDNA2008 +// compatibility. It is used by most browsers when resolving domain names. This +// option is only meaningful if combined with MapForLookup. +func Transitional(transitional bool) Option { + return func(o *options) { o.transitional = true } +} + +// VerifyDNSLength sets whether a Profile should fail if any of the IDN parts +// are longer than allowed by the RFC. +func VerifyDNSLength(verify bool) Option { + return func(o *options) { o.verifyDNSLength = verify } +} + +// ValidateLabels sets whether to check the mandatory label validation criteria +// as defined in Section 5.4 of RFC 5891. This includes testing for correct use +// of hyphens ('-'), normalization, validity of runes, and the context rules. +func ValidateLabels(enable bool) Option { + return func(o *options) { + // Don't override existing mappings, but set one that at least checks + // normalization if it is not set. + if o.mapping == nil && enable { + o.mapping = normalize } + o.trie = trie + o.validateLabels = enable + o.fromPuny = validateFromPunycode } - return strings.Join(labels, "."), nil +} + +// StrictDomainName limits the set of permissable ASCII characters to those +// allowed in domain names as defined in RFC 1034 (A-Z, a-z, 0-9 and the +// hyphen). This is set by default for MapForLookup and ValidateForRegistration. +// +// This option is useful, for instance, for browsers that allow characters +// outside this range, for example a '_' (U+005F LOW LINE). See +// http://www.rfc-editor.org/std/std3.txt for more details This option +// corresponds to the UseSTD3ASCIIRules option in UTS #46. +func StrictDomainName(use bool) Option { + return func(o *options) { + o.trie = trie + o.useSTD3Rules = use + o.fromPuny = validateFromPunycode + } +} + +// NOTE: the following options pull in tables. The tables should not be linked +// in as long as the options are not used. + +// BidiRule enables the Bidi rule as defined in RFC 5893. Any application +// that relies on proper validation of labels should include this rule. +func BidiRule() Option { + return func(o *options) { o.bidirule = bidirule.ValidString } +} + +// ValidateForRegistration sets validation options to verify that a given IDN is +// properly formatted for registration as defined by Section 4 of RFC 5891. +func ValidateForRegistration() Option { + return func(o *options) { + o.mapping = validateRegistration + StrictDomainName(true)(o) + ValidateLabels(true)(o) + VerifyDNSLength(true)(o) + BidiRule()(o) + } +} + +// MapForLookup sets validation and mapping options such that a given IDN is +// transformed for domain name lookup according to the requirements set out in +// Section 5 of RFC 5891. The mappings follow the recommendations of RFC 5894, +// RFC 5895 and UTS 46. It does not add the Bidi Rule. Use the BidiRule option +// to add this check. +// +// The mappings include normalization and mapping case, width and other +// compatibility mappings. +func MapForLookup() Option { + return func(o *options) { + o.mapping = validateAndMap + StrictDomainName(true)(o) + ValidateLabels(true)(o) + } +} + +type options struct { + transitional bool + useSTD3Rules bool + validateLabels bool + verifyDNSLength bool + + trie *idnaTrie + + // fromPuny calls validation rules when converting A-labels to U-labels. + fromPuny func(p *Profile, s string) error + + // mapping implements a validation and mapping step as defined in RFC 5895 + // or UTS 46, tailored to, for example, domain registration or lookup. + mapping func(p *Profile, s string) (string, error) + + // bidirule, if specified, checks whether s conforms to the Bidi Rule + // defined in RFC 5893. + bidirule func(s string) bool +} + +// A Profile defines the configuration of a IDNA mapper. +type Profile struct { + options +} + +func apply(o *options, opts []Option) { + for _, f := range opts { + f(o) + } +} + +// New creates a new Profile. +// +// With no options, the returned Profile is the most permissive and equals the +// Punycode Profile. Options can be passed to further restrict the Profile. The +// MapForLookup and ValidateForRegistration options set a collection of options, +// for lookup and registration purposes respectively, which can be tailored by +// adding more fine-grained options, where later options override earlier +// options. +func New(o ...Option) *Profile { + p := &Profile{} + apply(&p.options, o) + return p +} + +// ToASCII converts a domain or domain label to its ASCII form. For example, +// ToASCII("bücher.example.com") is "xn--bcher-kva.example.com", and +// ToASCII("golang") is "golang". If an error is encountered it will return +// an error and a (partially) processed result. +func (p *Profile) ToASCII(s string) (string, error) { + return p.process(s, true) } // ToUnicode converts a domain or domain label to its Unicode form. For example, // ToUnicode("xn--bcher-kva.example.com") is "bücher.example.com", and -// ToUnicode("golang") is "golang". -func ToUnicode(s string) (string, error) { - if !strings.Contains(s, acePrefix) { - return s, nil +// ToUnicode("golang") is "golang". If an error is encountered it will return +// an error and a (partially) processed result. +func (p *Profile) ToUnicode(s string) (string, error) { + pp := *p + pp.transitional = false + return pp.process(s, false) +} + +// String reports a string with a description of the profile for debugging +// purposes. The string format may change with different versions. +func (p *Profile) String() string { + s := "" + if p.transitional { + s = "Transitional" + } else { + s = "NonTransitional" + } + if p.useSTD3Rules { + s += ":UseSTD3Rules" + } + if p.validateLabels { + s += ":ValidateLabels" + } + if p.verifyDNSLength { + s += ":VerifyDNSLength" + } + return s +} + +var ( + // Punycode is a Profile that does raw punycode processing with a minimum + // of validation. + Punycode *Profile = punycode + + // Lookup is the recommended profile for looking up domain names, according + // to Section 5 of RFC 5891. The exact configuration of this profile may + // change over time. + Lookup *Profile = lookup + + // Display is the recommended profile for displaying domain names. + // The configuration of this profile may change over time. + Display *Profile = display + + // Registration is the recommended profile for checking whether a given + // IDN is valid for registration, according to Section 4 of RFC 5891. + Registration *Profile = registration + + punycode = &Profile{} + lookup = &Profile{options{ + transitional: true, + useSTD3Rules: true, + validateLabels: true, + trie: trie, + fromPuny: validateFromPunycode, + mapping: validateAndMap, + bidirule: bidirule.ValidString, + }} + display = &Profile{options{ + useSTD3Rules: true, + validateLabels: true, + trie: trie, + fromPuny: validateFromPunycode, + mapping: validateAndMap, + bidirule: bidirule.ValidString, + }} + registration = &Profile{options{ + useSTD3Rules: true, + validateLabels: true, + verifyDNSLength: true, + trie: trie, + fromPuny: validateFromPunycode, + mapping: validateRegistration, + bidirule: bidirule.ValidString, + }} + + // TODO: profiles + // Register: recommended for approving domain names: don't do any mappings + // but rather reject on invalid input. Bundle or block deviation characters. +) + +type labelError struct{ label, code_ string } + +func (e labelError) code() string { return e.code_ } +func (e labelError) Error() string { + return fmt.Sprintf("idna: invalid label %q", e.label) +} + +type runeError rune + +func (e runeError) code() string { return "P1" } +func (e runeError) Error() string { + return fmt.Sprintf("idna: disallowed rune %U", e) +} + +// process implements the algorithm described in section 4 of UTS #46, +// see http://www.unicode.org/reports/tr46. +func (p *Profile) process(s string, toASCII bool) (string, error) { + var err error + if p.mapping != nil { + s, err = p.mapping(p, s) + } + // Remove leading empty labels. + for ; len(s) > 0 && s[0] == '.'; s = s[1:] { + } + // It seems like we should only create this error on ToASCII, but the + // UTS 46 conformance tests suggests we should always check this. + if err == nil && p.verifyDNSLength && s == "" { + err = &labelError{s, "A4"} } - labels := strings.Split(s, ".") - for i, label := range labels { + labels := labelIter{orig: s} + for ; !labels.done(); labels.next() { + label := labels.label() + if label == "" { + // Empty labels are not okay. The label iterator skips the last + // label if it is empty. + if err == nil && p.verifyDNSLength { + err = &labelError{s, "A4"} + } + continue + } if strings.HasPrefix(label, acePrefix) { - u, err := decode(label[len(acePrefix):]) - if err != nil { - return "", err + u, err2 := decode(label[len(acePrefix):]) + if err2 != nil { + if err == nil { + err = err2 + } + // Spec says keep the old label. + continue + } + labels.set(u) + if err == nil && p.validateLabels { + err = p.fromPuny(p, u) + } + if err == nil { + // This should be called on NonTransitional, according to the + // spec, but that currently does not have any effect. Use the + // original profile to preserve options. + err = p.validateLabel(u) + } + } else if err == nil { + err = p.validateLabel(label) + } + } + if toASCII { + for labels.reset(); !labels.done(); labels.next() { + label := labels.label() + if !ascii(label) { + a, err2 := encode(acePrefix, label) + if err == nil { + err = err2 + } + label = a + labels.set(a) + } + n := len(label) + if p.verifyDNSLength && err == nil && (n == 0 || n > 63) { + err = &labelError{label, "A4"} + } + } + } + s = labels.result() + if toASCII && p.verifyDNSLength && err == nil { + // Compute the length of the domain name minus the root label and its dot. + n := len(s) + if n > 0 && s[n-1] == '.' { + n-- + } + if len(s) < 1 || n > 253 { + err = &labelError{s, "A4"} + } + } + return s, err +} + +func normalize(p *Profile, s string) (string, error) { + return norm.NFC.String(s), nil +} + +func validateRegistration(p *Profile, s string) (string, error) { + if !norm.NFC.IsNormalString(s) { + return s, &labelError{s, "V1"} + } + var err error + for i := 0; i < len(s); { + v, sz := trie.lookupString(s[i:]) + i += sz + // Copy bytes not copied so far. + switch p.simplify(info(v).category()) { + // TODO: handle the NV8 defined in the Unicode idna data set to allow + // for strict conformance to IDNA2008. + case valid, deviation: + case disallowed, mapped, unknown, ignored: + if err == nil { + r, _ := utf8.DecodeRuneInString(s[i:]) + err = runeError(r) + } + } + } + return s, err +} + +func validateAndMap(p *Profile, s string) (string, error) { + var ( + err error + b []byte + k int + ) + for i := 0; i < len(s); { + v, sz := trie.lookupString(s[i:]) + start := i + i += sz + // Copy bytes not copied so far. + switch p.simplify(info(v).category()) { + case valid: + continue + case disallowed: + if err == nil { + r, _ := utf8.DecodeRuneInString(s[i:]) + err = runeError(r) } - labels[i] = u + continue + case mapped, deviation: + b = append(b, s[k:start]...) + b = info(v).appendMapping(b, s[start:i]) + case ignored: + b = append(b, s[k:start]...) + // drop the rune + case unknown: + b = append(b, s[k:start]...) + b = append(b, "\ufffd"...) + } + k = i + } + if k == 0 { + // No changes so far. + s = norm.NFC.String(s) + } else { + b = append(b, s[k:]...) + if norm.NFC.QuickSpan(b) != len(b) { + b = norm.NFC.Bytes(b) } + // TODO: the punycode converters require strings as input. + s = string(b) + } + return s, err +} + +// A labelIter allows iterating over domain name labels. +type labelIter struct { + orig string + slice []string + curStart int + curEnd int + i int +} + +func (l *labelIter) reset() { + l.curStart = 0 + l.curEnd = 0 + l.i = 0 +} + +func (l *labelIter) done() bool { + return l.curStart >= len(l.orig) +} + +func (l *labelIter) result() string { + if l.slice != nil { + return strings.Join(l.slice, ".") + } + return l.orig +} + +func (l *labelIter) label() string { + if l.slice != nil { + return l.slice[l.i] + } + p := strings.IndexByte(l.orig[l.curStart:], '.') + l.curEnd = l.curStart + p + if p == -1 { + l.curEnd = len(l.orig) + } + return l.orig[l.curStart:l.curEnd] +} + +// next sets the value to the next label. It skips the last label if it is empty. +func (l *labelIter) next() { + l.i++ + if l.slice != nil { + if l.i >= len(l.slice) || l.i == len(l.slice)-1 && l.slice[l.i] == "" { + l.curStart = len(l.orig) + } + } else { + l.curStart = l.curEnd + 1 + if l.curStart == len(l.orig)-1 && l.orig[l.curStart] == '.' { + l.curStart = len(l.orig) + } + } +} + +func (l *labelIter) set(s string) { + if l.slice == nil { + l.slice = strings.Split(l.orig, ".") + } + l.slice[l.i] = s +} + +// acePrefix is the ASCII Compatible Encoding prefix. +const acePrefix = "xn--" + +func (p *Profile) simplify(cat category) category { + switch cat { + case disallowedSTD3Mapped: + if p.useSTD3Rules { + cat = disallowed + } else { + cat = mapped + } + case disallowedSTD3Valid: + if p.useSTD3Rules { + cat = disallowed + } else { + cat = valid + } + case deviation: + if !p.transitional { + cat = valid + } + case validNV8, validXV8: + // TODO: handle V2008 + cat = valid + } + return cat +} + +func validateFromPunycode(p *Profile, s string) error { + if !norm.NFC.IsNormalString(s) { + return &labelError{s, "V1"} + } + for i := 0; i < len(s); { + v, sz := trie.lookupString(s[i:]) + if c := p.simplify(info(v).category()); c != valid && c != deviation { + return &labelError{s, "V6"} + } + i += sz + } + return nil +} + +const ( + zwnj = "\u200c" + zwj = "\u200d" +) + +type joinState int8 + +const ( + stateStart joinState = iota + stateVirama + stateBefore + stateBeforeVirama + stateAfter + stateFAIL +) + +var joinStates = [][numJoinTypes]joinState{ + stateStart: { + joiningL: stateBefore, + joiningD: stateBefore, + joinZWNJ: stateFAIL, + joinZWJ: stateFAIL, + joinVirama: stateVirama, + }, + stateVirama: { + joiningL: stateBefore, + joiningD: stateBefore, + }, + stateBefore: { + joiningL: stateBefore, + joiningD: stateBefore, + joiningT: stateBefore, + joinZWNJ: stateAfter, + joinZWJ: stateFAIL, + joinVirama: stateBeforeVirama, + }, + stateBeforeVirama: { + joiningL: stateBefore, + joiningD: stateBefore, + joiningT: stateBefore, + }, + stateAfter: { + joiningL: stateFAIL, + joiningD: stateBefore, + joiningT: stateAfter, + joiningR: stateStart, + joinZWNJ: stateFAIL, + joinZWJ: stateFAIL, + joinVirama: stateAfter, // no-op as we can't accept joiners here + }, + stateFAIL: { + 0: stateFAIL, + joiningL: stateFAIL, + joiningD: stateFAIL, + joiningT: stateFAIL, + joiningR: stateFAIL, + joinZWNJ: stateFAIL, + joinZWJ: stateFAIL, + joinVirama: stateFAIL, + }, +} + +// validateLabel validates the criteria from Section 4.1. Item 1, 4, and 6 are +// already implicitly satisfied by the overall implementation. +func (p *Profile) validateLabel(s string) error { + if s == "" { + if p.verifyDNSLength { + return &labelError{s, "A4"} + } + return nil + } + if p.bidirule != nil && !p.bidirule(s) { + return &labelError{s, "B"} + } + if !p.validateLabels { + return nil + } + trie := p.trie // p.validateLabels is only set if trie is set. + if len(s) > 4 && s[2] == '-' && s[3] == '-' { + return &labelError{s, "V2"} + } + if s[0] == '-' || s[len(s)-1] == '-' { + return &labelError{s, "V3"} + } + // TODO: merge the use of this in the trie. + v, sz := trie.lookupString(s) + x := info(v) + if x.isModifier() { + return &labelError{s, "V5"} + } + // Quickly return in the absence of zero-width (non) joiners. + if strings.Index(s, zwj) == -1 && strings.Index(s, zwnj) == -1 { + return nil + } + st := stateStart + for i := 0; ; { + jt := x.joinType() + if s[i:i+sz] == zwj { + jt = joinZWJ + } else if s[i:i+sz] == zwnj { + jt = joinZWNJ + } + st = joinStates[st][jt] + if x.isViramaModifier() { + st = joinStates[st][joinVirama] + } + if i += sz; i == len(s) { + break + } + v, sz = trie.lookupString(s[i:]) + x = info(v) + } + if st == stateFAIL || st == stateAfter { + return &labelError{s, "C"} } - return strings.Join(labels, "."), nil + return nil } func ascii(s string) bool { diff --git a/libgo/go/golang_org/x/net/idna/punycode.go b/libgo/go/golang_org/x/net/idna/punycode.go index 92e733f6a7b..fab9229881a 100644 --- a/libgo/go/golang_org/x/net/idna/punycode.go +++ b/libgo/go/golang_org/x/net/idna/punycode.go @@ -1,4 +1,6 @@ -// Copyright 2012 The Go Authors. All rights reserved. +// Code generated by running "go run gen.go -core" in golang.org/x/text. DO NOT EDIT. + +// Copyright 2016 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. @@ -7,7 +9,6 @@ package idna // This file implements the Punycode algorithm from RFC 3492. import ( - "fmt" "math" "strings" "unicode/utf8" @@ -27,6 +28,8 @@ const ( tmin int32 = 1 ) +func punyError(s string) error { return &labelError{s, "A3"} } + // decode decodes a string as specified in section 6.2. func decode(encoded string) (string, error) { if encoded == "" { @@ -34,7 +37,7 @@ func decode(encoded string) (string, error) { } pos := 1 + strings.LastIndex(encoded, "-") if pos == 1 { - return "", fmt.Errorf("idna: invalid label %q", encoded) + return "", punyError(encoded) } if pos == len(encoded) { return encoded[:len(encoded)-1], nil @@ -50,16 +53,16 @@ func decode(encoded string) (string, error) { oldI, w := i, int32(1) for k := base; ; k += base { if pos == len(encoded) { - return "", fmt.Errorf("idna: invalid label %q", encoded) + return "", punyError(encoded) } digit, ok := decodeDigit(encoded[pos]) if !ok { - return "", fmt.Errorf("idna: invalid label %q", encoded) + return "", punyError(encoded) } pos++ i += digit * w if i < 0 { - return "", fmt.Errorf("idna: invalid label %q", encoded) + return "", punyError(encoded) } t := k - bias if t < tmin { @@ -72,7 +75,7 @@ func decode(encoded string) (string, error) { } w *= base - t if w >= math.MaxInt32/base { - return "", fmt.Errorf("idna: invalid label %q", encoded) + return "", punyError(encoded) } } x := int32(len(output) + 1) @@ -80,7 +83,7 @@ func decode(encoded string) (string, error) { n += i / x i %= x if n > utf8.MaxRune || len(output) >= 1024 { - return "", fmt.Errorf("idna: invalid label %q", encoded) + return "", punyError(encoded) } output = append(output, 0) copy(output[i+1:], output[i:]) @@ -121,14 +124,14 @@ func encode(prefix, s string) (string, error) { } delta += (m - n) * (h + 1) if delta < 0 { - return "", fmt.Errorf("idna: invalid label %q", s) + return "", punyError(s) } n = m for _, r := range s { if r < n { delta++ if delta < 0 { - return "", fmt.Errorf("idna: invalid label %q", s) + return "", punyError(s) } continue } diff --git a/libgo/go/golang_org/x/net/idna/tables.go b/libgo/go/golang_org/x/net/idna/tables.go new file mode 100644 index 00000000000..d57a3e25a9d --- /dev/null +++ b/libgo/go/golang_org/x/net/idna/tables.go @@ -0,0 +1,4479 @@ +// Code generated by running "go run gen.go -core" in golang.org/x/text. DO NOT EDIT. + +// Code generated by running "go generate" in golang_org/x/text. DO NOT EDIT. + +package idna + +// UnicodeVersion is the Unicode version from which the tables in this package are derived. +const UnicodeVersion = "9.0.0" + +var mappings string = "" + // Size: 8176 bytes + "\x00\x01 \x03 ̈\x01a\x03 ̄\x012\x013\x03 ́\x03 ̧\x011\x01o\x051⁄4\x051⁄2" + + "\x053⁄4\x03i̇\x03l·\x03ʼn\x01s\x03dž\x03ⱥ\x03ⱦ\x01h\x01j\x01r\x01w\x01y" + + "\x03 ̆\x03 ̇\x03 ̊\x03 ̨\x03 ̃\x03 ̋\x01l\x01x\x04̈́\x03 ι\x01;\x05 ̈́" + + "\x04եւ\x04اٴ\x04وٴ\x04ۇٴ\x04يٴ\x06क़\x06ख़\x06ग़\x06ज़\x06ड़\x06ढ़\x06फ़" + + "\x06य़\x06ড়\x06ঢ়\x06য়\x06ਲ਼\x06ਸ਼\x06ਖ਼\x06ਗ਼\x06ਜ਼\x06ਫ਼\x06ଡ଼\x06ଢ଼" + + "\x06ํา\x06ໍາ\x06ຫນ\x06ຫມ\x06གྷ\x06ཌྷ\x06དྷ\x06བྷ\x06ཛྷ\x06ཀྵ\x06ཱི\x06ཱུ" + + "\x06ྲྀ\x09ྲཱྀ\x06ླྀ\x09ླཱྀ\x06ཱྀ\x06ྒྷ\x06ྜྷ\x06ྡྷ\x06ྦྷ\x06ྫྷ\x06ྐྵ\x02" + + "в\x02д\x02о\x02с\x02т\x02ъ\x02ѣ\x02æ\x01b\x01d\x01e\x02ǝ\x01g\x01i\x01k" + + "\x01m\x01n\x02ȣ\x01p\x01t\x01u\x02ɐ\x02ɑ\x02ə\x02ɛ\x02ɜ\x02ŋ\x02ɔ\x02ɯ" + + "\x01v\x02β\x02γ\x02δ\x02φ\x02χ\x02ρ\x02н\x02ɒ\x01c\x02ɕ\x02ð\x01f\x02ɟ" + + "\x02ɡ\x02ɥ\x02ɨ\x02ɩ\x02ɪ\x02ʝ\x02ɭ\x02ʟ\x02ɱ\x02ɰ\x02ɲ\x02ɳ\x02ɴ\x02ɵ" + + "\x02ɸ\x02ʂ\x02ʃ\x02ƫ\x02ʉ\x02ʊ\x02ʋ\x02ʌ\x01z\x02ʐ\x02ʑ\x02ʒ\x02θ\x02ss" + + "\x02ά\x02έ\x02ή\x02ί\x02ό\x02ύ\x02ώ\x05ἀι\x05ἁι\x05ἂι\x05ἃι\x05ἄι\x05ἅι" + + "\x05ἆι\x05ἇι\x05ἠι\x05ἡι\x05ἢι\x05ἣι\x05ἤι\x05ἥι\x05ἦι\x05ἧι\x05ὠι\x05ὡι" + + "\x05ὢι\x05ὣι\x05ὤι\x05ὥι\x05ὦι\x05ὧι\x05ὰι\x04αι\x04άι\x05ᾶι\x02ι\x05 ̈͂" + + "\x05ὴι\x04ηι\x04ήι\x05ῆι\x05 ̓̀\x05 ̓́\x05 ̓͂\x02ΐ\x05 ̔̀\x05 ̔́\x05 ̔͂" + + "\x02ΰ\x05 ̈̀\x01`\x05ὼι\x04ωι\x04ώι\x05ῶι\x06′′\x09′′′\x06‵‵\x09‵‵‵\x02!" + + "!\x02??\x02?!\x02!?\x0c′′′′\x010\x014\x015\x016\x017\x018\x019\x01+\x01=" + + "\x01(\x01)\x02rs\x02ħ\x02no\x01q\x02sm\x02tm\x02ω\x02å\x02א\x02ב\x02ג" + + "\x02ד\x02π\x051⁄7\x051⁄9\x061⁄10\x051⁄3\x052⁄3\x051⁄5\x052⁄5\x053⁄5\x054" + + "⁄5\x051⁄6\x055⁄6\x051⁄8\x053⁄8\x055⁄8\x057⁄8\x041⁄\x02ii\x02iv\x02vi" + + "\x04viii\x02ix\x02xi\x050⁄3\x06∫∫\x09∫∫∫\x06∮∮\x09∮∮∮\x0210\x0211\x0212" + + "\x0213\x0214\x0215\x0216\x0217\x0218\x0219\x0220\x04(10)\x04(11)\x04(12)" + + "\x04(13)\x04(14)\x04(15)\x04(16)\x04(17)\x04(18)\x04(19)\x04(20)\x0c∫∫∫∫" + + "\x02==\x05⫝̸\x02ɫ\x02ɽ\x02ȿ\x02ɀ\x01.\x04 ゙\x04 ゚\x06より\x06コト\x05(ᄀ)\x05" + + "(ᄂ)\x05(ᄃ)\x05(ᄅ)\x05(ᄆ)\x05(ᄇ)\x05(ᄉ)\x05(ᄋ)\x05(ᄌ)\x05(ᄎ)\x05(ᄏ)\x05(ᄐ" + + ")\x05(ᄑ)\x05(ᄒ)\x05(가)\x05(나)\x05(다)\x05(라)\x05(마)\x05(바)\x05(사)\x05(아)" + + "\x05(자)\x05(차)\x05(카)\x05(타)\x05(파)\x05(하)\x05(주)\x08(오전)\x08(오후)\x05(一)" + + "\x05(二)\x05(三)\x05(四)\x05(五)\x05(六)\x05(七)\x05(八)\x05(九)\x05(十)\x05(月)" + + "\x05(火)\x05(水)\x05(木)\x05(金)\x05(土)\x05(日)\x05(株)\x05(有)\x05(社)\x05(名)" + + "\x05(特)\x05(財)\x05(祝)\x05(労)\x05(代)\x05(呼)\x05(学)\x05(監)\x05(企)\x05(資)" + + "\x05(協)\x05(祭)\x05(休)\x05(自)\x05(至)\x0221\x0222\x0223\x0224\x0225\x0226" + + "\x0227\x0228\x0229\x0230\x0231\x0232\x0233\x0234\x0235\x06참고\x06주의\x0236" + + "\x0237\x0238\x0239\x0240\x0241\x0242\x0243\x0244\x0245\x0246\x0247\x0248" + + "\x0249\x0250\x041月\x042月\x043月\x044月\x045月\x046月\x047月\x048月\x049月\x0510" + + "月\x0511月\x0512月\x02hg\x02ev\x0cアパート\x0cアルファ\x0cアンペア\x09アール\x0cイニング\x09" + + "インチ\x09ウォン\x0fエスクード\x0cエーカー\x09オンス\x09オーム\x09カイリ\x0cカラット\x0cカロリー\x09ガロ" + + "ン\x09ガンマ\x06ギガ\x09ギニー\x0cキュリー\x0cギルダー\x06キロ\x0fキログラム\x12キロメートル\x0fキロワッ" + + "ト\x09グラム\x0fグラムトン\x0fクルゼイロ\x0cクローネ\x09ケース\x09コルナ\x09コーポ\x0cサイクル\x0fサンチ" + + "ーム\x0cシリング\x09センチ\x09セント\x09ダース\x06デシ\x06ドル\x06トン\x06ナノ\x09ノット\x09ハイツ" + + "\x0fパーセント\x09パーツ\x0cバーレル\x0fピアストル\x09ピクル\x06ピコ\x06ビル\x0fファラッド\x0cフィート" + + "\x0fブッシェル\x09フラン\x0fヘクタール\x06ペソ\x09ペニヒ\x09ヘルツ\x09ペンス\x09ページ\x09ベータ\x0cポイ" + + "ント\x09ボルト\x06ホン\x09ポンド\x09ホール\x09ホーン\x0cマイクロ\x09マイル\x09マッハ\x09マルク\x0fマ" + + "ンション\x0cミクロン\x06ミリ\x0fミリバール\x06メガ\x0cメガトン\x0cメートル\x09ヤード\x09ヤール\x09ユアン" + + "\x0cリットル\x06リラ\x09ルピー\x0cルーブル\x06レム\x0fレントゲン\x09ワット\x040点\x041点\x042点" + + "\x043点\x044点\x045点\x046点\x047点\x048点\x049点\x0510点\x0511点\x0512点\x0513点" + + "\x0514点\x0515点\x0516点\x0517点\x0518点\x0519点\x0520点\x0521点\x0522点\x0523点" + + "\x0524点\x02da\x02au\x02ov\x02pc\x02dm\x02iu\x06平成\x06昭和\x06大正\x06明治\x0c株" + + "式会社\x02pa\x02na\x02ma\x02ka\x02kb\x02mb\x02gb\x04kcal\x02pf\x02nf\x02m" + + "g\x02kg\x02hz\x02ml\x02dl\x02kl\x02fm\x02nm\x02mm\x02cm\x02km\x02m2\x02m" + + "3\x05m∕s\x06m∕s2\x07rad∕s\x08rad∕s2\x02ps\x02ns\x02ms\x02pv\x02nv\x02mv" + + "\x02kv\x02pw\x02nw\x02mw\x02kw\x02bq\x02cc\x02cd\x06c∕kg\x02db\x02gy\x02" + + "ha\x02hp\x02in\x02kk\x02kt\x02lm\x02ln\x02lx\x02ph\x02pr\x02sr\x02sv\x02" + + "wb\x05v∕m\x05a∕m\x041日\x042日\x043日\x044日\x045日\x046日\x047日\x048日\x049日" + + "\x0510日\x0511日\x0512日\x0513日\x0514日\x0515日\x0516日\x0517日\x0518日\x0519日" + + "\x0520日\x0521日\x0522日\x0523日\x0524日\x0525日\x0526日\x0527日\x0528日\x0529日" + + "\x0530日\x0531日\x02ь\x02ɦ\x02ɬ\x02ʞ\x02ʇ\x02œ\x04𤋮\x04𢡊\x04𢡄\x04𣏕\x04𥉉" + + "\x04𥳐\x04𧻓\x02ff\x02fi\x02fl\x02st\x04մն\x04մե\x04մի\x04վն\x04մխ\x04יִ" + + "\x04ײַ\x02ע\x02ה\x02כ\x02ל\x02ם\x02ר\x02ת\x04שׁ\x04שׂ\x06שּׁ\x06שּׂ\x04א" + + "ַ\x04אָ\x04אּ\x04בּ\x04גּ\x04דּ\x04הּ\x04וּ\x04זּ\x04טּ\x04יּ\x04ךּ\x04" + + "כּ\x04לּ\x04מּ\x04נּ\x04סּ\x04ףּ\x04פּ\x04צּ\x04קּ\x04רּ\x04שּ\x04תּ" + + "\x04וֹ\x04בֿ\x04כֿ\x04פֿ\x04אל\x02ٱ\x02ٻ\x02پ\x02ڀ\x02ٺ\x02ٿ\x02ٹ\x02ڤ" + + "\x02ڦ\x02ڄ\x02ڃ\x02چ\x02ڇ\x02ڍ\x02ڌ\x02ڎ\x02ڈ\x02ژ\x02ڑ\x02ک\x02گ\x02ڳ" + + "\x02ڱ\x02ں\x02ڻ\x02ۀ\x02ہ\x02ھ\x02ے\x02ۓ\x02ڭ\x02ۇ\x02ۆ\x02ۈ\x02ۋ\x02ۅ" + + "\x02ۉ\x02ې\x02ى\x04ئا\x04ئە\x04ئو\x04ئۇ\x04ئۆ\x04ئۈ\x04ئې\x04ئى\x02ی\x04" + + "ئج\x04ئح\x04ئم\x04ئي\x04بج\x04بح\x04بخ\x04بم\x04بى\x04بي\x04تج\x04تح" + + "\x04تخ\x04تم\x04تى\x04تي\x04ثج\x04ثم\x04ثى\x04ثي\x04جح\x04جم\x04حج\x04حم" + + "\x04خج\x04خح\x04خم\x04سج\x04سح\x04سخ\x04سم\x04صح\x04صم\x04ضج\x04ضح\x04ضخ" + + "\x04ضم\x04طح\x04طم\x04ظم\x04عج\x04عم\x04غج\x04غم\x04فج\x04فح\x04فخ\x04فم" + + "\x04فى\x04في\x04قح\x04قم\x04قى\x04قي\x04كا\x04كج\x04كح\x04كخ\x04كل\x04كم" + + "\x04كى\x04كي\x04لج\x04لح\x04لخ\x04لم\x04لى\x04لي\x04مج\x04مح\x04مخ\x04مم" + + "\x04مى\x04مي\x04نج\x04نح\x04نخ\x04نم\x04نى\x04ني\x04هج\x04هم\x04هى\x04هي" + + "\x04يج\x04يح\x04يخ\x04يم\x04يى\x04يي\x04ذٰ\x04رٰ\x04ىٰ\x05 ٌّ\x05 ٍّ\x05" + + " َّ\x05 ُّ\x05 ِّ\x05 ّٰ\x04ئر\x04ئز\x04ئن\x04بر\x04بز\x04بن\x04تر\x04تز" + + "\x04تن\x04ثر\x04ثز\x04ثن\x04ما\x04نر\x04نز\x04نن\x04ير\x04يز\x04ين\x04ئخ" + + "\x04ئه\x04به\x04ته\x04صخ\x04له\x04نه\x04هٰ\x04يه\x04ثه\x04سه\x04شم\x04شه" + + "\x06ـَّ\x06ـُّ\x06ـِّ\x04طى\x04طي\x04عى\x04عي\x04غى\x04غي\x04سى\x04سي" + + "\x04شى\x04شي\x04حى\x04حي\x04جى\x04جي\x04خى\x04خي\x04صى\x04صي\x04ضى\x04ضي" + + "\x04شج\x04شح\x04شخ\x04شر\x04سر\x04صر\x04ضر\x04اً\x06تجم\x06تحج\x06تحم" + + "\x06تخم\x06تمج\x06تمح\x06تمخ\x06جمح\x06حمي\x06حمى\x06سحج\x06سجح\x06سجى" + + "\x06سمح\x06سمج\x06سمم\x06صحح\x06صمم\x06شحم\x06شجي\x06شمخ\x06شمم\x06ضحى" + + "\x06ضخم\x06طمح\x06طمم\x06طمي\x06عجم\x06عمم\x06عمى\x06غمم\x06غمي\x06غمى" + + "\x06فخم\x06قمح\x06قمم\x06لحم\x06لحي\x06لحى\x06لجج\x06لخم\x06لمح\x06محج" + + "\x06محم\x06محي\x06مجح\x06مجم\x06مخج\x06مخم\x06مجخ\x06همج\x06همم\x06نحم" + + "\x06نحى\x06نجم\x06نجى\x06نمي\x06نمى\x06يمم\x06بخي\x06تجي\x06تجى\x06تخي" + + "\x06تخى\x06تمي\x06تمى\x06جمي\x06جحى\x06جمى\x06سخى\x06صحي\x06شحي\x06ضحي" + + "\x06لجي\x06لمي\x06يحي\x06يجي\x06يمي\x06ممي\x06قمي\x06نحي\x06عمي\x06كمي" + + "\x06نجح\x06مخي\x06لجم\x06كمم\x06جحي\x06حجي\x06مجي\x06فمي\x06بحي\x06سخي" + + "\x06نجي\x06صلے\x06قلے\x08الله\x08اكبر\x08محمد\x08صلعم\x08رسول\x08عليه" + + "\x08وسلم\x06صلى!صلى الله عليه وسلم\x0fجل جلاله\x08ریال\x01,\x01:\x01!" + + "\x01?\x01_\x01{\x01}\x01[\x01]\x01#\x01&\x01*\x01-\x01<\x01>\x01\\\x01$" + + "\x01%\x01@\x04ـً\x04ـَ\x04ـُ\x04ـِ\x04ـّ\x04ـْ\x02ء\x02آ\x02أ\x02ؤ\x02إ" + + "\x02ئ\x02ا\x02ب\x02ة\x02ت\x02ث\x02ج\x02ح\x02خ\x02د\x02ذ\x02ر\x02ز\x02س" + + "\x02ش\x02ص\x02ض\x02ط\x02ظ\x02ع\x02غ\x02ف\x02ق\x02ك\x02ل\x02م\x02ن\x02ه" + + "\x02و\x02ي\x04لآ\x04لأ\x04لإ\x04لا\x01\x22\x01'\x01/\x01^\x01|\x01~\x02¢" + + "\x02£\x02¬\x02¦\x02¥\x08𝅗𝅥\x08𝅘𝅥\x0c𝅘𝅥𝅮\x0c𝅘𝅥𝅯\x0c𝅘𝅥𝅰\x0c𝅘𝅥𝅱\x0c𝅘𝅥𝅲\x08𝆹" + + "𝅥\x08𝆺𝅥\x0c𝆹𝅥𝅮\x0c𝆺𝅥𝅮\x0c𝆹𝅥𝅯\x0c𝆺𝅥𝅯\x02ı\x02ȷ\x02α\x02ε\x02ζ\x02η\x02" + + "κ\x02λ\x02μ\x02ν\x02ξ\x02ο\x02σ\x02τ\x02υ\x02ψ\x03∇\x03∂\x02ϝ\x02ٮ\x02ڡ" + + "\x02ٯ\x020,\x021,\x022,\x023,\x024,\x025,\x026,\x027,\x028,\x029,\x03(a)" + + "\x03(b)\x03(c)\x03(d)\x03(e)\x03(f)\x03(g)\x03(h)\x03(i)\x03(j)\x03(k)" + + "\x03(l)\x03(m)\x03(n)\x03(o)\x03(p)\x03(q)\x03(r)\x03(s)\x03(t)\x03(u)" + + "\x03(v)\x03(w)\x03(x)\x03(y)\x03(z)\x07〔s〕\x02wz\x02hv\x02sd\x03ppv\x02w" + + "c\x02mc\x02md\x02dj\x06ほか\x06ココ\x03サ\x03手\x03字\x03双\x03デ\x03二\x03多\x03解" + + "\x03天\x03交\x03映\x03無\x03料\x03前\x03後\x03再\x03新\x03初\x03終\x03生\x03販\x03声" + + "\x03吹\x03演\x03投\x03捕\x03一\x03三\x03遊\x03左\x03中\x03右\x03指\x03走\x03打\x03禁" + + "\x03空\x03合\x03満\x03有\x03月\x03申\x03割\x03営\x03配\x09〔本〕\x09〔三〕\x09〔二〕\x09〔安" + + "〕\x09〔点〕\x09〔打〕\x09〔盗〕\x09〔勝〕\x09〔敗〕\x03得\x03可\x03丽\x03丸\x03乁\x03你\x03" + + "侮\x03侻\x03倂\x03偺\x03備\x03僧\x03像\x03㒞\x03免\x03兔\x03兤\x03具\x03㒹\x03內\x03" + + "冗\x03冤\x03仌\x03冬\x03况\x03凵\x03刃\x03㓟\x03刻\x03剆\x03剷\x03㔕\x03勇\x03勉\x03" + + "勤\x03勺\x03包\x03匆\x03北\x03卉\x03卑\x03博\x03即\x03卽\x03卿\x03灰\x03及\x03叟\x03" + + "叫\x03叱\x03吆\x03咞\x03吸\x03呈\x03周\x03咢\x03哶\x03唐\x03啓\x03啣\x03善\x03喙\x03" + + "喫\x03喳\x03嗂\x03圖\x03嘆\x03圗\x03噑\x03噴\x03切\x03壮\x03城\x03埴\x03堍\x03型\x03" + + "堲\x03報\x03墬\x03売\x03壷\x03夆\x03夢\x03奢\x03姬\x03娛\x03娧\x03姘\x03婦\x03㛮\x03" + + "嬈\x03嬾\x03寃\x03寘\x03寧\x03寳\x03寿\x03将\x03尢\x03㞁\x03屠\x03屮\x03峀\x03岍\x03" + + "嵃\x03嵮\x03嵫\x03嵼\x03巡\x03巢\x03㠯\x03巽\x03帨\x03帽\x03幩\x03㡢\x03㡼\x03庰\x03" + + "庳\x03庶\x03廊\x03廾\x03舁\x03弢\x03㣇\x03形\x03彫\x03㣣\x03徚\x03忍\x03志\x03忹\x03" + + "悁\x03㤺\x03㤜\x03悔\x03惇\x03慈\x03慌\x03慎\x03慺\x03憎\x03憲\x03憤\x03憯\x03懞\x03" + + "懲\x03懶\x03成\x03戛\x03扝\x03抱\x03拔\x03捐\x03挽\x03拼\x03捨\x03掃\x03揤\x03搢\x03" + + "揅\x03掩\x03㨮\x03摩\x03摾\x03撝\x03摷\x03㩬\x03敏\x03敬\x03旣\x03書\x03晉\x03㬙\x03" + + "暑\x03㬈\x03㫤\x03冒\x03冕\x03最\x03暜\x03肭\x03䏙\x03朗\x03望\x03朡\x03杞\x03杓\x03" + + "㭉\x03柺\x03枅\x03桒\x03梅\x03梎\x03栟\x03椔\x03㮝\x03楂\x03榣\x03槪\x03檨\x03櫛\x03" + + "㰘\x03次\x03歔\x03㱎\x03歲\x03殟\x03殺\x03殻\x03汎\x03沿\x03泍\x03汧\x03洖\x03派\x03" + + "海\x03流\x03浩\x03浸\x03涅\x03洴\x03港\x03湮\x03㴳\x03滋\x03滇\x03淹\x03潮\x03濆\x03" + + "瀹\x03瀞\x03瀛\x03㶖\x03灊\x03災\x03灷\x03炭\x03煅\x03熜\x03爨\x03爵\x03牐\x03犀\x03" + + "犕\x03獺\x03王\x03㺬\x03玥\x03㺸\x03瑇\x03瑜\x03瑱\x03璅\x03瓊\x03㼛\x03甤\x03甾\x03" + + "異\x03瘐\x03㿼\x03䀈\x03直\x03眞\x03真\x03睊\x03䀹\x03瞋\x03䁆\x03䂖\x03硎\x03碌\x03" + + "磌\x03䃣\x03祖\x03福\x03秫\x03䄯\x03穀\x03穊\x03穏\x03䈂\x03篆\x03築\x03䈧\x03糒\x03" + + "䊠\x03糨\x03糣\x03紀\x03絣\x03䌁\x03緇\x03縂\x03繅\x03䌴\x03䍙\x03罺\x03羕\x03翺\x03" + + "者\x03聠\x03聰\x03䏕\x03育\x03脃\x03䐋\x03脾\x03媵\x03舄\x03辞\x03䑫\x03芑\x03芋\x03" + + "芝\x03劳\x03花\x03芳\x03芽\x03苦\x03若\x03茝\x03荣\x03莭\x03茣\x03莽\x03菧\x03著\x03" + + "荓\x03菊\x03菌\x03菜\x03䔫\x03蓱\x03蓳\x03蔖\x03蕤\x03䕝\x03䕡\x03䕫\x03虐\x03虜\x03" + + "虧\x03虩\x03蚩\x03蚈\x03蜎\x03蛢\x03蝹\x03蜨\x03蝫\x03螆\x03蟡\x03蠁\x03䗹\x03衠\x03" + + "衣\x03裗\x03裞\x03䘵\x03裺\x03㒻\x03䚾\x03䛇\x03誠\x03諭\x03變\x03豕\x03貫\x03賁\x03" + + "贛\x03起\x03跋\x03趼\x03跰\x03軔\x03輸\x03邔\x03郱\x03鄑\x03鄛\x03鈸\x03鋗\x03鋘\x03" + + "鉼\x03鏹\x03鐕\x03開\x03䦕\x03閷\x03䧦\x03雃\x03嶲\x03霣\x03䩮\x03䩶\x03韠\x03䪲\x03" + + "頋\x03頩\x03飢\x03䬳\x03餩\x03馧\x03駂\x03駾\x03䯎\x03鬒\x03鱀\x03鳽\x03䳎\x03䳭\x03" + + "鵧\x03䳸\x03麻\x03䵖\x03黹\x03黾\x03鼅\x03鼏\x03鼖\x03鼻" + +var xorData string = "" + // Size: 4855 bytes + "\x02\x0c\x09\x02\xb0\xec\x02\xad\xd8\x02\xad\xd9\x02\x06\x07\x02\x0f\x12" + + "\x02\x0f\x1f\x02\x0f\x1d\x02\x01\x13\x02\x0f\x16\x02\x0f\x0b\x02\x0f3" + + "\x02\x0f7\x02\x0f?\x02\x0f/\x02\x0f*\x02\x0c&\x02\x0c*\x02\x0c;\x02\x0c9" + + "\x02\x0c%\x02\xab\xed\x02\xab\xe2\x02\xab\xe3\x02\xa9\xe0\x02\xa9\xe1" + + "\x02\xa9\xe6\x02\xa3\xcb\x02\xa3\xc8\x02\xa3\xc9\x02\x01#\x02\x01\x08" + + "\x02\x0e>\x02\x0e'\x02\x0f\x03\x02\x03\x0d\x02\x03\x09\x02\x03\x17\x02" + + "\x03\x0e\x02\x02\x03\x02\x011\x02\x01\x00\x02\x01\x10\x02\x03<\x02\x07" + + "\x0d\x02\x02\x0c\x02\x0c0\x02\x01\x03\x02\x01\x01\x02\x01 \x02\x01\x22" + + "\x02\x01)\x02\x01\x0a\x02\x01\x0c\x02\x02\x06\x02\x02\x02\x02\x03\x10" + + "\x03\x037 \x03\x0b+\x03\x02\x01\x04\x02\x01\x02\x02\x019\x02\x03\x1c\x02" + + "\x02$\x03\x80p$\x02\x03:\x02\x03\x0a\x03\xc1r.\x03\xc1r,\x03\xc1r\x02" + + "\x02\x02:\x02\x02>\x02\x02,\x02\x02\x10\x02\x02\x00\x03\xc1s<\x03\xc1s*" + + "\x03\xc2L$\x03\xc2L;\x02\x09)\x02\x0a\x19\x03\x83\xab\xe3\x03\x83\xab" + + "\xf2\x03 4\xe0\x03\x81\xab\xea\x03\x81\xab\xf3\x03 4\xef\x03\x96\xe1\xcd" + + "\x03\x84\xe5\xc3\x02\x0d\x11\x03\x8b\xec\xcb\x03\x94\xec\xcf\x03\x9a\xec" + + "\xc2\x03\x8b\xec\xdb\x03\x94\xec\xdf\x03\x9a\xec\xd2\x03\x01\x0c!\x03" + + "\x01\x0c#\x03ʠ\x9d\x03ʣ\x9c\x03ʢ\x9f\x03ʥ\x9e\x03ʤ\x91\x03ʧ\x90\x03ʦ\x93" + + "\x03ʩ\x92\x03ʨ\x95\x03\xca\xf3\xb5\x03\xca\xf0\xb4\x03\xca\xf1\xb7\x03" + + "\xca\xf6\xb6\x03\xca\xf7\x89\x03\xca\xf4\x88\x03\xca\xf5\x8b\x03\xca\xfa" + + "\x8a\x03\xca\xfb\x8d\x03\xca\xf8\x8c\x03\xca\xf9\x8f\x03\xca\xfe\x8e\x03" + + "\xca\xff\x81\x03\xca\xfc\x80\x03\xca\xfd\x83\x03\xca\xe2\x82\x03\xca\xe3" + + "\x85\x03\xca\xe0\x84\x03\xca\xe1\x87\x03\xca\xe6\x86\x03\xca\xe7\x99\x03" + + "\xca\xe4\x98\x03\xca\xe5\x9b\x03\xca\xea\x9a\x03\xca\xeb\x9d\x03\xca\xe8" + + "\x9c\x03ؓ\x89\x03ߔ\x8b\x02\x010\x03\x03\x04\x1e\x03\x04\x15\x12\x03\x0b" + + "\x05,\x03\x06\x04\x00\x03\x06\x04)\x03\x06\x044\x03\x06\x04<\x03\x06\x05" + + "\x1d\x03\x06\x06\x00\x03\x06\x06\x0a\x03\x06\x06'\x03\x06\x062\x03\x0786" + + "\x03\x079/\x03\x079 \x03\x07:\x0e\x03\x07:\x1b\x03\x07:%\x03\x07;/\x03" + + "\x07;%\x03\x074\x11\x03\x076\x09\x03\x077*\x03\x070\x01\x03\x070\x0f\x03" + + "\x070.\x03\x071\x16\x03\x071\x04\x03\x0710\x03\x072\x18\x03\x072-\x03" + + "\x073\x14\x03\x073>\x03\x07'\x09\x03\x07 \x00\x03\x07\x1f\x0b\x03\x07" + + "\x18#\x03\x07\x18(\x03\x07\x186\x03\x07\x18\x03\x03\x07\x19\x16\x03\x07" + + "\x116\x03\x07\x12'\x03\x07\x13\x10\x03\x07\x0c&\x03\x07\x0c\x08\x03\x07" + + "\x0c\x13\x03\x07\x0d\x02\x03\x07\x0d\x1c\x03\x07\x0b5\x03\x07\x0b\x0a" + + "\x03\x07\x0b\x01\x03\x07\x0b\x0f\x03\x07\x05\x00\x03\x07\x05\x09\x03\x07" + + "\x05\x0b\x03\x07\x07\x01\x03\x07\x07\x08\x03\x07\x00<\x03\x07\x00+\x03" + + "\x07\x01)\x03\x07\x01\x1b\x03\x07\x01\x08\x03\x07\x03?\x03\x0445\x03\x04" + + "4\x08\x03\x0454\x03\x04)/\x03\x04)5\x03\x04+\x05\x03\x04+\x14\x03\x04+ " + + "\x03\x04+<\x03\x04*&\x03\x04*\x22\x03\x04&8\x03\x04!\x01\x03\x04!\x22" + + "\x03\x04\x11+\x03\x04\x10.\x03\x04\x104\x03\x04\x13=\x03\x04\x12\x04\x03" + + "\x04\x12\x0a\x03\x04\x0d\x1d\x03\x04\x0d\x07\x03\x04\x0d \x03\x05<>\x03" + + "\x055<\x03\x055!\x03\x055#\x03\x055&\x03\x054\x1d\x03\x054\x02\x03\x054" + + "\x07\x03\x0571\x03\x053\x1a\x03\x053\x16\x03\x05.<\x03\x05.\x07\x03\x05)" + + ":\x03\x05)<\x03\x05)\x0c\x03\x05)\x15\x03\x05+-\x03\x05+5\x03\x05$\x1e" + + "\x03\x05$\x14\x03\x05'\x04\x03\x05'\x14\x03\x05&\x02\x03\x05\x226\x03" + + "\x05\x22\x0c\x03\x05\x22\x1c\x03\x05\x19\x0a\x03\x05\x1b\x09\x03\x05\x1b" + + "\x0c\x03\x05\x14\x07\x03\x05\x16?\x03\x05\x16\x0c\x03\x05\x0c\x05\x03" + + "\x05\x0e\x0f\x03\x05\x01\x0e\x03\x05\x00(\x03\x05\x030\x03\x05\x03\x06" + + "\x03\x0a==\x03\x0a=1\x03\x0a=,\x03\x0a=\x0c\x03\x0a??\x03\x0a<\x08\x03" + + "\x0a9!\x03\x0a9)\x03\x0a97\x03\x0a99\x03\x0a6\x0a\x03\x0a6\x1c\x03\x0a6" + + "\x17\x03\x0a7'\x03\x0a78\x03\x0a73\x03\x0a'\x01\x03\x0a'&\x03\x0a\x1f" + + "\x0e\x03\x0a\x1f\x03\x03\x0a\x1f3\x03\x0a\x1b/\x03\x0a\x18\x19\x03\x0a" + + "\x19\x01\x03\x0a\x16\x14\x03\x0a\x0e\x22\x03\x0a\x0f\x10\x03\x0a\x0f\x02" + + "\x03\x0a\x0f \x03\x0a\x0c\x04\x03\x0a\x0b>\x03\x0a\x0b+\x03\x0a\x08/\x03" + + "\x0a\x046\x03\x0a\x05\x14\x03\x0a\x00\x04\x03\x0a\x00\x10\x03\x0a\x00" + + "\x14\x03\x0b<3\x03\x0b;*\x03\x0b9\x22\x03\x0b9)\x03\x0b97\x03\x0b+\x10" + + "\x03\x0b((\x03\x0b&5\x03\x0b$\x1c\x03\x0b$\x12\x03\x0b%\x04\x03\x0b#<" + + "\x03\x0b#0\x03\x0b#\x0d\x03\x0b#\x19\x03\x0b!:\x03\x0b!\x1f\x03\x0b!\x00" + + "\x03\x0b\x1e5\x03\x0b\x1c\x1d\x03\x0b\x1d-\x03\x0b\x1d(\x03\x0b\x18.\x03" + + "\x0b\x18 \x03\x0b\x18\x16\x03\x0b\x14\x13\x03\x0b\x15$\x03\x0b\x15\x22" + + "\x03\x0b\x12\x1b\x03\x0b\x12\x10\x03\x0b\x132\x03\x0b\x13=\x03\x0b\x12" + + "\x18\x03\x0b\x0c&\x03\x0b\x061\x03\x0b\x06:\x03\x0b\x05#\x03\x0b\x05<" + + "\x03\x0b\x04\x0b\x03\x0b\x04\x04\x03\x0b\x04\x1b\x03\x0b\x042\x03\x0b" + + "\x041\x03\x0b\x03\x03\x03\x0b\x03\x1d\x03\x0b\x03/\x03\x0b\x03+\x03\x0b" + + "\x02\x1b\x03\x0b\x02\x00\x03\x0b\x01\x1e\x03\x0b\x01\x08\x03\x0b\x015" + + "\x03\x06\x0d9\x03\x06\x0d=\x03\x06\x0d?\x03\x02\x001\x03\x02\x003\x03" + + "\x02\x02\x19\x03\x02\x006\x03\x02\x02\x1b\x03\x02\x004\x03\x02\x00<\x03" + + "\x02\x02\x0a\x03\x02\x02\x0e\x03\x02\x01\x1a\x03\x02\x01\x07\x03\x02\x01" + + "\x05\x03\x02\x01\x0b\x03\x02\x01%\x03\x02\x01\x0c\x03\x02\x01\x04\x03" + + "\x02\x01\x1c\x03\x02\x00.\x03\x02\x002\x03\x02\x00>\x03\x02\x00\x12\x03" + + "\x02\x00\x16\x03\x02\x011\x03\x02\x013\x03\x02\x02 \x03\x02\x02%\x03\x02" + + "\x02$\x03\x02\x028\x03\x02\x02;\x03\x02\x024\x03\x02\x012\x03\x02\x022" + + "\x03\x02\x02/\x03\x02\x01,\x03\x02\x01\x13\x03\x02\x01\x16\x03\x02\x01" + + "\x11\x03\x02\x01\x1e\x03\x02\x01\x15\x03\x02\x01\x17\x03\x02\x01\x0f\x03" + + "\x02\x01\x08\x03\x02\x00?\x03\x02\x03\x07\x03\x02\x03\x0d\x03\x02\x03" + + "\x13\x03\x02\x03\x1d\x03\x02\x03\x1f\x03\x02\x00\x03\x03\x02\x00\x0d\x03" + + "\x02\x00\x01\x03\x02\x00\x1b\x03\x02\x00\x19\x03\x02\x00\x18\x03\x02\x00" + + "\x13\x03\x02\x00/\x03\x07>\x12\x03\x07<\x1f\x03\x07>\x1d\x03\x06\x1d\x0e" + + "\x03\x07>\x1c\x03\x07>:\x03\x07>\x13\x03\x04\x12+\x03\x07?\x03\x03\x07>" + + "\x02\x03\x06\x224\x03\x06\x1a.\x03\x07<%\x03\x06\x1c\x0b\x03\x0609\x03" + + "\x05\x1f\x01\x03\x04'\x08\x03\x93\xfd\xf5\x03\x02\x0d \x03\x02\x0d#\x03" + + "\x02\x0d!\x03\x02\x0d&\x03\x02\x0d\x22\x03\x02\x0d/\x03\x02\x0d,\x03\x02" + + "\x0d$\x03\x02\x0d'\x03\x02\x0d%\x03\x02\x0d;\x03\x02\x0d=\x03\x02\x0d?" + + "\x03\x099.\x03\x08\x0b7\x03\x08\x02\x14\x03\x08\x14\x0d\x03\x08.:\x03" + + "\x089'\x03\x0f\x0b\x18\x03\x0f\x1c1\x03\x0f\x17&\x03\x0f9\x1f\x03\x0f0" + + "\x0c\x03\x0e\x0a9\x03\x0e\x056\x03\x0e\x1c#\x03\x0f\x13\x0e\x03\x072\x00" + + "\x03\x070\x0d\x03\x072\x0b\x03\x06\x11\x18\x03\x070\x10\x03\x06\x0f(\x03" + + "\x072\x05\x03\x06\x0f,\x03\x073\x15\x03\x06\x07\x08\x03\x05\x16\x02\x03" + + "\x04\x0b \x03\x05:8\x03\x05\x16%\x03\x0a\x0d\x1f\x03\x06\x16\x10\x03\x05" + + "\x1d5\x03\x05*;\x03\x05\x16\x1b\x03\x04.-\x03\x06\x1a\x19\x03\x04\x03," + + "\x03\x0b87\x03\x04/\x0a\x03\x06\x00,\x03\x04-\x01\x03\x04\x1e-\x03\x06/(" + + "\x03\x0a\x0b5\x03\x06\x0e7\x03\x06\x07.\x03\x0597\x03\x0a*%\x03\x0760" + + "\x03\x06\x0c;\x03\x05'\x00\x03\x072.\x03\x072\x08\x03\x06=\x01\x03\x06" + + "\x05\x1b\x03\x06\x06\x12\x03\x06$=\x03\x06'\x0d\x03\x04\x11\x0f\x03\x076" + + ",\x03\x06\x07;\x03\x06.,\x03\x86\xf9\xea\x03\x8f\xff\xeb\x02\x092\x02" + + "\x095\x02\x094\x02\x09;\x02\x09>\x02\x098\x02\x09*\x02\x09/\x02\x09,\x02" + + "\x09%\x02\x09&\x02\x09#\x02\x09 \x02\x08!\x02\x08%\x02\x08$\x02\x08+\x02" + + "\x08.\x02\x08*\x02\x08&\x02\x088\x02\x08>\x02\x084\x02\x086\x02\x080\x02" + + "\x08\x10\x02\x08\x17\x02\x08\x12\x02\x08\x1d\x02\x08\x1f\x02\x08\x13\x02" + + "\x08\x15\x02\x08\x14\x02\x08\x0c\x03\x8b\xfd\xd0\x03\x81\xec\xc6\x03\x87" + + "\xe0\x8a\x03-2\xe3\x03\x80\xef\xe4\x03-2\xea\x03\x88\xe6\xeb\x03\x8e\xe6" + + "\xe8\x03\x84\xe6\xe9\x03\x97\xe6\xee\x03-2\xf9\x03-2\xf6\x03\x8e\xe3\xad" + + "\x03\x80\xe3\x92\x03\x88\xe3\x90\x03\x8e\xe3\x90\x03\x80\xe3\x97\x03\x88" + + "\xe3\x95\x03\x88\xfe\xcb\x03\x8e\xfe\xca\x03\x84\xfe\xcd\x03\x91\xef\xc9" + + "\x03-2\xc1\x03-2\xc0\x03-2\xcb\x03\x88@\x09\x03\x8e@\x08\x03\x8f\xe0\xf5" + + "\x03\x8e\xe6\xf9\x03\x8e\xe0\xfa\x03\x93\xff\xf4\x03\x84\xee\xd3\x03\x0b" + + "(\x04\x023 \x021;\x02\x01*\x03\x0b#\x10\x03\x0b 0\x03\x0b!\x10\x03\x0b!0" + + "\x03\x07\x15\x08\x03\x09?5\x03\x07\x1f\x08\x03\x07\x17\x0b\x03\x09\x1f" + + "\x15\x03\x0b\x1c7\x03\x0a+#\x03\x06\x1a\x1b\x03\x06\x1a\x14\x03\x0a\x01" + + "\x18\x03\x06#\x1b\x03\x0a2\x0c\x03\x0a\x01\x04\x03\x09#;\x03\x08='\x03" + + "\x08\x1a\x0a\x03\x07\x03\x0a\x111\x03\x09\x1b\x09\x03\x073.\x03\x07\x01\x00" + + "\x03\x09/,\x03\x07#>\x03\x07\x048\x03\x0a\x1f\x22\x03\x098>\x03\x09\x11" + + "\x00\x03\x08/\x17\x03\x06'\x22\x03\x0b\x1a+\x03\x0a\x22\x19\x03\x0a/1" + + "\x03\x0974\x03\x09\x0f\x22\x03\x08,\x22\x03\x08?\x14\x03\x07$5\x03\x07<3" + + "\x03\x07=*\x03\x07\x13\x18\x03\x068\x0a\x03\x06\x09\x16\x03\x06\x13\x00" + + "\x03\x08\x067\x03\x08\x01\x03\x03\x08\x12\x1d\x03\x07+7\x03\x06(;\x03" + + "\x06\x1c?\x03\x07\x0e\x17\x03\x0a\x06\x1d\x03\x0a\x19\x07\x03\x08\x14$" + + "\x03\x07$;\x03\x08,$\x03\x08\x06\x0d\x03\x07\x16\x0a\x03\x06>>\x03\x0a" + + "\x06\x12\x03\x0a\x14)\x03\x09\x0d\x1f\x03\x09\x12\x17\x03\x09\x19\x01" + + "\x03\x08\x11 \x03\x08\x1d'\x03\x06<\x1a\x03\x0a.\x00\x03\x07'\x18\x03" + + "\x0a\x22\x08\x03\x08\x0d\x0a\x03\x08\x13)\x03\x07*)\x03\x06<,\x03\x07" + + "\x0b\x1a\x03\x09.\x14\x03\x09\x0d\x1e\x03\x07\x0e#\x03\x0b\x1d'\x03\x0a" + + "\x0a8\x03\x09%2\x03\x08+&\x03\x080\x12\x03\x0a)4\x03\x08\x06\x1f\x03\x0b" + + "\x1b\x1a\x03\x0a\x1b\x0f\x03\x0b\x1d*\x03\x09\x16$\x03\x090\x11\x03\x08" + + "\x11\x08\x03\x0a*(\x03\x0a\x042\x03\x089,\x03\x074'\x03\x07\x0f\x05\x03" + + "\x09\x0b\x0a\x03\x07\x1b\x01\x03\x09\x17:\x03\x09.\x0d\x03\x07.\x11\x03" + + "\x09+\x15\x03\x080\x13\x03\x0b\x1f\x19\x03\x0a \x11\x03\x0a\x220\x03\x09" + + "\x07;\x03\x08\x16\x1c\x03\x07,\x13\x03\x07\x0e/\x03\x06\x221\x03\x0a." + + "\x0a\x03\x0a7\x02\x03\x0a\x032\x03\x0a\x1d.\x03\x091\x06\x03\x09\x19:" + + "\x03\x08\x02/\x03\x060+\x03\x06\x0f-\x03\x06\x1c\x1f\x03\x06\x1d\x07\x03" + + "\x0a,\x11\x03\x09=\x0d\x03\x09\x0b;\x03\x07\x1b/\x03\x0a\x1f:\x03\x09 " + + "\x1f\x03\x09.\x10\x03\x094\x0b\x03\x09\x1a1\x03\x08#\x1a\x03\x084\x1d" + + "\x03\x08\x01\x1f\x03\x08\x11\x22\x03\x07'8\x03\x07\x1a>\x03\x0757\x03" + + "\x06&9\x03\x06+\x11\x03\x0a.\x0b\x03\x0a,>\x03\x0a4#\x03\x08%\x17\x03" + + "\x07\x05\x22\x03\x07\x0c\x0b\x03\x0a\x1d+\x03\x0a\x19\x16\x03\x09+\x1f" + + "\x03\x09\x08\x0b\x03\x08\x16\x18\x03\x08+\x12\x03\x0b\x1d\x0c\x03\x0a=" + + "\x10\x03\x0a\x09\x0d\x03\x0a\x10\x11\x03\x09&0\x03\x08(\x1f\x03\x087\x07" + + "\x03\x08\x185\x03\x07'6\x03\x06.\x05\x03\x06=\x04\x03\x06;;\x03\x06\x06," + + "\x03\x0b\x18>\x03\x08\x00\x18\x03\x06 \x03\x03\x06<\x00\x03\x09%\x18\x03" + + "\x0b\x1c<\x03\x0a%!\x03\x0a\x09\x12\x03\x0a\x16\x02\x03\x090'\x03\x09" + + "\x0e=\x03\x08 \x0e\x03\x08>\x03\x03\x074>\x03\x06&?\x03\x06\x19\x09\x03" + + "\x06?(\x03\x0a-\x0e\x03\x09:3\x03\x098:\x03\x09\x12\x0b\x03\x09\x1d\x17" + + "\x03\x087\x05\x03\x082\x14\x03\x08\x06%\x03\x08\x13\x1f\x03\x06\x06\x0e" + + "\x03\x0a\x22<\x03\x09/<\x03\x06>+\x03\x0a'?\x03\x0a\x13\x0c\x03\x09\x10<" + + "\x03\x07\x1b=\x03\x0a\x19\x13\x03\x09\x22\x1d\x03\x09\x07\x0d\x03\x08)" + + "\x1c\x03\x06=\x1a\x03\x0a/4\x03\x0a7\x11\x03\x0a\x16:\x03\x09?3\x03\x09:" + + "/\x03\x09\x05\x0a\x03\x09\x14\x06\x03\x087\x22\x03\x080\x07\x03\x08\x1a" + + "\x1f\x03\x07\x04(\x03\x07\x04\x09\x03\x06 %\x03\x06<\x08\x03\x0a+\x14" + + "\x03\x09\x1d\x16\x03\x0a70\x03\x08 >\x03\x0857\x03\x070\x0a\x03\x06=\x12" + + "\x03\x06\x16%\x03\x06\x1d,\x03\x099#\x03\x09\x10>\x03\x07 \x1e\x03\x08" + + "\x0c<\x03\x08\x0b\x18\x03\x08\x15+\x03\x08,:\x03\x08%\x22\x03\x07\x0a$" + + "\x03\x0b\x1c=\x03\x07+\x08\x03\x0a/\x05\x03\x0a \x07\x03\x0a\x12'\x03" + + "\x09#\x11\x03\x08\x1b\x15\x03\x0a\x06\x01\x03\x09\x1c\x1b\x03\x0922\x03" + + "\x07\x14<\x03\x07\x09\x04\x03\x061\x04\x03\x07\x0e\x01\x03\x0a\x13\x18" + + "\x03\x0a-\x0c\x03\x0a?\x0d\x03\x0a\x09\x0a\x03\x091&\x03\x0a/\x0b\x03" + + "\x08$<\x03\x083\x1d\x03\x08\x0c$\x03\x08\x0d\x07\x03\x08\x0d?\x03\x08" + + "\x0e\x14\x03\x065\x0a\x03\x08\x1a#\x03\x08\x16#\x03\x0702\x03\x07\x03" + + "\x1a\x03\x06(\x1d\x03\x06+\x1b\x03\x06\x0b\x05\x03\x06\x0b\x17\x03\x06" + + "\x0c\x04\x03\x06\x1e\x19\x03\x06+0\x03\x062\x18\x03\x0b\x16\x1e\x03\x0a+" + + "\x16\x03\x0a-?\x03\x0a#:\x03\x0a#\x10\x03\x0a%$\x03\x0a>+\x03\x0a01\x03" + + "\x0a1\x10\x03\x0a\x099\x03\x0a\x0a\x12\x03\x0a\x19\x1f\x03\x0a\x19\x12" + + "\x03\x09*)\x03\x09-\x16\x03\x09.1\x03\x09.2\x03\x09<\x0e\x03\x09> \x03" + + "\x093\x12\x03\x09\x0b\x01\x03\x09\x1c2\x03\x09\x11\x1c\x03\x09\x15%\x03" + + "\x08,&\x03\x08!\x22\x03\x089(\x03\x08\x0b\x1a\x03\x08\x0d2\x03\x08\x0c" + + "\x04\x03\x08\x0c\x06\x03\x08\x0c\x1f\x03\x08\x0c\x0c\x03\x08\x0f\x1f\x03" + + "\x08\x0f\x1d\x03\x08\x00\x14\x03\x08\x03\x14\x03\x08\x06\x16\x03\x08\x1e" + + "#\x03\x08\x11\x11\x03\x08\x10\x18\x03\x08\x14(\x03\x07)\x1e\x03\x07.1" + + "\x03\x07 $\x03\x07 '\x03\x078\x08\x03\x07\x0d0\x03\x07\x0f7\x03\x07\x05#" + + "\x03\x07\x05\x1a\x03\x07\x1a7\x03\x07\x1d-\x03\x07\x17\x10\x03\x06)\x1f" + + "\x03\x062\x0b\x03\x066\x16\x03\x06\x09\x11\x03\x09(\x1e\x03\x07!5\x03" + + "\x0b\x11\x16\x03\x0a/\x04\x03\x0a,\x1a\x03\x0b\x173\x03\x0a,1\x03\x0a/5" + + "\x03\x0a\x221\x03\x0a\x22\x0d\x03\x0a?%\x03\x0a<,\x03\x0a?#\x03\x0a>\x19" + + "\x03\x0a\x08&\x03\x0a\x0b\x0e\x03\x0a\x0c:\x03\x0a\x0c+\x03\x0a\x03\x22" + + "\x03\x0a\x06)\x03\x0a\x11\x10\x03\x0a\x11\x1a\x03\x0a\x17-\x03\x0a\x14(" + + "\x03\x09)\x1e\x03\x09/\x09\x03\x09.\x00\x03\x09,\x07\x03\x09/*\x03\x09-9" + + "\x03\x09\x228\x03\x09%\x09\x03\x09:\x12\x03\x09;\x1d\x03\x09?\x06\x03" + + "\x093%\x03\x096\x05\x03\x096\x08\x03\x097\x02\x03\x09\x07,\x03\x09\x04," + + "\x03\x09\x1f\x16\x03\x09\x11\x03\x03\x09\x11\x12\x03\x09\x168\x03\x08*" + + "\x05\x03\x08/2\x03\x084:\x03\x08\x22+\x03\x08 0\x03\x08&\x0a\x03\x08;" + + "\x10\x03\x08>$\x03\x08>\x18\x03\x0829\x03\x082:\x03\x081,\x03\x081<\x03" + + "\x081\x1c\x03\x087#\x03\x087*\x03\x08\x09'\x03\x08\x00\x1d\x03\x08\x05-" + + "\x03\x08\x1f4\x03\x08\x1d\x04\x03\x08\x16\x0f\x03\x07*7\x03\x07'!\x03" + + "\x07%\x1b\x03\x077\x0c\x03\x07\x0c1\x03\x07\x0c.\x03\x07\x00\x06\x03\x07" + + "\x01\x02\x03\x07\x010\x03\x07\x06=\x03\x07\x01\x03\x03\x07\x01\x13\x03" + + "\x07\x06\x06\x03\x07\x05\x0a\x03\x07\x1f\x09\x03\x07\x17:\x03\x06*1\x03" + + "\x06-\x1d\x03\x06\x223\x03\x062:\x03\x060$\x03\x066\x1e\x03\x064\x12\x03" + + "\x0645\x03\x06\x0b\x00\x03\x06\x0b7\x03\x06\x07\x1f\x03\x06\x15\x12\x03" + + "\x0c\x05\x0f\x03\x0b+\x0b\x03\x0b+-\x03\x06\x16\x1b\x03\x06\x15\x17\x03" + + "\x89\xca\xea\x03\x89\xca\xe8\x03\x0c8\x10\x03\x0c8\x01\x03\x0c8\x0f\x03" + + "\x0d8%\x03\x0d8!\x03\x0c8-\x03\x0c8/\x03\x0c8+\x03\x0c87\x03\x0c85\x03" + + "\x0c9\x09\x03\x0c9\x0d\x03\x0c9\x0f\x03\x0c9\x0b\x03\xcfu\x0c\x03\xcfu" + + "\x0f\x03\xcfu\x0e\x03\xcfu\x09\x03\x0c9\x10\x03\x0d9\x0c\x03\xcf`;\x03" + + "\xcf`>\x03\xcf`9\x03\xcf`8\x03\xcf`7\x03\xcf`*\x03\xcf`-\x03\xcf`,\x03" + + "\x0d\x1b\x1a\x03\x0d\x1b&\x03\x0c=.\x03\x0c=%\x03\x0c>\x1e\x03\x0c>\x14" + + "\x03\x0c?\x06\x03\x0c?\x0b\x03\x0c?\x0c\x03\x0c?\x0d\x03\x0c?\x02\x03" + + "\x0c>\x0f\x03\x0c>\x08\x03\x0c>\x09\x03\x0c>,\x03\x0c>\x0c\x03\x0c?\x13" + + "\x03\x0c?\x16\x03\x0c?\x15\x03\x0c?\x1c\x03\x0c?\x1f\x03\x0c?\x1d\x03" + + "\x0c?\x1a\x03\x0c?\x17\x03\x0c?\x08\x03\x0c?\x09\x03\x0c?\x0e\x03\x0c?" + + "\x04\x03\x0c?\x05\x03\x0c" + + "\x03\x0c=2\x03\x0c=6\x03\x0c<\x07\x03\x0c<\x05\x03\x0e:!\x03\x0e:#\x03" + + "\x0e8\x09\x03\x0e:&\x03\x0e8\x0b\x03\x0e:$\x03\x0e:,\x03\x0e8\x1a\x03" + + "\x0e8\x1e\x03\x0e:*\x03\x0e:7\x03\x0e:5\x03\x0e:;\x03\x0e:\x15\x03\x0e:<" + + "\x03\x0e:4\x03\x0e:'\x03\x0e:-\x03\x0e:%\x03\x0e:?\x03\x0e:=\x03\x0e:)" + + "\x03\x0e:/\x03\xcfs'\x03\x0d=\x0f\x03\x0d+*\x03\x0d99\x03\x0d9;\x03\x0d9" + + "?\x03\x0d)\x0d\x03\x0d(%\x02\x01\x18\x02\x01(\x02\x01\x1e\x03\x0f$!\x03" + + "\x0f87\x03\x0f4\x0e\x03\x0f5\x1d\x03\x06'\x03\x03\x0f\x08\x18\x03\x0f" + + "\x0d\x1b\x03\x0e2=\x03\x0e;\x08\x03\x0e:\x0b\x03\x0e\x06$\x03\x0e\x0d)" + + "\x03\x0e\x16\x1f\x03\x0e\x16\x1b\x03\x0d$\x0a\x03\x05,\x1d\x03\x0d. \x03" + + "\x0d.#\x03\x0c(/\x03\x09%\x02\x03\x0d90\x03\x0d\x0e4\x03\x0d\x0d\x0f\x03" + + "\x0c#\x00\x03\x0c,\x1e\x03\x0c2\x0e\x03\x0c\x01\x17\x03\x0c\x09:\x03\x0e" + + "\x173\x03\x0c\x08\x03\x03\x0c\x11\x07\x03\x0c\x10\x18\x03\x0c\x1f\x1c" + + "\x03\x0c\x19\x0e\x03\x0c\x1a\x1f\x03\x0f0>\x03\x0b->\x03\x0b<+\x03\x0b8" + + "\x13\x03\x0b\x043\x03\x0b\x14\x03\x03\x0b\x16%\x03\x0d\x22&\x03\x0b\x1a" + + "\x1a\x03\x0b\x1a\x04\x03\x0a%9\x03\x0a&2\x03\x0a&0\x03\x0a!\x1a\x03\x0a!" + + "7\x03\x0a5\x10\x03\x0a=4\x03\x0a?\x0e\x03\x0a>\x10\x03\x0a\x00 \x03\x0a" + + "\x0f:\x03\x0a\x0f9\x03\x0a\x0b\x0a\x03\x0a\x17%\x03\x0a\x1b-\x03\x09-" + + "\x1a\x03\x09,4\x03\x09.,\x03\x09)\x09\x03\x096!\x03\x091\x1f\x03\x093" + + "\x16\x03\x0c+\x1f\x03\x098 \x03\x098=\x03\x0c(\x1a\x03\x0c(\x16\x03\x09" + + "\x0a+\x03\x09\x16\x12\x03\x09\x13\x0e\x03\x09\x153\x03\x08)!\x03\x09\x1a" + + "\x01\x03\x09\x18\x01\x03\x08%#\x03\x08>\x22\x03\x08\x05%\x03\x08\x02*" + + "\x03\x08\x15;\x03\x08\x1b7\x03\x0f\x07\x1d\x03\x0f\x04\x03\x03\x070\x0c" + + "\x03\x07;\x0b\x03\x07\x08\x17\x03\x07\x12\x06\x03\x06/-\x03\x0671\x03" + + "\x065+\x03\x06>7\x03\x06\x049\x03\x05+\x1e\x03\x05,\x17\x03\x05 \x1d\x03" + + "\x05\x22\x05\x03\x050\x1d" + +// lookup returns the trie value for the first UTF-8 encoding in s and +// the width in bytes of this encoding. The size will be 0 if s does not +// hold enough bytes to complete the encoding. len(s) must be greater than 0. +func (t *idnaTrie) lookup(s []byte) (v uint16, sz int) { + c0 := s[0] + switch { + case c0 < 0x80: // is ASCII + return idnaValues[c0], 1 + case c0 < 0xC2: + return 0, 1 // Illegal UTF-8: not a starter, not ASCII. + case c0 < 0xE0: // 2-byte UTF-8 + if len(s) < 2 { + return 0, 0 + } + i := idnaIndex[c0] + c1 := s[1] + if c1 < 0x80 || 0xC0 <= c1 { + return 0, 1 // Illegal UTF-8: not a continuation byte. + } + return t.lookupValue(uint32(i), c1), 2 + case c0 < 0xF0: // 3-byte UTF-8 + if len(s) < 3 { + return 0, 0 + } + i := idnaIndex[c0] + c1 := s[1] + if c1 < 0x80 || 0xC0 <= c1 { + return 0, 1 // Illegal UTF-8: not a continuation byte. + } + o := uint32(i)<<6 + uint32(c1) + i = idnaIndex[o] + c2 := s[2] + if c2 < 0x80 || 0xC0 <= c2 { + return 0, 2 // Illegal UTF-8: not a continuation byte. + } + return t.lookupValue(uint32(i), c2), 3 + case c0 < 0xF8: // 4-byte UTF-8 + if len(s) < 4 { + return 0, 0 + } + i := idnaIndex[c0] + c1 := s[1] + if c1 < 0x80 || 0xC0 <= c1 { + return 0, 1 // Illegal UTF-8: not a continuation byte. + } + o := uint32(i)<<6 + uint32(c1) + i = idnaIndex[o] + c2 := s[2] + if c2 < 0x80 || 0xC0 <= c2 { + return 0, 2 // Illegal UTF-8: not a continuation byte. + } + o = uint32(i)<<6 + uint32(c2) + i = idnaIndex[o] + c3 := s[3] + if c3 < 0x80 || 0xC0 <= c3 { + return 0, 3 // Illegal UTF-8: not a continuation byte. + } + return t.lookupValue(uint32(i), c3), 4 + } + // Illegal rune + return 0, 1 +} + +// lookupUnsafe returns the trie value for the first UTF-8 encoding in s. +// s must start with a full and valid UTF-8 encoded rune. +func (t *idnaTrie) lookupUnsafe(s []byte) uint16 { + c0 := s[0] + if c0 < 0x80 { // is ASCII + return idnaValues[c0] + } + i := idnaIndex[c0] + if c0 < 0xE0 { // 2-byte UTF-8 + return t.lookupValue(uint32(i), s[1]) + } + i = idnaIndex[uint32(i)<<6+uint32(s[1])] + if c0 < 0xF0 { // 3-byte UTF-8 + return t.lookupValue(uint32(i), s[2]) + } + i = idnaIndex[uint32(i)<<6+uint32(s[2])] + if c0 < 0xF8 { // 4-byte UTF-8 + return t.lookupValue(uint32(i), s[3]) + } + return 0 +} + +// lookupString returns the trie value for the first UTF-8 encoding in s and +// the width in bytes of this encoding. The size will be 0 if s does not +// hold enough bytes to complete the encoding. len(s) must be greater than 0. +func (t *idnaTrie) lookupString(s string) (v uint16, sz int) { + c0 := s[0] + switch { + case c0 < 0x80: // is ASCII + return idnaValues[c0], 1 + case c0 < 0xC2: + return 0, 1 // Illegal UTF-8: not a starter, not ASCII. + case c0 < 0xE0: // 2-byte UTF-8 + if len(s) < 2 { + return 0, 0 + } + i := idnaIndex[c0] + c1 := s[1] + if c1 < 0x80 || 0xC0 <= c1 { + return 0, 1 // Illegal UTF-8: not a continuation byte. + } + return t.lookupValue(uint32(i), c1), 2 + case c0 < 0xF0: // 3-byte UTF-8 + if len(s) < 3 { + return 0, 0 + } + i := idnaIndex[c0] + c1 := s[1] + if c1 < 0x80 || 0xC0 <= c1 { + return 0, 1 // Illegal UTF-8: not a continuation byte. + } + o := uint32(i)<<6 + uint32(c1) + i = idnaIndex[o] + c2 := s[2] + if c2 < 0x80 || 0xC0 <= c2 { + return 0, 2 // Illegal UTF-8: not a continuation byte. + } + return t.lookupValue(uint32(i), c2), 3 + case c0 < 0xF8: // 4-byte UTF-8 + if len(s) < 4 { + return 0, 0 + } + i := idnaIndex[c0] + c1 := s[1] + if c1 < 0x80 || 0xC0 <= c1 { + return 0, 1 // Illegal UTF-8: not a continuation byte. + } + o := uint32(i)<<6 + uint32(c1) + i = idnaIndex[o] + c2 := s[2] + if c2 < 0x80 || 0xC0 <= c2 { + return 0, 2 // Illegal UTF-8: not a continuation byte. + } + o = uint32(i)<<6 + uint32(c2) + i = idnaIndex[o] + c3 := s[3] + if c3 < 0x80 || 0xC0 <= c3 { + return 0, 3 // Illegal UTF-8: not a continuation byte. + } + return t.lookupValue(uint32(i), c3), 4 + } + // Illegal rune + return 0, 1 +} + +// lookupStringUnsafe returns the trie value for the first UTF-8 encoding in s. +// s must start with a full and valid UTF-8 encoded rune. +func (t *idnaTrie) lookupStringUnsafe(s string) uint16 { + c0 := s[0] + if c0 < 0x80 { // is ASCII + return idnaValues[c0] + } + i := idnaIndex[c0] + if c0 < 0xE0 { // 2-byte UTF-8 + return t.lookupValue(uint32(i), s[1]) + } + i = idnaIndex[uint32(i)<<6+uint32(s[1])] + if c0 < 0xF0 { // 3-byte UTF-8 + return t.lookupValue(uint32(i), s[2]) + } + i = idnaIndex[uint32(i)<<6+uint32(s[2])] + if c0 < 0xF8 { // 4-byte UTF-8 + return t.lookupValue(uint32(i), s[3]) + } + return 0 +} + +// idnaTrie. Total size: 28496 bytes (27.83 KiB). Checksum: 43288b883596640e. +type idnaTrie struct{} + +func newIdnaTrie(i int) *idnaTrie { + return &idnaTrie{} +} + +// lookupValue determines the type of block n and looks up the value for b. +func (t *idnaTrie) lookupValue(n uint32, b byte) uint16 { + switch { + case n < 123: + return uint16(idnaValues[n<<6+uint32(b)]) + default: + n -= 123 + return uint16(idnaSparse.lookup(n, b)) + } +} + +// idnaValues: 125 blocks, 8000 entries, 16000 bytes +// The third block is the zero block. +var idnaValues = [8000]uint16{ + // Block 0x0, offset 0x0 + 0x00: 0x0080, 0x01: 0x0080, 0x02: 0x0080, 0x03: 0x0080, 0x04: 0x0080, 0x05: 0x0080, + 0x06: 0x0080, 0x07: 0x0080, 0x08: 0x0080, 0x09: 0x0080, 0x0a: 0x0080, 0x0b: 0x0080, + 0x0c: 0x0080, 0x0d: 0x0080, 0x0e: 0x0080, 0x0f: 0x0080, 0x10: 0x0080, 0x11: 0x0080, + 0x12: 0x0080, 0x13: 0x0080, 0x14: 0x0080, 0x15: 0x0080, 0x16: 0x0080, 0x17: 0x0080, + 0x18: 0x0080, 0x19: 0x0080, 0x1a: 0x0080, 0x1b: 0x0080, 0x1c: 0x0080, 0x1d: 0x0080, + 0x1e: 0x0080, 0x1f: 0x0080, 0x20: 0x0080, 0x21: 0x0080, 0x22: 0x0080, 0x23: 0x0080, + 0x24: 0x0080, 0x25: 0x0080, 0x26: 0x0080, 0x27: 0x0080, 0x28: 0x0080, 0x29: 0x0080, + 0x2a: 0x0080, 0x2b: 0x0080, 0x2c: 0x0080, 0x2d: 0x0008, 0x2e: 0x0008, 0x2f: 0x0080, + 0x30: 0x0008, 0x31: 0x0008, 0x32: 0x0008, 0x33: 0x0008, 0x34: 0x0008, 0x35: 0x0008, + 0x36: 0x0008, 0x37: 0x0008, 0x38: 0x0008, 0x39: 0x0008, 0x3a: 0x0080, 0x3b: 0x0080, + 0x3c: 0x0080, 0x3d: 0x0080, 0x3e: 0x0080, 0x3f: 0x0080, + // Block 0x1, offset 0x40 + 0x40: 0x0080, 0x41: 0xe105, 0x42: 0xe105, 0x43: 0xe105, 0x44: 0xe105, 0x45: 0xe105, + 0x46: 0xe105, 0x47: 0xe105, 0x48: 0xe105, 0x49: 0xe105, 0x4a: 0xe105, 0x4b: 0xe105, + 0x4c: 0xe105, 0x4d: 0xe105, 0x4e: 0xe105, 0x4f: 0xe105, 0x50: 0xe105, 0x51: 0xe105, + 0x52: 0xe105, 0x53: 0xe105, 0x54: 0xe105, 0x55: 0xe105, 0x56: 0xe105, 0x57: 0xe105, + 0x58: 0xe105, 0x59: 0xe105, 0x5a: 0xe105, 0x5b: 0x0080, 0x5c: 0x0080, 0x5d: 0x0080, + 0x5e: 0x0080, 0x5f: 0x0080, 0x60: 0x0080, 0x61: 0x0008, 0x62: 0x0008, 0x63: 0x0008, + 0x64: 0x0008, 0x65: 0x0008, 0x66: 0x0008, 0x67: 0x0008, 0x68: 0x0008, 0x69: 0x0008, + 0x6a: 0x0008, 0x6b: 0x0008, 0x6c: 0x0008, 0x6d: 0x0008, 0x6e: 0x0008, 0x6f: 0x0008, + 0x70: 0x0008, 0x71: 0x0008, 0x72: 0x0008, 0x73: 0x0008, 0x74: 0x0008, 0x75: 0x0008, + 0x76: 0x0008, 0x77: 0x0008, 0x78: 0x0008, 0x79: 0x0008, 0x7a: 0x0008, 0x7b: 0x0080, + 0x7c: 0x0080, 0x7d: 0x0080, 0x7e: 0x0080, 0x7f: 0x0080, + // Block 0x2, offset 0x80 + // Block 0x3, offset 0xc0 + 0xc0: 0x0040, 0xc1: 0x0040, 0xc2: 0x0040, 0xc3: 0x0040, 0xc4: 0x0040, 0xc5: 0x0040, + 0xc6: 0x0040, 0xc7: 0x0040, 0xc8: 0x0040, 0xc9: 0x0040, 0xca: 0x0040, 0xcb: 0x0040, + 0xcc: 0x0040, 0xcd: 0x0040, 0xce: 0x0040, 0xcf: 0x0040, 0xd0: 0x0040, 0xd1: 0x0040, + 0xd2: 0x0040, 0xd3: 0x0040, 0xd4: 0x0040, 0xd5: 0x0040, 0xd6: 0x0040, 0xd7: 0x0040, + 0xd8: 0x0040, 0xd9: 0x0040, 0xda: 0x0040, 0xdb: 0x0040, 0xdc: 0x0040, 0xdd: 0x0040, + 0xde: 0x0040, 0xdf: 0x0040, 0xe0: 0x000a, 0xe1: 0x0018, 0xe2: 0x0018, 0xe3: 0x0018, + 0xe4: 0x0018, 0xe5: 0x0018, 0xe6: 0x0018, 0xe7: 0x0018, 0xe8: 0x001a, 0xe9: 0x0018, + 0xea: 0x0039, 0xeb: 0x0018, 0xec: 0x0018, 0xed: 0x03c0, 0xee: 0x0018, 0xef: 0x004a, + 0xf0: 0x0018, 0xf1: 0x0018, 0xf2: 0x0069, 0xf3: 0x0079, 0xf4: 0x008a, 0xf5: 0x0005, + 0xf6: 0x0018, 0xf7: 0x0008, 0xf8: 0x00aa, 0xf9: 0x00c9, 0xfa: 0x00d9, 0xfb: 0x0018, + 0xfc: 0x00e9, 0xfd: 0x0119, 0xfe: 0x0149, 0xff: 0x0018, + // Block 0x4, offset 0x100 + 0x100: 0xe00d, 0x101: 0x0008, 0x102: 0xe00d, 0x103: 0x0008, 0x104: 0xe00d, 0x105: 0x0008, + 0x106: 0xe00d, 0x107: 0x0008, 0x108: 0xe00d, 0x109: 0x0008, 0x10a: 0xe00d, 0x10b: 0x0008, + 0x10c: 0xe00d, 0x10d: 0x0008, 0x10e: 0xe00d, 0x10f: 0x0008, 0x110: 0xe00d, 0x111: 0x0008, + 0x112: 0xe00d, 0x113: 0x0008, 0x114: 0xe00d, 0x115: 0x0008, 0x116: 0xe00d, 0x117: 0x0008, + 0x118: 0xe00d, 0x119: 0x0008, 0x11a: 0xe00d, 0x11b: 0x0008, 0x11c: 0xe00d, 0x11d: 0x0008, + 0x11e: 0xe00d, 0x11f: 0x0008, 0x120: 0xe00d, 0x121: 0x0008, 0x122: 0xe00d, 0x123: 0x0008, + 0x124: 0xe00d, 0x125: 0x0008, 0x126: 0xe00d, 0x127: 0x0008, 0x128: 0xe00d, 0x129: 0x0008, + 0x12a: 0xe00d, 0x12b: 0x0008, 0x12c: 0xe00d, 0x12d: 0x0008, 0x12e: 0xe00d, 0x12f: 0x0008, + 0x130: 0x0179, 0x131: 0x0008, 0x132: 0x0035, 0x133: 0x004d, 0x134: 0xe00d, 0x135: 0x0008, + 0x136: 0xe00d, 0x137: 0x0008, 0x138: 0x0008, 0x139: 0xe01d, 0x13a: 0x0008, 0x13b: 0xe03d, + 0x13c: 0x0008, 0x13d: 0xe01d, 0x13e: 0x0008, 0x13f: 0x0199, + // Block 0x5, offset 0x140 + 0x140: 0x0199, 0x141: 0xe01d, 0x142: 0x0008, 0x143: 0xe03d, 0x144: 0x0008, 0x145: 0xe01d, + 0x146: 0x0008, 0x147: 0xe07d, 0x148: 0x0008, 0x149: 0x01b9, 0x14a: 0xe00d, 0x14b: 0x0008, + 0x14c: 0xe00d, 0x14d: 0x0008, 0x14e: 0xe00d, 0x14f: 0x0008, 0x150: 0xe00d, 0x151: 0x0008, + 0x152: 0xe00d, 0x153: 0x0008, 0x154: 0xe00d, 0x155: 0x0008, 0x156: 0xe00d, 0x157: 0x0008, + 0x158: 0xe00d, 0x159: 0x0008, 0x15a: 0xe00d, 0x15b: 0x0008, 0x15c: 0xe00d, 0x15d: 0x0008, + 0x15e: 0xe00d, 0x15f: 0x0008, 0x160: 0xe00d, 0x161: 0x0008, 0x162: 0xe00d, 0x163: 0x0008, + 0x164: 0xe00d, 0x165: 0x0008, 0x166: 0xe00d, 0x167: 0x0008, 0x168: 0xe00d, 0x169: 0x0008, + 0x16a: 0xe00d, 0x16b: 0x0008, 0x16c: 0xe00d, 0x16d: 0x0008, 0x16e: 0xe00d, 0x16f: 0x0008, + 0x170: 0xe00d, 0x171: 0x0008, 0x172: 0xe00d, 0x173: 0x0008, 0x174: 0xe00d, 0x175: 0x0008, + 0x176: 0xe00d, 0x177: 0x0008, 0x178: 0x0065, 0x179: 0xe01d, 0x17a: 0x0008, 0x17b: 0xe03d, + 0x17c: 0x0008, 0x17d: 0xe01d, 0x17e: 0x0008, 0x17f: 0x01d9, + // Block 0x6, offset 0x180 + 0x180: 0x0008, 0x181: 0x007d, 0x182: 0xe00d, 0x183: 0x0008, 0x184: 0xe00d, 0x185: 0x0008, + 0x186: 0x007d, 0x187: 0xe07d, 0x188: 0x0008, 0x189: 0x0095, 0x18a: 0x00ad, 0x18b: 0xe03d, + 0x18c: 0x0008, 0x18d: 0x0008, 0x18e: 0x00c5, 0x18f: 0x00dd, 0x190: 0x00f5, 0x191: 0xe01d, + 0x192: 0x0008, 0x193: 0x010d, 0x194: 0x0125, 0x195: 0x0008, 0x196: 0x013d, 0x197: 0x013d, + 0x198: 0xe00d, 0x199: 0x0008, 0x19a: 0x0008, 0x19b: 0x0008, 0x19c: 0x010d, 0x19d: 0x0155, + 0x19e: 0x0008, 0x19f: 0x016d, 0x1a0: 0xe00d, 0x1a1: 0x0008, 0x1a2: 0xe00d, 0x1a3: 0x0008, + 0x1a4: 0xe00d, 0x1a5: 0x0008, 0x1a6: 0x0185, 0x1a7: 0xe07d, 0x1a8: 0x0008, 0x1a9: 0x019d, + 0x1aa: 0x0008, 0x1ab: 0x0008, 0x1ac: 0xe00d, 0x1ad: 0x0008, 0x1ae: 0x0185, 0x1af: 0xe0fd, + 0x1b0: 0x0008, 0x1b1: 0x01b5, 0x1b2: 0x01cd, 0x1b3: 0xe03d, 0x1b4: 0x0008, 0x1b5: 0xe01d, + 0x1b6: 0x0008, 0x1b7: 0x01e5, 0x1b8: 0xe00d, 0x1b9: 0x0008, 0x1ba: 0x0008, 0x1bb: 0x0008, + 0x1bc: 0xe00d, 0x1bd: 0x0008, 0x1be: 0x0008, 0x1bf: 0x0008, + // Block 0x7, offset 0x1c0 + 0x1c0: 0x0008, 0x1c1: 0x0008, 0x1c2: 0x0008, 0x1c3: 0x0008, 0x1c4: 0x01e9, 0x1c5: 0x01e9, + 0x1c6: 0x01e9, 0x1c7: 0x01fd, 0x1c8: 0x0215, 0x1c9: 0x022d, 0x1ca: 0x0245, 0x1cb: 0x025d, + 0x1cc: 0x0275, 0x1cd: 0xe01d, 0x1ce: 0x0008, 0x1cf: 0xe0fd, 0x1d0: 0x0008, 0x1d1: 0xe01d, + 0x1d2: 0x0008, 0x1d3: 0xe03d, 0x1d4: 0x0008, 0x1d5: 0xe01d, 0x1d6: 0x0008, 0x1d7: 0xe07d, + 0x1d8: 0x0008, 0x1d9: 0xe01d, 0x1da: 0x0008, 0x1db: 0xe03d, 0x1dc: 0x0008, 0x1dd: 0x0008, + 0x1de: 0xe00d, 0x1df: 0x0008, 0x1e0: 0xe00d, 0x1e1: 0x0008, 0x1e2: 0xe00d, 0x1e3: 0x0008, + 0x1e4: 0xe00d, 0x1e5: 0x0008, 0x1e6: 0xe00d, 0x1e7: 0x0008, 0x1e8: 0xe00d, 0x1e9: 0x0008, + 0x1ea: 0xe00d, 0x1eb: 0x0008, 0x1ec: 0xe00d, 0x1ed: 0x0008, 0x1ee: 0xe00d, 0x1ef: 0x0008, + 0x1f0: 0x0008, 0x1f1: 0x028d, 0x1f2: 0x02a5, 0x1f3: 0x02bd, 0x1f4: 0xe00d, 0x1f5: 0x0008, + 0x1f6: 0x02d5, 0x1f7: 0x02ed, 0x1f8: 0xe00d, 0x1f9: 0x0008, 0x1fa: 0xe00d, 0x1fb: 0x0008, + 0x1fc: 0xe00d, 0x1fd: 0x0008, 0x1fe: 0xe00d, 0x1ff: 0x0008, + // Block 0x8, offset 0x200 + 0x200: 0xe00d, 0x201: 0x0008, 0x202: 0xe00d, 0x203: 0x0008, 0x204: 0xe00d, 0x205: 0x0008, + 0x206: 0xe00d, 0x207: 0x0008, 0x208: 0xe00d, 0x209: 0x0008, 0x20a: 0xe00d, 0x20b: 0x0008, + 0x20c: 0xe00d, 0x20d: 0x0008, 0x20e: 0xe00d, 0x20f: 0x0008, 0x210: 0xe00d, 0x211: 0x0008, + 0x212: 0xe00d, 0x213: 0x0008, 0x214: 0xe00d, 0x215: 0x0008, 0x216: 0xe00d, 0x217: 0x0008, + 0x218: 0xe00d, 0x219: 0x0008, 0x21a: 0xe00d, 0x21b: 0x0008, 0x21c: 0xe00d, 0x21d: 0x0008, + 0x21e: 0xe00d, 0x21f: 0x0008, 0x220: 0x0305, 0x221: 0x0008, 0x222: 0xe00d, 0x223: 0x0008, + 0x224: 0xe00d, 0x225: 0x0008, 0x226: 0xe00d, 0x227: 0x0008, 0x228: 0xe00d, 0x229: 0x0008, + 0x22a: 0xe00d, 0x22b: 0x0008, 0x22c: 0xe00d, 0x22d: 0x0008, 0x22e: 0xe00d, 0x22f: 0x0008, + 0x230: 0xe00d, 0x231: 0x0008, 0x232: 0xe00d, 0x233: 0x0008, 0x234: 0x0008, 0x235: 0x0008, + 0x236: 0x0008, 0x237: 0x0008, 0x238: 0x0008, 0x239: 0x0008, 0x23a: 0x0209, 0x23b: 0xe03d, + 0x23c: 0x0008, 0x23d: 0x031d, 0x23e: 0x0229, 0x23f: 0x0008, + // Block 0x9, offset 0x240 + 0x240: 0x0008, 0x241: 0x0008, 0x242: 0x0018, 0x243: 0x0018, 0x244: 0x0018, 0x245: 0x0018, + 0x246: 0x0008, 0x247: 0x0008, 0x248: 0x0008, 0x249: 0x0008, 0x24a: 0x0008, 0x24b: 0x0008, + 0x24c: 0x0008, 0x24d: 0x0008, 0x24e: 0x0008, 0x24f: 0x0008, 0x250: 0x0008, 0x251: 0x0008, + 0x252: 0x0018, 0x253: 0x0018, 0x254: 0x0018, 0x255: 0x0018, 0x256: 0x0018, 0x257: 0x0018, + 0x258: 0x029a, 0x259: 0x02ba, 0x25a: 0x02da, 0x25b: 0x02fa, 0x25c: 0x031a, 0x25d: 0x033a, + 0x25e: 0x0018, 0x25f: 0x0018, 0x260: 0x03ad, 0x261: 0x0359, 0x262: 0x01d9, 0x263: 0x0369, + 0x264: 0x03c5, 0x265: 0x0018, 0x266: 0x0018, 0x267: 0x0018, 0x268: 0x0018, 0x269: 0x0018, + 0x26a: 0x0018, 0x26b: 0x0018, 0x26c: 0x0008, 0x26d: 0x0018, 0x26e: 0x0008, 0x26f: 0x0018, + 0x270: 0x0018, 0x271: 0x0018, 0x272: 0x0018, 0x273: 0x0018, 0x274: 0x0018, 0x275: 0x0018, + 0x276: 0x0018, 0x277: 0x0018, 0x278: 0x0018, 0x279: 0x0018, 0x27a: 0x0018, 0x27b: 0x0018, + 0x27c: 0x0018, 0x27d: 0x0018, 0x27e: 0x0018, 0x27f: 0x0018, + // Block 0xa, offset 0x280 + 0x280: 0x03dd, 0x281: 0x03dd, 0x282: 0x1308, 0x283: 0x03f5, 0x284: 0x0379, 0x285: 0x040d, + 0x286: 0x1308, 0x287: 0x1308, 0x288: 0x1308, 0x289: 0x1308, 0x28a: 0x1308, 0x28b: 0x1308, + 0x28c: 0x1308, 0x28d: 0x1308, 0x28e: 0x1308, 0x28f: 0x13c0, 0x290: 0x1308, 0x291: 0x1308, + 0x292: 0x1308, 0x293: 0x1308, 0x294: 0x1308, 0x295: 0x1308, 0x296: 0x1308, 0x297: 0x1308, + 0x298: 0x1308, 0x299: 0x1308, 0x29a: 0x1308, 0x29b: 0x1308, 0x29c: 0x1308, 0x29d: 0x1308, + 0x29e: 0x1308, 0x29f: 0x1308, 0x2a0: 0x1308, 0x2a1: 0x1308, 0x2a2: 0x1308, 0x2a3: 0x1308, + 0x2a4: 0x1308, 0x2a5: 0x1308, 0x2a6: 0x1308, 0x2a7: 0x1308, 0x2a8: 0x1308, 0x2a9: 0x1308, + 0x2aa: 0x1308, 0x2ab: 0x1308, 0x2ac: 0x1308, 0x2ad: 0x1308, 0x2ae: 0x1308, 0x2af: 0x1308, + 0x2b0: 0xe00d, 0x2b1: 0x0008, 0x2b2: 0xe00d, 0x2b3: 0x0008, 0x2b4: 0x0425, 0x2b5: 0x0008, + 0x2b6: 0xe00d, 0x2b7: 0x0008, 0x2b8: 0x0040, 0x2b9: 0x0040, 0x2ba: 0x03a2, 0x2bb: 0x0008, + 0x2bc: 0x0008, 0x2bd: 0x0008, 0x2be: 0x03c2, 0x2bf: 0x043d, + // Block 0xb, offset 0x2c0 + 0x2c0: 0x0040, 0x2c1: 0x0040, 0x2c2: 0x0040, 0x2c3: 0x0040, 0x2c4: 0x008a, 0x2c5: 0x03d2, + 0x2c6: 0xe155, 0x2c7: 0x0455, 0x2c8: 0xe12d, 0x2c9: 0xe13d, 0x2ca: 0xe12d, 0x2cb: 0x0040, + 0x2cc: 0x03dd, 0x2cd: 0x0040, 0x2ce: 0x046d, 0x2cf: 0x0485, 0x2d0: 0x0008, 0x2d1: 0xe105, + 0x2d2: 0xe105, 0x2d3: 0xe105, 0x2d4: 0xe105, 0x2d5: 0xe105, 0x2d6: 0xe105, 0x2d7: 0xe105, + 0x2d8: 0xe105, 0x2d9: 0xe105, 0x2da: 0xe105, 0x2db: 0xe105, 0x2dc: 0xe105, 0x2dd: 0xe105, + 0x2de: 0xe105, 0x2df: 0xe105, 0x2e0: 0x049d, 0x2e1: 0x049d, 0x2e2: 0x0040, 0x2e3: 0x049d, + 0x2e4: 0x049d, 0x2e5: 0x049d, 0x2e6: 0x049d, 0x2e7: 0x049d, 0x2e8: 0x049d, 0x2e9: 0x049d, + 0x2ea: 0x049d, 0x2eb: 0x049d, 0x2ec: 0x0008, 0x2ed: 0x0008, 0x2ee: 0x0008, 0x2ef: 0x0008, + 0x2f0: 0x0008, 0x2f1: 0x0008, 0x2f2: 0x0008, 0x2f3: 0x0008, 0x2f4: 0x0008, 0x2f5: 0x0008, + 0x2f6: 0x0008, 0x2f7: 0x0008, 0x2f8: 0x0008, 0x2f9: 0x0008, 0x2fa: 0x0008, 0x2fb: 0x0008, + 0x2fc: 0x0008, 0x2fd: 0x0008, 0x2fe: 0x0008, 0x2ff: 0x0008, + // Block 0xc, offset 0x300 + 0x300: 0x0008, 0x301: 0x0008, 0x302: 0xe00f, 0x303: 0x0008, 0x304: 0x0008, 0x305: 0x0008, + 0x306: 0x0008, 0x307: 0x0008, 0x308: 0x0008, 0x309: 0x0008, 0x30a: 0x0008, 0x30b: 0x0008, + 0x30c: 0x0008, 0x30d: 0x0008, 0x30e: 0x0008, 0x30f: 0xe0c5, 0x310: 0x04b5, 0x311: 0x04cd, + 0x312: 0xe0bd, 0x313: 0xe0f5, 0x314: 0xe0fd, 0x315: 0xe09d, 0x316: 0xe0b5, 0x317: 0x0008, + 0x318: 0xe00d, 0x319: 0x0008, 0x31a: 0xe00d, 0x31b: 0x0008, 0x31c: 0xe00d, 0x31d: 0x0008, + 0x31e: 0xe00d, 0x31f: 0x0008, 0x320: 0xe00d, 0x321: 0x0008, 0x322: 0xe00d, 0x323: 0x0008, + 0x324: 0xe00d, 0x325: 0x0008, 0x326: 0xe00d, 0x327: 0x0008, 0x328: 0xe00d, 0x329: 0x0008, + 0x32a: 0xe00d, 0x32b: 0x0008, 0x32c: 0xe00d, 0x32d: 0x0008, 0x32e: 0xe00d, 0x32f: 0x0008, + 0x330: 0x04e5, 0x331: 0xe185, 0x332: 0xe18d, 0x333: 0x0008, 0x334: 0x04fd, 0x335: 0x03dd, + 0x336: 0x0018, 0x337: 0xe07d, 0x338: 0x0008, 0x339: 0xe1d5, 0x33a: 0xe00d, 0x33b: 0x0008, + 0x33c: 0x0008, 0x33d: 0x0515, 0x33e: 0x052d, 0x33f: 0x052d, + // Block 0xd, offset 0x340 + 0x340: 0x0008, 0x341: 0x0008, 0x342: 0x0008, 0x343: 0x0008, 0x344: 0x0008, 0x345: 0x0008, + 0x346: 0x0008, 0x347: 0x0008, 0x348: 0x0008, 0x349: 0x0008, 0x34a: 0x0008, 0x34b: 0x0008, + 0x34c: 0x0008, 0x34d: 0x0008, 0x34e: 0x0008, 0x34f: 0x0008, 0x350: 0x0008, 0x351: 0x0008, + 0x352: 0x0008, 0x353: 0x0008, 0x354: 0x0008, 0x355: 0x0008, 0x356: 0x0008, 0x357: 0x0008, + 0x358: 0x0008, 0x359: 0x0008, 0x35a: 0x0008, 0x35b: 0x0008, 0x35c: 0x0008, 0x35d: 0x0008, + 0x35e: 0x0008, 0x35f: 0x0008, 0x360: 0xe00d, 0x361: 0x0008, 0x362: 0xe00d, 0x363: 0x0008, + 0x364: 0xe00d, 0x365: 0x0008, 0x366: 0xe00d, 0x367: 0x0008, 0x368: 0xe00d, 0x369: 0x0008, + 0x36a: 0xe00d, 0x36b: 0x0008, 0x36c: 0xe00d, 0x36d: 0x0008, 0x36e: 0xe00d, 0x36f: 0x0008, + 0x370: 0xe00d, 0x371: 0x0008, 0x372: 0xe00d, 0x373: 0x0008, 0x374: 0xe00d, 0x375: 0x0008, + 0x376: 0xe00d, 0x377: 0x0008, 0x378: 0xe00d, 0x379: 0x0008, 0x37a: 0xe00d, 0x37b: 0x0008, + 0x37c: 0xe00d, 0x37d: 0x0008, 0x37e: 0xe00d, 0x37f: 0x0008, + // Block 0xe, offset 0x380 + 0x380: 0xe00d, 0x381: 0x0008, 0x382: 0x0018, 0x383: 0x1308, 0x384: 0x1308, 0x385: 0x1308, + 0x386: 0x1308, 0x387: 0x1308, 0x388: 0x1318, 0x389: 0x1318, 0x38a: 0xe00d, 0x38b: 0x0008, + 0x38c: 0xe00d, 0x38d: 0x0008, 0x38e: 0xe00d, 0x38f: 0x0008, 0x390: 0xe00d, 0x391: 0x0008, + 0x392: 0xe00d, 0x393: 0x0008, 0x394: 0xe00d, 0x395: 0x0008, 0x396: 0xe00d, 0x397: 0x0008, + 0x398: 0xe00d, 0x399: 0x0008, 0x39a: 0xe00d, 0x39b: 0x0008, 0x39c: 0xe00d, 0x39d: 0x0008, + 0x39e: 0xe00d, 0x39f: 0x0008, 0x3a0: 0xe00d, 0x3a1: 0x0008, 0x3a2: 0xe00d, 0x3a3: 0x0008, + 0x3a4: 0xe00d, 0x3a5: 0x0008, 0x3a6: 0xe00d, 0x3a7: 0x0008, 0x3a8: 0xe00d, 0x3a9: 0x0008, + 0x3aa: 0xe00d, 0x3ab: 0x0008, 0x3ac: 0xe00d, 0x3ad: 0x0008, 0x3ae: 0xe00d, 0x3af: 0x0008, + 0x3b0: 0xe00d, 0x3b1: 0x0008, 0x3b2: 0xe00d, 0x3b3: 0x0008, 0x3b4: 0xe00d, 0x3b5: 0x0008, + 0x3b6: 0xe00d, 0x3b7: 0x0008, 0x3b8: 0xe00d, 0x3b9: 0x0008, 0x3ba: 0xe00d, 0x3bb: 0x0008, + 0x3bc: 0xe00d, 0x3bd: 0x0008, 0x3be: 0xe00d, 0x3bf: 0x0008, + // Block 0xf, offset 0x3c0 + 0x3c0: 0x0040, 0x3c1: 0xe01d, 0x3c2: 0x0008, 0x3c3: 0xe03d, 0x3c4: 0x0008, 0x3c5: 0xe01d, + 0x3c6: 0x0008, 0x3c7: 0xe07d, 0x3c8: 0x0008, 0x3c9: 0xe01d, 0x3ca: 0x0008, 0x3cb: 0xe03d, + 0x3cc: 0x0008, 0x3cd: 0xe01d, 0x3ce: 0x0008, 0x3cf: 0x0008, 0x3d0: 0xe00d, 0x3d1: 0x0008, + 0x3d2: 0xe00d, 0x3d3: 0x0008, 0x3d4: 0xe00d, 0x3d5: 0x0008, 0x3d6: 0xe00d, 0x3d7: 0x0008, + 0x3d8: 0xe00d, 0x3d9: 0x0008, 0x3da: 0xe00d, 0x3db: 0x0008, 0x3dc: 0xe00d, 0x3dd: 0x0008, + 0x3de: 0xe00d, 0x3df: 0x0008, 0x3e0: 0xe00d, 0x3e1: 0x0008, 0x3e2: 0xe00d, 0x3e3: 0x0008, + 0x3e4: 0xe00d, 0x3e5: 0x0008, 0x3e6: 0xe00d, 0x3e7: 0x0008, 0x3e8: 0xe00d, 0x3e9: 0x0008, + 0x3ea: 0xe00d, 0x3eb: 0x0008, 0x3ec: 0xe00d, 0x3ed: 0x0008, 0x3ee: 0xe00d, 0x3ef: 0x0008, + 0x3f0: 0xe00d, 0x3f1: 0x0008, 0x3f2: 0xe00d, 0x3f3: 0x0008, 0x3f4: 0xe00d, 0x3f5: 0x0008, + 0x3f6: 0xe00d, 0x3f7: 0x0008, 0x3f8: 0xe00d, 0x3f9: 0x0008, 0x3fa: 0xe00d, 0x3fb: 0x0008, + 0x3fc: 0xe00d, 0x3fd: 0x0008, 0x3fe: 0xe00d, 0x3ff: 0x0008, + // Block 0x10, offset 0x400 + 0x400: 0xe00d, 0x401: 0x0008, 0x402: 0xe00d, 0x403: 0x0008, 0x404: 0xe00d, 0x405: 0x0008, + 0x406: 0xe00d, 0x407: 0x0008, 0x408: 0xe00d, 0x409: 0x0008, 0x40a: 0xe00d, 0x40b: 0x0008, + 0x40c: 0xe00d, 0x40d: 0x0008, 0x40e: 0xe00d, 0x40f: 0x0008, 0x410: 0xe00d, 0x411: 0x0008, + 0x412: 0xe00d, 0x413: 0x0008, 0x414: 0xe00d, 0x415: 0x0008, 0x416: 0xe00d, 0x417: 0x0008, + 0x418: 0xe00d, 0x419: 0x0008, 0x41a: 0xe00d, 0x41b: 0x0008, 0x41c: 0xe00d, 0x41d: 0x0008, + 0x41e: 0xe00d, 0x41f: 0x0008, 0x420: 0xe00d, 0x421: 0x0008, 0x422: 0xe00d, 0x423: 0x0008, + 0x424: 0xe00d, 0x425: 0x0008, 0x426: 0xe00d, 0x427: 0x0008, 0x428: 0xe00d, 0x429: 0x0008, + 0x42a: 0xe00d, 0x42b: 0x0008, 0x42c: 0xe00d, 0x42d: 0x0008, 0x42e: 0xe00d, 0x42f: 0x0008, + 0x430: 0x0040, 0x431: 0x03f5, 0x432: 0x03f5, 0x433: 0x03f5, 0x434: 0x03f5, 0x435: 0x03f5, + 0x436: 0x03f5, 0x437: 0x03f5, 0x438: 0x03f5, 0x439: 0x03f5, 0x43a: 0x03f5, 0x43b: 0x03f5, + 0x43c: 0x03f5, 0x43d: 0x03f5, 0x43e: 0x03f5, 0x43f: 0x03f5, + // Block 0x11, offset 0x440 + 0x440: 0x0040, 0x441: 0x0040, 0x442: 0x0040, 0x443: 0x0040, 0x444: 0x0040, 0x445: 0x0040, + 0x446: 0x0018, 0x447: 0x0018, 0x448: 0x0018, 0x449: 0x0018, 0x44a: 0x0018, 0x44b: 0x0018, + 0x44c: 0x0018, 0x44d: 0x0018, 0x44e: 0x0018, 0x44f: 0x0018, 0x450: 0x1308, 0x451: 0x1308, + 0x452: 0x1308, 0x453: 0x1308, 0x454: 0x1308, 0x455: 0x1308, 0x456: 0x1308, 0x457: 0x1308, + 0x458: 0x1308, 0x459: 0x1308, 0x45a: 0x1308, 0x45b: 0x0018, 0x45c: 0x0340, 0x45d: 0x0040, + 0x45e: 0x0018, 0x45f: 0x0018, 0x460: 0x0208, 0x461: 0x0008, 0x462: 0x0408, 0x463: 0x0408, + 0x464: 0x0408, 0x465: 0x0408, 0x466: 0x0208, 0x467: 0x0408, 0x468: 0x0208, 0x469: 0x0408, + 0x46a: 0x0208, 0x46b: 0x0208, 0x46c: 0x0208, 0x46d: 0x0208, 0x46e: 0x0208, 0x46f: 0x0408, + 0x470: 0x0408, 0x471: 0x0408, 0x472: 0x0408, 0x473: 0x0208, 0x474: 0x0208, 0x475: 0x0208, + 0x476: 0x0208, 0x477: 0x0208, 0x478: 0x0208, 0x479: 0x0208, 0x47a: 0x0208, 0x47b: 0x0208, + 0x47c: 0x0208, 0x47d: 0x0208, 0x47e: 0x0208, 0x47f: 0x0208, + // Block 0x12, offset 0x480 + 0x480: 0x0408, 0x481: 0x0208, 0x482: 0x0208, 0x483: 0x0408, 0x484: 0x0408, 0x485: 0x0408, + 0x486: 0x0408, 0x487: 0x0408, 0x488: 0x0408, 0x489: 0x0408, 0x48a: 0x0408, 0x48b: 0x0408, + 0x48c: 0x0208, 0x48d: 0x0408, 0x48e: 0x0208, 0x48f: 0x0408, 0x490: 0x0208, 0x491: 0x0208, + 0x492: 0x0408, 0x493: 0x0408, 0x494: 0x0018, 0x495: 0x0408, 0x496: 0x1308, 0x497: 0x1308, + 0x498: 0x1308, 0x499: 0x1308, 0x49a: 0x1308, 0x49b: 0x1308, 0x49c: 0x1308, 0x49d: 0x0040, + 0x49e: 0x0018, 0x49f: 0x1308, 0x4a0: 0x1308, 0x4a1: 0x1308, 0x4a2: 0x1308, 0x4a3: 0x1308, + 0x4a4: 0x1308, 0x4a5: 0x0008, 0x4a6: 0x0008, 0x4a7: 0x1308, 0x4a8: 0x1308, 0x4a9: 0x0018, + 0x4aa: 0x1308, 0x4ab: 0x1308, 0x4ac: 0x1308, 0x4ad: 0x1308, 0x4ae: 0x0408, 0x4af: 0x0408, + 0x4b0: 0x0008, 0x4b1: 0x0008, 0x4b2: 0x0008, 0x4b3: 0x0008, 0x4b4: 0x0008, 0x4b5: 0x0008, + 0x4b6: 0x0008, 0x4b7: 0x0008, 0x4b8: 0x0008, 0x4b9: 0x0008, 0x4ba: 0x0208, 0x4bb: 0x0208, + 0x4bc: 0x0208, 0x4bd: 0x0008, 0x4be: 0x0008, 0x4bf: 0x0208, + // Block 0x13, offset 0x4c0 + 0x4c0: 0x0018, 0x4c1: 0x0018, 0x4c2: 0x0018, 0x4c3: 0x0018, 0x4c4: 0x0018, 0x4c5: 0x0018, + 0x4c6: 0x0018, 0x4c7: 0x0018, 0x4c8: 0x0018, 0x4c9: 0x0018, 0x4ca: 0x0018, 0x4cb: 0x0018, + 0x4cc: 0x0018, 0x4cd: 0x0018, 0x4ce: 0x0040, 0x4cf: 0x0340, 0x4d0: 0x0408, 0x4d1: 0x1308, + 0x4d2: 0x0208, 0x4d3: 0x0208, 0x4d4: 0x0208, 0x4d5: 0x0408, 0x4d6: 0x0408, 0x4d7: 0x0408, + 0x4d8: 0x0408, 0x4d9: 0x0408, 0x4da: 0x0208, 0x4db: 0x0208, 0x4dc: 0x0208, 0x4dd: 0x0208, + 0x4de: 0x0408, 0x4df: 0x0208, 0x4e0: 0x0208, 0x4e1: 0x0208, 0x4e2: 0x0208, 0x4e3: 0x0208, + 0x4e4: 0x0208, 0x4e5: 0x0208, 0x4e6: 0x0208, 0x4e7: 0x0208, 0x4e8: 0x0408, 0x4e9: 0x0208, + 0x4ea: 0x0408, 0x4eb: 0x0208, 0x4ec: 0x0408, 0x4ed: 0x0208, 0x4ee: 0x0208, 0x4ef: 0x0408, + 0x4f0: 0x1308, 0x4f1: 0x1308, 0x4f2: 0x1308, 0x4f3: 0x1308, 0x4f4: 0x1308, 0x4f5: 0x1308, + 0x4f6: 0x1308, 0x4f7: 0x1308, 0x4f8: 0x1308, 0x4f9: 0x1308, 0x4fa: 0x1308, 0x4fb: 0x1308, + 0x4fc: 0x1308, 0x4fd: 0x1308, 0x4fe: 0x1308, 0x4ff: 0x1308, + // Block 0x14, offset 0x500 + 0x500: 0x1008, 0x501: 0x1308, 0x502: 0x1308, 0x503: 0x1308, 0x504: 0x1308, 0x505: 0x1308, + 0x506: 0x1308, 0x507: 0x1308, 0x508: 0x1308, 0x509: 0x1008, 0x50a: 0x1008, 0x50b: 0x1008, + 0x50c: 0x1008, 0x50d: 0x1b08, 0x50e: 0x1008, 0x50f: 0x1008, 0x510: 0x0008, 0x511: 0x1308, + 0x512: 0x1308, 0x513: 0x1308, 0x514: 0x1308, 0x515: 0x1308, 0x516: 0x1308, 0x517: 0x1308, + 0x518: 0x04c9, 0x519: 0x0501, 0x51a: 0x0539, 0x51b: 0x0571, 0x51c: 0x05a9, 0x51d: 0x05e1, + 0x51e: 0x0619, 0x51f: 0x0651, 0x520: 0x0008, 0x521: 0x0008, 0x522: 0x1308, 0x523: 0x1308, + 0x524: 0x0018, 0x525: 0x0018, 0x526: 0x0008, 0x527: 0x0008, 0x528: 0x0008, 0x529: 0x0008, + 0x52a: 0x0008, 0x52b: 0x0008, 0x52c: 0x0008, 0x52d: 0x0008, 0x52e: 0x0008, 0x52f: 0x0008, + 0x530: 0x0018, 0x531: 0x0008, 0x532: 0x0008, 0x533: 0x0008, 0x534: 0x0008, 0x535: 0x0008, + 0x536: 0x0008, 0x537: 0x0008, 0x538: 0x0008, 0x539: 0x0008, 0x53a: 0x0008, 0x53b: 0x0008, + 0x53c: 0x0008, 0x53d: 0x0008, 0x53e: 0x0008, 0x53f: 0x0008, + // Block 0x15, offset 0x540 + 0x540: 0x0008, 0x541: 0x1308, 0x542: 0x1008, 0x543: 0x1008, 0x544: 0x0040, 0x545: 0x0008, + 0x546: 0x0008, 0x547: 0x0008, 0x548: 0x0008, 0x549: 0x0008, 0x54a: 0x0008, 0x54b: 0x0008, + 0x54c: 0x0008, 0x54d: 0x0040, 0x54e: 0x0040, 0x54f: 0x0008, 0x550: 0x0008, 0x551: 0x0040, + 0x552: 0x0040, 0x553: 0x0008, 0x554: 0x0008, 0x555: 0x0008, 0x556: 0x0008, 0x557: 0x0008, + 0x558: 0x0008, 0x559: 0x0008, 0x55a: 0x0008, 0x55b: 0x0008, 0x55c: 0x0008, 0x55d: 0x0008, + 0x55e: 0x0008, 0x55f: 0x0008, 0x560: 0x0008, 0x561: 0x0008, 0x562: 0x0008, 0x563: 0x0008, + 0x564: 0x0008, 0x565: 0x0008, 0x566: 0x0008, 0x567: 0x0008, 0x568: 0x0008, 0x569: 0x0040, + 0x56a: 0x0008, 0x56b: 0x0008, 0x56c: 0x0008, 0x56d: 0x0008, 0x56e: 0x0008, 0x56f: 0x0008, + 0x570: 0x0008, 0x571: 0x0040, 0x572: 0x0008, 0x573: 0x0040, 0x574: 0x0040, 0x575: 0x0040, + 0x576: 0x0008, 0x577: 0x0008, 0x578: 0x0008, 0x579: 0x0008, 0x57a: 0x0040, 0x57b: 0x0040, + 0x57c: 0x1308, 0x57d: 0x0008, 0x57e: 0x1008, 0x57f: 0x1008, + // Block 0x16, offset 0x580 + 0x580: 0x1008, 0x581: 0x1308, 0x582: 0x1308, 0x583: 0x1308, 0x584: 0x1308, 0x585: 0x0040, + 0x586: 0x0040, 0x587: 0x1008, 0x588: 0x1008, 0x589: 0x0040, 0x58a: 0x0040, 0x58b: 0x1008, + 0x58c: 0x1008, 0x58d: 0x1b08, 0x58e: 0x0008, 0x58f: 0x0040, 0x590: 0x0040, 0x591: 0x0040, + 0x592: 0x0040, 0x593: 0x0040, 0x594: 0x0040, 0x595: 0x0040, 0x596: 0x0040, 0x597: 0x1008, + 0x598: 0x0040, 0x599: 0x0040, 0x59a: 0x0040, 0x59b: 0x0040, 0x59c: 0x0689, 0x59d: 0x06c1, + 0x59e: 0x0040, 0x59f: 0x06f9, 0x5a0: 0x0008, 0x5a1: 0x0008, 0x5a2: 0x1308, 0x5a3: 0x1308, + 0x5a4: 0x0040, 0x5a5: 0x0040, 0x5a6: 0x0008, 0x5a7: 0x0008, 0x5a8: 0x0008, 0x5a9: 0x0008, + 0x5aa: 0x0008, 0x5ab: 0x0008, 0x5ac: 0x0008, 0x5ad: 0x0008, 0x5ae: 0x0008, 0x5af: 0x0008, + 0x5b0: 0x0008, 0x5b1: 0x0008, 0x5b2: 0x0018, 0x5b3: 0x0018, 0x5b4: 0x0018, 0x5b5: 0x0018, + 0x5b6: 0x0018, 0x5b7: 0x0018, 0x5b8: 0x0018, 0x5b9: 0x0018, 0x5ba: 0x0018, 0x5bb: 0x0018, + 0x5bc: 0x0040, 0x5bd: 0x0040, 0x5be: 0x0040, 0x5bf: 0x0040, + // Block 0x17, offset 0x5c0 + 0x5c0: 0x0040, 0x5c1: 0x1308, 0x5c2: 0x1308, 0x5c3: 0x1008, 0x5c4: 0x0040, 0x5c5: 0x0008, + 0x5c6: 0x0008, 0x5c7: 0x0008, 0x5c8: 0x0008, 0x5c9: 0x0008, 0x5ca: 0x0008, 0x5cb: 0x0040, + 0x5cc: 0x0040, 0x5cd: 0x0040, 0x5ce: 0x0040, 0x5cf: 0x0008, 0x5d0: 0x0008, 0x5d1: 0x0040, + 0x5d2: 0x0040, 0x5d3: 0x0008, 0x5d4: 0x0008, 0x5d5: 0x0008, 0x5d6: 0x0008, 0x5d7: 0x0008, + 0x5d8: 0x0008, 0x5d9: 0x0008, 0x5da: 0x0008, 0x5db: 0x0008, 0x5dc: 0x0008, 0x5dd: 0x0008, + 0x5de: 0x0008, 0x5df: 0x0008, 0x5e0: 0x0008, 0x5e1: 0x0008, 0x5e2: 0x0008, 0x5e3: 0x0008, + 0x5e4: 0x0008, 0x5e5: 0x0008, 0x5e6: 0x0008, 0x5e7: 0x0008, 0x5e8: 0x0008, 0x5e9: 0x0040, + 0x5ea: 0x0008, 0x5eb: 0x0008, 0x5ec: 0x0008, 0x5ed: 0x0008, 0x5ee: 0x0008, 0x5ef: 0x0008, + 0x5f0: 0x0008, 0x5f1: 0x0040, 0x5f2: 0x0008, 0x5f3: 0x0731, 0x5f4: 0x0040, 0x5f5: 0x0008, + 0x5f6: 0x0769, 0x5f7: 0x0040, 0x5f8: 0x0008, 0x5f9: 0x0008, 0x5fa: 0x0040, 0x5fb: 0x0040, + 0x5fc: 0x1308, 0x5fd: 0x0040, 0x5fe: 0x1008, 0x5ff: 0x1008, + // Block 0x18, offset 0x600 + 0x600: 0x1008, 0x601: 0x1308, 0x602: 0x1308, 0x603: 0x0040, 0x604: 0x0040, 0x605: 0x0040, + 0x606: 0x0040, 0x607: 0x1308, 0x608: 0x1308, 0x609: 0x0040, 0x60a: 0x0040, 0x60b: 0x1308, + 0x60c: 0x1308, 0x60d: 0x1b08, 0x60e: 0x0040, 0x60f: 0x0040, 0x610: 0x0040, 0x611: 0x1308, + 0x612: 0x0040, 0x613: 0x0040, 0x614: 0x0040, 0x615: 0x0040, 0x616: 0x0040, 0x617: 0x0040, + 0x618: 0x0040, 0x619: 0x07a1, 0x61a: 0x07d9, 0x61b: 0x0811, 0x61c: 0x0008, 0x61d: 0x0040, + 0x61e: 0x0849, 0x61f: 0x0040, 0x620: 0x0040, 0x621: 0x0040, 0x622: 0x0040, 0x623: 0x0040, + 0x624: 0x0040, 0x625: 0x0040, 0x626: 0x0008, 0x627: 0x0008, 0x628: 0x0008, 0x629: 0x0008, + 0x62a: 0x0008, 0x62b: 0x0008, 0x62c: 0x0008, 0x62d: 0x0008, 0x62e: 0x0008, 0x62f: 0x0008, + 0x630: 0x1308, 0x631: 0x1308, 0x632: 0x0008, 0x633: 0x0008, 0x634: 0x0008, 0x635: 0x1308, + 0x636: 0x0040, 0x637: 0x0040, 0x638: 0x0040, 0x639: 0x0040, 0x63a: 0x0040, 0x63b: 0x0040, + 0x63c: 0x0040, 0x63d: 0x0040, 0x63e: 0x0040, 0x63f: 0x0040, + // Block 0x19, offset 0x640 + 0x640: 0x0040, 0x641: 0x1308, 0x642: 0x1308, 0x643: 0x1008, 0x644: 0x0040, 0x645: 0x0008, + 0x646: 0x0008, 0x647: 0x0008, 0x648: 0x0008, 0x649: 0x0008, 0x64a: 0x0008, 0x64b: 0x0008, + 0x64c: 0x0008, 0x64d: 0x0008, 0x64e: 0x0040, 0x64f: 0x0008, 0x650: 0x0008, 0x651: 0x0008, + 0x652: 0x0040, 0x653: 0x0008, 0x654: 0x0008, 0x655: 0x0008, 0x656: 0x0008, 0x657: 0x0008, + 0x658: 0x0008, 0x659: 0x0008, 0x65a: 0x0008, 0x65b: 0x0008, 0x65c: 0x0008, 0x65d: 0x0008, + 0x65e: 0x0008, 0x65f: 0x0008, 0x660: 0x0008, 0x661: 0x0008, 0x662: 0x0008, 0x663: 0x0008, + 0x664: 0x0008, 0x665: 0x0008, 0x666: 0x0008, 0x667: 0x0008, 0x668: 0x0008, 0x669: 0x0040, + 0x66a: 0x0008, 0x66b: 0x0008, 0x66c: 0x0008, 0x66d: 0x0008, 0x66e: 0x0008, 0x66f: 0x0008, + 0x670: 0x0008, 0x671: 0x0040, 0x672: 0x0008, 0x673: 0x0008, 0x674: 0x0040, 0x675: 0x0008, + 0x676: 0x0008, 0x677: 0x0008, 0x678: 0x0008, 0x679: 0x0008, 0x67a: 0x0040, 0x67b: 0x0040, + 0x67c: 0x1308, 0x67d: 0x0008, 0x67e: 0x1008, 0x67f: 0x1008, + // Block 0x1a, offset 0x680 + 0x680: 0x1008, 0x681: 0x1308, 0x682: 0x1308, 0x683: 0x1308, 0x684: 0x1308, 0x685: 0x1308, + 0x686: 0x0040, 0x687: 0x1308, 0x688: 0x1308, 0x689: 0x1008, 0x68a: 0x0040, 0x68b: 0x1008, + 0x68c: 0x1008, 0x68d: 0x1b08, 0x68e: 0x0040, 0x68f: 0x0040, 0x690: 0x0008, 0x691: 0x0040, + 0x692: 0x0040, 0x693: 0x0040, 0x694: 0x0040, 0x695: 0x0040, 0x696: 0x0040, 0x697: 0x0040, + 0x698: 0x0040, 0x699: 0x0040, 0x69a: 0x0040, 0x69b: 0x0040, 0x69c: 0x0040, 0x69d: 0x0040, + 0x69e: 0x0040, 0x69f: 0x0040, 0x6a0: 0x0008, 0x6a1: 0x0008, 0x6a2: 0x1308, 0x6a3: 0x1308, + 0x6a4: 0x0040, 0x6a5: 0x0040, 0x6a6: 0x0008, 0x6a7: 0x0008, 0x6a8: 0x0008, 0x6a9: 0x0008, + 0x6aa: 0x0008, 0x6ab: 0x0008, 0x6ac: 0x0008, 0x6ad: 0x0008, 0x6ae: 0x0008, 0x6af: 0x0008, + 0x6b0: 0x0018, 0x6b1: 0x0018, 0x6b2: 0x0040, 0x6b3: 0x0040, 0x6b4: 0x0040, 0x6b5: 0x0040, + 0x6b6: 0x0040, 0x6b7: 0x0040, 0x6b8: 0x0040, 0x6b9: 0x0008, 0x6ba: 0x0040, 0x6bb: 0x0040, + 0x6bc: 0x0040, 0x6bd: 0x0040, 0x6be: 0x0040, 0x6bf: 0x0040, + // Block 0x1b, offset 0x6c0 + 0x6c0: 0x0040, 0x6c1: 0x1308, 0x6c2: 0x1008, 0x6c3: 0x1008, 0x6c4: 0x0040, 0x6c5: 0x0008, + 0x6c6: 0x0008, 0x6c7: 0x0008, 0x6c8: 0x0008, 0x6c9: 0x0008, 0x6ca: 0x0008, 0x6cb: 0x0008, + 0x6cc: 0x0008, 0x6cd: 0x0040, 0x6ce: 0x0040, 0x6cf: 0x0008, 0x6d0: 0x0008, 0x6d1: 0x0040, + 0x6d2: 0x0040, 0x6d3: 0x0008, 0x6d4: 0x0008, 0x6d5: 0x0008, 0x6d6: 0x0008, 0x6d7: 0x0008, + 0x6d8: 0x0008, 0x6d9: 0x0008, 0x6da: 0x0008, 0x6db: 0x0008, 0x6dc: 0x0008, 0x6dd: 0x0008, + 0x6de: 0x0008, 0x6df: 0x0008, 0x6e0: 0x0008, 0x6e1: 0x0008, 0x6e2: 0x0008, 0x6e3: 0x0008, + 0x6e4: 0x0008, 0x6e5: 0x0008, 0x6e6: 0x0008, 0x6e7: 0x0008, 0x6e8: 0x0008, 0x6e9: 0x0040, + 0x6ea: 0x0008, 0x6eb: 0x0008, 0x6ec: 0x0008, 0x6ed: 0x0008, 0x6ee: 0x0008, 0x6ef: 0x0008, + 0x6f0: 0x0008, 0x6f1: 0x0040, 0x6f2: 0x0008, 0x6f3: 0x0008, 0x6f4: 0x0040, 0x6f5: 0x0008, + 0x6f6: 0x0008, 0x6f7: 0x0008, 0x6f8: 0x0008, 0x6f9: 0x0008, 0x6fa: 0x0040, 0x6fb: 0x0040, + 0x6fc: 0x1308, 0x6fd: 0x0008, 0x6fe: 0x1008, 0x6ff: 0x1308, + // Block 0x1c, offset 0x700 + 0x700: 0x1008, 0x701: 0x1308, 0x702: 0x1308, 0x703: 0x1308, 0x704: 0x1308, 0x705: 0x0040, + 0x706: 0x0040, 0x707: 0x1008, 0x708: 0x1008, 0x709: 0x0040, 0x70a: 0x0040, 0x70b: 0x1008, + 0x70c: 0x1008, 0x70d: 0x1b08, 0x70e: 0x0040, 0x70f: 0x0040, 0x710: 0x0040, 0x711: 0x0040, + 0x712: 0x0040, 0x713: 0x0040, 0x714: 0x0040, 0x715: 0x0040, 0x716: 0x1308, 0x717: 0x1008, + 0x718: 0x0040, 0x719: 0x0040, 0x71a: 0x0040, 0x71b: 0x0040, 0x71c: 0x0881, 0x71d: 0x08b9, + 0x71e: 0x0040, 0x71f: 0x0008, 0x720: 0x0008, 0x721: 0x0008, 0x722: 0x1308, 0x723: 0x1308, + 0x724: 0x0040, 0x725: 0x0040, 0x726: 0x0008, 0x727: 0x0008, 0x728: 0x0008, 0x729: 0x0008, + 0x72a: 0x0008, 0x72b: 0x0008, 0x72c: 0x0008, 0x72d: 0x0008, 0x72e: 0x0008, 0x72f: 0x0008, + 0x730: 0x0018, 0x731: 0x0008, 0x732: 0x0018, 0x733: 0x0018, 0x734: 0x0018, 0x735: 0x0018, + 0x736: 0x0018, 0x737: 0x0018, 0x738: 0x0040, 0x739: 0x0040, 0x73a: 0x0040, 0x73b: 0x0040, + 0x73c: 0x0040, 0x73d: 0x0040, 0x73e: 0x0040, 0x73f: 0x0040, + // Block 0x1d, offset 0x740 + 0x740: 0x0040, 0x741: 0x0040, 0x742: 0x1308, 0x743: 0x0008, 0x744: 0x0040, 0x745: 0x0008, + 0x746: 0x0008, 0x747: 0x0008, 0x748: 0x0008, 0x749: 0x0008, 0x74a: 0x0008, 0x74b: 0x0040, + 0x74c: 0x0040, 0x74d: 0x0040, 0x74e: 0x0008, 0x74f: 0x0008, 0x750: 0x0008, 0x751: 0x0040, + 0x752: 0x0008, 0x753: 0x0008, 0x754: 0x0008, 0x755: 0x0008, 0x756: 0x0040, 0x757: 0x0040, + 0x758: 0x0040, 0x759: 0x0008, 0x75a: 0x0008, 0x75b: 0x0040, 0x75c: 0x0008, 0x75d: 0x0040, + 0x75e: 0x0008, 0x75f: 0x0008, 0x760: 0x0040, 0x761: 0x0040, 0x762: 0x0040, 0x763: 0x0008, + 0x764: 0x0008, 0x765: 0x0040, 0x766: 0x0040, 0x767: 0x0040, 0x768: 0x0008, 0x769: 0x0008, + 0x76a: 0x0008, 0x76b: 0x0040, 0x76c: 0x0040, 0x76d: 0x0040, 0x76e: 0x0008, 0x76f: 0x0008, + 0x770: 0x0008, 0x771: 0x0008, 0x772: 0x0008, 0x773: 0x0008, 0x774: 0x0008, 0x775: 0x0008, + 0x776: 0x0008, 0x777: 0x0008, 0x778: 0x0008, 0x779: 0x0008, 0x77a: 0x0040, 0x77b: 0x0040, + 0x77c: 0x0040, 0x77d: 0x0040, 0x77e: 0x1008, 0x77f: 0x1008, + // Block 0x1e, offset 0x780 + 0x780: 0x1308, 0x781: 0x1008, 0x782: 0x1008, 0x783: 0x1008, 0x784: 0x1008, 0x785: 0x0040, + 0x786: 0x1308, 0x787: 0x1308, 0x788: 0x1308, 0x789: 0x0040, 0x78a: 0x1308, 0x78b: 0x1308, + 0x78c: 0x1308, 0x78d: 0x1b08, 0x78e: 0x0040, 0x78f: 0x0040, 0x790: 0x0040, 0x791: 0x0040, + 0x792: 0x0040, 0x793: 0x0040, 0x794: 0x0040, 0x795: 0x1308, 0x796: 0x1308, 0x797: 0x0040, + 0x798: 0x0008, 0x799: 0x0008, 0x79a: 0x0008, 0x79b: 0x0040, 0x79c: 0x0040, 0x79d: 0x0040, + 0x79e: 0x0040, 0x79f: 0x0040, 0x7a0: 0x0008, 0x7a1: 0x0008, 0x7a2: 0x1308, 0x7a3: 0x1308, + 0x7a4: 0x0040, 0x7a5: 0x0040, 0x7a6: 0x0008, 0x7a7: 0x0008, 0x7a8: 0x0008, 0x7a9: 0x0008, + 0x7aa: 0x0008, 0x7ab: 0x0008, 0x7ac: 0x0008, 0x7ad: 0x0008, 0x7ae: 0x0008, 0x7af: 0x0008, + 0x7b0: 0x0040, 0x7b1: 0x0040, 0x7b2: 0x0040, 0x7b3: 0x0040, 0x7b4: 0x0040, 0x7b5: 0x0040, + 0x7b6: 0x0040, 0x7b7: 0x0040, 0x7b8: 0x0018, 0x7b9: 0x0018, 0x7ba: 0x0018, 0x7bb: 0x0018, + 0x7bc: 0x0018, 0x7bd: 0x0018, 0x7be: 0x0018, 0x7bf: 0x0018, + // Block 0x1f, offset 0x7c0 + 0x7c0: 0x0008, 0x7c1: 0x1308, 0x7c2: 0x1008, 0x7c3: 0x1008, 0x7c4: 0x0040, 0x7c5: 0x0008, + 0x7c6: 0x0008, 0x7c7: 0x0008, 0x7c8: 0x0008, 0x7c9: 0x0008, 0x7ca: 0x0008, 0x7cb: 0x0008, + 0x7cc: 0x0008, 0x7cd: 0x0040, 0x7ce: 0x0008, 0x7cf: 0x0008, 0x7d0: 0x0008, 0x7d1: 0x0040, + 0x7d2: 0x0008, 0x7d3: 0x0008, 0x7d4: 0x0008, 0x7d5: 0x0008, 0x7d6: 0x0008, 0x7d7: 0x0008, + 0x7d8: 0x0008, 0x7d9: 0x0008, 0x7da: 0x0008, 0x7db: 0x0008, 0x7dc: 0x0008, 0x7dd: 0x0008, + 0x7de: 0x0008, 0x7df: 0x0008, 0x7e0: 0x0008, 0x7e1: 0x0008, 0x7e2: 0x0008, 0x7e3: 0x0008, + 0x7e4: 0x0008, 0x7e5: 0x0008, 0x7e6: 0x0008, 0x7e7: 0x0008, 0x7e8: 0x0008, 0x7e9: 0x0040, + 0x7ea: 0x0008, 0x7eb: 0x0008, 0x7ec: 0x0008, 0x7ed: 0x0008, 0x7ee: 0x0008, 0x7ef: 0x0008, + 0x7f0: 0x0008, 0x7f1: 0x0008, 0x7f2: 0x0008, 0x7f3: 0x0008, 0x7f4: 0x0040, 0x7f5: 0x0008, + 0x7f6: 0x0008, 0x7f7: 0x0008, 0x7f8: 0x0008, 0x7f9: 0x0008, 0x7fa: 0x0040, 0x7fb: 0x0040, + 0x7fc: 0x1308, 0x7fd: 0x0008, 0x7fe: 0x1008, 0x7ff: 0x1308, + // Block 0x20, offset 0x800 + 0x800: 0x1008, 0x801: 0x1008, 0x802: 0x1008, 0x803: 0x1008, 0x804: 0x1008, 0x805: 0x0040, + 0x806: 0x1308, 0x807: 0x1008, 0x808: 0x1008, 0x809: 0x0040, 0x80a: 0x1008, 0x80b: 0x1008, + 0x80c: 0x1308, 0x80d: 0x1b08, 0x80e: 0x0040, 0x80f: 0x0040, 0x810: 0x0040, 0x811: 0x0040, + 0x812: 0x0040, 0x813: 0x0040, 0x814: 0x0040, 0x815: 0x1008, 0x816: 0x1008, 0x817: 0x0040, + 0x818: 0x0040, 0x819: 0x0040, 0x81a: 0x0040, 0x81b: 0x0040, 0x81c: 0x0040, 0x81d: 0x0040, + 0x81e: 0x0008, 0x81f: 0x0040, 0x820: 0x0008, 0x821: 0x0008, 0x822: 0x1308, 0x823: 0x1308, + 0x824: 0x0040, 0x825: 0x0040, 0x826: 0x0008, 0x827: 0x0008, 0x828: 0x0008, 0x829: 0x0008, + 0x82a: 0x0008, 0x82b: 0x0008, 0x82c: 0x0008, 0x82d: 0x0008, 0x82e: 0x0008, 0x82f: 0x0008, + 0x830: 0x0040, 0x831: 0x0008, 0x832: 0x0008, 0x833: 0x0040, 0x834: 0x0040, 0x835: 0x0040, + 0x836: 0x0040, 0x837: 0x0040, 0x838: 0x0040, 0x839: 0x0040, 0x83a: 0x0040, 0x83b: 0x0040, + 0x83c: 0x0040, 0x83d: 0x0040, 0x83e: 0x0040, 0x83f: 0x0040, + // Block 0x21, offset 0x840 + 0x840: 0x1008, 0x841: 0x1308, 0x842: 0x1308, 0x843: 0x1308, 0x844: 0x1308, 0x845: 0x0040, + 0x846: 0x1008, 0x847: 0x1008, 0x848: 0x1008, 0x849: 0x0040, 0x84a: 0x1008, 0x84b: 0x1008, + 0x84c: 0x1008, 0x84d: 0x1b08, 0x84e: 0x0008, 0x84f: 0x0018, 0x850: 0x0040, 0x851: 0x0040, + 0x852: 0x0040, 0x853: 0x0040, 0x854: 0x0008, 0x855: 0x0008, 0x856: 0x0008, 0x857: 0x1008, + 0x858: 0x0018, 0x859: 0x0018, 0x85a: 0x0018, 0x85b: 0x0018, 0x85c: 0x0018, 0x85d: 0x0018, + 0x85e: 0x0018, 0x85f: 0x0008, 0x860: 0x0008, 0x861: 0x0008, 0x862: 0x1308, 0x863: 0x1308, + 0x864: 0x0040, 0x865: 0x0040, 0x866: 0x0008, 0x867: 0x0008, 0x868: 0x0008, 0x869: 0x0008, + 0x86a: 0x0008, 0x86b: 0x0008, 0x86c: 0x0008, 0x86d: 0x0008, 0x86e: 0x0008, 0x86f: 0x0008, + 0x870: 0x0018, 0x871: 0x0018, 0x872: 0x0018, 0x873: 0x0018, 0x874: 0x0018, 0x875: 0x0018, + 0x876: 0x0018, 0x877: 0x0018, 0x878: 0x0018, 0x879: 0x0018, 0x87a: 0x0008, 0x87b: 0x0008, + 0x87c: 0x0008, 0x87d: 0x0008, 0x87e: 0x0008, 0x87f: 0x0008, + // Block 0x22, offset 0x880 + 0x880: 0x0040, 0x881: 0x0008, 0x882: 0x0008, 0x883: 0x0040, 0x884: 0x0008, 0x885: 0x0040, + 0x886: 0x0040, 0x887: 0x0008, 0x888: 0x0008, 0x889: 0x0040, 0x88a: 0x0008, 0x88b: 0x0040, + 0x88c: 0x0040, 0x88d: 0x0008, 0x88e: 0x0040, 0x88f: 0x0040, 0x890: 0x0040, 0x891: 0x0040, + 0x892: 0x0040, 0x893: 0x0040, 0x894: 0x0008, 0x895: 0x0008, 0x896: 0x0008, 0x897: 0x0008, + 0x898: 0x0040, 0x899: 0x0008, 0x89a: 0x0008, 0x89b: 0x0008, 0x89c: 0x0008, 0x89d: 0x0008, + 0x89e: 0x0008, 0x89f: 0x0008, 0x8a0: 0x0040, 0x8a1: 0x0008, 0x8a2: 0x0008, 0x8a3: 0x0008, + 0x8a4: 0x0040, 0x8a5: 0x0008, 0x8a6: 0x0040, 0x8a7: 0x0008, 0x8a8: 0x0040, 0x8a9: 0x0040, + 0x8aa: 0x0008, 0x8ab: 0x0008, 0x8ac: 0x0040, 0x8ad: 0x0008, 0x8ae: 0x0008, 0x8af: 0x0008, + 0x8b0: 0x0008, 0x8b1: 0x1308, 0x8b2: 0x0008, 0x8b3: 0x0929, 0x8b4: 0x1308, 0x8b5: 0x1308, + 0x8b6: 0x1308, 0x8b7: 0x1308, 0x8b8: 0x1308, 0x8b9: 0x1308, 0x8ba: 0x0040, 0x8bb: 0x1308, + 0x8bc: 0x1308, 0x8bd: 0x0008, 0x8be: 0x0040, 0x8bf: 0x0040, + // Block 0x23, offset 0x8c0 + 0x8c0: 0x0008, 0x8c1: 0x0008, 0x8c2: 0x0008, 0x8c3: 0x09d1, 0x8c4: 0x0008, 0x8c5: 0x0008, + 0x8c6: 0x0008, 0x8c7: 0x0008, 0x8c8: 0x0040, 0x8c9: 0x0008, 0x8ca: 0x0008, 0x8cb: 0x0008, + 0x8cc: 0x0008, 0x8cd: 0x0a09, 0x8ce: 0x0008, 0x8cf: 0x0008, 0x8d0: 0x0008, 0x8d1: 0x0008, + 0x8d2: 0x0a41, 0x8d3: 0x0008, 0x8d4: 0x0008, 0x8d5: 0x0008, 0x8d6: 0x0008, 0x8d7: 0x0a79, + 0x8d8: 0x0008, 0x8d9: 0x0008, 0x8da: 0x0008, 0x8db: 0x0008, 0x8dc: 0x0ab1, 0x8dd: 0x0008, + 0x8de: 0x0008, 0x8df: 0x0008, 0x8e0: 0x0008, 0x8e1: 0x0008, 0x8e2: 0x0008, 0x8e3: 0x0008, + 0x8e4: 0x0008, 0x8e5: 0x0008, 0x8e6: 0x0008, 0x8e7: 0x0008, 0x8e8: 0x0008, 0x8e9: 0x0ae9, + 0x8ea: 0x0008, 0x8eb: 0x0008, 0x8ec: 0x0008, 0x8ed: 0x0040, 0x8ee: 0x0040, 0x8ef: 0x0040, + 0x8f0: 0x0040, 0x8f1: 0x1308, 0x8f2: 0x1308, 0x8f3: 0x0b21, 0x8f4: 0x1308, 0x8f5: 0x0b59, + 0x8f6: 0x0b91, 0x8f7: 0x0bc9, 0x8f8: 0x0c19, 0x8f9: 0x0c51, 0x8fa: 0x1308, 0x8fb: 0x1308, + 0x8fc: 0x1308, 0x8fd: 0x1308, 0x8fe: 0x1308, 0x8ff: 0x1008, + // Block 0x24, offset 0x900 + 0x900: 0x1308, 0x901: 0x0ca1, 0x902: 0x1308, 0x903: 0x1308, 0x904: 0x1b08, 0x905: 0x0018, + 0x906: 0x1308, 0x907: 0x1308, 0x908: 0x0008, 0x909: 0x0008, 0x90a: 0x0008, 0x90b: 0x0008, + 0x90c: 0x0008, 0x90d: 0x1308, 0x90e: 0x1308, 0x90f: 0x1308, 0x910: 0x1308, 0x911: 0x1308, + 0x912: 0x1308, 0x913: 0x0cd9, 0x914: 0x1308, 0x915: 0x1308, 0x916: 0x1308, 0x917: 0x1308, + 0x918: 0x0040, 0x919: 0x1308, 0x91a: 0x1308, 0x91b: 0x1308, 0x91c: 0x1308, 0x91d: 0x0d11, + 0x91e: 0x1308, 0x91f: 0x1308, 0x920: 0x1308, 0x921: 0x1308, 0x922: 0x0d49, 0x923: 0x1308, + 0x924: 0x1308, 0x925: 0x1308, 0x926: 0x1308, 0x927: 0x0d81, 0x928: 0x1308, 0x929: 0x1308, + 0x92a: 0x1308, 0x92b: 0x1308, 0x92c: 0x0db9, 0x92d: 0x1308, 0x92e: 0x1308, 0x92f: 0x1308, + 0x930: 0x1308, 0x931: 0x1308, 0x932: 0x1308, 0x933: 0x1308, 0x934: 0x1308, 0x935: 0x1308, + 0x936: 0x1308, 0x937: 0x1308, 0x938: 0x1308, 0x939: 0x0df1, 0x93a: 0x1308, 0x93b: 0x1308, + 0x93c: 0x1308, 0x93d: 0x0040, 0x93e: 0x0018, 0x93f: 0x0018, + // Block 0x25, offset 0x940 + 0x940: 0x0008, 0x941: 0x0008, 0x942: 0x0008, 0x943: 0x0008, 0x944: 0x0008, 0x945: 0x0008, + 0x946: 0x0008, 0x947: 0x0008, 0x948: 0x0008, 0x949: 0x0008, 0x94a: 0x0008, 0x94b: 0x0008, + 0x94c: 0x0008, 0x94d: 0x0008, 0x94e: 0x0008, 0x94f: 0x0008, 0x950: 0x0008, 0x951: 0x0008, + 0x952: 0x0008, 0x953: 0x0008, 0x954: 0x0008, 0x955: 0x0008, 0x956: 0x0008, 0x957: 0x0008, + 0x958: 0x0008, 0x959: 0x0008, 0x95a: 0x0008, 0x95b: 0x0008, 0x95c: 0x0008, 0x95d: 0x0008, + 0x95e: 0x0008, 0x95f: 0x0008, 0x960: 0x0008, 0x961: 0x0008, 0x962: 0x0008, 0x963: 0x0008, + 0x964: 0x0008, 0x965: 0x0008, 0x966: 0x0008, 0x967: 0x0008, 0x968: 0x0008, 0x969: 0x0008, + 0x96a: 0x0008, 0x96b: 0x0008, 0x96c: 0x0039, 0x96d: 0x0ed1, 0x96e: 0x0ee9, 0x96f: 0x0008, + 0x970: 0x0ef9, 0x971: 0x0f09, 0x972: 0x0f19, 0x973: 0x0f31, 0x974: 0x0249, 0x975: 0x0f41, + 0x976: 0x0259, 0x977: 0x0f51, 0x978: 0x0359, 0x979: 0x0f61, 0x97a: 0x0f71, 0x97b: 0x0008, + 0x97c: 0x00d9, 0x97d: 0x0f81, 0x97e: 0x0f99, 0x97f: 0x0269, + // Block 0x26, offset 0x980 + 0x980: 0x0fa9, 0x981: 0x0fb9, 0x982: 0x0279, 0x983: 0x0039, 0x984: 0x0fc9, 0x985: 0x0fe1, + 0x986: 0x059d, 0x987: 0x0ee9, 0x988: 0x0ef9, 0x989: 0x0f09, 0x98a: 0x0ff9, 0x98b: 0x1011, + 0x98c: 0x1029, 0x98d: 0x0f31, 0x98e: 0x0008, 0x98f: 0x0f51, 0x990: 0x0f61, 0x991: 0x1041, + 0x992: 0x00d9, 0x993: 0x1059, 0x994: 0x05b5, 0x995: 0x05b5, 0x996: 0x0f99, 0x997: 0x0fa9, + 0x998: 0x0fb9, 0x999: 0x059d, 0x99a: 0x1071, 0x99b: 0x1089, 0x99c: 0x05cd, 0x99d: 0x1099, + 0x99e: 0x10b1, 0x99f: 0x10c9, 0x9a0: 0x10e1, 0x9a1: 0x10f9, 0x9a2: 0x0f41, 0x9a3: 0x0269, + 0x9a4: 0x0fb9, 0x9a5: 0x1089, 0x9a6: 0x1099, 0x9a7: 0x10b1, 0x9a8: 0x1111, 0x9a9: 0x10e1, + 0x9aa: 0x10f9, 0x9ab: 0x0008, 0x9ac: 0x0008, 0x9ad: 0x0008, 0x9ae: 0x0008, 0x9af: 0x0008, + 0x9b0: 0x0008, 0x9b1: 0x0008, 0x9b2: 0x0008, 0x9b3: 0x0008, 0x9b4: 0x0008, 0x9b5: 0x0008, + 0x9b6: 0x0008, 0x9b7: 0x0008, 0x9b8: 0x1129, 0x9b9: 0x0008, 0x9ba: 0x0008, 0x9bb: 0x0008, + 0x9bc: 0x0008, 0x9bd: 0x0008, 0x9be: 0x0008, 0x9bf: 0x0008, + // Block 0x27, offset 0x9c0 + 0x9c0: 0x0008, 0x9c1: 0x0008, 0x9c2: 0x0008, 0x9c3: 0x0008, 0x9c4: 0x0008, 0x9c5: 0x0008, + 0x9c6: 0x0008, 0x9c7: 0x0008, 0x9c8: 0x0008, 0x9c9: 0x0008, 0x9ca: 0x0008, 0x9cb: 0x0008, + 0x9cc: 0x0008, 0x9cd: 0x0008, 0x9ce: 0x0008, 0x9cf: 0x0008, 0x9d0: 0x0008, 0x9d1: 0x0008, + 0x9d2: 0x0008, 0x9d3: 0x0008, 0x9d4: 0x0008, 0x9d5: 0x0008, 0x9d6: 0x0008, 0x9d7: 0x0008, + 0x9d8: 0x0008, 0x9d9: 0x0008, 0x9da: 0x0008, 0x9db: 0x1141, 0x9dc: 0x1159, 0x9dd: 0x1169, + 0x9de: 0x1181, 0x9df: 0x1029, 0x9e0: 0x1199, 0x9e1: 0x11a9, 0x9e2: 0x11c1, 0x9e3: 0x11d9, + 0x9e4: 0x11f1, 0x9e5: 0x1209, 0x9e6: 0x1221, 0x9e7: 0x05e5, 0x9e8: 0x1239, 0x9e9: 0x1251, + 0x9ea: 0xe17d, 0x9eb: 0x1269, 0x9ec: 0x1281, 0x9ed: 0x1299, 0x9ee: 0x12b1, 0x9ef: 0x12c9, + 0x9f0: 0x12e1, 0x9f1: 0x12f9, 0x9f2: 0x1311, 0x9f3: 0x1329, 0x9f4: 0x1341, 0x9f5: 0x1359, + 0x9f6: 0x1371, 0x9f7: 0x1389, 0x9f8: 0x05fd, 0x9f9: 0x13a1, 0x9fa: 0x13b9, 0x9fb: 0x13d1, + 0x9fc: 0x13e1, 0x9fd: 0x13f9, 0x9fe: 0x1411, 0x9ff: 0x1429, + // Block 0x28, offset 0xa00 + 0xa00: 0xe00d, 0xa01: 0x0008, 0xa02: 0xe00d, 0xa03: 0x0008, 0xa04: 0xe00d, 0xa05: 0x0008, + 0xa06: 0xe00d, 0xa07: 0x0008, 0xa08: 0xe00d, 0xa09: 0x0008, 0xa0a: 0xe00d, 0xa0b: 0x0008, + 0xa0c: 0xe00d, 0xa0d: 0x0008, 0xa0e: 0xe00d, 0xa0f: 0x0008, 0xa10: 0xe00d, 0xa11: 0x0008, + 0xa12: 0xe00d, 0xa13: 0x0008, 0xa14: 0xe00d, 0xa15: 0x0008, 0xa16: 0xe00d, 0xa17: 0x0008, + 0xa18: 0xe00d, 0xa19: 0x0008, 0xa1a: 0xe00d, 0xa1b: 0x0008, 0xa1c: 0xe00d, 0xa1d: 0x0008, + 0xa1e: 0xe00d, 0xa1f: 0x0008, 0xa20: 0xe00d, 0xa21: 0x0008, 0xa22: 0xe00d, 0xa23: 0x0008, + 0xa24: 0xe00d, 0xa25: 0x0008, 0xa26: 0xe00d, 0xa27: 0x0008, 0xa28: 0xe00d, 0xa29: 0x0008, + 0xa2a: 0xe00d, 0xa2b: 0x0008, 0xa2c: 0xe00d, 0xa2d: 0x0008, 0xa2e: 0xe00d, 0xa2f: 0x0008, + 0xa30: 0xe00d, 0xa31: 0x0008, 0xa32: 0xe00d, 0xa33: 0x0008, 0xa34: 0xe00d, 0xa35: 0x0008, + 0xa36: 0xe00d, 0xa37: 0x0008, 0xa38: 0xe00d, 0xa39: 0x0008, 0xa3a: 0xe00d, 0xa3b: 0x0008, + 0xa3c: 0xe00d, 0xa3d: 0x0008, 0xa3e: 0xe00d, 0xa3f: 0x0008, + // Block 0x29, offset 0xa40 + 0xa40: 0xe00d, 0xa41: 0x0008, 0xa42: 0xe00d, 0xa43: 0x0008, 0xa44: 0xe00d, 0xa45: 0x0008, + 0xa46: 0xe00d, 0xa47: 0x0008, 0xa48: 0xe00d, 0xa49: 0x0008, 0xa4a: 0xe00d, 0xa4b: 0x0008, + 0xa4c: 0xe00d, 0xa4d: 0x0008, 0xa4e: 0xe00d, 0xa4f: 0x0008, 0xa50: 0xe00d, 0xa51: 0x0008, + 0xa52: 0xe00d, 0xa53: 0x0008, 0xa54: 0xe00d, 0xa55: 0x0008, 0xa56: 0x0008, 0xa57: 0x0008, + 0xa58: 0x0008, 0xa59: 0x0008, 0xa5a: 0x0615, 0xa5b: 0x0635, 0xa5c: 0x0008, 0xa5d: 0x0008, + 0xa5e: 0x1441, 0xa5f: 0x0008, 0xa60: 0xe00d, 0xa61: 0x0008, 0xa62: 0xe00d, 0xa63: 0x0008, + 0xa64: 0xe00d, 0xa65: 0x0008, 0xa66: 0xe00d, 0xa67: 0x0008, 0xa68: 0xe00d, 0xa69: 0x0008, + 0xa6a: 0xe00d, 0xa6b: 0x0008, 0xa6c: 0xe00d, 0xa6d: 0x0008, 0xa6e: 0xe00d, 0xa6f: 0x0008, + 0xa70: 0xe00d, 0xa71: 0x0008, 0xa72: 0xe00d, 0xa73: 0x0008, 0xa74: 0xe00d, 0xa75: 0x0008, + 0xa76: 0xe00d, 0xa77: 0x0008, 0xa78: 0xe00d, 0xa79: 0x0008, 0xa7a: 0xe00d, 0xa7b: 0x0008, + 0xa7c: 0xe00d, 0xa7d: 0x0008, 0xa7e: 0xe00d, 0xa7f: 0x0008, + // Block 0x2a, offset 0xa80 + 0xa80: 0x0008, 0xa81: 0x0008, 0xa82: 0x0008, 0xa83: 0x0008, 0xa84: 0x0008, 0xa85: 0x0008, + 0xa86: 0x0040, 0xa87: 0x0040, 0xa88: 0xe045, 0xa89: 0xe045, 0xa8a: 0xe045, 0xa8b: 0xe045, + 0xa8c: 0xe045, 0xa8d: 0xe045, 0xa8e: 0x0040, 0xa8f: 0x0040, 0xa90: 0x0008, 0xa91: 0x0008, + 0xa92: 0x0008, 0xa93: 0x0008, 0xa94: 0x0008, 0xa95: 0x0008, 0xa96: 0x0008, 0xa97: 0x0008, + 0xa98: 0x0040, 0xa99: 0xe045, 0xa9a: 0x0040, 0xa9b: 0xe045, 0xa9c: 0x0040, 0xa9d: 0xe045, + 0xa9e: 0x0040, 0xa9f: 0xe045, 0xaa0: 0x0008, 0xaa1: 0x0008, 0xaa2: 0x0008, 0xaa3: 0x0008, + 0xaa4: 0x0008, 0xaa5: 0x0008, 0xaa6: 0x0008, 0xaa7: 0x0008, 0xaa8: 0xe045, 0xaa9: 0xe045, + 0xaaa: 0xe045, 0xaab: 0xe045, 0xaac: 0xe045, 0xaad: 0xe045, 0xaae: 0xe045, 0xaaf: 0xe045, + 0xab0: 0x0008, 0xab1: 0x1459, 0xab2: 0x0008, 0xab3: 0x1471, 0xab4: 0x0008, 0xab5: 0x1489, + 0xab6: 0x0008, 0xab7: 0x14a1, 0xab8: 0x0008, 0xab9: 0x14b9, 0xaba: 0x0008, 0xabb: 0x14d1, + 0xabc: 0x0008, 0xabd: 0x14e9, 0xabe: 0x0040, 0xabf: 0x0040, + // Block 0x2b, offset 0xac0 + 0xac0: 0x1501, 0xac1: 0x1531, 0xac2: 0x1561, 0xac3: 0x1591, 0xac4: 0x15c1, 0xac5: 0x15f1, + 0xac6: 0x1621, 0xac7: 0x1651, 0xac8: 0x1501, 0xac9: 0x1531, 0xaca: 0x1561, 0xacb: 0x1591, + 0xacc: 0x15c1, 0xacd: 0x15f1, 0xace: 0x1621, 0xacf: 0x1651, 0xad0: 0x1681, 0xad1: 0x16b1, + 0xad2: 0x16e1, 0xad3: 0x1711, 0xad4: 0x1741, 0xad5: 0x1771, 0xad6: 0x17a1, 0xad7: 0x17d1, + 0xad8: 0x1681, 0xad9: 0x16b1, 0xada: 0x16e1, 0xadb: 0x1711, 0xadc: 0x1741, 0xadd: 0x1771, + 0xade: 0x17a1, 0xadf: 0x17d1, 0xae0: 0x1801, 0xae1: 0x1831, 0xae2: 0x1861, 0xae3: 0x1891, + 0xae4: 0x18c1, 0xae5: 0x18f1, 0xae6: 0x1921, 0xae7: 0x1951, 0xae8: 0x1801, 0xae9: 0x1831, + 0xaea: 0x1861, 0xaeb: 0x1891, 0xaec: 0x18c1, 0xaed: 0x18f1, 0xaee: 0x1921, 0xaef: 0x1951, + 0xaf0: 0x0008, 0xaf1: 0x0008, 0xaf2: 0x1981, 0xaf3: 0x19b1, 0xaf4: 0x19d9, 0xaf5: 0x0040, + 0xaf6: 0x0008, 0xaf7: 0x1a01, 0xaf8: 0xe045, 0xaf9: 0xe045, 0xafa: 0x064d, 0xafb: 0x1459, + 0xafc: 0x19b1, 0xafd: 0x0666, 0xafe: 0x1a31, 0xaff: 0x0686, + // Block 0x2c, offset 0xb00 + 0xb00: 0x06a6, 0xb01: 0x1a4a, 0xb02: 0x1a79, 0xb03: 0x1aa9, 0xb04: 0x1ad1, 0xb05: 0x0040, + 0xb06: 0x0008, 0xb07: 0x1af9, 0xb08: 0x06c5, 0xb09: 0x1471, 0xb0a: 0x06dd, 0xb0b: 0x1489, + 0xb0c: 0x1aa9, 0xb0d: 0x1b2a, 0xb0e: 0x1b5a, 0xb0f: 0x1b8a, 0xb10: 0x0008, 0xb11: 0x0008, + 0xb12: 0x0008, 0xb13: 0x1bb9, 0xb14: 0x0040, 0xb15: 0x0040, 0xb16: 0x0008, 0xb17: 0x0008, + 0xb18: 0xe045, 0xb19: 0xe045, 0xb1a: 0x06f5, 0xb1b: 0x14a1, 0xb1c: 0x0040, 0xb1d: 0x1bd2, + 0xb1e: 0x1c02, 0xb1f: 0x1c32, 0xb20: 0x0008, 0xb21: 0x0008, 0xb22: 0x0008, 0xb23: 0x1c61, + 0xb24: 0x0008, 0xb25: 0x0008, 0xb26: 0x0008, 0xb27: 0x0008, 0xb28: 0xe045, 0xb29: 0xe045, + 0xb2a: 0x070d, 0xb2b: 0x14d1, 0xb2c: 0xe04d, 0xb2d: 0x1c7a, 0xb2e: 0x03d2, 0xb2f: 0x1caa, + 0xb30: 0x0040, 0xb31: 0x0040, 0xb32: 0x1cb9, 0xb33: 0x1ce9, 0xb34: 0x1d11, 0xb35: 0x0040, + 0xb36: 0x0008, 0xb37: 0x1d39, 0xb38: 0x0725, 0xb39: 0x14b9, 0xb3a: 0x0515, 0xb3b: 0x14e9, + 0xb3c: 0x1ce9, 0xb3d: 0x073e, 0xb3e: 0x075e, 0xb3f: 0x0040, + // Block 0x2d, offset 0xb40 + 0xb40: 0x000a, 0xb41: 0x000a, 0xb42: 0x000a, 0xb43: 0x000a, 0xb44: 0x000a, 0xb45: 0x000a, + 0xb46: 0x000a, 0xb47: 0x000a, 0xb48: 0x000a, 0xb49: 0x000a, 0xb4a: 0x000a, 0xb4b: 0x03c0, + 0xb4c: 0x0003, 0xb4d: 0x0003, 0xb4e: 0x0340, 0xb4f: 0x0340, 0xb50: 0x0018, 0xb51: 0xe00d, + 0xb52: 0x0018, 0xb53: 0x0018, 0xb54: 0x0018, 0xb55: 0x0018, 0xb56: 0x0018, 0xb57: 0x077e, + 0xb58: 0x0018, 0xb59: 0x0018, 0xb5a: 0x0018, 0xb5b: 0x0018, 0xb5c: 0x0018, 0xb5d: 0x0018, + 0xb5e: 0x0018, 0xb5f: 0x0018, 0xb60: 0x0018, 0xb61: 0x0018, 0xb62: 0x0018, 0xb63: 0x0018, + 0xb64: 0x0040, 0xb65: 0x0040, 0xb66: 0x0040, 0xb67: 0x0018, 0xb68: 0x0040, 0xb69: 0x0040, + 0xb6a: 0x0340, 0xb6b: 0x0340, 0xb6c: 0x0340, 0xb6d: 0x0340, 0xb6e: 0x0340, 0xb6f: 0x000a, + 0xb70: 0x0018, 0xb71: 0x0018, 0xb72: 0x0018, 0xb73: 0x1d69, 0xb74: 0x1da1, 0xb75: 0x0018, + 0xb76: 0x1df1, 0xb77: 0x1e29, 0xb78: 0x0018, 0xb79: 0x0018, 0xb7a: 0x0018, 0xb7b: 0x0018, + 0xb7c: 0x1e7a, 0xb7d: 0x0018, 0xb7e: 0x079e, 0xb7f: 0x0018, + // Block 0x2e, offset 0xb80 + 0xb80: 0x0018, 0xb81: 0x0018, 0xb82: 0x0018, 0xb83: 0x0018, 0xb84: 0x0018, 0xb85: 0x0018, + 0xb86: 0x0018, 0xb87: 0x1e92, 0xb88: 0x1eaa, 0xb89: 0x1ec2, 0xb8a: 0x0018, 0xb8b: 0x0018, + 0xb8c: 0x0018, 0xb8d: 0x0018, 0xb8e: 0x0018, 0xb8f: 0x0018, 0xb90: 0x0018, 0xb91: 0x0018, + 0xb92: 0x0018, 0xb93: 0x0018, 0xb94: 0x0018, 0xb95: 0x0018, 0xb96: 0x0018, 0xb97: 0x1ed9, + 0xb98: 0x0018, 0xb99: 0x0018, 0xb9a: 0x0018, 0xb9b: 0x0018, 0xb9c: 0x0018, 0xb9d: 0x0018, + 0xb9e: 0x0018, 0xb9f: 0x000a, 0xba0: 0x03c0, 0xba1: 0x0340, 0xba2: 0x0340, 0xba3: 0x0340, + 0xba4: 0x03c0, 0xba5: 0x0040, 0xba6: 0x0040, 0xba7: 0x0040, 0xba8: 0x0040, 0xba9: 0x0040, + 0xbaa: 0x0340, 0xbab: 0x0340, 0xbac: 0x0340, 0xbad: 0x0340, 0xbae: 0x0340, 0xbaf: 0x0340, + 0xbb0: 0x1f41, 0xbb1: 0x0f41, 0xbb2: 0x0040, 0xbb3: 0x0040, 0xbb4: 0x1f51, 0xbb5: 0x1f61, + 0xbb6: 0x1f71, 0xbb7: 0x1f81, 0xbb8: 0x1f91, 0xbb9: 0x1fa1, 0xbba: 0x1fb2, 0xbbb: 0x07bd, + 0xbbc: 0x1fc2, 0xbbd: 0x1fd2, 0xbbe: 0x1fe2, 0xbbf: 0x0f71, + // Block 0x2f, offset 0xbc0 + 0xbc0: 0x1f41, 0xbc1: 0x00c9, 0xbc2: 0x0069, 0xbc3: 0x0079, 0xbc4: 0x1f51, 0xbc5: 0x1f61, + 0xbc6: 0x1f71, 0xbc7: 0x1f81, 0xbc8: 0x1f91, 0xbc9: 0x1fa1, 0xbca: 0x1fb2, 0xbcb: 0x07d5, + 0xbcc: 0x1fc2, 0xbcd: 0x1fd2, 0xbce: 0x1fe2, 0xbcf: 0x0040, 0xbd0: 0x0039, 0xbd1: 0x0f09, + 0xbd2: 0x00d9, 0xbd3: 0x0369, 0xbd4: 0x0ff9, 0xbd5: 0x0249, 0xbd6: 0x0f51, 0xbd7: 0x0359, + 0xbd8: 0x0f61, 0xbd9: 0x0f71, 0xbda: 0x0f99, 0xbdb: 0x01d9, 0xbdc: 0x0fa9, 0xbdd: 0x0040, + 0xbde: 0x0040, 0xbdf: 0x0040, 0xbe0: 0x0018, 0xbe1: 0x0018, 0xbe2: 0x0018, 0xbe3: 0x0018, + 0xbe4: 0x0018, 0xbe5: 0x0018, 0xbe6: 0x0018, 0xbe7: 0x0018, 0xbe8: 0x1ff1, 0xbe9: 0x0018, + 0xbea: 0x0018, 0xbeb: 0x0018, 0xbec: 0x0018, 0xbed: 0x0018, 0xbee: 0x0018, 0xbef: 0x0018, + 0xbf0: 0x0018, 0xbf1: 0x0018, 0xbf2: 0x0018, 0xbf3: 0x0018, 0xbf4: 0x0018, 0xbf5: 0x0018, + 0xbf6: 0x0018, 0xbf7: 0x0018, 0xbf8: 0x0018, 0xbf9: 0x0018, 0xbfa: 0x0018, 0xbfb: 0x0018, + 0xbfc: 0x0018, 0xbfd: 0x0018, 0xbfe: 0x0018, 0xbff: 0x0040, + // Block 0x30, offset 0xc00 + 0xc00: 0x07ee, 0xc01: 0x080e, 0xc02: 0x1159, 0xc03: 0x082d, 0xc04: 0x0018, 0xc05: 0x084e, + 0xc06: 0x086e, 0xc07: 0x1011, 0xc08: 0x0018, 0xc09: 0x088d, 0xc0a: 0x0f31, 0xc0b: 0x0249, + 0xc0c: 0x0249, 0xc0d: 0x0249, 0xc0e: 0x0249, 0xc0f: 0x2009, 0xc10: 0x0f41, 0xc11: 0x0f41, + 0xc12: 0x0359, 0xc13: 0x0359, 0xc14: 0x0018, 0xc15: 0x0f71, 0xc16: 0x2021, 0xc17: 0x0018, + 0xc18: 0x0018, 0xc19: 0x0f99, 0xc1a: 0x2039, 0xc1b: 0x0269, 0xc1c: 0x0269, 0xc1d: 0x0269, + 0xc1e: 0x0018, 0xc1f: 0x0018, 0xc20: 0x2049, 0xc21: 0x08ad, 0xc22: 0x2061, 0xc23: 0x0018, + 0xc24: 0x13d1, 0xc25: 0x0018, 0xc26: 0x2079, 0xc27: 0x0018, 0xc28: 0x13d1, 0xc29: 0x0018, + 0xc2a: 0x0f51, 0xc2b: 0x2091, 0xc2c: 0x0ee9, 0xc2d: 0x1159, 0xc2e: 0x0018, 0xc2f: 0x0f09, + 0xc30: 0x0f09, 0xc31: 0x1199, 0xc32: 0x0040, 0xc33: 0x0f61, 0xc34: 0x00d9, 0xc35: 0x20a9, + 0xc36: 0x20c1, 0xc37: 0x20d9, 0xc38: 0x20f1, 0xc39: 0x0f41, 0xc3a: 0x0018, 0xc3b: 0x08cd, + 0xc3c: 0x2109, 0xc3d: 0x10b1, 0xc3e: 0x10b1, 0xc3f: 0x2109, + // Block 0x31, offset 0xc40 + 0xc40: 0x08ed, 0xc41: 0x0018, 0xc42: 0x0018, 0xc43: 0x0018, 0xc44: 0x0018, 0xc45: 0x0ef9, + 0xc46: 0x0ef9, 0xc47: 0x0f09, 0xc48: 0x0f41, 0xc49: 0x0259, 0xc4a: 0x0018, 0xc4b: 0x0018, + 0xc4c: 0x0018, 0xc4d: 0x0018, 0xc4e: 0x0008, 0xc4f: 0x0018, 0xc50: 0x2121, 0xc51: 0x2151, + 0xc52: 0x2181, 0xc53: 0x21b9, 0xc54: 0x21e9, 0xc55: 0x2219, 0xc56: 0x2249, 0xc57: 0x2279, + 0xc58: 0x22a9, 0xc59: 0x22d9, 0xc5a: 0x2309, 0xc5b: 0x2339, 0xc5c: 0x2369, 0xc5d: 0x2399, + 0xc5e: 0x23c9, 0xc5f: 0x23f9, 0xc60: 0x0f41, 0xc61: 0x2421, 0xc62: 0x0905, 0xc63: 0x2439, + 0xc64: 0x1089, 0xc65: 0x2451, 0xc66: 0x0925, 0xc67: 0x2469, 0xc68: 0x2491, 0xc69: 0x0369, + 0xc6a: 0x24a9, 0xc6b: 0x0945, 0xc6c: 0x0359, 0xc6d: 0x1159, 0xc6e: 0x0ef9, 0xc6f: 0x0f61, + 0xc70: 0x0f41, 0xc71: 0x2421, 0xc72: 0x0965, 0xc73: 0x2439, 0xc74: 0x1089, 0xc75: 0x2451, + 0xc76: 0x0985, 0xc77: 0x2469, 0xc78: 0x2491, 0xc79: 0x0369, 0xc7a: 0x24a9, 0xc7b: 0x09a5, + 0xc7c: 0x0359, 0xc7d: 0x1159, 0xc7e: 0x0ef9, 0xc7f: 0x0f61, + // Block 0x32, offset 0xc80 + 0xc80: 0x0018, 0xc81: 0x0018, 0xc82: 0x0018, 0xc83: 0x0018, 0xc84: 0x0018, 0xc85: 0x0018, + 0xc86: 0x0018, 0xc87: 0x0018, 0xc88: 0x0018, 0xc89: 0x0018, 0xc8a: 0x0018, 0xc8b: 0x0040, + 0xc8c: 0x0040, 0xc8d: 0x0040, 0xc8e: 0x0040, 0xc8f: 0x0040, 0xc90: 0x0040, 0xc91: 0x0040, + 0xc92: 0x0040, 0xc93: 0x0040, 0xc94: 0x0040, 0xc95: 0x0040, 0xc96: 0x0040, 0xc97: 0x0040, + 0xc98: 0x0040, 0xc99: 0x0040, 0xc9a: 0x0040, 0xc9b: 0x0040, 0xc9c: 0x0040, 0xc9d: 0x0040, + 0xc9e: 0x0040, 0xc9f: 0x0040, 0xca0: 0x00c9, 0xca1: 0x0069, 0xca2: 0x0079, 0xca3: 0x1f51, + 0xca4: 0x1f61, 0xca5: 0x1f71, 0xca6: 0x1f81, 0xca7: 0x1f91, 0xca8: 0x1fa1, 0xca9: 0x2601, + 0xcaa: 0x2619, 0xcab: 0x2631, 0xcac: 0x2649, 0xcad: 0x2661, 0xcae: 0x2679, 0xcaf: 0x2691, + 0xcb0: 0x26a9, 0xcb1: 0x26c1, 0xcb2: 0x26d9, 0xcb3: 0x26f1, 0xcb4: 0x0a06, 0xcb5: 0x0a26, + 0xcb6: 0x0a46, 0xcb7: 0x0a66, 0xcb8: 0x0a86, 0xcb9: 0x0aa6, 0xcba: 0x0ac6, 0xcbb: 0x0ae6, + 0xcbc: 0x0b06, 0xcbd: 0x270a, 0xcbe: 0x2732, 0xcbf: 0x275a, + // Block 0x33, offset 0xcc0 + 0xcc0: 0x2782, 0xcc1: 0x27aa, 0xcc2: 0x27d2, 0xcc3: 0x27fa, 0xcc4: 0x2822, 0xcc5: 0x284a, + 0xcc6: 0x2872, 0xcc7: 0x289a, 0xcc8: 0x0040, 0xcc9: 0x0040, 0xcca: 0x0040, 0xccb: 0x0040, + 0xccc: 0x0040, 0xccd: 0x0040, 0xcce: 0x0040, 0xccf: 0x0040, 0xcd0: 0x0040, 0xcd1: 0x0040, + 0xcd2: 0x0040, 0xcd3: 0x0040, 0xcd4: 0x0040, 0xcd5: 0x0040, 0xcd6: 0x0040, 0xcd7: 0x0040, + 0xcd8: 0x0040, 0xcd9: 0x0040, 0xcda: 0x0040, 0xcdb: 0x0040, 0xcdc: 0x0b26, 0xcdd: 0x0b46, + 0xcde: 0x0b66, 0xcdf: 0x0b86, 0xce0: 0x0ba6, 0xce1: 0x0bc6, 0xce2: 0x0be6, 0xce3: 0x0c06, + 0xce4: 0x0c26, 0xce5: 0x0c46, 0xce6: 0x0c66, 0xce7: 0x0c86, 0xce8: 0x0ca6, 0xce9: 0x0cc6, + 0xcea: 0x0ce6, 0xceb: 0x0d06, 0xcec: 0x0d26, 0xced: 0x0d46, 0xcee: 0x0d66, 0xcef: 0x0d86, + 0xcf0: 0x0da6, 0xcf1: 0x0dc6, 0xcf2: 0x0de6, 0xcf3: 0x0e06, 0xcf4: 0x0e26, 0xcf5: 0x0e46, + 0xcf6: 0x0039, 0xcf7: 0x0ee9, 0xcf8: 0x1159, 0xcf9: 0x0ef9, 0xcfa: 0x0f09, 0xcfb: 0x1199, + 0xcfc: 0x0f31, 0xcfd: 0x0249, 0xcfe: 0x0f41, 0xcff: 0x0259, + // Block 0x34, offset 0xd00 + 0xd00: 0x0f51, 0xd01: 0x0359, 0xd02: 0x0f61, 0xd03: 0x0f71, 0xd04: 0x00d9, 0xd05: 0x0f99, + 0xd06: 0x2039, 0xd07: 0x0269, 0xd08: 0x01d9, 0xd09: 0x0fa9, 0xd0a: 0x0fb9, 0xd0b: 0x1089, + 0xd0c: 0x0279, 0xd0d: 0x0369, 0xd0e: 0x0289, 0xd0f: 0x13d1, 0xd10: 0x0039, 0xd11: 0x0ee9, + 0xd12: 0x1159, 0xd13: 0x0ef9, 0xd14: 0x0f09, 0xd15: 0x1199, 0xd16: 0x0f31, 0xd17: 0x0249, + 0xd18: 0x0f41, 0xd19: 0x0259, 0xd1a: 0x0f51, 0xd1b: 0x0359, 0xd1c: 0x0f61, 0xd1d: 0x0f71, + 0xd1e: 0x00d9, 0xd1f: 0x0f99, 0xd20: 0x2039, 0xd21: 0x0269, 0xd22: 0x01d9, 0xd23: 0x0fa9, + 0xd24: 0x0fb9, 0xd25: 0x1089, 0xd26: 0x0279, 0xd27: 0x0369, 0xd28: 0x0289, 0xd29: 0x13d1, + 0xd2a: 0x1f41, 0xd2b: 0x0018, 0xd2c: 0x0018, 0xd2d: 0x0018, 0xd2e: 0x0018, 0xd2f: 0x0018, + 0xd30: 0x0018, 0xd31: 0x0018, 0xd32: 0x0018, 0xd33: 0x0018, 0xd34: 0x0018, 0xd35: 0x0018, + 0xd36: 0x0018, 0xd37: 0x0018, 0xd38: 0x0018, 0xd39: 0x0018, 0xd3a: 0x0018, 0xd3b: 0x0018, + 0xd3c: 0x0018, 0xd3d: 0x0018, 0xd3e: 0x0018, 0xd3f: 0x0018, + // Block 0x35, offset 0xd40 + 0xd40: 0x0008, 0xd41: 0x0008, 0xd42: 0x0008, 0xd43: 0x0008, 0xd44: 0x0008, 0xd45: 0x0008, + 0xd46: 0x0008, 0xd47: 0x0008, 0xd48: 0x0008, 0xd49: 0x0008, 0xd4a: 0x0008, 0xd4b: 0x0008, + 0xd4c: 0x0008, 0xd4d: 0x0008, 0xd4e: 0x0008, 0xd4f: 0x0008, 0xd50: 0x0008, 0xd51: 0x0008, + 0xd52: 0x0008, 0xd53: 0x0008, 0xd54: 0x0008, 0xd55: 0x0008, 0xd56: 0x0008, 0xd57: 0x0008, + 0xd58: 0x0008, 0xd59: 0x0008, 0xd5a: 0x0008, 0xd5b: 0x0008, 0xd5c: 0x0008, 0xd5d: 0x0008, + 0xd5e: 0x0008, 0xd5f: 0x0040, 0xd60: 0xe00d, 0xd61: 0x0008, 0xd62: 0x2971, 0xd63: 0x0ebd, + 0xd64: 0x2989, 0xd65: 0x0008, 0xd66: 0x0008, 0xd67: 0xe07d, 0xd68: 0x0008, 0xd69: 0xe01d, + 0xd6a: 0x0008, 0xd6b: 0xe03d, 0xd6c: 0x0008, 0xd6d: 0x0fe1, 0xd6e: 0x1281, 0xd6f: 0x0fc9, + 0xd70: 0x1141, 0xd71: 0x0008, 0xd72: 0xe00d, 0xd73: 0x0008, 0xd74: 0x0008, 0xd75: 0xe01d, + 0xd76: 0x0008, 0xd77: 0x0008, 0xd78: 0x0008, 0xd79: 0x0008, 0xd7a: 0x0008, 0xd7b: 0x0008, + 0xd7c: 0x0259, 0xd7d: 0x1089, 0xd7e: 0x29a1, 0xd7f: 0x29b9, + // Block 0x36, offset 0xd80 + 0xd80: 0xe00d, 0xd81: 0x0008, 0xd82: 0xe00d, 0xd83: 0x0008, 0xd84: 0xe00d, 0xd85: 0x0008, + 0xd86: 0xe00d, 0xd87: 0x0008, 0xd88: 0xe00d, 0xd89: 0x0008, 0xd8a: 0xe00d, 0xd8b: 0x0008, + 0xd8c: 0xe00d, 0xd8d: 0x0008, 0xd8e: 0xe00d, 0xd8f: 0x0008, 0xd90: 0xe00d, 0xd91: 0x0008, + 0xd92: 0xe00d, 0xd93: 0x0008, 0xd94: 0xe00d, 0xd95: 0x0008, 0xd96: 0xe00d, 0xd97: 0x0008, + 0xd98: 0xe00d, 0xd99: 0x0008, 0xd9a: 0xe00d, 0xd9b: 0x0008, 0xd9c: 0xe00d, 0xd9d: 0x0008, + 0xd9e: 0xe00d, 0xd9f: 0x0008, 0xda0: 0xe00d, 0xda1: 0x0008, 0xda2: 0xe00d, 0xda3: 0x0008, + 0xda4: 0x0008, 0xda5: 0x0018, 0xda6: 0x0018, 0xda7: 0x0018, 0xda8: 0x0018, 0xda9: 0x0018, + 0xdaa: 0x0018, 0xdab: 0xe03d, 0xdac: 0x0008, 0xdad: 0xe01d, 0xdae: 0x0008, 0xdaf: 0x1308, + 0xdb0: 0x1308, 0xdb1: 0x1308, 0xdb2: 0xe00d, 0xdb3: 0x0008, 0xdb4: 0x0040, 0xdb5: 0x0040, + 0xdb6: 0x0040, 0xdb7: 0x0040, 0xdb8: 0x0040, 0xdb9: 0x0018, 0xdba: 0x0018, 0xdbb: 0x0018, + 0xdbc: 0x0018, 0xdbd: 0x0018, 0xdbe: 0x0018, 0xdbf: 0x0018, + // Block 0x37, offset 0xdc0 + 0xdc0: 0x26fd, 0xdc1: 0x271d, 0xdc2: 0x273d, 0xdc3: 0x275d, 0xdc4: 0x277d, 0xdc5: 0x279d, + 0xdc6: 0x27bd, 0xdc7: 0x27dd, 0xdc8: 0x27fd, 0xdc9: 0x281d, 0xdca: 0x283d, 0xdcb: 0x285d, + 0xdcc: 0x287d, 0xdcd: 0x289d, 0xdce: 0x28bd, 0xdcf: 0x28dd, 0xdd0: 0x28fd, 0xdd1: 0x291d, + 0xdd2: 0x293d, 0xdd3: 0x295d, 0xdd4: 0x297d, 0xdd5: 0x299d, 0xdd6: 0x0040, 0xdd7: 0x0040, + 0xdd8: 0x0040, 0xdd9: 0x0040, 0xdda: 0x0040, 0xddb: 0x0040, 0xddc: 0x0040, 0xddd: 0x0040, + 0xdde: 0x0040, 0xddf: 0x0040, 0xde0: 0x0040, 0xde1: 0x0040, 0xde2: 0x0040, 0xde3: 0x0040, + 0xde4: 0x0040, 0xde5: 0x0040, 0xde6: 0x0040, 0xde7: 0x0040, 0xde8: 0x0040, 0xde9: 0x0040, + 0xdea: 0x0040, 0xdeb: 0x0040, 0xdec: 0x0040, 0xded: 0x0040, 0xdee: 0x0040, 0xdef: 0x0040, + 0xdf0: 0x0040, 0xdf1: 0x0040, 0xdf2: 0x0040, 0xdf3: 0x0040, 0xdf4: 0x0040, 0xdf5: 0x0040, + 0xdf6: 0x0040, 0xdf7: 0x0040, 0xdf8: 0x0040, 0xdf9: 0x0040, 0xdfa: 0x0040, 0xdfb: 0x0040, + 0xdfc: 0x0040, 0xdfd: 0x0040, 0xdfe: 0x0040, 0xdff: 0x0040, + // Block 0x38, offset 0xe00 + 0xe00: 0x000a, 0xe01: 0x0018, 0xe02: 0x29d1, 0xe03: 0x0018, 0xe04: 0x0018, 0xe05: 0x0008, + 0xe06: 0x0008, 0xe07: 0x0008, 0xe08: 0x0018, 0xe09: 0x0018, 0xe0a: 0x0018, 0xe0b: 0x0018, + 0xe0c: 0x0018, 0xe0d: 0x0018, 0xe0e: 0x0018, 0xe0f: 0x0018, 0xe10: 0x0018, 0xe11: 0x0018, + 0xe12: 0x0018, 0xe13: 0x0018, 0xe14: 0x0018, 0xe15: 0x0018, 0xe16: 0x0018, 0xe17: 0x0018, + 0xe18: 0x0018, 0xe19: 0x0018, 0xe1a: 0x0018, 0xe1b: 0x0018, 0xe1c: 0x0018, 0xe1d: 0x0018, + 0xe1e: 0x0018, 0xe1f: 0x0018, 0xe20: 0x0018, 0xe21: 0x0018, 0xe22: 0x0018, 0xe23: 0x0018, + 0xe24: 0x0018, 0xe25: 0x0018, 0xe26: 0x0018, 0xe27: 0x0018, 0xe28: 0x0018, 0xe29: 0x0018, + 0xe2a: 0x1308, 0xe2b: 0x1308, 0xe2c: 0x1308, 0xe2d: 0x1308, 0xe2e: 0x1018, 0xe2f: 0x1018, + 0xe30: 0x0018, 0xe31: 0x0018, 0xe32: 0x0018, 0xe33: 0x0018, 0xe34: 0x0018, 0xe35: 0x0018, + 0xe36: 0xe125, 0xe37: 0x0018, 0xe38: 0x29bd, 0xe39: 0x29dd, 0xe3a: 0x29fd, 0xe3b: 0x0018, + 0xe3c: 0x0008, 0xe3d: 0x0018, 0xe3e: 0x0018, 0xe3f: 0x0018, + // Block 0x39, offset 0xe40 + 0xe40: 0x2b3d, 0xe41: 0x2b5d, 0xe42: 0x2b7d, 0xe43: 0x2b9d, 0xe44: 0x2bbd, 0xe45: 0x2bdd, + 0xe46: 0x2bdd, 0xe47: 0x2bdd, 0xe48: 0x2bfd, 0xe49: 0x2bfd, 0xe4a: 0x2bfd, 0xe4b: 0x2bfd, + 0xe4c: 0x2c1d, 0xe4d: 0x2c1d, 0xe4e: 0x2c1d, 0xe4f: 0x2c3d, 0xe50: 0x2c5d, 0xe51: 0x2c5d, + 0xe52: 0x2a7d, 0xe53: 0x2a7d, 0xe54: 0x2c5d, 0xe55: 0x2c5d, 0xe56: 0x2c7d, 0xe57: 0x2c7d, + 0xe58: 0x2c5d, 0xe59: 0x2c5d, 0xe5a: 0x2a7d, 0xe5b: 0x2a7d, 0xe5c: 0x2c5d, 0xe5d: 0x2c5d, + 0xe5e: 0x2c3d, 0xe5f: 0x2c3d, 0xe60: 0x2c9d, 0xe61: 0x2c9d, 0xe62: 0x2cbd, 0xe63: 0x2cbd, + 0xe64: 0x0040, 0xe65: 0x2cdd, 0xe66: 0x2cfd, 0xe67: 0x2d1d, 0xe68: 0x2d1d, 0xe69: 0x2d3d, + 0xe6a: 0x2d5d, 0xe6b: 0x2d7d, 0xe6c: 0x2d9d, 0xe6d: 0x2dbd, 0xe6e: 0x2ddd, 0xe6f: 0x2dfd, + 0xe70: 0x2e1d, 0xe71: 0x2e3d, 0xe72: 0x2e3d, 0xe73: 0x2e5d, 0xe74: 0x2e7d, 0xe75: 0x2e7d, + 0xe76: 0x2e9d, 0xe77: 0x2ebd, 0xe78: 0x2e5d, 0xe79: 0x2edd, 0xe7a: 0x2efd, 0xe7b: 0x2edd, + 0xe7c: 0x2e5d, 0xe7d: 0x2f1d, 0xe7e: 0x2f3d, 0xe7f: 0x2f5d, + // Block 0x3a, offset 0xe80 + 0xe80: 0x2f7d, 0xe81: 0x2f9d, 0xe82: 0x2cfd, 0xe83: 0x2cdd, 0xe84: 0x2fbd, 0xe85: 0x2fdd, + 0xe86: 0x2ffd, 0xe87: 0x301d, 0xe88: 0x303d, 0xe89: 0x305d, 0xe8a: 0x307d, 0xe8b: 0x309d, + 0xe8c: 0x30bd, 0xe8d: 0x30dd, 0xe8e: 0x30fd, 0xe8f: 0x0040, 0xe90: 0x0018, 0xe91: 0x0018, + 0xe92: 0x311d, 0xe93: 0x313d, 0xe94: 0x315d, 0xe95: 0x317d, 0xe96: 0x319d, 0xe97: 0x31bd, + 0xe98: 0x31dd, 0xe99: 0x31fd, 0xe9a: 0x321d, 0xe9b: 0x323d, 0xe9c: 0x315d, 0xe9d: 0x325d, + 0xe9e: 0x327d, 0xe9f: 0x329d, 0xea0: 0x0008, 0xea1: 0x0008, 0xea2: 0x0008, 0xea3: 0x0008, + 0xea4: 0x0008, 0xea5: 0x0008, 0xea6: 0x0008, 0xea7: 0x0008, 0xea8: 0x0008, 0xea9: 0x0008, + 0xeaa: 0x0008, 0xeab: 0x0008, 0xeac: 0x0008, 0xead: 0x0008, 0xeae: 0x0008, 0xeaf: 0x0008, + 0xeb0: 0x0008, 0xeb1: 0x0008, 0xeb2: 0x0008, 0xeb3: 0x0008, 0xeb4: 0x0008, 0xeb5: 0x0008, + 0xeb6: 0x0008, 0xeb7: 0x0008, 0xeb8: 0x0008, 0xeb9: 0x0008, 0xeba: 0x0008, 0xebb: 0x0040, + 0xebc: 0x0040, 0xebd: 0x0040, 0xebe: 0x0040, 0xebf: 0x0040, + // Block 0x3b, offset 0xec0 + 0xec0: 0x36a2, 0xec1: 0x36d2, 0xec2: 0x3702, 0xec3: 0x3732, 0xec4: 0x32bd, 0xec5: 0x32dd, + 0xec6: 0x32fd, 0xec7: 0x331d, 0xec8: 0x0018, 0xec9: 0x0018, 0xeca: 0x0018, 0xecb: 0x0018, + 0xecc: 0x0018, 0xecd: 0x0018, 0xece: 0x0018, 0xecf: 0x0018, 0xed0: 0x333d, 0xed1: 0x3761, + 0xed2: 0x3779, 0xed3: 0x3791, 0xed4: 0x37a9, 0xed5: 0x37c1, 0xed6: 0x37d9, 0xed7: 0x37f1, + 0xed8: 0x3809, 0xed9: 0x3821, 0xeda: 0x3839, 0xedb: 0x3851, 0xedc: 0x3869, 0xedd: 0x3881, + 0xede: 0x3899, 0xedf: 0x38b1, 0xee0: 0x335d, 0xee1: 0x337d, 0xee2: 0x339d, 0xee3: 0x33bd, + 0xee4: 0x33dd, 0xee5: 0x33dd, 0xee6: 0x33fd, 0xee7: 0x341d, 0xee8: 0x343d, 0xee9: 0x345d, + 0xeea: 0x347d, 0xeeb: 0x349d, 0xeec: 0x34bd, 0xeed: 0x34dd, 0xeee: 0x34fd, 0xeef: 0x351d, + 0xef0: 0x353d, 0xef1: 0x355d, 0xef2: 0x357d, 0xef3: 0x359d, 0xef4: 0x35bd, 0xef5: 0x35dd, + 0xef6: 0x35fd, 0xef7: 0x361d, 0xef8: 0x363d, 0xef9: 0x365d, 0xefa: 0x367d, 0xefb: 0x369d, + 0xefc: 0x38c9, 0xefd: 0x3901, 0xefe: 0x36bd, 0xeff: 0x0018, + // Block 0x3c, offset 0xf00 + 0xf00: 0x36dd, 0xf01: 0x36fd, 0xf02: 0x371d, 0xf03: 0x373d, 0xf04: 0x375d, 0xf05: 0x377d, + 0xf06: 0x379d, 0xf07: 0x37bd, 0xf08: 0x37dd, 0xf09: 0x37fd, 0xf0a: 0x381d, 0xf0b: 0x383d, + 0xf0c: 0x385d, 0xf0d: 0x387d, 0xf0e: 0x389d, 0xf0f: 0x38bd, 0xf10: 0x38dd, 0xf11: 0x38fd, + 0xf12: 0x391d, 0xf13: 0x393d, 0xf14: 0x395d, 0xf15: 0x397d, 0xf16: 0x399d, 0xf17: 0x39bd, + 0xf18: 0x39dd, 0xf19: 0x39fd, 0xf1a: 0x3a1d, 0xf1b: 0x3a3d, 0xf1c: 0x3a5d, 0xf1d: 0x3a7d, + 0xf1e: 0x3a9d, 0xf1f: 0x3abd, 0xf20: 0x3add, 0xf21: 0x3afd, 0xf22: 0x3b1d, 0xf23: 0x3b3d, + 0xf24: 0x3b5d, 0xf25: 0x3b7d, 0xf26: 0x127d, 0xf27: 0x3b9d, 0xf28: 0x3bbd, 0xf29: 0x3bdd, + 0xf2a: 0x3bfd, 0xf2b: 0x3c1d, 0xf2c: 0x3c3d, 0xf2d: 0x3c5d, 0xf2e: 0x239d, 0xf2f: 0x3c7d, + 0xf30: 0x3c9d, 0xf31: 0x3939, 0xf32: 0x3951, 0xf33: 0x3969, 0xf34: 0x3981, 0xf35: 0x3999, + 0xf36: 0x39b1, 0xf37: 0x39c9, 0xf38: 0x39e1, 0xf39: 0x39f9, 0xf3a: 0x3a11, 0xf3b: 0x3a29, + 0xf3c: 0x3a41, 0xf3d: 0x3a59, 0xf3e: 0x3a71, 0xf3f: 0x3a89, + // Block 0x3d, offset 0xf40 + 0xf40: 0x3aa1, 0xf41: 0x3ac9, 0xf42: 0x3af1, 0xf43: 0x3b19, 0xf44: 0x3b41, 0xf45: 0x3b69, + 0xf46: 0x3b91, 0xf47: 0x3bb9, 0xf48: 0x3be1, 0xf49: 0x3c09, 0xf4a: 0x3c39, 0xf4b: 0x3c69, + 0xf4c: 0x3c99, 0xf4d: 0x3cbd, 0xf4e: 0x3cb1, 0xf4f: 0x3cdd, 0xf50: 0x3cfd, 0xf51: 0x3d15, + 0xf52: 0x3d2d, 0xf53: 0x3d45, 0xf54: 0x3d5d, 0xf55: 0x3d5d, 0xf56: 0x3d45, 0xf57: 0x3d75, + 0xf58: 0x07bd, 0xf59: 0x3d8d, 0xf5a: 0x3da5, 0xf5b: 0x3dbd, 0xf5c: 0x3dd5, 0xf5d: 0x3ded, + 0xf5e: 0x3e05, 0xf5f: 0x3e1d, 0xf60: 0x3e35, 0xf61: 0x3e4d, 0xf62: 0x3e65, 0xf63: 0x3e7d, + 0xf64: 0x3e95, 0xf65: 0x3e95, 0xf66: 0x3ead, 0xf67: 0x3ead, 0xf68: 0x3ec5, 0xf69: 0x3ec5, + 0xf6a: 0x3edd, 0xf6b: 0x3ef5, 0xf6c: 0x3f0d, 0xf6d: 0x3f25, 0xf6e: 0x3f3d, 0xf6f: 0x3f3d, + 0xf70: 0x3f55, 0xf71: 0x3f55, 0xf72: 0x3f55, 0xf73: 0x3f6d, 0xf74: 0x3f85, 0xf75: 0x3f9d, + 0xf76: 0x3fb5, 0xf77: 0x3f9d, 0xf78: 0x3fcd, 0xf79: 0x3fe5, 0xf7a: 0x3f6d, 0xf7b: 0x3ffd, + 0xf7c: 0x4015, 0xf7d: 0x4015, 0xf7e: 0x4015, 0xf7f: 0x0040, + // Block 0x3e, offset 0xf80 + 0xf80: 0x3cc9, 0xf81: 0x3d31, 0xf82: 0x3d99, 0xf83: 0x3e01, 0xf84: 0x3e51, 0xf85: 0x3eb9, + 0xf86: 0x3f09, 0xf87: 0x3f59, 0xf88: 0x3fd9, 0xf89: 0x4041, 0xf8a: 0x4091, 0xf8b: 0x40e1, + 0xf8c: 0x4131, 0xf8d: 0x4199, 0xf8e: 0x4201, 0xf8f: 0x4251, 0xf90: 0x42a1, 0xf91: 0x42d9, + 0xf92: 0x4329, 0xf93: 0x4391, 0xf94: 0x43f9, 0xf95: 0x4431, 0xf96: 0x44b1, 0xf97: 0x4549, + 0xf98: 0x45c9, 0xf99: 0x4619, 0xf9a: 0x4699, 0xf9b: 0x4719, 0xf9c: 0x4781, 0xf9d: 0x47d1, + 0xf9e: 0x4821, 0xf9f: 0x4871, 0xfa0: 0x48d9, 0xfa1: 0x4959, 0xfa2: 0x49c1, 0xfa3: 0x4a11, + 0xfa4: 0x4a61, 0xfa5: 0x4ab1, 0xfa6: 0x4ae9, 0xfa7: 0x4b21, 0xfa8: 0x4b59, 0xfa9: 0x4b91, + 0xfaa: 0x4be1, 0xfab: 0x4c31, 0xfac: 0x4cb1, 0xfad: 0x4d01, 0xfae: 0x4d69, 0xfaf: 0x4de9, + 0xfb0: 0x4e39, 0xfb1: 0x4e71, 0xfb2: 0x4ea9, 0xfb3: 0x4f29, 0xfb4: 0x4f91, 0xfb5: 0x5011, + 0xfb6: 0x5061, 0xfb7: 0x50e1, 0xfb8: 0x5119, 0xfb9: 0x5169, 0xfba: 0x51b9, 0xfbb: 0x5209, + 0xfbc: 0x5259, 0xfbd: 0x52a9, 0xfbe: 0x5311, 0xfbf: 0x5361, + // Block 0x3f, offset 0xfc0 + 0xfc0: 0x5399, 0xfc1: 0x53e9, 0xfc2: 0x5439, 0xfc3: 0x5489, 0xfc4: 0x54f1, 0xfc5: 0x5541, + 0xfc6: 0x5591, 0xfc7: 0x55e1, 0xfc8: 0x5661, 0xfc9: 0x56c9, 0xfca: 0x5701, 0xfcb: 0x5781, + 0xfcc: 0x57b9, 0xfcd: 0x5821, 0xfce: 0x5889, 0xfcf: 0x58d9, 0xfd0: 0x5929, 0xfd1: 0x5979, + 0xfd2: 0x59e1, 0xfd3: 0x5a19, 0xfd4: 0x5a69, 0xfd5: 0x5ad1, 0xfd6: 0x5b09, 0xfd7: 0x5b89, + 0xfd8: 0x5bd9, 0xfd9: 0x5c01, 0xfda: 0x5c29, 0xfdb: 0x5c51, 0xfdc: 0x5c79, 0xfdd: 0x5ca1, + 0xfde: 0x5cc9, 0xfdf: 0x5cf1, 0xfe0: 0x5d19, 0xfe1: 0x5d41, 0xfe2: 0x5d69, 0xfe3: 0x5d99, + 0xfe4: 0x5dc9, 0xfe5: 0x5df9, 0xfe6: 0x5e29, 0xfe7: 0x5e59, 0xfe8: 0x5e89, 0xfe9: 0x5eb9, + 0xfea: 0x5ee9, 0xfeb: 0x5f19, 0xfec: 0x5f49, 0xfed: 0x5f79, 0xfee: 0x5fa9, 0xfef: 0x5fd9, + 0xff0: 0x6009, 0xff1: 0x402d, 0xff2: 0x6039, 0xff3: 0x6051, 0xff4: 0x404d, 0xff5: 0x6069, + 0xff6: 0x6081, 0xff7: 0x6099, 0xff8: 0x406d, 0xff9: 0x406d, 0xffa: 0x60b1, 0xffb: 0x60c9, + 0xffc: 0x6101, 0xffd: 0x6139, 0xffe: 0x6171, 0xfff: 0x61a9, + // Block 0x40, offset 0x1000 + 0x1000: 0x6211, 0x1001: 0x6229, 0x1002: 0x408d, 0x1003: 0x6241, 0x1004: 0x6259, 0x1005: 0x6271, + 0x1006: 0x6289, 0x1007: 0x62a1, 0x1008: 0x40ad, 0x1009: 0x62b9, 0x100a: 0x62e1, 0x100b: 0x62f9, + 0x100c: 0x40cd, 0x100d: 0x40cd, 0x100e: 0x6311, 0x100f: 0x6329, 0x1010: 0x6341, 0x1011: 0x40ed, + 0x1012: 0x410d, 0x1013: 0x412d, 0x1014: 0x414d, 0x1015: 0x416d, 0x1016: 0x6359, 0x1017: 0x6371, + 0x1018: 0x6389, 0x1019: 0x63a1, 0x101a: 0x63b9, 0x101b: 0x418d, 0x101c: 0x63d1, 0x101d: 0x63e9, + 0x101e: 0x6401, 0x101f: 0x41ad, 0x1020: 0x41cd, 0x1021: 0x6419, 0x1022: 0x41ed, 0x1023: 0x420d, + 0x1024: 0x422d, 0x1025: 0x6431, 0x1026: 0x424d, 0x1027: 0x6449, 0x1028: 0x6479, 0x1029: 0x6211, + 0x102a: 0x426d, 0x102b: 0x428d, 0x102c: 0x42ad, 0x102d: 0x42cd, 0x102e: 0x64b1, 0x102f: 0x64f1, + 0x1030: 0x6539, 0x1031: 0x6551, 0x1032: 0x42ed, 0x1033: 0x6569, 0x1034: 0x6581, 0x1035: 0x6599, + 0x1036: 0x430d, 0x1037: 0x65b1, 0x1038: 0x65c9, 0x1039: 0x65b1, 0x103a: 0x65e1, 0x103b: 0x65f9, + 0x103c: 0x432d, 0x103d: 0x6611, 0x103e: 0x6629, 0x103f: 0x6611, + // Block 0x41, offset 0x1040 + 0x1040: 0x434d, 0x1041: 0x436d, 0x1042: 0x0040, 0x1043: 0x6641, 0x1044: 0x6659, 0x1045: 0x6671, + 0x1046: 0x6689, 0x1047: 0x0040, 0x1048: 0x66c1, 0x1049: 0x66d9, 0x104a: 0x66f1, 0x104b: 0x6709, + 0x104c: 0x6721, 0x104d: 0x6739, 0x104e: 0x6401, 0x104f: 0x6751, 0x1050: 0x6769, 0x1051: 0x6781, + 0x1052: 0x438d, 0x1053: 0x6799, 0x1054: 0x6289, 0x1055: 0x43ad, 0x1056: 0x43cd, 0x1057: 0x67b1, + 0x1058: 0x0040, 0x1059: 0x43ed, 0x105a: 0x67c9, 0x105b: 0x67e1, 0x105c: 0x67f9, 0x105d: 0x6811, + 0x105e: 0x6829, 0x105f: 0x6859, 0x1060: 0x6889, 0x1061: 0x68b1, 0x1062: 0x68d9, 0x1063: 0x6901, + 0x1064: 0x6929, 0x1065: 0x6951, 0x1066: 0x6979, 0x1067: 0x69a1, 0x1068: 0x69c9, 0x1069: 0x69f1, + 0x106a: 0x6a21, 0x106b: 0x6a51, 0x106c: 0x6a81, 0x106d: 0x6ab1, 0x106e: 0x6ae1, 0x106f: 0x6b11, + 0x1070: 0x6b41, 0x1071: 0x6b71, 0x1072: 0x6ba1, 0x1073: 0x6bd1, 0x1074: 0x6c01, 0x1075: 0x6c31, + 0x1076: 0x6c61, 0x1077: 0x6c91, 0x1078: 0x6cc1, 0x1079: 0x6cf1, 0x107a: 0x6d21, 0x107b: 0x6d51, + 0x107c: 0x6d81, 0x107d: 0x6db1, 0x107e: 0x6de1, 0x107f: 0x440d, + // Block 0x42, offset 0x1080 + 0x1080: 0xe00d, 0x1081: 0x0008, 0x1082: 0xe00d, 0x1083: 0x0008, 0x1084: 0xe00d, 0x1085: 0x0008, + 0x1086: 0xe00d, 0x1087: 0x0008, 0x1088: 0xe00d, 0x1089: 0x0008, 0x108a: 0xe00d, 0x108b: 0x0008, + 0x108c: 0xe00d, 0x108d: 0x0008, 0x108e: 0xe00d, 0x108f: 0x0008, 0x1090: 0xe00d, 0x1091: 0x0008, + 0x1092: 0xe00d, 0x1093: 0x0008, 0x1094: 0xe00d, 0x1095: 0x0008, 0x1096: 0xe00d, 0x1097: 0x0008, + 0x1098: 0xe00d, 0x1099: 0x0008, 0x109a: 0xe00d, 0x109b: 0x0008, 0x109c: 0xe00d, 0x109d: 0x0008, + 0x109e: 0xe00d, 0x109f: 0x0008, 0x10a0: 0xe00d, 0x10a1: 0x0008, 0x10a2: 0xe00d, 0x10a3: 0x0008, + 0x10a4: 0xe00d, 0x10a5: 0x0008, 0x10a6: 0xe00d, 0x10a7: 0x0008, 0x10a8: 0xe00d, 0x10a9: 0x0008, + 0x10aa: 0xe00d, 0x10ab: 0x0008, 0x10ac: 0xe00d, 0x10ad: 0x0008, 0x10ae: 0x0008, 0x10af: 0x1308, + 0x10b0: 0x1318, 0x10b1: 0x1318, 0x10b2: 0x1318, 0x10b3: 0x0018, 0x10b4: 0x1308, 0x10b5: 0x1308, + 0x10b6: 0x1308, 0x10b7: 0x1308, 0x10b8: 0x1308, 0x10b9: 0x1308, 0x10ba: 0x1308, 0x10bb: 0x1308, + 0x10bc: 0x1308, 0x10bd: 0x1308, 0x10be: 0x0018, 0x10bf: 0x0008, + // Block 0x43, offset 0x10c0 + 0x10c0: 0xe00d, 0x10c1: 0x0008, 0x10c2: 0xe00d, 0x10c3: 0x0008, 0x10c4: 0xe00d, 0x10c5: 0x0008, + 0x10c6: 0xe00d, 0x10c7: 0x0008, 0x10c8: 0xe00d, 0x10c9: 0x0008, 0x10ca: 0xe00d, 0x10cb: 0x0008, + 0x10cc: 0xe00d, 0x10cd: 0x0008, 0x10ce: 0xe00d, 0x10cf: 0x0008, 0x10d0: 0xe00d, 0x10d1: 0x0008, + 0x10d2: 0xe00d, 0x10d3: 0x0008, 0x10d4: 0xe00d, 0x10d5: 0x0008, 0x10d6: 0xe00d, 0x10d7: 0x0008, + 0x10d8: 0xe00d, 0x10d9: 0x0008, 0x10da: 0xe00d, 0x10db: 0x0008, 0x10dc: 0x0ea1, 0x10dd: 0x6e11, + 0x10de: 0x1308, 0x10df: 0x1308, 0x10e0: 0x0008, 0x10e1: 0x0008, 0x10e2: 0x0008, 0x10e3: 0x0008, + 0x10e4: 0x0008, 0x10e5: 0x0008, 0x10e6: 0x0008, 0x10e7: 0x0008, 0x10e8: 0x0008, 0x10e9: 0x0008, + 0x10ea: 0x0008, 0x10eb: 0x0008, 0x10ec: 0x0008, 0x10ed: 0x0008, 0x10ee: 0x0008, 0x10ef: 0x0008, + 0x10f0: 0x0008, 0x10f1: 0x0008, 0x10f2: 0x0008, 0x10f3: 0x0008, 0x10f4: 0x0008, 0x10f5: 0x0008, + 0x10f6: 0x0008, 0x10f7: 0x0008, 0x10f8: 0x0008, 0x10f9: 0x0008, 0x10fa: 0x0008, 0x10fb: 0x0008, + 0x10fc: 0x0008, 0x10fd: 0x0008, 0x10fe: 0x0008, 0x10ff: 0x0008, + // Block 0x44, offset 0x1100 + 0x1100: 0x0018, 0x1101: 0x0018, 0x1102: 0x0018, 0x1103: 0x0018, 0x1104: 0x0018, 0x1105: 0x0018, + 0x1106: 0x0018, 0x1107: 0x0018, 0x1108: 0x0018, 0x1109: 0x0018, 0x110a: 0x0018, 0x110b: 0x0018, + 0x110c: 0x0018, 0x110d: 0x0018, 0x110e: 0x0018, 0x110f: 0x0018, 0x1110: 0x0018, 0x1111: 0x0018, + 0x1112: 0x0018, 0x1113: 0x0018, 0x1114: 0x0018, 0x1115: 0x0018, 0x1116: 0x0018, 0x1117: 0x0008, + 0x1118: 0x0008, 0x1119: 0x0008, 0x111a: 0x0008, 0x111b: 0x0008, 0x111c: 0x0008, 0x111d: 0x0008, + 0x111e: 0x0008, 0x111f: 0x0008, 0x1120: 0x0018, 0x1121: 0x0018, 0x1122: 0xe00d, 0x1123: 0x0008, + 0x1124: 0xe00d, 0x1125: 0x0008, 0x1126: 0xe00d, 0x1127: 0x0008, 0x1128: 0xe00d, 0x1129: 0x0008, + 0x112a: 0xe00d, 0x112b: 0x0008, 0x112c: 0xe00d, 0x112d: 0x0008, 0x112e: 0xe00d, 0x112f: 0x0008, + 0x1130: 0x0008, 0x1131: 0x0008, 0x1132: 0xe00d, 0x1133: 0x0008, 0x1134: 0xe00d, 0x1135: 0x0008, + 0x1136: 0xe00d, 0x1137: 0x0008, 0x1138: 0xe00d, 0x1139: 0x0008, 0x113a: 0xe00d, 0x113b: 0x0008, + 0x113c: 0xe00d, 0x113d: 0x0008, 0x113e: 0xe00d, 0x113f: 0x0008, + // Block 0x45, offset 0x1140 + 0x1140: 0xe00d, 0x1141: 0x0008, 0x1142: 0xe00d, 0x1143: 0x0008, 0x1144: 0xe00d, 0x1145: 0x0008, + 0x1146: 0xe00d, 0x1147: 0x0008, 0x1148: 0xe00d, 0x1149: 0x0008, 0x114a: 0xe00d, 0x114b: 0x0008, + 0x114c: 0xe00d, 0x114d: 0x0008, 0x114e: 0xe00d, 0x114f: 0x0008, 0x1150: 0xe00d, 0x1151: 0x0008, + 0x1152: 0xe00d, 0x1153: 0x0008, 0x1154: 0xe00d, 0x1155: 0x0008, 0x1156: 0xe00d, 0x1157: 0x0008, + 0x1158: 0xe00d, 0x1159: 0x0008, 0x115a: 0xe00d, 0x115b: 0x0008, 0x115c: 0xe00d, 0x115d: 0x0008, + 0x115e: 0xe00d, 0x115f: 0x0008, 0x1160: 0xe00d, 0x1161: 0x0008, 0x1162: 0xe00d, 0x1163: 0x0008, + 0x1164: 0xe00d, 0x1165: 0x0008, 0x1166: 0xe00d, 0x1167: 0x0008, 0x1168: 0xe00d, 0x1169: 0x0008, + 0x116a: 0xe00d, 0x116b: 0x0008, 0x116c: 0xe00d, 0x116d: 0x0008, 0x116e: 0xe00d, 0x116f: 0x0008, + 0x1170: 0xe0fd, 0x1171: 0x0008, 0x1172: 0x0008, 0x1173: 0x0008, 0x1174: 0x0008, 0x1175: 0x0008, + 0x1176: 0x0008, 0x1177: 0x0008, 0x1178: 0x0008, 0x1179: 0xe01d, 0x117a: 0x0008, 0x117b: 0xe03d, + 0x117c: 0x0008, 0x117d: 0x442d, 0x117e: 0xe00d, 0x117f: 0x0008, + // Block 0x46, offset 0x1180 + 0x1180: 0xe00d, 0x1181: 0x0008, 0x1182: 0xe00d, 0x1183: 0x0008, 0x1184: 0xe00d, 0x1185: 0x0008, + 0x1186: 0xe00d, 0x1187: 0x0008, 0x1188: 0x0008, 0x1189: 0x0018, 0x118a: 0x0018, 0x118b: 0xe03d, + 0x118c: 0x0008, 0x118d: 0x11d9, 0x118e: 0x0008, 0x118f: 0x0008, 0x1190: 0xe00d, 0x1191: 0x0008, + 0x1192: 0xe00d, 0x1193: 0x0008, 0x1194: 0x0008, 0x1195: 0x0008, 0x1196: 0xe00d, 0x1197: 0x0008, + 0x1198: 0xe00d, 0x1199: 0x0008, 0x119a: 0xe00d, 0x119b: 0x0008, 0x119c: 0xe00d, 0x119d: 0x0008, + 0x119e: 0xe00d, 0x119f: 0x0008, 0x11a0: 0xe00d, 0x11a1: 0x0008, 0x11a2: 0xe00d, 0x11a3: 0x0008, + 0x11a4: 0xe00d, 0x11a5: 0x0008, 0x11a6: 0xe00d, 0x11a7: 0x0008, 0x11a8: 0xe00d, 0x11a9: 0x0008, + 0x11aa: 0x6e29, 0x11ab: 0x1029, 0x11ac: 0x11c1, 0x11ad: 0x6e41, 0x11ae: 0x1221, 0x11af: 0x0040, + 0x11b0: 0x6e59, 0x11b1: 0x6e71, 0x11b2: 0x1239, 0x11b3: 0x444d, 0x11b4: 0xe00d, 0x11b5: 0x0008, + 0x11b6: 0xe00d, 0x11b7: 0x0008, 0x11b8: 0x0040, 0x11b9: 0x0040, 0x11ba: 0x0040, 0x11bb: 0x0040, + 0x11bc: 0x0040, 0x11bd: 0x0040, 0x11be: 0x0040, 0x11bf: 0x0040, + // Block 0x47, offset 0x11c0 + 0x11c0: 0x64d5, 0x11c1: 0x64f5, 0x11c2: 0x6515, 0x11c3: 0x6535, 0x11c4: 0x6555, 0x11c5: 0x6575, + 0x11c6: 0x6595, 0x11c7: 0x65b5, 0x11c8: 0x65d5, 0x11c9: 0x65f5, 0x11ca: 0x6615, 0x11cb: 0x6635, + 0x11cc: 0x6655, 0x11cd: 0x6675, 0x11ce: 0x0008, 0x11cf: 0x0008, 0x11d0: 0x6695, 0x11d1: 0x0008, + 0x11d2: 0x66b5, 0x11d3: 0x0008, 0x11d4: 0x0008, 0x11d5: 0x66d5, 0x11d6: 0x66f5, 0x11d7: 0x6715, + 0x11d8: 0x6735, 0x11d9: 0x6755, 0x11da: 0x6775, 0x11db: 0x6795, 0x11dc: 0x67b5, 0x11dd: 0x67d5, + 0x11de: 0x67f5, 0x11df: 0x0008, 0x11e0: 0x6815, 0x11e1: 0x0008, 0x11e2: 0x6835, 0x11e3: 0x0008, + 0x11e4: 0x0008, 0x11e5: 0x6855, 0x11e6: 0x6875, 0x11e7: 0x0008, 0x11e8: 0x0008, 0x11e9: 0x0008, + 0x11ea: 0x6895, 0x11eb: 0x68b5, 0x11ec: 0x68d5, 0x11ed: 0x68f5, 0x11ee: 0x6915, 0x11ef: 0x6935, + 0x11f0: 0x6955, 0x11f1: 0x6975, 0x11f2: 0x6995, 0x11f3: 0x69b5, 0x11f4: 0x69d5, 0x11f5: 0x69f5, + 0x11f6: 0x6a15, 0x11f7: 0x6a35, 0x11f8: 0x6a55, 0x11f9: 0x6a75, 0x11fa: 0x6a95, 0x11fb: 0x6ab5, + 0x11fc: 0x6ad5, 0x11fd: 0x6af5, 0x11fe: 0x6b15, 0x11ff: 0x6b35, + // Block 0x48, offset 0x1200 + 0x1200: 0x7a95, 0x1201: 0x7ab5, 0x1202: 0x7ad5, 0x1203: 0x7af5, 0x1204: 0x7b15, 0x1205: 0x7b35, + 0x1206: 0x7b55, 0x1207: 0x7b75, 0x1208: 0x7b95, 0x1209: 0x7bb5, 0x120a: 0x7bd5, 0x120b: 0x7bf5, + 0x120c: 0x7c15, 0x120d: 0x7c35, 0x120e: 0x7c55, 0x120f: 0x6ec9, 0x1210: 0x6ef1, 0x1211: 0x6f19, + 0x1212: 0x7c75, 0x1213: 0x7c95, 0x1214: 0x7cb5, 0x1215: 0x6f41, 0x1216: 0x6f69, 0x1217: 0x6f91, + 0x1218: 0x7cd5, 0x1219: 0x7cf5, 0x121a: 0x0040, 0x121b: 0x0040, 0x121c: 0x0040, 0x121d: 0x0040, + 0x121e: 0x0040, 0x121f: 0x0040, 0x1220: 0x0040, 0x1221: 0x0040, 0x1222: 0x0040, 0x1223: 0x0040, + 0x1224: 0x0040, 0x1225: 0x0040, 0x1226: 0x0040, 0x1227: 0x0040, 0x1228: 0x0040, 0x1229: 0x0040, + 0x122a: 0x0040, 0x122b: 0x0040, 0x122c: 0x0040, 0x122d: 0x0040, 0x122e: 0x0040, 0x122f: 0x0040, + 0x1230: 0x0040, 0x1231: 0x0040, 0x1232: 0x0040, 0x1233: 0x0040, 0x1234: 0x0040, 0x1235: 0x0040, + 0x1236: 0x0040, 0x1237: 0x0040, 0x1238: 0x0040, 0x1239: 0x0040, 0x123a: 0x0040, 0x123b: 0x0040, + 0x123c: 0x0040, 0x123d: 0x0040, 0x123e: 0x0040, 0x123f: 0x0040, + // Block 0x49, offset 0x1240 + 0x1240: 0x6fb9, 0x1241: 0x6fd1, 0x1242: 0x6fe9, 0x1243: 0x7d15, 0x1244: 0x7d35, 0x1245: 0x7001, + 0x1246: 0x7001, 0x1247: 0x0040, 0x1248: 0x0040, 0x1249: 0x0040, 0x124a: 0x0040, 0x124b: 0x0040, + 0x124c: 0x0040, 0x124d: 0x0040, 0x124e: 0x0040, 0x124f: 0x0040, 0x1250: 0x0040, 0x1251: 0x0040, + 0x1252: 0x0040, 0x1253: 0x7019, 0x1254: 0x7041, 0x1255: 0x7069, 0x1256: 0x7091, 0x1257: 0x70b9, + 0x1258: 0x0040, 0x1259: 0x0040, 0x125a: 0x0040, 0x125b: 0x0040, 0x125c: 0x0040, 0x125d: 0x70e1, + 0x125e: 0x1308, 0x125f: 0x7109, 0x1260: 0x7131, 0x1261: 0x20a9, 0x1262: 0x20f1, 0x1263: 0x7149, + 0x1264: 0x7161, 0x1265: 0x7179, 0x1266: 0x7191, 0x1267: 0x71a9, 0x1268: 0x71c1, 0x1269: 0x1fb2, + 0x126a: 0x71d9, 0x126b: 0x7201, 0x126c: 0x7229, 0x126d: 0x7261, 0x126e: 0x7299, 0x126f: 0x72c1, + 0x1270: 0x72e9, 0x1271: 0x7311, 0x1272: 0x7339, 0x1273: 0x7361, 0x1274: 0x7389, 0x1275: 0x73b1, + 0x1276: 0x73d9, 0x1277: 0x0040, 0x1278: 0x7401, 0x1279: 0x7429, 0x127a: 0x7451, 0x127b: 0x7479, + 0x127c: 0x74a1, 0x127d: 0x0040, 0x127e: 0x74c9, 0x127f: 0x0040, + // Block 0x4a, offset 0x1280 + 0x1280: 0x74f1, 0x1281: 0x7519, 0x1282: 0x0040, 0x1283: 0x7541, 0x1284: 0x7569, 0x1285: 0x0040, + 0x1286: 0x7591, 0x1287: 0x75b9, 0x1288: 0x75e1, 0x1289: 0x7609, 0x128a: 0x7631, 0x128b: 0x7659, + 0x128c: 0x7681, 0x128d: 0x76a9, 0x128e: 0x76d1, 0x128f: 0x76f9, 0x1290: 0x7721, 0x1291: 0x7721, + 0x1292: 0x7739, 0x1293: 0x7739, 0x1294: 0x7739, 0x1295: 0x7739, 0x1296: 0x7751, 0x1297: 0x7751, + 0x1298: 0x7751, 0x1299: 0x7751, 0x129a: 0x7769, 0x129b: 0x7769, 0x129c: 0x7769, 0x129d: 0x7769, + 0x129e: 0x7781, 0x129f: 0x7781, 0x12a0: 0x7781, 0x12a1: 0x7781, 0x12a2: 0x7799, 0x12a3: 0x7799, + 0x12a4: 0x7799, 0x12a5: 0x7799, 0x12a6: 0x77b1, 0x12a7: 0x77b1, 0x12a8: 0x77b1, 0x12a9: 0x77b1, + 0x12aa: 0x77c9, 0x12ab: 0x77c9, 0x12ac: 0x77c9, 0x12ad: 0x77c9, 0x12ae: 0x77e1, 0x12af: 0x77e1, + 0x12b0: 0x77e1, 0x12b1: 0x77e1, 0x12b2: 0x77f9, 0x12b3: 0x77f9, 0x12b4: 0x77f9, 0x12b5: 0x77f9, + 0x12b6: 0x7811, 0x12b7: 0x7811, 0x12b8: 0x7811, 0x12b9: 0x7811, 0x12ba: 0x7829, 0x12bb: 0x7829, + 0x12bc: 0x7829, 0x12bd: 0x7829, 0x12be: 0x7841, 0x12bf: 0x7841, + // Block 0x4b, offset 0x12c0 + 0x12c0: 0x7841, 0x12c1: 0x7841, 0x12c2: 0x7859, 0x12c3: 0x7859, 0x12c4: 0x7871, 0x12c5: 0x7871, + 0x12c6: 0x7889, 0x12c7: 0x7889, 0x12c8: 0x78a1, 0x12c9: 0x78a1, 0x12ca: 0x78b9, 0x12cb: 0x78b9, + 0x12cc: 0x78d1, 0x12cd: 0x78d1, 0x12ce: 0x78e9, 0x12cf: 0x78e9, 0x12d0: 0x78e9, 0x12d1: 0x78e9, + 0x12d2: 0x7901, 0x12d3: 0x7901, 0x12d4: 0x7901, 0x12d5: 0x7901, 0x12d6: 0x7919, 0x12d7: 0x7919, + 0x12d8: 0x7919, 0x12d9: 0x7919, 0x12da: 0x7931, 0x12db: 0x7931, 0x12dc: 0x7931, 0x12dd: 0x7931, + 0x12de: 0x7949, 0x12df: 0x7949, 0x12e0: 0x7961, 0x12e1: 0x7961, 0x12e2: 0x7961, 0x12e3: 0x7961, + 0x12e4: 0x7979, 0x12e5: 0x7979, 0x12e6: 0x7991, 0x12e7: 0x7991, 0x12e8: 0x7991, 0x12e9: 0x7991, + 0x12ea: 0x79a9, 0x12eb: 0x79a9, 0x12ec: 0x79a9, 0x12ed: 0x79a9, 0x12ee: 0x79c1, 0x12ef: 0x79c1, + 0x12f0: 0x79d9, 0x12f1: 0x79d9, 0x12f2: 0x0018, 0x12f3: 0x0018, 0x12f4: 0x0018, 0x12f5: 0x0018, + 0x12f6: 0x0018, 0x12f7: 0x0018, 0x12f8: 0x0018, 0x12f9: 0x0018, 0x12fa: 0x0018, 0x12fb: 0x0018, + 0x12fc: 0x0018, 0x12fd: 0x0018, 0x12fe: 0x0018, 0x12ff: 0x0018, + // Block 0x4c, offset 0x1300 + 0x1300: 0x0018, 0x1301: 0x0018, 0x1302: 0x0040, 0x1303: 0x0040, 0x1304: 0x0040, 0x1305: 0x0040, + 0x1306: 0x0040, 0x1307: 0x0040, 0x1308: 0x0040, 0x1309: 0x0040, 0x130a: 0x0040, 0x130b: 0x0040, + 0x130c: 0x0040, 0x130d: 0x0040, 0x130e: 0x0040, 0x130f: 0x0040, 0x1310: 0x0040, 0x1311: 0x0040, + 0x1312: 0x0040, 0x1313: 0x79f1, 0x1314: 0x79f1, 0x1315: 0x79f1, 0x1316: 0x79f1, 0x1317: 0x7a09, + 0x1318: 0x7a09, 0x1319: 0x7a21, 0x131a: 0x7a21, 0x131b: 0x7a39, 0x131c: 0x7a39, 0x131d: 0x0479, + 0x131e: 0x7a51, 0x131f: 0x7a51, 0x1320: 0x7a69, 0x1321: 0x7a69, 0x1322: 0x7a81, 0x1323: 0x7a81, + 0x1324: 0x7a99, 0x1325: 0x7a99, 0x1326: 0x7a99, 0x1327: 0x7a99, 0x1328: 0x7ab1, 0x1329: 0x7ab1, + 0x132a: 0x7ac9, 0x132b: 0x7ac9, 0x132c: 0x7af1, 0x132d: 0x7af1, 0x132e: 0x7b19, 0x132f: 0x7b19, + 0x1330: 0x7b41, 0x1331: 0x7b41, 0x1332: 0x7b69, 0x1333: 0x7b69, 0x1334: 0x7b91, 0x1335: 0x7b91, + 0x1336: 0x7bb9, 0x1337: 0x7bb9, 0x1338: 0x7bb9, 0x1339: 0x7be1, 0x133a: 0x7be1, 0x133b: 0x7be1, + 0x133c: 0x7c09, 0x133d: 0x7c09, 0x133e: 0x7c09, 0x133f: 0x7c09, + // Block 0x4d, offset 0x1340 + 0x1340: 0x85f9, 0x1341: 0x8621, 0x1342: 0x8649, 0x1343: 0x8671, 0x1344: 0x8699, 0x1345: 0x86c1, + 0x1346: 0x86e9, 0x1347: 0x8711, 0x1348: 0x8739, 0x1349: 0x8761, 0x134a: 0x8789, 0x134b: 0x87b1, + 0x134c: 0x87d9, 0x134d: 0x8801, 0x134e: 0x8829, 0x134f: 0x8851, 0x1350: 0x8879, 0x1351: 0x88a1, + 0x1352: 0x88c9, 0x1353: 0x88f1, 0x1354: 0x8919, 0x1355: 0x8941, 0x1356: 0x8969, 0x1357: 0x8991, + 0x1358: 0x89b9, 0x1359: 0x89e1, 0x135a: 0x8a09, 0x135b: 0x8a31, 0x135c: 0x8a59, 0x135d: 0x8a81, + 0x135e: 0x8aaa, 0x135f: 0x8ada, 0x1360: 0x8b0a, 0x1361: 0x8b3a, 0x1362: 0x8b6a, 0x1363: 0x8b9a, + 0x1364: 0x8bc9, 0x1365: 0x8bf1, 0x1366: 0x7c71, 0x1367: 0x8c19, 0x1368: 0x7be1, 0x1369: 0x7c99, + 0x136a: 0x8c41, 0x136b: 0x8c69, 0x136c: 0x7d39, 0x136d: 0x8c91, 0x136e: 0x7d61, 0x136f: 0x7d89, + 0x1370: 0x8cb9, 0x1371: 0x8ce1, 0x1372: 0x7e29, 0x1373: 0x8d09, 0x1374: 0x7e51, 0x1375: 0x7e79, + 0x1376: 0x8d31, 0x1377: 0x8d59, 0x1378: 0x7ec9, 0x1379: 0x8d81, 0x137a: 0x7ef1, 0x137b: 0x7f19, + 0x137c: 0x83a1, 0x137d: 0x83c9, 0x137e: 0x8441, 0x137f: 0x8469, + // Block 0x4e, offset 0x1380 + 0x1380: 0x8491, 0x1381: 0x8531, 0x1382: 0x8559, 0x1383: 0x8581, 0x1384: 0x85a9, 0x1385: 0x8649, + 0x1386: 0x8671, 0x1387: 0x8699, 0x1388: 0x8da9, 0x1389: 0x8739, 0x138a: 0x8dd1, 0x138b: 0x8df9, + 0x138c: 0x8829, 0x138d: 0x8e21, 0x138e: 0x8851, 0x138f: 0x8879, 0x1390: 0x8a81, 0x1391: 0x8e49, + 0x1392: 0x8e71, 0x1393: 0x89b9, 0x1394: 0x8e99, 0x1395: 0x89e1, 0x1396: 0x8a09, 0x1397: 0x7c21, + 0x1398: 0x7c49, 0x1399: 0x8ec1, 0x139a: 0x7c71, 0x139b: 0x8ee9, 0x139c: 0x7cc1, 0x139d: 0x7ce9, + 0x139e: 0x7d11, 0x139f: 0x7d39, 0x13a0: 0x8f11, 0x13a1: 0x7db1, 0x13a2: 0x7dd9, 0x13a3: 0x7e01, + 0x13a4: 0x7e29, 0x13a5: 0x8f39, 0x13a6: 0x7ec9, 0x13a7: 0x7f41, 0x13a8: 0x7f69, 0x13a9: 0x7f91, + 0x13aa: 0x7fb9, 0x13ab: 0x7fe1, 0x13ac: 0x8031, 0x13ad: 0x8059, 0x13ae: 0x8081, 0x13af: 0x80a9, + 0x13b0: 0x80d1, 0x13b1: 0x80f9, 0x13b2: 0x8f61, 0x13b3: 0x8121, 0x13b4: 0x8149, 0x13b5: 0x8171, + 0x13b6: 0x8199, 0x13b7: 0x81c1, 0x13b8: 0x81e9, 0x13b9: 0x8239, 0x13ba: 0x8261, 0x13bb: 0x8289, + 0x13bc: 0x82b1, 0x13bd: 0x82d9, 0x13be: 0x8301, 0x13bf: 0x8329, + // Block 0x4f, offset 0x13c0 + 0x13c0: 0x8351, 0x13c1: 0x8379, 0x13c2: 0x83f1, 0x13c3: 0x8419, 0x13c4: 0x84b9, 0x13c5: 0x84e1, + 0x13c6: 0x8509, 0x13c7: 0x8531, 0x13c8: 0x8559, 0x13c9: 0x85d1, 0x13ca: 0x85f9, 0x13cb: 0x8621, + 0x13cc: 0x8649, 0x13cd: 0x8f89, 0x13ce: 0x86c1, 0x13cf: 0x86e9, 0x13d0: 0x8711, 0x13d1: 0x8739, + 0x13d2: 0x87b1, 0x13d3: 0x87d9, 0x13d4: 0x8801, 0x13d5: 0x8829, 0x13d6: 0x8fb1, 0x13d7: 0x88a1, + 0x13d8: 0x88c9, 0x13d9: 0x8fd9, 0x13da: 0x8941, 0x13db: 0x8969, 0x13dc: 0x8991, 0x13dd: 0x89b9, + 0x13de: 0x9001, 0x13df: 0x7c71, 0x13e0: 0x8ee9, 0x13e1: 0x7d39, 0x13e2: 0x8f11, 0x13e3: 0x7e29, + 0x13e4: 0x8f39, 0x13e5: 0x7ec9, 0x13e6: 0x9029, 0x13e7: 0x80d1, 0x13e8: 0x9051, 0x13e9: 0x9079, + 0x13ea: 0x90a1, 0x13eb: 0x8531, 0x13ec: 0x8559, 0x13ed: 0x8649, 0x13ee: 0x8829, 0x13ef: 0x8fb1, + 0x13f0: 0x89b9, 0x13f1: 0x9001, 0x13f2: 0x90c9, 0x13f3: 0x9101, 0x13f4: 0x9139, 0x13f5: 0x9171, + 0x13f6: 0x9199, 0x13f7: 0x91c1, 0x13f8: 0x91e9, 0x13f9: 0x9211, 0x13fa: 0x9239, 0x13fb: 0x9261, + 0x13fc: 0x9289, 0x13fd: 0x92b1, 0x13fe: 0x92d9, 0x13ff: 0x9301, + // Block 0x50, offset 0x1400 + 0x1400: 0x9329, 0x1401: 0x9351, 0x1402: 0x9379, 0x1403: 0x93a1, 0x1404: 0x93c9, 0x1405: 0x93f1, + 0x1406: 0x9419, 0x1407: 0x9441, 0x1408: 0x9469, 0x1409: 0x9491, 0x140a: 0x94b9, 0x140b: 0x94e1, + 0x140c: 0x9079, 0x140d: 0x9509, 0x140e: 0x9531, 0x140f: 0x9559, 0x1410: 0x9581, 0x1411: 0x9171, + 0x1412: 0x9199, 0x1413: 0x91c1, 0x1414: 0x91e9, 0x1415: 0x9211, 0x1416: 0x9239, 0x1417: 0x9261, + 0x1418: 0x9289, 0x1419: 0x92b1, 0x141a: 0x92d9, 0x141b: 0x9301, 0x141c: 0x9329, 0x141d: 0x9351, + 0x141e: 0x9379, 0x141f: 0x93a1, 0x1420: 0x93c9, 0x1421: 0x93f1, 0x1422: 0x9419, 0x1423: 0x9441, + 0x1424: 0x9469, 0x1425: 0x9491, 0x1426: 0x94b9, 0x1427: 0x94e1, 0x1428: 0x9079, 0x1429: 0x9509, + 0x142a: 0x9531, 0x142b: 0x9559, 0x142c: 0x9581, 0x142d: 0x9491, 0x142e: 0x94b9, 0x142f: 0x94e1, + 0x1430: 0x9079, 0x1431: 0x9051, 0x1432: 0x90a1, 0x1433: 0x8211, 0x1434: 0x8059, 0x1435: 0x8081, + 0x1436: 0x80a9, 0x1437: 0x9491, 0x1438: 0x94b9, 0x1439: 0x94e1, 0x143a: 0x8211, 0x143b: 0x8239, + 0x143c: 0x95a9, 0x143d: 0x95a9, 0x143e: 0x0018, 0x143f: 0x0018, + // Block 0x51, offset 0x1440 + 0x1440: 0x0040, 0x1441: 0x0040, 0x1442: 0x0040, 0x1443: 0x0040, 0x1444: 0x0040, 0x1445: 0x0040, + 0x1446: 0x0040, 0x1447: 0x0040, 0x1448: 0x0040, 0x1449: 0x0040, 0x144a: 0x0040, 0x144b: 0x0040, + 0x144c: 0x0040, 0x144d: 0x0040, 0x144e: 0x0040, 0x144f: 0x0040, 0x1450: 0x95d1, 0x1451: 0x9609, + 0x1452: 0x9609, 0x1453: 0x9641, 0x1454: 0x9679, 0x1455: 0x96b1, 0x1456: 0x96e9, 0x1457: 0x9721, + 0x1458: 0x9759, 0x1459: 0x9759, 0x145a: 0x9791, 0x145b: 0x97c9, 0x145c: 0x9801, 0x145d: 0x9839, + 0x145e: 0x9871, 0x145f: 0x98a9, 0x1460: 0x98a9, 0x1461: 0x98e1, 0x1462: 0x9919, 0x1463: 0x9919, + 0x1464: 0x9951, 0x1465: 0x9951, 0x1466: 0x9989, 0x1467: 0x99c1, 0x1468: 0x99c1, 0x1469: 0x99f9, + 0x146a: 0x9a31, 0x146b: 0x9a31, 0x146c: 0x9a69, 0x146d: 0x9a69, 0x146e: 0x9aa1, 0x146f: 0x9ad9, + 0x1470: 0x9ad9, 0x1471: 0x9b11, 0x1472: 0x9b11, 0x1473: 0x9b49, 0x1474: 0x9b81, 0x1475: 0x9bb9, + 0x1476: 0x9bf1, 0x1477: 0x9bf1, 0x1478: 0x9c29, 0x1479: 0x9c61, 0x147a: 0x9c99, 0x147b: 0x9cd1, + 0x147c: 0x9d09, 0x147d: 0x9d09, 0x147e: 0x9d41, 0x147f: 0x9d79, + // Block 0x52, offset 0x1480 + 0x1480: 0xa949, 0x1481: 0xa981, 0x1482: 0xa9b9, 0x1483: 0xa8a1, 0x1484: 0x9bb9, 0x1485: 0x9989, + 0x1486: 0xa9f1, 0x1487: 0xaa29, 0x1488: 0x0040, 0x1489: 0x0040, 0x148a: 0x0040, 0x148b: 0x0040, + 0x148c: 0x0040, 0x148d: 0x0040, 0x148e: 0x0040, 0x148f: 0x0040, 0x1490: 0x0040, 0x1491: 0x0040, + 0x1492: 0x0040, 0x1493: 0x0040, 0x1494: 0x0040, 0x1495: 0x0040, 0x1496: 0x0040, 0x1497: 0x0040, + 0x1498: 0x0040, 0x1499: 0x0040, 0x149a: 0x0040, 0x149b: 0x0040, 0x149c: 0x0040, 0x149d: 0x0040, + 0x149e: 0x0040, 0x149f: 0x0040, 0x14a0: 0x0040, 0x14a1: 0x0040, 0x14a2: 0x0040, 0x14a3: 0x0040, + 0x14a4: 0x0040, 0x14a5: 0x0040, 0x14a6: 0x0040, 0x14a7: 0x0040, 0x14a8: 0x0040, 0x14a9: 0x0040, + 0x14aa: 0x0040, 0x14ab: 0x0040, 0x14ac: 0x0040, 0x14ad: 0x0040, 0x14ae: 0x0040, 0x14af: 0x0040, + 0x14b0: 0xaa61, 0x14b1: 0xaa99, 0x14b2: 0xaad1, 0x14b3: 0xab19, 0x14b4: 0xab61, 0x14b5: 0xaba9, + 0x14b6: 0xabf1, 0x14b7: 0xac39, 0x14b8: 0xac81, 0x14b9: 0xacc9, 0x14ba: 0xad02, 0x14bb: 0xae12, + 0x14bc: 0xae91, 0x14bd: 0x0018, 0x14be: 0x0040, 0x14bf: 0x0040, + // Block 0x53, offset 0x14c0 + 0x14c0: 0x13c0, 0x14c1: 0x13c0, 0x14c2: 0x13c0, 0x14c3: 0x13c0, 0x14c4: 0x13c0, 0x14c5: 0x13c0, + 0x14c6: 0x13c0, 0x14c7: 0x13c0, 0x14c8: 0x13c0, 0x14c9: 0x13c0, 0x14ca: 0x13c0, 0x14cb: 0x13c0, + 0x14cc: 0x13c0, 0x14cd: 0x13c0, 0x14ce: 0x13c0, 0x14cf: 0x13c0, 0x14d0: 0xaeda, 0x14d1: 0x7d55, + 0x14d2: 0x0040, 0x14d3: 0xaeea, 0x14d4: 0x03c2, 0x14d5: 0xaefa, 0x14d6: 0xaf0a, 0x14d7: 0x7d75, + 0x14d8: 0x7d95, 0x14d9: 0x0040, 0x14da: 0x0040, 0x14db: 0x0040, 0x14dc: 0x0040, 0x14dd: 0x0040, + 0x14de: 0x0040, 0x14df: 0x0040, 0x14e0: 0x1308, 0x14e1: 0x1308, 0x14e2: 0x1308, 0x14e3: 0x1308, + 0x14e4: 0x1308, 0x14e5: 0x1308, 0x14e6: 0x1308, 0x14e7: 0x1308, 0x14e8: 0x1308, 0x14e9: 0x1308, + 0x14ea: 0x1308, 0x14eb: 0x1308, 0x14ec: 0x1308, 0x14ed: 0x1308, 0x14ee: 0x1308, 0x14ef: 0x1308, + 0x14f0: 0x0040, 0x14f1: 0x7db5, 0x14f2: 0x7dd5, 0x14f3: 0xaf1a, 0x14f4: 0xaf1a, 0x14f5: 0x1fd2, + 0x14f6: 0x1fe2, 0x14f7: 0xaf2a, 0x14f8: 0xaf3a, 0x14f9: 0x7df5, 0x14fa: 0x7e15, 0x14fb: 0x7e35, + 0x14fc: 0x7df5, 0x14fd: 0x7e55, 0x14fe: 0x7e75, 0x14ff: 0x7e55, + // Block 0x54, offset 0x1500 + 0x1500: 0x7e95, 0x1501: 0x7eb5, 0x1502: 0x7ed5, 0x1503: 0x7eb5, 0x1504: 0x7ef5, 0x1505: 0x0018, + 0x1506: 0x0018, 0x1507: 0xaf4a, 0x1508: 0xaf5a, 0x1509: 0x7f16, 0x150a: 0x7f36, 0x150b: 0x7f56, + 0x150c: 0x7f76, 0x150d: 0xaf1a, 0x150e: 0xaf1a, 0x150f: 0xaf1a, 0x1510: 0xaeda, 0x1511: 0x7f95, + 0x1512: 0x0040, 0x1513: 0x0040, 0x1514: 0x03c2, 0x1515: 0xaeea, 0x1516: 0xaf0a, 0x1517: 0xaefa, + 0x1518: 0x7fb5, 0x1519: 0x1fd2, 0x151a: 0x1fe2, 0x151b: 0xaf2a, 0x151c: 0xaf3a, 0x151d: 0x7e95, + 0x151e: 0x7ef5, 0x151f: 0xaf6a, 0x1520: 0xaf7a, 0x1521: 0xaf8a, 0x1522: 0x1fb2, 0x1523: 0xaf99, + 0x1524: 0xafaa, 0x1525: 0xafba, 0x1526: 0x1fc2, 0x1527: 0x0040, 0x1528: 0xafca, 0x1529: 0xafda, + 0x152a: 0xafea, 0x152b: 0xaffa, 0x152c: 0x0040, 0x152d: 0x0040, 0x152e: 0x0040, 0x152f: 0x0040, + 0x1530: 0x7fd6, 0x1531: 0xb009, 0x1532: 0x7ff6, 0x1533: 0x0008, 0x1534: 0x8016, 0x1535: 0x0040, + 0x1536: 0x8036, 0x1537: 0xb031, 0x1538: 0x8056, 0x1539: 0xb059, 0x153a: 0x8076, 0x153b: 0xb081, + 0x153c: 0x8096, 0x153d: 0xb0a9, 0x153e: 0x80b6, 0x153f: 0xb0d1, + // Block 0x55, offset 0x1540 + 0x1540: 0xb0f9, 0x1541: 0xb111, 0x1542: 0xb111, 0x1543: 0xb129, 0x1544: 0xb129, 0x1545: 0xb141, + 0x1546: 0xb141, 0x1547: 0xb159, 0x1548: 0xb159, 0x1549: 0xb171, 0x154a: 0xb171, 0x154b: 0xb171, + 0x154c: 0xb171, 0x154d: 0xb189, 0x154e: 0xb189, 0x154f: 0xb1a1, 0x1550: 0xb1a1, 0x1551: 0xb1a1, + 0x1552: 0xb1a1, 0x1553: 0xb1b9, 0x1554: 0xb1b9, 0x1555: 0xb1d1, 0x1556: 0xb1d1, 0x1557: 0xb1d1, + 0x1558: 0xb1d1, 0x1559: 0xb1e9, 0x155a: 0xb1e9, 0x155b: 0xb1e9, 0x155c: 0xb1e9, 0x155d: 0xb201, + 0x155e: 0xb201, 0x155f: 0xb201, 0x1560: 0xb201, 0x1561: 0xb219, 0x1562: 0xb219, 0x1563: 0xb219, + 0x1564: 0xb219, 0x1565: 0xb231, 0x1566: 0xb231, 0x1567: 0xb231, 0x1568: 0xb231, 0x1569: 0xb249, + 0x156a: 0xb249, 0x156b: 0xb261, 0x156c: 0xb261, 0x156d: 0xb279, 0x156e: 0xb279, 0x156f: 0xb291, + 0x1570: 0xb291, 0x1571: 0xb2a9, 0x1572: 0xb2a9, 0x1573: 0xb2a9, 0x1574: 0xb2a9, 0x1575: 0xb2c1, + 0x1576: 0xb2c1, 0x1577: 0xb2c1, 0x1578: 0xb2c1, 0x1579: 0xb2d9, 0x157a: 0xb2d9, 0x157b: 0xb2d9, + 0x157c: 0xb2d9, 0x157d: 0xb2f1, 0x157e: 0xb2f1, 0x157f: 0xb2f1, + // Block 0x56, offset 0x1580 + 0x1580: 0xb2f1, 0x1581: 0xb309, 0x1582: 0xb309, 0x1583: 0xb309, 0x1584: 0xb309, 0x1585: 0xb321, + 0x1586: 0xb321, 0x1587: 0xb321, 0x1588: 0xb321, 0x1589: 0xb339, 0x158a: 0xb339, 0x158b: 0xb339, + 0x158c: 0xb339, 0x158d: 0xb351, 0x158e: 0xb351, 0x158f: 0xb351, 0x1590: 0xb351, 0x1591: 0xb369, + 0x1592: 0xb369, 0x1593: 0xb369, 0x1594: 0xb369, 0x1595: 0xb381, 0x1596: 0xb381, 0x1597: 0xb381, + 0x1598: 0xb381, 0x1599: 0xb399, 0x159a: 0xb399, 0x159b: 0xb399, 0x159c: 0xb399, 0x159d: 0xb3b1, + 0x159e: 0xb3b1, 0x159f: 0xb3b1, 0x15a0: 0xb3b1, 0x15a1: 0xb3c9, 0x15a2: 0xb3c9, 0x15a3: 0xb3c9, + 0x15a4: 0xb3c9, 0x15a5: 0xb3e1, 0x15a6: 0xb3e1, 0x15a7: 0xb3e1, 0x15a8: 0xb3e1, 0x15a9: 0xb3f9, + 0x15aa: 0xb3f9, 0x15ab: 0xb3f9, 0x15ac: 0xb3f9, 0x15ad: 0xb411, 0x15ae: 0xb411, 0x15af: 0x7ab1, + 0x15b0: 0x7ab1, 0x15b1: 0xb429, 0x15b2: 0xb429, 0x15b3: 0xb429, 0x15b4: 0xb429, 0x15b5: 0xb441, + 0x15b6: 0xb441, 0x15b7: 0xb469, 0x15b8: 0xb469, 0x15b9: 0xb491, 0x15ba: 0xb491, 0x15bb: 0xb4b9, + 0x15bc: 0xb4b9, 0x15bd: 0x0040, 0x15be: 0x0040, 0x15bf: 0x03c0, + // Block 0x57, offset 0x15c0 + 0x15c0: 0x0040, 0x15c1: 0xaefa, 0x15c2: 0xb4e2, 0x15c3: 0xaf6a, 0x15c4: 0xafda, 0x15c5: 0xafea, + 0x15c6: 0xaf7a, 0x15c7: 0xb4f2, 0x15c8: 0x1fd2, 0x15c9: 0x1fe2, 0x15ca: 0xaf8a, 0x15cb: 0x1fb2, + 0x15cc: 0xaeda, 0x15cd: 0xaf99, 0x15ce: 0x29d1, 0x15cf: 0xb502, 0x15d0: 0x1f41, 0x15d1: 0x00c9, + 0x15d2: 0x0069, 0x15d3: 0x0079, 0x15d4: 0x1f51, 0x15d5: 0x1f61, 0x15d6: 0x1f71, 0x15d7: 0x1f81, + 0x15d8: 0x1f91, 0x15d9: 0x1fa1, 0x15da: 0xaeea, 0x15db: 0x03c2, 0x15dc: 0xafaa, 0x15dd: 0x1fc2, + 0x15de: 0xafba, 0x15df: 0xaf0a, 0x15e0: 0xaffa, 0x15e1: 0x0039, 0x15e2: 0x0ee9, 0x15e3: 0x1159, + 0x15e4: 0x0ef9, 0x15e5: 0x0f09, 0x15e6: 0x1199, 0x15e7: 0x0f31, 0x15e8: 0x0249, 0x15e9: 0x0f41, + 0x15ea: 0x0259, 0x15eb: 0x0f51, 0x15ec: 0x0359, 0x15ed: 0x0f61, 0x15ee: 0x0f71, 0x15ef: 0x00d9, + 0x15f0: 0x0f99, 0x15f1: 0x2039, 0x15f2: 0x0269, 0x15f3: 0x01d9, 0x15f4: 0x0fa9, 0x15f5: 0x0fb9, + 0x15f6: 0x1089, 0x15f7: 0x0279, 0x15f8: 0x0369, 0x15f9: 0x0289, 0x15fa: 0x13d1, 0x15fb: 0xaf4a, + 0x15fc: 0xafca, 0x15fd: 0xaf5a, 0x15fe: 0xb512, 0x15ff: 0xaf1a, + // Block 0x58, offset 0x1600 + 0x1600: 0x1caa, 0x1601: 0x0039, 0x1602: 0x0ee9, 0x1603: 0x1159, 0x1604: 0x0ef9, 0x1605: 0x0f09, + 0x1606: 0x1199, 0x1607: 0x0f31, 0x1608: 0x0249, 0x1609: 0x0f41, 0x160a: 0x0259, 0x160b: 0x0f51, + 0x160c: 0x0359, 0x160d: 0x0f61, 0x160e: 0x0f71, 0x160f: 0x00d9, 0x1610: 0x0f99, 0x1611: 0x2039, + 0x1612: 0x0269, 0x1613: 0x01d9, 0x1614: 0x0fa9, 0x1615: 0x0fb9, 0x1616: 0x1089, 0x1617: 0x0279, + 0x1618: 0x0369, 0x1619: 0x0289, 0x161a: 0x13d1, 0x161b: 0xaf2a, 0x161c: 0xb522, 0x161d: 0xaf3a, + 0x161e: 0xb532, 0x161f: 0x80d5, 0x1620: 0x80f5, 0x1621: 0x29d1, 0x1622: 0x8115, 0x1623: 0x8115, + 0x1624: 0x8135, 0x1625: 0x8155, 0x1626: 0x8175, 0x1627: 0x8195, 0x1628: 0x81b5, 0x1629: 0x81d5, + 0x162a: 0x81f5, 0x162b: 0x8215, 0x162c: 0x8235, 0x162d: 0x8255, 0x162e: 0x8275, 0x162f: 0x8295, + 0x1630: 0x82b5, 0x1631: 0x82d5, 0x1632: 0x82f5, 0x1633: 0x8315, 0x1634: 0x8335, 0x1635: 0x8355, + 0x1636: 0x8375, 0x1637: 0x8395, 0x1638: 0x83b5, 0x1639: 0x83d5, 0x163a: 0x83f5, 0x163b: 0x8415, + 0x163c: 0x81b5, 0x163d: 0x8435, 0x163e: 0x8455, 0x163f: 0x8215, + // Block 0x59, offset 0x1640 + 0x1640: 0x8475, 0x1641: 0x8495, 0x1642: 0x84b5, 0x1643: 0x84d5, 0x1644: 0x84f5, 0x1645: 0x8515, + 0x1646: 0x8535, 0x1647: 0x8555, 0x1648: 0x84d5, 0x1649: 0x8575, 0x164a: 0x84d5, 0x164b: 0x8595, + 0x164c: 0x8595, 0x164d: 0x85b5, 0x164e: 0x85b5, 0x164f: 0x85d5, 0x1650: 0x8515, 0x1651: 0x85f5, + 0x1652: 0x8615, 0x1653: 0x85f5, 0x1654: 0x8635, 0x1655: 0x8615, 0x1656: 0x8655, 0x1657: 0x8655, + 0x1658: 0x8675, 0x1659: 0x8675, 0x165a: 0x8695, 0x165b: 0x8695, 0x165c: 0x8615, 0x165d: 0x8115, + 0x165e: 0x86b5, 0x165f: 0x86d5, 0x1660: 0x0040, 0x1661: 0x86f5, 0x1662: 0x8715, 0x1663: 0x8735, + 0x1664: 0x8755, 0x1665: 0x8735, 0x1666: 0x8775, 0x1667: 0x8795, 0x1668: 0x87b5, 0x1669: 0x87b5, + 0x166a: 0x87d5, 0x166b: 0x87d5, 0x166c: 0x87f5, 0x166d: 0x87f5, 0x166e: 0x87d5, 0x166f: 0x87d5, + 0x1670: 0x8815, 0x1671: 0x8835, 0x1672: 0x8855, 0x1673: 0x8875, 0x1674: 0x8895, 0x1675: 0x88b5, + 0x1676: 0x88b5, 0x1677: 0x88b5, 0x1678: 0x88d5, 0x1679: 0x88d5, 0x167a: 0x88d5, 0x167b: 0x88d5, + 0x167c: 0x87b5, 0x167d: 0x87b5, 0x167e: 0x87b5, 0x167f: 0x0040, + // Block 0x5a, offset 0x1680 + 0x1680: 0x0040, 0x1681: 0x0040, 0x1682: 0x8715, 0x1683: 0x86f5, 0x1684: 0x88f5, 0x1685: 0x86f5, + 0x1686: 0x8715, 0x1687: 0x86f5, 0x1688: 0x0040, 0x1689: 0x0040, 0x168a: 0x8915, 0x168b: 0x8715, + 0x168c: 0x8935, 0x168d: 0x88f5, 0x168e: 0x8935, 0x168f: 0x8715, 0x1690: 0x0040, 0x1691: 0x0040, + 0x1692: 0x8955, 0x1693: 0x8975, 0x1694: 0x8875, 0x1695: 0x8935, 0x1696: 0x88f5, 0x1697: 0x8935, + 0x1698: 0x0040, 0x1699: 0x0040, 0x169a: 0x8995, 0x169b: 0x89b5, 0x169c: 0x8995, 0x169d: 0x0040, + 0x169e: 0x0040, 0x169f: 0x0040, 0x16a0: 0xb541, 0x16a1: 0xb559, 0x16a2: 0xb571, 0x16a3: 0x89d6, + 0x16a4: 0xb589, 0x16a5: 0xb5a1, 0x16a6: 0x89f5, 0x16a7: 0x0040, 0x16a8: 0x8a15, 0x16a9: 0x8a35, + 0x16aa: 0x8a55, 0x16ab: 0x8a35, 0x16ac: 0x8a75, 0x16ad: 0x8a95, 0x16ae: 0x8ab5, 0x16af: 0x0040, + 0x16b0: 0x0040, 0x16b1: 0x0040, 0x16b2: 0x0040, 0x16b3: 0x0040, 0x16b4: 0x0040, 0x16b5: 0x0040, + 0x16b6: 0x0040, 0x16b7: 0x0040, 0x16b8: 0x0040, 0x16b9: 0x0340, 0x16ba: 0x0340, 0x16bb: 0x0340, + 0x16bc: 0x0040, 0x16bd: 0x0040, 0x16be: 0x0040, 0x16bf: 0x0040, + // Block 0x5b, offset 0x16c0 + 0x16c0: 0x0208, 0x16c1: 0x0208, 0x16c2: 0x0208, 0x16c3: 0x0208, 0x16c4: 0x0208, 0x16c5: 0x0408, + 0x16c6: 0x0008, 0x16c7: 0x0408, 0x16c8: 0x0018, 0x16c9: 0x0408, 0x16ca: 0x0408, 0x16cb: 0x0008, + 0x16cc: 0x0008, 0x16cd: 0x0108, 0x16ce: 0x0408, 0x16cf: 0x0408, 0x16d0: 0x0408, 0x16d1: 0x0408, + 0x16d2: 0x0408, 0x16d3: 0x0208, 0x16d4: 0x0208, 0x16d5: 0x0208, 0x16d6: 0x0208, 0x16d7: 0x0108, + 0x16d8: 0x0208, 0x16d9: 0x0208, 0x16da: 0x0208, 0x16db: 0x0208, 0x16dc: 0x0208, 0x16dd: 0x0408, + 0x16de: 0x0208, 0x16df: 0x0208, 0x16e0: 0x0208, 0x16e1: 0x0408, 0x16e2: 0x0008, 0x16e3: 0x0008, + 0x16e4: 0x0408, 0x16e5: 0x1308, 0x16e6: 0x1308, 0x16e7: 0x0040, 0x16e8: 0x0040, 0x16e9: 0x0040, + 0x16ea: 0x0040, 0x16eb: 0x0218, 0x16ec: 0x0218, 0x16ed: 0x0218, 0x16ee: 0x0218, 0x16ef: 0x0418, + 0x16f0: 0x0018, 0x16f1: 0x0018, 0x16f2: 0x0018, 0x16f3: 0x0018, 0x16f4: 0x0018, 0x16f5: 0x0018, + 0x16f6: 0x0018, 0x16f7: 0x0040, 0x16f8: 0x0040, 0x16f9: 0x0040, 0x16fa: 0x0040, 0x16fb: 0x0040, + 0x16fc: 0x0040, 0x16fd: 0x0040, 0x16fe: 0x0040, 0x16ff: 0x0040, + // Block 0x5c, offset 0x1700 + 0x1700: 0x0208, 0x1701: 0x0408, 0x1702: 0x0208, 0x1703: 0x0408, 0x1704: 0x0408, 0x1705: 0x0408, + 0x1706: 0x0208, 0x1707: 0x0208, 0x1708: 0x0208, 0x1709: 0x0408, 0x170a: 0x0208, 0x170b: 0x0208, + 0x170c: 0x0408, 0x170d: 0x0208, 0x170e: 0x0408, 0x170f: 0x0408, 0x1710: 0x0208, 0x1711: 0x0408, + 0x1712: 0x0040, 0x1713: 0x0040, 0x1714: 0x0040, 0x1715: 0x0040, 0x1716: 0x0040, 0x1717: 0x0040, + 0x1718: 0x0040, 0x1719: 0x0018, 0x171a: 0x0018, 0x171b: 0x0018, 0x171c: 0x0018, 0x171d: 0x0040, + 0x171e: 0x0040, 0x171f: 0x0040, 0x1720: 0x0040, 0x1721: 0x0040, 0x1722: 0x0040, 0x1723: 0x0040, + 0x1724: 0x0040, 0x1725: 0x0040, 0x1726: 0x0040, 0x1727: 0x0040, 0x1728: 0x0040, 0x1729: 0x0418, + 0x172a: 0x0418, 0x172b: 0x0418, 0x172c: 0x0418, 0x172d: 0x0218, 0x172e: 0x0218, 0x172f: 0x0018, + 0x1730: 0x0040, 0x1731: 0x0040, 0x1732: 0x0040, 0x1733: 0x0040, 0x1734: 0x0040, 0x1735: 0x0040, + 0x1736: 0x0040, 0x1737: 0x0040, 0x1738: 0x0040, 0x1739: 0x0040, 0x173a: 0x0040, 0x173b: 0x0040, + 0x173c: 0x0040, 0x173d: 0x0040, 0x173e: 0x0040, 0x173f: 0x0040, + // Block 0x5d, offset 0x1740 + 0x1740: 0x1308, 0x1741: 0x1308, 0x1742: 0x1008, 0x1743: 0x1008, 0x1744: 0x0040, 0x1745: 0x0008, + 0x1746: 0x0008, 0x1747: 0x0008, 0x1748: 0x0008, 0x1749: 0x0008, 0x174a: 0x0008, 0x174b: 0x0008, + 0x174c: 0x0008, 0x174d: 0x0040, 0x174e: 0x0040, 0x174f: 0x0008, 0x1750: 0x0008, 0x1751: 0x0040, + 0x1752: 0x0040, 0x1753: 0x0008, 0x1754: 0x0008, 0x1755: 0x0008, 0x1756: 0x0008, 0x1757: 0x0008, + 0x1758: 0x0008, 0x1759: 0x0008, 0x175a: 0x0008, 0x175b: 0x0008, 0x175c: 0x0008, 0x175d: 0x0008, + 0x175e: 0x0008, 0x175f: 0x0008, 0x1760: 0x0008, 0x1761: 0x0008, 0x1762: 0x0008, 0x1763: 0x0008, + 0x1764: 0x0008, 0x1765: 0x0008, 0x1766: 0x0008, 0x1767: 0x0008, 0x1768: 0x0008, 0x1769: 0x0040, + 0x176a: 0x0008, 0x176b: 0x0008, 0x176c: 0x0008, 0x176d: 0x0008, 0x176e: 0x0008, 0x176f: 0x0008, + 0x1770: 0x0008, 0x1771: 0x0040, 0x1772: 0x0008, 0x1773: 0x0008, 0x1774: 0x0040, 0x1775: 0x0008, + 0x1776: 0x0008, 0x1777: 0x0008, 0x1778: 0x0008, 0x1779: 0x0008, 0x177a: 0x0040, 0x177b: 0x0040, + 0x177c: 0x1308, 0x177d: 0x0008, 0x177e: 0x1008, 0x177f: 0x1008, + // Block 0x5e, offset 0x1780 + 0x1780: 0x1308, 0x1781: 0x1008, 0x1782: 0x1008, 0x1783: 0x1008, 0x1784: 0x1008, 0x1785: 0x0040, + 0x1786: 0x0040, 0x1787: 0x1008, 0x1788: 0x1008, 0x1789: 0x0040, 0x178a: 0x0040, 0x178b: 0x1008, + 0x178c: 0x1008, 0x178d: 0x1808, 0x178e: 0x0040, 0x178f: 0x0040, 0x1790: 0x0008, 0x1791: 0x0040, + 0x1792: 0x0040, 0x1793: 0x0040, 0x1794: 0x0040, 0x1795: 0x0040, 0x1796: 0x0040, 0x1797: 0x1008, + 0x1798: 0x0040, 0x1799: 0x0040, 0x179a: 0x0040, 0x179b: 0x0040, 0x179c: 0x0040, 0x179d: 0x0008, + 0x179e: 0x0008, 0x179f: 0x0008, 0x17a0: 0x0008, 0x17a1: 0x0008, 0x17a2: 0x1008, 0x17a3: 0x1008, + 0x17a4: 0x0040, 0x17a5: 0x0040, 0x17a6: 0x1308, 0x17a7: 0x1308, 0x17a8: 0x1308, 0x17a9: 0x1308, + 0x17aa: 0x1308, 0x17ab: 0x1308, 0x17ac: 0x1308, 0x17ad: 0x0040, 0x17ae: 0x0040, 0x17af: 0x0040, + 0x17b0: 0x1308, 0x17b1: 0x1308, 0x17b2: 0x1308, 0x17b3: 0x1308, 0x17b4: 0x1308, 0x17b5: 0x0040, + 0x17b6: 0x0040, 0x17b7: 0x0040, 0x17b8: 0x0040, 0x17b9: 0x0040, 0x17ba: 0x0040, 0x17bb: 0x0040, + 0x17bc: 0x0040, 0x17bd: 0x0040, 0x17be: 0x0040, 0x17bf: 0x0040, + // Block 0x5f, offset 0x17c0 + 0x17c0: 0x0039, 0x17c1: 0x0ee9, 0x17c2: 0x1159, 0x17c3: 0x0ef9, 0x17c4: 0x0f09, 0x17c5: 0x1199, + 0x17c6: 0x0f31, 0x17c7: 0x0249, 0x17c8: 0x0f41, 0x17c9: 0x0259, 0x17ca: 0x0f51, 0x17cb: 0x0359, + 0x17cc: 0x0f61, 0x17cd: 0x0f71, 0x17ce: 0x00d9, 0x17cf: 0x0f99, 0x17d0: 0x2039, 0x17d1: 0x0269, + 0x17d2: 0x01d9, 0x17d3: 0x0fa9, 0x17d4: 0x0fb9, 0x17d5: 0x1089, 0x17d6: 0x0279, 0x17d7: 0x0369, + 0x17d8: 0x0289, 0x17d9: 0x13d1, 0x17da: 0x0039, 0x17db: 0x0ee9, 0x17dc: 0x1159, 0x17dd: 0x0ef9, + 0x17de: 0x0f09, 0x17df: 0x1199, 0x17e0: 0x0f31, 0x17e1: 0x0249, 0x17e2: 0x0f41, 0x17e3: 0x0259, + 0x17e4: 0x0f51, 0x17e5: 0x0359, 0x17e6: 0x0f61, 0x17e7: 0x0f71, 0x17e8: 0x00d9, 0x17e9: 0x0f99, + 0x17ea: 0x2039, 0x17eb: 0x0269, 0x17ec: 0x01d9, 0x17ed: 0x0fa9, 0x17ee: 0x0fb9, 0x17ef: 0x1089, + 0x17f0: 0x0279, 0x17f1: 0x0369, 0x17f2: 0x0289, 0x17f3: 0x13d1, 0x17f4: 0x0039, 0x17f5: 0x0ee9, + 0x17f6: 0x1159, 0x17f7: 0x0ef9, 0x17f8: 0x0f09, 0x17f9: 0x1199, 0x17fa: 0x0f31, 0x17fb: 0x0249, + 0x17fc: 0x0f41, 0x17fd: 0x0259, 0x17fe: 0x0f51, 0x17ff: 0x0359, + // Block 0x60, offset 0x1800 + 0x1800: 0x0f61, 0x1801: 0x0f71, 0x1802: 0x00d9, 0x1803: 0x0f99, 0x1804: 0x2039, 0x1805: 0x0269, + 0x1806: 0x01d9, 0x1807: 0x0fa9, 0x1808: 0x0fb9, 0x1809: 0x1089, 0x180a: 0x0279, 0x180b: 0x0369, + 0x180c: 0x0289, 0x180d: 0x13d1, 0x180e: 0x0039, 0x180f: 0x0ee9, 0x1810: 0x1159, 0x1811: 0x0ef9, + 0x1812: 0x0f09, 0x1813: 0x1199, 0x1814: 0x0f31, 0x1815: 0x0040, 0x1816: 0x0f41, 0x1817: 0x0259, + 0x1818: 0x0f51, 0x1819: 0x0359, 0x181a: 0x0f61, 0x181b: 0x0f71, 0x181c: 0x00d9, 0x181d: 0x0f99, + 0x181e: 0x2039, 0x181f: 0x0269, 0x1820: 0x01d9, 0x1821: 0x0fa9, 0x1822: 0x0fb9, 0x1823: 0x1089, + 0x1824: 0x0279, 0x1825: 0x0369, 0x1826: 0x0289, 0x1827: 0x13d1, 0x1828: 0x0039, 0x1829: 0x0ee9, + 0x182a: 0x1159, 0x182b: 0x0ef9, 0x182c: 0x0f09, 0x182d: 0x1199, 0x182e: 0x0f31, 0x182f: 0x0249, + 0x1830: 0x0f41, 0x1831: 0x0259, 0x1832: 0x0f51, 0x1833: 0x0359, 0x1834: 0x0f61, 0x1835: 0x0f71, + 0x1836: 0x00d9, 0x1837: 0x0f99, 0x1838: 0x2039, 0x1839: 0x0269, 0x183a: 0x01d9, 0x183b: 0x0fa9, + 0x183c: 0x0fb9, 0x183d: 0x1089, 0x183e: 0x0279, 0x183f: 0x0369, + // Block 0x61, offset 0x1840 + 0x1840: 0x0289, 0x1841: 0x13d1, 0x1842: 0x0039, 0x1843: 0x0ee9, 0x1844: 0x1159, 0x1845: 0x0ef9, + 0x1846: 0x0f09, 0x1847: 0x1199, 0x1848: 0x0f31, 0x1849: 0x0249, 0x184a: 0x0f41, 0x184b: 0x0259, + 0x184c: 0x0f51, 0x184d: 0x0359, 0x184e: 0x0f61, 0x184f: 0x0f71, 0x1850: 0x00d9, 0x1851: 0x0f99, + 0x1852: 0x2039, 0x1853: 0x0269, 0x1854: 0x01d9, 0x1855: 0x0fa9, 0x1856: 0x0fb9, 0x1857: 0x1089, + 0x1858: 0x0279, 0x1859: 0x0369, 0x185a: 0x0289, 0x185b: 0x13d1, 0x185c: 0x0039, 0x185d: 0x0040, + 0x185e: 0x1159, 0x185f: 0x0ef9, 0x1860: 0x0040, 0x1861: 0x0040, 0x1862: 0x0f31, 0x1863: 0x0040, + 0x1864: 0x0040, 0x1865: 0x0259, 0x1866: 0x0f51, 0x1867: 0x0040, 0x1868: 0x0040, 0x1869: 0x0f71, + 0x186a: 0x00d9, 0x186b: 0x0f99, 0x186c: 0x2039, 0x186d: 0x0040, 0x186e: 0x01d9, 0x186f: 0x0fa9, + 0x1870: 0x0fb9, 0x1871: 0x1089, 0x1872: 0x0279, 0x1873: 0x0369, 0x1874: 0x0289, 0x1875: 0x13d1, + 0x1876: 0x0039, 0x1877: 0x0ee9, 0x1878: 0x1159, 0x1879: 0x0ef9, 0x187a: 0x0040, 0x187b: 0x1199, + 0x187c: 0x0040, 0x187d: 0x0249, 0x187e: 0x0f41, 0x187f: 0x0259, + // Block 0x62, offset 0x1880 + 0x1880: 0x0f51, 0x1881: 0x0359, 0x1882: 0x0f61, 0x1883: 0x0f71, 0x1884: 0x0040, 0x1885: 0x0f99, + 0x1886: 0x2039, 0x1887: 0x0269, 0x1888: 0x01d9, 0x1889: 0x0fa9, 0x188a: 0x0fb9, 0x188b: 0x1089, + 0x188c: 0x0279, 0x188d: 0x0369, 0x188e: 0x0289, 0x188f: 0x13d1, 0x1890: 0x0039, 0x1891: 0x0ee9, + 0x1892: 0x1159, 0x1893: 0x0ef9, 0x1894: 0x0f09, 0x1895: 0x1199, 0x1896: 0x0f31, 0x1897: 0x0249, + 0x1898: 0x0f41, 0x1899: 0x0259, 0x189a: 0x0f51, 0x189b: 0x0359, 0x189c: 0x0f61, 0x189d: 0x0f71, + 0x189e: 0x00d9, 0x189f: 0x0f99, 0x18a0: 0x2039, 0x18a1: 0x0269, 0x18a2: 0x01d9, 0x18a3: 0x0fa9, + 0x18a4: 0x0fb9, 0x18a5: 0x1089, 0x18a6: 0x0279, 0x18a7: 0x0369, 0x18a8: 0x0289, 0x18a9: 0x13d1, + 0x18aa: 0x0039, 0x18ab: 0x0ee9, 0x18ac: 0x1159, 0x18ad: 0x0ef9, 0x18ae: 0x0f09, 0x18af: 0x1199, + 0x18b0: 0x0f31, 0x18b1: 0x0249, 0x18b2: 0x0f41, 0x18b3: 0x0259, 0x18b4: 0x0f51, 0x18b5: 0x0359, + 0x18b6: 0x0f61, 0x18b7: 0x0f71, 0x18b8: 0x00d9, 0x18b9: 0x0f99, 0x18ba: 0x2039, 0x18bb: 0x0269, + 0x18bc: 0x01d9, 0x18bd: 0x0fa9, 0x18be: 0x0fb9, 0x18bf: 0x1089, + // Block 0x63, offset 0x18c0 + 0x18c0: 0x0279, 0x18c1: 0x0369, 0x18c2: 0x0289, 0x18c3: 0x13d1, 0x18c4: 0x0039, 0x18c5: 0x0ee9, + 0x18c6: 0x0040, 0x18c7: 0x0ef9, 0x18c8: 0x0f09, 0x18c9: 0x1199, 0x18ca: 0x0f31, 0x18cb: 0x0040, + 0x18cc: 0x0040, 0x18cd: 0x0259, 0x18ce: 0x0f51, 0x18cf: 0x0359, 0x18d0: 0x0f61, 0x18d1: 0x0f71, + 0x18d2: 0x00d9, 0x18d3: 0x0f99, 0x18d4: 0x2039, 0x18d5: 0x0040, 0x18d6: 0x01d9, 0x18d7: 0x0fa9, + 0x18d8: 0x0fb9, 0x18d9: 0x1089, 0x18da: 0x0279, 0x18db: 0x0369, 0x18dc: 0x0289, 0x18dd: 0x0040, + 0x18de: 0x0039, 0x18df: 0x0ee9, 0x18e0: 0x1159, 0x18e1: 0x0ef9, 0x18e2: 0x0f09, 0x18e3: 0x1199, + 0x18e4: 0x0f31, 0x18e5: 0x0249, 0x18e6: 0x0f41, 0x18e7: 0x0259, 0x18e8: 0x0f51, 0x18e9: 0x0359, + 0x18ea: 0x0f61, 0x18eb: 0x0f71, 0x18ec: 0x00d9, 0x18ed: 0x0f99, 0x18ee: 0x2039, 0x18ef: 0x0269, + 0x18f0: 0x01d9, 0x18f1: 0x0fa9, 0x18f2: 0x0fb9, 0x18f3: 0x1089, 0x18f4: 0x0279, 0x18f5: 0x0369, + 0x18f6: 0x0289, 0x18f7: 0x13d1, 0x18f8: 0x0039, 0x18f9: 0x0ee9, 0x18fa: 0x0040, 0x18fb: 0x0ef9, + 0x18fc: 0x0f09, 0x18fd: 0x1199, 0x18fe: 0x0f31, 0x18ff: 0x0040, + // Block 0x64, offset 0x1900 + 0x1900: 0x0f41, 0x1901: 0x0259, 0x1902: 0x0f51, 0x1903: 0x0359, 0x1904: 0x0f61, 0x1905: 0x0040, + 0x1906: 0x00d9, 0x1907: 0x0040, 0x1908: 0x0040, 0x1909: 0x0040, 0x190a: 0x01d9, 0x190b: 0x0fa9, + 0x190c: 0x0fb9, 0x190d: 0x1089, 0x190e: 0x0279, 0x190f: 0x0369, 0x1910: 0x0289, 0x1911: 0x0040, + 0x1912: 0x0039, 0x1913: 0x0ee9, 0x1914: 0x1159, 0x1915: 0x0ef9, 0x1916: 0x0f09, 0x1917: 0x1199, + 0x1918: 0x0f31, 0x1919: 0x0249, 0x191a: 0x0f41, 0x191b: 0x0259, 0x191c: 0x0f51, 0x191d: 0x0359, + 0x191e: 0x0f61, 0x191f: 0x0f71, 0x1920: 0x00d9, 0x1921: 0x0f99, 0x1922: 0x2039, 0x1923: 0x0269, + 0x1924: 0x01d9, 0x1925: 0x0fa9, 0x1926: 0x0fb9, 0x1927: 0x1089, 0x1928: 0x0279, 0x1929: 0x0369, + 0x192a: 0x0289, 0x192b: 0x13d1, 0x192c: 0x0039, 0x192d: 0x0ee9, 0x192e: 0x1159, 0x192f: 0x0ef9, + 0x1930: 0x0f09, 0x1931: 0x1199, 0x1932: 0x0f31, 0x1933: 0x0249, 0x1934: 0x0f41, 0x1935: 0x0259, + 0x1936: 0x0f51, 0x1937: 0x0359, 0x1938: 0x0f61, 0x1939: 0x0f71, 0x193a: 0x00d9, 0x193b: 0x0f99, + 0x193c: 0x2039, 0x193d: 0x0269, 0x193e: 0x01d9, 0x193f: 0x0fa9, + // Block 0x65, offset 0x1940 + 0x1940: 0x0fb9, 0x1941: 0x1089, 0x1942: 0x0279, 0x1943: 0x0369, 0x1944: 0x0289, 0x1945: 0x13d1, + 0x1946: 0x0039, 0x1947: 0x0ee9, 0x1948: 0x1159, 0x1949: 0x0ef9, 0x194a: 0x0f09, 0x194b: 0x1199, + 0x194c: 0x0f31, 0x194d: 0x0249, 0x194e: 0x0f41, 0x194f: 0x0259, 0x1950: 0x0f51, 0x1951: 0x0359, + 0x1952: 0x0f61, 0x1953: 0x0f71, 0x1954: 0x00d9, 0x1955: 0x0f99, 0x1956: 0x2039, 0x1957: 0x0269, + 0x1958: 0x01d9, 0x1959: 0x0fa9, 0x195a: 0x0fb9, 0x195b: 0x1089, 0x195c: 0x0279, 0x195d: 0x0369, + 0x195e: 0x0289, 0x195f: 0x13d1, 0x1960: 0x0039, 0x1961: 0x0ee9, 0x1962: 0x1159, 0x1963: 0x0ef9, + 0x1964: 0x0f09, 0x1965: 0x1199, 0x1966: 0x0f31, 0x1967: 0x0249, 0x1968: 0x0f41, 0x1969: 0x0259, + 0x196a: 0x0f51, 0x196b: 0x0359, 0x196c: 0x0f61, 0x196d: 0x0f71, 0x196e: 0x00d9, 0x196f: 0x0f99, + 0x1970: 0x2039, 0x1971: 0x0269, 0x1972: 0x01d9, 0x1973: 0x0fa9, 0x1974: 0x0fb9, 0x1975: 0x1089, + 0x1976: 0x0279, 0x1977: 0x0369, 0x1978: 0x0289, 0x1979: 0x13d1, 0x197a: 0x0039, 0x197b: 0x0ee9, + 0x197c: 0x1159, 0x197d: 0x0ef9, 0x197e: 0x0f09, 0x197f: 0x1199, + // Block 0x66, offset 0x1980 + 0x1980: 0x0f31, 0x1981: 0x0249, 0x1982: 0x0f41, 0x1983: 0x0259, 0x1984: 0x0f51, 0x1985: 0x0359, + 0x1986: 0x0f61, 0x1987: 0x0f71, 0x1988: 0x00d9, 0x1989: 0x0f99, 0x198a: 0x2039, 0x198b: 0x0269, + 0x198c: 0x01d9, 0x198d: 0x0fa9, 0x198e: 0x0fb9, 0x198f: 0x1089, 0x1990: 0x0279, 0x1991: 0x0369, + 0x1992: 0x0289, 0x1993: 0x13d1, 0x1994: 0x0039, 0x1995: 0x0ee9, 0x1996: 0x1159, 0x1997: 0x0ef9, + 0x1998: 0x0f09, 0x1999: 0x1199, 0x199a: 0x0f31, 0x199b: 0x0249, 0x199c: 0x0f41, 0x199d: 0x0259, + 0x199e: 0x0f51, 0x199f: 0x0359, 0x19a0: 0x0f61, 0x19a1: 0x0f71, 0x19a2: 0x00d9, 0x19a3: 0x0f99, + 0x19a4: 0x2039, 0x19a5: 0x0269, 0x19a6: 0x01d9, 0x19a7: 0x0fa9, 0x19a8: 0x0fb9, 0x19a9: 0x1089, + 0x19aa: 0x0279, 0x19ab: 0x0369, 0x19ac: 0x0289, 0x19ad: 0x13d1, 0x19ae: 0x0039, 0x19af: 0x0ee9, + 0x19b0: 0x1159, 0x19b1: 0x0ef9, 0x19b2: 0x0f09, 0x19b3: 0x1199, 0x19b4: 0x0f31, 0x19b5: 0x0249, + 0x19b6: 0x0f41, 0x19b7: 0x0259, 0x19b8: 0x0f51, 0x19b9: 0x0359, 0x19ba: 0x0f61, 0x19bb: 0x0f71, + 0x19bc: 0x00d9, 0x19bd: 0x0f99, 0x19be: 0x2039, 0x19bf: 0x0269, + // Block 0x67, offset 0x19c0 + 0x19c0: 0x01d9, 0x19c1: 0x0fa9, 0x19c2: 0x0fb9, 0x19c3: 0x1089, 0x19c4: 0x0279, 0x19c5: 0x0369, + 0x19c6: 0x0289, 0x19c7: 0x13d1, 0x19c8: 0x0039, 0x19c9: 0x0ee9, 0x19ca: 0x1159, 0x19cb: 0x0ef9, + 0x19cc: 0x0f09, 0x19cd: 0x1199, 0x19ce: 0x0f31, 0x19cf: 0x0249, 0x19d0: 0x0f41, 0x19d1: 0x0259, + 0x19d2: 0x0f51, 0x19d3: 0x0359, 0x19d4: 0x0f61, 0x19d5: 0x0f71, 0x19d6: 0x00d9, 0x19d7: 0x0f99, + 0x19d8: 0x2039, 0x19d9: 0x0269, 0x19da: 0x01d9, 0x19db: 0x0fa9, 0x19dc: 0x0fb9, 0x19dd: 0x1089, + 0x19de: 0x0279, 0x19df: 0x0369, 0x19e0: 0x0289, 0x19e1: 0x13d1, 0x19e2: 0x0039, 0x19e3: 0x0ee9, + 0x19e4: 0x1159, 0x19e5: 0x0ef9, 0x19e6: 0x0f09, 0x19e7: 0x1199, 0x19e8: 0x0f31, 0x19e9: 0x0249, + 0x19ea: 0x0f41, 0x19eb: 0x0259, 0x19ec: 0x0f51, 0x19ed: 0x0359, 0x19ee: 0x0f61, 0x19ef: 0x0f71, + 0x19f0: 0x00d9, 0x19f1: 0x0f99, 0x19f2: 0x2039, 0x19f3: 0x0269, 0x19f4: 0x01d9, 0x19f5: 0x0fa9, + 0x19f6: 0x0fb9, 0x19f7: 0x1089, 0x19f8: 0x0279, 0x19f9: 0x0369, 0x19fa: 0x0289, 0x19fb: 0x13d1, + 0x19fc: 0x0039, 0x19fd: 0x0ee9, 0x19fe: 0x1159, 0x19ff: 0x0ef9, + // Block 0x68, offset 0x1a00 + 0x1a00: 0x0f09, 0x1a01: 0x1199, 0x1a02: 0x0f31, 0x1a03: 0x0249, 0x1a04: 0x0f41, 0x1a05: 0x0259, + 0x1a06: 0x0f51, 0x1a07: 0x0359, 0x1a08: 0x0f61, 0x1a09: 0x0f71, 0x1a0a: 0x00d9, 0x1a0b: 0x0f99, + 0x1a0c: 0x2039, 0x1a0d: 0x0269, 0x1a0e: 0x01d9, 0x1a0f: 0x0fa9, 0x1a10: 0x0fb9, 0x1a11: 0x1089, + 0x1a12: 0x0279, 0x1a13: 0x0369, 0x1a14: 0x0289, 0x1a15: 0x13d1, 0x1a16: 0x0039, 0x1a17: 0x0ee9, + 0x1a18: 0x1159, 0x1a19: 0x0ef9, 0x1a1a: 0x0f09, 0x1a1b: 0x1199, 0x1a1c: 0x0f31, 0x1a1d: 0x0249, + 0x1a1e: 0x0f41, 0x1a1f: 0x0259, 0x1a20: 0x0f51, 0x1a21: 0x0359, 0x1a22: 0x0f61, 0x1a23: 0x0f71, + 0x1a24: 0x00d9, 0x1a25: 0x0f99, 0x1a26: 0x2039, 0x1a27: 0x0269, 0x1a28: 0x01d9, 0x1a29: 0x0fa9, + 0x1a2a: 0x0fb9, 0x1a2b: 0x1089, 0x1a2c: 0x0279, 0x1a2d: 0x0369, 0x1a2e: 0x0289, 0x1a2f: 0x13d1, + 0x1a30: 0x0039, 0x1a31: 0x0ee9, 0x1a32: 0x1159, 0x1a33: 0x0ef9, 0x1a34: 0x0f09, 0x1a35: 0x1199, + 0x1a36: 0x0f31, 0x1a37: 0x0249, 0x1a38: 0x0f41, 0x1a39: 0x0259, 0x1a3a: 0x0f51, 0x1a3b: 0x0359, + 0x1a3c: 0x0f61, 0x1a3d: 0x0f71, 0x1a3e: 0x00d9, 0x1a3f: 0x0f99, + // Block 0x69, offset 0x1a40 + 0x1a40: 0x2039, 0x1a41: 0x0269, 0x1a42: 0x01d9, 0x1a43: 0x0fa9, 0x1a44: 0x0fb9, 0x1a45: 0x1089, + 0x1a46: 0x0279, 0x1a47: 0x0369, 0x1a48: 0x0289, 0x1a49: 0x13d1, 0x1a4a: 0x0039, 0x1a4b: 0x0ee9, + 0x1a4c: 0x1159, 0x1a4d: 0x0ef9, 0x1a4e: 0x0f09, 0x1a4f: 0x1199, 0x1a50: 0x0f31, 0x1a51: 0x0249, + 0x1a52: 0x0f41, 0x1a53: 0x0259, 0x1a54: 0x0f51, 0x1a55: 0x0359, 0x1a56: 0x0f61, 0x1a57: 0x0f71, + 0x1a58: 0x00d9, 0x1a59: 0x0f99, 0x1a5a: 0x2039, 0x1a5b: 0x0269, 0x1a5c: 0x01d9, 0x1a5d: 0x0fa9, + 0x1a5e: 0x0fb9, 0x1a5f: 0x1089, 0x1a60: 0x0279, 0x1a61: 0x0369, 0x1a62: 0x0289, 0x1a63: 0x13d1, + 0x1a64: 0xba81, 0x1a65: 0xba99, 0x1a66: 0x0040, 0x1a67: 0x0040, 0x1a68: 0xbab1, 0x1a69: 0x1099, + 0x1a6a: 0x10b1, 0x1a6b: 0x10c9, 0x1a6c: 0xbac9, 0x1a6d: 0xbae1, 0x1a6e: 0xbaf9, 0x1a6f: 0x1429, + 0x1a70: 0x1a31, 0x1a71: 0xbb11, 0x1a72: 0xbb29, 0x1a73: 0xbb41, 0x1a74: 0xbb59, 0x1a75: 0xbb71, + 0x1a76: 0xbb89, 0x1a77: 0x2109, 0x1a78: 0x1111, 0x1a79: 0x1429, 0x1a7a: 0xbba1, 0x1a7b: 0xbbb9, + 0x1a7c: 0xbbd1, 0x1a7d: 0x10e1, 0x1a7e: 0x10f9, 0x1a7f: 0xbbe9, + // Block 0x6a, offset 0x1a80 + 0x1a80: 0x2079, 0x1a81: 0xbc01, 0x1a82: 0xbab1, 0x1a83: 0x1099, 0x1a84: 0x10b1, 0x1a85: 0x10c9, + 0x1a86: 0xbac9, 0x1a87: 0xbae1, 0x1a88: 0xbaf9, 0x1a89: 0x1429, 0x1a8a: 0x1a31, 0x1a8b: 0xbb11, + 0x1a8c: 0xbb29, 0x1a8d: 0xbb41, 0x1a8e: 0xbb59, 0x1a8f: 0xbb71, 0x1a90: 0xbb89, 0x1a91: 0x2109, + 0x1a92: 0x1111, 0x1a93: 0xbba1, 0x1a94: 0xbba1, 0x1a95: 0xbbb9, 0x1a96: 0xbbd1, 0x1a97: 0x10e1, + 0x1a98: 0x10f9, 0x1a99: 0xbbe9, 0x1a9a: 0x2079, 0x1a9b: 0xbc21, 0x1a9c: 0xbac9, 0x1a9d: 0x1429, + 0x1a9e: 0xbb11, 0x1a9f: 0x10e1, 0x1aa0: 0x1111, 0x1aa1: 0x2109, 0x1aa2: 0xbab1, 0x1aa3: 0x1099, + 0x1aa4: 0x10b1, 0x1aa5: 0x10c9, 0x1aa6: 0xbac9, 0x1aa7: 0xbae1, 0x1aa8: 0xbaf9, 0x1aa9: 0x1429, + 0x1aaa: 0x1a31, 0x1aab: 0xbb11, 0x1aac: 0xbb29, 0x1aad: 0xbb41, 0x1aae: 0xbb59, 0x1aaf: 0xbb71, + 0x1ab0: 0xbb89, 0x1ab1: 0x2109, 0x1ab2: 0x1111, 0x1ab3: 0x1429, 0x1ab4: 0xbba1, 0x1ab5: 0xbbb9, + 0x1ab6: 0xbbd1, 0x1ab7: 0x10e1, 0x1ab8: 0x10f9, 0x1ab9: 0xbbe9, 0x1aba: 0x2079, 0x1abb: 0xbc01, + 0x1abc: 0xbab1, 0x1abd: 0x1099, 0x1abe: 0x10b1, 0x1abf: 0x10c9, + // Block 0x6b, offset 0x1ac0 + 0x1ac0: 0xbac9, 0x1ac1: 0xbae1, 0x1ac2: 0xbaf9, 0x1ac3: 0x1429, 0x1ac4: 0x1a31, 0x1ac5: 0xbb11, + 0x1ac6: 0xbb29, 0x1ac7: 0xbb41, 0x1ac8: 0xbb59, 0x1ac9: 0xbb71, 0x1aca: 0xbb89, 0x1acb: 0x2109, + 0x1acc: 0x1111, 0x1acd: 0xbba1, 0x1ace: 0xbba1, 0x1acf: 0xbbb9, 0x1ad0: 0xbbd1, 0x1ad1: 0x10e1, + 0x1ad2: 0x10f9, 0x1ad3: 0xbbe9, 0x1ad4: 0x2079, 0x1ad5: 0xbc21, 0x1ad6: 0xbac9, 0x1ad7: 0x1429, + 0x1ad8: 0xbb11, 0x1ad9: 0x10e1, 0x1ada: 0x1111, 0x1adb: 0x2109, 0x1adc: 0xbab1, 0x1add: 0x1099, + 0x1ade: 0x10b1, 0x1adf: 0x10c9, 0x1ae0: 0xbac9, 0x1ae1: 0xbae1, 0x1ae2: 0xbaf9, 0x1ae3: 0x1429, + 0x1ae4: 0x1a31, 0x1ae5: 0xbb11, 0x1ae6: 0xbb29, 0x1ae7: 0xbb41, 0x1ae8: 0xbb59, 0x1ae9: 0xbb71, + 0x1aea: 0xbb89, 0x1aeb: 0x2109, 0x1aec: 0x1111, 0x1aed: 0x1429, 0x1aee: 0xbba1, 0x1aef: 0xbbb9, + 0x1af0: 0xbbd1, 0x1af1: 0x10e1, 0x1af2: 0x10f9, 0x1af3: 0xbbe9, 0x1af4: 0x2079, 0x1af5: 0xbc01, + 0x1af6: 0xbab1, 0x1af7: 0x1099, 0x1af8: 0x10b1, 0x1af9: 0x10c9, 0x1afa: 0xbac9, 0x1afb: 0xbae1, + 0x1afc: 0xbaf9, 0x1afd: 0x1429, 0x1afe: 0x1a31, 0x1aff: 0xbb11, + // Block 0x6c, offset 0x1b00 + 0x1b00: 0xbb29, 0x1b01: 0xbb41, 0x1b02: 0xbb59, 0x1b03: 0xbb71, 0x1b04: 0xbb89, 0x1b05: 0x2109, + 0x1b06: 0x1111, 0x1b07: 0xbba1, 0x1b08: 0xbba1, 0x1b09: 0xbbb9, 0x1b0a: 0xbbd1, 0x1b0b: 0x10e1, + 0x1b0c: 0x10f9, 0x1b0d: 0xbbe9, 0x1b0e: 0x2079, 0x1b0f: 0xbc21, 0x1b10: 0xbac9, 0x1b11: 0x1429, + 0x1b12: 0xbb11, 0x1b13: 0x10e1, 0x1b14: 0x1111, 0x1b15: 0x2109, 0x1b16: 0xbab1, 0x1b17: 0x1099, + 0x1b18: 0x10b1, 0x1b19: 0x10c9, 0x1b1a: 0xbac9, 0x1b1b: 0xbae1, 0x1b1c: 0xbaf9, 0x1b1d: 0x1429, + 0x1b1e: 0x1a31, 0x1b1f: 0xbb11, 0x1b20: 0xbb29, 0x1b21: 0xbb41, 0x1b22: 0xbb59, 0x1b23: 0xbb71, + 0x1b24: 0xbb89, 0x1b25: 0x2109, 0x1b26: 0x1111, 0x1b27: 0x1429, 0x1b28: 0xbba1, 0x1b29: 0xbbb9, + 0x1b2a: 0xbbd1, 0x1b2b: 0x10e1, 0x1b2c: 0x10f9, 0x1b2d: 0xbbe9, 0x1b2e: 0x2079, 0x1b2f: 0xbc01, + 0x1b30: 0xbab1, 0x1b31: 0x1099, 0x1b32: 0x10b1, 0x1b33: 0x10c9, 0x1b34: 0xbac9, 0x1b35: 0xbae1, + 0x1b36: 0xbaf9, 0x1b37: 0x1429, 0x1b38: 0x1a31, 0x1b39: 0xbb11, 0x1b3a: 0xbb29, 0x1b3b: 0xbb41, + 0x1b3c: 0xbb59, 0x1b3d: 0xbb71, 0x1b3e: 0xbb89, 0x1b3f: 0x2109, + // Block 0x6d, offset 0x1b40 + 0x1b40: 0x1111, 0x1b41: 0xbba1, 0x1b42: 0xbba1, 0x1b43: 0xbbb9, 0x1b44: 0xbbd1, 0x1b45: 0x10e1, + 0x1b46: 0x10f9, 0x1b47: 0xbbe9, 0x1b48: 0x2079, 0x1b49: 0xbc21, 0x1b4a: 0xbac9, 0x1b4b: 0x1429, + 0x1b4c: 0xbb11, 0x1b4d: 0x10e1, 0x1b4e: 0x1111, 0x1b4f: 0x2109, 0x1b50: 0xbab1, 0x1b51: 0x1099, + 0x1b52: 0x10b1, 0x1b53: 0x10c9, 0x1b54: 0xbac9, 0x1b55: 0xbae1, 0x1b56: 0xbaf9, 0x1b57: 0x1429, + 0x1b58: 0x1a31, 0x1b59: 0xbb11, 0x1b5a: 0xbb29, 0x1b5b: 0xbb41, 0x1b5c: 0xbb59, 0x1b5d: 0xbb71, + 0x1b5e: 0xbb89, 0x1b5f: 0x2109, 0x1b60: 0x1111, 0x1b61: 0x1429, 0x1b62: 0xbba1, 0x1b63: 0xbbb9, + 0x1b64: 0xbbd1, 0x1b65: 0x10e1, 0x1b66: 0x10f9, 0x1b67: 0xbbe9, 0x1b68: 0x2079, 0x1b69: 0xbc01, + 0x1b6a: 0xbab1, 0x1b6b: 0x1099, 0x1b6c: 0x10b1, 0x1b6d: 0x10c9, 0x1b6e: 0xbac9, 0x1b6f: 0xbae1, + 0x1b70: 0xbaf9, 0x1b71: 0x1429, 0x1b72: 0x1a31, 0x1b73: 0xbb11, 0x1b74: 0xbb29, 0x1b75: 0xbb41, + 0x1b76: 0xbb59, 0x1b77: 0xbb71, 0x1b78: 0xbb89, 0x1b79: 0x2109, 0x1b7a: 0x1111, 0x1b7b: 0xbba1, + 0x1b7c: 0xbba1, 0x1b7d: 0xbbb9, 0x1b7e: 0xbbd1, 0x1b7f: 0x10e1, + // Block 0x6e, offset 0x1b80 + 0x1b80: 0x10f9, 0x1b81: 0xbbe9, 0x1b82: 0x2079, 0x1b83: 0xbc21, 0x1b84: 0xbac9, 0x1b85: 0x1429, + 0x1b86: 0xbb11, 0x1b87: 0x10e1, 0x1b88: 0x1111, 0x1b89: 0x2109, 0x1b8a: 0xbc41, 0x1b8b: 0xbc41, + 0x1b8c: 0x0040, 0x1b8d: 0x0040, 0x1b8e: 0x1f41, 0x1b8f: 0x00c9, 0x1b90: 0x0069, 0x1b91: 0x0079, + 0x1b92: 0x1f51, 0x1b93: 0x1f61, 0x1b94: 0x1f71, 0x1b95: 0x1f81, 0x1b96: 0x1f91, 0x1b97: 0x1fa1, + 0x1b98: 0x1f41, 0x1b99: 0x00c9, 0x1b9a: 0x0069, 0x1b9b: 0x0079, 0x1b9c: 0x1f51, 0x1b9d: 0x1f61, + 0x1b9e: 0x1f71, 0x1b9f: 0x1f81, 0x1ba0: 0x1f91, 0x1ba1: 0x1fa1, 0x1ba2: 0x1f41, 0x1ba3: 0x00c9, + 0x1ba4: 0x0069, 0x1ba5: 0x0079, 0x1ba6: 0x1f51, 0x1ba7: 0x1f61, 0x1ba8: 0x1f71, 0x1ba9: 0x1f81, + 0x1baa: 0x1f91, 0x1bab: 0x1fa1, 0x1bac: 0x1f41, 0x1bad: 0x00c9, 0x1bae: 0x0069, 0x1baf: 0x0079, + 0x1bb0: 0x1f51, 0x1bb1: 0x1f61, 0x1bb2: 0x1f71, 0x1bb3: 0x1f81, 0x1bb4: 0x1f91, 0x1bb5: 0x1fa1, + 0x1bb6: 0x1f41, 0x1bb7: 0x00c9, 0x1bb8: 0x0069, 0x1bb9: 0x0079, 0x1bba: 0x1f51, 0x1bbb: 0x1f61, + 0x1bbc: 0x1f71, 0x1bbd: 0x1f81, 0x1bbe: 0x1f91, 0x1bbf: 0x1fa1, + // Block 0x6f, offset 0x1bc0 + 0x1bc0: 0xe115, 0x1bc1: 0xe115, 0x1bc2: 0xe135, 0x1bc3: 0xe135, 0x1bc4: 0xe115, 0x1bc5: 0xe115, + 0x1bc6: 0xe175, 0x1bc7: 0xe175, 0x1bc8: 0xe115, 0x1bc9: 0xe115, 0x1bca: 0xe135, 0x1bcb: 0xe135, + 0x1bcc: 0xe115, 0x1bcd: 0xe115, 0x1bce: 0xe1f5, 0x1bcf: 0xe1f5, 0x1bd0: 0xe115, 0x1bd1: 0xe115, + 0x1bd2: 0xe135, 0x1bd3: 0xe135, 0x1bd4: 0xe115, 0x1bd5: 0xe115, 0x1bd6: 0xe175, 0x1bd7: 0xe175, + 0x1bd8: 0xe115, 0x1bd9: 0xe115, 0x1bda: 0xe135, 0x1bdb: 0xe135, 0x1bdc: 0xe115, 0x1bdd: 0xe115, + 0x1bde: 0x8b05, 0x1bdf: 0x8b05, 0x1be0: 0x04b5, 0x1be1: 0x04b5, 0x1be2: 0x0208, 0x1be3: 0x0208, + 0x1be4: 0x0208, 0x1be5: 0x0208, 0x1be6: 0x0208, 0x1be7: 0x0208, 0x1be8: 0x0208, 0x1be9: 0x0208, + 0x1bea: 0x0208, 0x1beb: 0x0208, 0x1bec: 0x0208, 0x1bed: 0x0208, 0x1bee: 0x0208, 0x1bef: 0x0208, + 0x1bf0: 0x0208, 0x1bf1: 0x0208, 0x1bf2: 0x0208, 0x1bf3: 0x0208, 0x1bf4: 0x0208, 0x1bf5: 0x0208, + 0x1bf6: 0x0208, 0x1bf7: 0x0208, 0x1bf8: 0x0208, 0x1bf9: 0x0208, 0x1bfa: 0x0208, 0x1bfb: 0x0208, + 0x1bfc: 0x0208, 0x1bfd: 0x0208, 0x1bfe: 0x0208, 0x1bff: 0x0208, + // Block 0x70, offset 0x1c00 + 0x1c00: 0xb189, 0x1c01: 0xb1a1, 0x1c02: 0xb201, 0x1c03: 0xb249, 0x1c04: 0x0040, 0x1c05: 0xb411, + 0x1c06: 0xb291, 0x1c07: 0xb219, 0x1c08: 0xb309, 0x1c09: 0xb429, 0x1c0a: 0xb399, 0x1c0b: 0xb3b1, + 0x1c0c: 0xb3c9, 0x1c0d: 0xb3e1, 0x1c0e: 0xb2a9, 0x1c0f: 0xb339, 0x1c10: 0xb369, 0x1c11: 0xb2d9, + 0x1c12: 0xb381, 0x1c13: 0xb279, 0x1c14: 0xb2c1, 0x1c15: 0xb1d1, 0x1c16: 0xb1e9, 0x1c17: 0xb231, + 0x1c18: 0xb261, 0x1c19: 0xb2f1, 0x1c1a: 0xb321, 0x1c1b: 0xb351, 0x1c1c: 0xbc59, 0x1c1d: 0x7949, + 0x1c1e: 0xbc71, 0x1c1f: 0xbc89, 0x1c20: 0x0040, 0x1c21: 0xb1a1, 0x1c22: 0xb201, 0x1c23: 0x0040, + 0x1c24: 0xb3f9, 0x1c25: 0x0040, 0x1c26: 0x0040, 0x1c27: 0xb219, 0x1c28: 0x0040, 0x1c29: 0xb429, + 0x1c2a: 0xb399, 0x1c2b: 0xb3b1, 0x1c2c: 0xb3c9, 0x1c2d: 0xb3e1, 0x1c2e: 0xb2a9, 0x1c2f: 0xb339, + 0x1c30: 0xb369, 0x1c31: 0xb2d9, 0x1c32: 0xb381, 0x1c33: 0x0040, 0x1c34: 0xb2c1, 0x1c35: 0xb1d1, + 0x1c36: 0xb1e9, 0x1c37: 0xb231, 0x1c38: 0x0040, 0x1c39: 0xb2f1, 0x1c3a: 0x0040, 0x1c3b: 0xb351, + 0x1c3c: 0x0040, 0x1c3d: 0x0040, 0x1c3e: 0x0040, 0x1c3f: 0x0040, + // Block 0x71, offset 0x1c40 + 0x1c40: 0x0040, 0x1c41: 0x0040, 0x1c42: 0xb201, 0x1c43: 0x0040, 0x1c44: 0x0040, 0x1c45: 0x0040, + 0x1c46: 0x0040, 0x1c47: 0xb219, 0x1c48: 0x0040, 0x1c49: 0xb429, 0x1c4a: 0x0040, 0x1c4b: 0xb3b1, + 0x1c4c: 0x0040, 0x1c4d: 0xb3e1, 0x1c4e: 0xb2a9, 0x1c4f: 0xb339, 0x1c50: 0x0040, 0x1c51: 0xb2d9, + 0x1c52: 0xb381, 0x1c53: 0x0040, 0x1c54: 0xb2c1, 0x1c55: 0x0040, 0x1c56: 0x0040, 0x1c57: 0xb231, + 0x1c58: 0x0040, 0x1c59: 0xb2f1, 0x1c5a: 0x0040, 0x1c5b: 0xb351, 0x1c5c: 0x0040, 0x1c5d: 0x7949, + 0x1c5e: 0x0040, 0x1c5f: 0xbc89, 0x1c60: 0x0040, 0x1c61: 0xb1a1, 0x1c62: 0xb201, 0x1c63: 0x0040, + 0x1c64: 0xb3f9, 0x1c65: 0x0040, 0x1c66: 0x0040, 0x1c67: 0xb219, 0x1c68: 0xb309, 0x1c69: 0xb429, + 0x1c6a: 0xb399, 0x1c6b: 0x0040, 0x1c6c: 0xb3c9, 0x1c6d: 0xb3e1, 0x1c6e: 0xb2a9, 0x1c6f: 0xb339, + 0x1c70: 0xb369, 0x1c71: 0xb2d9, 0x1c72: 0xb381, 0x1c73: 0x0040, 0x1c74: 0xb2c1, 0x1c75: 0xb1d1, + 0x1c76: 0xb1e9, 0x1c77: 0xb231, 0x1c78: 0x0040, 0x1c79: 0xb2f1, 0x1c7a: 0xb321, 0x1c7b: 0xb351, + 0x1c7c: 0xbc59, 0x1c7d: 0x0040, 0x1c7e: 0xbc71, 0x1c7f: 0x0040, + // Block 0x72, offset 0x1c80 + 0x1c80: 0xb189, 0x1c81: 0xb1a1, 0x1c82: 0xb201, 0x1c83: 0xb249, 0x1c84: 0xb3f9, 0x1c85: 0xb411, + 0x1c86: 0xb291, 0x1c87: 0xb219, 0x1c88: 0xb309, 0x1c89: 0xb429, 0x1c8a: 0x0040, 0x1c8b: 0xb3b1, + 0x1c8c: 0xb3c9, 0x1c8d: 0xb3e1, 0x1c8e: 0xb2a9, 0x1c8f: 0xb339, 0x1c90: 0xb369, 0x1c91: 0xb2d9, + 0x1c92: 0xb381, 0x1c93: 0xb279, 0x1c94: 0xb2c1, 0x1c95: 0xb1d1, 0x1c96: 0xb1e9, 0x1c97: 0xb231, + 0x1c98: 0xb261, 0x1c99: 0xb2f1, 0x1c9a: 0xb321, 0x1c9b: 0xb351, 0x1c9c: 0x0040, 0x1c9d: 0x0040, + 0x1c9e: 0x0040, 0x1c9f: 0x0040, 0x1ca0: 0x0040, 0x1ca1: 0xb1a1, 0x1ca2: 0xb201, 0x1ca3: 0xb249, + 0x1ca4: 0x0040, 0x1ca5: 0xb411, 0x1ca6: 0xb291, 0x1ca7: 0xb219, 0x1ca8: 0xb309, 0x1ca9: 0xb429, + 0x1caa: 0x0040, 0x1cab: 0xb3b1, 0x1cac: 0xb3c9, 0x1cad: 0xb3e1, 0x1cae: 0xb2a9, 0x1caf: 0xb339, + 0x1cb0: 0xb369, 0x1cb1: 0xb2d9, 0x1cb2: 0xb381, 0x1cb3: 0xb279, 0x1cb4: 0xb2c1, 0x1cb5: 0xb1d1, + 0x1cb6: 0xb1e9, 0x1cb7: 0xb231, 0x1cb8: 0xb261, 0x1cb9: 0xb2f1, 0x1cba: 0xb321, 0x1cbb: 0xb351, + 0x1cbc: 0x0040, 0x1cbd: 0x0040, 0x1cbe: 0x0040, 0x1cbf: 0x0040, + // Block 0x73, offset 0x1cc0 + 0x1cc0: 0x0040, 0x1cc1: 0xbca2, 0x1cc2: 0xbcba, 0x1cc3: 0xbcd2, 0x1cc4: 0xbcea, 0x1cc5: 0xbd02, + 0x1cc6: 0xbd1a, 0x1cc7: 0xbd32, 0x1cc8: 0xbd4a, 0x1cc9: 0xbd62, 0x1cca: 0xbd7a, 0x1ccb: 0x0018, + 0x1ccc: 0x0018, 0x1ccd: 0x0040, 0x1cce: 0x0040, 0x1ccf: 0x0040, 0x1cd0: 0xbd92, 0x1cd1: 0xbdb2, + 0x1cd2: 0xbdd2, 0x1cd3: 0xbdf2, 0x1cd4: 0xbe12, 0x1cd5: 0xbe32, 0x1cd6: 0xbe52, 0x1cd7: 0xbe72, + 0x1cd8: 0xbe92, 0x1cd9: 0xbeb2, 0x1cda: 0xbed2, 0x1cdb: 0xbef2, 0x1cdc: 0xbf12, 0x1cdd: 0xbf32, + 0x1cde: 0xbf52, 0x1cdf: 0xbf72, 0x1ce0: 0xbf92, 0x1ce1: 0xbfb2, 0x1ce2: 0xbfd2, 0x1ce3: 0xbff2, + 0x1ce4: 0xc012, 0x1ce5: 0xc032, 0x1ce6: 0xc052, 0x1ce7: 0xc072, 0x1ce8: 0xc092, 0x1ce9: 0xc0b2, + 0x1cea: 0xc0d1, 0x1ceb: 0x1159, 0x1cec: 0x0269, 0x1ced: 0x6671, 0x1cee: 0xc111, 0x1cef: 0x0040, + 0x1cf0: 0x0039, 0x1cf1: 0x0ee9, 0x1cf2: 0x1159, 0x1cf3: 0x0ef9, 0x1cf4: 0x0f09, 0x1cf5: 0x1199, + 0x1cf6: 0x0f31, 0x1cf7: 0x0249, 0x1cf8: 0x0f41, 0x1cf9: 0x0259, 0x1cfa: 0x0f51, 0x1cfb: 0x0359, + 0x1cfc: 0x0f61, 0x1cfd: 0x0f71, 0x1cfe: 0x00d9, 0x1cff: 0x0f99, + // Block 0x74, offset 0x1d00 + 0x1d00: 0x2039, 0x1d01: 0x0269, 0x1d02: 0x01d9, 0x1d03: 0x0fa9, 0x1d04: 0x0fb9, 0x1d05: 0x1089, + 0x1d06: 0x0279, 0x1d07: 0x0369, 0x1d08: 0x0289, 0x1d09: 0x13d1, 0x1d0a: 0xc129, 0x1d0b: 0x65b1, + 0x1d0c: 0xc141, 0x1d0d: 0x1441, 0x1d0e: 0xc159, 0x1d0f: 0xc179, 0x1d10: 0x0018, 0x1d11: 0x0018, + 0x1d12: 0x0018, 0x1d13: 0x0018, 0x1d14: 0x0018, 0x1d15: 0x0018, 0x1d16: 0x0018, 0x1d17: 0x0018, + 0x1d18: 0x0018, 0x1d19: 0x0018, 0x1d1a: 0x0018, 0x1d1b: 0x0018, 0x1d1c: 0x0018, 0x1d1d: 0x0018, + 0x1d1e: 0x0018, 0x1d1f: 0x0018, 0x1d20: 0x0018, 0x1d21: 0x0018, 0x1d22: 0x0018, 0x1d23: 0x0018, + 0x1d24: 0x0018, 0x1d25: 0x0018, 0x1d26: 0x0018, 0x1d27: 0x0018, 0x1d28: 0x0018, 0x1d29: 0x0018, + 0x1d2a: 0xc191, 0x1d2b: 0xc1a9, 0x1d2c: 0x0040, 0x1d2d: 0x0040, 0x1d2e: 0x0040, 0x1d2f: 0x0040, + 0x1d30: 0x0018, 0x1d31: 0x0018, 0x1d32: 0x0018, 0x1d33: 0x0018, 0x1d34: 0x0018, 0x1d35: 0x0018, + 0x1d36: 0x0018, 0x1d37: 0x0018, 0x1d38: 0x0018, 0x1d39: 0x0018, 0x1d3a: 0x0018, 0x1d3b: 0x0018, + 0x1d3c: 0x0018, 0x1d3d: 0x0018, 0x1d3e: 0x0018, 0x1d3f: 0x0018, + // Block 0x75, offset 0x1d40 + 0x1d40: 0xc1d9, 0x1d41: 0xc211, 0x1d42: 0xc249, 0x1d43: 0x0040, 0x1d44: 0x0040, 0x1d45: 0x0040, + 0x1d46: 0x0040, 0x1d47: 0x0040, 0x1d48: 0x0040, 0x1d49: 0x0040, 0x1d4a: 0x0040, 0x1d4b: 0x0040, + 0x1d4c: 0x0040, 0x1d4d: 0x0040, 0x1d4e: 0x0040, 0x1d4f: 0x0040, 0x1d50: 0xc269, 0x1d51: 0xc289, + 0x1d52: 0xc2a9, 0x1d53: 0xc2c9, 0x1d54: 0xc2e9, 0x1d55: 0xc309, 0x1d56: 0xc329, 0x1d57: 0xc349, + 0x1d58: 0xc369, 0x1d59: 0xc389, 0x1d5a: 0xc3a9, 0x1d5b: 0xc3c9, 0x1d5c: 0xc3e9, 0x1d5d: 0xc409, + 0x1d5e: 0xc429, 0x1d5f: 0xc449, 0x1d60: 0xc469, 0x1d61: 0xc489, 0x1d62: 0xc4a9, 0x1d63: 0xc4c9, + 0x1d64: 0xc4e9, 0x1d65: 0xc509, 0x1d66: 0xc529, 0x1d67: 0xc549, 0x1d68: 0xc569, 0x1d69: 0xc589, + 0x1d6a: 0xc5a9, 0x1d6b: 0xc5c9, 0x1d6c: 0xc5e9, 0x1d6d: 0xc609, 0x1d6e: 0xc629, 0x1d6f: 0xc649, + 0x1d70: 0xc669, 0x1d71: 0xc689, 0x1d72: 0xc6a9, 0x1d73: 0xc6c9, 0x1d74: 0xc6e9, 0x1d75: 0xc709, + 0x1d76: 0xc729, 0x1d77: 0xc749, 0x1d78: 0xc769, 0x1d79: 0xc789, 0x1d7a: 0xc7a9, 0x1d7b: 0xc7c9, + 0x1d7c: 0x0040, 0x1d7d: 0x0040, 0x1d7e: 0x0040, 0x1d7f: 0x0040, + // Block 0x76, offset 0x1d80 + 0x1d80: 0xcaf9, 0x1d81: 0xcb19, 0x1d82: 0xcb39, 0x1d83: 0x8b1d, 0x1d84: 0xcb59, 0x1d85: 0xcb79, + 0x1d86: 0xcb99, 0x1d87: 0xcbb9, 0x1d88: 0xcbd9, 0x1d89: 0xcbf9, 0x1d8a: 0xcc19, 0x1d8b: 0xcc39, + 0x1d8c: 0xcc59, 0x1d8d: 0x8b3d, 0x1d8e: 0xcc79, 0x1d8f: 0xcc99, 0x1d90: 0xccb9, 0x1d91: 0xccd9, + 0x1d92: 0x8b5d, 0x1d93: 0xccf9, 0x1d94: 0xcd19, 0x1d95: 0xc429, 0x1d96: 0x8b7d, 0x1d97: 0xcd39, + 0x1d98: 0xcd59, 0x1d99: 0xcd79, 0x1d9a: 0xcd99, 0x1d9b: 0xcdb9, 0x1d9c: 0x8b9d, 0x1d9d: 0xcdd9, + 0x1d9e: 0xcdf9, 0x1d9f: 0xce19, 0x1da0: 0xce39, 0x1da1: 0xce59, 0x1da2: 0xc789, 0x1da3: 0xce79, + 0x1da4: 0xce99, 0x1da5: 0xceb9, 0x1da6: 0xced9, 0x1da7: 0xcef9, 0x1da8: 0xcf19, 0x1da9: 0xcf39, + 0x1daa: 0xcf59, 0x1dab: 0xcf79, 0x1dac: 0xcf99, 0x1dad: 0xcfb9, 0x1dae: 0xcfd9, 0x1daf: 0xcff9, + 0x1db0: 0xd019, 0x1db1: 0xd039, 0x1db2: 0xd039, 0x1db3: 0xd039, 0x1db4: 0x8bbd, 0x1db5: 0xd059, + 0x1db6: 0xd079, 0x1db7: 0xd099, 0x1db8: 0x8bdd, 0x1db9: 0xd0b9, 0x1dba: 0xd0d9, 0x1dbb: 0xd0f9, + 0x1dbc: 0xd119, 0x1dbd: 0xd139, 0x1dbe: 0xd159, 0x1dbf: 0xd179, + // Block 0x77, offset 0x1dc0 + 0x1dc0: 0xd199, 0x1dc1: 0xd1b9, 0x1dc2: 0xd1d9, 0x1dc3: 0xd1f9, 0x1dc4: 0xd219, 0x1dc5: 0xd239, + 0x1dc6: 0xd239, 0x1dc7: 0xd259, 0x1dc8: 0xd279, 0x1dc9: 0xd299, 0x1dca: 0xd2b9, 0x1dcb: 0xd2d9, + 0x1dcc: 0xd2f9, 0x1dcd: 0xd319, 0x1dce: 0xd339, 0x1dcf: 0xd359, 0x1dd0: 0xd379, 0x1dd1: 0xd399, + 0x1dd2: 0xd3b9, 0x1dd3: 0xd3d9, 0x1dd4: 0xd3f9, 0x1dd5: 0xd419, 0x1dd6: 0xd439, 0x1dd7: 0xd459, + 0x1dd8: 0xd479, 0x1dd9: 0x8bfd, 0x1dda: 0xd499, 0x1ddb: 0xd4b9, 0x1ddc: 0xd4d9, 0x1ddd: 0xc309, + 0x1dde: 0xd4f9, 0x1ddf: 0xd519, 0x1de0: 0x8c1d, 0x1de1: 0x8c3d, 0x1de2: 0xd539, 0x1de3: 0xd559, + 0x1de4: 0xd579, 0x1de5: 0xd599, 0x1de6: 0xd5b9, 0x1de7: 0xd5d9, 0x1de8: 0x0040, 0x1de9: 0xd5f9, + 0x1dea: 0xd619, 0x1deb: 0xd619, 0x1dec: 0x8c5d, 0x1ded: 0xd639, 0x1dee: 0xd659, 0x1def: 0xd679, + 0x1df0: 0xd699, 0x1df1: 0x8c7d, 0x1df2: 0xd6b9, 0x1df3: 0xd6d9, 0x1df4: 0x0040, 0x1df5: 0xd6f9, + 0x1df6: 0xd719, 0x1df7: 0xd739, 0x1df8: 0xd759, 0x1df9: 0xd779, 0x1dfa: 0xd799, 0x1dfb: 0x8c9d, + 0x1dfc: 0xd7b9, 0x1dfd: 0x8cbd, 0x1dfe: 0xd7d9, 0x1dff: 0xd7f9, + // Block 0x78, offset 0x1e00 + 0x1e00: 0xd819, 0x1e01: 0xd839, 0x1e02: 0xd859, 0x1e03: 0xd879, 0x1e04: 0xd899, 0x1e05: 0xd8b9, + 0x1e06: 0xd8d9, 0x1e07: 0xd8f9, 0x1e08: 0xd919, 0x1e09: 0x8cdd, 0x1e0a: 0xd939, 0x1e0b: 0xd959, + 0x1e0c: 0xd979, 0x1e0d: 0xd999, 0x1e0e: 0xd9b9, 0x1e0f: 0x8cfd, 0x1e10: 0xd9d9, 0x1e11: 0x8d1d, + 0x1e12: 0x8d3d, 0x1e13: 0xd9f9, 0x1e14: 0xda19, 0x1e15: 0xda19, 0x1e16: 0xda39, 0x1e17: 0x8d5d, + 0x1e18: 0x8d7d, 0x1e19: 0xda59, 0x1e1a: 0xda79, 0x1e1b: 0xda99, 0x1e1c: 0xdab9, 0x1e1d: 0xdad9, + 0x1e1e: 0xdaf9, 0x1e1f: 0xdb19, 0x1e20: 0xdb39, 0x1e21: 0xdb59, 0x1e22: 0xdb79, 0x1e23: 0xdb99, + 0x1e24: 0x8d9d, 0x1e25: 0xdbb9, 0x1e26: 0xdbd9, 0x1e27: 0xdbf9, 0x1e28: 0xdc19, 0x1e29: 0xdbf9, + 0x1e2a: 0xdc39, 0x1e2b: 0xdc59, 0x1e2c: 0xdc79, 0x1e2d: 0xdc99, 0x1e2e: 0xdcb9, 0x1e2f: 0xdcd9, + 0x1e30: 0xdcf9, 0x1e31: 0xdd19, 0x1e32: 0xdd39, 0x1e33: 0xdd59, 0x1e34: 0xdd79, 0x1e35: 0xdd99, + 0x1e36: 0xddb9, 0x1e37: 0xddd9, 0x1e38: 0x8dbd, 0x1e39: 0xddf9, 0x1e3a: 0xde19, 0x1e3b: 0xde39, + 0x1e3c: 0xde59, 0x1e3d: 0xde79, 0x1e3e: 0x8ddd, 0x1e3f: 0xde99, + // Block 0x79, offset 0x1e40 + 0x1e40: 0xe599, 0x1e41: 0xe5b9, 0x1e42: 0xe5d9, 0x1e43: 0xe5f9, 0x1e44: 0xe619, 0x1e45: 0xe639, + 0x1e46: 0x8efd, 0x1e47: 0xe659, 0x1e48: 0xe679, 0x1e49: 0xe699, 0x1e4a: 0xe6b9, 0x1e4b: 0xe6d9, + 0x1e4c: 0xe6f9, 0x1e4d: 0x8f1d, 0x1e4e: 0xe719, 0x1e4f: 0xe739, 0x1e50: 0x8f3d, 0x1e51: 0x8f5d, + 0x1e52: 0xe759, 0x1e53: 0xe779, 0x1e54: 0xe799, 0x1e55: 0xe7b9, 0x1e56: 0xe7d9, 0x1e57: 0xe7f9, + 0x1e58: 0xe819, 0x1e59: 0xe839, 0x1e5a: 0xe859, 0x1e5b: 0x8f7d, 0x1e5c: 0xe879, 0x1e5d: 0x8f9d, + 0x1e5e: 0xe899, 0x1e5f: 0x0040, 0x1e60: 0xe8b9, 0x1e61: 0xe8d9, 0x1e62: 0xe8f9, 0x1e63: 0x8fbd, + 0x1e64: 0xe919, 0x1e65: 0xe939, 0x1e66: 0x8fdd, 0x1e67: 0x8ffd, 0x1e68: 0xe959, 0x1e69: 0xe979, + 0x1e6a: 0xe999, 0x1e6b: 0xe9b9, 0x1e6c: 0xe9d9, 0x1e6d: 0xe9d9, 0x1e6e: 0xe9f9, 0x1e6f: 0xea19, + 0x1e70: 0xea39, 0x1e71: 0xea59, 0x1e72: 0xea79, 0x1e73: 0xea99, 0x1e74: 0xeab9, 0x1e75: 0x901d, + 0x1e76: 0xead9, 0x1e77: 0x903d, 0x1e78: 0xeaf9, 0x1e79: 0x905d, 0x1e7a: 0xeb19, 0x1e7b: 0x907d, + 0x1e7c: 0x909d, 0x1e7d: 0x90bd, 0x1e7e: 0xeb39, 0x1e7f: 0xeb59, + // Block 0x7a, offset 0x1e80 + 0x1e80: 0xeb79, 0x1e81: 0x90dd, 0x1e82: 0x90fd, 0x1e83: 0x911d, 0x1e84: 0x913d, 0x1e85: 0xeb99, + 0x1e86: 0xebb9, 0x1e87: 0xebb9, 0x1e88: 0xebd9, 0x1e89: 0xebf9, 0x1e8a: 0xec19, 0x1e8b: 0xec39, + 0x1e8c: 0xec59, 0x1e8d: 0x915d, 0x1e8e: 0xec79, 0x1e8f: 0xec99, 0x1e90: 0xecb9, 0x1e91: 0xecd9, + 0x1e92: 0x917d, 0x1e93: 0xecf9, 0x1e94: 0x919d, 0x1e95: 0x91bd, 0x1e96: 0xed19, 0x1e97: 0xed39, + 0x1e98: 0xed59, 0x1e99: 0xed79, 0x1e9a: 0xed99, 0x1e9b: 0xedb9, 0x1e9c: 0x91dd, 0x1e9d: 0x91fd, + 0x1e9e: 0x921d, 0x1e9f: 0x0040, 0x1ea0: 0xedd9, 0x1ea1: 0x923d, 0x1ea2: 0xedf9, 0x1ea3: 0xee19, + 0x1ea4: 0xee39, 0x1ea5: 0x925d, 0x1ea6: 0xee59, 0x1ea7: 0xee79, 0x1ea8: 0xee99, 0x1ea9: 0xeeb9, + 0x1eaa: 0xeed9, 0x1eab: 0x927d, 0x1eac: 0xeef9, 0x1ead: 0xef19, 0x1eae: 0xef39, 0x1eaf: 0xef59, + 0x1eb0: 0xef79, 0x1eb1: 0xef99, 0x1eb2: 0x929d, 0x1eb3: 0x92bd, 0x1eb4: 0xefb9, 0x1eb5: 0x92dd, + 0x1eb6: 0xefd9, 0x1eb7: 0x92fd, 0x1eb8: 0xeff9, 0x1eb9: 0xf019, 0x1eba: 0xf039, 0x1ebb: 0x931d, + 0x1ebc: 0x933d, 0x1ebd: 0xf059, 0x1ebe: 0x935d, 0x1ebf: 0xf079, + // Block 0x7b, offset 0x1ec0 + 0x1ec0: 0xf6b9, 0x1ec1: 0xf6d9, 0x1ec2: 0xf6f9, 0x1ec3: 0xf719, 0x1ec4: 0xf739, 0x1ec5: 0x951d, + 0x1ec6: 0xf759, 0x1ec7: 0xf779, 0x1ec8: 0xf799, 0x1ec9: 0xf7b9, 0x1eca: 0xf7d9, 0x1ecb: 0x953d, + 0x1ecc: 0x955d, 0x1ecd: 0xf7f9, 0x1ece: 0xf819, 0x1ecf: 0xf839, 0x1ed0: 0xf859, 0x1ed1: 0xf879, + 0x1ed2: 0xf899, 0x1ed3: 0x957d, 0x1ed4: 0xf8b9, 0x1ed5: 0xf8d9, 0x1ed6: 0xf8f9, 0x1ed7: 0xf919, + 0x1ed8: 0x959d, 0x1ed9: 0x95bd, 0x1eda: 0xf939, 0x1edb: 0xf959, 0x1edc: 0xf979, 0x1edd: 0x95dd, + 0x1ede: 0xf999, 0x1edf: 0xf9b9, 0x1ee0: 0x6815, 0x1ee1: 0x95fd, 0x1ee2: 0xf9d9, 0x1ee3: 0xf9f9, + 0x1ee4: 0xfa19, 0x1ee5: 0x961d, 0x1ee6: 0xfa39, 0x1ee7: 0xfa59, 0x1ee8: 0xfa79, 0x1ee9: 0xfa99, + 0x1eea: 0xfab9, 0x1eeb: 0xfad9, 0x1eec: 0xfaf9, 0x1eed: 0x963d, 0x1eee: 0xfb19, 0x1eef: 0xfb39, + 0x1ef0: 0xfb59, 0x1ef1: 0x965d, 0x1ef2: 0xfb79, 0x1ef3: 0xfb99, 0x1ef4: 0xfbb9, 0x1ef5: 0xfbd9, + 0x1ef6: 0x7b35, 0x1ef7: 0x967d, 0x1ef8: 0xfbf9, 0x1ef9: 0xfc19, 0x1efa: 0xfc39, 0x1efb: 0x969d, + 0x1efc: 0xfc59, 0x1efd: 0x96bd, 0x1efe: 0xfc79, 0x1eff: 0xfc79, + // Block 0x7c, offset 0x1f00 + 0x1f00: 0xfc99, 0x1f01: 0x96dd, 0x1f02: 0xfcb9, 0x1f03: 0xfcd9, 0x1f04: 0xfcf9, 0x1f05: 0xfd19, + 0x1f06: 0xfd39, 0x1f07: 0xfd59, 0x1f08: 0xfd79, 0x1f09: 0x96fd, 0x1f0a: 0xfd99, 0x1f0b: 0xfdb9, + 0x1f0c: 0xfdd9, 0x1f0d: 0xfdf9, 0x1f0e: 0xfe19, 0x1f0f: 0xfe39, 0x1f10: 0x971d, 0x1f11: 0xfe59, + 0x1f12: 0x973d, 0x1f13: 0x975d, 0x1f14: 0x977d, 0x1f15: 0xfe79, 0x1f16: 0xfe99, 0x1f17: 0xfeb9, + 0x1f18: 0xfed9, 0x1f19: 0xfef9, 0x1f1a: 0xff19, 0x1f1b: 0xff39, 0x1f1c: 0xff59, 0x1f1d: 0x979d, + 0x1f1e: 0x0040, 0x1f1f: 0x0040, 0x1f20: 0x0040, 0x1f21: 0x0040, 0x1f22: 0x0040, 0x1f23: 0x0040, + 0x1f24: 0x0040, 0x1f25: 0x0040, 0x1f26: 0x0040, 0x1f27: 0x0040, 0x1f28: 0x0040, 0x1f29: 0x0040, + 0x1f2a: 0x0040, 0x1f2b: 0x0040, 0x1f2c: 0x0040, 0x1f2d: 0x0040, 0x1f2e: 0x0040, 0x1f2f: 0x0040, + 0x1f30: 0x0040, 0x1f31: 0x0040, 0x1f32: 0x0040, 0x1f33: 0x0040, 0x1f34: 0x0040, 0x1f35: 0x0040, + 0x1f36: 0x0040, 0x1f37: 0x0040, 0x1f38: 0x0040, 0x1f39: 0x0040, 0x1f3a: 0x0040, 0x1f3b: 0x0040, + 0x1f3c: 0x0040, 0x1f3d: 0x0040, 0x1f3e: 0x0040, 0x1f3f: 0x0040, +} + +// idnaIndex: 35 blocks, 2240 entries, 4480 bytes +// Block 0 is the zero block. +var idnaIndex = [2240]uint16{ + // Block 0x0, offset 0x0 + // Block 0x1, offset 0x40 + // Block 0x2, offset 0x80 + // Block 0x3, offset 0xc0 + 0xc2: 0x01, 0xc3: 0x7b, 0xc4: 0x02, 0xc5: 0x03, 0xc6: 0x04, 0xc7: 0x05, + 0xc8: 0x06, 0xc9: 0x7c, 0xca: 0x7d, 0xcb: 0x07, 0xcc: 0x7e, 0xcd: 0x08, 0xce: 0x09, 0xcf: 0x0a, + 0xd0: 0x7f, 0xd1: 0x0b, 0xd2: 0x0c, 0xd3: 0x0d, 0xd4: 0x0e, 0xd5: 0x80, 0xd6: 0x81, 0xd7: 0x82, + 0xd8: 0x0f, 0xd9: 0x83, 0xda: 0x84, 0xdb: 0x10, 0xdc: 0x11, 0xdd: 0x85, 0xde: 0x86, 0xdf: 0x87, + 0xe0: 0x02, 0xe1: 0x03, 0xe2: 0x04, 0xe3: 0x05, 0xe4: 0x06, 0xe5: 0x07, 0xe6: 0x07, 0xe7: 0x07, + 0xe8: 0x07, 0xe9: 0x08, 0xea: 0x09, 0xeb: 0x07, 0xec: 0x07, 0xed: 0x0a, 0xee: 0x0b, 0xef: 0x0c, + 0xf0: 0x1c, 0xf1: 0x1d, 0xf2: 0x1d, 0xf3: 0x1f, 0xf4: 0x20, + // Block 0x4, offset 0x100 + 0x120: 0x88, 0x121: 0x89, 0x122: 0x8a, 0x123: 0x8b, 0x124: 0x8c, 0x125: 0x12, 0x126: 0x13, 0x127: 0x14, + 0x128: 0x15, 0x129: 0x16, 0x12a: 0x17, 0x12b: 0x18, 0x12c: 0x19, 0x12d: 0x1a, 0x12e: 0x1b, 0x12f: 0x8d, + 0x130: 0x8e, 0x131: 0x1c, 0x132: 0x1d, 0x133: 0x1e, 0x134: 0x8f, 0x135: 0x1f, 0x136: 0x90, 0x137: 0x91, + 0x138: 0x92, 0x139: 0x93, 0x13a: 0x20, 0x13b: 0x94, 0x13c: 0x95, 0x13d: 0x21, 0x13e: 0x22, 0x13f: 0x96, + // Block 0x5, offset 0x140 + 0x140: 0x97, 0x141: 0x98, 0x142: 0x99, 0x143: 0x9a, 0x144: 0x9b, 0x145: 0x9c, 0x146: 0x9b, 0x147: 0x9b, + 0x148: 0x9d, 0x149: 0x9e, 0x14a: 0x9f, 0x14b: 0xa0, 0x14c: 0xa1, 0x14d: 0xa2, 0x14e: 0xa3, 0x14f: 0xa4, + 0x150: 0xa5, 0x151: 0x9d, 0x152: 0x9d, 0x153: 0x9d, 0x154: 0x9d, 0x155: 0x9d, 0x156: 0x9d, 0x157: 0x9d, + 0x158: 0x9d, 0x159: 0xa6, 0x15a: 0xa7, 0x15b: 0xa8, 0x15c: 0xa9, 0x15d: 0xaa, 0x15e: 0xab, 0x15f: 0xac, + 0x160: 0xad, 0x161: 0xae, 0x162: 0xaf, 0x163: 0xb0, 0x164: 0xb1, 0x165: 0xb2, 0x166: 0xb3, 0x167: 0xb4, + 0x168: 0xb5, 0x169: 0xb6, 0x16a: 0xb7, 0x16b: 0xb8, 0x16c: 0xb9, 0x16d: 0xba, 0x16e: 0xbb, 0x16f: 0xbc, + 0x170: 0xbd, 0x171: 0xbe, 0x172: 0xbf, 0x173: 0xc0, 0x174: 0x23, 0x175: 0x24, 0x176: 0x25, 0x177: 0xc1, + 0x178: 0x26, 0x179: 0x26, 0x17a: 0x27, 0x17b: 0x26, 0x17c: 0xc2, 0x17d: 0x28, 0x17e: 0x29, 0x17f: 0x2a, + // Block 0x6, offset 0x180 + 0x180: 0x2b, 0x181: 0x2c, 0x182: 0x2d, 0x183: 0xc3, 0x184: 0x2e, 0x185: 0x2f, 0x186: 0xc4, 0x187: 0x9b, + 0x188: 0xc5, 0x189: 0xc6, 0x18a: 0x9b, 0x18b: 0x9b, 0x18c: 0xc7, 0x18d: 0x9b, 0x18e: 0x9b, 0x18f: 0xc8, + 0x190: 0xc9, 0x191: 0x30, 0x192: 0x31, 0x193: 0x32, 0x194: 0x9b, 0x195: 0x9b, 0x196: 0x9b, 0x197: 0x9b, + 0x198: 0x9b, 0x199: 0x9b, 0x19a: 0x9b, 0x19b: 0x9b, 0x19c: 0x9b, 0x19d: 0x9b, 0x19e: 0x9b, 0x19f: 0x9b, + 0x1a0: 0x9b, 0x1a1: 0x9b, 0x1a2: 0x9b, 0x1a3: 0x9b, 0x1a4: 0x9b, 0x1a5: 0x9b, 0x1a6: 0x9b, 0x1a7: 0x9b, + 0x1a8: 0xca, 0x1a9: 0xcb, 0x1aa: 0x9b, 0x1ab: 0xcc, 0x1ac: 0x9b, 0x1ad: 0xcd, 0x1ae: 0xce, 0x1af: 0xcf, + 0x1b0: 0xd0, 0x1b1: 0x33, 0x1b2: 0x26, 0x1b3: 0x34, 0x1b4: 0xd1, 0x1b5: 0xd2, 0x1b6: 0xd3, 0x1b7: 0xd4, + 0x1b8: 0xd5, 0x1b9: 0xd6, 0x1ba: 0xd7, 0x1bb: 0xd8, 0x1bc: 0xd9, 0x1bd: 0xda, 0x1be: 0xdb, 0x1bf: 0x35, + // Block 0x7, offset 0x1c0 + 0x1c0: 0x36, 0x1c1: 0xdc, 0x1c2: 0xdd, 0x1c3: 0xde, 0x1c4: 0xdf, 0x1c5: 0x37, 0x1c6: 0x38, 0x1c7: 0xe0, + 0x1c8: 0xe1, 0x1c9: 0x39, 0x1ca: 0x3a, 0x1cb: 0x3b, 0x1cc: 0x3c, 0x1cd: 0x3d, 0x1ce: 0x3e, 0x1cf: 0x3f, + 0x1d0: 0x9d, 0x1d1: 0x9d, 0x1d2: 0x9d, 0x1d3: 0x9d, 0x1d4: 0x9d, 0x1d5: 0x9d, 0x1d6: 0x9d, 0x1d7: 0x9d, + 0x1d8: 0x9d, 0x1d9: 0x9d, 0x1da: 0x9d, 0x1db: 0x9d, 0x1dc: 0x9d, 0x1dd: 0x9d, 0x1de: 0x9d, 0x1df: 0x9d, + 0x1e0: 0x9d, 0x1e1: 0x9d, 0x1e2: 0x9d, 0x1e3: 0x9d, 0x1e4: 0x9d, 0x1e5: 0x9d, 0x1e6: 0x9d, 0x1e7: 0x9d, + 0x1e8: 0x9d, 0x1e9: 0x9d, 0x1ea: 0x9d, 0x1eb: 0x9d, 0x1ec: 0x9d, 0x1ed: 0x9d, 0x1ee: 0x9d, 0x1ef: 0x9d, + 0x1f0: 0x9d, 0x1f1: 0x9d, 0x1f2: 0x9d, 0x1f3: 0x9d, 0x1f4: 0x9d, 0x1f5: 0x9d, 0x1f6: 0x9d, 0x1f7: 0x9d, + 0x1f8: 0x9d, 0x1f9: 0x9d, 0x1fa: 0x9d, 0x1fb: 0x9d, 0x1fc: 0x9d, 0x1fd: 0x9d, 0x1fe: 0x9d, 0x1ff: 0x9d, + // Block 0x8, offset 0x200 + 0x200: 0x9d, 0x201: 0x9d, 0x202: 0x9d, 0x203: 0x9d, 0x204: 0x9d, 0x205: 0x9d, 0x206: 0x9d, 0x207: 0x9d, + 0x208: 0x9d, 0x209: 0x9d, 0x20a: 0x9d, 0x20b: 0x9d, 0x20c: 0x9d, 0x20d: 0x9d, 0x20e: 0x9d, 0x20f: 0x9d, + 0x210: 0x9d, 0x211: 0x9d, 0x212: 0x9d, 0x213: 0x9d, 0x214: 0x9d, 0x215: 0x9d, 0x216: 0x9d, 0x217: 0x9d, + 0x218: 0x9d, 0x219: 0x9d, 0x21a: 0x9d, 0x21b: 0x9d, 0x21c: 0x9d, 0x21d: 0x9d, 0x21e: 0x9d, 0x21f: 0x9d, + 0x220: 0x9d, 0x221: 0x9d, 0x222: 0x9d, 0x223: 0x9d, 0x224: 0x9d, 0x225: 0x9d, 0x226: 0x9d, 0x227: 0x9d, + 0x228: 0x9d, 0x229: 0x9d, 0x22a: 0x9d, 0x22b: 0x9d, 0x22c: 0x9d, 0x22d: 0x9d, 0x22e: 0x9d, 0x22f: 0x9d, + 0x230: 0x9d, 0x231: 0x9d, 0x232: 0x9d, 0x233: 0x9d, 0x234: 0x9d, 0x235: 0x9d, 0x236: 0xb0, 0x237: 0x9b, + 0x238: 0x9d, 0x239: 0x9d, 0x23a: 0x9d, 0x23b: 0x9d, 0x23c: 0x9d, 0x23d: 0x9d, 0x23e: 0x9d, 0x23f: 0x9d, + // Block 0x9, offset 0x240 + 0x240: 0x9d, 0x241: 0x9d, 0x242: 0x9d, 0x243: 0x9d, 0x244: 0x9d, 0x245: 0x9d, 0x246: 0x9d, 0x247: 0x9d, + 0x248: 0x9d, 0x249: 0x9d, 0x24a: 0x9d, 0x24b: 0x9d, 0x24c: 0x9d, 0x24d: 0x9d, 0x24e: 0x9d, 0x24f: 0x9d, + 0x250: 0x9d, 0x251: 0x9d, 0x252: 0x9d, 0x253: 0x9d, 0x254: 0x9d, 0x255: 0x9d, 0x256: 0x9d, 0x257: 0x9d, + 0x258: 0x9d, 0x259: 0x9d, 0x25a: 0x9d, 0x25b: 0x9d, 0x25c: 0x9d, 0x25d: 0x9d, 0x25e: 0x9d, 0x25f: 0x9d, + 0x260: 0x9d, 0x261: 0x9d, 0x262: 0x9d, 0x263: 0x9d, 0x264: 0x9d, 0x265: 0x9d, 0x266: 0x9d, 0x267: 0x9d, + 0x268: 0x9d, 0x269: 0x9d, 0x26a: 0x9d, 0x26b: 0x9d, 0x26c: 0x9d, 0x26d: 0x9d, 0x26e: 0x9d, 0x26f: 0x9d, + 0x270: 0x9d, 0x271: 0x9d, 0x272: 0x9d, 0x273: 0x9d, 0x274: 0x9d, 0x275: 0x9d, 0x276: 0x9d, 0x277: 0x9d, + 0x278: 0x9d, 0x279: 0x9d, 0x27a: 0x9d, 0x27b: 0x9d, 0x27c: 0x9d, 0x27d: 0x9d, 0x27e: 0x9d, 0x27f: 0x9d, + // Block 0xa, offset 0x280 + 0x280: 0x9d, 0x281: 0x9d, 0x282: 0x9d, 0x283: 0x9d, 0x284: 0x9d, 0x285: 0x9d, 0x286: 0x9d, 0x287: 0x9d, + 0x288: 0x9d, 0x289: 0x9d, 0x28a: 0x9d, 0x28b: 0x9d, 0x28c: 0x9d, 0x28d: 0x9d, 0x28e: 0x9d, 0x28f: 0x9d, + 0x290: 0x9d, 0x291: 0x9d, 0x292: 0x9d, 0x293: 0x9d, 0x294: 0x9d, 0x295: 0x9d, 0x296: 0x9d, 0x297: 0x9d, + 0x298: 0x9d, 0x299: 0x9d, 0x29a: 0x9d, 0x29b: 0x9d, 0x29c: 0x9d, 0x29d: 0x9d, 0x29e: 0x9d, 0x29f: 0x9d, + 0x2a0: 0x9d, 0x2a1: 0x9d, 0x2a2: 0x9d, 0x2a3: 0x9d, 0x2a4: 0x9d, 0x2a5: 0x9d, 0x2a6: 0x9d, 0x2a7: 0x9d, + 0x2a8: 0x9d, 0x2a9: 0x9d, 0x2aa: 0x9d, 0x2ab: 0x9d, 0x2ac: 0x9d, 0x2ad: 0x9d, 0x2ae: 0x9d, 0x2af: 0x9d, + 0x2b0: 0x9d, 0x2b1: 0x9d, 0x2b2: 0x9d, 0x2b3: 0x9d, 0x2b4: 0x9d, 0x2b5: 0x9d, 0x2b6: 0x9d, 0x2b7: 0x9d, + 0x2b8: 0x9d, 0x2b9: 0x9d, 0x2ba: 0x9d, 0x2bb: 0x9d, 0x2bc: 0x9d, 0x2bd: 0x9d, 0x2be: 0x9d, 0x2bf: 0xe2, + // Block 0xb, offset 0x2c0 + 0x2c0: 0x9d, 0x2c1: 0x9d, 0x2c2: 0x9d, 0x2c3: 0x9d, 0x2c4: 0x9d, 0x2c5: 0x9d, 0x2c6: 0x9d, 0x2c7: 0x9d, + 0x2c8: 0x9d, 0x2c9: 0x9d, 0x2ca: 0x9d, 0x2cb: 0x9d, 0x2cc: 0x9d, 0x2cd: 0x9d, 0x2ce: 0x9d, 0x2cf: 0x9d, + 0x2d0: 0x9d, 0x2d1: 0x9d, 0x2d2: 0xe3, 0x2d3: 0xe4, 0x2d4: 0x9d, 0x2d5: 0x9d, 0x2d6: 0x9d, 0x2d7: 0x9d, + 0x2d8: 0xe5, 0x2d9: 0x40, 0x2da: 0x41, 0x2db: 0xe6, 0x2dc: 0x42, 0x2dd: 0x43, 0x2de: 0x44, 0x2df: 0xe7, + 0x2e0: 0xe8, 0x2e1: 0xe9, 0x2e2: 0xea, 0x2e3: 0xeb, 0x2e4: 0xec, 0x2e5: 0xed, 0x2e6: 0xee, 0x2e7: 0xef, + 0x2e8: 0xf0, 0x2e9: 0xf1, 0x2ea: 0xf2, 0x2eb: 0xf3, 0x2ec: 0xf4, 0x2ed: 0xf5, 0x2ee: 0xf6, 0x2ef: 0xf7, + 0x2f0: 0x9d, 0x2f1: 0x9d, 0x2f2: 0x9d, 0x2f3: 0x9d, 0x2f4: 0x9d, 0x2f5: 0x9d, 0x2f6: 0x9d, 0x2f7: 0x9d, + 0x2f8: 0x9d, 0x2f9: 0x9d, 0x2fa: 0x9d, 0x2fb: 0x9d, 0x2fc: 0x9d, 0x2fd: 0x9d, 0x2fe: 0x9d, 0x2ff: 0x9d, + // Block 0xc, offset 0x300 + 0x300: 0x9d, 0x301: 0x9d, 0x302: 0x9d, 0x303: 0x9d, 0x304: 0x9d, 0x305: 0x9d, 0x306: 0x9d, 0x307: 0x9d, + 0x308: 0x9d, 0x309: 0x9d, 0x30a: 0x9d, 0x30b: 0x9d, 0x30c: 0x9d, 0x30d: 0x9d, 0x30e: 0x9d, 0x30f: 0x9d, + 0x310: 0x9d, 0x311: 0x9d, 0x312: 0x9d, 0x313: 0x9d, 0x314: 0x9d, 0x315: 0x9d, 0x316: 0x9d, 0x317: 0x9d, + 0x318: 0x9d, 0x319: 0x9d, 0x31a: 0x9d, 0x31b: 0x9d, 0x31c: 0x9d, 0x31d: 0x9d, 0x31e: 0xf8, 0x31f: 0xf9, + // Block 0xd, offset 0x340 + 0x340: 0xb8, 0x341: 0xb8, 0x342: 0xb8, 0x343: 0xb8, 0x344: 0xb8, 0x345: 0xb8, 0x346: 0xb8, 0x347: 0xb8, + 0x348: 0xb8, 0x349: 0xb8, 0x34a: 0xb8, 0x34b: 0xb8, 0x34c: 0xb8, 0x34d: 0xb8, 0x34e: 0xb8, 0x34f: 0xb8, + 0x350: 0xb8, 0x351: 0xb8, 0x352: 0xb8, 0x353: 0xb8, 0x354: 0xb8, 0x355: 0xb8, 0x356: 0xb8, 0x357: 0xb8, + 0x358: 0xb8, 0x359: 0xb8, 0x35a: 0xb8, 0x35b: 0xb8, 0x35c: 0xb8, 0x35d: 0xb8, 0x35e: 0xb8, 0x35f: 0xb8, + 0x360: 0xb8, 0x361: 0xb8, 0x362: 0xb8, 0x363: 0xb8, 0x364: 0xb8, 0x365: 0xb8, 0x366: 0xb8, 0x367: 0xb8, + 0x368: 0xb8, 0x369: 0xb8, 0x36a: 0xb8, 0x36b: 0xb8, 0x36c: 0xb8, 0x36d: 0xb8, 0x36e: 0xb8, 0x36f: 0xb8, + 0x370: 0xb8, 0x371: 0xb8, 0x372: 0xb8, 0x373: 0xb8, 0x374: 0xb8, 0x375: 0xb8, 0x376: 0xb8, 0x377: 0xb8, + 0x378: 0xb8, 0x379: 0xb8, 0x37a: 0xb8, 0x37b: 0xb8, 0x37c: 0xb8, 0x37d: 0xb8, 0x37e: 0xb8, 0x37f: 0xb8, + // Block 0xe, offset 0x380 + 0x380: 0xb8, 0x381: 0xb8, 0x382: 0xb8, 0x383: 0xb8, 0x384: 0xb8, 0x385: 0xb8, 0x386: 0xb8, 0x387: 0xb8, + 0x388: 0xb8, 0x389: 0xb8, 0x38a: 0xb8, 0x38b: 0xb8, 0x38c: 0xb8, 0x38d: 0xb8, 0x38e: 0xb8, 0x38f: 0xb8, + 0x390: 0xb8, 0x391: 0xb8, 0x392: 0xb8, 0x393: 0xb8, 0x394: 0xb8, 0x395: 0xb8, 0x396: 0xb8, 0x397: 0xb8, + 0x398: 0xb8, 0x399: 0xb8, 0x39a: 0xb8, 0x39b: 0xb8, 0x39c: 0xb8, 0x39d: 0xb8, 0x39e: 0xb8, 0x39f: 0xb8, + 0x3a0: 0xb8, 0x3a1: 0xb8, 0x3a2: 0xb8, 0x3a3: 0xb8, 0x3a4: 0xfa, 0x3a5: 0xfb, 0x3a6: 0xfc, 0x3a7: 0xfd, + 0x3a8: 0x45, 0x3a9: 0xfe, 0x3aa: 0xff, 0x3ab: 0x46, 0x3ac: 0x47, 0x3ad: 0x48, 0x3ae: 0x49, 0x3af: 0x4a, + 0x3b0: 0x100, 0x3b1: 0x4b, 0x3b2: 0x4c, 0x3b3: 0x4d, 0x3b4: 0x4e, 0x3b5: 0x4f, 0x3b6: 0x101, 0x3b7: 0x50, + 0x3b8: 0x51, 0x3b9: 0x52, 0x3ba: 0x53, 0x3bb: 0x54, 0x3bc: 0x55, 0x3bd: 0x56, 0x3be: 0x57, 0x3bf: 0x58, + // Block 0xf, offset 0x3c0 + 0x3c0: 0x102, 0x3c1: 0x103, 0x3c2: 0x9d, 0x3c3: 0x104, 0x3c4: 0x105, 0x3c5: 0x9b, 0x3c6: 0x106, 0x3c7: 0x107, + 0x3c8: 0xb8, 0x3c9: 0xb8, 0x3ca: 0x108, 0x3cb: 0x109, 0x3cc: 0x10a, 0x3cd: 0x10b, 0x3ce: 0x10c, 0x3cf: 0x10d, + 0x3d0: 0x10e, 0x3d1: 0x9d, 0x3d2: 0x10f, 0x3d3: 0x110, 0x3d4: 0x111, 0x3d5: 0x112, 0x3d6: 0xb8, 0x3d7: 0xb8, + 0x3d8: 0x9d, 0x3d9: 0x9d, 0x3da: 0x9d, 0x3db: 0x9d, 0x3dc: 0x113, 0x3dd: 0x114, 0x3de: 0xb8, 0x3df: 0xb8, + 0x3e0: 0x115, 0x3e1: 0x116, 0x3e2: 0x117, 0x3e3: 0x118, 0x3e4: 0x119, 0x3e5: 0xb8, 0x3e6: 0x11a, 0x3e7: 0x11b, + 0x3e8: 0x11c, 0x3e9: 0x11d, 0x3ea: 0x11e, 0x3eb: 0x59, 0x3ec: 0x11f, 0x3ed: 0x120, 0x3ee: 0x5a, 0x3ef: 0xb8, + 0x3f0: 0x9d, 0x3f1: 0x121, 0x3f2: 0x122, 0x3f3: 0x123, 0x3f4: 0xb8, 0x3f5: 0xb8, 0x3f6: 0xb8, 0x3f7: 0xb8, + 0x3f8: 0xb8, 0x3f9: 0x124, 0x3fa: 0xb8, 0x3fb: 0xb8, 0x3fc: 0xb8, 0x3fd: 0xb8, 0x3fe: 0xb8, 0x3ff: 0xb8, + // Block 0x10, offset 0x400 + 0x400: 0x125, 0x401: 0x126, 0x402: 0x127, 0x403: 0x128, 0x404: 0x129, 0x405: 0x12a, 0x406: 0x12b, 0x407: 0x12c, + 0x408: 0x12d, 0x409: 0xb8, 0x40a: 0x12e, 0x40b: 0x12f, 0x40c: 0x5b, 0x40d: 0x5c, 0x40e: 0xb8, 0x40f: 0xb8, + 0x410: 0x130, 0x411: 0x131, 0x412: 0x132, 0x413: 0x133, 0x414: 0xb8, 0x415: 0xb8, 0x416: 0x134, 0x417: 0x135, + 0x418: 0x136, 0x419: 0x137, 0x41a: 0x138, 0x41b: 0x139, 0x41c: 0x13a, 0x41d: 0xb8, 0x41e: 0xb8, 0x41f: 0xb8, + 0x420: 0xb8, 0x421: 0xb8, 0x422: 0x13b, 0x423: 0x13c, 0x424: 0xb8, 0x425: 0xb8, 0x426: 0xb8, 0x427: 0xb8, + 0x428: 0xb8, 0x429: 0xb8, 0x42a: 0xb8, 0x42b: 0x13d, 0x42c: 0xb8, 0x42d: 0xb8, 0x42e: 0xb8, 0x42f: 0xb8, + 0x430: 0x13e, 0x431: 0x13f, 0x432: 0x140, 0x433: 0xb8, 0x434: 0xb8, 0x435: 0xb8, 0x436: 0xb8, 0x437: 0xb8, + 0x438: 0xb8, 0x439: 0xb8, 0x43a: 0xb8, 0x43b: 0xb8, 0x43c: 0xb8, 0x43d: 0xb8, 0x43e: 0xb8, 0x43f: 0xb8, + // Block 0x11, offset 0x440 + 0x440: 0x9d, 0x441: 0x9d, 0x442: 0x9d, 0x443: 0x9d, 0x444: 0x9d, 0x445: 0x9d, 0x446: 0x9d, 0x447: 0x9d, + 0x448: 0x9d, 0x449: 0x9d, 0x44a: 0x9d, 0x44b: 0x9d, 0x44c: 0x9d, 0x44d: 0x9d, 0x44e: 0x141, 0x44f: 0xb8, + 0x450: 0x9b, 0x451: 0x142, 0x452: 0x9d, 0x453: 0x9d, 0x454: 0x9d, 0x455: 0x143, 0x456: 0xb8, 0x457: 0xb8, + 0x458: 0xb8, 0x459: 0xb8, 0x45a: 0xb8, 0x45b: 0xb8, 0x45c: 0xb8, 0x45d: 0xb8, 0x45e: 0xb8, 0x45f: 0xb8, + 0x460: 0xb8, 0x461: 0xb8, 0x462: 0xb8, 0x463: 0xb8, 0x464: 0xb8, 0x465: 0xb8, 0x466: 0xb8, 0x467: 0xb8, + 0x468: 0xb8, 0x469: 0xb8, 0x46a: 0xb8, 0x46b: 0xb8, 0x46c: 0xb8, 0x46d: 0xb8, 0x46e: 0xb8, 0x46f: 0xb8, + 0x470: 0xb8, 0x471: 0xb8, 0x472: 0xb8, 0x473: 0xb8, 0x474: 0xb8, 0x475: 0xb8, 0x476: 0xb8, 0x477: 0xb8, + 0x478: 0xb8, 0x479: 0xb8, 0x47a: 0xb8, 0x47b: 0xb8, 0x47c: 0xb8, 0x47d: 0xb8, 0x47e: 0xb8, 0x47f: 0xb8, + // Block 0x12, offset 0x480 + 0x480: 0x9d, 0x481: 0x9d, 0x482: 0x9d, 0x483: 0x9d, 0x484: 0x9d, 0x485: 0x9d, 0x486: 0x9d, 0x487: 0x9d, + 0x488: 0x9d, 0x489: 0x9d, 0x48a: 0x9d, 0x48b: 0x9d, 0x48c: 0x9d, 0x48d: 0x9d, 0x48e: 0x9d, 0x48f: 0x9d, + 0x490: 0x144, 0x491: 0xb8, 0x492: 0xb8, 0x493: 0xb8, 0x494: 0xb8, 0x495: 0xb8, 0x496: 0xb8, 0x497: 0xb8, + 0x498: 0xb8, 0x499: 0xb8, 0x49a: 0xb8, 0x49b: 0xb8, 0x49c: 0xb8, 0x49d: 0xb8, 0x49e: 0xb8, 0x49f: 0xb8, + 0x4a0: 0xb8, 0x4a1: 0xb8, 0x4a2: 0xb8, 0x4a3: 0xb8, 0x4a4: 0xb8, 0x4a5: 0xb8, 0x4a6: 0xb8, 0x4a7: 0xb8, + 0x4a8: 0xb8, 0x4a9: 0xb8, 0x4aa: 0xb8, 0x4ab: 0xb8, 0x4ac: 0xb8, 0x4ad: 0xb8, 0x4ae: 0xb8, 0x4af: 0xb8, + 0x4b0: 0xb8, 0x4b1: 0xb8, 0x4b2: 0xb8, 0x4b3: 0xb8, 0x4b4: 0xb8, 0x4b5: 0xb8, 0x4b6: 0xb8, 0x4b7: 0xb8, + 0x4b8: 0xb8, 0x4b9: 0xb8, 0x4ba: 0xb8, 0x4bb: 0xb8, 0x4bc: 0xb8, 0x4bd: 0xb8, 0x4be: 0xb8, 0x4bf: 0xb8, + // Block 0x13, offset 0x4c0 + 0x4c0: 0xb8, 0x4c1: 0xb8, 0x4c2: 0xb8, 0x4c3: 0xb8, 0x4c4: 0xb8, 0x4c5: 0xb8, 0x4c6: 0xb8, 0x4c7: 0xb8, + 0x4c8: 0xb8, 0x4c9: 0xb8, 0x4ca: 0xb8, 0x4cb: 0xb8, 0x4cc: 0xb8, 0x4cd: 0xb8, 0x4ce: 0xb8, 0x4cf: 0xb8, + 0x4d0: 0x9d, 0x4d1: 0x9d, 0x4d2: 0x9d, 0x4d3: 0x9d, 0x4d4: 0x9d, 0x4d5: 0x9d, 0x4d6: 0x9d, 0x4d7: 0x9d, + 0x4d8: 0x9d, 0x4d9: 0x145, 0x4da: 0xb8, 0x4db: 0xb8, 0x4dc: 0xb8, 0x4dd: 0xb8, 0x4de: 0xb8, 0x4df: 0xb8, + 0x4e0: 0xb8, 0x4e1: 0xb8, 0x4e2: 0xb8, 0x4e3: 0xb8, 0x4e4: 0xb8, 0x4e5: 0xb8, 0x4e6: 0xb8, 0x4e7: 0xb8, + 0x4e8: 0xb8, 0x4e9: 0xb8, 0x4ea: 0xb8, 0x4eb: 0xb8, 0x4ec: 0xb8, 0x4ed: 0xb8, 0x4ee: 0xb8, 0x4ef: 0xb8, + 0x4f0: 0xb8, 0x4f1: 0xb8, 0x4f2: 0xb8, 0x4f3: 0xb8, 0x4f4: 0xb8, 0x4f5: 0xb8, 0x4f6: 0xb8, 0x4f7: 0xb8, + 0x4f8: 0xb8, 0x4f9: 0xb8, 0x4fa: 0xb8, 0x4fb: 0xb8, 0x4fc: 0xb8, 0x4fd: 0xb8, 0x4fe: 0xb8, 0x4ff: 0xb8, + // Block 0x14, offset 0x500 + 0x500: 0xb8, 0x501: 0xb8, 0x502: 0xb8, 0x503: 0xb8, 0x504: 0xb8, 0x505: 0xb8, 0x506: 0xb8, 0x507: 0xb8, + 0x508: 0xb8, 0x509: 0xb8, 0x50a: 0xb8, 0x50b: 0xb8, 0x50c: 0xb8, 0x50d: 0xb8, 0x50e: 0xb8, 0x50f: 0xb8, + 0x510: 0xb8, 0x511: 0xb8, 0x512: 0xb8, 0x513: 0xb8, 0x514: 0xb8, 0x515: 0xb8, 0x516: 0xb8, 0x517: 0xb8, + 0x518: 0xb8, 0x519: 0xb8, 0x51a: 0xb8, 0x51b: 0xb8, 0x51c: 0xb8, 0x51d: 0xb8, 0x51e: 0xb8, 0x51f: 0xb8, + 0x520: 0x9d, 0x521: 0x9d, 0x522: 0x9d, 0x523: 0x9d, 0x524: 0x9d, 0x525: 0x9d, 0x526: 0x9d, 0x527: 0x9d, + 0x528: 0x13d, 0x529: 0x146, 0x52a: 0xb8, 0x52b: 0x147, 0x52c: 0x148, 0x52d: 0x149, 0x52e: 0x14a, 0x52f: 0xb8, + 0x530: 0xb8, 0x531: 0xb8, 0x532: 0xb8, 0x533: 0xb8, 0x534: 0xb8, 0x535: 0xb8, 0x536: 0xb8, 0x537: 0xb8, + 0x538: 0xb8, 0x539: 0xb8, 0x53a: 0xb8, 0x53b: 0xb8, 0x53c: 0x9d, 0x53d: 0x14b, 0x53e: 0x14c, 0x53f: 0x14d, + // Block 0x15, offset 0x540 + 0x540: 0x9d, 0x541: 0x9d, 0x542: 0x9d, 0x543: 0x9d, 0x544: 0x9d, 0x545: 0x9d, 0x546: 0x9d, 0x547: 0x9d, + 0x548: 0x9d, 0x549: 0x9d, 0x54a: 0x9d, 0x54b: 0x9d, 0x54c: 0x9d, 0x54d: 0x9d, 0x54e: 0x9d, 0x54f: 0x9d, + 0x550: 0x9d, 0x551: 0x9d, 0x552: 0x9d, 0x553: 0x9d, 0x554: 0x9d, 0x555: 0x9d, 0x556: 0x9d, 0x557: 0x9d, + 0x558: 0x9d, 0x559: 0x9d, 0x55a: 0x9d, 0x55b: 0x9d, 0x55c: 0x9d, 0x55d: 0x9d, 0x55e: 0x9d, 0x55f: 0x14e, + 0x560: 0x9d, 0x561: 0x9d, 0x562: 0x9d, 0x563: 0x9d, 0x564: 0x9d, 0x565: 0x9d, 0x566: 0x9d, 0x567: 0x9d, + 0x568: 0x9d, 0x569: 0x9d, 0x56a: 0x9d, 0x56b: 0x14f, 0x56c: 0xb8, 0x56d: 0xb8, 0x56e: 0xb8, 0x56f: 0xb8, + 0x570: 0xb8, 0x571: 0xb8, 0x572: 0xb8, 0x573: 0xb8, 0x574: 0xb8, 0x575: 0xb8, 0x576: 0xb8, 0x577: 0xb8, + 0x578: 0xb8, 0x579: 0xb8, 0x57a: 0xb8, 0x57b: 0xb8, 0x57c: 0xb8, 0x57d: 0xb8, 0x57e: 0xb8, 0x57f: 0xb8, + // Block 0x16, offset 0x580 + 0x580: 0x150, 0x581: 0xb8, 0x582: 0xb8, 0x583: 0xb8, 0x584: 0xb8, 0x585: 0xb8, 0x586: 0xb8, 0x587: 0xb8, + 0x588: 0xb8, 0x589: 0xb8, 0x58a: 0xb8, 0x58b: 0xb8, 0x58c: 0xb8, 0x58d: 0xb8, 0x58e: 0xb8, 0x58f: 0xb8, + 0x590: 0xb8, 0x591: 0xb8, 0x592: 0xb8, 0x593: 0xb8, 0x594: 0xb8, 0x595: 0xb8, 0x596: 0xb8, 0x597: 0xb8, + 0x598: 0xb8, 0x599: 0xb8, 0x59a: 0xb8, 0x59b: 0xb8, 0x59c: 0xb8, 0x59d: 0xb8, 0x59e: 0xb8, 0x59f: 0xb8, + 0x5a0: 0xb8, 0x5a1: 0xb8, 0x5a2: 0xb8, 0x5a3: 0xb8, 0x5a4: 0xb8, 0x5a5: 0xb8, 0x5a6: 0xb8, 0x5a7: 0xb8, + 0x5a8: 0xb8, 0x5a9: 0xb8, 0x5aa: 0xb8, 0x5ab: 0xb8, 0x5ac: 0xb8, 0x5ad: 0xb8, 0x5ae: 0xb8, 0x5af: 0xb8, + 0x5b0: 0x9d, 0x5b1: 0x151, 0x5b2: 0x152, 0x5b3: 0xb8, 0x5b4: 0xb8, 0x5b5: 0xb8, 0x5b6: 0xb8, 0x5b7: 0xb8, + 0x5b8: 0xb8, 0x5b9: 0xb8, 0x5ba: 0xb8, 0x5bb: 0xb8, 0x5bc: 0xb8, 0x5bd: 0xb8, 0x5be: 0xb8, 0x5bf: 0xb8, + // Block 0x17, offset 0x5c0 + 0x5c0: 0x9b, 0x5c1: 0x9b, 0x5c2: 0x9b, 0x5c3: 0x153, 0x5c4: 0x154, 0x5c5: 0x155, 0x5c6: 0x156, 0x5c7: 0x157, + 0x5c8: 0x9b, 0x5c9: 0x158, 0x5ca: 0xb8, 0x5cb: 0xb8, 0x5cc: 0x9b, 0x5cd: 0x159, 0x5ce: 0xb8, 0x5cf: 0xb8, + 0x5d0: 0x5d, 0x5d1: 0x5e, 0x5d2: 0x5f, 0x5d3: 0x60, 0x5d4: 0x61, 0x5d5: 0x62, 0x5d6: 0x63, 0x5d7: 0x64, + 0x5d8: 0x65, 0x5d9: 0x66, 0x5da: 0x67, 0x5db: 0x68, 0x5dc: 0x69, 0x5dd: 0x6a, 0x5de: 0x6b, 0x5df: 0x6c, + 0x5e0: 0x9b, 0x5e1: 0x9b, 0x5e2: 0x9b, 0x5e3: 0x9b, 0x5e4: 0x9b, 0x5e5: 0x9b, 0x5e6: 0x9b, 0x5e7: 0x9b, + 0x5e8: 0x15a, 0x5e9: 0x15b, 0x5ea: 0x15c, 0x5eb: 0xb8, 0x5ec: 0xb8, 0x5ed: 0xb8, 0x5ee: 0xb8, 0x5ef: 0xb8, + 0x5f0: 0xb8, 0x5f1: 0xb8, 0x5f2: 0xb8, 0x5f3: 0xb8, 0x5f4: 0xb8, 0x5f5: 0xb8, 0x5f6: 0xb8, 0x5f7: 0xb8, + 0x5f8: 0xb8, 0x5f9: 0xb8, 0x5fa: 0xb8, 0x5fb: 0xb8, 0x5fc: 0xb8, 0x5fd: 0xb8, 0x5fe: 0xb8, 0x5ff: 0xb8, + // Block 0x18, offset 0x600 + 0x600: 0x15d, 0x601: 0xb8, 0x602: 0xb8, 0x603: 0xb8, 0x604: 0xb8, 0x605: 0xb8, 0x606: 0xb8, 0x607: 0xb8, + 0x608: 0xb8, 0x609: 0xb8, 0x60a: 0xb8, 0x60b: 0xb8, 0x60c: 0xb8, 0x60d: 0xb8, 0x60e: 0xb8, 0x60f: 0xb8, + 0x610: 0xb8, 0x611: 0xb8, 0x612: 0xb8, 0x613: 0xb8, 0x614: 0xb8, 0x615: 0xb8, 0x616: 0xb8, 0x617: 0xb8, + 0x618: 0xb8, 0x619: 0xb8, 0x61a: 0xb8, 0x61b: 0xb8, 0x61c: 0xb8, 0x61d: 0xb8, 0x61e: 0xb8, 0x61f: 0xb8, + 0x620: 0x9d, 0x621: 0x9d, 0x622: 0x9d, 0x623: 0x15e, 0x624: 0x6d, 0x625: 0x15f, 0x626: 0xb8, 0x627: 0xb8, + 0x628: 0xb8, 0x629: 0xb8, 0x62a: 0xb8, 0x62b: 0xb8, 0x62c: 0xb8, 0x62d: 0xb8, 0x62e: 0xb8, 0x62f: 0xb8, + 0x630: 0xb8, 0x631: 0xb8, 0x632: 0xb8, 0x633: 0xb8, 0x634: 0xb8, 0x635: 0xb8, 0x636: 0xb8, 0x637: 0xb8, + 0x638: 0x6e, 0x639: 0x6f, 0x63a: 0x70, 0x63b: 0x160, 0x63c: 0xb8, 0x63d: 0xb8, 0x63e: 0xb8, 0x63f: 0xb8, + // Block 0x19, offset 0x640 + 0x640: 0x161, 0x641: 0x9b, 0x642: 0x162, 0x643: 0x163, 0x644: 0x71, 0x645: 0x72, 0x646: 0x164, 0x647: 0x165, + 0x648: 0x73, 0x649: 0x166, 0x64a: 0xb8, 0x64b: 0xb8, 0x64c: 0x9b, 0x64d: 0x9b, 0x64e: 0x9b, 0x64f: 0x9b, + 0x650: 0x9b, 0x651: 0x9b, 0x652: 0x9b, 0x653: 0x9b, 0x654: 0x9b, 0x655: 0x9b, 0x656: 0x9b, 0x657: 0x9b, + 0x658: 0x9b, 0x659: 0x9b, 0x65a: 0x9b, 0x65b: 0x167, 0x65c: 0x9b, 0x65d: 0x168, 0x65e: 0x9b, 0x65f: 0x169, + 0x660: 0x16a, 0x661: 0x16b, 0x662: 0x16c, 0x663: 0xb8, 0x664: 0x16d, 0x665: 0x16e, 0x666: 0x16f, 0x667: 0x170, + 0x668: 0xb8, 0x669: 0xb8, 0x66a: 0xb8, 0x66b: 0xb8, 0x66c: 0xb8, 0x66d: 0xb8, 0x66e: 0xb8, 0x66f: 0xb8, + 0x670: 0xb8, 0x671: 0xb8, 0x672: 0xb8, 0x673: 0xb8, 0x674: 0xb8, 0x675: 0xb8, 0x676: 0xb8, 0x677: 0xb8, + 0x678: 0xb8, 0x679: 0xb8, 0x67a: 0xb8, 0x67b: 0xb8, 0x67c: 0xb8, 0x67d: 0xb8, 0x67e: 0xb8, 0x67f: 0xb8, + // Block 0x1a, offset 0x680 + 0x680: 0x9d, 0x681: 0x9d, 0x682: 0x9d, 0x683: 0x9d, 0x684: 0x9d, 0x685: 0x9d, 0x686: 0x9d, 0x687: 0x9d, + 0x688: 0x9d, 0x689: 0x9d, 0x68a: 0x9d, 0x68b: 0x9d, 0x68c: 0x9d, 0x68d: 0x9d, 0x68e: 0x9d, 0x68f: 0x9d, + 0x690: 0x9d, 0x691: 0x9d, 0x692: 0x9d, 0x693: 0x9d, 0x694: 0x9d, 0x695: 0x9d, 0x696: 0x9d, 0x697: 0x9d, + 0x698: 0x9d, 0x699: 0x9d, 0x69a: 0x9d, 0x69b: 0x171, 0x69c: 0x9d, 0x69d: 0x9d, 0x69e: 0x9d, 0x69f: 0x9d, + 0x6a0: 0x9d, 0x6a1: 0x9d, 0x6a2: 0x9d, 0x6a3: 0x9d, 0x6a4: 0x9d, 0x6a5: 0x9d, 0x6a6: 0x9d, 0x6a7: 0x9d, + 0x6a8: 0x9d, 0x6a9: 0x9d, 0x6aa: 0x9d, 0x6ab: 0x9d, 0x6ac: 0x9d, 0x6ad: 0x9d, 0x6ae: 0x9d, 0x6af: 0x9d, + 0x6b0: 0x9d, 0x6b1: 0x9d, 0x6b2: 0x9d, 0x6b3: 0x9d, 0x6b4: 0x9d, 0x6b5: 0x9d, 0x6b6: 0x9d, 0x6b7: 0x9d, + 0x6b8: 0x9d, 0x6b9: 0x9d, 0x6ba: 0x9d, 0x6bb: 0x9d, 0x6bc: 0x9d, 0x6bd: 0x9d, 0x6be: 0x9d, 0x6bf: 0x9d, + // Block 0x1b, offset 0x6c0 + 0x6c0: 0x9d, 0x6c1: 0x9d, 0x6c2: 0x9d, 0x6c3: 0x9d, 0x6c4: 0x9d, 0x6c5: 0x9d, 0x6c6: 0x9d, 0x6c7: 0x9d, + 0x6c8: 0x9d, 0x6c9: 0x9d, 0x6ca: 0x9d, 0x6cb: 0x9d, 0x6cc: 0x9d, 0x6cd: 0x9d, 0x6ce: 0x9d, 0x6cf: 0x9d, + 0x6d0: 0x9d, 0x6d1: 0x9d, 0x6d2: 0x9d, 0x6d3: 0x9d, 0x6d4: 0x9d, 0x6d5: 0x9d, 0x6d6: 0x9d, 0x6d7: 0x9d, + 0x6d8: 0x9d, 0x6d9: 0x9d, 0x6da: 0x9d, 0x6db: 0x9d, 0x6dc: 0x172, 0x6dd: 0x9d, 0x6de: 0x9d, 0x6df: 0x9d, + 0x6e0: 0x173, 0x6e1: 0x9d, 0x6e2: 0x9d, 0x6e3: 0x9d, 0x6e4: 0x9d, 0x6e5: 0x9d, 0x6e6: 0x9d, 0x6e7: 0x9d, + 0x6e8: 0x9d, 0x6e9: 0x9d, 0x6ea: 0x9d, 0x6eb: 0x9d, 0x6ec: 0x9d, 0x6ed: 0x9d, 0x6ee: 0x9d, 0x6ef: 0x9d, + 0x6f0: 0x9d, 0x6f1: 0x9d, 0x6f2: 0x9d, 0x6f3: 0x9d, 0x6f4: 0x9d, 0x6f5: 0x9d, 0x6f6: 0x9d, 0x6f7: 0x9d, + 0x6f8: 0x9d, 0x6f9: 0x9d, 0x6fa: 0x9d, 0x6fb: 0x9d, 0x6fc: 0x9d, 0x6fd: 0x9d, 0x6fe: 0x9d, 0x6ff: 0x9d, + // Block 0x1c, offset 0x700 + 0x700: 0x9d, 0x701: 0x9d, 0x702: 0x9d, 0x703: 0x9d, 0x704: 0x9d, 0x705: 0x9d, 0x706: 0x9d, 0x707: 0x9d, + 0x708: 0x9d, 0x709: 0x9d, 0x70a: 0x9d, 0x70b: 0x9d, 0x70c: 0x9d, 0x70d: 0x9d, 0x70e: 0x9d, 0x70f: 0x9d, + 0x710: 0x9d, 0x711: 0x9d, 0x712: 0x9d, 0x713: 0x9d, 0x714: 0x9d, 0x715: 0x9d, 0x716: 0x9d, 0x717: 0x9d, + 0x718: 0x9d, 0x719: 0x9d, 0x71a: 0x9d, 0x71b: 0x9d, 0x71c: 0x9d, 0x71d: 0x9d, 0x71e: 0x9d, 0x71f: 0x9d, + 0x720: 0x9d, 0x721: 0x9d, 0x722: 0x9d, 0x723: 0x9d, 0x724: 0x9d, 0x725: 0x9d, 0x726: 0x9d, 0x727: 0x9d, + 0x728: 0x9d, 0x729: 0x9d, 0x72a: 0x9d, 0x72b: 0x9d, 0x72c: 0x9d, 0x72d: 0x9d, 0x72e: 0x9d, 0x72f: 0x9d, + 0x730: 0x9d, 0x731: 0x9d, 0x732: 0x9d, 0x733: 0x9d, 0x734: 0x9d, 0x735: 0x9d, 0x736: 0x9d, 0x737: 0x9d, + 0x738: 0x9d, 0x739: 0x9d, 0x73a: 0x174, 0x73b: 0xb8, 0x73c: 0xb8, 0x73d: 0xb8, 0x73e: 0xb8, 0x73f: 0xb8, + // Block 0x1d, offset 0x740 + 0x740: 0xb8, 0x741: 0xb8, 0x742: 0xb8, 0x743: 0xb8, 0x744: 0xb8, 0x745: 0xb8, 0x746: 0xb8, 0x747: 0xb8, + 0x748: 0xb8, 0x749: 0xb8, 0x74a: 0xb8, 0x74b: 0xb8, 0x74c: 0xb8, 0x74d: 0xb8, 0x74e: 0xb8, 0x74f: 0xb8, + 0x750: 0xb8, 0x751: 0xb8, 0x752: 0xb8, 0x753: 0xb8, 0x754: 0xb8, 0x755: 0xb8, 0x756: 0xb8, 0x757: 0xb8, + 0x758: 0xb8, 0x759: 0xb8, 0x75a: 0xb8, 0x75b: 0xb8, 0x75c: 0xb8, 0x75d: 0xb8, 0x75e: 0xb8, 0x75f: 0xb8, + 0x760: 0x74, 0x761: 0x75, 0x762: 0x76, 0x763: 0x175, 0x764: 0x77, 0x765: 0x78, 0x766: 0x176, 0x767: 0x79, + 0x768: 0x7a, 0x769: 0xb8, 0x76a: 0xb8, 0x76b: 0xb8, 0x76c: 0xb8, 0x76d: 0xb8, 0x76e: 0xb8, 0x76f: 0xb8, + 0x770: 0xb8, 0x771: 0xb8, 0x772: 0xb8, 0x773: 0xb8, 0x774: 0xb8, 0x775: 0xb8, 0x776: 0xb8, 0x777: 0xb8, + 0x778: 0xb8, 0x779: 0xb8, 0x77a: 0xb8, 0x77b: 0xb8, 0x77c: 0xb8, 0x77d: 0xb8, 0x77e: 0xb8, 0x77f: 0xb8, + // Block 0x1e, offset 0x780 + 0x790: 0x0d, 0x791: 0x0e, 0x792: 0x0f, 0x793: 0x10, 0x794: 0x11, 0x795: 0x0b, 0x796: 0x12, 0x797: 0x07, + 0x798: 0x13, 0x799: 0x0b, 0x79a: 0x0b, 0x79b: 0x14, 0x79c: 0x0b, 0x79d: 0x15, 0x79e: 0x16, 0x79f: 0x17, + 0x7a0: 0x07, 0x7a1: 0x07, 0x7a2: 0x07, 0x7a3: 0x07, 0x7a4: 0x07, 0x7a5: 0x07, 0x7a6: 0x07, 0x7a7: 0x07, + 0x7a8: 0x07, 0x7a9: 0x07, 0x7aa: 0x18, 0x7ab: 0x19, 0x7ac: 0x1a, 0x7ad: 0x0b, 0x7ae: 0x0b, 0x7af: 0x1b, + 0x7b0: 0x0b, 0x7b1: 0x0b, 0x7b2: 0x0b, 0x7b3: 0x0b, 0x7b4: 0x0b, 0x7b5: 0x0b, 0x7b6: 0x0b, 0x7b7: 0x0b, + 0x7b8: 0x0b, 0x7b9: 0x0b, 0x7ba: 0x0b, 0x7bb: 0x0b, 0x7bc: 0x0b, 0x7bd: 0x0b, 0x7be: 0x0b, 0x7bf: 0x0b, + // Block 0x1f, offset 0x7c0 + 0x7c0: 0x0b, 0x7c1: 0x0b, 0x7c2: 0x0b, 0x7c3: 0x0b, 0x7c4: 0x0b, 0x7c5: 0x0b, 0x7c6: 0x0b, 0x7c7: 0x0b, + 0x7c8: 0x0b, 0x7c9: 0x0b, 0x7ca: 0x0b, 0x7cb: 0x0b, 0x7cc: 0x0b, 0x7cd: 0x0b, 0x7ce: 0x0b, 0x7cf: 0x0b, + 0x7d0: 0x0b, 0x7d1: 0x0b, 0x7d2: 0x0b, 0x7d3: 0x0b, 0x7d4: 0x0b, 0x7d5: 0x0b, 0x7d6: 0x0b, 0x7d7: 0x0b, + 0x7d8: 0x0b, 0x7d9: 0x0b, 0x7da: 0x0b, 0x7db: 0x0b, 0x7dc: 0x0b, 0x7dd: 0x0b, 0x7de: 0x0b, 0x7df: 0x0b, + 0x7e0: 0x0b, 0x7e1: 0x0b, 0x7e2: 0x0b, 0x7e3: 0x0b, 0x7e4: 0x0b, 0x7e5: 0x0b, 0x7e6: 0x0b, 0x7e7: 0x0b, + 0x7e8: 0x0b, 0x7e9: 0x0b, 0x7ea: 0x0b, 0x7eb: 0x0b, 0x7ec: 0x0b, 0x7ed: 0x0b, 0x7ee: 0x0b, 0x7ef: 0x0b, + 0x7f0: 0x0b, 0x7f1: 0x0b, 0x7f2: 0x0b, 0x7f3: 0x0b, 0x7f4: 0x0b, 0x7f5: 0x0b, 0x7f6: 0x0b, 0x7f7: 0x0b, + 0x7f8: 0x0b, 0x7f9: 0x0b, 0x7fa: 0x0b, 0x7fb: 0x0b, 0x7fc: 0x0b, 0x7fd: 0x0b, 0x7fe: 0x0b, 0x7ff: 0x0b, + // Block 0x20, offset 0x800 + 0x800: 0x177, 0x801: 0x178, 0x802: 0xb8, 0x803: 0xb8, 0x804: 0x179, 0x805: 0x179, 0x806: 0x179, 0x807: 0x17a, + 0x808: 0xb8, 0x809: 0xb8, 0x80a: 0xb8, 0x80b: 0xb8, 0x80c: 0xb8, 0x80d: 0xb8, 0x80e: 0xb8, 0x80f: 0xb8, + 0x810: 0xb8, 0x811: 0xb8, 0x812: 0xb8, 0x813: 0xb8, 0x814: 0xb8, 0x815: 0xb8, 0x816: 0xb8, 0x817: 0xb8, + 0x818: 0xb8, 0x819: 0xb8, 0x81a: 0xb8, 0x81b: 0xb8, 0x81c: 0xb8, 0x81d: 0xb8, 0x81e: 0xb8, 0x81f: 0xb8, + 0x820: 0xb8, 0x821: 0xb8, 0x822: 0xb8, 0x823: 0xb8, 0x824: 0xb8, 0x825: 0xb8, 0x826: 0xb8, 0x827: 0xb8, + 0x828: 0xb8, 0x829: 0xb8, 0x82a: 0xb8, 0x82b: 0xb8, 0x82c: 0xb8, 0x82d: 0xb8, 0x82e: 0xb8, 0x82f: 0xb8, + 0x830: 0xb8, 0x831: 0xb8, 0x832: 0xb8, 0x833: 0xb8, 0x834: 0xb8, 0x835: 0xb8, 0x836: 0xb8, 0x837: 0xb8, + 0x838: 0xb8, 0x839: 0xb8, 0x83a: 0xb8, 0x83b: 0xb8, 0x83c: 0xb8, 0x83d: 0xb8, 0x83e: 0xb8, 0x83f: 0xb8, + // Block 0x21, offset 0x840 + 0x840: 0x0b, 0x841: 0x0b, 0x842: 0x0b, 0x843: 0x0b, 0x844: 0x0b, 0x845: 0x0b, 0x846: 0x0b, 0x847: 0x0b, + 0x848: 0x0b, 0x849: 0x0b, 0x84a: 0x0b, 0x84b: 0x0b, 0x84c: 0x0b, 0x84d: 0x0b, 0x84e: 0x0b, 0x84f: 0x0b, + 0x850: 0x0b, 0x851: 0x0b, 0x852: 0x0b, 0x853: 0x0b, 0x854: 0x0b, 0x855: 0x0b, 0x856: 0x0b, 0x857: 0x0b, + 0x858: 0x0b, 0x859: 0x0b, 0x85a: 0x0b, 0x85b: 0x0b, 0x85c: 0x0b, 0x85d: 0x0b, 0x85e: 0x0b, 0x85f: 0x0b, + 0x860: 0x1e, 0x861: 0x0b, 0x862: 0x0b, 0x863: 0x0b, 0x864: 0x0b, 0x865: 0x0b, 0x866: 0x0b, 0x867: 0x0b, + 0x868: 0x0b, 0x869: 0x0b, 0x86a: 0x0b, 0x86b: 0x0b, 0x86c: 0x0b, 0x86d: 0x0b, 0x86e: 0x0b, 0x86f: 0x0b, + 0x870: 0x0b, 0x871: 0x0b, 0x872: 0x0b, 0x873: 0x0b, 0x874: 0x0b, 0x875: 0x0b, 0x876: 0x0b, 0x877: 0x0b, + 0x878: 0x0b, 0x879: 0x0b, 0x87a: 0x0b, 0x87b: 0x0b, 0x87c: 0x0b, 0x87d: 0x0b, 0x87e: 0x0b, 0x87f: 0x0b, + // Block 0x22, offset 0x880 + 0x880: 0x0b, 0x881: 0x0b, 0x882: 0x0b, 0x883: 0x0b, 0x884: 0x0b, 0x885: 0x0b, 0x886: 0x0b, 0x887: 0x0b, + 0x888: 0x0b, 0x889: 0x0b, 0x88a: 0x0b, 0x88b: 0x0b, 0x88c: 0x0b, 0x88d: 0x0b, 0x88e: 0x0b, 0x88f: 0x0b, +} + +// idnaSparseOffset: 256 entries, 512 bytes +var idnaSparseOffset = []uint16{0x0, 0x8, 0x19, 0x25, 0x27, 0x2c, 0x34, 0x3f, 0x4b, 0x5c, 0x60, 0x6f, 0x74, 0x7b, 0x87, 0x95, 0xa3, 0xa8, 0xb1, 0xc1, 0xcf, 0xdc, 0xe8, 0xf9, 0x103, 0x10a, 0x117, 0x128, 0x12f, 0x13a, 0x149, 0x157, 0x161, 0x163, 0x167, 0x169, 0x175, 0x180, 0x188, 0x18e, 0x194, 0x199, 0x19e, 0x1a1, 0x1a5, 0x1ab, 0x1b0, 0x1bc, 0x1c6, 0x1cc, 0x1dd, 0x1e7, 0x1ea, 0x1f2, 0x1f5, 0x202, 0x20a, 0x20e, 0x215, 0x21d, 0x22d, 0x239, 0x23b, 0x245, 0x251, 0x25d, 0x269, 0x271, 0x276, 0x280, 0x291, 0x295, 0x2a0, 0x2a4, 0x2ad, 0x2b5, 0x2bb, 0x2c0, 0x2c3, 0x2c6, 0x2ca, 0x2d0, 0x2d4, 0x2d8, 0x2de, 0x2e5, 0x2eb, 0x2f3, 0x2fa, 0x305, 0x30f, 0x313, 0x316, 0x31c, 0x320, 0x322, 0x325, 0x327, 0x32a, 0x334, 0x337, 0x346, 0x34a, 0x34f, 0x352, 0x356, 0x35b, 0x360, 0x366, 0x36c, 0x37b, 0x381, 0x385, 0x394, 0x399, 0x3a1, 0x3ab, 0x3b6, 0x3be, 0x3cf, 0x3d8, 0x3e8, 0x3f5, 0x3ff, 0x404, 0x411, 0x415, 0x41a, 0x41c, 0x420, 0x422, 0x426, 0x42f, 0x435, 0x439, 0x449, 0x453, 0x458, 0x45b, 0x461, 0x468, 0x46d, 0x471, 0x477, 0x47c, 0x485, 0x48a, 0x490, 0x497, 0x49e, 0x4a5, 0x4a9, 0x4ae, 0x4b1, 0x4b6, 0x4c2, 0x4c8, 0x4cd, 0x4d4, 0x4dc, 0x4e1, 0x4e5, 0x4f5, 0x4fc, 0x500, 0x504, 0x50b, 0x50e, 0x511, 0x515, 0x519, 0x51f, 0x528, 0x534, 0x53b, 0x544, 0x54c, 0x553, 0x561, 0x56e, 0x57b, 0x584, 0x588, 0x596, 0x59e, 0x5a9, 0x5b2, 0x5b8, 0x5c0, 0x5c9, 0x5d3, 0x5d6, 0x5e2, 0x5e5, 0x5ea, 0x5ed, 0x5f7, 0x600, 0x60c, 0x60f, 0x614, 0x617, 0x61a, 0x61d, 0x624, 0x62b, 0x62f, 0x63a, 0x63d, 0x643, 0x648, 0x64c, 0x64f, 0x652, 0x655, 0x65a, 0x664, 0x667, 0x66b, 0x67a, 0x686, 0x68a, 0x68f, 0x694, 0x698, 0x69d, 0x6a6, 0x6b1, 0x6b7, 0x6bf, 0x6c3, 0x6c7, 0x6cd, 0x6d3, 0x6d8, 0x6db, 0x6e9, 0x6f0, 0x6f3, 0x6f6, 0x6fa, 0x700, 0x705, 0x70f, 0x714, 0x717, 0x71a, 0x71d, 0x720, 0x724, 0x727, 0x737, 0x748, 0x74d, 0x74f, 0x751} + +// idnaSparseValues: 1876 entries, 7504 bytes +var idnaSparseValues = [1876]valueRange{ + // Block 0x0, offset 0x0 + {value: 0x0000, lo: 0x07}, + {value: 0xe105, lo: 0x80, hi: 0x96}, + {value: 0x0018, lo: 0x97, hi: 0x97}, + {value: 0xe105, lo: 0x98, hi: 0x9e}, + {value: 0x001f, lo: 0x9f, hi: 0x9f}, + {value: 0x0008, lo: 0xa0, hi: 0xb6}, + {value: 0x0018, lo: 0xb7, hi: 0xb7}, + {value: 0x0008, lo: 0xb8, hi: 0xbf}, + // Block 0x1, offset 0x8 + {value: 0x0000, lo: 0x10}, + {value: 0x0008, lo: 0x80, hi: 0x80}, + {value: 0xe01d, lo: 0x81, hi: 0x81}, + {value: 0x0008, lo: 0x82, hi: 0x82}, + {value: 0x0335, lo: 0x83, hi: 0x83}, + {value: 0x034d, lo: 0x84, hi: 0x84}, + {value: 0x0365, lo: 0x85, hi: 0x85}, + {value: 0xe00d, lo: 0x86, hi: 0x86}, + {value: 0x0008, lo: 0x87, hi: 0x87}, + {value: 0xe00d, lo: 0x88, hi: 0x88}, + {value: 0x0008, lo: 0x89, hi: 0x89}, + {value: 0xe00d, lo: 0x8a, hi: 0x8a}, + {value: 0x0008, lo: 0x8b, hi: 0x8b}, + {value: 0xe00d, lo: 0x8c, hi: 0x8c}, + {value: 0x0008, lo: 0x8d, hi: 0x8d}, + {value: 0xe00d, lo: 0x8e, hi: 0x8e}, + {value: 0x0008, lo: 0x8f, hi: 0xbf}, + // Block 0x2, offset 0x19 + {value: 0x0000, lo: 0x0b}, + {value: 0x0008, lo: 0x80, hi: 0xaf}, + {value: 0x0249, lo: 0xb0, hi: 0xb0}, + {value: 0x037d, lo: 0xb1, hi: 0xb1}, + {value: 0x0259, lo: 0xb2, hi: 0xb2}, + {value: 0x0269, lo: 0xb3, hi: 0xb3}, + {value: 0x034d, lo: 0xb4, hi: 0xb4}, + {value: 0x0395, lo: 0xb5, hi: 0xb5}, + {value: 0xe1bd, lo: 0xb6, hi: 0xb6}, + {value: 0x0279, lo: 0xb7, hi: 0xb7}, + {value: 0x0289, lo: 0xb8, hi: 0xb8}, + {value: 0x0008, lo: 0xb9, hi: 0xbf}, + // Block 0x3, offset 0x25 + {value: 0x0000, lo: 0x01}, + {value: 0x1308, lo: 0x80, hi: 0xbf}, + // Block 0x4, offset 0x27 + {value: 0x0000, lo: 0x04}, + {value: 0x03f5, lo: 0x80, hi: 0x8f}, + {value: 0xe105, lo: 0x90, hi: 0x9f}, + {value: 0x049d, lo: 0xa0, hi: 0xaf}, + {value: 0x0008, lo: 0xb0, hi: 0xbf}, + // Block 0x5, offset 0x2c + {value: 0x0000, lo: 0x07}, + {value: 0xe185, lo: 0x80, hi: 0x8f}, + {value: 0x0545, lo: 0x90, hi: 0x96}, + {value: 0x0040, lo: 0x97, hi: 0x98}, + {value: 0x0008, lo: 0x99, hi: 0x99}, + {value: 0x0018, lo: 0x9a, hi: 0x9f}, + {value: 0x0040, lo: 0xa0, hi: 0xa0}, + {value: 0x0008, lo: 0xa1, hi: 0xbf}, + // Block 0x6, offset 0x34 + {value: 0x0000, lo: 0x0a}, + {value: 0x0008, lo: 0x80, hi: 0x86}, + {value: 0x0401, lo: 0x87, hi: 0x87}, + {value: 0x0040, lo: 0x88, hi: 0x88}, + {value: 0x0018, lo: 0x89, hi: 0x8a}, + {value: 0x0040, lo: 0x8b, hi: 0x8c}, + {value: 0x0018, lo: 0x8d, hi: 0x8f}, + {value: 0x0040, lo: 0x90, hi: 0x90}, + {value: 0x1308, lo: 0x91, hi: 0xbd}, + {value: 0x0018, lo: 0xbe, hi: 0xbe}, + {value: 0x1308, lo: 0xbf, hi: 0xbf}, + // Block 0x7, offset 0x3f + {value: 0x0000, lo: 0x0b}, + {value: 0x0018, lo: 0x80, hi: 0x80}, + {value: 0x1308, lo: 0x81, hi: 0x82}, + {value: 0x0018, lo: 0x83, hi: 0x83}, + {value: 0x1308, lo: 0x84, hi: 0x85}, + {value: 0x0018, lo: 0x86, hi: 0x86}, + {value: 0x1308, lo: 0x87, hi: 0x87}, + {value: 0x0040, lo: 0x88, hi: 0x8f}, + {value: 0x0008, lo: 0x90, hi: 0xaa}, + {value: 0x0040, lo: 0xab, hi: 0xaf}, + {value: 0x0008, lo: 0xb0, hi: 0xb4}, + {value: 0x0040, lo: 0xb5, hi: 0xbf}, + // Block 0x8, offset 0x4b + {value: 0x0000, lo: 0x10}, + {value: 0x0018, lo: 0x80, hi: 0x80}, + {value: 0x0208, lo: 0x81, hi: 0x87}, + {value: 0x0408, lo: 0x88, hi: 0x88}, + {value: 0x0208, lo: 0x89, hi: 0x8a}, + {value: 0x1308, lo: 0x8b, hi: 0x9f}, + {value: 0x0008, lo: 0xa0, hi: 0xa9}, + {value: 0x0018, lo: 0xaa, hi: 0xad}, + {value: 0x0208, lo: 0xae, hi: 0xaf}, + {value: 0x1308, lo: 0xb0, hi: 0xb0}, + {value: 0x0408, lo: 0xb1, hi: 0xb3}, + {value: 0x0008, lo: 0xb4, hi: 0xb4}, + {value: 0x0429, lo: 0xb5, hi: 0xb5}, + {value: 0x0451, lo: 0xb6, hi: 0xb6}, + {value: 0x0479, lo: 0xb7, hi: 0xb7}, + {value: 0x04a1, lo: 0xb8, hi: 0xb8}, + {value: 0x0208, lo: 0xb9, hi: 0xbf}, + // Block 0x9, offset 0x5c + {value: 0x0000, lo: 0x03}, + {value: 0x0208, lo: 0x80, hi: 0x87}, + {value: 0x0408, lo: 0x88, hi: 0x99}, + {value: 0x0208, lo: 0x9a, hi: 0xbf}, + // Block 0xa, offset 0x60 + {value: 0x0000, lo: 0x0e}, + {value: 0x1308, lo: 0x80, hi: 0x8a}, + {value: 0x0040, lo: 0x8b, hi: 0x8c}, + {value: 0x0408, lo: 0x8d, hi: 0x8d}, + {value: 0x0208, lo: 0x8e, hi: 0x98}, + {value: 0x0408, lo: 0x99, hi: 0x9b}, + {value: 0x0208, lo: 0x9c, hi: 0xaa}, + {value: 0x0408, lo: 0xab, hi: 0xac}, + {value: 0x0208, lo: 0xad, hi: 0xb0}, + {value: 0x0408, lo: 0xb1, hi: 0xb1}, + {value: 0x0208, lo: 0xb2, hi: 0xb2}, + {value: 0x0408, lo: 0xb3, hi: 0xb4}, + {value: 0x0208, lo: 0xb5, hi: 0xb7}, + {value: 0x0408, lo: 0xb8, hi: 0xb9}, + {value: 0x0208, lo: 0xba, hi: 0xbf}, + // Block 0xb, offset 0x6f + {value: 0x0000, lo: 0x04}, + {value: 0x0008, lo: 0x80, hi: 0xa5}, + {value: 0x1308, lo: 0xa6, hi: 0xb0}, + {value: 0x0008, lo: 0xb1, hi: 0xb1}, + {value: 0x0040, lo: 0xb2, hi: 0xbf}, + // Block 0xc, offset 0x74 + {value: 0x0000, lo: 0x06}, + {value: 0x0008, lo: 0x80, hi: 0x89}, + {value: 0x0208, lo: 0x8a, hi: 0xaa}, + {value: 0x1308, lo: 0xab, hi: 0xb3}, + {value: 0x0008, lo: 0xb4, hi: 0xb5}, + {value: 0x0018, lo: 0xb6, hi: 0xba}, + {value: 0x0040, lo: 0xbb, hi: 0xbf}, + // Block 0xd, offset 0x7b + {value: 0x0000, lo: 0x0b}, + {value: 0x0008, lo: 0x80, hi: 0x95}, + {value: 0x1308, lo: 0x96, hi: 0x99}, + {value: 0x0008, lo: 0x9a, hi: 0x9a}, + {value: 0x1308, lo: 0x9b, hi: 0xa3}, + {value: 0x0008, lo: 0xa4, hi: 0xa4}, + {value: 0x1308, lo: 0xa5, hi: 0xa7}, + {value: 0x0008, lo: 0xa8, hi: 0xa8}, + {value: 0x1308, lo: 0xa9, hi: 0xad}, + {value: 0x0040, lo: 0xae, hi: 0xaf}, + {value: 0x0018, lo: 0xb0, hi: 0xbe}, + {value: 0x0040, lo: 0xbf, hi: 0xbf}, + // Block 0xe, offset 0x87 + {value: 0x0000, lo: 0x0d}, + {value: 0x0408, lo: 0x80, hi: 0x80}, + {value: 0x0208, lo: 0x81, hi: 0x85}, + {value: 0x0408, lo: 0x86, hi: 0x87}, + {value: 0x0208, lo: 0x88, hi: 0x88}, + {value: 0x0408, lo: 0x89, hi: 0x89}, + {value: 0x0208, lo: 0x8a, hi: 0x93}, + {value: 0x0408, lo: 0x94, hi: 0x94}, + {value: 0x0208, lo: 0x95, hi: 0x95}, + {value: 0x0008, lo: 0x96, hi: 0x98}, + {value: 0x1308, lo: 0x99, hi: 0x9b}, + {value: 0x0040, lo: 0x9c, hi: 0x9d}, + {value: 0x0018, lo: 0x9e, hi: 0x9e}, + {value: 0x0040, lo: 0x9f, hi: 0xbf}, + // Block 0xf, offset 0x95 + {value: 0x0000, lo: 0x0d}, + {value: 0x0040, lo: 0x80, hi: 0x9f}, + {value: 0x0208, lo: 0xa0, hi: 0xa9}, + {value: 0x0408, lo: 0xaa, hi: 0xac}, + {value: 0x0008, lo: 0xad, hi: 0xad}, + {value: 0x0408, lo: 0xae, hi: 0xae}, + {value: 0x0208, lo: 0xaf, hi: 0xb0}, + {value: 0x0408, lo: 0xb1, hi: 0xb2}, + {value: 0x0208, lo: 0xb3, hi: 0xb4}, + {value: 0x0040, lo: 0xb5, hi: 0xb5}, + {value: 0x0208, lo: 0xb6, hi: 0xb8}, + {value: 0x0408, lo: 0xb9, hi: 0xb9}, + {value: 0x0208, lo: 0xba, hi: 0xbd}, + {value: 0x0040, lo: 0xbe, hi: 0xbf}, + // Block 0x10, offset 0xa3 + {value: 0x0000, lo: 0x04}, + {value: 0x0040, lo: 0x80, hi: 0x93}, + {value: 0x1308, lo: 0x94, hi: 0xa1}, + {value: 0x0040, lo: 0xa2, hi: 0xa2}, + {value: 0x1308, lo: 0xa3, hi: 0xbf}, + // Block 0x11, offset 0xa8 + {value: 0x0000, lo: 0x08}, + {value: 0x1308, lo: 0x80, hi: 0x82}, + {value: 0x1008, lo: 0x83, hi: 0x83}, + {value: 0x0008, lo: 0x84, hi: 0xb9}, + {value: 0x1308, lo: 0xba, hi: 0xba}, + {value: 0x1008, lo: 0xbb, hi: 0xbb}, + {value: 0x1308, lo: 0xbc, hi: 0xbc}, + {value: 0x0008, lo: 0xbd, hi: 0xbd}, + {value: 0x1008, lo: 0xbe, hi: 0xbf}, + // Block 0x12, offset 0xb1 + {value: 0x0000, lo: 0x0f}, + {value: 0x1308, lo: 0x80, hi: 0x80}, + {value: 0x1008, lo: 0x81, hi: 0x82}, + {value: 0x0040, lo: 0x83, hi: 0x85}, + {value: 0x1008, lo: 0x86, hi: 0x88}, + {value: 0x0040, lo: 0x89, hi: 0x89}, + {value: 0x1008, lo: 0x8a, hi: 0x8c}, + {value: 0x1b08, lo: 0x8d, hi: 0x8d}, + {value: 0x0040, lo: 0x8e, hi: 0x8f}, + {value: 0x0008, lo: 0x90, hi: 0x90}, + {value: 0x0040, lo: 0x91, hi: 0x96}, + {value: 0x1008, lo: 0x97, hi: 0x97}, + {value: 0x0040, lo: 0x98, hi: 0xa5}, + {value: 0x0008, lo: 0xa6, hi: 0xaf}, + {value: 0x0018, lo: 0xb0, hi: 0xba}, + {value: 0x0040, lo: 0xbb, hi: 0xbf}, + // Block 0x13, offset 0xc1 + {value: 0x0000, lo: 0x0d}, + {value: 0x1308, lo: 0x80, hi: 0x80}, + {value: 0x1008, lo: 0x81, hi: 0x83}, + {value: 0x0040, lo: 0x84, hi: 0x84}, + {value: 0x0008, lo: 0x85, hi: 0x8c}, + {value: 0x0040, lo: 0x8d, hi: 0x8d}, + {value: 0x0008, lo: 0x8e, hi: 0x90}, + {value: 0x0040, lo: 0x91, hi: 0x91}, + {value: 0x0008, lo: 0x92, hi: 0xa8}, + {value: 0x0040, lo: 0xa9, hi: 0xa9}, + {value: 0x0008, lo: 0xaa, hi: 0xb9}, + {value: 0x0040, lo: 0xba, hi: 0xbc}, + {value: 0x0008, lo: 0xbd, hi: 0xbd}, + {value: 0x1308, lo: 0xbe, hi: 0xbf}, + // Block 0x14, offset 0xcf + {value: 0x0000, lo: 0x0c}, + {value: 0x0040, lo: 0x80, hi: 0x80}, + {value: 0x1308, lo: 0x81, hi: 0x81}, + {value: 0x1008, lo: 0x82, hi: 0x83}, + {value: 0x0040, lo: 0x84, hi: 0x84}, + {value: 0x0008, lo: 0x85, hi: 0x8c}, + {value: 0x0040, lo: 0x8d, hi: 0x8d}, + {value: 0x0008, lo: 0x8e, hi: 0x90}, + {value: 0x0040, lo: 0x91, hi: 0x91}, + {value: 0x0008, lo: 0x92, hi: 0xba}, + {value: 0x0040, lo: 0xbb, hi: 0xbc}, + {value: 0x0008, lo: 0xbd, hi: 0xbd}, + {value: 0x1008, lo: 0xbe, hi: 0xbf}, + // Block 0x15, offset 0xdc + {value: 0x0000, lo: 0x0b}, + {value: 0x0040, lo: 0x80, hi: 0x81}, + {value: 0x1008, lo: 0x82, hi: 0x83}, + {value: 0x0040, lo: 0x84, hi: 0x84}, + {value: 0x0008, lo: 0x85, hi: 0x96}, + {value: 0x0040, lo: 0x97, hi: 0x99}, + {value: 0x0008, lo: 0x9a, hi: 0xb1}, + {value: 0x0040, lo: 0xb2, hi: 0xb2}, + {value: 0x0008, lo: 0xb3, hi: 0xbb}, + {value: 0x0040, lo: 0xbc, hi: 0xbc}, + {value: 0x0008, lo: 0xbd, hi: 0xbd}, + {value: 0x0040, lo: 0xbe, hi: 0xbf}, + // Block 0x16, offset 0xe8 + {value: 0x0000, lo: 0x10}, + {value: 0x0008, lo: 0x80, hi: 0x86}, + {value: 0x0040, lo: 0x87, hi: 0x89}, + {value: 0x1b08, lo: 0x8a, hi: 0x8a}, + {value: 0x0040, lo: 0x8b, hi: 0x8e}, + {value: 0x1008, lo: 0x8f, hi: 0x91}, + {value: 0x1308, lo: 0x92, hi: 0x94}, + {value: 0x0040, lo: 0x95, hi: 0x95}, + {value: 0x1308, lo: 0x96, hi: 0x96}, + {value: 0x0040, lo: 0x97, hi: 0x97}, + {value: 0x1008, lo: 0x98, hi: 0x9f}, + {value: 0x0040, lo: 0xa0, hi: 0xa5}, + {value: 0x0008, lo: 0xa6, hi: 0xaf}, + {value: 0x0040, lo: 0xb0, hi: 0xb1}, + {value: 0x1008, lo: 0xb2, hi: 0xb3}, + {value: 0x0018, lo: 0xb4, hi: 0xb4}, + {value: 0x0040, lo: 0xb5, hi: 0xbf}, + // Block 0x17, offset 0xf9 + {value: 0x0000, lo: 0x09}, + {value: 0x0040, lo: 0x80, hi: 0x80}, + {value: 0x0008, lo: 0x81, hi: 0xb0}, + {value: 0x1308, lo: 0xb1, hi: 0xb1}, + {value: 0x0008, lo: 0xb2, hi: 0xb2}, + {value: 0x08f1, lo: 0xb3, hi: 0xb3}, + {value: 0x1308, lo: 0xb4, hi: 0xb9}, + {value: 0x1b08, lo: 0xba, hi: 0xba}, + {value: 0x0040, lo: 0xbb, hi: 0xbe}, + {value: 0x0018, lo: 0xbf, hi: 0xbf}, + // Block 0x18, offset 0x103 + {value: 0x0000, lo: 0x06}, + {value: 0x0008, lo: 0x80, hi: 0x86}, + {value: 0x1308, lo: 0x87, hi: 0x8e}, + {value: 0x0018, lo: 0x8f, hi: 0x8f}, + {value: 0x0008, lo: 0x90, hi: 0x99}, + {value: 0x0018, lo: 0x9a, hi: 0x9b}, + {value: 0x0040, lo: 0x9c, hi: 0xbf}, + // Block 0x19, offset 0x10a + {value: 0x0000, lo: 0x0c}, + {value: 0x0008, lo: 0x80, hi: 0x84}, + {value: 0x0040, lo: 0x85, hi: 0x85}, + {value: 0x0008, lo: 0x86, hi: 0x86}, + {value: 0x0040, lo: 0x87, hi: 0x87}, + {value: 0x1308, lo: 0x88, hi: 0x8d}, + {value: 0x0040, lo: 0x8e, hi: 0x8f}, + {value: 0x0008, lo: 0x90, hi: 0x99}, + {value: 0x0040, lo: 0x9a, hi: 0x9b}, + {value: 0x0961, lo: 0x9c, hi: 0x9c}, + {value: 0x0999, lo: 0x9d, hi: 0x9d}, + {value: 0x0008, lo: 0x9e, hi: 0x9f}, + {value: 0x0040, lo: 0xa0, hi: 0xbf}, + // Block 0x1a, offset 0x117 + {value: 0x0000, lo: 0x10}, + {value: 0x0008, lo: 0x80, hi: 0x80}, + {value: 0x0018, lo: 0x81, hi: 0x8a}, + {value: 0x0008, lo: 0x8b, hi: 0x8b}, + {value: 0xe03d, lo: 0x8c, hi: 0x8c}, + {value: 0x0018, lo: 0x8d, hi: 0x97}, + {value: 0x1308, lo: 0x98, hi: 0x99}, + {value: 0x0018, lo: 0x9a, hi: 0x9f}, + {value: 0x0008, lo: 0xa0, hi: 0xa9}, + {value: 0x0018, lo: 0xaa, hi: 0xb4}, + {value: 0x1308, lo: 0xb5, hi: 0xb5}, + {value: 0x0018, lo: 0xb6, hi: 0xb6}, + {value: 0x1308, lo: 0xb7, hi: 0xb7}, + {value: 0x0018, lo: 0xb8, hi: 0xb8}, + {value: 0x1308, lo: 0xb9, hi: 0xb9}, + {value: 0x0018, lo: 0xba, hi: 0xbd}, + {value: 0x1008, lo: 0xbe, hi: 0xbf}, + // Block 0x1b, offset 0x128 + {value: 0x0000, lo: 0x06}, + {value: 0x0018, lo: 0x80, hi: 0x85}, + {value: 0x1308, lo: 0x86, hi: 0x86}, + {value: 0x0018, lo: 0x87, hi: 0x8c}, + {value: 0x0040, lo: 0x8d, hi: 0x8d}, + {value: 0x0018, lo: 0x8e, hi: 0x9a}, + {value: 0x0040, lo: 0x9b, hi: 0xbf}, + // Block 0x1c, offset 0x12f + {value: 0x0000, lo: 0x0a}, + {value: 0x0008, lo: 0x80, hi: 0xaa}, + {value: 0x1008, lo: 0xab, hi: 0xac}, + {value: 0x1308, lo: 0xad, hi: 0xb0}, + {value: 0x1008, lo: 0xb1, hi: 0xb1}, + {value: 0x1308, lo: 0xb2, hi: 0xb7}, + {value: 0x1008, lo: 0xb8, hi: 0xb8}, + {value: 0x1b08, lo: 0xb9, hi: 0xba}, + {value: 0x1008, lo: 0xbb, hi: 0xbc}, + {value: 0x1308, lo: 0xbd, hi: 0xbe}, + {value: 0x0008, lo: 0xbf, hi: 0xbf}, + // Block 0x1d, offset 0x13a + {value: 0x0000, lo: 0x0e}, + {value: 0x0008, lo: 0x80, hi: 0x89}, + {value: 0x0018, lo: 0x8a, hi: 0x8f}, + {value: 0x0008, lo: 0x90, hi: 0x95}, + {value: 0x1008, lo: 0x96, hi: 0x97}, + {value: 0x1308, lo: 0x98, hi: 0x99}, + {value: 0x0008, lo: 0x9a, hi: 0x9d}, + {value: 0x1308, lo: 0x9e, hi: 0xa0}, + {value: 0x0008, lo: 0xa1, hi: 0xa1}, + {value: 0x1008, lo: 0xa2, hi: 0xa4}, + {value: 0x0008, lo: 0xa5, hi: 0xa6}, + {value: 0x1008, lo: 0xa7, hi: 0xad}, + {value: 0x0008, lo: 0xae, hi: 0xb0}, + {value: 0x1308, lo: 0xb1, hi: 0xb4}, + {value: 0x0008, lo: 0xb5, hi: 0xbf}, + // Block 0x1e, offset 0x149 + {value: 0x0000, lo: 0x0d}, + {value: 0x0008, lo: 0x80, hi: 0x81}, + {value: 0x1308, lo: 0x82, hi: 0x82}, + {value: 0x1008, lo: 0x83, hi: 0x84}, + {value: 0x1308, lo: 0x85, hi: 0x86}, + {value: 0x1008, lo: 0x87, hi: 0x8c}, + {value: 0x1308, lo: 0x8d, hi: 0x8d}, + {value: 0x0008, lo: 0x8e, hi: 0x8e}, + {value: 0x1008, lo: 0x8f, hi: 0x8f}, + {value: 0x0008, lo: 0x90, hi: 0x99}, + {value: 0x1008, lo: 0x9a, hi: 0x9c}, + {value: 0x1308, lo: 0x9d, hi: 0x9d}, + {value: 0x0018, lo: 0x9e, hi: 0x9f}, + {value: 0x0040, lo: 0xa0, hi: 0xbf}, + // Block 0x1f, offset 0x157 + {value: 0x0000, lo: 0x09}, + {value: 0x0040, lo: 0x80, hi: 0x86}, + {value: 0x055d, lo: 0x87, hi: 0x87}, + {value: 0x0040, lo: 0x88, hi: 0x8c}, + {value: 0x055d, lo: 0x8d, hi: 0x8d}, + {value: 0x0040, lo: 0x8e, hi: 0x8f}, + {value: 0x0008, lo: 0x90, hi: 0xba}, + {value: 0x0018, lo: 0xbb, hi: 0xbb}, + {value: 0xe105, lo: 0xbc, hi: 0xbc}, + {value: 0x0008, lo: 0xbd, hi: 0xbf}, + // Block 0x20, offset 0x161 + {value: 0x0000, lo: 0x01}, + {value: 0x0018, lo: 0x80, hi: 0xbf}, + // Block 0x21, offset 0x163 + {value: 0x0000, lo: 0x03}, + {value: 0x0018, lo: 0x80, hi: 0x9e}, + {value: 0x0040, lo: 0x9f, hi: 0xa0}, + {value: 0x0018, lo: 0xa1, hi: 0xbf}, + // Block 0x22, offset 0x167 + {value: 0x0000, lo: 0x01}, + {value: 0x0008, lo: 0x80, hi: 0xbf}, + // Block 0x23, offset 0x169 + {value: 0x0000, lo: 0x0b}, + {value: 0x0008, lo: 0x80, hi: 0x88}, + {value: 0x0040, lo: 0x89, hi: 0x89}, + {value: 0x0008, lo: 0x8a, hi: 0x8d}, + {value: 0x0040, lo: 0x8e, hi: 0x8f}, + {value: 0x0008, lo: 0x90, hi: 0x96}, + {value: 0x0040, lo: 0x97, hi: 0x97}, + {value: 0x0008, lo: 0x98, hi: 0x98}, + {value: 0x0040, lo: 0x99, hi: 0x99}, + {value: 0x0008, lo: 0x9a, hi: 0x9d}, + {value: 0x0040, lo: 0x9e, hi: 0x9f}, + {value: 0x0008, lo: 0xa0, hi: 0xbf}, + // Block 0x24, offset 0x175 + {value: 0x0000, lo: 0x0a}, + {value: 0x0008, lo: 0x80, hi: 0x88}, + {value: 0x0040, lo: 0x89, hi: 0x89}, + {value: 0x0008, lo: 0x8a, hi: 0x8d}, + {value: 0x0040, lo: 0x8e, hi: 0x8f}, + {value: 0x0008, lo: 0x90, hi: 0xb0}, + {value: 0x0040, lo: 0xb1, hi: 0xb1}, + {value: 0x0008, lo: 0xb2, hi: 0xb5}, + {value: 0x0040, lo: 0xb6, hi: 0xb7}, + {value: 0x0008, lo: 0xb8, hi: 0xbe}, + {value: 0x0040, lo: 0xbf, hi: 0xbf}, + // Block 0x25, offset 0x180 + {value: 0x0000, lo: 0x07}, + {value: 0x0008, lo: 0x80, hi: 0x80}, + {value: 0x0040, lo: 0x81, hi: 0x81}, + {value: 0x0008, lo: 0x82, hi: 0x85}, + {value: 0x0040, lo: 0x86, hi: 0x87}, + {value: 0x0008, lo: 0x88, hi: 0x96}, + {value: 0x0040, lo: 0x97, hi: 0x97}, + {value: 0x0008, lo: 0x98, hi: 0xbf}, + // Block 0x26, offset 0x188 + {value: 0x0000, lo: 0x05}, + {value: 0x0008, lo: 0x80, hi: 0x90}, + {value: 0x0040, lo: 0x91, hi: 0x91}, + {value: 0x0008, lo: 0x92, hi: 0x95}, + {value: 0x0040, lo: 0x96, hi: 0x97}, + {value: 0x0008, lo: 0x98, hi: 0xbf}, + // Block 0x27, offset 0x18e + {value: 0x0000, lo: 0x05}, + {value: 0x0008, lo: 0x80, hi: 0x9a}, + {value: 0x0040, lo: 0x9b, hi: 0x9c}, + {value: 0x1308, lo: 0x9d, hi: 0x9f}, + {value: 0x0018, lo: 0xa0, hi: 0xbc}, + {value: 0x0040, lo: 0xbd, hi: 0xbf}, + // Block 0x28, offset 0x194 + {value: 0x0000, lo: 0x04}, + {value: 0x0008, lo: 0x80, hi: 0x8f}, + {value: 0x0018, lo: 0x90, hi: 0x99}, + {value: 0x0040, lo: 0x9a, hi: 0x9f}, + {value: 0x0008, lo: 0xa0, hi: 0xbf}, + // Block 0x29, offset 0x199 + {value: 0x0000, lo: 0x04}, + {value: 0x0008, lo: 0x80, hi: 0xb5}, + {value: 0x0040, lo: 0xb6, hi: 0xb7}, + {value: 0xe045, lo: 0xb8, hi: 0xbd}, + {value: 0x0040, lo: 0xbe, hi: 0xbf}, + // Block 0x2a, offset 0x19e + {value: 0x0000, lo: 0x02}, + {value: 0x0018, lo: 0x80, hi: 0x80}, + {value: 0x0008, lo: 0x81, hi: 0xbf}, + // Block 0x2b, offset 0x1a1 + {value: 0x0000, lo: 0x03}, + {value: 0x0008, lo: 0x80, hi: 0xac}, + {value: 0x0018, lo: 0xad, hi: 0xae}, + {value: 0x0008, lo: 0xaf, hi: 0xbf}, + // Block 0x2c, offset 0x1a5 + {value: 0x0000, lo: 0x05}, + {value: 0x0040, lo: 0x80, hi: 0x80}, + {value: 0x0008, lo: 0x81, hi: 0x9a}, + {value: 0x0018, lo: 0x9b, hi: 0x9c}, + {value: 0x0040, lo: 0x9d, hi: 0x9f}, + {value: 0x0008, lo: 0xa0, hi: 0xbf}, + // Block 0x2d, offset 0x1ab + {value: 0x0000, lo: 0x04}, + {value: 0x0008, lo: 0x80, hi: 0xaa}, + {value: 0x0018, lo: 0xab, hi: 0xb0}, + {value: 0x0008, lo: 0xb1, hi: 0xb8}, + {value: 0x0040, lo: 0xb9, hi: 0xbf}, + // Block 0x2e, offset 0x1b0 + {value: 0x0000, lo: 0x0b}, + {value: 0x0008, lo: 0x80, hi: 0x8c}, + {value: 0x0040, lo: 0x8d, hi: 0x8d}, + {value: 0x0008, lo: 0x8e, hi: 0x91}, + {value: 0x1308, lo: 0x92, hi: 0x93}, + {value: 0x1b08, lo: 0x94, hi: 0x94}, + {value: 0x0040, lo: 0x95, hi: 0x9f}, + {value: 0x0008, lo: 0xa0, hi: 0xb1}, + {value: 0x1308, lo: 0xb2, hi: 0xb3}, + {value: 0x1b08, lo: 0xb4, hi: 0xb4}, + {value: 0x0018, lo: 0xb5, hi: 0xb6}, + {value: 0x0040, lo: 0xb7, hi: 0xbf}, + // Block 0x2f, offset 0x1bc + {value: 0x0000, lo: 0x09}, + {value: 0x0008, lo: 0x80, hi: 0x91}, + {value: 0x1308, lo: 0x92, hi: 0x93}, + {value: 0x0040, lo: 0x94, hi: 0x9f}, + {value: 0x0008, lo: 0xa0, hi: 0xac}, + {value: 0x0040, lo: 0xad, hi: 0xad}, + {value: 0x0008, lo: 0xae, hi: 0xb0}, + {value: 0x0040, lo: 0xb1, hi: 0xb1}, + {value: 0x1308, lo: 0xb2, hi: 0xb3}, + {value: 0x0040, lo: 0xb4, hi: 0xbf}, + // Block 0x30, offset 0x1c6 + {value: 0x0000, lo: 0x05}, + {value: 0x0008, lo: 0x80, hi: 0xb3}, + {value: 0x1340, lo: 0xb4, hi: 0xb5}, + {value: 0x1008, lo: 0xb6, hi: 0xb6}, + {value: 0x1308, lo: 0xb7, hi: 0xbd}, + {value: 0x1008, lo: 0xbe, hi: 0xbf}, + // Block 0x31, offset 0x1cc + {value: 0x0000, lo: 0x10}, + {value: 0x1008, lo: 0x80, hi: 0x85}, + {value: 0x1308, lo: 0x86, hi: 0x86}, + {value: 0x1008, lo: 0x87, hi: 0x88}, + {value: 0x1308, lo: 0x89, hi: 0x91}, + {value: 0x1b08, lo: 0x92, hi: 0x92}, + {value: 0x1308, lo: 0x93, hi: 0x93}, + {value: 0x0018, lo: 0x94, hi: 0x96}, + {value: 0x0008, lo: 0x97, hi: 0x97}, + {value: 0x0018, lo: 0x98, hi: 0x9b}, + {value: 0x0008, lo: 0x9c, hi: 0x9c}, + {value: 0x1308, lo: 0x9d, hi: 0x9d}, + {value: 0x0040, lo: 0x9e, hi: 0x9f}, + {value: 0x0008, lo: 0xa0, hi: 0xa9}, + {value: 0x0040, lo: 0xaa, hi: 0xaf}, + {value: 0x0018, lo: 0xb0, hi: 0xb9}, + {value: 0x0040, lo: 0xba, hi: 0xbf}, + // Block 0x32, offset 0x1dd + {value: 0x0000, lo: 0x09}, + {value: 0x0018, lo: 0x80, hi: 0x85}, + {value: 0x0040, lo: 0x86, hi: 0x86}, + {value: 0x0218, lo: 0x87, hi: 0x87}, + {value: 0x0018, lo: 0x88, hi: 0x8a}, + {value: 0x13c0, lo: 0x8b, hi: 0x8d}, + {value: 0x0040, lo: 0x8e, hi: 0x8f}, + {value: 0x0008, lo: 0x90, hi: 0x99}, + {value: 0x0040, lo: 0x9a, hi: 0x9f}, + {value: 0x0208, lo: 0xa0, hi: 0xbf}, + // Block 0x33, offset 0x1e7 + {value: 0x0000, lo: 0x02}, + {value: 0x0208, lo: 0x80, hi: 0xb7}, + {value: 0x0040, lo: 0xb8, hi: 0xbf}, + // Block 0x34, offset 0x1ea + {value: 0x0000, lo: 0x07}, + {value: 0x0008, lo: 0x80, hi: 0x84}, + {value: 0x1308, lo: 0x85, hi: 0x86}, + {value: 0x0208, lo: 0x87, hi: 0xa8}, + {value: 0x1308, lo: 0xa9, hi: 0xa9}, + {value: 0x0208, lo: 0xaa, hi: 0xaa}, + {value: 0x0040, lo: 0xab, hi: 0xaf}, + {value: 0x0008, lo: 0xb0, hi: 0xbf}, + // Block 0x35, offset 0x1f2 + {value: 0x0000, lo: 0x02}, + {value: 0x0008, lo: 0x80, hi: 0xb5}, + {value: 0x0040, lo: 0xb6, hi: 0xbf}, + // Block 0x36, offset 0x1f5 + {value: 0x0000, lo: 0x0c}, + {value: 0x0008, lo: 0x80, hi: 0x9e}, + {value: 0x0040, lo: 0x9f, hi: 0x9f}, + {value: 0x1308, lo: 0xa0, hi: 0xa2}, + {value: 0x1008, lo: 0xa3, hi: 0xa6}, + {value: 0x1308, lo: 0xa7, hi: 0xa8}, + {value: 0x1008, lo: 0xa9, hi: 0xab}, + {value: 0x0040, lo: 0xac, hi: 0xaf}, + {value: 0x1008, lo: 0xb0, hi: 0xb1}, + {value: 0x1308, lo: 0xb2, hi: 0xb2}, + {value: 0x1008, lo: 0xb3, hi: 0xb8}, + {value: 0x1308, lo: 0xb9, hi: 0xbb}, + {value: 0x0040, lo: 0xbc, hi: 0xbf}, + // Block 0x37, offset 0x202 + {value: 0x0000, lo: 0x07}, + {value: 0x0018, lo: 0x80, hi: 0x80}, + {value: 0x0040, lo: 0x81, hi: 0x83}, + {value: 0x0018, lo: 0x84, hi: 0x85}, + {value: 0x0008, lo: 0x86, hi: 0xad}, + {value: 0x0040, lo: 0xae, hi: 0xaf}, + {value: 0x0008, lo: 0xb0, hi: 0xb4}, + {value: 0x0040, lo: 0xb5, hi: 0xbf}, + // Block 0x38, offset 0x20a + {value: 0x0000, lo: 0x03}, + {value: 0x0008, lo: 0x80, hi: 0xab}, + {value: 0x0040, lo: 0xac, hi: 0xaf}, + {value: 0x0008, lo: 0xb0, hi: 0xbf}, + // Block 0x39, offset 0x20e + {value: 0x0000, lo: 0x06}, + {value: 0x0008, lo: 0x80, hi: 0x89}, + {value: 0x0040, lo: 0x8a, hi: 0x8f}, + {value: 0x0008, lo: 0x90, hi: 0x99}, + {value: 0x0028, lo: 0x9a, hi: 0x9a}, + {value: 0x0040, lo: 0x9b, hi: 0x9d}, + {value: 0x0018, lo: 0x9e, hi: 0xbf}, + // Block 0x3a, offset 0x215 + {value: 0x0000, lo: 0x07}, + {value: 0x0008, lo: 0x80, hi: 0x96}, + {value: 0x1308, lo: 0x97, hi: 0x98}, + {value: 0x1008, lo: 0x99, hi: 0x9a}, + {value: 0x1308, lo: 0x9b, hi: 0x9b}, + {value: 0x0040, lo: 0x9c, hi: 0x9d}, + {value: 0x0018, lo: 0x9e, hi: 0x9f}, + {value: 0x0008, lo: 0xa0, hi: 0xbf}, + // Block 0x3b, offset 0x21d + {value: 0x0000, lo: 0x0f}, + {value: 0x0008, lo: 0x80, hi: 0x94}, + {value: 0x1008, lo: 0x95, hi: 0x95}, + {value: 0x1308, lo: 0x96, hi: 0x96}, + {value: 0x1008, lo: 0x97, hi: 0x97}, + {value: 0x1308, lo: 0x98, hi: 0x9e}, + {value: 0x0040, lo: 0x9f, hi: 0x9f}, + {value: 0x1b08, lo: 0xa0, hi: 0xa0}, + {value: 0x1008, lo: 0xa1, hi: 0xa1}, + {value: 0x1308, lo: 0xa2, hi: 0xa2}, + {value: 0x1008, lo: 0xa3, hi: 0xa4}, + {value: 0x1308, lo: 0xa5, hi: 0xac}, + {value: 0x1008, lo: 0xad, hi: 0xb2}, + {value: 0x1308, lo: 0xb3, hi: 0xbc}, + {value: 0x0040, lo: 0xbd, hi: 0xbe}, + {value: 0x1308, lo: 0xbf, hi: 0xbf}, + // Block 0x3c, offset 0x22d + {value: 0x0000, lo: 0x0b}, + {value: 0x0008, lo: 0x80, hi: 0x89}, + {value: 0x0040, lo: 0x8a, hi: 0x8f}, + {value: 0x0008, lo: 0x90, hi: 0x99}, + {value: 0x0040, lo: 0x9a, hi: 0x9f}, + {value: 0x0018, lo: 0xa0, hi: 0xa6}, + {value: 0x0008, lo: 0xa7, hi: 0xa7}, + {value: 0x0018, lo: 0xa8, hi: 0xad}, + {value: 0x0040, lo: 0xae, hi: 0xaf}, + {value: 0x1308, lo: 0xb0, hi: 0xbd}, + {value: 0x1318, lo: 0xbe, hi: 0xbe}, + {value: 0x0040, lo: 0xbf, hi: 0xbf}, + // Block 0x3d, offset 0x239 + {value: 0x0000, lo: 0x01}, + {value: 0x0040, lo: 0x80, hi: 0xbf}, + // Block 0x3e, offset 0x23b + {value: 0x0000, lo: 0x09}, + {value: 0x1308, lo: 0x80, hi: 0x83}, + {value: 0x1008, lo: 0x84, hi: 0x84}, + {value: 0x0008, lo: 0x85, hi: 0xb3}, + {value: 0x1308, lo: 0xb4, hi: 0xb4}, + {value: 0x1008, lo: 0xb5, hi: 0xb5}, + {value: 0x1308, lo: 0xb6, hi: 0xba}, + {value: 0x1008, lo: 0xbb, hi: 0xbb}, + {value: 0x1308, lo: 0xbc, hi: 0xbc}, + {value: 0x1008, lo: 0xbd, hi: 0xbf}, + // Block 0x3f, offset 0x245 + {value: 0x0000, lo: 0x0b}, + {value: 0x1008, lo: 0x80, hi: 0x81}, + {value: 0x1308, lo: 0x82, hi: 0x82}, + {value: 0x1008, lo: 0x83, hi: 0x83}, + {value: 0x1808, lo: 0x84, hi: 0x84}, + {value: 0x0008, lo: 0x85, hi: 0x8b}, + {value: 0x0040, lo: 0x8c, hi: 0x8f}, + {value: 0x0008, lo: 0x90, hi: 0x99}, + {value: 0x0018, lo: 0x9a, hi: 0xaa}, + {value: 0x1308, lo: 0xab, hi: 0xb3}, + {value: 0x0018, lo: 0xb4, hi: 0xbc}, + {value: 0x0040, lo: 0xbd, hi: 0xbf}, + // Block 0x40, offset 0x251 + {value: 0x0000, lo: 0x0b}, + {value: 0x1308, lo: 0x80, hi: 0x81}, + {value: 0x1008, lo: 0x82, hi: 0x82}, + {value: 0x0008, lo: 0x83, hi: 0xa0}, + {value: 0x1008, lo: 0xa1, hi: 0xa1}, + {value: 0x1308, lo: 0xa2, hi: 0xa5}, + {value: 0x1008, lo: 0xa6, hi: 0xa7}, + {value: 0x1308, lo: 0xa8, hi: 0xa9}, + {value: 0x1808, lo: 0xaa, hi: 0xaa}, + {value: 0x1b08, lo: 0xab, hi: 0xab}, + {value: 0x1308, lo: 0xac, hi: 0xad}, + {value: 0x0008, lo: 0xae, hi: 0xbf}, + // Block 0x41, offset 0x25d + {value: 0x0000, lo: 0x0b}, + {value: 0x0008, lo: 0x80, hi: 0xa5}, + {value: 0x1308, lo: 0xa6, hi: 0xa6}, + {value: 0x1008, lo: 0xa7, hi: 0xa7}, + {value: 0x1308, lo: 0xa8, hi: 0xa9}, + {value: 0x1008, lo: 0xaa, hi: 0xac}, + {value: 0x1308, lo: 0xad, hi: 0xad}, + {value: 0x1008, lo: 0xae, hi: 0xae}, + {value: 0x1308, lo: 0xaf, hi: 0xb1}, + {value: 0x1808, lo: 0xb2, hi: 0xb3}, + {value: 0x0040, lo: 0xb4, hi: 0xbb}, + {value: 0x0018, lo: 0xbc, hi: 0xbf}, + // Block 0x42, offset 0x269 + {value: 0x0000, lo: 0x07}, + {value: 0x0008, lo: 0x80, hi: 0xa3}, + {value: 0x1008, lo: 0xa4, hi: 0xab}, + {value: 0x1308, lo: 0xac, hi: 0xb3}, + {value: 0x1008, lo: 0xb4, hi: 0xb5}, + {value: 0x1308, lo: 0xb6, hi: 0xb7}, + {value: 0x0040, lo: 0xb8, hi: 0xba}, + {value: 0x0018, lo: 0xbb, hi: 0xbf}, + // Block 0x43, offset 0x271 + {value: 0x0000, lo: 0x04}, + {value: 0x0008, lo: 0x80, hi: 0x89}, + {value: 0x0040, lo: 0x8a, hi: 0x8c}, + {value: 0x0008, lo: 0x8d, hi: 0xbd}, + {value: 0x0018, lo: 0xbe, hi: 0xbf}, + // Block 0x44, offset 0x276 + {value: 0x0000, lo: 0x09}, + {value: 0x0e29, lo: 0x80, hi: 0x80}, + {value: 0x0e41, lo: 0x81, hi: 0x81}, + {value: 0x0e59, lo: 0x82, hi: 0x82}, + {value: 0x0e71, lo: 0x83, hi: 0x83}, + {value: 0x0e89, lo: 0x84, hi: 0x85}, + {value: 0x0ea1, lo: 0x86, hi: 0x86}, + {value: 0x0eb9, lo: 0x87, hi: 0x87}, + {value: 0x057d, lo: 0x88, hi: 0x88}, + {value: 0x0040, lo: 0x89, hi: 0xbf}, + // Block 0x45, offset 0x280 + {value: 0x0000, lo: 0x10}, + {value: 0x0018, lo: 0x80, hi: 0x87}, + {value: 0x0040, lo: 0x88, hi: 0x8f}, + {value: 0x1308, lo: 0x90, hi: 0x92}, + {value: 0x0018, lo: 0x93, hi: 0x93}, + {value: 0x1308, lo: 0x94, hi: 0xa0}, + {value: 0x1008, lo: 0xa1, hi: 0xa1}, + {value: 0x1308, lo: 0xa2, hi: 0xa8}, + {value: 0x0008, lo: 0xa9, hi: 0xac}, + {value: 0x1308, lo: 0xad, hi: 0xad}, + {value: 0x0008, lo: 0xae, hi: 0xb1}, + {value: 0x1008, lo: 0xb2, hi: 0xb3}, + {value: 0x1308, lo: 0xb4, hi: 0xb4}, + {value: 0x0008, lo: 0xb5, hi: 0xb6}, + {value: 0x0040, lo: 0xb7, hi: 0xb7}, + {value: 0x1308, lo: 0xb8, hi: 0xb9}, + {value: 0x0040, lo: 0xba, hi: 0xbf}, + // Block 0x46, offset 0x291 + {value: 0x0000, lo: 0x03}, + {value: 0x1308, lo: 0x80, hi: 0xb5}, + {value: 0x0040, lo: 0xb6, hi: 0xba}, + {value: 0x1308, lo: 0xbb, hi: 0xbf}, + // Block 0x47, offset 0x295 + {value: 0x0000, lo: 0x0a}, + {value: 0x0008, lo: 0x80, hi: 0x87}, + {value: 0xe045, lo: 0x88, hi: 0x8f}, + {value: 0x0008, lo: 0x90, hi: 0x95}, + {value: 0x0040, lo: 0x96, hi: 0x97}, + {value: 0xe045, lo: 0x98, hi: 0x9d}, + {value: 0x0040, lo: 0x9e, hi: 0x9f}, + {value: 0x0008, lo: 0xa0, hi: 0xa7}, + {value: 0xe045, lo: 0xa8, hi: 0xaf}, + {value: 0x0008, lo: 0xb0, hi: 0xb7}, + {value: 0xe045, lo: 0xb8, hi: 0xbf}, + // Block 0x48, offset 0x2a0 + {value: 0x0000, lo: 0x03}, + {value: 0x0040, lo: 0x80, hi: 0x8f}, + {value: 0x1318, lo: 0x90, hi: 0xb0}, + {value: 0x0040, lo: 0xb1, hi: 0xbf}, + // Block 0x49, offset 0x2a4 + {value: 0x0000, lo: 0x08}, + {value: 0x0018, lo: 0x80, hi: 0x82}, + {value: 0x0040, lo: 0x83, hi: 0x83}, + {value: 0x0008, lo: 0x84, hi: 0x84}, + {value: 0x0018, lo: 0x85, hi: 0x88}, + {value: 0x24c1, lo: 0x89, hi: 0x89}, + {value: 0x0018, lo: 0x8a, hi: 0x8b}, + {value: 0x0040, lo: 0x8c, hi: 0x8f}, + {value: 0x0018, lo: 0x90, hi: 0xbf}, + // Block 0x4a, offset 0x2ad + {value: 0x0000, lo: 0x07}, + {value: 0x0018, lo: 0x80, hi: 0xab}, + {value: 0x24f1, lo: 0xac, hi: 0xac}, + {value: 0x2529, lo: 0xad, hi: 0xad}, + {value: 0x0018, lo: 0xae, hi: 0xae}, + {value: 0x2579, lo: 0xaf, hi: 0xaf}, + {value: 0x25b1, lo: 0xb0, hi: 0xb0}, + {value: 0x0018, lo: 0xb1, hi: 0xbf}, + // Block 0x4b, offset 0x2b5 + {value: 0x0000, lo: 0x05}, + {value: 0x0018, lo: 0x80, hi: 0x9f}, + {value: 0x0080, lo: 0xa0, hi: 0xa0}, + {value: 0x0018, lo: 0xa1, hi: 0xad}, + {value: 0x0080, lo: 0xae, hi: 0xaf}, + {value: 0x0018, lo: 0xb0, hi: 0xbf}, + // Block 0x4c, offset 0x2bb + {value: 0x0000, lo: 0x04}, + {value: 0x0018, lo: 0x80, hi: 0xa8}, + {value: 0x09c5, lo: 0xa9, hi: 0xa9}, + {value: 0x09e5, lo: 0xaa, hi: 0xaa}, + {value: 0x0018, lo: 0xab, hi: 0xbf}, + // Block 0x4d, offset 0x2c0 + {value: 0x0000, lo: 0x02}, + {value: 0x0018, lo: 0x80, hi: 0xbe}, + {value: 0x0040, lo: 0xbf, hi: 0xbf}, + // Block 0x4e, offset 0x2c3 + {value: 0x0000, lo: 0x02}, + {value: 0x0018, lo: 0x80, hi: 0xa6}, + {value: 0x0040, lo: 0xa7, hi: 0xbf}, + // Block 0x4f, offset 0x2c6 + {value: 0x0000, lo: 0x03}, + {value: 0x0018, lo: 0x80, hi: 0x8b}, + {value: 0x28c1, lo: 0x8c, hi: 0x8c}, + {value: 0x0018, lo: 0x8d, hi: 0xbf}, + // Block 0x50, offset 0x2ca + {value: 0x0000, lo: 0x05}, + {value: 0x0018, lo: 0x80, hi: 0xb3}, + {value: 0x0e66, lo: 0xb4, hi: 0xb4}, + {value: 0x292a, lo: 0xb5, hi: 0xb5}, + {value: 0x0e86, lo: 0xb6, hi: 0xb6}, + {value: 0x0018, lo: 0xb7, hi: 0xbf}, + // Block 0x51, offset 0x2d0 + {value: 0x0000, lo: 0x03}, + {value: 0x0018, lo: 0x80, hi: 0x9b}, + {value: 0x2941, lo: 0x9c, hi: 0x9c}, + {value: 0x0018, lo: 0x9d, hi: 0xbf}, + // Block 0x52, offset 0x2d4 + {value: 0x0000, lo: 0x03}, + {value: 0x0018, lo: 0x80, hi: 0xb3}, + {value: 0x0040, lo: 0xb4, hi: 0xb5}, + {value: 0x0018, lo: 0xb6, hi: 0xbf}, + // Block 0x53, offset 0x2d8 + {value: 0x0000, lo: 0x05}, + {value: 0x0018, lo: 0x80, hi: 0x95}, + {value: 0x0040, lo: 0x96, hi: 0x97}, + {value: 0x0018, lo: 0x98, hi: 0xb9}, + {value: 0x0040, lo: 0xba, hi: 0xbc}, + {value: 0x0018, lo: 0xbd, hi: 0xbf}, + // Block 0x54, offset 0x2de + {value: 0x0000, lo: 0x06}, + {value: 0x0018, lo: 0x80, hi: 0x88}, + {value: 0x0040, lo: 0x89, hi: 0x89}, + {value: 0x0018, lo: 0x8a, hi: 0x91}, + {value: 0x0040, lo: 0x92, hi: 0xab}, + {value: 0x0018, lo: 0xac, hi: 0xaf}, + {value: 0x0040, lo: 0xb0, hi: 0xbf}, + // Block 0x55, offset 0x2e5 + {value: 0x0000, lo: 0x05}, + {value: 0xe185, lo: 0x80, hi: 0x8f}, + {value: 0x03f5, lo: 0x90, hi: 0x9f}, + {value: 0x0ea5, lo: 0xa0, hi: 0xae}, + {value: 0x0040, lo: 0xaf, hi: 0xaf}, + {value: 0x0008, lo: 0xb0, hi: 0xbf}, + // Block 0x56, offset 0x2eb + {value: 0x0000, lo: 0x07}, + {value: 0x0008, lo: 0x80, hi: 0xa5}, + {value: 0x0040, lo: 0xa6, hi: 0xa6}, + {value: 0x0008, lo: 0xa7, hi: 0xa7}, + {value: 0x0040, lo: 0xa8, hi: 0xac}, + {value: 0x0008, lo: 0xad, hi: 0xad}, + {value: 0x0040, lo: 0xae, hi: 0xaf}, + {value: 0x0008, lo: 0xb0, hi: 0xbf}, + // Block 0x57, offset 0x2f3 + {value: 0x0000, lo: 0x06}, + {value: 0x0008, lo: 0x80, hi: 0xa7}, + {value: 0x0040, lo: 0xa8, hi: 0xae}, + {value: 0xe075, lo: 0xaf, hi: 0xaf}, + {value: 0x0018, lo: 0xb0, hi: 0xb0}, + {value: 0x0040, lo: 0xb1, hi: 0xbe}, + {value: 0x1b08, lo: 0xbf, hi: 0xbf}, + // Block 0x58, offset 0x2fa + {value: 0x0000, lo: 0x0a}, + {value: 0x0008, lo: 0x80, hi: 0x96}, + {value: 0x0040, lo: 0x97, hi: 0x9f}, + {value: 0x0008, lo: 0xa0, hi: 0xa6}, + {value: 0x0040, lo: 0xa7, hi: 0xa7}, + {value: 0x0008, lo: 0xa8, hi: 0xae}, + {value: 0x0040, lo: 0xaf, hi: 0xaf}, + {value: 0x0008, lo: 0xb0, hi: 0xb6}, + {value: 0x0040, lo: 0xb7, hi: 0xb7}, + {value: 0x0008, lo: 0xb8, hi: 0xbe}, + {value: 0x0040, lo: 0xbf, hi: 0xbf}, + // Block 0x59, offset 0x305 + {value: 0x0000, lo: 0x09}, + {value: 0x0008, lo: 0x80, hi: 0x86}, + {value: 0x0040, lo: 0x87, hi: 0x87}, + {value: 0x0008, lo: 0x88, hi: 0x8e}, + {value: 0x0040, lo: 0x8f, hi: 0x8f}, + {value: 0x0008, lo: 0x90, hi: 0x96}, + {value: 0x0040, lo: 0x97, hi: 0x97}, + {value: 0x0008, lo: 0x98, hi: 0x9e}, + {value: 0x0040, lo: 0x9f, hi: 0x9f}, + {value: 0x1308, lo: 0xa0, hi: 0xbf}, + // Block 0x5a, offset 0x30f + {value: 0x0000, lo: 0x03}, + {value: 0x0018, lo: 0x80, hi: 0xae}, + {value: 0x0008, lo: 0xaf, hi: 0xaf}, + {value: 0x0018, lo: 0xb0, hi: 0xbf}, + // Block 0x5b, offset 0x313 + {value: 0x0000, lo: 0x02}, + {value: 0x0018, lo: 0x80, hi: 0x84}, + {value: 0x0040, lo: 0x85, hi: 0xbf}, + // Block 0x5c, offset 0x316 + {value: 0x0000, lo: 0x05}, + {value: 0x0018, lo: 0x80, hi: 0x99}, + {value: 0x0040, lo: 0x9a, hi: 0x9a}, + {value: 0x0018, lo: 0x9b, hi: 0x9e}, + {value: 0x0edd, lo: 0x9f, hi: 0x9f}, + {value: 0x0018, lo: 0xa0, hi: 0xbf}, + // Block 0x5d, offset 0x31c + {value: 0x0000, lo: 0x03}, + {value: 0x0018, lo: 0x80, hi: 0xb2}, + {value: 0x0efd, lo: 0xb3, hi: 0xb3}, + {value: 0x0040, lo: 0xb4, hi: 0xbf}, + // Block 0x5e, offset 0x320 + {value: 0x0020, lo: 0x01}, + {value: 0x0f1d, lo: 0x80, hi: 0xbf}, + // Block 0x5f, offset 0x322 + {value: 0x0020, lo: 0x02}, + {value: 0x171d, lo: 0x80, hi: 0x8f}, + {value: 0x18fd, lo: 0x90, hi: 0xbf}, + // Block 0x60, offset 0x325 + {value: 0x0020, lo: 0x01}, + {value: 0x1efd, lo: 0x80, hi: 0xbf}, + // Block 0x61, offset 0x327 + {value: 0x0000, lo: 0x02}, + {value: 0x0040, lo: 0x80, hi: 0x80}, + {value: 0x0008, lo: 0x81, hi: 0xbf}, + // Block 0x62, offset 0x32a + {value: 0x0000, lo: 0x09}, + {value: 0x0008, lo: 0x80, hi: 0x96}, + {value: 0x0040, lo: 0x97, hi: 0x98}, + {value: 0x1308, lo: 0x99, hi: 0x9a}, + {value: 0x29e2, lo: 0x9b, hi: 0x9b}, + {value: 0x2a0a, lo: 0x9c, hi: 0x9c}, + {value: 0x0008, lo: 0x9d, hi: 0x9e}, + {value: 0x2a31, lo: 0x9f, hi: 0x9f}, + {value: 0x0018, lo: 0xa0, hi: 0xa0}, + {value: 0x0008, lo: 0xa1, hi: 0xbf}, + // Block 0x63, offset 0x334 + {value: 0x0000, lo: 0x02}, + {value: 0x0008, lo: 0x80, hi: 0xbe}, + {value: 0x2a69, lo: 0xbf, hi: 0xbf}, + // Block 0x64, offset 0x337 + {value: 0x0000, lo: 0x0e}, + {value: 0x0040, lo: 0x80, hi: 0x84}, + {value: 0x0008, lo: 0x85, hi: 0xad}, + {value: 0x0040, lo: 0xae, hi: 0xb0}, + {value: 0x2a1d, lo: 0xb1, hi: 0xb1}, + {value: 0x2a3d, lo: 0xb2, hi: 0xb2}, + {value: 0x2a5d, lo: 0xb3, hi: 0xb3}, + {value: 0x2a7d, lo: 0xb4, hi: 0xb4}, + {value: 0x2a5d, lo: 0xb5, hi: 0xb5}, + {value: 0x2a9d, lo: 0xb6, hi: 0xb6}, + {value: 0x2abd, lo: 0xb7, hi: 0xb7}, + {value: 0x2add, lo: 0xb8, hi: 0xb9}, + {value: 0x2afd, lo: 0xba, hi: 0xbb}, + {value: 0x2b1d, lo: 0xbc, hi: 0xbd}, + {value: 0x2afd, lo: 0xbe, hi: 0xbf}, + // Block 0x65, offset 0x346 + {value: 0x0000, lo: 0x03}, + {value: 0x0018, lo: 0x80, hi: 0xa3}, + {value: 0x0040, lo: 0xa4, hi: 0xaf}, + {value: 0x0008, lo: 0xb0, hi: 0xbf}, + // Block 0x66, offset 0x34a + {value: 0x0030, lo: 0x04}, + {value: 0x2aa2, lo: 0x80, hi: 0x9d}, + {value: 0x305a, lo: 0x9e, hi: 0x9e}, + {value: 0x0040, lo: 0x9f, hi: 0x9f}, + {value: 0x30a2, lo: 0xa0, hi: 0xbf}, + // Block 0x67, offset 0x34f + {value: 0x0000, lo: 0x02}, + {value: 0x0008, lo: 0x80, hi: 0x95}, + {value: 0x0040, lo: 0x96, hi: 0xbf}, + // Block 0x68, offset 0x352 + {value: 0x0000, lo: 0x03}, + {value: 0x0008, lo: 0x80, hi: 0x8c}, + {value: 0x0040, lo: 0x8d, hi: 0x8f}, + {value: 0x0018, lo: 0x90, hi: 0xbf}, + // Block 0x69, offset 0x356 + {value: 0x0000, lo: 0x04}, + {value: 0x0018, lo: 0x80, hi: 0x86}, + {value: 0x0040, lo: 0x87, hi: 0x8f}, + {value: 0x0008, lo: 0x90, hi: 0xbd}, + {value: 0x0018, lo: 0xbe, hi: 0xbf}, + // Block 0x6a, offset 0x35b + {value: 0x0000, lo: 0x04}, + {value: 0x0008, lo: 0x80, hi: 0x8c}, + {value: 0x0018, lo: 0x8d, hi: 0x8f}, + {value: 0x0008, lo: 0x90, hi: 0xab}, + {value: 0x0040, lo: 0xac, hi: 0xbf}, + // Block 0x6b, offset 0x360 + {value: 0x0000, lo: 0x05}, + {value: 0x0008, lo: 0x80, hi: 0xa5}, + {value: 0x0018, lo: 0xa6, hi: 0xaf}, + {value: 0x1308, lo: 0xb0, hi: 0xb1}, + {value: 0x0018, lo: 0xb2, hi: 0xb7}, + {value: 0x0040, lo: 0xb8, hi: 0xbf}, + // Block 0x6c, offset 0x366 + {value: 0x0000, lo: 0x05}, + {value: 0x0040, lo: 0x80, hi: 0xb6}, + {value: 0x0008, lo: 0xb7, hi: 0xb7}, + {value: 0x2009, lo: 0xb8, hi: 0xb8}, + {value: 0x6e89, lo: 0xb9, hi: 0xb9}, + {value: 0x0008, lo: 0xba, hi: 0xbf}, + // Block 0x6d, offset 0x36c + {value: 0x0000, lo: 0x0e}, + {value: 0x0008, lo: 0x80, hi: 0x81}, + {value: 0x1308, lo: 0x82, hi: 0x82}, + {value: 0x0008, lo: 0x83, hi: 0x85}, + {value: 0x1b08, lo: 0x86, hi: 0x86}, + {value: 0x0008, lo: 0x87, hi: 0x8a}, + {value: 0x1308, lo: 0x8b, hi: 0x8b}, + {value: 0x0008, lo: 0x8c, hi: 0xa2}, + {value: 0x1008, lo: 0xa3, hi: 0xa4}, + {value: 0x1308, lo: 0xa5, hi: 0xa6}, + {value: 0x1008, lo: 0xa7, hi: 0xa7}, + {value: 0x0018, lo: 0xa8, hi: 0xab}, + {value: 0x0040, lo: 0xac, hi: 0xaf}, + {value: 0x0018, lo: 0xb0, hi: 0xb9}, + {value: 0x0040, lo: 0xba, hi: 0xbf}, + // Block 0x6e, offset 0x37b + {value: 0x0000, lo: 0x05}, + {value: 0x0208, lo: 0x80, hi: 0xb1}, + {value: 0x0108, lo: 0xb2, hi: 0xb2}, + {value: 0x0008, lo: 0xb3, hi: 0xb3}, + {value: 0x0018, lo: 0xb4, hi: 0xb7}, + {value: 0x0040, lo: 0xb8, hi: 0xbf}, + // Block 0x6f, offset 0x381 + {value: 0x0000, lo: 0x03}, + {value: 0x1008, lo: 0x80, hi: 0x81}, + {value: 0x0008, lo: 0x82, hi: 0xb3}, + {value: 0x1008, lo: 0xb4, hi: 0xbf}, + // Block 0x70, offset 0x385 + {value: 0x0000, lo: 0x0e}, + {value: 0x1008, lo: 0x80, hi: 0x83}, + {value: 0x1b08, lo: 0x84, hi: 0x84}, + {value: 0x1308, lo: 0x85, hi: 0x85}, + {value: 0x0040, lo: 0x86, hi: 0x8d}, + {value: 0x0018, lo: 0x8e, hi: 0x8f}, + {value: 0x0008, lo: 0x90, hi: 0x99}, + {value: 0x0040, lo: 0x9a, hi: 0x9f}, + {value: 0x1308, lo: 0xa0, hi: 0xb1}, + {value: 0x0008, lo: 0xb2, hi: 0xb7}, + {value: 0x0018, lo: 0xb8, hi: 0xba}, + {value: 0x0008, lo: 0xbb, hi: 0xbb}, + {value: 0x0018, lo: 0xbc, hi: 0xbc}, + {value: 0x0008, lo: 0xbd, hi: 0xbd}, + {value: 0x0040, lo: 0xbe, hi: 0xbf}, + // Block 0x71, offset 0x394 + {value: 0x0000, lo: 0x04}, + {value: 0x0008, lo: 0x80, hi: 0xa5}, + {value: 0x1308, lo: 0xa6, hi: 0xad}, + {value: 0x0018, lo: 0xae, hi: 0xaf}, + {value: 0x0008, lo: 0xb0, hi: 0xbf}, + // Block 0x72, offset 0x399 + {value: 0x0000, lo: 0x07}, + {value: 0x0008, lo: 0x80, hi: 0x86}, + {value: 0x1308, lo: 0x87, hi: 0x91}, + {value: 0x1008, lo: 0x92, hi: 0x92}, + {value: 0x1808, lo: 0x93, hi: 0x93}, + {value: 0x0040, lo: 0x94, hi: 0x9e}, + {value: 0x0018, lo: 0x9f, hi: 0xbc}, + {value: 0x0040, lo: 0xbd, hi: 0xbf}, + // Block 0x73, offset 0x3a1 + {value: 0x0000, lo: 0x09}, + {value: 0x1308, lo: 0x80, hi: 0x82}, + {value: 0x1008, lo: 0x83, hi: 0x83}, + {value: 0x0008, lo: 0x84, hi: 0xb2}, + {value: 0x1308, lo: 0xb3, hi: 0xb3}, + {value: 0x1008, lo: 0xb4, hi: 0xb5}, + {value: 0x1308, lo: 0xb6, hi: 0xb9}, + {value: 0x1008, lo: 0xba, hi: 0xbb}, + {value: 0x1308, lo: 0xbc, hi: 0xbc}, + {value: 0x1008, lo: 0xbd, hi: 0xbf}, + // Block 0x74, offset 0x3ab + {value: 0x0000, lo: 0x0a}, + {value: 0x1808, lo: 0x80, hi: 0x80}, + {value: 0x0018, lo: 0x81, hi: 0x8d}, + {value: 0x0040, lo: 0x8e, hi: 0x8e}, + {value: 0x0008, lo: 0x8f, hi: 0x99}, + {value: 0x0040, lo: 0x9a, hi: 0x9d}, + {value: 0x0018, lo: 0x9e, hi: 0x9f}, + {value: 0x0008, lo: 0xa0, hi: 0xa4}, + {value: 0x1308, lo: 0xa5, hi: 0xa5}, + {value: 0x0008, lo: 0xa6, hi: 0xbe}, + {value: 0x0040, lo: 0xbf, hi: 0xbf}, + // Block 0x75, offset 0x3b6 + {value: 0x0000, lo: 0x07}, + {value: 0x0008, lo: 0x80, hi: 0xa8}, + {value: 0x1308, lo: 0xa9, hi: 0xae}, + {value: 0x1008, lo: 0xaf, hi: 0xb0}, + {value: 0x1308, lo: 0xb1, hi: 0xb2}, + {value: 0x1008, lo: 0xb3, hi: 0xb4}, + {value: 0x1308, lo: 0xb5, hi: 0xb6}, + {value: 0x0040, lo: 0xb7, hi: 0xbf}, + // Block 0x76, offset 0x3be + {value: 0x0000, lo: 0x10}, + {value: 0x0008, lo: 0x80, hi: 0x82}, + {value: 0x1308, lo: 0x83, hi: 0x83}, + {value: 0x0008, lo: 0x84, hi: 0x8b}, + {value: 0x1308, lo: 0x8c, hi: 0x8c}, + {value: 0x1008, lo: 0x8d, hi: 0x8d}, + {value: 0x0040, lo: 0x8e, hi: 0x8f}, + {value: 0x0008, lo: 0x90, hi: 0x99}, + {value: 0x0040, lo: 0x9a, hi: 0x9b}, + {value: 0x0018, lo: 0x9c, hi: 0x9f}, + {value: 0x0008, lo: 0xa0, hi: 0xb6}, + {value: 0x0018, lo: 0xb7, hi: 0xb9}, + {value: 0x0008, lo: 0xba, hi: 0xba}, + {value: 0x1008, lo: 0xbb, hi: 0xbb}, + {value: 0x1308, lo: 0xbc, hi: 0xbc}, + {value: 0x1008, lo: 0xbd, hi: 0xbd}, + {value: 0x0008, lo: 0xbe, hi: 0xbf}, + // Block 0x77, offset 0x3cf + {value: 0x0000, lo: 0x08}, + {value: 0x0008, lo: 0x80, hi: 0xaf}, + {value: 0x1308, lo: 0xb0, hi: 0xb0}, + {value: 0x0008, lo: 0xb1, hi: 0xb1}, + {value: 0x1308, lo: 0xb2, hi: 0xb4}, + {value: 0x0008, lo: 0xb5, hi: 0xb6}, + {value: 0x1308, lo: 0xb7, hi: 0xb8}, + {value: 0x0008, lo: 0xb9, hi: 0xbd}, + {value: 0x1308, lo: 0xbe, hi: 0xbf}, + // Block 0x78, offset 0x3d8 + {value: 0x0000, lo: 0x0f}, + {value: 0x0008, lo: 0x80, hi: 0x80}, + {value: 0x1308, lo: 0x81, hi: 0x81}, + {value: 0x0008, lo: 0x82, hi: 0x82}, + {value: 0x0040, lo: 0x83, hi: 0x9a}, + {value: 0x0008, lo: 0x9b, hi: 0x9d}, + {value: 0x0018, lo: 0x9e, hi: 0x9f}, + {value: 0x0008, lo: 0xa0, hi: 0xaa}, + {value: 0x1008, lo: 0xab, hi: 0xab}, + {value: 0x1308, lo: 0xac, hi: 0xad}, + {value: 0x1008, lo: 0xae, hi: 0xaf}, + {value: 0x0018, lo: 0xb0, hi: 0xb1}, + {value: 0x0008, lo: 0xb2, hi: 0xb4}, + {value: 0x1008, lo: 0xb5, hi: 0xb5}, + {value: 0x1b08, lo: 0xb6, hi: 0xb6}, + {value: 0x0040, lo: 0xb7, hi: 0xbf}, + // Block 0x79, offset 0x3e8 + {value: 0x0000, lo: 0x0c}, + {value: 0x0040, lo: 0x80, hi: 0x80}, + {value: 0x0008, lo: 0x81, hi: 0x86}, + {value: 0x0040, lo: 0x87, hi: 0x88}, + {value: 0x0008, lo: 0x89, hi: 0x8e}, + {value: 0x0040, lo: 0x8f, hi: 0x90}, + {value: 0x0008, lo: 0x91, hi: 0x96}, + {value: 0x0040, lo: 0x97, hi: 0x9f}, + {value: 0x0008, lo: 0xa0, hi: 0xa6}, + {value: 0x0040, lo: 0xa7, hi: 0xa7}, + {value: 0x0008, lo: 0xa8, hi: 0xae}, + {value: 0x0040, lo: 0xaf, hi: 0xaf}, + {value: 0x0008, lo: 0xb0, hi: 0xbf}, + // Block 0x7a, offset 0x3f5 + {value: 0x0000, lo: 0x09}, + {value: 0x0008, lo: 0x80, hi: 0x9a}, + {value: 0x0018, lo: 0x9b, hi: 0x9b}, + {value: 0x4465, lo: 0x9c, hi: 0x9c}, + {value: 0x447d, lo: 0x9d, hi: 0x9d}, + {value: 0x2971, lo: 0x9e, hi: 0x9e}, + {value: 0xe06d, lo: 0x9f, hi: 0x9f}, + {value: 0x0008, lo: 0xa0, hi: 0xa5}, + {value: 0x0040, lo: 0xa6, hi: 0xaf}, + {value: 0x4495, lo: 0xb0, hi: 0xbf}, + // Block 0x7b, offset 0x3ff + {value: 0x0000, lo: 0x04}, + {value: 0x44b5, lo: 0x80, hi: 0x8f}, + {value: 0x44d5, lo: 0x90, hi: 0x9f}, + {value: 0x44f5, lo: 0xa0, hi: 0xaf}, + {value: 0x44d5, lo: 0xb0, hi: 0xbf}, + // Block 0x7c, offset 0x404 + {value: 0x0000, lo: 0x0c}, + {value: 0x0008, lo: 0x80, hi: 0xa2}, + {value: 0x1008, lo: 0xa3, hi: 0xa4}, + {value: 0x1308, lo: 0xa5, hi: 0xa5}, + {value: 0x1008, lo: 0xa6, hi: 0xa7}, + {value: 0x1308, lo: 0xa8, hi: 0xa8}, + {value: 0x1008, lo: 0xa9, hi: 0xaa}, + {value: 0x0018, lo: 0xab, hi: 0xab}, + {value: 0x1008, lo: 0xac, hi: 0xac}, + {value: 0x1b08, lo: 0xad, hi: 0xad}, + {value: 0x0040, lo: 0xae, hi: 0xaf}, + {value: 0x0008, lo: 0xb0, hi: 0xb9}, + {value: 0x0040, lo: 0xba, hi: 0xbf}, + // Block 0x7d, offset 0x411 + {value: 0x0000, lo: 0x03}, + {value: 0x0008, lo: 0x80, hi: 0xa3}, + {value: 0x0040, lo: 0xa4, hi: 0xaf}, + {value: 0x0018, lo: 0xb0, hi: 0xbf}, + // Block 0x7e, offset 0x415 + {value: 0x0000, lo: 0x04}, + {value: 0x0018, lo: 0x80, hi: 0x86}, + {value: 0x0040, lo: 0x87, hi: 0x8a}, + {value: 0x0018, lo: 0x8b, hi: 0xbb}, + {value: 0x0040, lo: 0xbc, hi: 0xbf}, + // Block 0x7f, offset 0x41a + {value: 0x0020, lo: 0x01}, + {value: 0x4515, lo: 0x80, hi: 0xbf}, + // Block 0x80, offset 0x41c + {value: 0x0020, lo: 0x03}, + {value: 0x4d15, lo: 0x80, hi: 0x94}, + {value: 0x4ad5, lo: 0x95, hi: 0x95}, + {value: 0x4fb5, lo: 0x96, hi: 0xbf}, + // Block 0x81, offset 0x420 + {value: 0x0020, lo: 0x01}, + {value: 0x54f5, lo: 0x80, hi: 0xbf}, + // Block 0x82, offset 0x422 + {value: 0x0020, lo: 0x03}, + {value: 0x5cf5, lo: 0x80, hi: 0x84}, + {value: 0x5655, lo: 0x85, hi: 0x85}, + {value: 0x5d95, lo: 0x86, hi: 0xbf}, + // Block 0x83, offset 0x426 + {value: 0x0020, lo: 0x08}, + {value: 0x6b55, lo: 0x80, hi: 0x8f}, + {value: 0x6d15, lo: 0x90, hi: 0x90}, + {value: 0x6d55, lo: 0x91, hi: 0xab}, + {value: 0x6ea1, lo: 0xac, hi: 0xac}, + {value: 0x70b5, lo: 0xad, hi: 0xad}, + {value: 0x0040, lo: 0xae, hi: 0xae}, + {value: 0x0040, lo: 0xaf, hi: 0xaf}, + {value: 0x70d5, lo: 0xb0, hi: 0xbf}, + // Block 0x84, offset 0x42f + {value: 0x0020, lo: 0x05}, + {value: 0x72d5, lo: 0x80, hi: 0xad}, + {value: 0x6535, lo: 0xae, hi: 0xae}, + {value: 0x7895, lo: 0xaf, hi: 0xb5}, + {value: 0x6f55, lo: 0xb6, hi: 0xb6}, + {value: 0x7975, lo: 0xb7, hi: 0xbf}, + // Block 0x85, offset 0x435 + {value: 0x0028, lo: 0x03}, + {value: 0x7c21, lo: 0x80, hi: 0x82}, + {value: 0x7be1, lo: 0x83, hi: 0x83}, + {value: 0x7c99, lo: 0x84, hi: 0xbf}, + // Block 0x86, offset 0x439 + {value: 0x0038, lo: 0x0f}, + {value: 0x9db1, lo: 0x80, hi: 0x83}, + {value: 0x9e59, lo: 0x84, hi: 0x85}, + {value: 0x9e91, lo: 0x86, hi: 0x87}, + {value: 0x9ec9, lo: 0x88, hi: 0x8f}, + {value: 0x0040, lo: 0x90, hi: 0x90}, + {value: 0x0040, lo: 0x91, hi: 0x91}, + {value: 0xa089, lo: 0x92, hi: 0x97}, + {value: 0xa1a1, lo: 0x98, hi: 0x9c}, + {value: 0xa281, lo: 0x9d, hi: 0xb3}, + {value: 0x9d41, lo: 0xb4, hi: 0xb4}, + {value: 0x9db1, lo: 0xb5, hi: 0xb5}, + {value: 0xa789, lo: 0xb6, hi: 0xbb}, + {value: 0xa869, lo: 0xbc, hi: 0xbc}, + {value: 0xa7f9, lo: 0xbd, hi: 0xbd}, + {value: 0xa8d9, lo: 0xbe, hi: 0xbf}, + // Block 0x87, offset 0x449 + {value: 0x0000, lo: 0x09}, + {value: 0x0008, lo: 0x80, hi: 0x8b}, + {value: 0x0040, lo: 0x8c, hi: 0x8c}, + {value: 0x0008, lo: 0x8d, hi: 0xa6}, + {value: 0x0040, lo: 0xa7, hi: 0xa7}, + {value: 0x0008, lo: 0xa8, hi: 0xba}, + {value: 0x0040, lo: 0xbb, hi: 0xbb}, + {value: 0x0008, lo: 0xbc, hi: 0xbd}, + {value: 0x0040, lo: 0xbe, hi: 0xbe}, + {value: 0x0008, lo: 0xbf, hi: 0xbf}, + // Block 0x88, offset 0x453 + {value: 0x0000, lo: 0x04}, + {value: 0x0008, lo: 0x80, hi: 0x8d}, + {value: 0x0040, lo: 0x8e, hi: 0x8f}, + {value: 0x0008, lo: 0x90, hi: 0x9d}, + {value: 0x0040, lo: 0x9e, hi: 0xbf}, + // Block 0x89, offset 0x458 + {value: 0x0000, lo: 0x02}, + {value: 0x0008, lo: 0x80, hi: 0xba}, + {value: 0x0040, lo: 0xbb, hi: 0xbf}, + // Block 0x8a, offset 0x45b + {value: 0x0000, lo: 0x05}, + {value: 0x0018, lo: 0x80, hi: 0x82}, + {value: 0x0040, lo: 0x83, hi: 0x86}, + {value: 0x0018, lo: 0x87, hi: 0xb3}, + {value: 0x0040, lo: 0xb4, hi: 0xb6}, + {value: 0x0018, lo: 0xb7, hi: 0xbf}, + // Block 0x8b, offset 0x461 + {value: 0x0000, lo: 0x06}, + {value: 0x0018, lo: 0x80, hi: 0x8e}, + {value: 0x0040, lo: 0x8f, hi: 0x8f}, + {value: 0x0018, lo: 0x90, hi: 0x9b}, + {value: 0x0040, lo: 0x9c, hi: 0x9f}, + {value: 0x0018, lo: 0xa0, hi: 0xa0}, + {value: 0x0040, lo: 0xa1, hi: 0xbf}, + // Block 0x8c, offset 0x468 + {value: 0x0000, lo: 0x04}, + {value: 0x0040, lo: 0x80, hi: 0x8f}, + {value: 0x0018, lo: 0x90, hi: 0xbc}, + {value: 0x1308, lo: 0xbd, hi: 0xbd}, + {value: 0x0040, lo: 0xbe, hi: 0xbf}, + // Block 0x8d, offset 0x46d + {value: 0x0000, lo: 0x03}, + {value: 0x0008, lo: 0x80, hi: 0x9c}, + {value: 0x0040, lo: 0x9d, hi: 0x9f}, + {value: 0x0008, lo: 0xa0, hi: 0xbf}, + // Block 0x8e, offset 0x471 + {value: 0x0000, lo: 0x05}, + {value: 0x0008, lo: 0x80, hi: 0x90}, + {value: 0x0040, lo: 0x91, hi: 0x9f}, + {value: 0x1308, lo: 0xa0, hi: 0xa0}, + {value: 0x0018, lo: 0xa1, hi: 0xbb}, + {value: 0x0040, lo: 0xbc, hi: 0xbf}, + // Block 0x8f, offset 0x477 + {value: 0x0000, lo: 0x04}, + {value: 0x0008, lo: 0x80, hi: 0x9f}, + {value: 0x0018, lo: 0xa0, hi: 0xa3}, + {value: 0x0040, lo: 0xa4, hi: 0xaf}, + {value: 0x0008, lo: 0xb0, hi: 0xbf}, + // Block 0x90, offset 0x47c + {value: 0x0000, lo: 0x08}, + {value: 0x0008, lo: 0x80, hi: 0x80}, + {value: 0x0018, lo: 0x81, hi: 0x81}, + {value: 0x0008, lo: 0x82, hi: 0x89}, + {value: 0x0018, lo: 0x8a, hi: 0x8a}, + {value: 0x0040, lo: 0x8b, hi: 0x8f}, + {value: 0x0008, lo: 0x90, hi: 0xb5}, + {value: 0x1308, lo: 0xb6, hi: 0xba}, + {value: 0x0040, lo: 0xbb, hi: 0xbf}, + // Block 0x91, offset 0x485 + {value: 0x0000, lo: 0x04}, + {value: 0x0008, lo: 0x80, hi: 0x9d}, + {value: 0x0040, lo: 0x9e, hi: 0x9e}, + {value: 0x0018, lo: 0x9f, hi: 0x9f}, + {value: 0x0008, lo: 0xa0, hi: 0xbf}, + // Block 0x92, offset 0x48a + {value: 0x0000, lo: 0x05}, + {value: 0x0008, lo: 0x80, hi: 0x83}, + {value: 0x0040, lo: 0x84, hi: 0x87}, + {value: 0x0008, lo: 0x88, hi: 0x8f}, + {value: 0x0018, lo: 0x90, hi: 0x95}, + {value: 0x0040, lo: 0x96, hi: 0xbf}, + // Block 0x93, offset 0x490 + {value: 0x0000, lo: 0x06}, + {value: 0xe145, lo: 0x80, hi: 0x87}, + {value: 0xe1c5, lo: 0x88, hi: 0x8f}, + {value: 0xe145, lo: 0x90, hi: 0x97}, + {value: 0x8ad5, lo: 0x98, hi: 0x9f}, + {value: 0x8aed, lo: 0xa0, hi: 0xa7}, + {value: 0x0008, lo: 0xa8, hi: 0xbf}, + // Block 0x94, offset 0x497 + {value: 0x0000, lo: 0x06}, + {value: 0x0008, lo: 0x80, hi: 0x9d}, + {value: 0x0040, lo: 0x9e, hi: 0x9f}, + {value: 0x0008, lo: 0xa0, hi: 0xa9}, + {value: 0x0040, lo: 0xaa, hi: 0xaf}, + {value: 0x8aed, lo: 0xb0, hi: 0xb7}, + {value: 0x8ad5, lo: 0xb8, hi: 0xbf}, + // Block 0x95, offset 0x49e + {value: 0x0000, lo: 0x06}, + {value: 0xe145, lo: 0x80, hi: 0x87}, + {value: 0xe1c5, lo: 0x88, hi: 0x8f}, + {value: 0xe145, lo: 0x90, hi: 0x93}, + {value: 0x0040, lo: 0x94, hi: 0x97}, + {value: 0x0008, lo: 0x98, hi: 0xbb}, + {value: 0x0040, lo: 0xbc, hi: 0xbf}, + // Block 0x96, offset 0x4a5 + {value: 0x0000, lo: 0x03}, + {value: 0x0008, lo: 0x80, hi: 0xa7}, + {value: 0x0040, lo: 0xa8, hi: 0xaf}, + {value: 0x0008, lo: 0xb0, hi: 0xbf}, + // Block 0x97, offset 0x4a9 + {value: 0x0000, lo: 0x04}, + {value: 0x0008, lo: 0x80, hi: 0xa3}, + {value: 0x0040, lo: 0xa4, hi: 0xae}, + {value: 0x0018, lo: 0xaf, hi: 0xaf}, + {value: 0x0040, lo: 0xb0, hi: 0xbf}, + // Block 0x98, offset 0x4ae + {value: 0x0000, lo: 0x02}, + {value: 0x0008, lo: 0x80, hi: 0xb6}, + {value: 0x0040, lo: 0xb7, hi: 0xbf}, + // Block 0x99, offset 0x4b1 + {value: 0x0000, lo: 0x04}, + {value: 0x0008, lo: 0x80, hi: 0x95}, + {value: 0x0040, lo: 0x96, hi: 0x9f}, + {value: 0x0008, lo: 0xa0, hi: 0xa7}, + {value: 0x0040, lo: 0xa8, hi: 0xbf}, + // Block 0x9a, offset 0x4b6 + {value: 0x0000, lo: 0x0b}, + {value: 0x0008, lo: 0x80, hi: 0x85}, + {value: 0x0040, lo: 0x86, hi: 0x87}, + {value: 0x0008, lo: 0x88, hi: 0x88}, + {value: 0x0040, lo: 0x89, hi: 0x89}, + {value: 0x0008, lo: 0x8a, hi: 0xb5}, + {value: 0x0040, lo: 0xb6, hi: 0xb6}, + {value: 0x0008, lo: 0xb7, hi: 0xb8}, + {value: 0x0040, lo: 0xb9, hi: 0xbb}, + {value: 0x0008, lo: 0xbc, hi: 0xbc}, + {value: 0x0040, lo: 0xbd, hi: 0xbe}, + {value: 0x0008, lo: 0xbf, hi: 0xbf}, + // Block 0x9b, offset 0x4c2 + {value: 0x0000, lo: 0x05}, + {value: 0x0008, lo: 0x80, hi: 0x95}, + {value: 0x0040, lo: 0x96, hi: 0x96}, + {value: 0x0018, lo: 0x97, hi: 0x9f}, + {value: 0x0008, lo: 0xa0, hi: 0xb6}, + {value: 0x0018, lo: 0xb7, hi: 0xbf}, + // Block 0x9c, offset 0x4c8 + {value: 0x0000, lo: 0x04}, + {value: 0x0008, lo: 0x80, hi: 0x9e}, + {value: 0x0040, lo: 0x9f, hi: 0xa6}, + {value: 0x0018, lo: 0xa7, hi: 0xaf}, + {value: 0x0040, lo: 0xb0, hi: 0xbf}, + // Block 0x9d, offset 0x4cd + {value: 0x0000, lo: 0x06}, + {value: 0x0040, lo: 0x80, hi: 0x9f}, + {value: 0x0008, lo: 0xa0, hi: 0xb2}, + {value: 0x0040, lo: 0xb3, hi: 0xb3}, + {value: 0x0008, lo: 0xb4, hi: 0xb5}, + {value: 0x0040, lo: 0xb6, hi: 0xba}, + {value: 0x0018, lo: 0xbb, hi: 0xbf}, + // Block 0x9e, offset 0x4d4 + {value: 0x0000, lo: 0x07}, + {value: 0x0008, lo: 0x80, hi: 0x95}, + {value: 0x0018, lo: 0x96, hi: 0x9b}, + {value: 0x0040, lo: 0x9c, hi: 0x9e}, + {value: 0x0018, lo: 0x9f, hi: 0x9f}, + {value: 0x0008, lo: 0xa0, hi: 0xb9}, + {value: 0x0040, lo: 0xba, hi: 0xbe}, + {value: 0x0018, lo: 0xbf, hi: 0xbf}, + // Block 0x9f, offset 0x4dc + {value: 0x0000, lo: 0x04}, + {value: 0x0008, lo: 0x80, hi: 0xb7}, + {value: 0x0040, lo: 0xb8, hi: 0xbb}, + {value: 0x0018, lo: 0xbc, hi: 0xbd}, + {value: 0x0008, lo: 0xbe, hi: 0xbf}, + // Block 0xa0, offset 0x4e1 + {value: 0x0000, lo: 0x03}, + {value: 0x0018, lo: 0x80, hi: 0x8f}, + {value: 0x0040, lo: 0x90, hi: 0x91}, + {value: 0x0018, lo: 0x92, hi: 0xbf}, + // Block 0xa1, offset 0x4e5 + {value: 0x0000, lo: 0x0f}, + {value: 0x0008, lo: 0x80, hi: 0x80}, + {value: 0x1308, lo: 0x81, hi: 0x83}, + {value: 0x0040, lo: 0x84, hi: 0x84}, + {value: 0x1308, lo: 0x85, hi: 0x86}, + {value: 0x0040, lo: 0x87, hi: 0x8b}, + {value: 0x1308, lo: 0x8c, hi: 0x8f}, + {value: 0x0008, lo: 0x90, hi: 0x93}, + {value: 0x0040, lo: 0x94, hi: 0x94}, + {value: 0x0008, lo: 0x95, hi: 0x97}, + {value: 0x0040, lo: 0x98, hi: 0x98}, + {value: 0x0008, lo: 0x99, hi: 0xb3}, + {value: 0x0040, lo: 0xb4, hi: 0xb7}, + {value: 0x1308, lo: 0xb8, hi: 0xba}, + {value: 0x0040, lo: 0xbb, hi: 0xbe}, + {value: 0x1b08, lo: 0xbf, hi: 0xbf}, + // Block 0xa2, offset 0x4f5 + {value: 0x0000, lo: 0x06}, + {value: 0x0018, lo: 0x80, hi: 0x87}, + {value: 0x0040, lo: 0x88, hi: 0x8f}, + {value: 0x0018, lo: 0x90, hi: 0x98}, + {value: 0x0040, lo: 0x99, hi: 0x9f}, + {value: 0x0008, lo: 0xa0, hi: 0xbc}, + {value: 0x0018, lo: 0xbd, hi: 0xbf}, + // Block 0xa3, offset 0x4fc + {value: 0x0000, lo: 0x03}, + {value: 0x0008, lo: 0x80, hi: 0x9c}, + {value: 0x0018, lo: 0x9d, hi: 0x9f}, + {value: 0x0040, lo: 0xa0, hi: 0xbf}, + // Block 0xa4, offset 0x500 + {value: 0x0000, lo: 0x03}, + {value: 0x0008, lo: 0x80, hi: 0xb5}, + {value: 0x0040, lo: 0xb6, hi: 0xb8}, + {value: 0x0018, lo: 0xb9, hi: 0xbf}, + // Block 0xa5, offset 0x504 + {value: 0x0000, lo: 0x06}, + {value: 0x0008, lo: 0x80, hi: 0x95}, + {value: 0x0040, lo: 0x96, hi: 0x97}, + {value: 0x0018, lo: 0x98, hi: 0x9f}, + {value: 0x0008, lo: 0xa0, hi: 0xb2}, + {value: 0x0040, lo: 0xb3, hi: 0xb7}, + {value: 0x0018, lo: 0xb8, hi: 0xbf}, + // Block 0xa6, offset 0x50b + {value: 0x0000, lo: 0x02}, + {value: 0x0008, lo: 0x80, hi: 0x88}, + {value: 0x0040, lo: 0x89, hi: 0xbf}, + // Block 0xa7, offset 0x50e + {value: 0x0000, lo: 0x02}, + {value: 0x03dd, lo: 0x80, hi: 0xb2}, + {value: 0x0040, lo: 0xb3, hi: 0xbf}, + // Block 0xa8, offset 0x511 + {value: 0x0000, lo: 0x03}, + {value: 0x0008, lo: 0x80, hi: 0xb2}, + {value: 0x0040, lo: 0xb3, hi: 0xb9}, + {value: 0x0018, lo: 0xba, hi: 0xbf}, + // Block 0xa9, offset 0x515 + {value: 0x0000, lo: 0x03}, + {value: 0x0040, lo: 0x80, hi: 0x9f}, + {value: 0x0018, lo: 0xa0, hi: 0xbe}, + {value: 0x0040, lo: 0xbf, hi: 0xbf}, + // Block 0xaa, offset 0x519 + {value: 0x0000, lo: 0x05}, + {value: 0x1008, lo: 0x80, hi: 0x80}, + {value: 0x1308, lo: 0x81, hi: 0x81}, + {value: 0x1008, lo: 0x82, hi: 0x82}, + {value: 0x0008, lo: 0x83, hi: 0xb7}, + {value: 0x1308, lo: 0xb8, hi: 0xbf}, + // Block 0xab, offset 0x51f + {value: 0x0000, lo: 0x08}, + {value: 0x1308, lo: 0x80, hi: 0x85}, + {value: 0x1b08, lo: 0x86, hi: 0x86}, + {value: 0x0018, lo: 0x87, hi: 0x8d}, + {value: 0x0040, lo: 0x8e, hi: 0x91}, + {value: 0x0018, lo: 0x92, hi: 0xa5}, + {value: 0x0008, lo: 0xa6, hi: 0xaf}, + {value: 0x0040, lo: 0xb0, hi: 0xbe}, + {value: 0x1b08, lo: 0xbf, hi: 0xbf}, + // Block 0xac, offset 0x528 + {value: 0x0000, lo: 0x0b}, + {value: 0x1308, lo: 0x80, hi: 0x81}, + {value: 0x1008, lo: 0x82, hi: 0x82}, + {value: 0x0008, lo: 0x83, hi: 0xaf}, + {value: 0x1008, lo: 0xb0, hi: 0xb2}, + {value: 0x1308, lo: 0xb3, hi: 0xb6}, + {value: 0x1008, lo: 0xb7, hi: 0xb8}, + {value: 0x1b08, lo: 0xb9, hi: 0xb9}, + {value: 0x1308, lo: 0xba, hi: 0xba}, + {value: 0x0018, lo: 0xbb, hi: 0xbc}, + {value: 0x0340, lo: 0xbd, hi: 0xbd}, + {value: 0x0018, lo: 0xbe, hi: 0xbf}, + // Block 0xad, offset 0x534 + {value: 0x0000, lo: 0x06}, + {value: 0x0018, lo: 0x80, hi: 0x81}, + {value: 0x0040, lo: 0x82, hi: 0x8f}, + {value: 0x0008, lo: 0x90, hi: 0xa8}, + {value: 0x0040, lo: 0xa9, hi: 0xaf}, + {value: 0x0008, lo: 0xb0, hi: 0xb9}, + {value: 0x0040, lo: 0xba, hi: 0xbf}, + // Block 0xae, offset 0x53b + {value: 0x0000, lo: 0x08}, + {value: 0x1308, lo: 0x80, hi: 0x82}, + {value: 0x0008, lo: 0x83, hi: 0xa6}, + {value: 0x1308, lo: 0xa7, hi: 0xab}, + {value: 0x1008, lo: 0xac, hi: 0xac}, + {value: 0x1308, lo: 0xad, hi: 0xb2}, + {value: 0x1b08, lo: 0xb3, hi: 0xb4}, + {value: 0x0040, lo: 0xb5, hi: 0xb5}, + {value: 0x0008, lo: 0xb6, hi: 0xbf}, + // Block 0xaf, offset 0x544 + {value: 0x0000, lo: 0x07}, + {value: 0x0018, lo: 0x80, hi: 0x83}, + {value: 0x0040, lo: 0x84, hi: 0x8f}, + {value: 0x0008, lo: 0x90, hi: 0xb2}, + {value: 0x1308, lo: 0xb3, hi: 0xb3}, + {value: 0x0018, lo: 0xb4, hi: 0xb5}, + {value: 0x0008, lo: 0xb6, hi: 0xb6}, + {value: 0x0040, lo: 0xb7, hi: 0xbf}, + // Block 0xb0, offset 0x54c + {value: 0x0000, lo: 0x06}, + {value: 0x1308, lo: 0x80, hi: 0x81}, + {value: 0x1008, lo: 0x82, hi: 0x82}, + {value: 0x0008, lo: 0x83, hi: 0xb2}, + {value: 0x1008, lo: 0xb3, hi: 0xb5}, + {value: 0x1308, lo: 0xb6, hi: 0xbe}, + {value: 0x1008, lo: 0xbf, hi: 0xbf}, + // Block 0xb1, offset 0x553 + {value: 0x0000, lo: 0x0d}, + {value: 0x1808, lo: 0x80, hi: 0x80}, + {value: 0x0008, lo: 0x81, hi: 0x84}, + {value: 0x0018, lo: 0x85, hi: 0x89}, + {value: 0x1308, lo: 0x8a, hi: 0x8c}, + {value: 0x0018, lo: 0x8d, hi: 0x8d}, + {value: 0x0040, lo: 0x8e, hi: 0x8f}, + {value: 0x0008, lo: 0x90, hi: 0x9a}, + {value: 0x0018, lo: 0x9b, hi: 0x9b}, + {value: 0x0008, lo: 0x9c, hi: 0x9c}, + {value: 0x0018, lo: 0x9d, hi: 0x9f}, + {value: 0x0040, lo: 0xa0, hi: 0xa0}, + {value: 0x0018, lo: 0xa1, hi: 0xb4}, + {value: 0x0040, lo: 0xb5, hi: 0xbf}, + // Block 0xb2, offset 0x561 + {value: 0x0000, lo: 0x0c}, + {value: 0x0008, lo: 0x80, hi: 0x91}, + {value: 0x0040, lo: 0x92, hi: 0x92}, + {value: 0x0008, lo: 0x93, hi: 0xab}, + {value: 0x1008, lo: 0xac, hi: 0xae}, + {value: 0x1308, lo: 0xaf, hi: 0xb1}, + {value: 0x1008, lo: 0xb2, hi: 0xb3}, + {value: 0x1308, lo: 0xb4, hi: 0xb4}, + {value: 0x1808, lo: 0xb5, hi: 0xb5}, + {value: 0x1308, lo: 0xb6, hi: 0xb7}, + {value: 0x0018, lo: 0xb8, hi: 0xbd}, + {value: 0x1308, lo: 0xbe, hi: 0xbe}, + {value: 0x0040, lo: 0xbf, hi: 0xbf}, + // Block 0xb3, offset 0x56e + {value: 0x0000, lo: 0x0c}, + {value: 0x0008, lo: 0x80, hi: 0x86}, + {value: 0x0040, lo: 0x87, hi: 0x87}, + {value: 0x0008, lo: 0x88, hi: 0x88}, + {value: 0x0040, lo: 0x89, hi: 0x89}, + {value: 0x0008, lo: 0x8a, hi: 0x8d}, + {value: 0x0040, lo: 0x8e, hi: 0x8e}, + {value: 0x0008, lo: 0x8f, hi: 0x9d}, + {value: 0x0040, lo: 0x9e, hi: 0x9e}, + {value: 0x0008, lo: 0x9f, hi: 0xa8}, + {value: 0x0018, lo: 0xa9, hi: 0xa9}, + {value: 0x0040, lo: 0xaa, hi: 0xaf}, + {value: 0x0008, lo: 0xb0, hi: 0xbf}, + // Block 0xb4, offset 0x57b + {value: 0x0000, lo: 0x08}, + {value: 0x0008, lo: 0x80, hi: 0x9e}, + {value: 0x1308, lo: 0x9f, hi: 0x9f}, + {value: 0x1008, lo: 0xa0, hi: 0xa2}, + {value: 0x1308, lo: 0xa3, hi: 0xa9}, + {value: 0x1b08, lo: 0xaa, hi: 0xaa}, + {value: 0x0040, lo: 0xab, hi: 0xaf}, + {value: 0x0008, lo: 0xb0, hi: 0xb9}, + {value: 0x0040, lo: 0xba, hi: 0xbf}, + // Block 0xb5, offset 0x584 + {value: 0x0000, lo: 0x03}, + {value: 0x0008, lo: 0x80, hi: 0xb4}, + {value: 0x1008, lo: 0xb5, hi: 0xb7}, + {value: 0x1308, lo: 0xb8, hi: 0xbf}, + // Block 0xb6, offset 0x588 + {value: 0x0000, lo: 0x0d}, + {value: 0x1008, lo: 0x80, hi: 0x81}, + {value: 0x1b08, lo: 0x82, hi: 0x82}, + {value: 0x1308, lo: 0x83, hi: 0x84}, + {value: 0x1008, lo: 0x85, hi: 0x85}, + {value: 0x1308, lo: 0x86, hi: 0x86}, + {value: 0x0008, lo: 0x87, hi: 0x8a}, + {value: 0x0018, lo: 0x8b, hi: 0x8f}, + {value: 0x0008, lo: 0x90, hi: 0x99}, + {value: 0x0040, lo: 0x9a, hi: 0x9a}, + {value: 0x0018, lo: 0x9b, hi: 0x9b}, + {value: 0x0040, lo: 0x9c, hi: 0x9c}, + {value: 0x0018, lo: 0x9d, hi: 0x9d}, + {value: 0x0040, lo: 0x9e, hi: 0xbf}, + // Block 0xb7, offset 0x596 + {value: 0x0000, lo: 0x07}, + {value: 0x0008, lo: 0x80, hi: 0xaf}, + {value: 0x1008, lo: 0xb0, hi: 0xb2}, + {value: 0x1308, lo: 0xb3, hi: 0xb8}, + {value: 0x1008, lo: 0xb9, hi: 0xb9}, + {value: 0x1308, lo: 0xba, hi: 0xba}, + {value: 0x1008, lo: 0xbb, hi: 0xbe}, + {value: 0x1308, lo: 0xbf, hi: 0xbf}, + // Block 0xb8, offset 0x59e + {value: 0x0000, lo: 0x0a}, + {value: 0x1308, lo: 0x80, hi: 0x80}, + {value: 0x1008, lo: 0x81, hi: 0x81}, + {value: 0x1b08, lo: 0x82, hi: 0x82}, + {value: 0x1308, lo: 0x83, hi: 0x83}, + {value: 0x0008, lo: 0x84, hi: 0x85}, + {value: 0x0018, lo: 0x86, hi: 0x86}, + {value: 0x0008, lo: 0x87, hi: 0x87}, + {value: 0x0040, lo: 0x88, hi: 0x8f}, + {value: 0x0008, lo: 0x90, hi: 0x99}, + {value: 0x0040, lo: 0x9a, hi: 0xbf}, + // Block 0xb9, offset 0x5a9 + {value: 0x0000, lo: 0x08}, + {value: 0x0008, lo: 0x80, hi: 0xae}, + {value: 0x1008, lo: 0xaf, hi: 0xb1}, + {value: 0x1308, lo: 0xb2, hi: 0xb5}, + {value: 0x0040, lo: 0xb6, hi: 0xb7}, + {value: 0x1008, lo: 0xb8, hi: 0xbb}, + {value: 0x1308, lo: 0xbc, hi: 0xbd}, + {value: 0x1008, lo: 0xbe, hi: 0xbe}, + {value: 0x1b08, lo: 0xbf, hi: 0xbf}, + // Block 0xba, offset 0x5b2 + {value: 0x0000, lo: 0x05}, + {value: 0x1308, lo: 0x80, hi: 0x80}, + {value: 0x0018, lo: 0x81, hi: 0x97}, + {value: 0x0008, lo: 0x98, hi: 0x9b}, + {value: 0x1308, lo: 0x9c, hi: 0x9d}, + {value: 0x0040, lo: 0x9e, hi: 0xbf}, + // Block 0xbb, offset 0x5b8 + {value: 0x0000, lo: 0x07}, + {value: 0x0008, lo: 0x80, hi: 0xaf}, + {value: 0x1008, lo: 0xb0, hi: 0xb2}, + {value: 0x1308, lo: 0xb3, hi: 0xba}, + {value: 0x1008, lo: 0xbb, hi: 0xbc}, + {value: 0x1308, lo: 0xbd, hi: 0xbd}, + {value: 0x1008, lo: 0xbe, hi: 0xbe}, + {value: 0x1b08, lo: 0xbf, hi: 0xbf}, + // Block 0xbc, offset 0x5c0 + {value: 0x0000, lo: 0x08}, + {value: 0x1308, lo: 0x80, hi: 0x80}, + {value: 0x0018, lo: 0x81, hi: 0x83}, + {value: 0x0008, lo: 0x84, hi: 0x84}, + {value: 0x0040, lo: 0x85, hi: 0x8f}, + {value: 0x0008, lo: 0x90, hi: 0x99}, + {value: 0x0040, lo: 0x9a, hi: 0x9f}, + {value: 0x0018, lo: 0xa0, hi: 0xac}, + {value: 0x0040, lo: 0xad, hi: 0xbf}, + // Block 0xbd, offset 0x5c9 + {value: 0x0000, lo: 0x09}, + {value: 0x0008, lo: 0x80, hi: 0xaa}, + {value: 0x1308, lo: 0xab, hi: 0xab}, + {value: 0x1008, lo: 0xac, hi: 0xac}, + {value: 0x1308, lo: 0xad, hi: 0xad}, + {value: 0x1008, lo: 0xae, hi: 0xaf}, + {value: 0x1308, lo: 0xb0, hi: 0xb5}, + {value: 0x1808, lo: 0xb6, hi: 0xb6}, + {value: 0x1308, lo: 0xb7, hi: 0xb7}, + {value: 0x0040, lo: 0xb8, hi: 0xbf}, + // Block 0xbe, offset 0x5d3 + {value: 0x0000, lo: 0x02}, + {value: 0x0008, lo: 0x80, hi: 0x89}, + {value: 0x0040, lo: 0x8a, hi: 0xbf}, + // Block 0xbf, offset 0x5d6 + {value: 0x0000, lo: 0x0b}, + {value: 0x0008, lo: 0x80, hi: 0x99}, + {value: 0x0040, lo: 0x9a, hi: 0x9c}, + {value: 0x1308, lo: 0x9d, hi: 0x9f}, + {value: 0x1008, lo: 0xa0, hi: 0xa1}, + {value: 0x1308, lo: 0xa2, hi: 0xa5}, + {value: 0x1008, lo: 0xa6, hi: 0xa6}, + {value: 0x1308, lo: 0xa7, hi: 0xaa}, + {value: 0x1b08, lo: 0xab, hi: 0xab}, + {value: 0x0040, lo: 0xac, hi: 0xaf}, + {value: 0x0008, lo: 0xb0, hi: 0xb9}, + {value: 0x0018, lo: 0xba, hi: 0xbf}, + // Block 0xc0, offset 0x5e2 + {value: 0x0000, lo: 0x02}, + {value: 0x0040, lo: 0x80, hi: 0x9f}, + {value: 0x049d, lo: 0xa0, hi: 0xbf}, + // Block 0xc1, offset 0x5e5 + {value: 0x0000, lo: 0x04}, + {value: 0x0008, lo: 0x80, hi: 0xa9}, + {value: 0x0018, lo: 0xaa, hi: 0xb2}, + {value: 0x0040, lo: 0xb3, hi: 0xbe}, + {value: 0x0008, lo: 0xbf, hi: 0xbf}, + // Block 0xc2, offset 0x5ea + {value: 0x0000, lo: 0x02}, + {value: 0x0008, lo: 0x80, hi: 0xb8}, + {value: 0x0040, lo: 0xb9, hi: 0xbf}, + // Block 0xc3, offset 0x5ed + {value: 0x0000, lo: 0x09}, + {value: 0x0008, lo: 0x80, hi: 0x88}, + {value: 0x0040, lo: 0x89, hi: 0x89}, + {value: 0x0008, lo: 0x8a, hi: 0xae}, + {value: 0x1008, lo: 0xaf, hi: 0xaf}, + {value: 0x1308, lo: 0xb0, hi: 0xb6}, + {value: 0x0040, lo: 0xb7, hi: 0xb7}, + {value: 0x1308, lo: 0xb8, hi: 0xbd}, + {value: 0x1008, lo: 0xbe, hi: 0xbe}, + {value: 0x1b08, lo: 0xbf, hi: 0xbf}, + // Block 0xc4, offset 0x5f7 + {value: 0x0000, lo: 0x08}, + {value: 0x0008, lo: 0x80, hi: 0x80}, + {value: 0x0018, lo: 0x81, hi: 0x85}, + {value: 0x0040, lo: 0x86, hi: 0x8f}, + {value: 0x0008, lo: 0x90, hi: 0x99}, + {value: 0x0018, lo: 0x9a, hi: 0xac}, + {value: 0x0040, lo: 0xad, hi: 0xaf}, + {value: 0x0018, lo: 0xb0, hi: 0xb1}, + {value: 0x0008, lo: 0xb2, hi: 0xbf}, + // Block 0xc5, offset 0x600 + {value: 0x0000, lo: 0x0b}, + {value: 0x0008, lo: 0x80, hi: 0x8f}, + {value: 0x0040, lo: 0x90, hi: 0x91}, + {value: 0x1308, lo: 0x92, hi: 0xa7}, + {value: 0x0040, lo: 0xa8, hi: 0xa8}, + {value: 0x1008, lo: 0xa9, hi: 0xa9}, + {value: 0x1308, lo: 0xaa, hi: 0xb0}, + {value: 0x1008, lo: 0xb1, hi: 0xb1}, + {value: 0x1308, lo: 0xb2, hi: 0xb3}, + {value: 0x1008, lo: 0xb4, hi: 0xb4}, + {value: 0x1308, lo: 0xb5, hi: 0xb6}, + {value: 0x0040, lo: 0xb7, hi: 0xbf}, + // Block 0xc6, offset 0x60c + {value: 0x0000, lo: 0x02}, + {value: 0x0008, lo: 0x80, hi: 0x99}, + {value: 0x0040, lo: 0x9a, hi: 0xbf}, + // Block 0xc7, offset 0x60f + {value: 0x0000, lo: 0x04}, + {value: 0x0018, lo: 0x80, hi: 0xae}, + {value: 0x0040, lo: 0xaf, hi: 0xaf}, + {value: 0x0018, lo: 0xb0, hi: 0xb4}, + {value: 0x0040, lo: 0xb5, hi: 0xbf}, + // Block 0xc8, offset 0x614 + {value: 0x0000, lo: 0x02}, + {value: 0x0008, lo: 0x80, hi: 0x83}, + {value: 0x0040, lo: 0x84, hi: 0xbf}, + // Block 0xc9, offset 0x617 + {value: 0x0000, lo: 0x02}, + {value: 0x0008, lo: 0x80, hi: 0xae}, + {value: 0x0040, lo: 0xaf, hi: 0xbf}, + // Block 0xca, offset 0x61a + {value: 0x0000, lo: 0x02}, + {value: 0x0008, lo: 0x80, hi: 0x86}, + {value: 0x0040, lo: 0x87, hi: 0xbf}, + // Block 0xcb, offset 0x61d + {value: 0x0000, lo: 0x06}, + {value: 0x0008, lo: 0x80, hi: 0x9e}, + {value: 0x0040, lo: 0x9f, hi: 0x9f}, + {value: 0x0008, lo: 0xa0, hi: 0xa9}, + {value: 0x0040, lo: 0xaa, hi: 0xad}, + {value: 0x0018, lo: 0xae, hi: 0xaf}, + {value: 0x0040, lo: 0xb0, hi: 0xbf}, + // Block 0xcc, offset 0x624 + {value: 0x0000, lo: 0x06}, + {value: 0x0040, lo: 0x80, hi: 0x8f}, + {value: 0x0008, lo: 0x90, hi: 0xad}, + {value: 0x0040, lo: 0xae, hi: 0xaf}, + {value: 0x1308, lo: 0xb0, hi: 0xb4}, + {value: 0x0018, lo: 0xb5, hi: 0xb5}, + {value: 0x0040, lo: 0xb6, hi: 0xbf}, + // Block 0xcd, offset 0x62b + {value: 0x0000, lo: 0x03}, + {value: 0x0008, lo: 0x80, hi: 0xaf}, + {value: 0x1308, lo: 0xb0, hi: 0xb6}, + {value: 0x0018, lo: 0xb7, hi: 0xbf}, + // Block 0xce, offset 0x62f + {value: 0x0000, lo: 0x0a}, + {value: 0x0008, lo: 0x80, hi: 0x83}, + {value: 0x0018, lo: 0x84, hi: 0x85}, + {value: 0x0040, lo: 0x86, hi: 0x8f}, + {value: 0x0008, lo: 0x90, hi: 0x99}, + {value: 0x0040, lo: 0x9a, hi: 0x9a}, + {value: 0x0018, lo: 0x9b, hi: 0xa1}, + {value: 0x0040, lo: 0xa2, hi: 0xa2}, + {value: 0x0008, lo: 0xa3, hi: 0xb7}, + {value: 0x0040, lo: 0xb8, hi: 0xbc}, + {value: 0x0008, lo: 0xbd, hi: 0xbf}, + // Block 0xcf, offset 0x63a + {value: 0x0000, lo: 0x02}, + {value: 0x0008, lo: 0x80, hi: 0x8f}, + {value: 0x0040, lo: 0x90, hi: 0xbf}, + // Block 0xd0, offset 0x63d + {value: 0x0000, lo: 0x05}, + {value: 0x0008, lo: 0x80, hi: 0x84}, + {value: 0x0040, lo: 0x85, hi: 0x8f}, + {value: 0x0008, lo: 0x90, hi: 0x90}, + {value: 0x1008, lo: 0x91, hi: 0xbe}, + {value: 0x0040, lo: 0xbf, hi: 0xbf}, + // Block 0xd1, offset 0x643 + {value: 0x0000, lo: 0x04}, + {value: 0x0040, lo: 0x80, hi: 0x8e}, + {value: 0x1308, lo: 0x8f, hi: 0x92}, + {value: 0x0008, lo: 0x93, hi: 0x9f}, + {value: 0x0040, lo: 0xa0, hi: 0xbf}, + // Block 0xd2, offset 0x648 + {value: 0x0000, lo: 0x03}, + {value: 0x0040, lo: 0x80, hi: 0x9f}, + {value: 0x0008, lo: 0xa0, hi: 0xa0}, + {value: 0x0040, lo: 0xa1, hi: 0xbf}, + // Block 0xd3, offset 0x64c + {value: 0x0000, lo: 0x02}, + {value: 0x0008, lo: 0x80, hi: 0xac}, + {value: 0x0040, lo: 0xad, hi: 0xbf}, + // Block 0xd4, offset 0x64f + {value: 0x0000, lo: 0x02}, + {value: 0x0008, lo: 0x80, hi: 0xb2}, + {value: 0x0040, lo: 0xb3, hi: 0xbf}, + // Block 0xd5, offset 0x652 + {value: 0x0000, lo: 0x02}, + {value: 0x0008, lo: 0x80, hi: 0x81}, + {value: 0x0040, lo: 0x82, hi: 0xbf}, + // Block 0xd6, offset 0x655 + {value: 0x0000, lo: 0x04}, + {value: 0x0008, lo: 0x80, hi: 0xaa}, + {value: 0x0040, lo: 0xab, hi: 0xaf}, + {value: 0x0008, lo: 0xb0, hi: 0xbc}, + {value: 0x0040, lo: 0xbd, hi: 0xbf}, + // Block 0xd7, offset 0x65a + {value: 0x0000, lo: 0x09}, + {value: 0x0008, lo: 0x80, hi: 0x88}, + {value: 0x0040, lo: 0x89, hi: 0x8f}, + {value: 0x0008, lo: 0x90, hi: 0x99}, + {value: 0x0040, lo: 0x9a, hi: 0x9b}, + {value: 0x0018, lo: 0x9c, hi: 0x9c}, + {value: 0x1308, lo: 0x9d, hi: 0x9e}, + {value: 0x0018, lo: 0x9f, hi: 0x9f}, + {value: 0x03c0, lo: 0xa0, hi: 0xa3}, + {value: 0x0040, lo: 0xa4, hi: 0xbf}, + // Block 0xd8, offset 0x664 + {value: 0x0000, lo: 0x02}, + {value: 0x0018, lo: 0x80, hi: 0xb5}, + {value: 0x0040, lo: 0xb6, hi: 0xbf}, + // Block 0xd9, offset 0x667 + {value: 0x0000, lo: 0x03}, + {value: 0x0018, lo: 0x80, hi: 0xa6}, + {value: 0x0040, lo: 0xa7, hi: 0xa8}, + {value: 0x0018, lo: 0xa9, hi: 0xbf}, + // Block 0xda, offset 0x66b + {value: 0x0000, lo: 0x0e}, + {value: 0x0018, lo: 0x80, hi: 0x9d}, + {value: 0xb5b9, lo: 0x9e, hi: 0x9e}, + {value: 0xb601, lo: 0x9f, hi: 0x9f}, + {value: 0xb649, lo: 0xa0, hi: 0xa0}, + {value: 0xb6b1, lo: 0xa1, hi: 0xa1}, + {value: 0xb719, lo: 0xa2, hi: 0xa2}, + {value: 0xb781, lo: 0xa3, hi: 0xa3}, + {value: 0xb7e9, lo: 0xa4, hi: 0xa4}, + {value: 0x1018, lo: 0xa5, hi: 0xa6}, + {value: 0x1318, lo: 0xa7, hi: 0xa9}, + {value: 0x0018, lo: 0xaa, hi: 0xac}, + {value: 0x1018, lo: 0xad, hi: 0xb2}, + {value: 0x0340, lo: 0xb3, hi: 0xba}, + {value: 0x1318, lo: 0xbb, hi: 0xbf}, + // Block 0xdb, offset 0x67a + {value: 0x0000, lo: 0x0b}, + {value: 0x1318, lo: 0x80, hi: 0x82}, + {value: 0x0018, lo: 0x83, hi: 0x84}, + {value: 0x1318, lo: 0x85, hi: 0x8b}, + {value: 0x0018, lo: 0x8c, hi: 0xa9}, + {value: 0x1318, lo: 0xaa, hi: 0xad}, + {value: 0x0018, lo: 0xae, hi: 0xba}, + {value: 0xb851, lo: 0xbb, hi: 0xbb}, + {value: 0xb899, lo: 0xbc, hi: 0xbc}, + {value: 0xb8e1, lo: 0xbd, hi: 0xbd}, + {value: 0xb949, lo: 0xbe, hi: 0xbe}, + {value: 0xb9b1, lo: 0xbf, hi: 0xbf}, + // Block 0xdc, offset 0x686 + {value: 0x0000, lo: 0x03}, + {value: 0xba19, lo: 0x80, hi: 0x80}, + {value: 0x0018, lo: 0x81, hi: 0xa8}, + {value: 0x0040, lo: 0xa9, hi: 0xbf}, + // Block 0xdd, offset 0x68a + {value: 0x0000, lo: 0x04}, + {value: 0x0018, lo: 0x80, hi: 0x81}, + {value: 0x1318, lo: 0x82, hi: 0x84}, + {value: 0x0018, lo: 0x85, hi: 0x85}, + {value: 0x0040, lo: 0x86, hi: 0xbf}, + // Block 0xde, offset 0x68f + {value: 0x0000, lo: 0x04}, + {value: 0x0018, lo: 0x80, hi: 0x96}, + {value: 0x0040, lo: 0x97, hi: 0x9f}, + {value: 0x0018, lo: 0xa0, hi: 0xb1}, + {value: 0x0040, lo: 0xb2, hi: 0xbf}, + // Block 0xdf, offset 0x694 + {value: 0x0000, lo: 0x03}, + {value: 0x1308, lo: 0x80, hi: 0xb6}, + {value: 0x0018, lo: 0xb7, hi: 0xba}, + {value: 0x1308, lo: 0xbb, hi: 0xbf}, + // Block 0xe0, offset 0x698 + {value: 0x0000, lo: 0x04}, + {value: 0x1308, lo: 0x80, hi: 0xac}, + {value: 0x0018, lo: 0xad, hi: 0xb4}, + {value: 0x1308, lo: 0xb5, hi: 0xb5}, + {value: 0x0018, lo: 0xb6, hi: 0xbf}, + // Block 0xe1, offset 0x69d + {value: 0x0000, lo: 0x08}, + {value: 0x0018, lo: 0x80, hi: 0x83}, + {value: 0x1308, lo: 0x84, hi: 0x84}, + {value: 0x0018, lo: 0x85, hi: 0x8b}, + {value: 0x0040, lo: 0x8c, hi: 0x9a}, + {value: 0x1308, lo: 0x9b, hi: 0x9f}, + {value: 0x0040, lo: 0xa0, hi: 0xa0}, + {value: 0x1308, lo: 0xa1, hi: 0xaf}, + {value: 0x0040, lo: 0xb0, hi: 0xbf}, + // Block 0xe2, offset 0x6a6 + {value: 0x0000, lo: 0x0a}, + {value: 0x1308, lo: 0x80, hi: 0x86}, + {value: 0x0040, lo: 0x87, hi: 0x87}, + {value: 0x1308, lo: 0x88, hi: 0x98}, + {value: 0x0040, lo: 0x99, hi: 0x9a}, + {value: 0x1308, lo: 0x9b, hi: 0xa1}, + {value: 0x0040, lo: 0xa2, hi: 0xa2}, + {value: 0x1308, lo: 0xa3, hi: 0xa4}, + {value: 0x0040, lo: 0xa5, hi: 0xa5}, + {value: 0x1308, lo: 0xa6, hi: 0xaa}, + {value: 0x0040, lo: 0xab, hi: 0xbf}, + // Block 0xe3, offset 0x6b1 + {value: 0x0000, lo: 0x05}, + {value: 0x0008, lo: 0x80, hi: 0x84}, + {value: 0x0040, lo: 0x85, hi: 0x86}, + {value: 0x0018, lo: 0x87, hi: 0x8f}, + {value: 0x1308, lo: 0x90, hi: 0x96}, + {value: 0x0040, lo: 0x97, hi: 0xbf}, + // Block 0xe4, offset 0x6b7 + {value: 0x0000, lo: 0x07}, + {value: 0x0208, lo: 0x80, hi: 0x83}, + {value: 0x1308, lo: 0x84, hi: 0x8a}, + {value: 0x0040, lo: 0x8b, hi: 0x8f}, + {value: 0x0008, lo: 0x90, hi: 0x99}, + {value: 0x0040, lo: 0x9a, hi: 0x9d}, + {value: 0x0018, lo: 0x9e, hi: 0x9f}, + {value: 0x0040, lo: 0xa0, hi: 0xbf}, + // Block 0xe5, offset 0x6bf + {value: 0x0000, lo: 0x03}, + {value: 0x0040, lo: 0x80, hi: 0xaf}, + {value: 0x0018, lo: 0xb0, hi: 0xb1}, + {value: 0x0040, lo: 0xb2, hi: 0xbf}, + // Block 0xe6, offset 0x6c3 + {value: 0x0000, lo: 0x03}, + {value: 0x0018, lo: 0x80, hi: 0xab}, + {value: 0x0040, lo: 0xac, hi: 0xaf}, + {value: 0x0018, lo: 0xb0, hi: 0xbf}, + // Block 0xe7, offset 0x6c7 + {value: 0x0000, lo: 0x05}, + {value: 0x0018, lo: 0x80, hi: 0x93}, + {value: 0x0040, lo: 0x94, hi: 0x9f}, + {value: 0x0018, lo: 0xa0, hi: 0xae}, + {value: 0x0040, lo: 0xaf, hi: 0xb0}, + {value: 0x0018, lo: 0xb1, hi: 0xbf}, + // Block 0xe8, offset 0x6cd + {value: 0x0000, lo: 0x05}, + {value: 0x0040, lo: 0x80, hi: 0x80}, + {value: 0x0018, lo: 0x81, hi: 0x8f}, + {value: 0x0040, lo: 0x90, hi: 0x90}, + {value: 0x0018, lo: 0x91, hi: 0xb5}, + {value: 0x0040, lo: 0xb6, hi: 0xbf}, + // Block 0xe9, offset 0x6d3 + {value: 0x0000, lo: 0x04}, + {value: 0x0018, lo: 0x80, hi: 0x8f}, + {value: 0xc1c1, lo: 0x90, hi: 0x90}, + {value: 0x0018, lo: 0x91, hi: 0xac}, + {value: 0x0040, lo: 0xad, hi: 0xbf}, + // Block 0xea, offset 0x6d8 + {value: 0x0000, lo: 0x02}, + {value: 0x0040, lo: 0x80, hi: 0xa5}, + {value: 0x0018, lo: 0xa6, hi: 0xbf}, + // Block 0xeb, offset 0x6db + {value: 0x0000, lo: 0x0d}, + {value: 0xc7e9, lo: 0x80, hi: 0x80}, + {value: 0xc839, lo: 0x81, hi: 0x81}, + {value: 0xc889, lo: 0x82, hi: 0x82}, + {value: 0xc8d9, lo: 0x83, hi: 0x83}, + {value: 0xc929, lo: 0x84, hi: 0x84}, + {value: 0xc979, lo: 0x85, hi: 0x85}, + {value: 0xc9c9, lo: 0x86, hi: 0x86}, + {value: 0xca19, lo: 0x87, hi: 0x87}, + {value: 0xca69, lo: 0x88, hi: 0x88}, + {value: 0x0040, lo: 0x89, hi: 0x8f}, + {value: 0xcab9, lo: 0x90, hi: 0x90}, + {value: 0xcad9, lo: 0x91, hi: 0x91}, + {value: 0x0040, lo: 0x92, hi: 0xbf}, + // Block 0xec, offset 0x6e9 + {value: 0x0000, lo: 0x06}, + {value: 0x0018, lo: 0x80, hi: 0x92}, + {value: 0x0040, lo: 0x93, hi: 0x9f}, + {value: 0x0018, lo: 0xa0, hi: 0xac}, + {value: 0x0040, lo: 0xad, hi: 0xaf}, + {value: 0x0018, lo: 0xb0, hi: 0xb6}, + {value: 0x0040, lo: 0xb7, hi: 0xbf}, + // Block 0xed, offset 0x6f0 + {value: 0x0000, lo: 0x02}, + {value: 0x0018, lo: 0x80, hi: 0xb3}, + {value: 0x0040, lo: 0xb4, hi: 0xbf}, + // Block 0xee, offset 0x6f3 + {value: 0x0000, lo: 0x02}, + {value: 0x0018, lo: 0x80, hi: 0x94}, + {value: 0x0040, lo: 0x95, hi: 0xbf}, + // Block 0xef, offset 0x6f6 + {value: 0x0000, lo: 0x03}, + {value: 0x0018, lo: 0x80, hi: 0x8b}, + {value: 0x0040, lo: 0x8c, hi: 0x8f}, + {value: 0x0018, lo: 0x90, hi: 0xbf}, + // Block 0xf0, offset 0x6fa + {value: 0x0000, lo: 0x05}, + {value: 0x0018, lo: 0x80, hi: 0x87}, + {value: 0x0040, lo: 0x88, hi: 0x8f}, + {value: 0x0018, lo: 0x90, hi: 0x99}, + {value: 0x0040, lo: 0x9a, hi: 0x9f}, + {value: 0x0018, lo: 0xa0, hi: 0xbf}, + // Block 0xf1, offset 0x700 + {value: 0x0000, lo: 0x04}, + {value: 0x0018, lo: 0x80, hi: 0x87}, + {value: 0x0040, lo: 0x88, hi: 0x8f}, + {value: 0x0018, lo: 0x90, hi: 0xad}, + {value: 0x0040, lo: 0xae, hi: 0xbf}, + // Block 0xf2, offset 0x705 + {value: 0x0000, lo: 0x09}, + {value: 0x0040, lo: 0x80, hi: 0x8f}, + {value: 0x0018, lo: 0x90, hi: 0x9e}, + {value: 0x0040, lo: 0x9f, hi: 0x9f}, + {value: 0x0018, lo: 0xa0, hi: 0xa7}, + {value: 0x0040, lo: 0xa8, hi: 0xaf}, + {value: 0x0018, lo: 0xb0, hi: 0xb0}, + {value: 0x0040, lo: 0xb1, hi: 0xb2}, + {value: 0x0018, lo: 0xb3, hi: 0xbe}, + {value: 0x0040, lo: 0xbf, hi: 0xbf}, + // Block 0xf3, offset 0x70f + {value: 0x0000, lo: 0x04}, + {value: 0x0018, lo: 0x80, hi: 0x8b}, + {value: 0x0040, lo: 0x8c, hi: 0x8f}, + {value: 0x0018, lo: 0x90, hi: 0x9e}, + {value: 0x0040, lo: 0x9f, hi: 0xbf}, + // Block 0xf4, offset 0x714 + {value: 0x0000, lo: 0x02}, + {value: 0x0018, lo: 0x80, hi: 0x91}, + {value: 0x0040, lo: 0x92, hi: 0xbf}, + // Block 0xf5, offset 0x717 + {value: 0x0000, lo: 0x02}, + {value: 0x0018, lo: 0x80, hi: 0x80}, + {value: 0x0040, lo: 0x81, hi: 0xbf}, + // Block 0xf6, offset 0x71a + {value: 0x0000, lo: 0x02}, + {value: 0x0008, lo: 0x80, hi: 0x96}, + {value: 0x0040, lo: 0x97, hi: 0xbf}, + // Block 0xf7, offset 0x71d + {value: 0x0000, lo: 0x02}, + {value: 0x0008, lo: 0x80, hi: 0xb4}, + {value: 0x0040, lo: 0xb5, hi: 0xbf}, + // Block 0xf8, offset 0x720 + {value: 0x0000, lo: 0x03}, + {value: 0x0008, lo: 0x80, hi: 0x9d}, + {value: 0x0040, lo: 0x9e, hi: 0x9f}, + {value: 0x0008, lo: 0xa0, hi: 0xbf}, + // Block 0xf9, offset 0x724 + {value: 0x0000, lo: 0x02}, + {value: 0x0008, lo: 0x80, hi: 0xa1}, + {value: 0x0040, lo: 0xa2, hi: 0xbf}, + // Block 0xfa, offset 0x727 + {value: 0x0020, lo: 0x0f}, + {value: 0xdeb9, lo: 0x80, hi: 0x89}, + {value: 0x8dfd, lo: 0x8a, hi: 0x8a}, + {value: 0xdff9, lo: 0x8b, hi: 0x9c}, + {value: 0x8e1d, lo: 0x9d, hi: 0x9d}, + {value: 0xe239, lo: 0x9e, hi: 0xa2}, + {value: 0x8e3d, lo: 0xa3, hi: 0xa3}, + {value: 0xe2d9, lo: 0xa4, hi: 0xab}, + {value: 0x7ed5, lo: 0xac, hi: 0xac}, + {value: 0xe3d9, lo: 0xad, hi: 0xaf}, + {value: 0x8e5d, lo: 0xb0, hi: 0xb0}, + {value: 0xe439, lo: 0xb1, hi: 0xb6}, + {value: 0x8e7d, lo: 0xb7, hi: 0xb9}, + {value: 0xe4f9, lo: 0xba, hi: 0xba}, + {value: 0x8edd, lo: 0xbb, hi: 0xbb}, + {value: 0xe519, lo: 0xbc, hi: 0xbf}, + // Block 0xfb, offset 0x737 + {value: 0x0020, lo: 0x10}, + {value: 0x937d, lo: 0x80, hi: 0x80}, + {value: 0xf099, lo: 0x81, hi: 0x86}, + {value: 0x939d, lo: 0x87, hi: 0x8a}, + {value: 0xd9f9, lo: 0x8b, hi: 0x8b}, + {value: 0xf159, lo: 0x8c, hi: 0x96}, + {value: 0x941d, lo: 0x97, hi: 0x97}, + {value: 0xf2b9, lo: 0x98, hi: 0xa3}, + {value: 0x943d, lo: 0xa4, hi: 0xa6}, + {value: 0xf439, lo: 0xa7, hi: 0xaa}, + {value: 0x949d, lo: 0xab, hi: 0xab}, + {value: 0xf4b9, lo: 0xac, hi: 0xac}, + {value: 0x94bd, lo: 0xad, hi: 0xad}, + {value: 0xf4d9, lo: 0xae, hi: 0xaf}, + {value: 0x94dd, lo: 0xb0, hi: 0xb1}, + {value: 0xf519, lo: 0xb2, hi: 0xbe}, + {value: 0x0040, lo: 0xbf, hi: 0xbf}, + // Block 0xfc, offset 0x748 + {value: 0x0000, lo: 0x04}, + {value: 0x0040, lo: 0x80, hi: 0x80}, + {value: 0x0340, lo: 0x81, hi: 0x81}, + {value: 0x0040, lo: 0x82, hi: 0x9f}, + {value: 0x0340, lo: 0xa0, hi: 0xbf}, + // Block 0xfd, offset 0x74d + {value: 0x0000, lo: 0x01}, + {value: 0x0340, lo: 0x80, hi: 0xbf}, + // Block 0xfe, offset 0x74f + {value: 0x0000, lo: 0x01}, + {value: 0x13c0, lo: 0x80, hi: 0xbf}, + // Block 0xff, offset 0x751 + {value: 0x0000, lo: 0x02}, + {value: 0x13c0, lo: 0x80, hi: 0xaf}, + {value: 0x0040, lo: 0xb0, hi: 0xbf}, +} + +// Total table size 41559 bytes (40KiB); checksum: F4A1FA4E diff --git a/libgo/go/golang_org/x/net/idna/trie.go b/libgo/go/golang_org/x/net/idna/trie.go new file mode 100644 index 00000000000..000fb979a0b --- /dev/null +++ b/libgo/go/golang_org/x/net/idna/trie.go @@ -0,0 +1,72 @@ +// Code generated by running "go run gen.go -core" in golang.org/x/text. DO NOT EDIT. + +// Copyright 2016 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package idna + +// appendMapping appends the mapping for the respective rune. isMapped must be +// true. A mapping is a categorization of a rune as defined in UTS #46. +func (c info) appendMapping(b []byte, s string) []byte { + index := int(c >> indexShift) + if c&xorBit == 0 { + s := mappings[index:] + return append(b, s[1:s[0]+1]...) + } + b = append(b, s...) + if c&inlineXOR == inlineXOR { + // TODO: support and handle two-byte inline masks + b[len(b)-1] ^= byte(index) + } else { + for p := len(b) - int(xorData[index]); p < len(b); p++ { + index++ + b[p] ^= xorData[index] + } + } + return b +} + +// Sparse block handling code. + +type valueRange struct { + value uint16 // header: value:stride + lo, hi byte // header: lo:n +} + +type sparseBlocks struct { + values []valueRange + offset []uint16 +} + +var idnaSparse = sparseBlocks{ + values: idnaSparseValues[:], + offset: idnaSparseOffset[:], +} + +// Don't use newIdnaTrie to avoid unconditional linking in of the table. +var trie = &idnaTrie{} + +// lookup determines the type of block n and looks up the value for b. +// For n < t.cutoff, the block is a simple lookup table. Otherwise, the block +// is a list of ranges with an accompanying value. Given a matching range r, +// the value for b is by r.value + (b - r.lo) * stride. +func (t *sparseBlocks) lookup(n uint32, b byte) uint16 { + offset := t.offset[n] + header := t.values[offset] + lo := offset + 1 + hi := lo + uint16(header.lo) + for lo < hi { + m := lo + (hi-lo)/2 + r := t.values[m] + if r.lo <= b && b <= r.hi { + return r.value + uint16(b-r.lo)*header.value + } + if b < r.lo { + hi = m + } else { + lo = m + 1 + } + } + return 0 +} diff --git a/libgo/go/golang_org/x/net/idna/trieval.go b/libgo/go/golang_org/x/net/idna/trieval.go new file mode 100644 index 00000000000..cd88e4d6bda --- /dev/null +++ b/libgo/go/golang_org/x/net/idna/trieval.go @@ -0,0 +1,116 @@ +// Code generated by running "go run gen.go -core" in golang.org/x/text. DO NOT EDIT. + +// Code generated by running "go generate" in golang_org/x/text. DO NOT EDIT. + +package idna + +// This file contains definitions for interpreting the trie value of the idna +// trie generated by "go run gen*.go". It is shared by both the generator +// program and the resultant package. Sharing is achieved by the generator +// copying gen_trieval.go to trieval.go and changing what's above this comment. + +// info holds information from the IDNA mapping table for a single rune. It is +// the value returned by a trie lookup. In most cases, all information fits in +// a 16-bit value. For mappings, this value may contain an index into a slice +// with the mapped string. Such mappings can consist of the actual mapped value +// or an XOR pattern to be applied to the bytes of the UTF8 encoding of the +// input rune. This technique is used by the cases packages and reduces the +// table size significantly. +// +// The per-rune values have the following format: +// +// if mapped { +// if inlinedXOR { +// 15..13 inline XOR marker +// 12..11 unused +// 10..3 inline XOR mask +// } else { +// 15..3 index into xor or mapping table +// } +// } else { +// 15..13 unused +// 12 modifier (including virama) +// 11 virama modifier +// 10..8 joining type +// 7..3 category type +// } +// 2 use xor pattern +// 1..0 mapped category +// +// See the definitions below for a more detailed description of the various +// bits. +type info uint16 + +const ( + catSmallMask = 0x3 + catBigMask = 0xF8 + indexShift = 3 + xorBit = 0x4 // interpret the index as an xor pattern + inlineXOR = 0xE000 // These bits are set if the XOR pattern is inlined. + + joinShift = 8 + joinMask = 0x07 + + viramaModifier = 0x0800 + modifier = 0x1000 +) + +// A category corresponds to a category defined in the IDNA mapping table. +type category uint16 + +const ( + unknown category = 0 // not defined currently in unicode. + mapped category = 1 + disallowedSTD3Mapped category = 2 + deviation category = 3 +) + +const ( + valid category = 0x08 + validNV8 category = 0x18 + validXV8 category = 0x28 + disallowed category = 0x40 + disallowedSTD3Valid category = 0x80 + ignored category = 0xC0 +) + +// join types and additional rune information +const ( + joiningL = (iota + 1) + joiningD + joiningT + joiningR + + //the following types are derived during processing + joinZWJ + joinZWNJ + joinVirama + numJoinTypes +) + +func (c info) isMapped() bool { + return c&0x3 != 0 +} + +func (c info) category() category { + small := c & catSmallMask + if small != 0 { + return category(small) + } + return category(c & catBigMask) +} + +func (c info) joinType() info { + if c.isMapped() { + return 0 + } + return (c >> joinShift) & joinMask +} + +func (c info) isModifier() bool { + return c&(modifier|catSmallMask) == modifier +} + +func (c info) isViramaModifier() bool { + return c&(viramaModifier|catSmallMask) == viramaModifier +} diff --git a/libgo/go/golang_org/x/net/lif/address.go b/libgo/go/golang_org/x/net/lif/address.go index f9b34aed03e..afb957fd8e1 100644 --- a/libgo/go/golang_org/x/net/lif/address.go +++ b/libgo/go/golang_org/x/net/lif/address.go @@ -67,7 +67,7 @@ func Addrs(af int, name string) ([]Addr, error) { continue } sa := (*sockaddrStorage)(unsafe.Pointer(&lifr.Lifru[0])) - l := int(littleEndian.Uint32(lifr.Lifru1[:4])) + l := int(nativeEndian.Uint32(lifr.Lifru1[:4])) if l == 0 { continue } @@ -77,7 +77,7 @@ func Addrs(af int, name string) ([]Addr, error) { copy(a.IP[:], lifr.Lifru[4:8]) as = append(as, a) case sysAF_INET6: - a := &Inet6Addr{PrefixLen: l, ZoneID: int(littleEndian.Uint32(lifr.Lifru[24:28]))} + a := &Inet6Addr{PrefixLen: l, ZoneID: int(nativeEndian.Uint32(lifr.Lifru[24:28]))} copy(a.IP[:], lifr.Lifru[8:24]) as = append(as, a) } diff --git a/libgo/go/golang_org/x/net/lif/address_test.go b/libgo/go/golang_org/x/net/lif/address_test.go index f62ed934713..a25f10b67af 100644 --- a/libgo/go/golang_org/x/net/lif/address_test.go +++ b/libgo/go/golang_org/x/net/lif/address_test.go @@ -78,15 +78,17 @@ type addrPack struct { } func addrPacks() ([]addrPack, error) { + var lastErr error var aps []addrPack for _, af := range [...]int{sysAF_UNSPEC, sysAF_INET, sysAF_INET6} { as, err := Addrs(af, "") if err != nil { - return nil, err + lastErr = err + continue } aps = append(aps, addrPack{af: af, as: as}) } - return aps, nil + return aps, lastErr } func TestAddrs(t *testing.T) { diff --git a/libgo/go/golang_org/x/net/lif/binary.go b/libgo/go/golang_org/x/net/lif/binary.go index aade9eafa25..738a94f4224 100644 --- a/libgo/go/golang_org/x/net/lif/binary.go +++ b/libgo/go/golang_org/x/net/lif/binary.go @@ -12,7 +12,10 @@ package lif // library. Therefore the package set used in the package must be the // same as net package. -var littleEndian binaryLittleEndian +var ( + littleEndian binaryLittleEndian + bigEndian binaryBigEndian +) type binaryByteOrder interface { Uint16([]byte) uint16 @@ -66,3 +69,47 @@ func (binaryLittleEndian) PutUint64(b []byte, v uint64) { b[6] = byte(v >> 48) b[7] = byte(v >> 56) } + +type binaryBigEndian struct{} + +func (binaryBigEndian) Uint16(b []byte) uint16 { + _ = b[1] // bounds check hint to compiler; see golang.org/issue/14808 + return uint16(b[1]) | uint16(b[0])<<8 +} + +func (binaryBigEndian) PutUint16(b []byte, v uint16) { + _ = b[1] // early bounds check to guarantee safety of writes below + b[0] = byte(v >> 8) + b[1] = byte(v) +} + +func (binaryBigEndian) Uint32(b []byte) uint32 { + _ = b[3] // bounds check hint to compiler; see golang.org/issue/14808 + return uint32(b[3]) | uint32(b[2])<<8 | uint32(b[1])<<16 | uint32(b[0])<<24 +} + +func (binaryBigEndian) PutUint32(b []byte, v uint32) { + _ = b[3] // early bounds check to guarantee safety of writes below + b[0] = byte(v >> 24) + b[1] = byte(v >> 16) + b[2] = byte(v >> 8) + b[3] = byte(v) +} + +func (binaryBigEndian) Uint64(b []byte) uint64 { + _ = b[7] // bounds check hint to compiler; see golang.org/issue/14808 + return uint64(b[7]) | uint64(b[6])<<8 | uint64(b[5])<<16 | uint64(b[4])<<24 | + uint64(b[3])<<32 | uint64(b[2])<<40 | uint64(b[1])<<48 | uint64(b[0])<<56 +} + +func (binaryBigEndian) PutUint64(b []byte, v uint64) { + _ = b[7] // early bounds check to guarantee safety of writes below + b[0] = byte(v >> 56) + b[1] = byte(v >> 48) + b[2] = byte(v >> 40) + b[3] = byte(v >> 32) + b[4] = byte(v >> 24) + b[5] = byte(v >> 16) + b[6] = byte(v >> 8) + b[7] = byte(v) +} diff --git a/libgo/go/golang_org/x/net/lif/defs_solaris.go b/libgo/go/golang_org/x/net/lif/defs_solaris.go index 8b84ba5e33f..02c19981d20 100644 --- a/libgo/go/golang_org/x/net/lif/defs_solaris.go +++ b/libgo/go/golang_org/x/net/lif/defs_solaris.go @@ -75,7 +75,7 @@ const ( sizeofLifIfinfoReq = C.sizeof_struct_lif_ifinfo_req ) -type sysLifnum C.struct_lifnum +type lifnum C.struct_lifnum type lifreq C.struct_lifreq diff --git a/libgo/go/golang_org/x/net/lif/link.go b/libgo/go/golang_org/x/net/lif/link.go index 6a77a8f5d80..70e0262781d 100644 --- a/libgo/go/golang_org/x/net/lif/link.go +++ b/libgo/go/golang_org/x/net/lif/link.go @@ -31,15 +31,15 @@ func (ll *Link) fetch(s uintptr) { } ioc := int64(sysSIOCGLIFINDEX) if err := ioctl(s, uintptr(ioc), unsafe.Pointer(&lifr)); err == nil { - ll.Index = int(littleEndian.Uint32(lifr.Lifru[:4])) + ll.Index = int(nativeEndian.Uint32(lifr.Lifru[:4])) } ioc = int64(sysSIOCGLIFFLAGS) if err := ioctl(s, uintptr(ioc), unsafe.Pointer(&lifr)); err == nil { - ll.Flags = int(littleEndian.Uint64(lifr.Lifru[:8])) + ll.Flags = int(nativeEndian.Uint64(lifr.Lifru[:8])) } ioc = int64(sysSIOCGLIFMTU) if err := ioctl(s, uintptr(ioc), unsafe.Pointer(&lifr)); err == nil { - ll.MTU = int(littleEndian.Uint32(lifr.Lifru[:4])) + ll.MTU = int(nativeEndian.Uint32(lifr.Lifru[:4])) } switch ll.Type { case sysIFT_IPV4, sysIFT_IPV6, sysIFT_6TO4: @@ -70,7 +70,7 @@ func Links(af int, name string) ([]Link, error) { func links(eps []endpoint, name string) ([]Link, error) { var lls []Link - lifn := sysLifnum{Flags: sysLIFC_NOXMIT | sysLIFC_TEMPORARY | sysLIFC_ALLZONES | sysLIFC_UNDER_IPMP} + lifn := lifnum{Flags: sysLIFC_NOXMIT | sysLIFC_TEMPORARY | sysLIFC_ALLZONES | sysLIFC_UNDER_IPMP} lifc := lifconf{Flags: sysLIFC_NOXMIT | sysLIFC_TEMPORARY | sysLIFC_ALLZONES | sysLIFC_UNDER_IPMP} for _, ep := range eps { lifn.Family = uint16(ep.af) diff --git a/libgo/go/golang_org/x/net/lif/link_test.go b/libgo/go/golang_org/x/net/lif/link_test.go index 8fb2bf6f345..0cb9b95c699 100644 --- a/libgo/go/golang_org/x/net/lif/link_test.go +++ b/libgo/go/golang_org/x/net/lif/link_test.go @@ -21,15 +21,17 @@ type linkPack struct { } func linkPacks() ([]linkPack, error) { + var lastErr error var lps []linkPack for _, af := range [...]int{sysAF_UNSPEC, sysAF_INET, sysAF_INET6} { lls, err := Links(af, "") if err != nil { - return nil, err + lastErr = err + continue } lps = append(lps, linkPack{af: af, lls: lls}) } - return lps, nil + return lps, lastErr } func TestLinks(t *testing.T) { diff --git a/libgo/go/golang_org/x/net/lif/sys.go b/libgo/go/golang_org/x/net/lif/sys.go new file mode 100644 index 00000000000..c896041b7b4 --- /dev/null +++ b/libgo/go/golang_org/x/net/lif/sys.go @@ -0,0 +1,21 @@ +// Copyright 2017 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build solaris + +package lif + +import "unsafe" + +var nativeEndian binaryByteOrder + +func init() { + i := uint32(1) + b := (*[4]byte)(unsafe.Pointer(&i)) + if b[0] == 1 { + nativeEndian = littleEndian + } else { + nativeEndian = bigEndian + } +} diff --git a/libgo/go/golang_org/x/net/nettest/conntest.go b/libgo/go/golang_org/x/net/nettest/conntest.go new file mode 100644 index 00000000000..5bd3a8c68c5 --- /dev/null +++ b/libgo/go/golang_org/x/net/nettest/conntest.go @@ -0,0 +1,456 @@ +// Copyright 2016 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package nettest provides utilities for network testing. +package nettest + +import ( + "bytes" + "encoding/binary" + "io" + "io/ioutil" + "math/rand" + "net" + "runtime" + "sync" + "testing" + "time" +) + +var ( + aLongTimeAgo = time.Unix(233431200, 0) + neverTimeout = time.Time{} +) + +// MakePipe creates a connection between two endpoints and returns the pair +// as c1 and c2, such that anything written to c1 is read by c2 and vice-versa. +// The stop function closes all resources, including c1, c2, and the underlying +// net.Listener (if there is one), and should not be nil. +type MakePipe func() (c1, c2 net.Conn, stop func(), err error) + +// TestConn tests that a net.Conn implementation properly satisfies the interface. +// The tests should not produce any false positives, but may experience +// false negatives. Thus, some issues may only be detected when the test is +// run multiple times. For maximal effectiveness, run the tests under the +// race detector. +func TestConn(t *testing.T, mp MakePipe) { + testConn(t, mp) +} + +type connTester func(t *testing.T, c1, c2 net.Conn) + +func timeoutWrapper(t *testing.T, mp MakePipe, f connTester) { + c1, c2, stop, err := mp() + if err != nil { + t.Fatalf("unable to make pipe: %v", err) + } + var once sync.Once + defer once.Do(func() { stop() }) + timer := time.AfterFunc(time.Minute, func() { + once.Do(func() { + t.Error("test timed out; terminating pipe") + stop() + }) + }) + defer timer.Stop() + f(t, c1, c2) +} + +// testBasicIO tests that the data sent on c1 is properly received on c2. +func testBasicIO(t *testing.T, c1, c2 net.Conn) { + want := make([]byte, 1<<20) + rand.New(rand.NewSource(0)).Read(want) + + dataCh := make(chan []byte) + go func() { + rd := bytes.NewReader(want) + if err := chunkedCopy(c1, rd); err != nil { + t.Errorf("unexpected c1.Write error: %v", err) + } + if err := c1.Close(); err != nil { + t.Errorf("unexpected c1.Close error: %v", err) + } + }() + + go func() { + wr := new(bytes.Buffer) + if err := chunkedCopy(wr, c2); err != nil { + t.Errorf("unexpected c2.Read error: %v", err) + } + if err := c2.Close(); err != nil { + t.Errorf("unexpected c2.Close error: %v", err) + } + dataCh <- wr.Bytes() + }() + + if got := <-dataCh; !bytes.Equal(got, want) { + t.Errorf("transmitted data differs") + } +} + +// testPingPong tests that the two endpoints can synchronously send data to +// each other in a typical request-response pattern. +func testPingPong(t *testing.T, c1, c2 net.Conn) { + var wg sync.WaitGroup + defer wg.Wait() + + pingPonger := func(c net.Conn) { + defer wg.Done() + buf := make([]byte, 8) + var prev uint64 + for { + if _, err := io.ReadFull(c, buf); err != nil { + if err == io.EOF { + break + } + t.Errorf("unexpected Read error: %v", err) + } + + v := binary.LittleEndian.Uint64(buf) + binary.LittleEndian.PutUint64(buf, v+1) + if prev != 0 && prev+2 != v { + t.Errorf("mismatching value: got %d, want %d", v, prev+2) + } + prev = v + if v == 1000 { + break + } + + if _, err := c.Write(buf); err != nil { + t.Errorf("unexpected Write error: %v", err) + break + } + } + if err := c.Close(); err != nil { + t.Errorf("unexpected Close error: %v", err) + } + } + + wg.Add(2) + go pingPonger(c1) + go pingPonger(c2) + + // Start off the chain reaction. + if _, err := c1.Write(make([]byte, 8)); err != nil { + t.Errorf("unexpected c1.Write error: %v", err) + } +} + +// testRacyRead tests that it is safe to mutate the input Read buffer +// immediately after cancelation has occurred. +func testRacyRead(t *testing.T, c1, c2 net.Conn) { + go chunkedCopy(c2, rand.New(rand.NewSource(0))) + + var wg sync.WaitGroup + defer wg.Wait() + + c1.SetReadDeadline(time.Now().Add(time.Millisecond)) + for i := 0; i < 10; i++ { + wg.Add(1) + go func() { + defer wg.Done() + + b1 := make([]byte, 1024) + b2 := make([]byte, 1024) + for j := 0; j < 100; j++ { + _, err := c1.Read(b1) + copy(b1, b2) // Mutate b1 to trigger potential race + if err != nil { + checkForTimeoutError(t, err) + c1.SetReadDeadline(time.Now().Add(time.Millisecond)) + } + } + }() + } +} + +// testRacyWrite tests that it is safe to mutate the input Write buffer +// immediately after cancelation has occurred. +func testRacyWrite(t *testing.T, c1, c2 net.Conn) { + go chunkedCopy(ioutil.Discard, c2) + + var wg sync.WaitGroup + defer wg.Wait() + + c1.SetWriteDeadline(time.Now().Add(time.Millisecond)) + for i := 0; i < 10; i++ { + wg.Add(1) + go func() { + defer wg.Done() + + b1 := make([]byte, 1024) + b2 := make([]byte, 1024) + for j := 0; j < 100; j++ { + _, err := c1.Write(b1) + copy(b1, b2) // Mutate b1 to trigger potential race + if err != nil { + checkForTimeoutError(t, err) + c1.SetWriteDeadline(time.Now().Add(time.Millisecond)) + } + } + }() + } +} + +// testReadTimeout tests that Read timeouts do not affect Write. +func testReadTimeout(t *testing.T, c1, c2 net.Conn) { + go chunkedCopy(ioutil.Discard, c2) + + c1.SetReadDeadline(aLongTimeAgo) + _, err := c1.Read(make([]byte, 1024)) + checkForTimeoutError(t, err) + if _, err := c1.Write(make([]byte, 1024)); err != nil { + t.Errorf("unexpected Write error: %v", err) + } +} + +// testWriteTimeout tests that Write timeouts do not affect Read. +func testWriteTimeout(t *testing.T, c1, c2 net.Conn) { + go chunkedCopy(c2, rand.New(rand.NewSource(0))) + + c1.SetWriteDeadline(aLongTimeAgo) + _, err := c1.Write(make([]byte, 1024)) + checkForTimeoutError(t, err) + if _, err := c1.Read(make([]byte, 1024)); err != nil { + t.Errorf("unexpected Read error: %v", err) + } +} + +// testPastTimeout tests that a deadline set in the past immediately times out +// Read and Write requests. +func testPastTimeout(t *testing.T, c1, c2 net.Conn) { + go chunkedCopy(c2, c2) + + testRoundtrip(t, c1) + + c1.SetDeadline(aLongTimeAgo) + n, err := c1.Write(make([]byte, 1024)) + if n != 0 { + t.Errorf("unexpected Write count: got %d, want 0", n) + } + checkForTimeoutError(t, err) + n, err = c1.Read(make([]byte, 1024)) + if n != 0 { + t.Errorf("unexpected Read count: got %d, want 0", n) + } + checkForTimeoutError(t, err) + + testRoundtrip(t, c1) +} + +// testPresentTimeout tests that a deadline set while there are pending +// Read and Write operations immediately times out those operations. +func testPresentTimeout(t *testing.T, c1, c2 net.Conn) { + var wg sync.WaitGroup + defer wg.Wait() + wg.Add(3) + + deadlineSet := make(chan bool, 1) + go func() { + defer wg.Done() + time.Sleep(100 * time.Millisecond) + deadlineSet <- true + c1.SetReadDeadline(aLongTimeAgo) + c1.SetWriteDeadline(aLongTimeAgo) + }() + go func() { + defer wg.Done() + n, err := c1.Read(make([]byte, 1024)) + if n != 0 { + t.Errorf("unexpected Read count: got %d, want 0", n) + } + checkForTimeoutError(t, err) + if len(deadlineSet) == 0 { + t.Error("Read timed out before deadline is set") + } + }() + go func() { + defer wg.Done() + var err error + for err == nil { + _, err = c1.Write(make([]byte, 1024)) + } + checkForTimeoutError(t, err) + if len(deadlineSet) == 0 { + t.Error("Write timed out before deadline is set") + } + }() +} + +// testFutureTimeout tests that a future deadline will eventually time out +// Read and Write operations. +func testFutureTimeout(t *testing.T, c1, c2 net.Conn) { + var wg sync.WaitGroup + wg.Add(2) + + c1.SetDeadline(time.Now().Add(100 * time.Millisecond)) + go func() { + defer wg.Done() + _, err := c1.Read(make([]byte, 1024)) + checkForTimeoutError(t, err) + }() + go func() { + defer wg.Done() + var err error + for err == nil { + _, err = c1.Write(make([]byte, 1024)) + } + checkForTimeoutError(t, err) + }() + wg.Wait() + + go chunkedCopy(c2, c2) + resyncConn(t, c1) + testRoundtrip(t, c1) +} + +// testCloseTimeout tests that calling Close immediately times out pending +// Read and Write operations. +func testCloseTimeout(t *testing.T, c1, c2 net.Conn) { + go chunkedCopy(c2, c2) + + var wg sync.WaitGroup + defer wg.Wait() + wg.Add(3) + + // Test for cancelation upon connection closure. + c1.SetDeadline(neverTimeout) + go func() { + defer wg.Done() + time.Sleep(100 * time.Millisecond) + c1.Close() + }() + go func() { + defer wg.Done() + var err error + buf := make([]byte, 1024) + for err == nil { + _, err = c1.Read(buf) + } + }() + go func() { + defer wg.Done() + var err error + buf := make([]byte, 1024) + for err == nil { + _, err = c1.Write(buf) + } + }() +} + +// testConcurrentMethods tests that the methods of net.Conn can safely +// be called concurrently. +func testConcurrentMethods(t *testing.T, c1, c2 net.Conn) { + if runtime.GOOS == "plan9" { + t.Skip("skipping on plan9; see https://golang.org/issue/20489") + } + go chunkedCopy(c2, c2) + + // The results of the calls may be nonsensical, but this should + // not trigger a race detector warning. + var wg sync.WaitGroup + for i := 0; i < 100; i++ { + wg.Add(7) + go func() { + defer wg.Done() + c1.Read(make([]byte, 1024)) + }() + go func() { + defer wg.Done() + c1.Write(make([]byte, 1024)) + }() + go func() { + defer wg.Done() + c1.SetDeadline(time.Now().Add(10 * time.Millisecond)) + }() + go func() { + defer wg.Done() + c1.SetReadDeadline(aLongTimeAgo) + }() + go func() { + defer wg.Done() + c1.SetWriteDeadline(aLongTimeAgo) + }() + go func() { + defer wg.Done() + c1.LocalAddr() + }() + go func() { + defer wg.Done() + c1.RemoteAddr() + }() + } + wg.Wait() // At worst, the deadline is set 10ms into the future + + resyncConn(t, c1) + testRoundtrip(t, c1) +} + +// checkForTimeoutError checks that the error satisfies the Error interface +// and that Timeout returns true. +func checkForTimeoutError(t *testing.T, err error) { + if nerr, ok := err.(net.Error); ok { + if !nerr.Timeout() { + t.Errorf("err.Timeout() = false, want true") + } + } else { + t.Errorf("got %T, want net.Error", err) + } +} + +// testRoundtrip writes something into c and reads it back. +// It assumes that everything written into c is echoed back to itself. +func testRoundtrip(t *testing.T, c net.Conn) { + if err := c.SetDeadline(neverTimeout); err != nil { + t.Errorf("roundtrip SetDeadline error: %v", err) + } + + const s = "Hello, world!" + buf := []byte(s) + if _, err := c.Write(buf); err != nil { + t.Errorf("roundtrip Write error: %v", err) + } + if _, err := io.ReadFull(c, buf); err != nil { + t.Errorf("roundtrip Read error: %v", err) + } + if string(buf) != s { + t.Errorf("roundtrip data mismatch: got %q, want %q", buf, s) + } +} + +// resyncConn resynchronizes the connection into a sane state. +// It assumes that everything written into c is echoed back to itself. +// It assumes that 0xff is not currently on the wire or in the read buffer. +func resyncConn(t *testing.T, c net.Conn) { + c.SetDeadline(neverTimeout) + errCh := make(chan error) + go func() { + _, err := c.Write([]byte{0xff}) + errCh <- err + }() + buf := make([]byte, 1024) + for { + n, err := c.Read(buf) + if n > 0 && bytes.IndexByte(buf[:n], 0xff) == n-1 { + break + } + if err != nil { + t.Errorf("unexpected Read error: %v", err) + break + } + } + if err := <-errCh; err != nil { + t.Errorf("unexpected Write error: %v", err) + } +} + +// chunkedCopy copies from r to w in fixed-width chunks to avoid +// causing a Write that exceeds the maximum packet size for packet-based +// connections like "unixpacket". +// We assume that the maximum packet size is at least 1024. +func chunkedCopy(w io.Writer, r io.Reader) error { + b := make([]byte, 1024) + _, err := io.CopyBuffer(struct{ io.Writer }{w}, struct{ io.Reader }{r}, b) + return err +} diff --git a/libgo/go/golang_org/x/net/nettest/conntest_go16.go b/libgo/go/golang_org/x/net/nettest/conntest_go16.go new file mode 100644 index 00000000000..4cbf48e35e1 --- /dev/null +++ b/libgo/go/golang_org/x/net/nettest/conntest_go16.go @@ -0,0 +1,24 @@ +// Copyright 2016 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build !go1.7 + +package nettest + +import "testing" + +func testConn(t *testing.T, mp MakePipe) { + // Avoid using subtests on Go 1.6 and below. + timeoutWrapper(t, mp, testBasicIO) + timeoutWrapper(t, mp, testPingPong) + timeoutWrapper(t, mp, testRacyRead) + timeoutWrapper(t, mp, testRacyWrite) + timeoutWrapper(t, mp, testReadTimeout) + timeoutWrapper(t, mp, testWriteTimeout) + timeoutWrapper(t, mp, testPastTimeout) + timeoutWrapper(t, mp, testPresentTimeout) + timeoutWrapper(t, mp, testFutureTimeout) + timeoutWrapper(t, mp, testCloseTimeout) + timeoutWrapper(t, mp, testConcurrentMethods) +} diff --git a/libgo/go/golang_org/x/net/nettest/conntest_go17.go b/libgo/go/golang_org/x/net/nettest/conntest_go17.go new file mode 100644 index 00000000000..fa039f03fc9 --- /dev/null +++ b/libgo/go/golang_org/x/net/nettest/conntest_go17.go @@ -0,0 +1,24 @@ +// Copyright 2016 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build go1.7 + +package nettest + +import "testing" + +func testConn(t *testing.T, mp MakePipe) { + // Use subtests on Go 1.7 and above since it is better organized. + t.Run("BasicIO", func(t *testing.T) { timeoutWrapper(t, mp, testBasicIO) }) + t.Run("PingPong", func(t *testing.T) { timeoutWrapper(t, mp, testPingPong) }) + t.Run("RacyRead", func(t *testing.T) { timeoutWrapper(t, mp, testRacyRead) }) + t.Run("RacyWrite", func(t *testing.T) { timeoutWrapper(t, mp, testRacyWrite) }) + t.Run("ReadTimeout", func(t *testing.T) { timeoutWrapper(t, mp, testReadTimeout) }) + t.Run("WriteTimeout", func(t *testing.T) { timeoutWrapper(t, mp, testWriteTimeout) }) + t.Run("PastTimeout", func(t *testing.T) { timeoutWrapper(t, mp, testPastTimeout) }) + t.Run("PresentTimeout", func(t *testing.T) { timeoutWrapper(t, mp, testPresentTimeout) }) + t.Run("FutureTimeout", func(t *testing.T) { timeoutWrapper(t, mp, testFutureTimeout) }) + t.Run("CloseTimeout", func(t *testing.T) { timeoutWrapper(t, mp, testCloseTimeout) }) + t.Run("ConcurrentMethods", func(t *testing.T) { timeoutWrapper(t, mp, testConcurrentMethods) }) +} diff --git a/libgo/go/golang_org/x/net/nettest/conntest_test.go b/libgo/go/golang_org/x/net/nettest/conntest_test.go new file mode 100644 index 00000000000..23bd69fc630 --- /dev/null +++ b/libgo/go/golang_org/x/net/nettest/conntest_test.go @@ -0,0 +1,126 @@ +// Copyright 2016 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build go1.8 + +package nettest + +import ( + "fmt" + "io/ioutil" + "net" + "os" + "runtime" + "testing" +) + +// testUnixAddr uses ioutil.TempFile to get a name that is unique. +// It also uses /tmp directory in case it is prohibited to create UNIX +// sockets in TMPDIR. +func testUnixAddr() string { + f, err := ioutil.TempFile("", "go-nettest") + if err != nil { + panic(err) + } + addr := f.Name() + f.Close() + os.Remove(addr) + return addr +} + +// testableNetwork reports whether network is testable on the current +// platform configuration. +// This is based on logic from standard library's net/platform_test.go. +func testableNetwork(network string) bool { + switch network { + case "unix": + switch runtime.GOOS { + case "android", "nacl", "plan9", "windows": + return false + } + if runtime.GOOS == "darwin" && (runtime.GOARCH == "arm" || runtime.GOARCH == "arm64") { + return false + } + case "unixpacket": + switch runtime.GOOS { + case "android", "darwin", "nacl", "plan9", "windows", "freebsd": + return false + } + } + return true +} + +func newLocalListener(network string) (net.Listener, error) { + switch network { + case "tcp": + ln, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + ln, err = net.Listen("tcp6", "[::1]:0") + } + return ln, err + case "unix", "unixpacket": + return net.Listen(network, testUnixAddr()) + } + return nil, fmt.Errorf("%s is not supported", network) +} + +func TestTestConn(t *testing.T) { + tests := []struct{ name, network string }{ + {"TCP", "tcp"}, + {"UnixPipe", "unix"}, + {"UnixPacketPipe", "unixpacket"}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if !testableNetwork(tt.network) { + t.Skipf("not supported on %s", runtime.GOOS) + } + + mp := func() (c1, c2 net.Conn, stop func(), err error) { + ln, err := newLocalListener(tt.network) + if err != nil { + return nil, nil, nil, err + } + + // Start a connection between two endpoints. + var err1, err2 error + done := make(chan bool) + go func() { + c2, err2 = ln.Accept() + close(done) + }() + c1, err1 = net.Dial(ln.Addr().Network(), ln.Addr().String()) + <-done + + stop = func() { + if err1 == nil { + c1.Close() + } + if err2 == nil { + c2.Close() + } + ln.Close() + switch tt.network { + case "unix", "unixpacket": + os.Remove(ln.Addr().String()) + } + } + + switch { + case err1 != nil: + stop() + return nil, nil, nil, err1 + case err2 != nil: + stop() + return nil, nil, nil, err2 + default: + return c1, c2, stop, nil + } + } + + TestConn(t, mp) + }) + } +} diff --git a/libgo/go/golang_org/x/net/proxy/direct.go b/libgo/go/golang_org/x/net/proxy/direct.go new file mode 100644 index 00000000000..4c5ad88b1e7 --- /dev/null +++ b/libgo/go/golang_org/x/net/proxy/direct.go @@ -0,0 +1,18 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package proxy + +import ( + "net" +) + +type direct struct{} + +// Direct is a direct proxy: one that makes network connections directly. +var Direct = direct{} + +func (direct) Dial(network, addr string) (net.Conn, error) { + return net.Dial(network, addr) +} diff --git a/libgo/go/golang_org/x/net/proxy/per_host.go b/libgo/go/golang_org/x/net/proxy/per_host.go new file mode 100644 index 00000000000..f540b196f7d --- /dev/null +++ b/libgo/go/golang_org/x/net/proxy/per_host.go @@ -0,0 +1,140 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package proxy + +import ( + "net" + "strings" +) + +// A PerHost directs connections to a default Dialer unless the hostname +// requested matches one of a number of exceptions. +type PerHost struct { + def, bypass Dialer + + bypassNetworks []*net.IPNet + bypassIPs []net.IP + bypassZones []string + bypassHosts []string +} + +// NewPerHost returns a PerHost Dialer that directs connections to either +// defaultDialer or bypass, depending on whether the connection matches one of +// the configured rules. +func NewPerHost(defaultDialer, bypass Dialer) *PerHost { + return &PerHost{ + def: defaultDialer, + bypass: bypass, + } +} + +// Dial connects to the address addr on the given network through either +// defaultDialer or bypass. +func (p *PerHost) Dial(network, addr string) (c net.Conn, err error) { + host, _, err := net.SplitHostPort(addr) + if err != nil { + return nil, err + } + + return p.dialerForRequest(host).Dial(network, addr) +} + +func (p *PerHost) dialerForRequest(host string) Dialer { + if ip := net.ParseIP(host); ip != nil { + for _, net := range p.bypassNetworks { + if net.Contains(ip) { + return p.bypass + } + } + for _, bypassIP := range p.bypassIPs { + if bypassIP.Equal(ip) { + return p.bypass + } + } + return p.def + } + + for _, zone := range p.bypassZones { + if strings.HasSuffix(host, zone) { + return p.bypass + } + if host == zone[1:] { + // For a zone "example.com", we match "example.com" + // too. + return p.bypass + } + } + for _, bypassHost := range p.bypassHosts { + if bypassHost == host { + return p.bypass + } + } + return p.def +} + +// AddFromString parses a string that contains comma-separated values +// specifying hosts that should use the bypass proxy. Each value is either an +// IP address, a CIDR range, a zone (*.example.com) or a hostname +// (localhost). A best effort is made to parse the string and errors are +// ignored. +func (p *PerHost) AddFromString(s string) { + hosts := strings.Split(s, ",") + for _, host := range hosts { + host = strings.TrimSpace(host) + if len(host) == 0 { + continue + } + if strings.Contains(host, "/") { + // We assume that it's a CIDR address like 127.0.0.0/8 + if _, net, err := net.ParseCIDR(host); err == nil { + p.AddNetwork(net) + } + continue + } + if ip := net.ParseIP(host); ip != nil { + p.AddIP(ip) + continue + } + if strings.HasPrefix(host, "*.") { + p.AddZone(host[1:]) + continue + } + p.AddHost(host) + } +} + +// AddIP specifies an IP address that will use the bypass proxy. Note that +// this will only take effect if a literal IP address is dialed. A connection +// to a named host will never match an IP. +func (p *PerHost) AddIP(ip net.IP) { + p.bypassIPs = append(p.bypassIPs, ip) +} + +// AddNetwork specifies an IP range that will use the bypass proxy. Note that +// this will only take effect if a literal IP address is dialed. A connection +// to a named host will never match. +func (p *PerHost) AddNetwork(net *net.IPNet) { + p.bypassNetworks = append(p.bypassNetworks, net) +} + +// AddZone specifies a DNS suffix that will use the bypass proxy. A zone of +// "example.com" matches "example.com" and all of its subdomains. +func (p *PerHost) AddZone(zone string) { + if strings.HasSuffix(zone, ".") { + zone = zone[:len(zone)-1] + } + if !strings.HasPrefix(zone, ".") { + zone = "." + zone + } + p.bypassZones = append(p.bypassZones, zone) +} + +// AddHost specifies a hostname that will use the bypass proxy. +func (p *PerHost) AddHost(host string) { + if strings.HasSuffix(host, ".") { + host = host[:len(host)-1] + } + p.bypassHosts = append(p.bypassHosts, host) +} diff --git a/libgo/go/golang_org/x/net/proxy/per_host_test.go b/libgo/go/golang_org/x/net/proxy/per_host_test.go new file mode 100644 index 00000000000..a7d80957113 --- /dev/null +++ b/libgo/go/golang_org/x/net/proxy/per_host_test.go @@ -0,0 +1,55 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package proxy + +import ( + "errors" + "net" + "reflect" + "testing" +) + +type recordingProxy struct { + addrs []string +} + +func (r *recordingProxy) Dial(network, addr string) (net.Conn, error) { + r.addrs = append(r.addrs, addr) + return nil, errors.New("recordingProxy") +} + +func TestPerHost(t *testing.T) { + var def, bypass recordingProxy + perHost := NewPerHost(&def, &bypass) + perHost.AddFromString("localhost,*.zone,127.0.0.1,10.0.0.1/8,1000::/16") + + expectedDef := []string{ + "example.com:123", + "1.2.3.4:123", + "[1001::]:123", + } + expectedBypass := []string{ + "localhost:123", + "zone:123", + "foo.zone:123", + "127.0.0.1:123", + "10.1.2.3:123", + "[1000::]:123", + } + + for _, addr := range expectedDef { + perHost.Dial("tcp", addr) + } + for _, addr := range expectedBypass { + perHost.Dial("tcp", addr) + } + + if !reflect.DeepEqual(expectedDef, def.addrs) { + t.Errorf("Hosts which went to the default proxy didn't match. Got %v, want %v", def.addrs, expectedDef) + } + if !reflect.DeepEqual(expectedBypass, bypass.addrs) { + t.Errorf("Hosts which went to the bypass proxy didn't match. Got %v, want %v", bypass.addrs, expectedBypass) + } +} diff --git a/libgo/go/golang_org/x/net/proxy/proxy.go b/libgo/go/golang_org/x/net/proxy/proxy.go new file mode 100644 index 00000000000..78a8b7bee97 --- /dev/null +++ b/libgo/go/golang_org/x/net/proxy/proxy.go @@ -0,0 +1,94 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package proxy provides support for a variety of protocols to proxy network +// data. +package proxy // import "golang.org/x/net/proxy" + +import ( + "errors" + "net" + "net/url" + "os" +) + +// A Dialer is a means to establish a connection. +type Dialer interface { + // Dial connects to the given address via the proxy. + Dial(network, addr string) (c net.Conn, err error) +} + +// Auth contains authentication parameters that specific Dialers may require. +type Auth struct { + User, Password string +} + +// FromEnvironment returns the dialer specified by the proxy related variables in +// the environment. +func FromEnvironment() Dialer { + allProxy := os.Getenv("all_proxy") + if len(allProxy) == 0 { + return Direct + } + + proxyURL, err := url.Parse(allProxy) + if err != nil { + return Direct + } + proxy, err := FromURL(proxyURL, Direct) + if err != nil { + return Direct + } + + noProxy := os.Getenv("no_proxy") + if len(noProxy) == 0 { + return proxy + } + + perHost := NewPerHost(proxy, Direct) + perHost.AddFromString(noProxy) + return perHost +} + +// proxySchemes is a map from URL schemes to a function that creates a Dialer +// from a URL with such a scheme. +var proxySchemes map[string]func(*url.URL, Dialer) (Dialer, error) + +// RegisterDialerType takes a URL scheme and a function to generate Dialers from +// a URL with that scheme and a forwarding Dialer. Registered schemes are used +// by FromURL. +func RegisterDialerType(scheme string, f func(*url.URL, Dialer) (Dialer, error)) { + if proxySchemes == nil { + proxySchemes = make(map[string]func(*url.URL, Dialer) (Dialer, error)) + } + proxySchemes[scheme] = f +} + +// FromURL returns a Dialer given a URL specification and an underlying +// Dialer for it to make network requests. +func FromURL(u *url.URL, forward Dialer) (Dialer, error) { + var auth *Auth + if u.User != nil { + auth = new(Auth) + auth.User = u.User.Username() + if p, ok := u.User.Password(); ok { + auth.Password = p + } + } + + switch u.Scheme { + case "socks5": + return SOCKS5("tcp", u.Host, auth, forward) + } + + // If the scheme doesn't match any of the built-in schemes, see if it + // was registered by another package. + if proxySchemes != nil { + if f, ok := proxySchemes[u.Scheme]; ok { + return f(u, forward) + } + } + + return nil, errors.New("proxy: unknown scheme: " + u.Scheme) +} diff --git a/libgo/go/golang_org/x/net/proxy/proxy_test.go b/libgo/go/golang_org/x/net/proxy/proxy_test.go new file mode 100644 index 00000000000..c19a5c06353 --- /dev/null +++ b/libgo/go/golang_org/x/net/proxy/proxy_test.go @@ -0,0 +1,142 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package proxy + +import ( + "io" + "net" + "net/url" + "strconv" + "sync" + "testing" +) + +func TestFromURL(t *testing.T) { + endSystem, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatalf("net.Listen failed: %v", err) + } + defer endSystem.Close() + gateway, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatalf("net.Listen failed: %v", err) + } + defer gateway.Close() + + var wg sync.WaitGroup + wg.Add(1) + go socks5Gateway(t, gateway, endSystem, socks5Domain, &wg) + + url, err := url.Parse("socks5://user:password@" + gateway.Addr().String()) + if err != nil { + t.Fatalf("url.Parse failed: %v", err) + } + proxy, err := FromURL(url, Direct) + if err != nil { + t.Fatalf("FromURL failed: %v", err) + } + _, port, err := net.SplitHostPort(endSystem.Addr().String()) + if err != nil { + t.Fatalf("net.SplitHostPort failed: %v", err) + } + if c, err := proxy.Dial("tcp", "localhost:"+port); err != nil { + t.Fatalf("FromURL.Dial failed: %v", err) + } else { + c.Close() + } + + wg.Wait() +} + +func TestSOCKS5(t *testing.T) { + endSystem, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatalf("net.Listen failed: %v", err) + } + defer endSystem.Close() + gateway, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatalf("net.Listen failed: %v", err) + } + defer gateway.Close() + + var wg sync.WaitGroup + wg.Add(1) + go socks5Gateway(t, gateway, endSystem, socks5IP4, &wg) + + proxy, err := SOCKS5("tcp", gateway.Addr().String(), nil, Direct) + if err != nil { + t.Fatalf("SOCKS5 failed: %v", err) + } + if c, err := proxy.Dial("tcp", endSystem.Addr().String()); err != nil { + t.Fatalf("SOCKS5.Dial failed: %v", err) + } else { + c.Close() + } + + wg.Wait() +} + +func socks5Gateway(t *testing.T, gateway, endSystem net.Listener, typ byte, wg *sync.WaitGroup) { + defer wg.Done() + + c, err := gateway.Accept() + if err != nil { + t.Errorf("net.Listener.Accept failed: %v", err) + return + } + defer c.Close() + + b := make([]byte, 32) + var n int + if typ == socks5Domain { + n = 4 + } else { + n = 3 + } + if _, err := io.ReadFull(c, b[:n]); err != nil { + t.Errorf("io.ReadFull failed: %v", err) + return + } + if _, err := c.Write([]byte{socks5Version, socks5AuthNone}); err != nil { + t.Errorf("net.Conn.Write failed: %v", err) + return + } + if typ == socks5Domain { + n = 16 + } else { + n = 10 + } + if _, err := io.ReadFull(c, b[:n]); err != nil { + t.Errorf("io.ReadFull failed: %v", err) + return + } + if b[0] != socks5Version || b[1] != socks5Connect || b[2] != 0x00 || b[3] != typ { + t.Errorf("got an unexpected packet: %#02x %#02x %#02x %#02x", b[0], b[1], b[2], b[3]) + return + } + if typ == socks5Domain { + copy(b[:5], []byte{socks5Version, 0x00, 0x00, socks5Domain, 9}) + b = append(b, []byte("localhost")...) + } else { + copy(b[:4], []byte{socks5Version, 0x00, 0x00, socks5IP4}) + } + host, port, err := net.SplitHostPort(endSystem.Addr().String()) + if err != nil { + t.Errorf("net.SplitHostPort failed: %v", err) + return + } + b = append(b, []byte(net.ParseIP(host).To4())...) + p, err := strconv.Atoi(port) + if err != nil { + t.Errorf("strconv.Atoi failed: %v", err) + return + } + b = append(b, []byte{byte(p >> 8), byte(p)}...) + if _, err := c.Write(b); err != nil { + t.Errorf("net.Conn.Write failed: %v", err) + return + } +} diff --git a/libgo/go/golang_org/x/net/proxy/socks5.go b/libgo/go/golang_org/x/net/proxy/socks5.go new file mode 100644 index 00000000000..973f57f1970 --- /dev/null +++ b/libgo/go/golang_org/x/net/proxy/socks5.go @@ -0,0 +1,213 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package proxy + +import ( + "errors" + "io" + "net" + "strconv" +) + +// SOCKS5 returns a Dialer that makes SOCKSv5 connections to the given address +// with an optional username and password. See RFC 1928. +func SOCKS5(network, addr string, auth *Auth, forward Dialer) (Dialer, error) { + s := &socks5{ + network: network, + addr: addr, + forward: forward, + } + if auth != nil { + s.user = auth.User + s.password = auth.Password + } + + return s, nil +} + +type socks5 struct { + user, password string + network, addr string + forward Dialer +} + +const socks5Version = 5 + +const ( + socks5AuthNone = 0 + socks5AuthPassword = 2 +) + +const socks5Connect = 1 + +const ( + socks5IP4 = 1 + socks5Domain = 3 + socks5IP6 = 4 +) + +var socks5Errors = []string{ + "", + "general failure", + "connection forbidden", + "network unreachable", + "host unreachable", + "connection refused", + "TTL expired", + "command not supported", + "address type not supported", +} + +// Dial connects to the address addr on the network net via the SOCKS5 proxy. +func (s *socks5) Dial(network, addr string) (net.Conn, error) { + switch network { + case "tcp", "tcp6", "tcp4": + default: + return nil, errors.New("proxy: no support for SOCKS5 proxy connections of type " + network) + } + + conn, err := s.forward.Dial(s.network, s.addr) + if err != nil { + return nil, err + } + if err := s.connect(conn, addr); err != nil { + conn.Close() + return nil, err + } + return conn, nil +} + +// connect takes an existing connection to a socks5 proxy server, +// and commands the server to extend that connection to target, +// which must be a canonical address with a host and port. +func (s *socks5) connect(conn net.Conn, target string) error { + host, portStr, err := net.SplitHostPort(target) + if err != nil { + return err + } + + port, err := strconv.Atoi(portStr) + if err != nil { + return errors.New("proxy: failed to parse port number: " + portStr) + } + if port < 1 || port > 0xffff { + return errors.New("proxy: port number out of range: " + portStr) + } + + // the size here is just an estimate + buf := make([]byte, 0, 6+len(host)) + + buf = append(buf, socks5Version) + if len(s.user) > 0 && len(s.user) < 256 && len(s.password) < 256 { + buf = append(buf, 2 /* num auth methods */, socks5AuthNone, socks5AuthPassword) + } else { + buf = append(buf, 1 /* num auth methods */, socks5AuthNone) + } + + if _, err := conn.Write(buf); err != nil { + return errors.New("proxy: failed to write greeting to SOCKS5 proxy at " + s.addr + ": " + err.Error()) + } + + if _, err := io.ReadFull(conn, buf[:2]); err != nil { + return errors.New("proxy: failed to read greeting from SOCKS5 proxy at " + s.addr + ": " + err.Error()) + } + if buf[0] != 5 { + return errors.New("proxy: SOCKS5 proxy at " + s.addr + " has unexpected version " + strconv.Itoa(int(buf[0]))) + } + if buf[1] == 0xff { + return errors.New("proxy: SOCKS5 proxy at " + s.addr + " requires authentication") + } + + if buf[1] == socks5AuthPassword { + buf = buf[:0] + buf = append(buf, 1 /* password protocol version */) + buf = append(buf, uint8(len(s.user))) + buf = append(buf, s.user...) + buf = append(buf, uint8(len(s.password))) + buf = append(buf, s.password...) + + if _, err := conn.Write(buf); err != nil { + return errors.New("proxy: failed to write authentication request to SOCKS5 proxy at " + s.addr + ": " + err.Error()) + } + + if _, err := io.ReadFull(conn, buf[:2]); err != nil { + return errors.New("proxy: failed to read authentication reply from SOCKS5 proxy at " + s.addr + ": " + err.Error()) + } + + if buf[1] != 0 { + return errors.New("proxy: SOCKS5 proxy at " + s.addr + " rejected username/password") + } + } + + buf = buf[:0] + buf = append(buf, socks5Version, socks5Connect, 0 /* reserved */) + + if ip := net.ParseIP(host); ip != nil { + if ip4 := ip.To4(); ip4 != nil { + buf = append(buf, socks5IP4) + ip = ip4 + } else { + buf = append(buf, socks5IP6) + } + buf = append(buf, ip...) + } else { + if len(host) > 255 { + return errors.New("proxy: destination hostname too long: " + host) + } + buf = append(buf, socks5Domain) + buf = append(buf, byte(len(host))) + buf = append(buf, host...) + } + buf = append(buf, byte(port>>8), byte(port)) + + if _, err := conn.Write(buf); err != nil { + return errors.New("proxy: failed to write connect request to SOCKS5 proxy at " + s.addr + ": " + err.Error()) + } + + if _, err := io.ReadFull(conn, buf[:4]); err != nil { + return errors.New("proxy: failed to read connect reply from SOCKS5 proxy at " + s.addr + ": " + err.Error()) + } + + failure := "unknown error" + if int(buf[1]) < len(socks5Errors) { + failure = socks5Errors[buf[1]] + } + + if len(failure) > 0 { + return errors.New("proxy: SOCKS5 proxy at " + s.addr + " failed to connect: " + failure) + } + + bytesToDiscard := 0 + switch buf[3] { + case socks5IP4: + bytesToDiscard = net.IPv4len + case socks5IP6: + bytesToDiscard = net.IPv6len + case socks5Domain: + _, err := io.ReadFull(conn, buf[:1]) + if err != nil { + return errors.New("proxy: failed to read domain length from SOCKS5 proxy at " + s.addr + ": " + err.Error()) + } + bytesToDiscard = int(buf[0]) + default: + return errors.New("proxy: got unknown address type " + strconv.Itoa(int(buf[3])) + " from SOCKS5 proxy at " + s.addr) + } + + if cap(buf) < bytesToDiscard { + buf = make([]byte, bytesToDiscard) + } else { + buf = buf[:bytesToDiscard] + } + if _, err := io.ReadFull(conn, buf); err != nil { + return errors.New("proxy: failed to read address from SOCKS5 proxy at " + s.addr + ": " + err.Error()) + } + + // Also need to discard the port number + if _, err := io.ReadFull(conn, buf[:2]); err != nil { + return errors.New("proxy: failed to read port from SOCKS5 proxy at " + s.addr + ": " + err.Error()) + } + + return nil +} diff --git a/libgo/go/golang_org/x/net/route/address.go b/libgo/go/golang_org/x/net/route/address.go index a56909c105b..e6bfa39e93b 100644 --- a/libgo/go/golang_org/x/net/route/address.go +++ b/libgo/go/golang_org/x/net/route/address.go @@ -24,6 +24,39 @@ type LinkAddr struct { // Family implements the Family method of Addr interface. func (a *LinkAddr) Family() int { return sysAF_LINK } +func (a *LinkAddr) lenAndSpace() (int, int) { + l := 8 + len(a.Name) + len(a.Addr) + return l, roundup(l) +} + +func (a *LinkAddr) marshal(b []byte) (int, error) { + l, ll := a.lenAndSpace() + if len(b) < ll { + return 0, errShortBuffer + } + nlen, alen := len(a.Name), len(a.Addr) + if nlen > 255 || alen > 255 { + return 0, errInvalidAddr + } + b[0] = byte(l) + b[1] = sysAF_LINK + if a.Index > 0 { + nativeEndian.PutUint16(b[2:4], uint16(a.Index)) + } + data := b[8:] + if nlen > 0 { + b[5] = byte(nlen) + copy(data[:nlen], a.Addr) + data = data[nlen:] + } + if alen > 0 { + b[6] = byte(alen) + copy(data[:alen], a.Name) + data = data[alen:] + } + return ll, nil +} + func parseLinkAddr(b []byte) (Addr, error) { if len(b) < 8 { return nil, errInvalidAddr @@ -90,6 +123,21 @@ type Inet4Addr struct { // Family implements the Family method of Addr interface. func (a *Inet4Addr) Family() int { return sysAF_INET } +func (a *Inet4Addr) lenAndSpace() (int, int) { + return sizeofSockaddrInet, roundup(sizeofSockaddrInet) +} + +func (a *Inet4Addr) marshal(b []byte) (int, error) { + l, ll := a.lenAndSpace() + if len(b) < ll { + return 0, errShortBuffer + } + b[0] = byte(l) + b[1] = sysAF_INET + copy(b[4:8], a.IP[:]) + return ll, nil +} + // An Inet6Addr represents an internet address for IPv6. type Inet6Addr struct { IP [16]byte // IP address @@ -99,18 +147,36 @@ type Inet6Addr struct { // Family implements the Family method of Addr interface. func (a *Inet6Addr) Family() int { return sysAF_INET6 } +func (a *Inet6Addr) lenAndSpace() (int, int) { + return sizeofSockaddrInet6, roundup(sizeofSockaddrInet6) +} + +func (a *Inet6Addr) marshal(b []byte) (int, error) { + l, ll := a.lenAndSpace() + if len(b) < ll { + return 0, errShortBuffer + } + b[0] = byte(l) + b[1] = sysAF_INET6 + copy(b[8:24], a.IP[:]) + if a.ZoneID > 0 { + nativeEndian.PutUint32(b[24:28], uint32(a.ZoneID)) + } + return ll, nil +} + // parseInetAddr parses b as an internet address for IPv4 or IPv6. func parseInetAddr(af int, b []byte) (Addr, error) { switch af { case sysAF_INET: - if len(b) < 16 { + if len(b) < sizeofSockaddrInet { return nil, errInvalidAddr } a := &Inet4Addr{} copy(a.IP[:], b[4:8]) return a, nil case sysAF_INET6: - if len(b) < 28 { + if len(b) < sizeofSockaddrInet6 { return nil, errInvalidAddr } a := &Inet6Addr{ZoneID: int(nativeEndian.Uint32(b[24:28]))} @@ -174,7 +240,7 @@ func parseKernelInetAddr(af int, b []byte) (int, Addr, error) { off6 = 8 // offset of in6_addr ) switch { - case b[0] == 28: // size of sockaddr_in6 + case b[0] == sizeofSockaddrInet6: a := &Inet6Addr{} copy(a.IP[:], b[off6:off6+16]) return int(b[0]), a, nil @@ -186,7 +252,7 @@ func parseKernelInetAddr(af int, b []byte) (int, Addr, error) { copy(a.IP[:], b[l-off6:l]) } return int(b[0]), a, nil - case b[0] == 16: // size of sockaddr_in + case b[0] == sizeofSockaddrInet: a := &Inet4Addr{} copy(a.IP[:], b[off4:off4+4]) return int(b[0]), a, nil @@ -211,6 +277,24 @@ type DefaultAddr struct { // Family implements the Family method of Addr interface. func (a *DefaultAddr) Family() int { return a.af } +func (a *DefaultAddr) lenAndSpace() (int, int) { + l := len(a.Raw) + return l, roundup(l) +} + +func (a *DefaultAddr) marshal(b []byte) (int, error) { + l, ll := a.lenAndSpace() + if len(b) < ll { + return 0, errShortBuffer + } + if l > 255 { + return 0, errInvalidAddr + } + b[1] = byte(l) + copy(b[:l], a.Raw) + return ll, nil +} + func parseDefaultAddr(b []byte) (Addr, error) { if len(b) < 2 || len(b) < int(b[0]) { return nil, errInvalidAddr @@ -219,6 +303,66 @@ func parseDefaultAddr(b []byte) (Addr, error) { return a, nil } +func addrsSpace(as []Addr) int { + var l int + for _, a := range as { + switch a := a.(type) { + case *LinkAddr: + _, ll := a.lenAndSpace() + l += ll + case *Inet4Addr: + _, ll := a.lenAndSpace() + l += ll + case *Inet6Addr: + _, ll := a.lenAndSpace() + l += ll + case *DefaultAddr: + _, ll := a.lenAndSpace() + l += ll + } + } + return l +} + +// marshalAddrs marshals as and returns a bitmap indicating which +// address is stored in b. +func marshalAddrs(b []byte, as []Addr) (uint, error) { + var attrs uint + for i, a := range as { + switch a := a.(type) { + case *LinkAddr: + l, err := a.marshal(b) + if err != nil { + return 0, err + } + b = b[l:] + attrs |= 1 << uint(i) + case *Inet4Addr: + l, err := a.marshal(b) + if err != nil { + return 0, err + } + b = b[l:] + attrs |= 1 << uint(i) + case *Inet6Addr: + l, err := a.marshal(b) + if err != nil { + return 0, err + } + b = b[l:] + attrs |= 1 << uint(i) + case *DefaultAddr: + l, err := a.marshal(b) + if err != nil { + return 0, err + } + b = b[l:] + attrs |= 1 << uint(i) + } + } + return attrs, nil +} + func parseAddrs(attrs uint, fn func(int, []byte) (int, Addr, error), b []byte) ([]Addr, error) { var as [sysRTAX_MAX]Addr af := int(sysAF_UNSPEC) diff --git a/libgo/go/golang_org/x/net/route/binary.go b/libgo/go/golang_org/x/net/route/binary.go index 4c561631b94..6910520eca5 100644 --- a/libgo/go/golang_org/x/net/route/binary.go +++ b/libgo/go/golang_org/x/net/route/binary.go @@ -9,7 +9,7 @@ package route // This file contains duplicates of encoding/binary package. // // This package is supposed to be used by the net package of standard -// library. Therefore a package set used in the package must be the +// library. Therefore the package set used in the package must be the // same as net package. var ( diff --git a/libgo/go/golang_org/x/net/route/defs_darwin.go b/libgo/go/golang_org/x/net/route/defs_darwin.go index f452ad14ce6..e7716442d21 100644 --- a/libgo/go/golang_org/x/net/route/defs_darwin.go +++ b/libgo/go/golang_org/x/net/route/defs_darwin.go @@ -13,6 +13,8 @@ package route #include #include #include + +#include */ import "C" @@ -23,6 +25,8 @@ const ( sysAF_LINK = C.AF_LINK sysAF_INET6 = C.AF_INET6 + sysSOCK_RAW = C.SOCK_RAW + sysNET_RT_DUMP = C.NET_RT_DUMP sysNET_RT_FLAGS = C.NET_RT_FLAGS sysNET_RT_IFLIST = C.NET_RT_IFLIST @@ -103,4 +107,8 @@ const ( sizeofRtMsghdrDarwin15 = C.sizeof_struct_rt_msghdr sizeofRtMsghdr2Darwin15 = C.sizeof_struct_rt_msghdr2 sizeofRtMetricsDarwin15 = C.sizeof_struct_rt_metrics + + sizeofSockaddrStorage = C.sizeof_struct_sockaddr_storage + sizeofSockaddrInet = C.sizeof_struct_sockaddr_in + sizeofSockaddrInet6 = C.sizeof_struct_sockaddr_in6 ) diff --git a/libgo/go/golang_org/x/net/route/defs_dragonfly.go b/libgo/go/golang_org/x/net/route/defs_dragonfly.go index c737751d76d..dd31de269a8 100644 --- a/libgo/go/golang_org/x/net/route/defs_dragonfly.go +++ b/libgo/go/golang_org/x/net/route/defs_dragonfly.go @@ -13,6 +13,8 @@ package route #include #include #include + +#include */ import "C" @@ -23,6 +25,8 @@ const ( sysAF_LINK = C.AF_LINK sysAF_INET6 = C.AF_INET6 + sysSOCK_RAW = C.SOCK_RAW + sysNET_RT_DUMP = C.NET_RT_DUMP sysNET_RT_FLAGS = C.NET_RT_FLAGS sysNET_RT_IFLIST = C.NET_RT_IFLIST @@ -102,4 +106,8 @@ const ( sizeofRtMsghdrDragonFlyBSD4 = C.sizeof_struct_rt_msghdr sizeofRtMetricsDragonFlyBSD4 = C.sizeof_struct_rt_metrics + + sizeofSockaddrStorage = C.sizeof_struct_sockaddr_storage + sizeofSockaddrInet = C.sizeof_struct_sockaddr_in + sizeofSockaddrInet6 = C.sizeof_struct_sockaddr_in6 ) diff --git a/libgo/go/golang_org/x/net/route/defs_freebsd.go b/libgo/go/golang_org/x/net/route/defs_freebsd.go index 8f834e81db5..d95594d8ea5 100644 --- a/libgo/go/golang_org/x/net/route/defs_freebsd.go +++ b/libgo/go/golang_org/x/net/route/defs_freebsd.go @@ -14,6 +14,8 @@ package route #include #include +#include + struct if_data_freebsd7 { u_char ifi_type; u_char ifi_physical; @@ -222,6 +224,8 @@ const ( sysAF_LINK = C.AF_LINK sysAF_INET6 = C.AF_INET6 + sysSOCK_RAW = C.SOCK_RAW + sysNET_RT_DUMP = C.NET_RT_DUMP sysNET_RT_FLAGS = C.NET_RT_FLAGS sysNET_RT_IFLIST = C.NET_RT_IFLIST @@ -326,4 +330,8 @@ const ( sizeofIfDataFreeBSD9Emu = C.sizeof_struct_if_data_freebsd9 sizeofIfDataFreeBSD10Emu = C.sizeof_struct_if_data_freebsd10 sizeofIfDataFreeBSD11Emu = C.sizeof_struct_if_data_freebsd11 + + sizeofSockaddrStorage = C.sizeof_struct_sockaddr_storage + sizeofSockaddrInet = C.sizeof_struct_sockaddr_in + sizeofSockaddrInet6 = C.sizeof_struct_sockaddr_in6 ) diff --git a/libgo/go/golang_org/x/net/route/defs_netbsd.go b/libgo/go/golang_org/x/net/route/defs_netbsd.go index b18d85e0165..b0abd549a09 100644 --- a/libgo/go/golang_org/x/net/route/defs_netbsd.go +++ b/libgo/go/golang_org/x/net/route/defs_netbsd.go @@ -13,6 +13,8 @@ package route #include #include #include + +#include */ import "C" @@ -23,6 +25,8 @@ const ( sysAF_LINK = C.AF_LINK sysAF_INET6 = C.AF_INET6 + sysSOCK_RAW = C.SOCK_RAW + sysNET_RT_DUMP = C.NET_RT_DUMP sysNET_RT_FLAGS = C.NET_RT_FLAGS sysNET_RT_IFLIST = C.NET_RT_IFLIST @@ -101,4 +105,8 @@ const ( sizeofRtMsghdrNetBSD7 = C.sizeof_struct_rt_msghdr sizeofRtMetricsNetBSD7 = C.sizeof_struct_rt_metrics + + sizeofSockaddrStorage = C.sizeof_struct_sockaddr_storage + sizeofSockaddrInet = C.sizeof_struct_sockaddr_in + sizeofSockaddrInet6 = C.sizeof_struct_sockaddr_in6 ) diff --git a/libgo/go/golang_org/x/net/route/defs_openbsd.go b/libgo/go/golang_org/x/net/route/defs_openbsd.go index 5df7a43bc3e..0f66d3619c7 100644 --- a/libgo/go/golang_org/x/net/route/defs_openbsd.go +++ b/libgo/go/golang_org/x/net/route/defs_openbsd.go @@ -13,6 +13,8 @@ package route #include #include #include + +#include */ import "C" @@ -23,6 +25,8 @@ const ( sysAF_LINK = C.AF_LINK sysAF_INET6 = C.AF_INET6 + sysSOCK_RAW = C.SOCK_RAW + sysNET_RT_DUMP = C.NET_RT_DUMP sysNET_RT_FLAGS = C.NET_RT_FLAGS sysNET_RT_IFLIST = C.NET_RT_IFLIST @@ -91,3 +95,11 @@ const ( sysRTAX_LABEL = C.RTAX_LABEL sysRTAX_MAX = C.RTAX_MAX ) + +const ( + sizeofRtMsghdr = C.sizeof_struct_rt_msghdr + + sizeofSockaddrStorage = C.sizeof_struct_sockaddr_storage + sizeofSockaddrInet = C.sizeof_struct_sockaddr_in + sizeofSockaddrInet6 = C.sizeof_struct_sockaddr_in6 +) diff --git a/libgo/go/golang_org/x/net/route/message.go b/libgo/go/golang_org/x/net/route/message.go index d7ae0eb50fc..0fa7e09f468 100644 --- a/libgo/go/golang_org/x/net/route/message.go +++ b/libgo/go/golang_org/x/net/route/message.go @@ -7,9 +7,6 @@ package route // A Message represents a routing message. -// -// Note: This interface will be changed to support Marshal method in -// future version. type Message interface { // Sys returns operating system-specific information. Sys() []Sys @@ -52,11 +49,10 @@ func ParseRIB(typ RIBType, b []byte) ([]Message, error) { b = b[l:] continue } - mtyp := int(b[3]) - if fn, ok := parseFns[mtyp]; !ok { + if w, ok := wireFormats[int(b[3])]; !ok { nskips++ } else { - m, err := fn(typ, b) + m, err := w.parse(typ, b) if err != nil { return nil, err } diff --git a/libgo/go/golang_org/x/net/route/message_darwin_test.go b/libgo/go/golang_org/x/net/route/message_darwin_test.go index 3fdd12df553..316aa75071d 100644 --- a/libgo/go/golang_org/x/net/route/message_darwin_test.go +++ b/libgo/go/golang_org/x/net/route/message_darwin_test.go @@ -7,21 +7,28 @@ package route import "testing" func TestFetchAndParseRIBOnDarwin(t *testing.T) { - for _, af := range []int{sysAF_UNSPEC, sysAF_INET, sysAF_INET6} { - for _, typ := range []RIBType{sysNET_RT_FLAGS, sysNET_RT_DUMP2, sysNET_RT_IFLIST2} { - ms, err := fetchAndParseRIB(af, typ) + for _, typ := range []RIBType{sysNET_RT_FLAGS, sysNET_RT_DUMP2, sysNET_RT_IFLIST2} { + var lastErr error + var ms []Message + for _, af := range []int{sysAF_UNSPEC, sysAF_INET, sysAF_INET6} { + rs, err := fetchAndParseRIB(af, typ) if err != nil { - t.Error(err) + lastErr = err continue } - ss, err := msgs(ms).validate() - if err != nil { - t.Errorf("%v %d %v", addrFamily(af), typ, err) - continue - } - for _, s := range ss { - t.Log(s) - } + ms = append(ms, rs...) + } + if len(ms) == 0 && lastErr != nil { + t.Error(typ, lastErr) + continue + } + ss, err := msgs(ms).validate() + if err != nil { + t.Error(typ, err) + continue + } + for _, s := range ss { + t.Log(s) } } } diff --git a/libgo/go/golang_org/x/net/route/message_freebsd_test.go b/libgo/go/golang_org/x/net/route/message_freebsd_test.go index 785c273f656..db4b56752cf 100644 --- a/libgo/go/golang_org/x/net/route/message_freebsd_test.go +++ b/libgo/go/golang_org/x/net/route/message_freebsd_test.go @@ -6,26 +6,32 @@ package route import ( "testing" - "time" "unsafe" ) func TestFetchAndParseRIBOnFreeBSD(t *testing.T) { - for _, af := range []int{sysAF_UNSPEC, sysAF_INET, sysAF_INET6} { - for _, typ := range []RIBType{sysNET_RT_IFMALIST} { - ms, err := fetchAndParseRIB(af, typ) - if err != nil { - t.Error(err) - continue - } - ss, err := msgs(ms).validate() + for _, typ := range []RIBType{sysNET_RT_IFMALIST} { + var lastErr error + var ms []Message + for _, af := range []int{sysAF_UNSPEC, sysAF_INET, sysAF_INET6} { + rs, err := fetchAndParseRIB(af, typ) if err != nil { - t.Errorf("%v %d %v", addrFamily(af), typ, err) + lastErr = err continue } - for _, s := range ss { - t.Log(s) - } + ms = append(ms, rs...) + } + if len(ms) == 0 && lastErr != nil { + t.Error(typ, lastErr) + continue + } + ss, err := msgs(ms).validate() + if err != nil { + t.Error(typ, err) + continue + } + for _, s := range ss { + t.Log(s) } } } @@ -48,58 +54,38 @@ func TestFetchAndParseRIBOnFreeBSD10AndAbove(t *testing.T) { {typ: sysNET_RT_IFLIST}, {typ: sysNET_RT_IFLISTL}, } - for _, af := range []int{sysAF_UNSPEC, sysAF_INET, sysAF_INET6} { + for i := range tests { var lastErr error - for i := 0; i < 3; i++ { - for j := range tests { - var err error - if tests[j].b, err = FetchRIB(af, tests[j].typ, 0); err != nil { - lastErr = err - time.Sleep(10 * time.Millisecond) - } - } - if lastErr == nil { - break + for _, af := range []int{sysAF_UNSPEC, sysAF_INET, sysAF_INET6} { + rs, err := fetchAndParseRIB(af, tests[i].typ) + if err != nil { + lastErr = err + continue } + tests[i].msgs = append(tests[i].msgs, rs...) } - if lastErr != nil { - t.Error(af, lastErr) + if len(tests[i].msgs) == 0 && lastErr != nil { + t.Error(tests[i].typ, lastErr) continue } - for i := range tests { - var err error - if tests[i].msgs, err = ParseRIB(tests[i].typ, tests[i].b); err != nil { - lastErr = err - t.Error(af, err) - } - } + tests[i].ss, lastErr = msgs(tests[i].msgs).validate() if lastErr != nil { + t.Error(tests[i].typ, lastErr) continue } - for i := range tests { - var err error - tests[i].ss, err = msgs(tests[i].msgs).validate() - if err != nil { - lastErr = err - t.Error(af, err) - } - for _, s := range tests[i].ss { - t.Log(s) - } + for _, s := range tests[i].ss { + t.Log(s) } - if lastErr != nil { + } + for i := len(tests) - 1; i > 0; i-- { + if len(tests[i].ss) != len(tests[i-1].ss) { + t.Errorf("got %v; want %v", tests[i].ss, tests[i-1].ss) continue } - for i := len(tests) - 1; i > 0; i-- { - if len(tests[i].ss) != len(tests[i-1].ss) { - t.Errorf("got %v; want %v", tests[i].ss, tests[i-1].ss) - continue - } - for j, s1 := range tests[i].ss { - s0 := tests[i-1].ss[j] - if s1 != s0 { - t.Errorf("got %s; want %s", s1, s0) - } + for j, s1 := range tests[i].ss { + s0 := tests[i-1].ss[j] + if s1 != s0 { + t.Errorf("got %s; want %s", s1, s0) } } } diff --git a/libgo/go/golang_org/x/net/route/message_test.go b/libgo/go/golang_org/x/net/route/message_test.go index c0c7c57a9a5..e848dabf4f9 100644 --- a/libgo/go/golang_org/x/net/route/message_test.go +++ b/libgo/go/golang_org/x/net/route/message_test.go @@ -14,30 +14,54 @@ import ( ) func TestFetchAndParseRIB(t *testing.T) { - for _, af := range []int{sysAF_UNSPEC, sysAF_INET, sysAF_INET6} { - for _, typ := range []RIBType{sysNET_RT_DUMP, sysNET_RT_IFLIST} { - ms, err := fetchAndParseRIB(af, typ) + for _, typ := range []RIBType{sysNET_RT_DUMP, sysNET_RT_IFLIST} { + var lastErr error + var ms []Message + for _, af := range []int{sysAF_UNSPEC, sysAF_INET, sysAF_INET6} { + rs, err := fetchAndParseRIB(af, typ) if err != nil { - t.Error(err) - continue - } - ss, err := msgs(ms).validate() - if err != nil { - t.Errorf("%v %d %v", addrFamily(af), typ, err) + lastErr = err continue } - for _, s := range ss { - t.Log(s) - } + ms = append(ms, rs...) + } + if len(ms) == 0 && lastErr != nil { + t.Error(typ, lastErr) + continue + } + ss, err := msgs(ms).validate() + if err != nil { + t.Error(typ, err) + continue + } + for _, s := range ss { + t.Log(typ, s) } } } +var ( + rtmonSock int + rtmonErr error +) + +func init() { + // We need to keep rtmonSock alive to avoid treading on + // recycled socket descriptors. + rtmonSock, rtmonErr = syscall.Socket(sysAF_ROUTE, sysSOCK_RAW, sysAF_UNSPEC) +} + +// TestMonitorAndParseRIB leaks a worker goroutine and a socket +// descriptor but that's intentional. func TestMonitorAndParseRIB(t *testing.T) { if testing.Short() || os.Getuid() != 0 { t.Skip("must be root") } + if rtmonErr != nil { + t.Fatal(rtmonErr) + } + // We suppose that using an IPv4 link-local address and the // dot1Q ID for Token Ring and FDDI doesn't harm anyone. pv := &propVirtual{addr: "169.254.0.1", mask: "255.255.255.0"} @@ -49,16 +73,18 @@ func TestMonitorAndParseRIB(t *testing.T) { } pv.teardown() - s, err := syscall.Socket(syscall.AF_ROUTE, syscall.SOCK_RAW, syscall.AF_UNSPEC) - if err != nil { - t.Fatal(err) - } - defer syscall.Close(s) - go func() { b := make([]byte, os.Getpagesize()) for { - n, err := syscall.Read(s, b) + // There's no easy way to unblock this read + // call because the routing message exchange + // over routing socket is a connectionless + // message-oriented protocol, no control plane + // for signaling connectivity, and we cannot + // use the net package of standard library due + // to the lack of support for routing socket + // and circular dependency. + n, err := syscall.Read(rtmonSock, b) if err != nil { return } @@ -116,3 +142,98 @@ func TestParseRIBWithFuzz(t *testing.T) { } } } + +func TestRouteMessage(t *testing.T) { + s, err := syscall.Socket(sysAF_ROUTE, sysSOCK_RAW, sysAF_UNSPEC) + if err != nil { + t.Fatal(err) + } + defer syscall.Close(s) + + var ms []RouteMessage + for _, af := range []int{sysAF_INET, sysAF_INET6} { + if _, err := fetchAndParseRIB(af, sysNET_RT_DUMP); err != nil { + t.Log(err) + continue + } + switch af { + case sysAF_INET: + ms = append(ms, []RouteMessage{ + { + Type: sysRTM_GET, + Addrs: []Addr{ + &Inet4Addr{IP: [4]byte{127, 0, 0, 1}}, + nil, + nil, + nil, + &LinkAddr{}, + &Inet4Addr{}, + nil, + &Inet4Addr{}, + }, + }, + { + Type: sysRTM_GET, + Addrs: []Addr{ + &Inet4Addr{IP: [4]byte{127, 0, 0, 1}}, + }, + }, + }...) + case sysAF_INET6: + ms = append(ms, []RouteMessage{ + { + Type: sysRTM_GET, + Addrs: []Addr{ + &Inet6Addr{IP: [16]byte{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1}}, + nil, + nil, + nil, + &LinkAddr{}, + &Inet6Addr{}, + nil, + &Inet6Addr{}, + }, + }, + { + Type: sysRTM_GET, + Addrs: []Addr{ + &Inet6Addr{IP: [16]byte{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1}}, + }, + }, + }...) + } + } + for i, m := range ms { + m.ID = uintptr(os.Getpid()) + m.Seq = i + 1 + wb, err := m.Marshal() + if err != nil { + t.Fatalf("%v: %v", m, err) + } + if _, err := syscall.Write(s, wb); err != nil { + t.Fatalf("%v: %v", m, err) + } + rb := make([]byte, os.Getpagesize()) + n, err := syscall.Read(s, rb) + if err != nil { + t.Fatalf("%v: %v", m, err) + } + rms, err := ParseRIB(0, rb[:n]) + if err != nil { + t.Fatalf("%v: %v", m, err) + } + for _, rm := range rms { + err := rm.(*RouteMessage).Err + if err != nil { + t.Errorf("%v: %v", m, err) + } + } + ss, err := msgs(rms).validate() + if err != nil { + t.Fatalf("%v: %v", m, err) + } + for _, s := range ss { + t.Log(s) + } + } +} diff --git a/libgo/go/golang_org/x/net/route/route.go b/libgo/go/golang_org/x/net/route/route.go index c986e29ebc9..081da0d5c18 100644 --- a/libgo/go/golang_org/x/net/route/route.go +++ b/libgo/go/golang_org/x/net/route/route.go @@ -24,21 +24,70 @@ var ( errMessageTooShort = errors.New("message too short") errInvalidMessage = errors.New("invalid message") errInvalidAddr = errors.New("invalid address") + errShortBuffer = errors.New("short buffer") ) // A RouteMessage represents a message conveying an address prefix, a // nexthop address and an output interface. +// +// Unlike other messages, this message can be used to query adjacency +// information for the given address prefix, to add a new route, and +// to delete or modify the existing route from the routing information +// base inside the kernel by writing and reading route messages on a +// routing socket. +// +// For the manipulation of routing information, the route message must +// contain appropriate fields that include: +// +// Version = +// Type = +// Flags = +// Index = +// ID = +// Seq = +// Addrs = +// +// The Type field specifies a type of manipulation, the Flags field +// specifies a class of target information and the Addrs field +// specifies target information like the following: +// +// route.RouteMessage{ +// Version: RTM_VERSION, +// Type: RTM_GET, +// Flags: RTF_UP | RTF_HOST, +// ID: uintptr(os.Getpid()), +// Seq: 1, +// Addrs: []route.Addrs{ +// RTAX_DST: &route.Inet4Addr{ ... }, +// RTAX_IFP: &route.LinkAddr{ ... }, +// RTAX_BRD: &route.Inet4Addr{ ... }, +// }, +// } +// +// The values for the above fields depend on the implementation of +// each operating system. +// +// The Err field on a response message contains an error value on the +// requested operation. If non-nil, the requested operation is failed. type RouteMessage struct { - Version int // message version - Type int // message type - Flags int // route flags - Index int // interface index when atatched - Addrs []Addr // addresses + Version int // message version + Type int // message type + Flags int // route flags + Index int // interface index when atatched + ID uintptr // sender's identifier; usually process ID + Seq int // sequence number + Err error // error on requested operation + Addrs []Addr // addresses extOff int // offset of header extension raw []byte // raw message } +// Marshal returns the binary encoding of m. +func (m *RouteMessage) Marshal() ([]byte, error) { + return m.marshal() +} + // A RIBType reprensents a type of routing information base. type RIBType int diff --git a/libgo/go/golang_org/x/net/route/route_classic.go b/libgo/go/golang_org/x/net/route/route_classic.go index d333c6aa526..61b2bb4add0 100644 --- a/libgo/go/golang_org/x/net/route/route_classic.go +++ b/libgo/go/golang_org/x/net/route/route_classic.go @@ -6,6 +6,36 @@ package route +import "syscall" + +func (m *RouteMessage) marshal() ([]byte, error) { + w, ok := wireFormats[m.Type] + if !ok { + return nil, errUnsupportedMessage + } + l := w.bodyOff + addrsSpace(m.Addrs) + b := make([]byte, l) + nativeEndian.PutUint16(b[:2], uint16(l)) + if m.Version == 0 { + b[2] = sysRTM_VERSION + } else { + b[2] = byte(m.Version) + } + b[3] = byte(m.Type) + nativeEndian.PutUint32(b[8:12], uint32(m.Flags)) + nativeEndian.PutUint16(b[4:6], uint16(m.Index)) + nativeEndian.PutUint32(b[16:20], uint32(m.ID)) + nativeEndian.PutUint32(b[20:24], uint32(m.Seq)) + attrs, err := marshalAddrs(b[w.bodyOff:], m.Addrs) + if err != nil { + return nil, err + } + if attrs > 0 { + nativeEndian.PutUint32(b[12:16], uint32(attrs)) + } + return b, nil +} + func (w *wireFormat) parseRouteMessage(typ RIBType, b []byte) (Message, error) { if len(b) < w.bodyOff { return nil, errMessageTooShort @@ -19,9 +49,15 @@ func (w *wireFormat) parseRouteMessage(typ RIBType, b []byte) (Message, error) { Type: int(b[3]), Flags: int(nativeEndian.Uint32(b[8:12])), Index: int(nativeEndian.Uint16(b[4:6])), + ID: uintptr(nativeEndian.Uint32(b[16:20])), + Seq: int(nativeEndian.Uint32(b[20:24])), extOff: w.extOff, raw: b[:l], } + errno := syscall.Errno(nativeEndian.Uint32(b[28:32])) + if errno != 0 { + m.Err = errno + } var err error m.Addrs, err = parseAddrs(uint(nativeEndian.Uint32(b[12:16])), parseKernelInetAddr, b[w.bodyOff:]) if err != nil { diff --git a/libgo/go/golang_org/x/net/route/route_openbsd.go b/libgo/go/golang_org/x/net/route/route_openbsd.go index 76eae40d806..daf2e90c476 100644 --- a/libgo/go/golang_org/x/net/route/route_openbsd.go +++ b/libgo/go/golang_org/x/net/route/route_openbsd.go @@ -4,8 +4,35 @@ package route +import "syscall" + +func (m *RouteMessage) marshal() ([]byte, error) { + l := sizeofRtMsghdr + addrsSpace(m.Addrs) + b := make([]byte, l) + nativeEndian.PutUint16(b[:2], uint16(l)) + if m.Version == 0 { + b[2] = sysRTM_VERSION + } else { + b[2] = byte(m.Version) + } + b[3] = byte(m.Type) + nativeEndian.PutUint16(b[4:6], uint16(sizeofRtMsghdr)) + nativeEndian.PutUint32(b[16:20], uint32(m.Flags)) + nativeEndian.PutUint16(b[6:8], uint16(m.Index)) + nativeEndian.PutUint32(b[24:28], uint32(m.ID)) + nativeEndian.PutUint32(b[28:32], uint32(m.Seq)) + attrs, err := marshalAddrs(b[sizeofRtMsghdr:], m.Addrs) + if err != nil { + return nil, err + } + if attrs > 0 { + nativeEndian.PutUint32(b[12:16], uint32(attrs)) + } + return b, nil +} + func (*wireFormat) parseRouteMessage(_ RIBType, b []byte) (Message, error) { - if len(b) < 40 { + if len(b) < sizeofRtMsghdr { return nil, errMessageTooShort } l := int(nativeEndian.Uint16(b[:2])) @@ -17,12 +44,18 @@ func (*wireFormat) parseRouteMessage(_ RIBType, b []byte) (Message, error) { Type: int(b[3]), Flags: int(nativeEndian.Uint32(b[16:20])), Index: int(nativeEndian.Uint16(b[6:8])), + ID: uintptr(nativeEndian.Uint32(b[24:28])), + Seq: int(nativeEndian.Uint32(b[28:32])), raw: b[:l], } ll := int(nativeEndian.Uint16(b[4:6])) if len(b) < ll { return nil, errInvalidMessage } + errno := syscall.Errno(nativeEndian.Uint32(b[32:36])) + if errno != 0 { + m.Err = errno + } as, err := parseAddrs(uint(nativeEndian.Uint32(b[12:16])), parseKernelInetAddr, b[ll:]) if err != nil { return nil, err diff --git a/libgo/go/golang_org/x/net/route/sys.go b/libgo/go/golang_org/x/net/route/sys.go index 80ca83ae137..3d0ee9b1481 100644 --- a/libgo/go/golang_org/x/net/route/sys.go +++ b/libgo/go/golang_org/x/net/route/sys.go @@ -11,7 +11,7 @@ import "unsafe" var ( nativeEndian binaryByteOrder kernelAlign int - parseFns map[int]parseFn + wireFormats map[int]*wireFormat ) func init() { @@ -22,7 +22,7 @@ func init() { } else { nativeEndian = bigEndian } - kernelAlign, parseFns = probeRoutingStack() + kernelAlign, wireFormats = probeRoutingStack() } func roundup(l int) int { @@ -32,9 +32,8 @@ func roundup(l int) int { return (l + kernelAlign - 1) & ^(kernelAlign - 1) } -type parseFn func(RIBType, []byte) (Message, error) - type wireFormat struct { extOff int // offset of header extension bodyOff int // offset of message body + parse func(RIBType, []byte) (Message, error) } diff --git a/libgo/go/golang_org/x/net/route/sys_darwin.go b/libgo/go/golang_org/x/net/route/sys_darwin.go index fff3a0fd1db..e742c919db8 100644 --- a/libgo/go/golang_org/x/net/route/sys_darwin.go +++ b/libgo/go/golang_org/x/net/route/sys_darwin.go @@ -49,32 +49,39 @@ func (m *InterfaceMessage) Sys() []Sys { } } -func probeRoutingStack() (int, map[int]parseFn) { +func probeRoutingStack() (int, map[int]*wireFormat) { rtm := &wireFormat{extOff: 36, bodyOff: sizeofRtMsghdrDarwin15} + rtm.parse = rtm.parseRouteMessage rtm2 := &wireFormat{extOff: 36, bodyOff: sizeofRtMsghdr2Darwin15} + rtm2.parse = rtm2.parseRouteMessage ifm := &wireFormat{extOff: 16, bodyOff: sizeofIfMsghdrDarwin15} + ifm.parse = ifm.parseInterfaceMessage ifm2 := &wireFormat{extOff: 32, bodyOff: sizeofIfMsghdr2Darwin15} + ifm2.parse = ifm2.parseInterfaceMessage ifam := &wireFormat{extOff: sizeofIfaMsghdrDarwin15, bodyOff: sizeofIfaMsghdrDarwin15} + ifam.parse = ifam.parseInterfaceAddrMessage ifmam := &wireFormat{extOff: sizeofIfmaMsghdrDarwin15, bodyOff: sizeofIfmaMsghdrDarwin15} + ifmam.parse = ifmam.parseInterfaceMulticastAddrMessage ifmam2 := &wireFormat{extOff: sizeofIfmaMsghdr2Darwin15, bodyOff: sizeofIfmaMsghdr2Darwin15} + ifmam2.parse = ifmam2.parseInterfaceMulticastAddrMessage // Darwin kernels require 32-bit aligned access to routing facilities. - return 4, map[int]parseFn{ - sysRTM_ADD: rtm.parseRouteMessage, - sysRTM_DELETE: rtm.parseRouteMessage, - sysRTM_CHANGE: rtm.parseRouteMessage, - sysRTM_GET: rtm.parseRouteMessage, - sysRTM_LOSING: rtm.parseRouteMessage, - sysRTM_REDIRECT: rtm.parseRouteMessage, - sysRTM_MISS: rtm.parseRouteMessage, - sysRTM_LOCK: rtm.parseRouteMessage, - sysRTM_RESOLVE: rtm.parseRouteMessage, - sysRTM_NEWADDR: ifam.parseInterfaceAddrMessage, - sysRTM_DELADDR: ifam.parseInterfaceAddrMessage, - sysRTM_IFINFO: ifm.parseInterfaceMessage, - sysRTM_NEWMADDR: ifmam.parseInterfaceMulticastAddrMessage, - sysRTM_DELMADDR: ifmam.parseInterfaceMulticastAddrMessage, - sysRTM_IFINFO2: ifm2.parseInterfaceMessage, - sysRTM_NEWMADDR2: ifmam2.parseInterfaceMulticastAddrMessage, - sysRTM_GET2: rtm2.parseRouteMessage, + return 4, map[int]*wireFormat{ + sysRTM_ADD: rtm, + sysRTM_DELETE: rtm, + sysRTM_CHANGE: rtm, + sysRTM_GET: rtm, + sysRTM_LOSING: rtm, + sysRTM_REDIRECT: rtm, + sysRTM_MISS: rtm, + sysRTM_LOCK: rtm, + sysRTM_RESOLVE: rtm, + sysRTM_NEWADDR: ifam, + sysRTM_DELADDR: ifam, + sysRTM_IFINFO: ifm, + sysRTM_NEWMADDR: ifmam, + sysRTM_DELMADDR: ifmam, + sysRTM_IFINFO2: ifm2, + sysRTM_NEWMADDR2: ifmam2, + sysRTM_GET2: rtm2, } } diff --git a/libgo/go/golang_org/x/net/route/sys_dragonfly.go b/libgo/go/golang_org/x/net/route/sys_dragonfly.go index da848b3d076..b175cb18cee 100644 --- a/libgo/go/golang_org/x/net/route/sys_dragonfly.go +++ b/libgo/go/golang_org/x/net/route/sys_dragonfly.go @@ -44,28 +44,33 @@ func (m *InterfaceMessage) Sys() []Sys { } } -func probeRoutingStack() (int, map[int]parseFn) { +func probeRoutingStack() (int, map[int]*wireFormat) { var p uintptr rtm := &wireFormat{extOff: 40, bodyOff: sizeofRtMsghdrDragonFlyBSD4} + rtm.parse = rtm.parseRouteMessage ifm := &wireFormat{extOff: 16, bodyOff: sizeofIfMsghdrDragonFlyBSD4} + ifm.parse = ifm.parseInterfaceMessage ifam := &wireFormat{extOff: sizeofIfaMsghdrDragonFlyBSD4, bodyOff: sizeofIfaMsghdrDragonFlyBSD4} + ifam.parse = ifam.parseInterfaceAddrMessage ifmam := &wireFormat{extOff: sizeofIfmaMsghdrDragonFlyBSD4, bodyOff: sizeofIfmaMsghdrDragonFlyBSD4} + ifmam.parse = ifmam.parseInterfaceMulticastAddrMessage ifanm := &wireFormat{extOff: sizeofIfAnnouncemsghdrDragonFlyBSD4, bodyOff: sizeofIfAnnouncemsghdrDragonFlyBSD4} - return int(unsafe.Sizeof(p)), map[int]parseFn{ - sysRTM_ADD: rtm.parseRouteMessage, - sysRTM_DELETE: rtm.parseRouteMessage, - sysRTM_CHANGE: rtm.parseRouteMessage, - sysRTM_GET: rtm.parseRouteMessage, - sysRTM_LOSING: rtm.parseRouteMessage, - sysRTM_REDIRECT: rtm.parseRouteMessage, - sysRTM_MISS: rtm.parseRouteMessage, - sysRTM_LOCK: rtm.parseRouteMessage, - sysRTM_RESOLVE: rtm.parseRouteMessage, - sysRTM_NEWADDR: ifam.parseInterfaceAddrMessage, - sysRTM_DELADDR: ifam.parseInterfaceAddrMessage, - sysRTM_IFINFO: ifm.parseInterfaceMessage, - sysRTM_NEWMADDR: ifmam.parseInterfaceMulticastAddrMessage, - sysRTM_DELMADDR: ifmam.parseInterfaceMulticastAddrMessage, - sysRTM_IFANNOUNCE: ifanm.parseInterfaceAnnounceMessage, + ifanm.parse = ifanm.parseInterfaceAnnounceMessage + return int(unsafe.Sizeof(p)), map[int]*wireFormat{ + sysRTM_ADD: rtm, + sysRTM_DELETE: rtm, + sysRTM_CHANGE: rtm, + sysRTM_GET: rtm, + sysRTM_LOSING: rtm, + sysRTM_REDIRECT: rtm, + sysRTM_MISS: rtm, + sysRTM_LOCK: rtm, + sysRTM_RESOLVE: rtm, + sysRTM_NEWADDR: ifam, + sysRTM_DELADDR: ifam, + sysRTM_IFINFO: ifm, + sysRTM_NEWMADDR: ifmam, + sysRTM_DELMADDR: ifmam, + sysRTM_IFANNOUNCE: ifanm, } } diff --git a/libgo/go/golang_org/x/net/route/sys_freebsd.go b/libgo/go/golang_org/x/net/route/sys_freebsd.go index 7b05c1a5a05..010d4ae7827 100644 --- a/libgo/go/golang_org/x/net/route/sys_freebsd.go +++ b/libgo/go/golang_org/x/net/route/sys_freebsd.go @@ -54,7 +54,7 @@ func (m *InterfaceMessage) Sys() []Sys { } } -func probeRoutingStack() (int, map[int]parseFn) { +func probeRoutingStack() (int, map[int]*wireFormat) { var p uintptr wordSize := int(unsafe.Sizeof(p)) align := int(unsafe.Sizeof(p)) @@ -130,21 +130,26 @@ func probeRoutingStack() (int, map[int]parseFn) { ifm.bodyOff = sizeofIfMsghdrFreeBSD11 } } - return align, map[int]parseFn{ - sysRTM_ADD: rtm.parseRouteMessage, - sysRTM_DELETE: rtm.parseRouteMessage, - sysRTM_CHANGE: rtm.parseRouteMessage, - sysRTM_GET: rtm.parseRouteMessage, - sysRTM_LOSING: rtm.parseRouteMessage, - sysRTM_REDIRECT: rtm.parseRouteMessage, - sysRTM_MISS: rtm.parseRouteMessage, - sysRTM_LOCK: rtm.parseRouteMessage, - sysRTM_RESOLVE: rtm.parseRouteMessage, - sysRTM_NEWADDR: ifam.parseInterfaceAddrMessage, - sysRTM_DELADDR: ifam.parseInterfaceAddrMessage, - sysRTM_IFINFO: ifm.parseInterfaceMessage, - sysRTM_NEWMADDR: ifmam.parseInterfaceMulticastAddrMessage, - sysRTM_DELMADDR: ifmam.parseInterfaceMulticastAddrMessage, - sysRTM_IFANNOUNCE: ifanm.parseInterfaceAnnounceMessage, + rtm.parse = rtm.parseRouteMessage + ifm.parse = ifm.parseInterfaceMessage + ifam.parse = ifam.parseInterfaceAddrMessage + ifmam.parse = ifmam.parseInterfaceMulticastAddrMessage + ifanm.parse = ifanm.parseInterfaceAnnounceMessage + return align, map[int]*wireFormat{ + sysRTM_ADD: rtm, + sysRTM_DELETE: rtm, + sysRTM_CHANGE: rtm, + sysRTM_GET: rtm, + sysRTM_LOSING: rtm, + sysRTM_REDIRECT: rtm, + sysRTM_MISS: rtm, + sysRTM_LOCK: rtm, + sysRTM_RESOLVE: rtm, + sysRTM_NEWADDR: ifam, + sysRTM_DELADDR: ifam, + sysRTM_IFINFO: ifm, + sysRTM_NEWMADDR: ifmam, + sysRTM_DELMADDR: ifmam, + sysRTM_IFANNOUNCE: ifanm, } } diff --git a/libgo/go/golang_org/x/net/route/sys_netbsd.go b/libgo/go/golang_org/x/net/route/sys_netbsd.go index 4d8076b518d..b4e33014031 100644 --- a/libgo/go/golang_org/x/net/route/sys_netbsd.go +++ b/libgo/go/golang_org/x/net/route/sys_netbsd.go @@ -42,26 +42,30 @@ func (m *InterfaceMessage) Sys() []Sys { } } -func probeRoutingStack() (int, map[int]parseFn) { +func probeRoutingStack() (int, map[int]*wireFormat) { rtm := &wireFormat{extOff: 40, bodyOff: sizeofRtMsghdrNetBSD7} + rtm.parse = rtm.parseRouteMessage ifm := &wireFormat{extOff: 16, bodyOff: sizeofIfMsghdrNetBSD7} + ifm.parse = ifm.parseInterfaceMessage ifam := &wireFormat{extOff: sizeofIfaMsghdrNetBSD7, bodyOff: sizeofIfaMsghdrNetBSD7} + ifam.parse = ifam.parseInterfaceAddrMessage ifanm := &wireFormat{extOff: sizeofIfAnnouncemsghdrNetBSD7, bodyOff: sizeofIfAnnouncemsghdrNetBSD7} + ifanm.parse = ifanm.parseInterfaceAnnounceMessage // NetBSD 6 and above kernels require 64-bit aligned access to // routing facilities. - return 8, map[int]parseFn{ - sysRTM_ADD: rtm.parseRouteMessage, - sysRTM_DELETE: rtm.parseRouteMessage, - sysRTM_CHANGE: rtm.parseRouteMessage, - sysRTM_GET: rtm.parseRouteMessage, - sysRTM_LOSING: rtm.parseRouteMessage, - sysRTM_REDIRECT: rtm.parseRouteMessage, - sysRTM_MISS: rtm.parseRouteMessage, - sysRTM_LOCK: rtm.parseRouteMessage, - sysRTM_RESOLVE: rtm.parseRouteMessage, - sysRTM_NEWADDR: ifam.parseInterfaceAddrMessage, - sysRTM_DELADDR: ifam.parseInterfaceAddrMessage, - sysRTM_IFANNOUNCE: ifanm.parseInterfaceAnnounceMessage, - sysRTM_IFINFO: ifm.parseInterfaceMessage, + return 8, map[int]*wireFormat{ + sysRTM_ADD: rtm, + sysRTM_DELETE: rtm, + sysRTM_CHANGE: rtm, + sysRTM_GET: rtm, + sysRTM_LOSING: rtm, + sysRTM_REDIRECT: rtm, + sysRTM_MISS: rtm, + sysRTM_LOCK: rtm, + sysRTM_RESOLVE: rtm, + sysRTM_NEWADDR: ifam, + sysRTM_DELADDR: ifam, + sysRTM_IFANNOUNCE: ifanm, + sysRTM_IFINFO: ifm, } } diff --git a/libgo/go/golang_org/x/net/route/sys_openbsd.go b/libgo/go/golang_org/x/net/route/sys_openbsd.go index 26d04386961..8798dc4ca37 100644 --- a/libgo/go/golang_org/x/net/route/sys_openbsd.go +++ b/libgo/go/golang_org/x/net/route/sys_openbsd.go @@ -51,22 +51,29 @@ func (m *InterfaceMessage) Sys() []Sys { } } -func probeRoutingStack() (int, map[int]parseFn) { +func probeRoutingStack() (int, map[int]*wireFormat) { var p uintptr - nooff := &wireFormat{extOff: -1, bodyOff: -1} - return int(unsafe.Sizeof(p)), map[int]parseFn{ - sysRTM_ADD: nooff.parseRouteMessage, - sysRTM_DELETE: nooff.parseRouteMessage, - sysRTM_CHANGE: nooff.parseRouteMessage, - sysRTM_GET: nooff.parseRouteMessage, - sysRTM_LOSING: nooff.parseRouteMessage, - sysRTM_REDIRECT: nooff.parseRouteMessage, - sysRTM_MISS: nooff.parseRouteMessage, - sysRTM_LOCK: nooff.parseRouteMessage, - sysRTM_RESOLVE: nooff.parseRouteMessage, - sysRTM_NEWADDR: nooff.parseInterfaceAddrMessage, - sysRTM_DELADDR: nooff.parseInterfaceAddrMessage, - sysRTM_IFINFO: nooff.parseInterfaceMessage, - sysRTM_IFANNOUNCE: nooff.parseInterfaceAnnounceMessage, + rtm := &wireFormat{extOff: -1, bodyOff: -1} + rtm.parse = rtm.parseRouteMessage + ifm := &wireFormat{extOff: -1, bodyOff: -1} + ifm.parse = ifm.parseInterfaceMessage + ifam := &wireFormat{extOff: -1, bodyOff: -1} + ifam.parse = ifam.parseInterfaceAddrMessage + ifanm := &wireFormat{extOff: -1, bodyOff: -1} + ifanm.parse = ifanm.parseInterfaceAnnounceMessage + return int(unsafe.Sizeof(p)), map[int]*wireFormat{ + sysRTM_ADD: rtm, + sysRTM_DELETE: rtm, + sysRTM_CHANGE: rtm, + sysRTM_GET: rtm, + sysRTM_LOSING: rtm, + sysRTM_REDIRECT: rtm, + sysRTM_MISS: rtm, + sysRTM_LOCK: rtm, + sysRTM_RESOLVE: rtm, + sysRTM_NEWADDR: ifam, + sysRTM_DELADDR: ifam, + sysRTM_IFINFO: ifm, + sysRTM_IFANNOUNCE: ifanm, } } diff --git a/libgo/go/golang_org/x/net/route/syscall.go b/libgo/go/golang_org/x/net/route/syscall.go index d136325a30a..c211188b10b 100644 --- a/libgo/go/golang_org/x/net/route/syscall.go +++ b/libgo/go/golang_org/x/net/route/syscall.go @@ -11,10 +11,6 @@ import ( "unsafe" ) -// TODO: replace with runtime.KeepAlive when available -//go:noescape -func keepAlive(p unsafe.Pointer) - var zero uintptr func sysctl(mib []int32, old *byte, oldlen *uintptr, new *byte, newlen uintptr) error { @@ -25,7 +21,6 @@ func sysctl(mib []int32, old *byte, oldlen *uintptr, new *byte, newlen uintptr) p = unsafe.Pointer(&zero) } _, _, errno := syscall.Syscall6(syscall.SYS___SYSCTL, uintptr(p), uintptr(len(mib)), uintptr(unsafe.Pointer(old)), uintptr(unsafe.Pointer(oldlen)), uintptr(unsafe.Pointer(new)), uintptr(newlen)) - keepAlive(p) if errno != 0 { return error(errno) } diff --git a/libgo/go/golang_org/x/net/route/zsys_darwin.go b/libgo/go/golang_org/x/net/route/zsys_darwin.go index 265b81cd50e..4e2e1ab090c 100644 --- a/libgo/go/golang_org/x/net/route/zsys_darwin.go +++ b/libgo/go/golang_org/x/net/route/zsys_darwin.go @@ -10,6 +10,8 @@ const ( sysAF_LINK = 0x12 sysAF_INET6 = 0x1e + sysSOCK_RAW = 0x3 + sysNET_RT_DUMP = 0x1 sysNET_RT_FLAGS = 0x2 sysNET_RT_IFLIST = 0x3 @@ -90,4 +92,8 @@ const ( sizeofRtMsghdrDarwin15 = 0x5c sizeofRtMsghdr2Darwin15 = 0x5c sizeofRtMetricsDarwin15 = 0x38 + + sizeofSockaddrStorage = 0x80 + sizeofSockaddrInet = 0x10 + sizeofSockaddrInet6 = 0x1c ) diff --git a/libgo/go/golang_org/x/net/route/zsys_dragonfly.go b/libgo/go/golang_org/x/net/route/zsys_dragonfly.go index dd36dece0f6..719c88d11f8 100644 --- a/libgo/go/golang_org/x/net/route/zsys_dragonfly.go +++ b/libgo/go/golang_org/x/net/route/zsys_dragonfly.go @@ -10,6 +10,8 @@ const ( sysAF_LINK = 0x12 sysAF_INET6 = 0x1c + sysSOCK_RAW = 0x3 + sysNET_RT_DUMP = 0x1 sysNET_RT_FLAGS = 0x2 sysNET_RT_IFLIST = 0x3 @@ -89,4 +91,8 @@ const ( sizeofRtMsghdrDragonFlyBSD4 = 0x98 sizeofRtMetricsDragonFlyBSD4 = 0x70 + + sizeofSockaddrStorage = 0x80 + sizeofSockaddrInet = 0x10 + sizeofSockaddrInet6 = 0x1c ) diff --git a/libgo/go/golang_org/x/net/route/zsys_freebsd_386.go b/libgo/go/golang_org/x/net/route/zsys_freebsd_386.go index 9bac2e3900f..b03bc01f654 100644 --- a/libgo/go/golang_org/x/net/route/zsys_freebsd_386.go +++ b/libgo/go/golang_org/x/net/route/zsys_freebsd_386.go @@ -10,6 +10,8 @@ const ( sysAF_LINK = 0x12 sysAF_INET6 = 0x1c + sysSOCK_RAW = 0x3 + sysNET_RT_DUMP = 0x1 sysNET_RT_FLAGS = 0x2 sysNET_RT_IFLIST = 0x3 @@ -117,4 +119,8 @@ const ( sizeofIfDataFreeBSD9Emu = 0x98 sizeofIfDataFreeBSD10Emu = 0x98 sizeofIfDataFreeBSD11Emu = 0x98 + + sizeofSockaddrStorage = 0x80 + sizeofSockaddrInet = 0x10 + sizeofSockaddrInet6 = 0x1c ) diff --git a/libgo/go/golang_org/x/net/route/zsys_freebsd_amd64.go b/libgo/go/golang_org/x/net/route/zsys_freebsd_amd64.go index b1920d7ac16..0b675b3d3f9 100644 --- a/libgo/go/golang_org/x/net/route/zsys_freebsd_amd64.go +++ b/libgo/go/golang_org/x/net/route/zsys_freebsd_amd64.go @@ -10,6 +10,8 @@ const ( sysAF_LINK = 0x12 sysAF_INET6 = 0x1c + sysSOCK_RAW = 0x3 + sysNET_RT_DUMP = 0x1 sysNET_RT_FLAGS = 0x2 sysNET_RT_IFLIST = 0x3 @@ -114,4 +116,8 @@ const ( sizeofIfDataFreeBSD9Emu = 0x98 sizeofIfDataFreeBSD10Emu = 0x98 sizeofIfDataFreeBSD11Emu = 0x98 + + sizeofSockaddrStorage = 0x80 + sizeofSockaddrInet = 0x10 + sizeofSockaddrInet6 = 0x1c ) diff --git a/libgo/go/golang_org/x/net/route/zsys_freebsd_arm.go b/libgo/go/golang_org/x/net/route/zsys_freebsd_arm.go index a034d6fcbf2..58f8ea16f25 100644 --- a/libgo/go/golang_org/x/net/route/zsys_freebsd_arm.go +++ b/libgo/go/golang_org/x/net/route/zsys_freebsd_arm.go @@ -10,6 +10,8 @@ const ( sysAF_LINK = 0x12 sysAF_INET6 = 0x1c + sysSOCK_RAW = 0x3 + sysNET_RT_DUMP = 0x1 sysNET_RT_FLAGS = 0x2 sysNET_RT_IFLIST = 0x3 @@ -114,4 +116,8 @@ const ( sizeofIfDataFreeBSD9Emu = 0x60 sizeofIfDataFreeBSD10Emu = 0x60 sizeofIfDataFreeBSD11Emu = 0x98 + + sizeofSockaddrStorage = 0x80 + sizeofSockaddrInet = 0x10 + sizeofSockaddrInet6 = 0x1c ) diff --git a/libgo/go/golang_org/x/net/route/zsys_netbsd.go b/libgo/go/golang_org/x/net/route/zsys_netbsd.go index aa4aad1613e..e0df45e8b55 100644 --- a/libgo/go/golang_org/x/net/route/zsys_netbsd.go +++ b/libgo/go/golang_org/x/net/route/zsys_netbsd.go @@ -10,6 +10,8 @@ const ( sysAF_LINK = 0x12 sysAF_INET6 = 0x18 + sysSOCK_RAW = 0x3 + sysNET_RT_DUMP = 0x1 sysNET_RT_FLAGS = 0x2 sysNET_RT_IFLIST = 0x5 @@ -88,4 +90,8 @@ const ( sizeofRtMsghdrNetBSD7 = 0x78 sizeofRtMetricsNetBSD7 = 0x50 + + sizeofSockaddrStorage = 0x80 + sizeofSockaddrInet = 0x10 + sizeofSockaddrInet6 = 0x1c ) diff --git a/libgo/go/golang_org/x/net/route/zsys_openbsd.go b/libgo/go/golang_org/x/net/route/zsys_openbsd.go index 4fadc4e8fa1..f5a1ff96731 100644 --- a/libgo/go/golang_org/x/net/route/zsys_openbsd.go +++ b/libgo/go/golang_org/x/net/route/zsys_openbsd.go @@ -10,6 +10,8 @@ const ( sysAF_LINK = 0x12 sysAF_INET6 = 0x18 + sysSOCK_RAW = 0x3 + sysNET_RT_DUMP = 0x1 sysNET_RT_FLAGS = 0x2 sysNET_RT_IFLIST = 0x3 @@ -78,3 +80,11 @@ const ( sysRTAX_LABEL = 0xa sysRTAX_MAX = 0xb ) + +const ( + sizeofRtMsghdr = 0x60 + + sizeofSockaddrStorage = 0x100 + sizeofSockaddrInet = 0x10 + sizeofSockaddrInet6 = 0x1c +) diff --git a/libgo/go/golang_org/x/text/secure/bidirule/bidirule.go b/libgo/go/golang_org/x/text/secure/bidirule/bidirule.go new file mode 100644 index 00000000000..9f9594e6179 --- /dev/null +++ b/libgo/go/golang_org/x/text/secure/bidirule/bidirule.go @@ -0,0 +1,344 @@ +// Code generated by running "go run gen.go -core" in golang.org/x/text. DO NOT EDIT. + +// Copyright 2016 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package bidirule implements the Bidi Rule defined by RFC 5893. +// +// This package is under development. The API may change without notice and +// without preserving backward compatibility. +package bidirule + +import ( + "errors" + "unicode/utf8" + + "golang_org/x/text/transform" + "golang_org/x/text/unicode/bidi" +) + +// This file contains an implementation of RFC 5893: Right-to-Left Scripts for +// Internationalized Domain Names for Applications (IDNA) +// +// A label is an individual component of a domain name. Labels are usually +// shown separated by dots; for example, the domain name "www.example.com" is +// composed of three labels: "www", "example", and "com". +// +// An RTL label is a label that contains at least one character of class R, AL, +// or AN. An LTR label is any label that is not an RTL label. +// +// A "Bidi domain name" is a domain name that contains at least one RTL label. +// +// The following guarantees can be made based on the above: +// +// o In a domain name consisting of only labels that satisfy the rule, +// the requirements of Section 3 are satisfied. Note that even LTR +// labels and pure ASCII labels have to be tested. +// +// o In a domain name consisting of only LDH labels (as defined in the +// Definitions document [RFC5890]) and labels that satisfy the rule, +// the requirements of Section 3 are satisfied as long as a label +// that starts with an ASCII digit does not come after a +// right-to-left label. +// +// No guarantee is given for other combinations. + +// ErrInvalid indicates a label is invalid according to the Bidi Rule. +var ErrInvalid = errors.New("bidirule: failed Bidi Rule") + +type ruleState uint8 + +const ( + ruleInitial ruleState = iota + ruleLTR + ruleLTRFinal + ruleRTL + ruleRTLFinal + ruleInvalid +) + +type ruleTransition struct { + next ruleState + mask uint16 +} + +var transitions = [...][2]ruleTransition{ + // [2.1] The first character must be a character with Bidi property L, R, or + // AL. If it has the R or AL property, it is an RTL label; if it has the L + // property, it is an LTR label. + ruleInitial: { + {ruleLTRFinal, 1 << bidi.L}, + {ruleRTLFinal, 1< 0; count-- { + p.openers.Remove(p.openers.Front()) + } + break + } + } + sort.Sort(p.pairPositions) + // if we get here, the closing bracket matched no openers + // and gets ignored + } + } +} + +// Bracket pairs within an isolating run sequence are processed as units so +// that both the opening and the closing paired bracket in a pair resolve to +// the same direction. +// +// N0. Process bracket pairs in an isolating run sequence sequentially in +// the logical order of the text positions of the opening paired brackets +// using the logic given below. Within this scope, bidirectional types EN +// and AN are treated as R. +// +// Identify the bracket pairs in the current isolating run sequence +// according to BD16. For each bracket-pair element in the list of pairs of +// text positions: +// +// a Inspect the bidirectional types of the characters enclosed within the +// bracket pair. +// +// b If any strong type (either L or R) matching the embedding direction is +// found, set the type for both brackets in the pair to match the embedding +// direction. +// +// o [ e ] o -> o e e e o +// +// o [ o e ] -> o e o e e +// +// o [ NI e ] -> o e NI e e +// +// c Otherwise, if a strong type (opposite the embedding direction) is +// found, test for adjacent strong types as follows: 1 First, check +// backwards before the opening paired bracket until the first strong type +// (L, R, or sos) is found. If that first preceding strong type is opposite +// the embedding direction, then set the type for both brackets in the pair +// to that type. 2 Otherwise, set the type for both brackets in the pair to +// the embedding direction. +// +// o [ o ] e -> o o o o e +// +// o [ o NI ] o -> o o o NI o o +// +// e [ o ] o -> e e o e o +// +// e [ o ] e -> e e o e e +// +// e ( o [ o ] NI ) e -> e e o o o o NI e e +// +// d Otherwise, do not set the type for the current bracket pair. Note that +// if the enclosed text contains no strong types the paired brackets will +// both resolve to the same level when resolved individually using rules N1 +// and N2. +// +// e ( NI ) o -> e ( NI ) o + +// getStrongTypeN0 maps character's directional code to strong type as required +// by rule N0. +// +// TODO: have separate type for "strong" directionality. +func (p *bracketPairer) getStrongTypeN0(index int) Class { + switch p.codesIsolatedRun[index] { + // in the scope of N0, number types are treated as R + case EN, AN, AL, R: + return R + case L: + return L + default: + return ON + } +} + +// classifyPairContent reports the strong types contained inside a Bracket Pair, +// assuming the given embedding direction. +// +// It returns ON if no strong type is found. If a single strong type is found, +// it returns this this type. Otherwise it returns the embedding direction. +// +// TODO: use separate type for "strong" directionality. +func (p *bracketPairer) classifyPairContent(loc bracketPair, dirEmbed Class) Class { + dirOpposite := ON + for i := loc.opener + 1; i < loc.closer; i++ { + dir := p.getStrongTypeN0(i) + if dir == ON { + continue + } + if dir == dirEmbed { + return dir // type matching embedding direction found + } + dirOpposite = dir + } + // return ON if no strong type found, or class opposite to dirEmbed + return dirOpposite +} + +// classBeforePair determines which strong types are present before a Bracket +// Pair. Return R or L if strong type found, otherwise ON. +func (p *bracketPairer) classBeforePair(loc bracketPair) Class { + for i := loc.opener - 1; i >= 0; i-- { + if dir := p.getStrongTypeN0(i); dir != ON { + return dir + } + } + // no strong types found, return sos + return p.sos +} + +// assignBracketType implements rule N0 for a single bracket pair. +func (p *bracketPairer) assignBracketType(loc bracketPair, dirEmbed Class, initialTypes []Class) { + // rule "N0, a", inspect contents of pair + dirPair := p.classifyPairContent(loc, dirEmbed) + + // dirPair is now L, R, or N (no strong type found) + + // the following logical tests are performed out of order compared to + // the statement of the rules but yield the same results + if dirPair == ON { + return // case "d" - nothing to do + } + + if dirPair != dirEmbed { + // case "c": strong type found, opposite - check before (c.1) + dirPair = p.classBeforePair(loc) + if dirPair == dirEmbed || dirPair == ON { + // no strong opposite type found before - use embedding (c.2) + dirPair = dirEmbed + } + } + // else: case "b", strong type found matching embedding, + // no explicit action needed, as dirPair is already set to embedding + // direction + + // set the bracket types to the type found + p.setBracketsToType(loc, dirPair, initialTypes) +} + +func (p *bracketPairer) setBracketsToType(loc bracketPair, dirPair Class, initialTypes []Class) { + p.codesIsolatedRun[loc.opener] = dirPair + p.codesIsolatedRun[loc.closer] = dirPair + + for i := loc.opener + 1; i < loc.closer; i++ { + index := p.indexes[i] + if initialTypes[index] != NSM { + break + } + p.codesIsolatedRun[i] = dirPair + } + + for i := loc.closer + 1; i < len(p.indexes); i++ { + index := p.indexes[i] + if initialTypes[index] != NSM { + break + } + p.codesIsolatedRun[i] = dirPair + } +} + +// resolveBrackets implements rule N0 for a list of pairs. +func (p *bracketPairer) resolveBrackets(dirEmbed Class, initialTypes []Class) { + for _, loc := range p.pairPositions { + p.assignBracketType(loc, dirEmbed, initialTypes) + } +} diff --git a/libgo/go/golang_org/x/text/unicode/bidi/core.go b/libgo/go/golang_org/x/text/unicode/bidi/core.go new file mode 100644 index 00000000000..a352ad622a9 --- /dev/null +++ b/libgo/go/golang_org/x/text/unicode/bidi/core.go @@ -0,0 +1,1060 @@ +// Code generated by running "go run gen.go -core" in golang.org/x/text. DO NOT EDIT. + +// Copyright 2015 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package bidi + +import "log" + +// This implementation is a port based on the reference implementation found at: +// http://www.unicode.org/Public/PROGRAMS/BidiReferenceJava/ +// +// described in Unicode Bidirectional Algorithm (UAX #9). +// +// Input: +// There are two levels of input to the algorithm, since clients may prefer to +// supply some information from out-of-band sources rather than relying on the +// default behavior. +// +// - Bidi class array +// - Bidi class array, with externally supplied base line direction +// +// Output: +// Output is separated into several stages: +// +// - levels array over entire paragraph +// - reordering array over entire paragraph +// - levels array over line +// - reordering array over line +// +// Note that for conformance to the Unicode Bidirectional Algorithm, +// implementations are only required to generate correct reordering and +// character directionality (odd or even levels) over a line. Generating +// identical level arrays over a line is not required. Bidi explicit format +// codes (LRE, RLE, LRO, RLO, PDF) and BN can be assigned arbitrary levels and +// positions as long as the rest of the input is properly reordered. +// +// As the algorithm is defined to operate on a single paragraph at a time, this +// implementation is written to handle single paragraphs. Thus rule P1 is +// presumed by this implementation-- the data provided to the implementation is +// assumed to be a single paragraph, and either contains no 'B' codes, or a +// single 'B' code at the end of the input. 'B' is allowed as input to +// illustrate how the algorithm assigns it a level. +// +// Also note that rules L3 and L4 depend on the rendering engine that uses the +// result of the bidi algorithm. This implementation assumes that the rendering +// engine expects combining marks in visual order (e.g. to the left of their +// base character in RTL runs) and that it adjusts the glyphs used to render +// mirrored characters that are in RTL runs so that they render appropriately. + +// level is the embedding level of a character. Even embedding levels indicate +// left-to-right order and odd levels indicate right-to-left order. The special +// level of -1 is reserved for undefined order. +type level int8 + +const implicitLevel level = -1 + +// in returns if x is equal to any of the values in set. +func (c Class) in(set ...Class) bool { + for _, s := range set { + if c == s { + return true + } + } + return false +} + +// A paragraph contains the state of a paragraph. +type paragraph struct { + initialTypes []Class + + // Arrays of properties needed for paired bracket evaluation in N0 + pairTypes []bracketType // paired Bracket types for paragraph + pairValues []rune // rune for opening bracket or pbOpen and pbClose; 0 for pbNone + + embeddingLevel level // default: = implicitLevel; + + // at the paragraph levels + resultTypes []Class + resultLevels []level + + // Index of matching PDI for isolate initiator characters. For other + // characters, the value of matchingPDI will be set to -1. For isolate + // initiators with no matching PDI, matchingPDI will be set to the length of + // the input string. + matchingPDI []int + + // Index of matching isolate initiator for PDI characters. For other + // characters, and for PDIs with no matching isolate initiator, the value of + // matchingIsolateInitiator will be set to -1. + matchingIsolateInitiator []int +} + +// newParagraph initializes a paragraph. The user needs to supply a few arrays +// corresponding to the preprocessed text input. The types correspond to the +// Unicode BiDi classes for each rune. pairTypes indicates the bracket type for +// each rune. pairValues provides a unique bracket class identifier for each +// rune (suggested is the rune of the open bracket for opening and matching +// close brackets, after normalization). The embedding levels are optional, but +// may be supplied to encode embedding levels of styled text. +// +// TODO: return an error. +func newParagraph(types []Class, pairTypes []bracketType, pairValues []rune, levels level) *paragraph { + validateTypes(types) + validatePbTypes(pairTypes) + validatePbValues(pairValues, pairTypes) + validateParagraphEmbeddingLevel(levels) + + p := ¶graph{ + initialTypes: append([]Class(nil), types...), + embeddingLevel: levels, + + pairTypes: pairTypes, + pairValues: pairValues, + + resultTypes: append([]Class(nil), types...), + } + p.run() + return p +} + +func (p *paragraph) Len() int { return len(p.initialTypes) } + +// The algorithm. Does not include line-based processing (Rules L1, L2). +// These are applied later in the line-based phase of the algorithm. +func (p *paragraph) run() { + p.determineMatchingIsolates() + + // 1) determining the paragraph level + // Rule P1 is the requirement for entering this algorithm. + // Rules P2, P3. + // If no externally supplied paragraph embedding level, use default. + if p.embeddingLevel == implicitLevel { + p.embeddingLevel = p.determineParagraphEmbeddingLevel(0, p.Len()) + } + + // Initialize result levels to paragraph embedding level. + p.resultLevels = make([]level, p.Len()) + setLevels(p.resultLevels, p.embeddingLevel) + + // 2) Explicit levels and directions + // Rules X1-X8. + p.determineExplicitEmbeddingLevels() + + // Rule X9. + // We do not remove the embeddings, the overrides, the PDFs, and the BNs + // from the string explicitly. But they are not copied into isolating run + // sequences when they are created, so they are removed for all + // practical purposes. + + // Rule X10. + // Run remainder of algorithm one isolating run sequence at a time + for _, seq := range p.determineIsolatingRunSequences() { + // 3) resolving weak types + // Rules W1-W7. + seq.resolveWeakTypes() + + // 4a) resolving paired brackets + // Rule N0 + resolvePairedBrackets(seq) + + // 4b) resolving neutral types + // Rules N1-N3. + seq.resolveNeutralTypes() + + // 5) resolving implicit embedding levels + // Rules I1, I2. + seq.resolveImplicitLevels() + + // Apply the computed levels and types + seq.applyLevelsAndTypes() + } + + // Assign appropriate levels to 'hide' LREs, RLEs, LROs, RLOs, PDFs, and + // BNs. This is for convenience, so the resulting level array will have + // a value for every character. + p.assignLevelsToCharactersRemovedByX9() +} + +// determineMatchingIsolates determines the matching PDI for each isolate +// initiator and vice versa. +// +// Definition BD9. +// +// At the end of this function: +// +// - The member variable matchingPDI is set to point to the index of the +// matching PDI character for each isolate initiator character. If there is +// no matching PDI, it is set to the length of the input text. For other +// characters, it is set to -1. +// - The member variable matchingIsolateInitiator is set to point to the +// index of the matching isolate initiator character for each PDI character. +// If there is no matching isolate initiator, or the character is not a PDI, +// it is set to -1. +func (p *paragraph) determineMatchingIsolates() { + p.matchingPDI = make([]int, p.Len()) + p.matchingIsolateInitiator = make([]int, p.Len()) + + for i := range p.matchingIsolateInitiator { + p.matchingIsolateInitiator[i] = -1 + } + + for i := range p.matchingPDI { + p.matchingPDI[i] = -1 + + if t := p.resultTypes[i]; t.in(LRI, RLI, FSI) { + depthCounter := 1 + for j := i + 1; j < p.Len(); j++ { + if u := p.resultTypes[j]; u.in(LRI, RLI, FSI) { + depthCounter++ + } else if u == PDI { + if depthCounter--; depthCounter == 0 { + p.matchingPDI[i] = j + p.matchingIsolateInitiator[j] = i + break + } + } + } + if p.matchingPDI[i] == -1 { + p.matchingPDI[i] = p.Len() + } + } + } +} + +// determineParagraphEmbeddingLevel reports the resolved paragraph direction of +// the substring limited by the given range [start, end). +// +// Determines the paragraph level based on rules P2, P3. This is also used +// in rule X5c to find if an FSI should resolve to LRI or RLI. +func (p *paragraph) determineParagraphEmbeddingLevel(start, end int) level { + var strongType Class = unknownClass + + // Rule P2. + for i := start; i < end; i++ { + if t := p.resultTypes[i]; t.in(L, AL, R) { + strongType = t + break + } else if t.in(FSI, LRI, RLI) { + i = p.matchingPDI[i] // skip over to the matching PDI + if i > end { + log.Panic("assert (i <= end)") + } + } + } + // Rule P3. + switch strongType { + case unknownClass: // none found + // default embedding level when no strong types found is 0. + return 0 + case L: + return 0 + default: // AL, R + return 1 + } +} + +const maxDepth = 125 + +// This stack will store the embedding levels and override and isolated +// statuses +type directionalStatusStack struct { + stackCounter int + embeddingLevelStack [maxDepth + 1]level + overrideStatusStack [maxDepth + 1]Class + isolateStatusStack [maxDepth + 1]bool +} + +func (s *directionalStatusStack) empty() { s.stackCounter = 0 } +func (s *directionalStatusStack) pop() { s.stackCounter-- } +func (s *directionalStatusStack) depth() int { return s.stackCounter } + +func (s *directionalStatusStack) push(level level, overrideStatus Class, isolateStatus bool) { + s.embeddingLevelStack[s.stackCounter] = level + s.overrideStatusStack[s.stackCounter] = overrideStatus + s.isolateStatusStack[s.stackCounter] = isolateStatus + s.stackCounter++ +} + +func (s *directionalStatusStack) lastEmbeddingLevel() level { + return s.embeddingLevelStack[s.stackCounter-1] +} + +func (s *directionalStatusStack) lastDirectionalOverrideStatus() Class { + return s.overrideStatusStack[s.stackCounter-1] +} + +func (s *directionalStatusStack) lastDirectionalIsolateStatus() bool { + return s.isolateStatusStack[s.stackCounter-1] +} + +// Determine explicit levels using rules X1 - X8 +func (p *paragraph) determineExplicitEmbeddingLevels() { + var stack directionalStatusStack + var overflowIsolateCount, overflowEmbeddingCount, validIsolateCount int + + // Rule X1. + stack.push(p.embeddingLevel, ON, false) + + for i, t := range p.resultTypes { + // Rules X2, X3, X4, X5, X5a, X5b, X5c + switch t { + case RLE, LRE, RLO, LRO, RLI, LRI, FSI: + isIsolate := t.in(RLI, LRI, FSI) + isRTL := t.in(RLE, RLO, RLI) + + // override if this is an FSI that resolves to RLI + if t == FSI { + isRTL = (p.determineParagraphEmbeddingLevel(i+1, p.matchingPDI[i]) == 1) + } + if isIsolate { + p.resultLevels[i] = stack.lastEmbeddingLevel() + if stack.lastDirectionalOverrideStatus() != ON { + p.resultTypes[i] = stack.lastDirectionalOverrideStatus() + } + } + + var newLevel level + if isRTL { + // least greater odd + newLevel = (stack.lastEmbeddingLevel() + 1) | 1 + } else { + // least greater even + newLevel = (stack.lastEmbeddingLevel() + 2) &^ 1 + } + + if newLevel <= maxDepth && overflowIsolateCount == 0 && overflowEmbeddingCount == 0 { + if isIsolate { + validIsolateCount++ + } + // Push new embedding level, override status, and isolated + // status. + // No check for valid stack counter, since the level check + // suffices. + switch t { + case LRO: + stack.push(newLevel, L, isIsolate) + case RLO: + stack.push(newLevel, R, isIsolate) + default: + stack.push(newLevel, ON, isIsolate) + } + // Not really part of the spec + if !isIsolate { + p.resultLevels[i] = newLevel + } + } else { + // This is an invalid explicit formatting character, + // so apply the "Otherwise" part of rules X2-X5b. + if isIsolate { + overflowIsolateCount++ + } else { // !isIsolate + if overflowIsolateCount == 0 { + overflowEmbeddingCount++ + } + } + } + + // Rule X6a + case PDI: + if overflowIsolateCount > 0 { + overflowIsolateCount-- + } else if validIsolateCount == 0 { + // do nothing + } else { + overflowEmbeddingCount = 0 + for !stack.lastDirectionalIsolateStatus() { + stack.pop() + } + stack.pop() + validIsolateCount-- + } + p.resultLevels[i] = stack.lastEmbeddingLevel() + + // Rule X7 + case PDF: + // Not really part of the spec + p.resultLevels[i] = stack.lastEmbeddingLevel() + + if overflowIsolateCount > 0 { + // do nothing + } else if overflowEmbeddingCount > 0 { + overflowEmbeddingCount-- + } else if !stack.lastDirectionalIsolateStatus() && stack.depth() >= 2 { + stack.pop() + } + + case B: // paragraph separator. + // Rule X8. + + // These values are reset for clarity, in this implementation B + // can only occur as the last code in the array. + stack.empty() + overflowIsolateCount = 0 + overflowEmbeddingCount = 0 + validIsolateCount = 0 + p.resultLevels[i] = p.embeddingLevel + + default: + p.resultLevels[i] = stack.lastEmbeddingLevel() + if stack.lastDirectionalOverrideStatus() != ON { + p.resultTypes[i] = stack.lastDirectionalOverrideStatus() + } + } + } +} + +type isolatingRunSequence struct { + p *paragraph + + indexes []int // indexes to the original string + + types []Class // type of each character using the index + resolvedLevels []level // resolved levels after application of rules + level level + sos, eos Class +} + +func (i *isolatingRunSequence) Len() int { return len(i.indexes) } + +func maxLevel(a, b level) level { + if a > b { + return a + } + return b +} + +// Rule X10, second bullet: Determine the start-of-sequence (sos) and end-of-sequence (eos) types, +// either L or R, for each isolating run sequence. +func (p *paragraph) isolatingRunSequence(indexes []int) *isolatingRunSequence { + length := len(indexes) + types := make([]Class, length) + for i, x := range indexes { + types[i] = p.resultTypes[x] + } + + // assign level, sos and eos + prevChar := indexes[0] - 1 + for prevChar >= 0 && isRemovedByX9(p.initialTypes[prevChar]) { + prevChar-- + } + prevLevel := p.embeddingLevel + if prevChar >= 0 { + prevLevel = p.resultLevels[prevChar] + } + + var succLevel level + lastType := types[length-1] + if lastType.in(LRI, RLI, FSI) { + succLevel = p.embeddingLevel + } else { + // the first character after the end of run sequence + limit := indexes[length-1] + 1 + for ; limit < p.Len() && isRemovedByX9(p.initialTypes[limit]); limit++ { + + } + succLevel = p.embeddingLevel + if limit < p.Len() { + succLevel = p.resultLevels[limit] + } + } + level := p.resultLevels[indexes[0]] + return &isolatingRunSequence{ + p: p, + indexes: indexes, + types: types, + level: level, + sos: typeForLevel(maxLevel(prevLevel, level)), + eos: typeForLevel(maxLevel(succLevel, level)), + } +} + +// Resolving weak types Rules W1-W7. +// +// Note that some weak types (EN, AN) remain after this processing is +// complete. +func (s *isolatingRunSequence) resolveWeakTypes() { + + // on entry, only these types remain + s.assertOnly(L, R, AL, EN, ES, ET, AN, CS, B, S, WS, ON, NSM, LRI, RLI, FSI, PDI) + + // Rule W1. + // Changes all NSMs. + preceedingCharacterType := s.sos + for i, t := range s.types { + if t == NSM { + s.types[i] = preceedingCharacterType + } else { + if t.in(LRI, RLI, FSI, PDI) { + preceedingCharacterType = ON + } + preceedingCharacterType = t + } + } + + // Rule W2. + // EN does not change at the start of the run, because sos != AL. + for i, t := range s.types { + if t == EN { + for j := i - 1; j >= 0; j-- { + if t := s.types[j]; t.in(L, R, AL) { + if t == AL { + s.types[i] = AN + } + break + } + } + } + } + + // Rule W3. + for i, t := range s.types { + if t == AL { + s.types[i] = R + } + } + + // Rule W4. + // Since there must be values on both sides for this rule to have an + // effect, the scan skips the first and last value. + // + // Although the scan proceeds left to right, and changes the type + // values in a way that would appear to affect the computations + // later in the scan, there is actually no problem. A change in the + // current value can only affect the value to its immediate right, + // and only affect it if it is ES or CS. But the current value can + // only change if the value to its right is not ES or CS. Thus + // either the current value will not change, or its change will have + // no effect on the remainder of the analysis. + + for i := 1; i < s.Len()-1; i++ { + t := s.types[i] + if t == ES || t == CS { + prevSepType := s.types[i-1] + succSepType := s.types[i+1] + if prevSepType == EN && succSepType == EN { + s.types[i] = EN + } else if s.types[i] == CS && prevSepType == AN && succSepType == AN { + s.types[i] = AN + } + } + } + + // Rule W5. + for i, t := range s.types { + if t == ET { + // locate end of sequence + runStart := i + runEnd := s.findRunLimit(runStart, ET) + + // check values at ends of sequence + t := s.sos + if runStart > 0 { + t = s.types[runStart-1] + } + if t != EN { + t = s.eos + if runEnd < len(s.types) { + t = s.types[runEnd] + } + } + if t == EN { + setTypes(s.types[runStart:runEnd], EN) + } + // continue at end of sequence + i = runEnd + } + } + + // Rule W6. + for i, t := range s.types { + if t.in(ES, ET, CS) { + s.types[i] = ON + } + } + + // Rule W7. + for i, t := range s.types { + if t == EN { + // set default if we reach start of run + prevStrongType := s.sos + for j := i - 1; j >= 0; j-- { + t = s.types[j] + if t == L || t == R { // AL's have been changed to R + prevStrongType = t + break + } + } + if prevStrongType == L { + s.types[i] = L + } + } + } +} + +// 6) resolving neutral types Rules N1-N2. +func (s *isolatingRunSequence) resolveNeutralTypes() { + + // on entry, only these types can be in resultTypes + s.assertOnly(L, R, EN, AN, B, S, WS, ON, RLI, LRI, FSI, PDI) + + for i, t := range s.types { + switch t { + case WS, ON, B, S, RLI, LRI, FSI, PDI: + // find bounds of run of neutrals + runStart := i + runEnd := s.findRunLimit(runStart, B, S, WS, ON, RLI, LRI, FSI, PDI) + + // determine effective types at ends of run + var leadType, trailType Class + + // Note that the character found can only be L, R, AN, or + // EN. + if runStart == 0 { + leadType = s.sos + } else { + leadType = s.types[runStart-1] + if leadType.in(AN, EN) { + leadType = R + } + } + if runEnd == len(s.types) { + trailType = s.eos + } else { + trailType = s.types[runEnd] + if trailType.in(AN, EN) { + trailType = R + } + } + + var resolvedType Class + if leadType == trailType { + // Rule N1. + resolvedType = leadType + } else { + // Rule N2. + // Notice the embedding level of the run is used, not + // the paragraph embedding level. + resolvedType = typeForLevel(s.level) + } + + setTypes(s.types[runStart:runEnd], resolvedType) + + // skip over run of (former) neutrals + i = runEnd + } + } +} + +func setLevels(levels []level, newLevel level) { + for i := range levels { + levels[i] = newLevel + } +} + +func setTypes(types []Class, newType Class) { + for i := range types { + types[i] = newType + } +} + +// 7) resolving implicit embedding levels Rules I1, I2. +func (s *isolatingRunSequence) resolveImplicitLevels() { + + // on entry, only these types can be in resultTypes + s.assertOnly(L, R, EN, AN) + + s.resolvedLevels = make([]level, len(s.types)) + setLevels(s.resolvedLevels, s.level) + + if (s.level & 1) == 0 { // even level + for i, t := range s.types { + // Rule I1. + if t == L { + // no change + } else if t == R { + s.resolvedLevels[i] += 1 + } else { // t == AN || t == EN + s.resolvedLevels[i] += 2 + } + } + } else { // odd level + for i, t := range s.types { + // Rule I2. + if t == R { + // no change + } else { // t == L || t == AN || t == EN + s.resolvedLevels[i] += 1 + } + } + } +} + +// Applies the levels and types resolved in rules W1-I2 to the +// resultLevels array. +func (s *isolatingRunSequence) applyLevelsAndTypes() { + for i, x := range s.indexes { + s.p.resultTypes[x] = s.types[i] + s.p.resultLevels[x] = s.resolvedLevels[i] + } +} + +// Return the limit of the run consisting only of the types in validSet +// starting at index. This checks the value at index, and will return +// index if that value is not in validSet. +func (s *isolatingRunSequence) findRunLimit(index int, validSet ...Class) int { +loop: + for ; index < len(s.types); index++ { + t := s.types[index] + for _, valid := range validSet { + if t == valid { + continue loop + } + } + return index // didn't find a match in validSet + } + return len(s.types) +} + +// Algorithm validation. Assert that all values in types are in the +// provided set. +func (s *isolatingRunSequence) assertOnly(codes ...Class) { +loop: + for i, t := range s.types { + for _, c := range codes { + if t == c { + continue loop + } + } + log.Panicf("invalid bidi code %v present in assertOnly at position %d", t, s.indexes[i]) + } +} + +// determineLevelRuns returns an array of level runs. Each level run is +// described as an array of indexes into the input string. +// +// Determines the level runs. Rule X9 will be applied in determining the +// runs, in the way that makes sure the characters that are supposed to be +// removed are not included in the runs. +func (p *paragraph) determineLevelRuns() [][]int { + run := []int{} + allRuns := [][]int{} + currentLevel := implicitLevel + + for i := range p.initialTypes { + if !isRemovedByX9(p.initialTypes[i]) { + if p.resultLevels[i] != currentLevel { + // we just encountered a new run; wrap up last run + if currentLevel >= 0 { // only wrap it up if there was a run + allRuns = append(allRuns, run) + run = nil + } + // Start new run + currentLevel = p.resultLevels[i] + } + run = append(run, i) + } + } + // Wrap up the final run, if any + if len(run) > 0 { + allRuns = append(allRuns, run) + } + return allRuns +} + +// Definition BD13. Determine isolating run sequences. +func (p *paragraph) determineIsolatingRunSequences() []*isolatingRunSequence { + levelRuns := p.determineLevelRuns() + + // Compute the run that each character belongs to + runForCharacter := make([]int, p.Len()) + for i, run := range levelRuns { + for _, index := range run { + runForCharacter[index] = i + } + } + + sequences := []*isolatingRunSequence{} + + var currentRunSequence []int + + for _, run := range levelRuns { + first := run[0] + if p.initialTypes[first] != PDI || p.matchingIsolateInitiator[first] == -1 { + currentRunSequence = nil + // int run = i; + for { + // Copy this level run into currentRunSequence + currentRunSequence = append(currentRunSequence, run...) + + last := currentRunSequence[len(currentRunSequence)-1] + lastT := p.initialTypes[last] + if lastT.in(LRI, RLI, FSI) && p.matchingPDI[last] != p.Len() { + run = levelRuns[runForCharacter[p.matchingPDI[last]]] + } else { + break + } + } + sequences = append(sequences, p.isolatingRunSequence(currentRunSequence)) + } + } + return sequences +} + +// Assign level information to characters removed by rule X9. This is for +// ease of relating the level information to the original input data. Note +// that the levels assigned to these codes are arbitrary, they're chosen so +// as to avoid breaking level runs. +func (p *paragraph) assignLevelsToCharactersRemovedByX9() { + for i, t := range p.initialTypes { + if t.in(LRE, RLE, LRO, RLO, PDF, BN) { + p.resultTypes[i] = t + p.resultLevels[i] = -1 + } + } + // now propagate forward the levels information (could have + // propagated backward, the main thing is not to introduce a level + // break where one doesn't already exist). + + if p.resultLevels[0] == -1 { + p.resultLevels[0] = p.embeddingLevel + } + for i := 1; i < len(p.initialTypes); i++ { + if p.resultLevels[i] == -1 { + p.resultLevels[i] = p.resultLevels[i-1] + } + } + // Embedding information is for informational purposes only so need not be + // adjusted. +} + +// +// Output +// + +// getLevels computes levels array breaking lines at offsets in linebreaks. +// Rule L1. +// +// The linebreaks array must include at least one value. The values must be +// in strictly increasing order (no duplicates) between 1 and the length of +// the text, inclusive. The last value must be the length of the text. +func (p *paragraph) getLevels(linebreaks []int) []level { + // Note that since the previous processing has removed all + // P, S, and WS values from resultTypes, the values referred to + // in these rules are the initial types, before any processing + // has been applied (including processing of overrides). + // + // This example implementation has reinserted explicit format codes + // and BN, in order that the levels array correspond to the + // initial text. Their final placement is not normative. + // These codes are treated like WS in this implementation, + // so they don't interrupt sequences of WS. + + validateLineBreaks(linebreaks, p.Len()) + + result := append([]level(nil), p.resultLevels...) + + // don't worry about linebreaks since if there is a break within + // a series of WS values preceding S, the linebreak itself + // causes the reset. + for i, t := range p.initialTypes { + if t.in(B, S) { + // Rule L1, clauses one and two. + result[i] = p.embeddingLevel + + // Rule L1, clause three. + for j := i - 1; j >= 0; j-- { + if isWhitespace(p.initialTypes[j]) { // including format codes + result[j] = p.embeddingLevel + } else { + break + } + } + } + } + + // Rule L1, clause four. + start := 0 + for _, limit := range linebreaks { + for j := limit - 1; j >= start; j-- { + if isWhitespace(p.initialTypes[j]) { // including format codes + result[j] = p.embeddingLevel + } else { + break + } + } + start = limit + } + + return result +} + +// getReordering returns the reordering of lines from a visual index to a +// logical index for line breaks at the given offsets. +// +// Lines are concatenated from left to right. So for example, the fifth +// character from the left on the third line is +// +// getReordering(linebreaks)[linebreaks[1] + 4] +// +// (linebreaks[1] is the position after the last character of the second +// line, which is also the index of the first character on the third line, +// and adding four gets the fifth character from the left). +// +// The linebreaks array must include at least one value. The values must be +// in strictly increasing order (no duplicates) between 1 and the length of +// the text, inclusive. The last value must be the length of the text. +func (p *paragraph) getReordering(linebreaks []int) []int { + validateLineBreaks(linebreaks, p.Len()) + + return computeMultilineReordering(p.getLevels(linebreaks), linebreaks) +} + +// Return multiline reordering array for a given level array. Reordering +// does not occur across a line break. +func computeMultilineReordering(levels []level, linebreaks []int) []int { + result := make([]int, len(levels)) + + start := 0 + for _, limit := range linebreaks { + tempLevels := make([]level, limit-start) + copy(tempLevels, levels[start:]) + + for j, order := range computeReordering(tempLevels) { + result[start+j] = order + start + } + start = limit + } + return result +} + +// Return reordering array for a given level array. This reorders a single +// line. The reordering is a visual to logical map. For example, the +// leftmost char is string.charAt(order[0]). Rule L2. +func computeReordering(levels []level) []int { + result := make([]int, len(levels)) + // initialize order + for i := range result { + result[i] = i + } + + // locate highest level found on line. + // Note the rules say text, but no reordering across line bounds is + // performed, so this is sufficient. + highestLevel := level(0) + lowestOddLevel := level(maxDepth + 2) + for _, level := range levels { + if level > highestLevel { + highestLevel = level + } + if level&1 != 0 && level < lowestOddLevel { + lowestOddLevel = level + } + } + + for level := highestLevel; level >= lowestOddLevel; level-- { + for i := 0; i < len(levels); i++ { + if levels[i] >= level { + // find range of text at or above this level + start := i + limit := i + 1 + for limit < len(levels) && levels[limit] >= level { + limit++ + } + + for j, k := start, limit-1; j < k; j, k = j+1, k-1 { + result[j], result[k] = result[k], result[j] + } + // skip to end of level run + i = limit + } + } + } + + return result +} + +// isWhitespace reports whether the type is considered a whitespace type for the +// line break rules. +func isWhitespace(c Class) bool { + switch c { + case LRE, RLE, LRO, RLO, PDF, LRI, RLI, FSI, PDI, BN, WS: + return true + } + return false +} + +// isRemovedByX9 reports whether the type is one of the types removed in X9. +func isRemovedByX9(c Class) bool { + switch c { + case LRE, RLE, LRO, RLO, PDF, BN: + return true + } + return false +} + +// typeForLevel reports the strong type (L or R) corresponding to the level. +func typeForLevel(level level) Class { + if (level & 0x1) == 0 { + return L + } + return R +} + +// TODO: change validation to not panic + +func validateTypes(types []Class) { + if len(types) == 0 { + log.Panic("types is null") + } + for i, t := range types[:len(types)-1] { + if t == B { + log.Panicf("B type before end of paragraph at index: %d", i) + } + } +} + +func validateParagraphEmbeddingLevel(embeddingLevel level) { + if embeddingLevel != implicitLevel && + embeddingLevel != 0 && + embeddingLevel != 1 { + log.Panicf("illegal paragraph embedding level: %d", embeddingLevel) + } +} + +func validateLineBreaks(linebreaks []int, textLength int) { + prev := 0 + for i, next := range linebreaks { + if next <= prev { + log.Panicf("bad linebreak: %d at index: %d", next, i) + } + prev = next + } + if prev != textLength { + log.Panicf("last linebreak was %d, want %d", prev, textLength) + } +} + +func validatePbTypes(pairTypes []bracketType) { + if len(pairTypes) == 0 { + log.Panic("pairTypes is null") + } + for i, pt := range pairTypes { + switch pt { + case bpNone, bpOpen, bpClose: + default: + log.Panicf("illegal pairType value at %d: %v", i, pairTypes[i]) + } + } +} + +func validatePbValues(pairValues []rune, pairTypes []bracketType) { + if pairValues == nil { + log.Panic("pairValues is null") + } + if len(pairTypes) != len(pairValues) { + log.Panic("pairTypes is different length from pairValues") + } +} diff --git a/libgo/go/golang_org/x/text/unicode/bidi/example_test.go b/libgo/go/golang_org/x/text/unicode/bidi/example_test.go new file mode 100644 index 00000000000..e1739598d43 --- /dev/null +++ b/libgo/go/golang_org/x/text/unicode/bidi/example_test.go @@ -0,0 +1,185 @@ +// Code generated by running "go run gen.go -core" in golang.org/x/text. DO NOT EDIT. + +// +build ignore + +package bidi_test + +import ( + "fmt" + "log" + + "golang_org/x/text/bidi" +) + +func foo() { + var sa StringAttributes + var p Paragraph + n, _ := p.SetString(s) + for i, o := 0, p.Ordering(); i < o.NumRuns(); i++ { + b := o.Run(i).Bytes() + + start, end := o.Run(i).Pos() + for p := start; p < end; { + style, n := sa.StyleAt(start) + render() + p += n + } + + } +} + +type style int + +const ( + styleNormal = 0 + styleSelected = 1 << (iota - 1) + styleBold + styleItalics +) + +type styleRun struct { + end int + style style +} + +func getTextWidth(text string, styleRuns []styleRun) int { + // simplistic way to compute the width + return len([]rune(text)) +} + +// set limit and StyleRun limit for a line +// from text[start] and from styleRuns[styleRunStart] +// using Bidi.getLogicalRun(...) +// returns line width +func getLineBreak(p *bidi.Paragraph, start int, styles []styleRun) (n int) { + // dummy return + return 0 +} + +// render runs on a line sequentially, always from left to right + +// prepare rendering a new line +func startLine(d bidi.Direction, lineWidth int) { + fmt.Println() +} + +// render a run of text and advance to the right by the run width +// the text[start..limit-1] is always in logical order +func renderRun(text string, d bidi.Direction, styl style) { +} + +// We could compute a cross-product +// from the style runs with the directional runs +// and then reorder it. +// Instead, here we iterate over each run type +// and render the intersections - +// with shortcuts in simple (and common) cases. +// renderParagraph() is the main function. + +// render a directional run with +// (possibly) multiple style runs intersecting with it +func renderDirectionalRun(text string, offset int, d bidi.Direction, styles []styleRun) { + start, end := offset, len(text)+offset + // iterate over style runs + if run.Direction() == bidi.LeftToRight { + styleEnd := 0 + for _, sr := range styles { + styleEnd = styleRuns[i].end + if start < styleEnd { + if styleEnd > end { + styleEnd = end + } + renderRun(text[start-offset:styleEnd-offset], run.Direction(), styles[i].style) + if styleEnd == end { + break + } + start = styleEnd + } + } + } else { + styleStart := 0 + for i := len(styles) - 1; i >= 0; i-- { + if i > 0 { + styleStart = styles[i-1].end + } else { + styleStart = 0 + } + if end >= styleStart { + if styleStart < start { + styleStart = start + } + renderRun(text[styleStart-offset:end-offset], run.Direction(), styles[i].style) + if styleStart == start { + break + } + end = styleStart + } + } + } +} + +// the line object represents text[start..limit-1] +func renderLine(line *bidi.Runs, text string, offset int, styles []styleRun) { + if dir := line.Direction(); dir != bidi.Mixed { + if len(styles) == 1 { + renderRun(text, dir, styles[0].style) + } else { + for i := 0; i < line.NumRuns(); i++ { + renderDirectionalRun(text, offset, dir, styles) + } + } + } else { + // iterate over both directional and style runs + for i := 0; i < line.Len(); i++ { + run := line.Run(i) + start, _ := run.Pos() + renderDirectionalRun(text[start-offset:], start, run.Direction(), styles) + } + } +} + +func renderParagraph(text string, d bidi.Direction, styles []styleRun, int lineWidth) { + var p bidi.Paragraph + if err := p.SetString(text, bidi.DefaultDirection(d)); err != nil { + log.Fatal(err) + } + + if len(styles) == 0 { + styles = append(styles, []styleRun{len(text), styleNormal}) + } + + if width := getTextWidth(text, styles); width <= lineWidth { + // everything fits onto one line + + runs, err := p.Runs() + if err != nil { + log.Fatal(err) + } + + // prepare rendering a new line from either left or right + startLine(p.Direction(), width) + renderLine(&runs, text, styles) + } else { + // we need to render several lines + + for start, end := 0, 0; start < len(text); start = end { + for start >= styles[0].end { + styles = styles[1:] + } + end = getLineBreak(p, start, styles[startStyles:]) + + runs, err := p.Line(start, end) + if err != nil { + log.Fatal(err) + } + + startLine(p.Direction(), end-start) + renderLine(&runs, text[start:end], styles[startStyles:]) + } + } +} + +func main() { + renderParagraph("Some Latin text...", bidi.LeftToRight, nil, 80) + renderParagraph("Some Hebrew text...", bidi.RightToLeft, nil, 60) +} diff --git a/libgo/go/golang_org/x/text/unicode/bidi/prop.go b/libgo/go/golang_org/x/text/unicode/bidi/prop.go new file mode 100644 index 00000000000..ed191c220d3 --- /dev/null +++ b/libgo/go/golang_org/x/text/unicode/bidi/prop.go @@ -0,0 +1,208 @@ +// Code generated by running "go run gen.go -core" in golang.org/x/text. DO NOT EDIT. + +// Copyright 2016 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package bidi + +import "unicode/utf8" + +// Properties provides access to BiDi properties of runes. +type Properties struct { + entry uint8 + last uint8 +} + +var trie = newBidiTrie(0) + +// TODO: using this for bidirule reduces the running time by about 5%. Consider +// if this is worth exposing or if we can find a way to speed up the Class +// method. +// +// // CompactClass is like Class, but maps all of the BiDi control classes +// // (LRO, RLO, LRE, RLE, PDF, LRI, RLI, FSI, PDI) to the class Control. +// func (p Properties) CompactClass() Class { +// return Class(p.entry & 0x0F) +// } + +// Class returns the Bidi class for p. +func (p Properties) Class() Class { + c := Class(p.entry & 0x0F) + if c == Control { + c = controlByteToClass[p.last&0xF] + } + return c +} + +// IsBracket reports whether the rune is a bracket. +func (p Properties) IsBracket() bool { return p.entry&0xF0 != 0 } + +// IsOpeningBracket reports whether the rune is an opening bracket. +// IsBracket must return true. +func (p Properties) IsOpeningBracket() bool { return p.entry&openMask != 0 } + +// TODO: find a better API and expose. +func (p Properties) reverseBracket(r rune) rune { + return xorMasks[p.entry>>xorMaskShift] ^ r +} + +var controlByteToClass = [16]Class{ + 0xD: LRO, // U+202D LeftToRightOverride, + 0xE: RLO, // U+202E RightToLeftOverride, + 0xA: LRE, // U+202A LeftToRightEmbedding, + 0xB: RLE, // U+202B RightToLeftEmbedding, + 0xC: PDF, // U+202C PopDirectionalFormat, + 0x6: LRI, // U+2066 LeftToRightIsolate, + 0x7: RLI, // U+2067 RightToLeftIsolate, + 0x8: FSI, // U+2068 FirstStrongIsolate, + 0x9: PDI, // U+2069 PopDirectionalIsolate, +} + +// LookupRune returns properties for r. +func LookupRune(r rune) (p Properties, size int) { + var buf [4]byte + n := utf8.EncodeRune(buf[:], r) + return Lookup(buf[:n]) +} + +// TODO: these lookup methods are based on the generated trie code. The returned +// sizes have slightly different semantics from the generated code, in that it +// always returns size==1 for an illegal UTF-8 byte (instead of the length +// of the maximum invalid subsequence). Most Transformers, like unicode/norm, +// leave invalid UTF-8 untouched, in which case it has performance benefits to +// do so (without changing the semantics). Bidi requires the semantics used here +// for the bidirule implementation to be compatible with the Go semantics. +// They ultimately should perhaps be adopted by all trie implementations, for +// convenience sake. +// This unrolled code also boosts performance of the secure/bidirule package by +// about 30%. +// So, to remove this code: +// - add option to trie generator to define return type. +// - always return 1 byte size for ill-formed UTF-8 runes. + +// Lookup returns properties for the first rune in s and the width in bytes of +// its encoding. The size will be 0 if s does not hold enough bytes to complete +// the encoding. +func Lookup(s []byte) (p Properties, sz int) { + c0 := s[0] + switch { + case c0 < 0x80: // is ASCII + return Properties{entry: bidiValues[c0]}, 1 + case c0 < 0xC2: + return Properties{}, 1 + case c0 < 0xE0: // 2-byte UTF-8 + if len(s) < 2 { + return Properties{}, 0 + } + i := bidiIndex[c0] + c1 := s[1] + if c1 < 0x80 || 0xC0 <= c1 { + return Properties{}, 1 + } + return Properties{entry: trie.lookupValue(uint32(i), c1)}, 2 + case c0 < 0xF0: // 3-byte UTF-8 + if len(s) < 3 { + return Properties{}, 0 + } + i := bidiIndex[c0] + c1 := s[1] + if c1 < 0x80 || 0xC0 <= c1 { + return Properties{}, 1 + } + o := uint32(i)<<6 + uint32(c1) + i = bidiIndex[o] + c2 := s[2] + if c2 < 0x80 || 0xC0 <= c2 { + return Properties{}, 1 + } + return Properties{entry: trie.lookupValue(uint32(i), c2), last: c2}, 3 + case c0 < 0xF8: // 4-byte UTF-8 + if len(s) < 4 { + return Properties{}, 0 + } + i := bidiIndex[c0] + c1 := s[1] + if c1 < 0x80 || 0xC0 <= c1 { + return Properties{}, 1 + } + o := uint32(i)<<6 + uint32(c1) + i = bidiIndex[o] + c2 := s[2] + if c2 < 0x80 || 0xC0 <= c2 { + return Properties{}, 1 + } + o = uint32(i)<<6 + uint32(c2) + i = bidiIndex[o] + c3 := s[3] + if c3 < 0x80 || 0xC0 <= c3 { + return Properties{}, 1 + } + return Properties{entry: trie.lookupValue(uint32(i), c3)}, 4 + } + // Illegal rune + return Properties{}, 1 +} + +// LookupString returns properties for the first rune in s and the width in +// bytes of its encoding. The size will be 0 if s does not hold enough bytes to +// complete the encoding. +func LookupString(s string) (p Properties, sz int) { + c0 := s[0] + switch { + case c0 < 0x80: // is ASCII + return Properties{entry: bidiValues[c0]}, 1 + case c0 < 0xC2: + return Properties{}, 1 + case c0 < 0xE0: // 2-byte UTF-8 + if len(s) < 2 { + return Properties{}, 0 + } + i := bidiIndex[c0] + c1 := s[1] + if c1 < 0x80 || 0xC0 <= c1 { + return Properties{}, 1 + } + return Properties{entry: trie.lookupValue(uint32(i), c1)}, 2 + case c0 < 0xF0: // 3-byte UTF-8 + if len(s) < 3 { + return Properties{}, 0 + } + i := bidiIndex[c0] + c1 := s[1] + if c1 < 0x80 || 0xC0 <= c1 { + return Properties{}, 1 + } + o := uint32(i)<<6 + uint32(c1) + i = bidiIndex[o] + c2 := s[2] + if c2 < 0x80 || 0xC0 <= c2 { + return Properties{}, 1 + } + return Properties{entry: trie.lookupValue(uint32(i), c2), last: c2}, 3 + case c0 < 0xF8: // 4-byte UTF-8 + if len(s) < 4 { + return Properties{}, 0 + } + i := bidiIndex[c0] + c1 := s[1] + if c1 < 0x80 || 0xC0 <= c1 { + return Properties{}, 1 + } + o := uint32(i)<<6 + uint32(c1) + i = bidiIndex[o] + c2 := s[2] + if c2 < 0x80 || 0xC0 <= c2 { + return Properties{}, 1 + } + o = uint32(i)<<6 + uint32(c2) + i = bidiIndex[o] + c3 := s[3] + if c3 < 0x80 || 0xC0 <= c3 { + return Properties{}, 1 + } + return Properties{entry: trie.lookupValue(uint32(i), c3)}, 4 + } + // Illegal rune + return Properties{}, 1 +} diff --git a/libgo/go/golang_org/x/text/unicode/bidi/tables.go b/libgo/go/golang_org/x/text/unicode/bidi/tables.go new file mode 100644 index 00000000000..124531574c3 --- /dev/null +++ b/libgo/go/golang_org/x/text/unicode/bidi/tables.go @@ -0,0 +1,1781 @@ +// Code generated by running "go run gen.go -core" in golang.org/x/text. DO NOT EDIT. + +// Code generated by running "go generate" in golang_org/x/text. DO NOT EDIT. + +package bidi + +// UnicodeVersion is the Unicode version from which the tables in this package are derived. +const UnicodeVersion = "9.0.0" + +// xorMasks contains masks to be xor-ed with brackets to get the reverse +// version. +var xorMasks = []int32{ // 8 elements + 0, 1, 6, 7, 3, 15, 29, 63, +} // Size: 56 bytes + +// lookup returns the trie value for the first UTF-8 encoding in s and +// the width in bytes of this encoding. The size will be 0 if s does not +// hold enough bytes to complete the encoding. len(s) must be greater than 0. +func (t *bidiTrie) lookup(s []byte) (v uint8, sz int) { + c0 := s[0] + switch { + case c0 < 0x80: // is ASCII + return bidiValues[c0], 1 + case c0 < 0xC2: + return 0, 1 // Illegal UTF-8: not a starter, not ASCII. + case c0 < 0xE0: // 2-byte UTF-8 + if len(s) < 2 { + return 0, 0 + } + i := bidiIndex[c0] + c1 := s[1] + if c1 < 0x80 || 0xC0 <= c1 { + return 0, 1 // Illegal UTF-8: not a continuation byte. + } + return t.lookupValue(uint32(i), c1), 2 + case c0 < 0xF0: // 3-byte UTF-8 + if len(s) < 3 { + return 0, 0 + } + i := bidiIndex[c0] + c1 := s[1] + if c1 < 0x80 || 0xC0 <= c1 { + return 0, 1 // Illegal UTF-8: not a continuation byte. + } + o := uint32(i)<<6 + uint32(c1) + i = bidiIndex[o] + c2 := s[2] + if c2 < 0x80 || 0xC0 <= c2 { + return 0, 2 // Illegal UTF-8: not a continuation byte. + } + return t.lookupValue(uint32(i), c2), 3 + case c0 < 0xF8: // 4-byte UTF-8 + if len(s) < 4 { + return 0, 0 + } + i := bidiIndex[c0] + c1 := s[1] + if c1 < 0x80 || 0xC0 <= c1 { + return 0, 1 // Illegal UTF-8: not a continuation byte. + } + o := uint32(i)<<6 + uint32(c1) + i = bidiIndex[o] + c2 := s[2] + if c2 < 0x80 || 0xC0 <= c2 { + return 0, 2 // Illegal UTF-8: not a continuation byte. + } + o = uint32(i)<<6 + uint32(c2) + i = bidiIndex[o] + c3 := s[3] + if c3 < 0x80 || 0xC0 <= c3 { + return 0, 3 // Illegal UTF-8: not a continuation byte. + } + return t.lookupValue(uint32(i), c3), 4 + } + // Illegal rune + return 0, 1 +} + +// lookupUnsafe returns the trie value for the first UTF-8 encoding in s. +// s must start with a full and valid UTF-8 encoded rune. +func (t *bidiTrie) lookupUnsafe(s []byte) uint8 { + c0 := s[0] + if c0 < 0x80 { // is ASCII + return bidiValues[c0] + } + i := bidiIndex[c0] + if c0 < 0xE0 { // 2-byte UTF-8 + return t.lookupValue(uint32(i), s[1]) + } + i = bidiIndex[uint32(i)<<6+uint32(s[1])] + if c0 < 0xF0 { // 3-byte UTF-8 + return t.lookupValue(uint32(i), s[2]) + } + i = bidiIndex[uint32(i)<<6+uint32(s[2])] + if c0 < 0xF8 { // 4-byte UTF-8 + return t.lookupValue(uint32(i), s[3]) + } + return 0 +} + +// lookupString returns the trie value for the first UTF-8 encoding in s and +// the width in bytes of this encoding. The size will be 0 if s does not +// hold enough bytes to complete the encoding. len(s) must be greater than 0. +func (t *bidiTrie) lookupString(s string) (v uint8, sz int) { + c0 := s[0] + switch { + case c0 < 0x80: // is ASCII + return bidiValues[c0], 1 + case c0 < 0xC2: + return 0, 1 // Illegal UTF-8: not a starter, not ASCII. + case c0 < 0xE0: // 2-byte UTF-8 + if len(s) < 2 { + return 0, 0 + } + i := bidiIndex[c0] + c1 := s[1] + if c1 < 0x80 || 0xC0 <= c1 { + return 0, 1 // Illegal UTF-8: not a continuation byte. + } + return t.lookupValue(uint32(i), c1), 2 + case c0 < 0xF0: // 3-byte UTF-8 + if len(s) < 3 { + return 0, 0 + } + i := bidiIndex[c0] + c1 := s[1] + if c1 < 0x80 || 0xC0 <= c1 { + return 0, 1 // Illegal UTF-8: not a continuation byte. + } + o := uint32(i)<<6 + uint32(c1) + i = bidiIndex[o] + c2 := s[2] + if c2 < 0x80 || 0xC0 <= c2 { + return 0, 2 // Illegal UTF-8: not a continuation byte. + } + return t.lookupValue(uint32(i), c2), 3 + case c0 < 0xF8: // 4-byte UTF-8 + if len(s) < 4 { + return 0, 0 + } + i := bidiIndex[c0] + c1 := s[1] + if c1 < 0x80 || 0xC0 <= c1 { + return 0, 1 // Illegal UTF-8: not a continuation byte. + } + o := uint32(i)<<6 + uint32(c1) + i = bidiIndex[o] + c2 := s[2] + if c2 < 0x80 || 0xC0 <= c2 { + return 0, 2 // Illegal UTF-8: not a continuation byte. + } + o = uint32(i)<<6 + uint32(c2) + i = bidiIndex[o] + c3 := s[3] + if c3 < 0x80 || 0xC0 <= c3 { + return 0, 3 // Illegal UTF-8: not a continuation byte. + } + return t.lookupValue(uint32(i), c3), 4 + } + // Illegal rune + return 0, 1 +} + +// lookupStringUnsafe returns the trie value for the first UTF-8 encoding in s. +// s must start with a full and valid UTF-8 encoded rune. +func (t *bidiTrie) lookupStringUnsafe(s string) uint8 { + c0 := s[0] + if c0 < 0x80 { // is ASCII + return bidiValues[c0] + } + i := bidiIndex[c0] + if c0 < 0xE0 { // 2-byte UTF-8 + return t.lookupValue(uint32(i), s[1]) + } + i = bidiIndex[uint32(i)<<6+uint32(s[1])] + if c0 < 0xF0 { // 3-byte UTF-8 + return t.lookupValue(uint32(i), s[2]) + } + i = bidiIndex[uint32(i)<<6+uint32(s[2])] + if c0 < 0xF8 { // 4-byte UTF-8 + return t.lookupValue(uint32(i), s[3]) + } + return 0 +} + +// bidiTrie. Total size: 15744 bytes (15.38 KiB). Checksum: b4c3b70954803b86. +type bidiTrie struct{} + +func newBidiTrie(i int) *bidiTrie { + return &bidiTrie{} +} + +// lookupValue determines the type of block n and looks up the value for b. +func (t *bidiTrie) lookupValue(n uint32, b byte) uint8 { + switch { + default: + return uint8(bidiValues[n<<6+uint32(b)]) + } +} + +// bidiValues: 222 blocks, 14208 entries, 14208 bytes +// The third block is the zero block. +var bidiValues = [14208]uint8{ + // Block 0x0, offset 0x0 + 0x00: 0x000b, 0x01: 0x000b, 0x02: 0x000b, 0x03: 0x000b, 0x04: 0x000b, 0x05: 0x000b, + 0x06: 0x000b, 0x07: 0x000b, 0x08: 0x000b, 0x09: 0x0008, 0x0a: 0x0007, 0x0b: 0x0008, + 0x0c: 0x0009, 0x0d: 0x0007, 0x0e: 0x000b, 0x0f: 0x000b, 0x10: 0x000b, 0x11: 0x000b, + 0x12: 0x000b, 0x13: 0x000b, 0x14: 0x000b, 0x15: 0x000b, 0x16: 0x000b, 0x17: 0x000b, + 0x18: 0x000b, 0x19: 0x000b, 0x1a: 0x000b, 0x1b: 0x000b, 0x1c: 0x0007, 0x1d: 0x0007, + 0x1e: 0x0007, 0x1f: 0x0008, 0x20: 0x0009, 0x21: 0x000a, 0x22: 0x000a, 0x23: 0x0004, + 0x24: 0x0004, 0x25: 0x0004, 0x26: 0x000a, 0x27: 0x000a, 0x28: 0x003a, 0x29: 0x002a, + 0x2a: 0x000a, 0x2b: 0x0003, 0x2c: 0x0006, 0x2d: 0x0003, 0x2e: 0x0006, 0x2f: 0x0006, + 0x30: 0x0002, 0x31: 0x0002, 0x32: 0x0002, 0x33: 0x0002, 0x34: 0x0002, 0x35: 0x0002, + 0x36: 0x0002, 0x37: 0x0002, 0x38: 0x0002, 0x39: 0x0002, 0x3a: 0x0006, 0x3b: 0x000a, + 0x3c: 0x000a, 0x3d: 0x000a, 0x3e: 0x000a, 0x3f: 0x000a, + // Block 0x1, offset 0x40 + 0x40: 0x000a, + 0x5b: 0x005a, 0x5c: 0x000a, 0x5d: 0x004a, + 0x5e: 0x000a, 0x5f: 0x000a, 0x60: 0x000a, + 0x7b: 0x005a, + 0x7c: 0x000a, 0x7d: 0x004a, 0x7e: 0x000a, 0x7f: 0x000b, + // Block 0x2, offset 0x80 + // Block 0x3, offset 0xc0 + 0xc0: 0x000b, 0xc1: 0x000b, 0xc2: 0x000b, 0xc3: 0x000b, 0xc4: 0x000b, 0xc5: 0x0007, + 0xc6: 0x000b, 0xc7: 0x000b, 0xc8: 0x000b, 0xc9: 0x000b, 0xca: 0x000b, 0xcb: 0x000b, + 0xcc: 0x000b, 0xcd: 0x000b, 0xce: 0x000b, 0xcf: 0x000b, 0xd0: 0x000b, 0xd1: 0x000b, + 0xd2: 0x000b, 0xd3: 0x000b, 0xd4: 0x000b, 0xd5: 0x000b, 0xd6: 0x000b, 0xd7: 0x000b, + 0xd8: 0x000b, 0xd9: 0x000b, 0xda: 0x000b, 0xdb: 0x000b, 0xdc: 0x000b, 0xdd: 0x000b, + 0xde: 0x000b, 0xdf: 0x000b, 0xe0: 0x0006, 0xe1: 0x000a, 0xe2: 0x0004, 0xe3: 0x0004, + 0xe4: 0x0004, 0xe5: 0x0004, 0xe6: 0x000a, 0xe7: 0x000a, 0xe8: 0x000a, 0xe9: 0x000a, + 0xeb: 0x000a, 0xec: 0x000a, 0xed: 0x000b, 0xee: 0x000a, 0xef: 0x000a, + 0xf0: 0x0004, 0xf1: 0x0004, 0xf2: 0x0002, 0xf3: 0x0002, 0xf4: 0x000a, + 0xf6: 0x000a, 0xf7: 0x000a, 0xf8: 0x000a, 0xf9: 0x0002, 0xfb: 0x000a, + 0xfc: 0x000a, 0xfd: 0x000a, 0xfe: 0x000a, 0xff: 0x000a, + // Block 0x4, offset 0x100 + 0x117: 0x000a, + 0x137: 0x000a, + // Block 0x5, offset 0x140 + 0x179: 0x000a, 0x17a: 0x000a, + // Block 0x6, offset 0x180 + 0x182: 0x000a, 0x183: 0x000a, 0x184: 0x000a, 0x185: 0x000a, + 0x186: 0x000a, 0x187: 0x000a, 0x188: 0x000a, 0x189: 0x000a, 0x18a: 0x000a, 0x18b: 0x000a, + 0x18c: 0x000a, 0x18d: 0x000a, 0x18e: 0x000a, 0x18f: 0x000a, + 0x192: 0x000a, 0x193: 0x000a, 0x194: 0x000a, 0x195: 0x000a, 0x196: 0x000a, 0x197: 0x000a, + 0x198: 0x000a, 0x199: 0x000a, 0x19a: 0x000a, 0x19b: 0x000a, 0x19c: 0x000a, 0x19d: 0x000a, + 0x19e: 0x000a, 0x19f: 0x000a, + 0x1a5: 0x000a, 0x1a6: 0x000a, 0x1a7: 0x000a, 0x1a8: 0x000a, 0x1a9: 0x000a, + 0x1aa: 0x000a, 0x1ab: 0x000a, 0x1ac: 0x000a, 0x1ad: 0x000a, 0x1af: 0x000a, + 0x1b0: 0x000a, 0x1b1: 0x000a, 0x1b2: 0x000a, 0x1b3: 0x000a, 0x1b4: 0x000a, 0x1b5: 0x000a, + 0x1b6: 0x000a, 0x1b7: 0x000a, 0x1b8: 0x000a, 0x1b9: 0x000a, 0x1ba: 0x000a, 0x1bb: 0x000a, + 0x1bc: 0x000a, 0x1bd: 0x000a, 0x1be: 0x000a, 0x1bf: 0x000a, + // Block 0x7, offset 0x1c0 + 0x1c0: 0x000c, 0x1c1: 0x000c, 0x1c2: 0x000c, 0x1c3: 0x000c, 0x1c4: 0x000c, 0x1c5: 0x000c, + 0x1c6: 0x000c, 0x1c7: 0x000c, 0x1c8: 0x000c, 0x1c9: 0x000c, 0x1ca: 0x000c, 0x1cb: 0x000c, + 0x1cc: 0x000c, 0x1cd: 0x000c, 0x1ce: 0x000c, 0x1cf: 0x000c, 0x1d0: 0x000c, 0x1d1: 0x000c, + 0x1d2: 0x000c, 0x1d3: 0x000c, 0x1d4: 0x000c, 0x1d5: 0x000c, 0x1d6: 0x000c, 0x1d7: 0x000c, + 0x1d8: 0x000c, 0x1d9: 0x000c, 0x1da: 0x000c, 0x1db: 0x000c, 0x1dc: 0x000c, 0x1dd: 0x000c, + 0x1de: 0x000c, 0x1df: 0x000c, 0x1e0: 0x000c, 0x1e1: 0x000c, 0x1e2: 0x000c, 0x1e3: 0x000c, + 0x1e4: 0x000c, 0x1e5: 0x000c, 0x1e6: 0x000c, 0x1e7: 0x000c, 0x1e8: 0x000c, 0x1e9: 0x000c, + 0x1ea: 0x000c, 0x1eb: 0x000c, 0x1ec: 0x000c, 0x1ed: 0x000c, 0x1ee: 0x000c, 0x1ef: 0x000c, + 0x1f0: 0x000c, 0x1f1: 0x000c, 0x1f2: 0x000c, 0x1f3: 0x000c, 0x1f4: 0x000c, 0x1f5: 0x000c, + 0x1f6: 0x000c, 0x1f7: 0x000c, 0x1f8: 0x000c, 0x1f9: 0x000c, 0x1fa: 0x000c, 0x1fb: 0x000c, + 0x1fc: 0x000c, 0x1fd: 0x000c, 0x1fe: 0x000c, 0x1ff: 0x000c, + // Block 0x8, offset 0x200 + 0x200: 0x000c, 0x201: 0x000c, 0x202: 0x000c, 0x203: 0x000c, 0x204: 0x000c, 0x205: 0x000c, + 0x206: 0x000c, 0x207: 0x000c, 0x208: 0x000c, 0x209: 0x000c, 0x20a: 0x000c, 0x20b: 0x000c, + 0x20c: 0x000c, 0x20d: 0x000c, 0x20e: 0x000c, 0x20f: 0x000c, 0x210: 0x000c, 0x211: 0x000c, + 0x212: 0x000c, 0x213: 0x000c, 0x214: 0x000c, 0x215: 0x000c, 0x216: 0x000c, 0x217: 0x000c, + 0x218: 0x000c, 0x219: 0x000c, 0x21a: 0x000c, 0x21b: 0x000c, 0x21c: 0x000c, 0x21d: 0x000c, + 0x21e: 0x000c, 0x21f: 0x000c, 0x220: 0x000c, 0x221: 0x000c, 0x222: 0x000c, 0x223: 0x000c, + 0x224: 0x000c, 0x225: 0x000c, 0x226: 0x000c, 0x227: 0x000c, 0x228: 0x000c, 0x229: 0x000c, + 0x22a: 0x000c, 0x22b: 0x000c, 0x22c: 0x000c, 0x22d: 0x000c, 0x22e: 0x000c, 0x22f: 0x000c, + 0x234: 0x000a, 0x235: 0x000a, + 0x23e: 0x000a, + // Block 0x9, offset 0x240 + 0x244: 0x000a, 0x245: 0x000a, + 0x247: 0x000a, + // Block 0xa, offset 0x280 + 0x2b6: 0x000a, + // Block 0xb, offset 0x2c0 + 0x2c3: 0x000c, 0x2c4: 0x000c, 0x2c5: 0x000c, + 0x2c6: 0x000c, 0x2c7: 0x000c, 0x2c8: 0x000c, 0x2c9: 0x000c, + // Block 0xc, offset 0x300 + 0x30a: 0x000a, + 0x30d: 0x000a, 0x30e: 0x000a, 0x30f: 0x0004, 0x310: 0x0001, 0x311: 0x000c, + 0x312: 0x000c, 0x313: 0x000c, 0x314: 0x000c, 0x315: 0x000c, 0x316: 0x000c, 0x317: 0x000c, + 0x318: 0x000c, 0x319: 0x000c, 0x31a: 0x000c, 0x31b: 0x000c, 0x31c: 0x000c, 0x31d: 0x000c, + 0x31e: 0x000c, 0x31f: 0x000c, 0x320: 0x000c, 0x321: 0x000c, 0x322: 0x000c, 0x323: 0x000c, + 0x324: 0x000c, 0x325: 0x000c, 0x326: 0x000c, 0x327: 0x000c, 0x328: 0x000c, 0x329: 0x000c, + 0x32a: 0x000c, 0x32b: 0x000c, 0x32c: 0x000c, 0x32d: 0x000c, 0x32e: 0x000c, 0x32f: 0x000c, + 0x330: 0x000c, 0x331: 0x000c, 0x332: 0x000c, 0x333: 0x000c, 0x334: 0x000c, 0x335: 0x000c, + 0x336: 0x000c, 0x337: 0x000c, 0x338: 0x000c, 0x339: 0x000c, 0x33a: 0x000c, 0x33b: 0x000c, + 0x33c: 0x000c, 0x33d: 0x000c, 0x33e: 0x0001, 0x33f: 0x000c, + // Block 0xd, offset 0x340 + 0x340: 0x0001, 0x341: 0x000c, 0x342: 0x000c, 0x343: 0x0001, 0x344: 0x000c, 0x345: 0x000c, + 0x346: 0x0001, 0x347: 0x000c, 0x348: 0x0001, 0x349: 0x0001, 0x34a: 0x0001, 0x34b: 0x0001, + 0x34c: 0x0001, 0x34d: 0x0001, 0x34e: 0x0001, 0x34f: 0x0001, 0x350: 0x0001, 0x351: 0x0001, + 0x352: 0x0001, 0x353: 0x0001, 0x354: 0x0001, 0x355: 0x0001, 0x356: 0x0001, 0x357: 0x0001, + 0x358: 0x0001, 0x359: 0x0001, 0x35a: 0x0001, 0x35b: 0x0001, 0x35c: 0x0001, 0x35d: 0x0001, + 0x35e: 0x0001, 0x35f: 0x0001, 0x360: 0x0001, 0x361: 0x0001, 0x362: 0x0001, 0x363: 0x0001, + 0x364: 0x0001, 0x365: 0x0001, 0x366: 0x0001, 0x367: 0x0001, 0x368: 0x0001, 0x369: 0x0001, + 0x36a: 0x0001, 0x36b: 0x0001, 0x36c: 0x0001, 0x36d: 0x0001, 0x36e: 0x0001, 0x36f: 0x0001, + 0x370: 0x0001, 0x371: 0x0001, 0x372: 0x0001, 0x373: 0x0001, 0x374: 0x0001, 0x375: 0x0001, + 0x376: 0x0001, 0x377: 0x0001, 0x378: 0x0001, 0x379: 0x0001, 0x37a: 0x0001, 0x37b: 0x0001, + 0x37c: 0x0001, 0x37d: 0x0001, 0x37e: 0x0001, 0x37f: 0x0001, + // Block 0xe, offset 0x380 + 0x380: 0x0005, 0x381: 0x0005, 0x382: 0x0005, 0x383: 0x0005, 0x384: 0x0005, 0x385: 0x0005, + 0x386: 0x000a, 0x387: 0x000a, 0x388: 0x000d, 0x389: 0x0004, 0x38a: 0x0004, 0x38b: 0x000d, + 0x38c: 0x0006, 0x38d: 0x000d, 0x38e: 0x000a, 0x38f: 0x000a, 0x390: 0x000c, 0x391: 0x000c, + 0x392: 0x000c, 0x393: 0x000c, 0x394: 0x000c, 0x395: 0x000c, 0x396: 0x000c, 0x397: 0x000c, + 0x398: 0x000c, 0x399: 0x000c, 0x39a: 0x000c, 0x39b: 0x000d, 0x39c: 0x000d, 0x39d: 0x000d, + 0x39e: 0x000d, 0x39f: 0x000d, 0x3a0: 0x000d, 0x3a1: 0x000d, 0x3a2: 0x000d, 0x3a3: 0x000d, + 0x3a4: 0x000d, 0x3a5: 0x000d, 0x3a6: 0x000d, 0x3a7: 0x000d, 0x3a8: 0x000d, 0x3a9: 0x000d, + 0x3aa: 0x000d, 0x3ab: 0x000d, 0x3ac: 0x000d, 0x3ad: 0x000d, 0x3ae: 0x000d, 0x3af: 0x000d, + 0x3b0: 0x000d, 0x3b1: 0x000d, 0x3b2: 0x000d, 0x3b3: 0x000d, 0x3b4: 0x000d, 0x3b5: 0x000d, + 0x3b6: 0x000d, 0x3b7: 0x000d, 0x3b8: 0x000d, 0x3b9: 0x000d, 0x3ba: 0x000d, 0x3bb: 0x000d, + 0x3bc: 0x000d, 0x3bd: 0x000d, 0x3be: 0x000d, 0x3bf: 0x000d, + // Block 0xf, offset 0x3c0 + 0x3c0: 0x000d, 0x3c1: 0x000d, 0x3c2: 0x000d, 0x3c3: 0x000d, 0x3c4: 0x000d, 0x3c5: 0x000d, + 0x3c6: 0x000d, 0x3c7: 0x000d, 0x3c8: 0x000d, 0x3c9: 0x000d, 0x3ca: 0x000d, 0x3cb: 0x000c, + 0x3cc: 0x000c, 0x3cd: 0x000c, 0x3ce: 0x000c, 0x3cf: 0x000c, 0x3d0: 0x000c, 0x3d1: 0x000c, + 0x3d2: 0x000c, 0x3d3: 0x000c, 0x3d4: 0x000c, 0x3d5: 0x000c, 0x3d6: 0x000c, 0x3d7: 0x000c, + 0x3d8: 0x000c, 0x3d9: 0x000c, 0x3da: 0x000c, 0x3db: 0x000c, 0x3dc: 0x000c, 0x3dd: 0x000c, + 0x3de: 0x000c, 0x3df: 0x000c, 0x3e0: 0x0005, 0x3e1: 0x0005, 0x3e2: 0x0005, 0x3e3: 0x0005, + 0x3e4: 0x0005, 0x3e5: 0x0005, 0x3e6: 0x0005, 0x3e7: 0x0005, 0x3e8: 0x0005, 0x3e9: 0x0005, + 0x3ea: 0x0004, 0x3eb: 0x0005, 0x3ec: 0x0005, 0x3ed: 0x000d, 0x3ee: 0x000d, 0x3ef: 0x000d, + 0x3f0: 0x000c, 0x3f1: 0x000d, 0x3f2: 0x000d, 0x3f3: 0x000d, 0x3f4: 0x000d, 0x3f5: 0x000d, + 0x3f6: 0x000d, 0x3f7: 0x000d, 0x3f8: 0x000d, 0x3f9: 0x000d, 0x3fa: 0x000d, 0x3fb: 0x000d, + 0x3fc: 0x000d, 0x3fd: 0x000d, 0x3fe: 0x000d, 0x3ff: 0x000d, + // Block 0x10, offset 0x400 + 0x400: 0x000d, 0x401: 0x000d, 0x402: 0x000d, 0x403: 0x000d, 0x404: 0x000d, 0x405: 0x000d, + 0x406: 0x000d, 0x407: 0x000d, 0x408: 0x000d, 0x409: 0x000d, 0x40a: 0x000d, 0x40b: 0x000d, + 0x40c: 0x000d, 0x40d: 0x000d, 0x40e: 0x000d, 0x40f: 0x000d, 0x410: 0x000d, 0x411: 0x000d, + 0x412: 0x000d, 0x413: 0x000d, 0x414: 0x000d, 0x415: 0x000d, 0x416: 0x000d, 0x417: 0x000d, + 0x418: 0x000d, 0x419: 0x000d, 0x41a: 0x000d, 0x41b: 0x000d, 0x41c: 0x000d, 0x41d: 0x000d, + 0x41e: 0x000d, 0x41f: 0x000d, 0x420: 0x000d, 0x421: 0x000d, 0x422: 0x000d, 0x423: 0x000d, + 0x424: 0x000d, 0x425: 0x000d, 0x426: 0x000d, 0x427: 0x000d, 0x428: 0x000d, 0x429: 0x000d, + 0x42a: 0x000d, 0x42b: 0x000d, 0x42c: 0x000d, 0x42d: 0x000d, 0x42e: 0x000d, 0x42f: 0x000d, + 0x430: 0x000d, 0x431: 0x000d, 0x432: 0x000d, 0x433: 0x000d, 0x434: 0x000d, 0x435: 0x000d, + 0x436: 0x000d, 0x437: 0x000d, 0x438: 0x000d, 0x439: 0x000d, 0x43a: 0x000d, 0x43b: 0x000d, + 0x43c: 0x000d, 0x43d: 0x000d, 0x43e: 0x000d, 0x43f: 0x000d, + // Block 0x11, offset 0x440 + 0x440: 0x000d, 0x441: 0x000d, 0x442: 0x000d, 0x443: 0x000d, 0x444: 0x000d, 0x445: 0x000d, + 0x446: 0x000d, 0x447: 0x000d, 0x448: 0x000d, 0x449: 0x000d, 0x44a: 0x000d, 0x44b: 0x000d, + 0x44c: 0x000d, 0x44d: 0x000d, 0x44e: 0x000d, 0x44f: 0x000d, 0x450: 0x000d, 0x451: 0x000d, + 0x452: 0x000d, 0x453: 0x000d, 0x454: 0x000d, 0x455: 0x000d, 0x456: 0x000c, 0x457: 0x000c, + 0x458: 0x000c, 0x459: 0x000c, 0x45a: 0x000c, 0x45b: 0x000c, 0x45c: 0x000c, 0x45d: 0x0005, + 0x45e: 0x000a, 0x45f: 0x000c, 0x460: 0x000c, 0x461: 0x000c, 0x462: 0x000c, 0x463: 0x000c, + 0x464: 0x000c, 0x465: 0x000d, 0x466: 0x000d, 0x467: 0x000c, 0x468: 0x000c, 0x469: 0x000a, + 0x46a: 0x000c, 0x46b: 0x000c, 0x46c: 0x000c, 0x46d: 0x000c, 0x46e: 0x000d, 0x46f: 0x000d, + 0x470: 0x0002, 0x471: 0x0002, 0x472: 0x0002, 0x473: 0x0002, 0x474: 0x0002, 0x475: 0x0002, + 0x476: 0x0002, 0x477: 0x0002, 0x478: 0x0002, 0x479: 0x0002, 0x47a: 0x000d, 0x47b: 0x000d, + 0x47c: 0x000d, 0x47d: 0x000d, 0x47e: 0x000d, 0x47f: 0x000d, + // Block 0x12, offset 0x480 + 0x480: 0x000d, 0x481: 0x000d, 0x482: 0x000d, 0x483: 0x000d, 0x484: 0x000d, 0x485: 0x000d, + 0x486: 0x000d, 0x487: 0x000d, 0x488: 0x000d, 0x489: 0x000d, 0x48a: 0x000d, 0x48b: 0x000d, + 0x48c: 0x000d, 0x48d: 0x000d, 0x48e: 0x000d, 0x48f: 0x000d, 0x490: 0x000d, 0x491: 0x000c, + 0x492: 0x000d, 0x493: 0x000d, 0x494: 0x000d, 0x495: 0x000d, 0x496: 0x000d, 0x497: 0x000d, + 0x498: 0x000d, 0x499: 0x000d, 0x49a: 0x000d, 0x49b: 0x000d, 0x49c: 0x000d, 0x49d: 0x000d, + 0x49e: 0x000d, 0x49f: 0x000d, 0x4a0: 0x000d, 0x4a1: 0x000d, 0x4a2: 0x000d, 0x4a3: 0x000d, + 0x4a4: 0x000d, 0x4a5: 0x000d, 0x4a6: 0x000d, 0x4a7: 0x000d, 0x4a8: 0x000d, 0x4a9: 0x000d, + 0x4aa: 0x000d, 0x4ab: 0x000d, 0x4ac: 0x000d, 0x4ad: 0x000d, 0x4ae: 0x000d, 0x4af: 0x000d, + 0x4b0: 0x000c, 0x4b1: 0x000c, 0x4b2: 0x000c, 0x4b3: 0x000c, 0x4b4: 0x000c, 0x4b5: 0x000c, + 0x4b6: 0x000c, 0x4b7: 0x000c, 0x4b8: 0x000c, 0x4b9: 0x000c, 0x4ba: 0x000c, 0x4bb: 0x000c, + 0x4bc: 0x000c, 0x4bd: 0x000c, 0x4be: 0x000c, 0x4bf: 0x000c, + // Block 0x13, offset 0x4c0 + 0x4c0: 0x000c, 0x4c1: 0x000c, 0x4c2: 0x000c, 0x4c3: 0x000c, 0x4c4: 0x000c, 0x4c5: 0x000c, + 0x4c6: 0x000c, 0x4c7: 0x000c, 0x4c8: 0x000c, 0x4c9: 0x000c, 0x4ca: 0x000c, 0x4cb: 0x000d, + 0x4cc: 0x000d, 0x4cd: 0x000d, 0x4ce: 0x000d, 0x4cf: 0x000d, 0x4d0: 0x000d, 0x4d1: 0x000d, + 0x4d2: 0x000d, 0x4d3: 0x000d, 0x4d4: 0x000d, 0x4d5: 0x000d, 0x4d6: 0x000d, 0x4d7: 0x000d, + 0x4d8: 0x000d, 0x4d9: 0x000d, 0x4da: 0x000d, 0x4db: 0x000d, 0x4dc: 0x000d, 0x4dd: 0x000d, + 0x4de: 0x000d, 0x4df: 0x000d, 0x4e0: 0x000d, 0x4e1: 0x000d, 0x4e2: 0x000d, 0x4e3: 0x000d, + 0x4e4: 0x000d, 0x4e5: 0x000d, 0x4e6: 0x000d, 0x4e7: 0x000d, 0x4e8: 0x000d, 0x4e9: 0x000d, + 0x4ea: 0x000d, 0x4eb: 0x000d, 0x4ec: 0x000d, 0x4ed: 0x000d, 0x4ee: 0x000d, 0x4ef: 0x000d, + 0x4f0: 0x000d, 0x4f1: 0x000d, 0x4f2: 0x000d, 0x4f3: 0x000d, 0x4f4: 0x000d, 0x4f5: 0x000d, + 0x4f6: 0x000d, 0x4f7: 0x000d, 0x4f8: 0x000d, 0x4f9: 0x000d, 0x4fa: 0x000d, 0x4fb: 0x000d, + 0x4fc: 0x000d, 0x4fd: 0x000d, 0x4fe: 0x000d, 0x4ff: 0x000d, + // Block 0x14, offset 0x500 + 0x500: 0x000d, 0x501: 0x000d, 0x502: 0x000d, 0x503: 0x000d, 0x504: 0x000d, 0x505: 0x000d, + 0x506: 0x000d, 0x507: 0x000d, 0x508: 0x000d, 0x509: 0x000d, 0x50a: 0x000d, 0x50b: 0x000d, + 0x50c: 0x000d, 0x50d: 0x000d, 0x50e: 0x000d, 0x50f: 0x000d, 0x510: 0x000d, 0x511: 0x000d, + 0x512: 0x000d, 0x513: 0x000d, 0x514: 0x000d, 0x515: 0x000d, 0x516: 0x000d, 0x517: 0x000d, + 0x518: 0x000d, 0x519: 0x000d, 0x51a: 0x000d, 0x51b: 0x000d, 0x51c: 0x000d, 0x51d: 0x000d, + 0x51e: 0x000d, 0x51f: 0x000d, 0x520: 0x000d, 0x521: 0x000d, 0x522: 0x000d, 0x523: 0x000d, + 0x524: 0x000d, 0x525: 0x000d, 0x526: 0x000c, 0x527: 0x000c, 0x528: 0x000c, 0x529: 0x000c, + 0x52a: 0x000c, 0x52b: 0x000c, 0x52c: 0x000c, 0x52d: 0x000c, 0x52e: 0x000c, 0x52f: 0x000c, + 0x530: 0x000c, 0x531: 0x000d, 0x532: 0x000d, 0x533: 0x000d, 0x534: 0x000d, 0x535: 0x000d, + 0x536: 0x000d, 0x537: 0x000d, 0x538: 0x000d, 0x539: 0x000d, 0x53a: 0x000d, 0x53b: 0x000d, + 0x53c: 0x000d, 0x53d: 0x000d, 0x53e: 0x000d, 0x53f: 0x000d, + // Block 0x15, offset 0x540 + 0x540: 0x0001, 0x541: 0x0001, 0x542: 0x0001, 0x543: 0x0001, 0x544: 0x0001, 0x545: 0x0001, + 0x546: 0x0001, 0x547: 0x0001, 0x548: 0x0001, 0x549: 0x0001, 0x54a: 0x0001, 0x54b: 0x0001, + 0x54c: 0x0001, 0x54d: 0x0001, 0x54e: 0x0001, 0x54f: 0x0001, 0x550: 0x0001, 0x551: 0x0001, + 0x552: 0x0001, 0x553: 0x0001, 0x554: 0x0001, 0x555: 0x0001, 0x556: 0x0001, 0x557: 0x0001, + 0x558: 0x0001, 0x559: 0x0001, 0x55a: 0x0001, 0x55b: 0x0001, 0x55c: 0x0001, 0x55d: 0x0001, + 0x55e: 0x0001, 0x55f: 0x0001, 0x560: 0x0001, 0x561: 0x0001, 0x562: 0x0001, 0x563: 0x0001, + 0x564: 0x0001, 0x565: 0x0001, 0x566: 0x0001, 0x567: 0x0001, 0x568: 0x0001, 0x569: 0x0001, + 0x56a: 0x0001, 0x56b: 0x000c, 0x56c: 0x000c, 0x56d: 0x000c, 0x56e: 0x000c, 0x56f: 0x000c, + 0x570: 0x000c, 0x571: 0x000c, 0x572: 0x000c, 0x573: 0x000c, 0x574: 0x0001, 0x575: 0x0001, + 0x576: 0x000a, 0x577: 0x000a, 0x578: 0x000a, 0x579: 0x000a, 0x57a: 0x0001, 0x57b: 0x0001, + 0x57c: 0x0001, 0x57d: 0x0001, 0x57e: 0x0001, 0x57f: 0x0001, + // Block 0x16, offset 0x580 + 0x580: 0x0001, 0x581: 0x0001, 0x582: 0x0001, 0x583: 0x0001, 0x584: 0x0001, 0x585: 0x0001, + 0x586: 0x0001, 0x587: 0x0001, 0x588: 0x0001, 0x589: 0x0001, 0x58a: 0x0001, 0x58b: 0x0001, + 0x58c: 0x0001, 0x58d: 0x0001, 0x58e: 0x0001, 0x58f: 0x0001, 0x590: 0x0001, 0x591: 0x0001, + 0x592: 0x0001, 0x593: 0x0001, 0x594: 0x0001, 0x595: 0x0001, 0x596: 0x000c, 0x597: 0x000c, + 0x598: 0x000c, 0x599: 0x000c, 0x59a: 0x0001, 0x59b: 0x000c, 0x59c: 0x000c, 0x59d: 0x000c, + 0x59e: 0x000c, 0x59f: 0x000c, 0x5a0: 0x000c, 0x5a1: 0x000c, 0x5a2: 0x000c, 0x5a3: 0x000c, + 0x5a4: 0x0001, 0x5a5: 0x000c, 0x5a6: 0x000c, 0x5a7: 0x000c, 0x5a8: 0x0001, 0x5a9: 0x000c, + 0x5aa: 0x000c, 0x5ab: 0x000c, 0x5ac: 0x000c, 0x5ad: 0x000c, 0x5ae: 0x0001, 0x5af: 0x0001, + 0x5b0: 0x0001, 0x5b1: 0x0001, 0x5b2: 0x0001, 0x5b3: 0x0001, 0x5b4: 0x0001, 0x5b5: 0x0001, + 0x5b6: 0x0001, 0x5b7: 0x0001, 0x5b8: 0x0001, 0x5b9: 0x0001, 0x5ba: 0x0001, 0x5bb: 0x0001, + 0x5bc: 0x0001, 0x5bd: 0x0001, 0x5be: 0x0001, 0x5bf: 0x0001, + // Block 0x17, offset 0x5c0 + 0x5c0: 0x0001, 0x5c1: 0x0001, 0x5c2: 0x0001, 0x5c3: 0x0001, 0x5c4: 0x0001, 0x5c5: 0x0001, + 0x5c6: 0x0001, 0x5c7: 0x0001, 0x5c8: 0x0001, 0x5c9: 0x0001, 0x5ca: 0x0001, 0x5cb: 0x0001, + 0x5cc: 0x0001, 0x5cd: 0x0001, 0x5ce: 0x0001, 0x5cf: 0x0001, 0x5d0: 0x0001, 0x5d1: 0x0001, + 0x5d2: 0x0001, 0x5d3: 0x0001, 0x5d4: 0x0001, 0x5d5: 0x0001, 0x5d6: 0x0001, 0x5d7: 0x0001, + 0x5d8: 0x0001, 0x5d9: 0x000c, 0x5da: 0x000c, 0x5db: 0x000c, 0x5dc: 0x0001, 0x5dd: 0x0001, + 0x5de: 0x0001, 0x5df: 0x0001, 0x5e0: 0x0001, 0x5e1: 0x0001, 0x5e2: 0x0001, 0x5e3: 0x0001, + 0x5e4: 0x0001, 0x5e5: 0x0001, 0x5e6: 0x0001, 0x5e7: 0x0001, 0x5e8: 0x0001, 0x5e9: 0x0001, + 0x5ea: 0x0001, 0x5eb: 0x0001, 0x5ec: 0x0001, 0x5ed: 0x0001, 0x5ee: 0x0001, 0x5ef: 0x0001, + 0x5f0: 0x0001, 0x5f1: 0x0001, 0x5f2: 0x0001, 0x5f3: 0x0001, 0x5f4: 0x0001, 0x5f5: 0x0001, + 0x5f6: 0x0001, 0x5f7: 0x0001, 0x5f8: 0x0001, 0x5f9: 0x0001, 0x5fa: 0x0001, 0x5fb: 0x0001, + 0x5fc: 0x0001, 0x5fd: 0x0001, 0x5fe: 0x0001, 0x5ff: 0x0001, + // Block 0x18, offset 0x600 + 0x600: 0x0001, 0x601: 0x0001, 0x602: 0x0001, 0x603: 0x0001, 0x604: 0x0001, 0x605: 0x0001, + 0x606: 0x0001, 0x607: 0x0001, 0x608: 0x0001, 0x609: 0x0001, 0x60a: 0x0001, 0x60b: 0x0001, + 0x60c: 0x0001, 0x60d: 0x0001, 0x60e: 0x0001, 0x60f: 0x0001, 0x610: 0x0001, 0x611: 0x0001, + 0x612: 0x0001, 0x613: 0x0001, 0x614: 0x0001, 0x615: 0x0001, 0x616: 0x0001, 0x617: 0x0001, + 0x618: 0x0001, 0x619: 0x0001, 0x61a: 0x0001, 0x61b: 0x0001, 0x61c: 0x0001, 0x61d: 0x0001, + 0x61e: 0x0001, 0x61f: 0x0001, 0x620: 0x000d, 0x621: 0x000d, 0x622: 0x000d, 0x623: 0x000d, + 0x624: 0x000d, 0x625: 0x000d, 0x626: 0x000d, 0x627: 0x000d, 0x628: 0x000d, 0x629: 0x000d, + 0x62a: 0x000d, 0x62b: 0x000d, 0x62c: 0x000d, 0x62d: 0x000d, 0x62e: 0x000d, 0x62f: 0x000d, + 0x630: 0x000d, 0x631: 0x000d, 0x632: 0x000d, 0x633: 0x000d, 0x634: 0x000d, 0x635: 0x000d, + 0x636: 0x000d, 0x637: 0x000d, 0x638: 0x000d, 0x639: 0x000d, 0x63a: 0x000d, 0x63b: 0x000d, + 0x63c: 0x000d, 0x63d: 0x000d, 0x63e: 0x000d, 0x63f: 0x000d, + // Block 0x19, offset 0x640 + 0x640: 0x000d, 0x641: 0x000d, 0x642: 0x000d, 0x643: 0x000d, 0x644: 0x000d, 0x645: 0x000d, + 0x646: 0x000d, 0x647: 0x000d, 0x648: 0x000d, 0x649: 0x000d, 0x64a: 0x000d, 0x64b: 0x000d, + 0x64c: 0x000d, 0x64d: 0x000d, 0x64e: 0x000d, 0x64f: 0x000d, 0x650: 0x000d, 0x651: 0x000d, + 0x652: 0x000d, 0x653: 0x000d, 0x654: 0x000c, 0x655: 0x000c, 0x656: 0x000c, 0x657: 0x000c, + 0x658: 0x000c, 0x659: 0x000c, 0x65a: 0x000c, 0x65b: 0x000c, 0x65c: 0x000c, 0x65d: 0x000c, + 0x65e: 0x000c, 0x65f: 0x000c, 0x660: 0x000c, 0x661: 0x000c, 0x662: 0x0005, 0x663: 0x000c, + 0x664: 0x000c, 0x665: 0x000c, 0x666: 0x000c, 0x667: 0x000c, 0x668: 0x000c, 0x669: 0x000c, + 0x66a: 0x000c, 0x66b: 0x000c, 0x66c: 0x000c, 0x66d: 0x000c, 0x66e: 0x000c, 0x66f: 0x000c, + 0x670: 0x000c, 0x671: 0x000c, 0x672: 0x000c, 0x673: 0x000c, 0x674: 0x000c, 0x675: 0x000c, + 0x676: 0x000c, 0x677: 0x000c, 0x678: 0x000c, 0x679: 0x000c, 0x67a: 0x000c, 0x67b: 0x000c, + 0x67c: 0x000c, 0x67d: 0x000c, 0x67e: 0x000c, 0x67f: 0x000c, + // Block 0x1a, offset 0x680 + 0x680: 0x000c, 0x681: 0x000c, 0x682: 0x000c, + 0x6ba: 0x000c, + 0x6bc: 0x000c, + // Block 0x1b, offset 0x6c0 + 0x6c1: 0x000c, 0x6c2: 0x000c, 0x6c3: 0x000c, 0x6c4: 0x000c, 0x6c5: 0x000c, + 0x6c6: 0x000c, 0x6c7: 0x000c, 0x6c8: 0x000c, + 0x6cd: 0x000c, 0x6d1: 0x000c, + 0x6d2: 0x000c, 0x6d3: 0x000c, 0x6d4: 0x000c, 0x6d5: 0x000c, 0x6d6: 0x000c, 0x6d7: 0x000c, + 0x6e2: 0x000c, 0x6e3: 0x000c, + // Block 0x1c, offset 0x700 + 0x701: 0x000c, + 0x73c: 0x000c, + // Block 0x1d, offset 0x740 + 0x741: 0x000c, 0x742: 0x000c, 0x743: 0x000c, 0x744: 0x000c, + 0x74d: 0x000c, + 0x762: 0x000c, 0x763: 0x000c, + 0x772: 0x0004, 0x773: 0x0004, + 0x77b: 0x0004, + // Block 0x1e, offset 0x780 + 0x781: 0x000c, 0x782: 0x000c, + 0x7bc: 0x000c, + // Block 0x1f, offset 0x7c0 + 0x7c1: 0x000c, 0x7c2: 0x000c, + 0x7c7: 0x000c, 0x7c8: 0x000c, 0x7cb: 0x000c, + 0x7cc: 0x000c, 0x7cd: 0x000c, 0x7d1: 0x000c, + 0x7f0: 0x000c, 0x7f1: 0x000c, 0x7f5: 0x000c, + // Block 0x20, offset 0x800 + 0x801: 0x000c, 0x802: 0x000c, 0x803: 0x000c, 0x804: 0x000c, 0x805: 0x000c, + 0x807: 0x000c, 0x808: 0x000c, + 0x80d: 0x000c, + 0x822: 0x000c, 0x823: 0x000c, + 0x831: 0x0004, + // Block 0x21, offset 0x840 + 0x841: 0x000c, + 0x87c: 0x000c, 0x87f: 0x000c, + // Block 0x22, offset 0x880 + 0x881: 0x000c, 0x882: 0x000c, 0x883: 0x000c, 0x884: 0x000c, + 0x88d: 0x000c, + 0x896: 0x000c, + 0x8a2: 0x000c, 0x8a3: 0x000c, + // Block 0x23, offset 0x8c0 + 0x8c2: 0x000c, + // Block 0x24, offset 0x900 + 0x900: 0x000c, + 0x90d: 0x000c, + 0x933: 0x000a, 0x934: 0x000a, 0x935: 0x000a, + 0x936: 0x000a, 0x937: 0x000a, 0x938: 0x000a, 0x939: 0x0004, 0x93a: 0x000a, + // Block 0x25, offset 0x940 + 0x940: 0x000c, + 0x97e: 0x000c, 0x97f: 0x000c, + // Block 0x26, offset 0x980 + 0x980: 0x000c, + 0x986: 0x000c, 0x987: 0x000c, 0x988: 0x000c, 0x98a: 0x000c, 0x98b: 0x000c, + 0x98c: 0x000c, 0x98d: 0x000c, + 0x995: 0x000c, 0x996: 0x000c, + 0x9a2: 0x000c, 0x9a3: 0x000c, + 0x9b8: 0x000a, 0x9b9: 0x000a, 0x9ba: 0x000a, 0x9bb: 0x000a, + 0x9bc: 0x000a, 0x9bd: 0x000a, 0x9be: 0x000a, + // Block 0x27, offset 0x9c0 + 0x9cc: 0x000c, 0x9cd: 0x000c, + 0x9e2: 0x000c, 0x9e3: 0x000c, + // Block 0x28, offset 0xa00 + 0xa01: 0x000c, + // Block 0x29, offset 0xa40 + 0xa41: 0x000c, 0xa42: 0x000c, 0xa43: 0x000c, 0xa44: 0x000c, + 0xa4d: 0x000c, + 0xa62: 0x000c, 0xa63: 0x000c, + // Block 0x2a, offset 0xa80 + 0xa8a: 0x000c, + 0xa92: 0x000c, 0xa93: 0x000c, 0xa94: 0x000c, 0xa96: 0x000c, + // Block 0x2b, offset 0xac0 + 0xaf1: 0x000c, 0xaf4: 0x000c, 0xaf5: 0x000c, + 0xaf6: 0x000c, 0xaf7: 0x000c, 0xaf8: 0x000c, 0xaf9: 0x000c, 0xafa: 0x000c, + 0xaff: 0x0004, + // Block 0x2c, offset 0xb00 + 0xb07: 0x000c, 0xb08: 0x000c, 0xb09: 0x000c, 0xb0a: 0x000c, 0xb0b: 0x000c, + 0xb0c: 0x000c, 0xb0d: 0x000c, 0xb0e: 0x000c, + // Block 0x2d, offset 0xb40 + 0xb71: 0x000c, 0xb74: 0x000c, 0xb75: 0x000c, + 0xb76: 0x000c, 0xb77: 0x000c, 0xb78: 0x000c, 0xb79: 0x000c, 0xb7b: 0x000c, + 0xb7c: 0x000c, + // Block 0x2e, offset 0xb80 + 0xb88: 0x000c, 0xb89: 0x000c, 0xb8a: 0x000c, 0xb8b: 0x000c, + 0xb8c: 0x000c, 0xb8d: 0x000c, + // Block 0x2f, offset 0xbc0 + 0xbd8: 0x000c, 0xbd9: 0x000c, + 0xbf5: 0x000c, + 0xbf7: 0x000c, 0xbf9: 0x000c, 0xbfa: 0x003a, 0xbfb: 0x002a, + 0xbfc: 0x003a, 0xbfd: 0x002a, + // Block 0x30, offset 0xc00 + 0xc31: 0x000c, 0xc32: 0x000c, 0xc33: 0x000c, 0xc34: 0x000c, 0xc35: 0x000c, + 0xc36: 0x000c, 0xc37: 0x000c, 0xc38: 0x000c, 0xc39: 0x000c, 0xc3a: 0x000c, 0xc3b: 0x000c, + 0xc3c: 0x000c, 0xc3d: 0x000c, 0xc3e: 0x000c, + // Block 0x31, offset 0xc40 + 0xc40: 0x000c, 0xc41: 0x000c, 0xc42: 0x000c, 0xc43: 0x000c, 0xc44: 0x000c, + 0xc46: 0x000c, 0xc47: 0x000c, + 0xc4d: 0x000c, 0xc4e: 0x000c, 0xc4f: 0x000c, 0xc50: 0x000c, 0xc51: 0x000c, + 0xc52: 0x000c, 0xc53: 0x000c, 0xc54: 0x000c, 0xc55: 0x000c, 0xc56: 0x000c, 0xc57: 0x000c, + 0xc59: 0x000c, 0xc5a: 0x000c, 0xc5b: 0x000c, 0xc5c: 0x000c, 0xc5d: 0x000c, + 0xc5e: 0x000c, 0xc5f: 0x000c, 0xc60: 0x000c, 0xc61: 0x000c, 0xc62: 0x000c, 0xc63: 0x000c, + 0xc64: 0x000c, 0xc65: 0x000c, 0xc66: 0x000c, 0xc67: 0x000c, 0xc68: 0x000c, 0xc69: 0x000c, + 0xc6a: 0x000c, 0xc6b: 0x000c, 0xc6c: 0x000c, 0xc6d: 0x000c, 0xc6e: 0x000c, 0xc6f: 0x000c, + 0xc70: 0x000c, 0xc71: 0x000c, 0xc72: 0x000c, 0xc73: 0x000c, 0xc74: 0x000c, 0xc75: 0x000c, + 0xc76: 0x000c, 0xc77: 0x000c, 0xc78: 0x000c, 0xc79: 0x000c, 0xc7a: 0x000c, 0xc7b: 0x000c, + 0xc7c: 0x000c, + // Block 0x32, offset 0xc80 + 0xc86: 0x000c, + // Block 0x33, offset 0xcc0 + 0xced: 0x000c, 0xcee: 0x000c, 0xcef: 0x000c, + 0xcf0: 0x000c, 0xcf2: 0x000c, 0xcf3: 0x000c, 0xcf4: 0x000c, 0xcf5: 0x000c, + 0xcf6: 0x000c, 0xcf7: 0x000c, 0xcf9: 0x000c, 0xcfa: 0x000c, + 0xcfd: 0x000c, 0xcfe: 0x000c, + // Block 0x34, offset 0xd00 + 0xd18: 0x000c, 0xd19: 0x000c, + 0xd1e: 0x000c, 0xd1f: 0x000c, 0xd20: 0x000c, + 0xd31: 0x000c, 0xd32: 0x000c, 0xd33: 0x000c, 0xd34: 0x000c, + // Block 0x35, offset 0xd40 + 0xd42: 0x000c, 0xd45: 0x000c, + 0xd46: 0x000c, + 0xd4d: 0x000c, + 0xd5d: 0x000c, + // Block 0x36, offset 0xd80 + 0xd9d: 0x000c, + 0xd9e: 0x000c, 0xd9f: 0x000c, + // Block 0x37, offset 0xdc0 + 0xdd0: 0x000a, 0xdd1: 0x000a, + 0xdd2: 0x000a, 0xdd3: 0x000a, 0xdd4: 0x000a, 0xdd5: 0x000a, 0xdd6: 0x000a, 0xdd7: 0x000a, + 0xdd8: 0x000a, 0xdd9: 0x000a, + // Block 0x38, offset 0xe00 + 0xe00: 0x000a, + // Block 0x39, offset 0xe40 + 0xe40: 0x0009, + 0xe5b: 0x007a, 0xe5c: 0x006a, + // Block 0x3a, offset 0xe80 + 0xe92: 0x000c, 0xe93: 0x000c, 0xe94: 0x000c, + 0xeb2: 0x000c, 0xeb3: 0x000c, 0xeb4: 0x000c, + // Block 0x3b, offset 0xec0 + 0xed2: 0x000c, 0xed3: 0x000c, + 0xef2: 0x000c, 0xef3: 0x000c, + // Block 0x3c, offset 0xf00 + 0xf34: 0x000c, 0xf35: 0x000c, + 0xf37: 0x000c, 0xf38: 0x000c, 0xf39: 0x000c, 0xf3a: 0x000c, 0xf3b: 0x000c, + 0xf3c: 0x000c, 0xf3d: 0x000c, + // Block 0x3d, offset 0xf40 + 0xf46: 0x000c, 0xf49: 0x000c, 0xf4a: 0x000c, 0xf4b: 0x000c, + 0xf4c: 0x000c, 0xf4d: 0x000c, 0xf4e: 0x000c, 0xf4f: 0x000c, 0xf50: 0x000c, 0xf51: 0x000c, + 0xf52: 0x000c, 0xf53: 0x000c, + 0xf5b: 0x0004, 0xf5d: 0x000c, + 0xf70: 0x000a, 0xf71: 0x000a, 0xf72: 0x000a, 0xf73: 0x000a, 0xf74: 0x000a, 0xf75: 0x000a, + 0xf76: 0x000a, 0xf77: 0x000a, 0xf78: 0x000a, 0xf79: 0x000a, + // Block 0x3e, offset 0xf80 + 0xf80: 0x000a, 0xf81: 0x000a, 0xf82: 0x000a, 0xf83: 0x000a, 0xf84: 0x000a, 0xf85: 0x000a, + 0xf86: 0x000a, 0xf87: 0x000a, 0xf88: 0x000a, 0xf89: 0x000a, 0xf8a: 0x000a, 0xf8b: 0x000c, + 0xf8c: 0x000c, 0xf8d: 0x000c, 0xf8e: 0x000b, + // Block 0x3f, offset 0xfc0 + 0xfc5: 0x000c, + 0xfc6: 0x000c, + 0xfe9: 0x000c, + // Block 0x40, offset 0x1000 + 0x1020: 0x000c, 0x1021: 0x000c, 0x1022: 0x000c, + 0x1027: 0x000c, 0x1028: 0x000c, + 0x1032: 0x000c, + 0x1039: 0x000c, 0x103a: 0x000c, 0x103b: 0x000c, + // Block 0x41, offset 0x1040 + 0x1040: 0x000a, 0x1044: 0x000a, 0x1045: 0x000a, + // Block 0x42, offset 0x1080 + 0x109e: 0x000a, 0x109f: 0x000a, 0x10a0: 0x000a, 0x10a1: 0x000a, 0x10a2: 0x000a, 0x10a3: 0x000a, + 0x10a4: 0x000a, 0x10a5: 0x000a, 0x10a6: 0x000a, 0x10a7: 0x000a, 0x10a8: 0x000a, 0x10a9: 0x000a, + 0x10aa: 0x000a, 0x10ab: 0x000a, 0x10ac: 0x000a, 0x10ad: 0x000a, 0x10ae: 0x000a, 0x10af: 0x000a, + 0x10b0: 0x000a, 0x10b1: 0x000a, 0x10b2: 0x000a, 0x10b3: 0x000a, 0x10b4: 0x000a, 0x10b5: 0x000a, + 0x10b6: 0x000a, 0x10b7: 0x000a, 0x10b8: 0x000a, 0x10b9: 0x000a, 0x10ba: 0x000a, 0x10bb: 0x000a, + 0x10bc: 0x000a, 0x10bd: 0x000a, 0x10be: 0x000a, 0x10bf: 0x000a, + // Block 0x43, offset 0x10c0 + 0x10d7: 0x000c, + 0x10d8: 0x000c, 0x10db: 0x000c, + // Block 0x44, offset 0x1100 + 0x1116: 0x000c, + 0x1118: 0x000c, 0x1119: 0x000c, 0x111a: 0x000c, 0x111b: 0x000c, 0x111c: 0x000c, 0x111d: 0x000c, + 0x111e: 0x000c, 0x1120: 0x000c, 0x1122: 0x000c, + 0x1125: 0x000c, 0x1126: 0x000c, 0x1127: 0x000c, 0x1128: 0x000c, 0x1129: 0x000c, + 0x112a: 0x000c, 0x112b: 0x000c, 0x112c: 0x000c, + 0x1133: 0x000c, 0x1134: 0x000c, 0x1135: 0x000c, + 0x1136: 0x000c, 0x1137: 0x000c, 0x1138: 0x000c, 0x1139: 0x000c, 0x113a: 0x000c, 0x113b: 0x000c, + 0x113c: 0x000c, 0x113f: 0x000c, + // Block 0x45, offset 0x1140 + 0x1170: 0x000c, 0x1171: 0x000c, 0x1172: 0x000c, 0x1173: 0x000c, 0x1174: 0x000c, 0x1175: 0x000c, + 0x1176: 0x000c, 0x1177: 0x000c, 0x1178: 0x000c, 0x1179: 0x000c, 0x117a: 0x000c, 0x117b: 0x000c, + 0x117c: 0x000c, 0x117d: 0x000c, 0x117e: 0x000c, + // Block 0x46, offset 0x1180 + 0x1180: 0x000c, 0x1181: 0x000c, 0x1182: 0x000c, 0x1183: 0x000c, + 0x11b4: 0x000c, + 0x11b6: 0x000c, 0x11b7: 0x000c, 0x11b8: 0x000c, 0x11b9: 0x000c, 0x11ba: 0x000c, + 0x11bc: 0x000c, + // Block 0x47, offset 0x11c0 + 0x11c2: 0x000c, + 0x11eb: 0x000c, 0x11ec: 0x000c, 0x11ed: 0x000c, 0x11ee: 0x000c, 0x11ef: 0x000c, + 0x11f0: 0x000c, 0x11f1: 0x000c, 0x11f2: 0x000c, 0x11f3: 0x000c, + // Block 0x48, offset 0x1200 + 0x1200: 0x000c, 0x1201: 0x000c, + 0x1222: 0x000c, 0x1223: 0x000c, + 0x1224: 0x000c, 0x1225: 0x000c, 0x1228: 0x000c, 0x1229: 0x000c, + 0x122b: 0x000c, 0x122c: 0x000c, 0x122d: 0x000c, + // Block 0x49, offset 0x1240 + 0x1266: 0x000c, 0x1268: 0x000c, 0x1269: 0x000c, + 0x126d: 0x000c, 0x126f: 0x000c, + 0x1270: 0x000c, 0x1271: 0x000c, + // Block 0x4a, offset 0x1280 + 0x12ac: 0x000c, 0x12ad: 0x000c, 0x12ae: 0x000c, 0x12af: 0x000c, + 0x12b0: 0x000c, 0x12b1: 0x000c, 0x12b2: 0x000c, 0x12b3: 0x000c, + 0x12b6: 0x000c, 0x12b7: 0x000c, + // Block 0x4b, offset 0x12c0 + 0x12d0: 0x000c, 0x12d1: 0x000c, + 0x12d2: 0x000c, 0x12d4: 0x000c, 0x12d5: 0x000c, 0x12d6: 0x000c, 0x12d7: 0x000c, + 0x12d8: 0x000c, 0x12d9: 0x000c, 0x12da: 0x000c, 0x12db: 0x000c, 0x12dc: 0x000c, 0x12dd: 0x000c, + 0x12de: 0x000c, 0x12df: 0x000c, 0x12e0: 0x000c, 0x12e2: 0x000c, 0x12e3: 0x000c, + 0x12e4: 0x000c, 0x12e5: 0x000c, 0x12e6: 0x000c, 0x12e7: 0x000c, 0x12e8: 0x000c, + 0x12ed: 0x000c, + 0x12f4: 0x000c, + 0x12f8: 0x000c, 0x12f9: 0x000c, + // Block 0x4c, offset 0x1300 + 0x1300: 0x000c, 0x1301: 0x000c, 0x1302: 0x000c, 0x1303: 0x000c, 0x1304: 0x000c, 0x1305: 0x000c, + 0x1306: 0x000c, 0x1307: 0x000c, 0x1308: 0x000c, 0x1309: 0x000c, 0x130a: 0x000c, 0x130b: 0x000c, + 0x130c: 0x000c, 0x130d: 0x000c, 0x130e: 0x000c, 0x130f: 0x000c, 0x1310: 0x000c, 0x1311: 0x000c, + 0x1312: 0x000c, 0x1313: 0x000c, 0x1314: 0x000c, 0x1315: 0x000c, 0x1316: 0x000c, 0x1317: 0x000c, + 0x1318: 0x000c, 0x1319: 0x000c, 0x131a: 0x000c, 0x131b: 0x000c, 0x131c: 0x000c, 0x131d: 0x000c, + 0x131e: 0x000c, 0x131f: 0x000c, 0x1320: 0x000c, 0x1321: 0x000c, 0x1322: 0x000c, 0x1323: 0x000c, + 0x1324: 0x000c, 0x1325: 0x000c, 0x1326: 0x000c, 0x1327: 0x000c, 0x1328: 0x000c, 0x1329: 0x000c, + 0x132a: 0x000c, 0x132b: 0x000c, 0x132c: 0x000c, 0x132d: 0x000c, 0x132e: 0x000c, 0x132f: 0x000c, + 0x1330: 0x000c, 0x1331: 0x000c, 0x1332: 0x000c, 0x1333: 0x000c, 0x1334: 0x000c, 0x1335: 0x000c, + 0x133b: 0x000c, + 0x133c: 0x000c, 0x133d: 0x000c, 0x133e: 0x000c, 0x133f: 0x000c, + // Block 0x4d, offset 0x1340 + 0x137d: 0x000a, 0x137f: 0x000a, + // Block 0x4e, offset 0x1380 + 0x1380: 0x000a, 0x1381: 0x000a, + 0x138d: 0x000a, 0x138e: 0x000a, 0x138f: 0x000a, + 0x139d: 0x000a, + 0x139e: 0x000a, 0x139f: 0x000a, + 0x13ad: 0x000a, 0x13ae: 0x000a, 0x13af: 0x000a, + 0x13bd: 0x000a, 0x13be: 0x000a, + // Block 0x4f, offset 0x13c0 + 0x13c0: 0x0009, 0x13c1: 0x0009, 0x13c2: 0x0009, 0x13c3: 0x0009, 0x13c4: 0x0009, 0x13c5: 0x0009, + 0x13c6: 0x0009, 0x13c7: 0x0009, 0x13c8: 0x0009, 0x13c9: 0x0009, 0x13ca: 0x0009, 0x13cb: 0x000b, + 0x13cc: 0x000b, 0x13cd: 0x000b, 0x13cf: 0x0001, 0x13d0: 0x000a, 0x13d1: 0x000a, + 0x13d2: 0x000a, 0x13d3: 0x000a, 0x13d4: 0x000a, 0x13d5: 0x000a, 0x13d6: 0x000a, 0x13d7: 0x000a, + 0x13d8: 0x000a, 0x13d9: 0x000a, 0x13da: 0x000a, 0x13db: 0x000a, 0x13dc: 0x000a, 0x13dd: 0x000a, + 0x13de: 0x000a, 0x13df: 0x000a, 0x13e0: 0x000a, 0x13e1: 0x000a, 0x13e2: 0x000a, 0x13e3: 0x000a, + 0x13e4: 0x000a, 0x13e5: 0x000a, 0x13e6: 0x000a, 0x13e7: 0x000a, 0x13e8: 0x0009, 0x13e9: 0x0007, + 0x13ea: 0x000e, 0x13eb: 0x000e, 0x13ec: 0x000e, 0x13ed: 0x000e, 0x13ee: 0x000e, 0x13ef: 0x0006, + 0x13f0: 0x0004, 0x13f1: 0x0004, 0x13f2: 0x0004, 0x13f3: 0x0004, 0x13f4: 0x0004, 0x13f5: 0x000a, + 0x13f6: 0x000a, 0x13f7: 0x000a, 0x13f8: 0x000a, 0x13f9: 0x000a, 0x13fa: 0x000a, 0x13fb: 0x000a, + 0x13fc: 0x000a, 0x13fd: 0x000a, 0x13fe: 0x000a, 0x13ff: 0x000a, + // Block 0x50, offset 0x1400 + 0x1400: 0x000a, 0x1401: 0x000a, 0x1402: 0x000a, 0x1403: 0x000a, 0x1404: 0x0006, 0x1405: 0x009a, + 0x1406: 0x008a, 0x1407: 0x000a, 0x1408: 0x000a, 0x1409: 0x000a, 0x140a: 0x000a, 0x140b: 0x000a, + 0x140c: 0x000a, 0x140d: 0x000a, 0x140e: 0x000a, 0x140f: 0x000a, 0x1410: 0x000a, 0x1411: 0x000a, + 0x1412: 0x000a, 0x1413: 0x000a, 0x1414: 0x000a, 0x1415: 0x000a, 0x1416: 0x000a, 0x1417: 0x000a, + 0x1418: 0x000a, 0x1419: 0x000a, 0x141a: 0x000a, 0x141b: 0x000a, 0x141c: 0x000a, 0x141d: 0x000a, + 0x141e: 0x000a, 0x141f: 0x0009, 0x1420: 0x000b, 0x1421: 0x000b, 0x1422: 0x000b, 0x1423: 0x000b, + 0x1424: 0x000b, 0x1425: 0x000b, 0x1426: 0x000e, 0x1427: 0x000e, 0x1428: 0x000e, 0x1429: 0x000e, + 0x142a: 0x000b, 0x142b: 0x000b, 0x142c: 0x000b, 0x142d: 0x000b, 0x142e: 0x000b, 0x142f: 0x000b, + 0x1430: 0x0002, 0x1434: 0x0002, 0x1435: 0x0002, + 0x1436: 0x0002, 0x1437: 0x0002, 0x1438: 0x0002, 0x1439: 0x0002, 0x143a: 0x0003, 0x143b: 0x0003, + 0x143c: 0x000a, 0x143d: 0x009a, 0x143e: 0x008a, + // Block 0x51, offset 0x1440 + 0x1440: 0x0002, 0x1441: 0x0002, 0x1442: 0x0002, 0x1443: 0x0002, 0x1444: 0x0002, 0x1445: 0x0002, + 0x1446: 0x0002, 0x1447: 0x0002, 0x1448: 0x0002, 0x1449: 0x0002, 0x144a: 0x0003, 0x144b: 0x0003, + 0x144c: 0x000a, 0x144d: 0x009a, 0x144e: 0x008a, + 0x1460: 0x0004, 0x1461: 0x0004, 0x1462: 0x0004, 0x1463: 0x0004, + 0x1464: 0x0004, 0x1465: 0x0004, 0x1466: 0x0004, 0x1467: 0x0004, 0x1468: 0x0004, 0x1469: 0x0004, + 0x146a: 0x0004, 0x146b: 0x0004, 0x146c: 0x0004, 0x146d: 0x0004, 0x146e: 0x0004, 0x146f: 0x0004, + 0x1470: 0x0004, 0x1471: 0x0004, 0x1472: 0x0004, 0x1473: 0x0004, 0x1474: 0x0004, 0x1475: 0x0004, + 0x1476: 0x0004, 0x1477: 0x0004, 0x1478: 0x0004, 0x1479: 0x0004, 0x147a: 0x0004, 0x147b: 0x0004, + 0x147c: 0x0004, 0x147d: 0x0004, 0x147e: 0x0004, 0x147f: 0x0004, + // Block 0x52, offset 0x1480 + 0x1480: 0x0004, 0x1481: 0x0004, 0x1482: 0x0004, 0x1483: 0x0004, 0x1484: 0x0004, 0x1485: 0x0004, + 0x1486: 0x0004, 0x1487: 0x0004, 0x1488: 0x0004, 0x1489: 0x0004, 0x148a: 0x0004, 0x148b: 0x0004, + 0x148c: 0x0004, 0x148d: 0x0004, 0x148e: 0x0004, 0x148f: 0x0004, 0x1490: 0x000c, 0x1491: 0x000c, + 0x1492: 0x000c, 0x1493: 0x000c, 0x1494: 0x000c, 0x1495: 0x000c, 0x1496: 0x000c, 0x1497: 0x000c, + 0x1498: 0x000c, 0x1499: 0x000c, 0x149a: 0x000c, 0x149b: 0x000c, 0x149c: 0x000c, 0x149d: 0x000c, + 0x149e: 0x000c, 0x149f: 0x000c, 0x14a0: 0x000c, 0x14a1: 0x000c, 0x14a2: 0x000c, 0x14a3: 0x000c, + 0x14a4: 0x000c, 0x14a5: 0x000c, 0x14a6: 0x000c, 0x14a7: 0x000c, 0x14a8: 0x000c, 0x14a9: 0x000c, + 0x14aa: 0x000c, 0x14ab: 0x000c, 0x14ac: 0x000c, 0x14ad: 0x000c, 0x14ae: 0x000c, 0x14af: 0x000c, + 0x14b0: 0x000c, + // Block 0x53, offset 0x14c0 + 0x14c0: 0x000a, 0x14c1: 0x000a, 0x14c3: 0x000a, 0x14c4: 0x000a, 0x14c5: 0x000a, + 0x14c6: 0x000a, 0x14c8: 0x000a, 0x14c9: 0x000a, + 0x14d4: 0x000a, 0x14d6: 0x000a, 0x14d7: 0x000a, + 0x14d8: 0x000a, + 0x14de: 0x000a, 0x14df: 0x000a, 0x14e0: 0x000a, 0x14e1: 0x000a, 0x14e2: 0x000a, 0x14e3: 0x000a, + 0x14e5: 0x000a, 0x14e7: 0x000a, 0x14e9: 0x000a, + 0x14ee: 0x0004, + 0x14fa: 0x000a, 0x14fb: 0x000a, + // Block 0x54, offset 0x1500 + 0x1500: 0x000a, 0x1501: 0x000a, 0x1502: 0x000a, 0x1503: 0x000a, 0x1504: 0x000a, + 0x150a: 0x000a, 0x150b: 0x000a, + 0x150c: 0x000a, 0x150d: 0x000a, 0x1510: 0x000a, 0x1511: 0x000a, + 0x1512: 0x000a, 0x1513: 0x000a, 0x1514: 0x000a, 0x1515: 0x000a, 0x1516: 0x000a, 0x1517: 0x000a, + 0x1518: 0x000a, 0x1519: 0x000a, 0x151a: 0x000a, 0x151b: 0x000a, 0x151c: 0x000a, 0x151d: 0x000a, + 0x151e: 0x000a, 0x151f: 0x000a, + // Block 0x55, offset 0x1540 + 0x1549: 0x000a, 0x154a: 0x000a, 0x154b: 0x000a, + 0x1550: 0x000a, 0x1551: 0x000a, + 0x1552: 0x000a, 0x1553: 0x000a, 0x1554: 0x000a, 0x1555: 0x000a, 0x1556: 0x000a, 0x1557: 0x000a, + 0x1558: 0x000a, 0x1559: 0x000a, 0x155a: 0x000a, 0x155b: 0x000a, 0x155c: 0x000a, 0x155d: 0x000a, + 0x155e: 0x000a, 0x155f: 0x000a, 0x1560: 0x000a, 0x1561: 0x000a, 0x1562: 0x000a, 0x1563: 0x000a, + 0x1564: 0x000a, 0x1565: 0x000a, 0x1566: 0x000a, 0x1567: 0x000a, 0x1568: 0x000a, 0x1569: 0x000a, + 0x156a: 0x000a, 0x156b: 0x000a, 0x156c: 0x000a, 0x156d: 0x000a, 0x156e: 0x000a, 0x156f: 0x000a, + 0x1570: 0x000a, 0x1571: 0x000a, 0x1572: 0x000a, 0x1573: 0x000a, 0x1574: 0x000a, 0x1575: 0x000a, + 0x1576: 0x000a, 0x1577: 0x000a, 0x1578: 0x000a, 0x1579: 0x000a, 0x157a: 0x000a, 0x157b: 0x000a, + 0x157c: 0x000a, 0x157d: 0x000a, 0x157e: 0x000a, 0x157f: 0x000a, + // Block 0x56, offset 0x1580 + 0x1580: 0x000a, 0x1581: 0x000a, 0x1582: 0x000a, 0x1583: 0x000a, 0x1584: 0x000a, 0x1585: 0x000a, + 0x1586: 0x000a, 0x1587: 0x000a, 0x1588: 0x000a, 0x1589: 0x000a, 0x158a: 0x000a, 0x158b: 0x000a, + 0x158c: 0x000a, 0x158d: 0x000a, 0x158e: 0x000a, 0x158f: 0x000a, 0x1590: 0x000a, 0x1591: 0x000a, + 0x1592: 0x000a, 0x1593: 0x000a, 0x1594: 0x000a, 0x1595: 0x000a, 0x1596: 0x000a, 0x1597: 0x000a, + 0x1598: 0x000a, 0x1599: 0x000a, 0x159a: 0x000a, 0x159b: 0x000a, 0x159c: 0x000a, 0x159d: 0x000a, + 0x159e: 0x000a, 0x159f: 0x000a, 0x15a0: 0x000a, 0x15a1: 0x000a, 0x15a2: 0x000a, 0x15a3: 0x000a, + 0x15a4: 0x000a, 0x15a5: 0x000a, 0x15a6: 0x000a, 0x15a7: 0x000a, 0x15a8: 0x000a, 0x15a9: 0x000a, + 0x15aa: 0x000a, 0x15ab: 0x000a, 0x15ac: 0x000a, 0x15ad: 0x000a, 0x15ae: 0x000a, 0x15af: 0x000a, + 0x15b0: 0x000a, 0x15b1: 0x000a, 0x15b2: 0x000a, 0x15b3: 0x000a, 0x15b4: 0x000a, 0x15b5: 0x000a, + 0x15b6: 0x000a, 0x15b7: 0x000a, 0x15b8: 0x000a, 0x15b9: 0x000a, 0x15ba: 0x000a, 0x15bb: 0x000a, + 0x15bc: 0x000a, 0x15bd: 0x000a, 0x15be: 0x000a, 0x15bf: 0x000a, + // Block 0x57, offset 0x15c0 + 0x15c0: 0x000a, 0x15c1: 0x000a, 0x15c2: 0x000a, 0x15c3: 0x000a, 0x15c4: 0x000a, 0x15c5: 0x000a, + 0x15c6: 0x000a, 0x15c7: 0x000a, 0x15c8: 0x000a, 0x15c9: 0x000a, 0x15ca: 0x000a, 0x15cb: 0x000a, + 0x15cc: 0x000a, 0x15cd: 0x000a, 0x15ce: 0x000a, 0x15cf: 0x000a, 0x15d0: 0x000a, 0x15d1: 0x000a, + 0x15d2: 0x0003, 0x15d3: 0x0004, 0x15d4: 0x000a, 0x15d5: 0x000a, 0x15d6: 0x000a, 0x15d7: 0x000a, + 0x15d8: 0x000a, 0x15d9: 0x000a, 0x15da: 0x000a, 0x15db: 0x000a, 0x15dc: 0x000a, 0x15dd: 0x000a, + 0x15de: 0x000a, 0x15df: 0x000a, 0x15e0: 0x000a, 0x15e1: 0x000a, 0x15e2: 0x000a, 0x15e3: 0x000a, + 0x15e4: 0x000a, 0x15e5: 0x000a, 0x15e6: 0x000a, 0x15e7: 0x000a, 0x15e8: 0x000a, 0x15e9: 0x000a, + 0x15ea: 0x000a, 0x15eb: 0x000a, 0x15ec: 0x000a, 0x15ed: 0x000a, 0x15ee: 0x000a, 0x15ef: 0x000a, + 0x15f0: 0x000a, 0x15f1: 0x000a, 0x15f2: 0x000a, 0x15f3: 0x000a, 0x15f4: 0x000a, 0x15f5: 0x000a, + 0x15f6: 0x000a, 0x15f7: 0x000a, 0x15f8: 0x000a, 0x15f9: 0x000a, 0x15fa: 0x000a, 0x15fb: 0x000a, + 0x15fc: 0x000a, 0x15fd: 0x000a, 0x15fe: 0x000a, 0x15ff: 0x000a, + // Block 0x58, offset 0x1600 + 0x1600: 0x000a, 0x1601: 0x000a, 0x1602: 0x000a, 0x1603: 0x000a, 0x1604: 0x000a, 0x1605: 0x000a, + 0x1606: 0x000a, 0x1607: 0x000a, 0x1608: 0x003a, 0x1609: 0x002a, 0x160a: 0x003a, 0x160b: 0x002a, + 0x160c: 0x000a, 0x160d: 0x000a, 0x160e: 0x000a, 0x160f: 0x000a, 0x1610: 0x000a, 0x1611: 0x000a, + 0x1612: 0x000a, 0x1613: 0x000a, 0x1614: 0x000a, 0x1615: 0x000a, 0x1616: 0x000a, 0x1617: 0x000a, + 0x1618: 0x000a, 0x1619: 0x000a, 0x161a: 0x000a, 0x161b: 0x000a, 0x161c: 0x000a, 0x161d: 0x000a, + 0x161e: 0x000a, 0x161f: 0x000a, 0x1620: 0x000a, 0x1621: 0x000a, 0x1622: 0x000a, 0x1623: 0x000a, + 0x1624: 0x000a, 0x1625: 0x000a, 0x1626: 0x000a, 0x1627: 0x000a, 0x1628: 0x000a, 0x1629: 0x009a, + 0x162a: 0x008a, 0x162b: 0x000a, 0x162c: 0x000a, 0x162d: 0x000a, 0x162e: 0x000a, 0x162f: 0x000a, + 0x1630: 0x000a, 0x1631: 0x000a, 0x1632: 0x000a, 0x1633: 0x000a, 0x1634: 0x000a, 0x1635: 0x000a, + // Block 0x59, offset 0x1640 + 0x167b: 0x000a, + 0x167c: 0x000a, 0x167d: 0x000a, 0x167e: 0x000a, 0x167f: 0x000a, + // Block 0x5a, offset 0x1680 + 0x1680: 0x000a, 0x1681: 0x000a, 0x1682: 0x000a, 0x1683: 0x000a, 0x1684: 0x000a, 0x1685: 0x000a, + 0x1686: 0x000a, 0x1687: 0x000a, 0x1688: 0x000a, 0x1689: 0x000a, 0x168a: 0x000a, 0x168b: 0x000a, + 0x168c: 0x000a, 0x168d: 0x000a, 0x168e: 0x000a, 0x168f: 0x000a, 0x1690: 0x000a, 0x1691: 0x000a, + 0x1692: 0x000a, 0x1693: 0x000a, 0x1694: 0x000a, 0x1696: 0x000a, 0x1697: 0x000a, + 0x1698: 0x000a, 0x1699: 0x000a, 0x169a: 0x000a, 0x169b: 0x000a, 0x169c: 0x000a, 0x169d: 0x000a, + 0x169e: 0x000a, 0x169f: 0x000a, 0x16a0: 0x000a, 0x16a1: 0x000a, 0x16a2: 0x000a, 0x16a3: 0x000a, + 0x16a4: 0x000a, 0x16a5: 0x000a, 0x16a6: 0x000a, 0x16a7: 0x000a, 0x16a8: 0x000a, 0x16a9: 0x000a, + 0x16aa: 0x000a, 0x16ab: 0x000a, 0x16ac: 0x000a, 0x16ad: 0x000a, 0x16ae: 0x000a, 0x16af: 0x000a, + 0x16b0: 0x000a, 0x16b1: 0x000a, 0x16b2: 0x000a, 0x16b3: 0x000a, 0x16b4: 0x000a, 0x16b5: 0x000a, + 0x16b6: 0x000a, 0x16b7: 0x000a, 0x16b8: 0x000a, 0x16b9: 0x000a, 0x16ba: 0x000a, 0x16bb: 0x000a, + 0x16bc: 0x000a, 0x16bd: 0x000a, 0x16be: 0x000a, 0x16bf: 0x000a, + // Block 0x5b, offset 0x16c0 + 0x16c0: 0x000a, 0x16c1: 0x000a, 0x16c2: 0x000a, 0x16c3: 0x000a, 0x16c4: 0x000a, 0x16c5: 0x000a, + 0x16c6: 0x000a, 0x16c7: 0x000a, 0x16c8: 0x000a, 0x16c9: 0x000a, 0x16ca: 0x000a, 0x16cb: 0x000a, + 0x16cc: 0x000a, 0x16cd: 0x000a, 0x16ce: 0x000a, 0x16cf: 0x000a, 0x16d0: 0x000a, 0x16d1: 0x000a, + 0x16d2: 0x000a, 0x16d3: 0x000a, 0x16d4: 0x000a, 0x16d5: 0x000a, 0x16d6: 0x000a, 0x16d7: 0x000a, + 0x16d8: 0x000a, 0x16d9: 0x000a, 0x16da: 0x000a, 0x16db: 0x000a, 0x16dc: 0x000a, 0x16dd: 0x000a, + 0x16de: 0x000a, 0x16df: 0x000a, 0x16e0: 0x000a, 0x16e1: 0x000a, 0x16e2: 0x000a, 0x16e3: 0x000a, + 0x16e4: 0x000a, 0x16e5: 0x000a, 0x16e6: 0x000a, 0x16e7: 0x000a, 0x16e8: 0x000a, 0x16e9: 0x000a, + 0x16ea: 0x000a, 0x16eb: 0x000a, 0x16ec: 0x000a, 0x16ed: 0x000a, 0x16ee: 0x000a, 0x16ef: 0x000a, + 0x16f0: 0x000a, 0x16f1: 0x000a, 0x16f2: 0x000a, 0x16f3: 0x000a, 0x16f4: 0x000a, 0x16f5: 0x000a, + 0x16f6: 0x000a, 0x16f7: 0x000a, 0x16f8: 0x000a, 0x16f9: 0x000a, 0x16fa: 0x000a, 0x16fb: 0x000a, + 0x16fc: 0x000a, 0x16fd: 0x000a, 0x16fe: 0x000a, + // Block 0x5c, offset 0x1700 + 0x1700: 0x000a, 0x1701: 0x000a, 0x1702: 0x000a, 0x1703: 0x000a, 0x1704: 0x000a, 0x1705: 0x000a, + 0x1706: 0x000a, 0x1707: 0x000a, 0x1708: 0x000a, 0x1709: 0x000a, 0x170a: 0x000a, 0x170b: 0x000a, + 0x170c: 0x000a, 0x170d: 0x000a, 0x170e: 0x000a, 0x170f: 0x000a, 0x1710: 0x000a, 0x1711: 0x000a, + 0x1712: 0x000a, 0x1713: 0x000a, 0x1714: 0x000a, 0x1715: 0x000a, 0x1716: 0x000a, 0x1717: 0x000a, + 0x1718: 0x000a, 0x1719: 0x000a, 0x171a: 0x000a, 0x171b: 0x000a, 0x171c: 0x000a, 0x171d: 0x000a, + 0x171e: 0x000a, 0x171f: 0x000a, 0x1720: 0x000a, 0x1721: 0x000a, 0x1722: 0x000a, 0x1723: 0x000a, + 0x1724: 0x000a, 0x1725: 0x000a, 0x1726: 0x000a, + // Block 0x5d, offset 0x1740 + 0x1740: 0x000a, 0x1741: 0x000a, 0x1742: 0x000a, 0x1743: 0x000a, 0x1744: 0x000a, 0x1745: 0x000a, + 0x1746: 0x000a, 0x1747: 0x000a, 0x1748: 0x000a, 0x1749: 0x000a, 0x174a: 0x000a, + 0x1760: 0x000a, 0x1761: 0x000a, 0x1762: 0x000a, 0x1763: 0x000a, + 0x1764: 0x000a, 0x1765: 0x000a, 0x1766: 0x000a, 0x1767: 0x000a, 0x1768: 0x000a, 0x1769: 0x000a, + 0x176a: 0x000a, 0x176b: 0x000a, 0x176c: 0x000a, 0x176d: 0x000a, 0x176e: 0x000a, 0x176f: 0x000a, + 0x1770: 0x000a, 0x1771: 0x000a, 0x1772: 0x000a, 0x1773: 0x000a, 0x1774: 0x000a, 0x1775: 0x000a, + 0x1776: 0x000a, 0x1777: 0x000a, 0x1778: 0x000a, 0x1779: 0x000a, 0x177a: 0x000a, 0x177b: 0x000a, + 0x177c: 0x000a, 0x177d: 0x000a, 0x177e: 0x000a, 0x177f: 0x000a, + // Block 0x5e, offset 0x1780 + 0x1780: 0x000a, 0x1781: 0x000a, 0x1782: 0x000a, 0x1783: 0x000a, 0x1784: 0x000a, 0x1785: 0x000a, + 0x1786: 0x000a, 0x1787: 0x000a, 0x1788: 0x0002, 0x1789: 0x0002, 0x178a: 0x0002, 0x178b: 0x0002, + 0x178c: 0x0002, 0x178d: 0x0002, 0x178e: 0x0002, 0x178f: 0x0002, 0x1790: 0x0002, 0x1791: 0x0002, + 0x1792: 0x0002, 0x1793: 0x0002, 0x1794: 0x0002, 0x1795: 0x0002, 0x1796: 0x0002, 0x1797: 0x0002, + 0x1798: 0x0002, 0x1799: 0x0002, 0x179a: 0x0002, 0x179b: 0x0002, + // Block 0x5f, offset 0x17c0 + 0x17ea: 0x000a, 0x17eb: 0x000a, 0x17ec: 0x000a, 0x17ed: 0x000a, 0x17ee: 0x000a, 0x17ef: 0x000a, + 0x17f0: 0x000a, 0x17f1: 0x000a, 0x17f2: 0x000a, 0x17f3: 0x000a, 0x17f4: 0x000a, 0x17f5: 0x000a, + 0x17f6: 0x000a, 0x17f7: 0x000a, 0x17f8: 0x000a, 0x17f9: 0x000a, 0x17fa: 0x000a, 0x17fb: 0x000a, + 0x17fc: 0x000a, 0x17fd: 0x000a, 0x17fe: 0x000a, 0x17ff: 0x000a, + // Block 0x60, offset 0x1800 + 0x1800: 0x000a, 0x1801: 0x000a, 0x1802: 0x000a, 0x1803: 0x000a, 0x1804: 0x000a, 0x1805: 0x000a, + 0x1806: 0x000a, 0x1807: 0x000a, 0x1808: 0x000a, 0x1809: 0x000a, 0x180a: 0x000a, 0x180b: 0x000a, + 0x180c: 0x000a, 0x180d: 0x000a, 0x180e: 0x000a, 0x180f: 0x000a, 0x1810: 0x000a, 0x1811: 0x000a, + 0x1812: 0x000a, 0x1813: 0x000a, 0x1814: 0x000a, 0x1815: 0x000a, 0x1816: 0x000a, 0x1817: 0x000a, + 0x1818: 0x000a, 0x1819: 0x000a, 0x181a: 0x000a, 0x181b: 0x000a, 0x181c: 0x000a, 0x181d: 0x000a, + 0x181e: 0x000a, 0x181f: 0x000a, 0x1820: 0x000a, 0x1821: 0x000a, 0x1822: 0x000a, 0x1823: 0x000a, + 0x1824: 0x000a, 0x1825: 0x000a, 0x1826: 0x000a, 0x1827: 0x000a, 0x1828: 0x000a, 0x1829: 0x000a, + 0x182a: 0x000a, 0x182b: 0x000a, 0x182d: 0x000a, 0x182e: 0x000a, 0x182f: 0x000a, + 0x1830: 0x000a, 0x1831: 0x000a, 0x1832: 0x000a, 0x1833: 0x000a, 0x1834: 0x000a, 0x1835: 0x000a, + 0x1836: 0x000a, 0x1837: 0x000a, 0x1838: 0x000a, 0x1839: 0x000a, 0x183a: 0x000a, 0x183b: 0x000a, + 0x183c: 0x000a, 0x183d: 0x000a, 0x183e: 0x000a, 0x183f: 0x000a, + // Block 0x61, offset 0x1840 + 0x1840: 0x000a, 0x1841: 0x000a, 0x1842: 0x000a, 0x1843: 0x000a, 0x1844: 0x000a, 0x1845: 0x000a, + 0x1846: 0x000a, 0x1847: 0x000a, 0x1848: 0x000a, 0x1849: 0x000a, 0x184a: 0x000a, 0x184b: 0x000a, + 0x184c: 0x000a, 0x184d: 0x000a, 0x184e: 0x000a, 0x184f: 0x000a, 0x1850: 0x000a, 0x1851: 0x000a, + 0x1852: 0x000a, 0x1853: 0x000a, 0x1854: 0x000a, 0x1855: 0x000a, 0x1856: 0x000a, 0x1857: 0x000a, + 0x1858: 0x000a, 0x1859: 0x000a, 0x185a: 0x000a, 0x185b: 0x000a, 0x185c: 0x000a, 0x185d: 0x000a, + 0x185e: 0x000a, 0x185f: 0x000a, 0x1860: 0x000a, 0x1861: 0x000a, 0x1862: 0x000a, 0x1863: 0x000a, + 0x1864: 0x000a, 0x1865: 0x000a, 0x1866: 0x000a, 0x1867: 0x000a, 0x1868: 0x003a, 0x1869: 0x002a, + 0x186a: 0x003a, 0x186b: 0x002a, 0x186c: 0x003a, 0x186d: 0x002a, 0x186e: 0x003a, 0x186f: 0x002a, + 0x1870: 0x003a, 0x1871: 0x002a, 0x1872: 0x003a, 0x1873: 0x002a, 0x1874: 0x003a, 0x1875: 0x002a, + 0x1876: 0x000a, 0x1877: 0x000a, 0x1878: 0x000a, 0x1879: 0x000a, 0x187a: 0x000a, 0x187b: 0x000a, + 0x187c: 0x000a, 0x187d: 0x000a, 0x187e: 0x000a, 0x187f: 0x000a, + // Block 0x62, offset 0x1880 + 0x1880: 0x000a, 0x1881: 0x000a, 0x1882: 0x000a, 0x1883: 0x000a, 0x1884: 0x000a, 0x1885: 0x009a, + 0x1886: 0x008a, 0x1887: 0x000a, 0x1888: 0x000a, 0x1889: 0x000a, 0x188a: 0x000a, 0x188b: 0x000a, + 0x188c: 0x000a, 0x188d: 0x000a, 0x188e: 0x000a, 0x188f: 0x000a, 0x1890: 0x000a, 0x1891: 0x000a, + 0x1892: 0x000a, 0x1893: 0x000a, 0x1894: 0x000a, 0x1895: 0x000a, 0x1896: 0x000a, 0x1897: 0x000a, + 0x1898: 0x000a, 0x1899: 0x000a, 0x189a: 0x000a, 0x189b: 0x000a, 0x189c: 0x000a, 0x189d: 0x000a, + 0x189e: 0x000a, 0x189f: 0x000a, 0x18a0: 0x000a, 0x18a1: 0x000a, 0x18a2: 0x000a, 0x18a3: 0x000a, + 0x18a4: 0x000a, 0x18a5: 0x000a, 0x18a6: 0x003a, 0x18a7: 0x002a, 0x18a8: 0x003a, 0x18a9: 0x002a, + 0x18aa: 0x003a, 0x18ab: 0x002a, 0x18ac: 0x003a, 0x18ad: 0x002a, 0x18ae: 0x003a, 0x18af: 0x002a, + 0x18b0: 0x000a, 0x18b1: 0x000a, 0x18b2: 0x000a, 0x18b3: 0x000a, 0x18b4: 0x000a, 0x18b5: 0x000a, + 0x18b6: 0x000a, 0x18b7: 0x000a, 0x18b8: 0x000a, 0x18b9: 0x000a, 0x18ba: 0x000a, 0x18bb: 0x000a, + 0x18bc: 0x000a, 0x18bd: 0x000a, 0x18be: 0x000a, 0x18bf: 0x000a, + // Block 0x63, offset 0x18c0 + 0x18c0: 0x000a, 0x18c1: 0x000a, 0x18c2: 0x000a, 0x18c3: 0x007a, 0x18c4: 0x006a, 0x18c5: 0x009a, + 0x18c6: 0x008a, 0x18c7: 0x00ba, 0x18c8: 0x00aa, 0x18c9: 0x009a, 0x18ca: 0x008a, 0x18cb: 0x007a, + 0x18cc: 0x006a, 0x18cd: 0x00da, 0x18ce: 0x002a, 0x18cf: 0x003a, 0x18d0: 0x00ca, 0x18d1: 0x009a, + 0x18d2: 0x008a, 0x18d3: 0x007a, 0x18d4: 0x006a, 0x18d5: 0x009a, 0x18d6: 0x008a, 0x18d7: 0x00ba, + 0x18d8: 0x00aa, 0x18d9: 0x000a, 0x18da: 0x000a, 0x18db: 0x000a, 0x18dc: 0x000a, 0x18dd: 0x000a, + 0x18de: 0x000a, 0x18df: 0x000a, 0x18e0: 0x000a, 0x18e1: 0x000a, 0x18e2: 0x000a, 0x18e3: 0x000a, + 0x18e4: 0x000a, 0x18e5: 0x000a, 0x18e6: 0x000a, 0x18e7: 0x000a, 0x18e8: 0x000a, 0x18e9: 0x000a, + 0x18ea: 0x000a, 0x18eb: 0x000a, 0x18ec: 0x000a, 0x18ed: 0x000a, 0x18ee: 0x000a, 0x18ef: 0x000a, + 0x18f0: 0x000a, 0x18f1: 0x000a, 0x18f2: 0x000a, 0x18f3: 0x000a, 0x18f4: 0x000a, 0x18f5: 0x000a, + 0x18f6: 0x000a, 0x18f7: 0x000a, 0x18f8: 0x000a, 0x18f9: 0x000a, 0x18fa: 0x000a, 0x18fb: 0x000a, + 0x18fc: 0x000a, 0x18fd: 0x000a, 0x18fe: 0x000a, 0x18ff: 0x000a, + // Block 0x64, offset 0x1900 + 0x1900: 0x000a, 0x1901: 0x000a, 0x1902: 0x000a, 0x1903: 0x000a, 0x1904: 0x000a, 0x1905: 0x000a, + 0x1906: 0x000a, 0x1907: 0x000a, 0x1908: 0x000a, 0x1909: 0x000a, 0x190a: 0x000a, 0x190b: 0x000a, + 0x190c: 0x000a, 0x190d: 0x000a, 0x190e: 0x000a, 0x190f: 0x000a, 0x1910: 0x000a, 0x1911: 0x000a, + 0x1912: 0x000a, 0x1913: 0x000a, 0x1914: 0x000a, 0x1915: 0x000a, 0x1916: 0x000a, 0x1917: 0x000a, + 0x1918: 0x003a, 0x1919: 0x002a, 0x191a: 0x003a, 0x191b: 0x002a, 0x191c: 0x000a, 0x191d: 0x000a, + 0x191e: 0x000a, 0x191f: 0x000a, 0x1920: 0x000a, 0x1921: 0x000a, 0x1922: 0x000a, 0x1923: 0x000a, + 0x1924: 0x000a, 0x1925: 0x000a, 0x1926: 0x000a, 0x1927: 0x000a, 0x1928: 0x000a, 0x1929: 0x000a, + 0x192a: 0x000a, 0x192b: 0x000a, 0x192c: 0x000a, 0x192d: 0x000a, 0x192e: 0x000a, 0x192f: 0x000a, + 0x1930: 0x000a, 0x1931: 0x000a, 0x1932: 0x000a, 0x1933: 0x000a, 0x1934: 0x000a, 0x1935: 0x000a, + 0x1936: 0x000a, 0x1937: 0x000a, 0x1938: 0x000a, 0x1939: 0x000a, 0x193a: 0x000a, 0x193b: 0x000a, + 0x193c: 0x003a, 0x193d: 0x002a, 0x193e: 0x000a, 0x193f: 0x000a, + // Block 0x65, offset 0x1940 + 0x1940: 0x000a, 0x1941: 0x000a, 0x1942: 0x000a, 0x1943: 0x000a, 0x1944: 0x000a, 0x1945: 0x000a, + 0x1946: 0x000a, 0x1947: 0x000a, 0x1948: 0x000a, 0x1949: 0x000a, 0x194a: 0x000a, 0x194b: 0x000a, + 0x194c: 0x000a, 0x194d: 0x000a, 0x194e: 0x000a, 0x194f: 0x000a, 0x1950: 0x000a, 0x1951: 0x000a, + 0x1952: 0x000a, 0x1953: 0x000a, 0x1954: 0x000a, 0x1955: 0x000a, 0x1956: 0x000a, 0x1957: 0x000a, + 0x1958: 0x000a, 0x1959: 0x000a, 0x195a: 0x000a, 0x195b: 0x000a, 0x195c: 0x000a, 0x195d: 0x000a, + 0x195e: 0x000a, 0x195f: 0x000a, 0x1960: 0x000a, 0x1961: 0x000a, 0x1962: 0x000a, 0x1963: 0x000a, + 0x1964: 0x000a, 0x1965: 0x000a, 0x1966: 0x000a, 0x1967: 0x000a, 0x1968: 0x000a, 0x1969: 0x000a, + 0x196a: 0x000a, 0x196b: 0x000a, 0x196c: 0x000a, 0x196d: 0x000a, 0x196e: 0x000a, 0x196f: 0x000a, + 0x1970: 0x000a, 0x1971: 0x000a, 0x1972: 0x000a, 0x1973: 0x000a, + 0x1976: 0x000a, 0x1977: 0x000a, 0x1978: 0x000a, 0x1979: 0x000a, 0x197a: 0x000a, 0x197b: 0x000a, + 0x197c: 0x000a, 0x197d: 0x000a, 0x197e: 0x000a, 0x197f: 0x000a, + // Block 0x66, offset 0x1980 + 0x1980: 0x000a, 0x1981: 0x000a, 0x1982: 0x000a, 0x1983: 0x000a, 0x1984: 0x000a, 0x1985: 0x000a, + 0x1986: 0x000a, 0x1987: 0x000a, 0x1988: 0x000a, 0x1989: 0x000a, 0x198a: 0x000a, 0x198b: 0x000a, + 0x198c: 0x000a, 0x198d: 0x000a, 0x198e: 0x000a, 0x198f: 0x000a, 0x1990: 0x000a, 0x1991: 0x000a, + 0x1992: 0x000a, 0x1993: 0x000a, 0x1994: 0x000a, 0x1995: 0x000a, + 0x1998: 0x000a, 0x1999: 0x000a, 0x199a: 0x000a, 0x199b: 0x000a, 0x199c: 0x000a, 0x199d: 0x000a, + 0x199e: 0x000a, 0x199f: 0x000a, 0x19a0: 0x000a, 0x19a1: 0x000a, 0x19a2: 0x000a, 0x19a3: 0x000a, + 0x19a4: 0x000a, 0x19a5: 0x000a, 0x19a6: 0x000a, 0x19a7: 0x000a, 0x19a8: 0x000a, 0x19a9: 0x000a, + 0x19aa: 0x000a, 0x19ab: 0x000a, 0x19ac: 0x000a, 0x19ad: 0x000a, 0x19ae: 0x000a, 0x19af: 0x000a, + 0x19b0: 0x000a, 0x19b1: 0x000a, 0x19b2: 0x000a, 0x19b3: 0x000a, 0x19b4: 0x000a, 0x19b5: 0x000a, + 0x19b6: 0x000a, 0x19b7: 0x000a, 0x19b8: 0x000a, 0x19b9: 0x000a, + 0x19bd: 0x000a, 0x19be: 0x000a, 0x19bf: 0x000a, + // Block 0x67, offset 0x19c0 + 0x19c0: 0x000a, 0x19c1: 0x000a, 0x19c2: 0x000a, 0x19c3: 0x000a, 0x19c4: 0x000a, 0x19c5: 0x000a, + 0x19c6: 0x000a, 0x19c7: 0x000a, 0x19c8: 0x000a, 0x19ca: 0x000a, 0x19cb: 0x000a, + 0x19cc: 0x000a, 0x19cd: 0x000a, 0x19ce: 0x000a, 0x19cf: 0x000a, 0x19d0: 0x000a, 0x19d1: 0x000a, + 0x19ec: 0x000a, 0x19ed: 0x000a, 0x19ee: 0x000a, 0x19ef: 0x000a, + // Block 0x68, offset 0x1a00 + 0x1a25: 0x000a, 0x1a26: 0x000a, 0x1a27: 0x000a, 0x1a28: 0x000a, 0x1a29: 0x000a, + 0x1a2a: 0x000a, 0x1a2f: 0x000c, + 0x1a30: 0x000c, 0x1a31: 0x000c, + 0x1a39: 0x000a, 0x1a3a: 0x000a, 0x1a3b: 0x000a, + 0x1a3c: 0x000a, 0x1a3d: 0x000a, 0x1a3e: 0x000a, 0x1a3f: 0x000a, + // Block 0x69, offset 0x1a40 + 0x1a7f: 0x000c, + // Block 0x6a, offset 0x1a80 + 0x1aa0: 0x000c, 0x1aa1: 0x000c, 0x1aa2: 0x000c, 0x1aa3: 0x000c, + 0x1aa4: 0x000c, 0x1aa5: 0x000c, 0x1aa6: 0x000c, 0x1aa7: 0x000c, 0x1aa8: 0x000c, 0x1aa9: 0x000c, + 0x1aaa: 0x000c, 0x1aab: 0x000c, 0x1aac: 0x000c, 0x1aad: 0x000c, 0x1aae: 0x000c, 0x1aaf: 0x000c, + 0x1ab0: 0x000c, 0x1ab1: 0x000c, 0x1ab2: 0x000c, 0x1ab3: 0x000c, 0x1ab4: 0x000c, 0x1ab5: 0x000c, + 0x1ab6: 0x000c, 0x1ab7: 0x000c, 0x1ab8: 0x000c, 0x1ab9: 0x000c, 0x1aba: 0x000c, 0x1abb: 0x000c, + 0x1abc: 0x000c, 0x1abd: 0x000c, 0x1abe: 0x000c, 0x1abf: 0x000c, + // Block 0x6b, offset 0x1ac0 + 0x1ac0: 0x000a, 0x1ac1: 0x000a, 0x1ac2: 0x000a, 0x1ac3: 0x000a, 0x1ac4: 0x000a, 0x1ac5: 0x000a, + 0x1ac6: 0x000a, 0x1ac7: 0x000a, 0x1ac8: 0x000a, 0x1ac9: 0x000a, 0x1aca: 0x000a, 0x1acb: 0x000a, + 0x1acc: 0x000a, 0x1acd: 0x000a, 0x1ace: 0x000a, 0x1acf: 0x000a, 0x1ad0: 0x000a, 0x1ad1: 0x000a, + 0x1ad2: 0x000a, 0x1ad3: 0x000a, 0x1ad4: 0x000a, 0x1ad5: 0x000a, 0x1ad6: 0x000a, 0x1ad7: 0x000a, + 0x1ad8: 0x000a, 0x1ad9: 0x000a, 0x1ada: 0x000a, 0x1adb: 0x000a, 0x1adc: 0x000a, 0x1add: 0x000a, + 0x1ade: 0x000a, 0x1adf: 0x000a, 0x1ae0: 0x000a, 0x1ae1: 0x000a, 0x1ae2: 0x003a, 0x1ae3: 0x002a, + 0x1ae4: 0x003a, 0x1ae5: 0x002a, 0x1ae6: 0x003a, 0x1ae7: 0x002a, 0x1ae8: 0x003a, 0x1ae9: 0x002a, + 0x1aea: 0x000a, 0x1aeb: 0x000a, 0x1aec: 0x000a, 0x1aed: 0x000a, 0x1aee: 0x000a, 0x1aef: 0x000a, + 0x1af0: 0x000a, 0x1af1: 0x000a, 0x1af2: 0x000a, 0x1af3: 0x000a, 0x1af4: 0x000a, 0x1af5: 0x000a, + 0x1af6: 0x000a, 0x1af7: 0x000a, 0x1af8: 0x000a, 0x1af9: 0x000a, 0x1afa: 0x000a, 0x1afb: 0x000a, + 0x1afc: 0x000a, 0x1afd: 0x000a, 0x1afe: 0x000a, 0x1aff: 0x000a, + // Block 0x6c, offset 0x1b00 + 0x1b00: 0x000a, 0x1b01: 0x000a, 0x1b02: 0x000a, 0x1b03: 0x000a, 0x1b04: 0x000a, + // Block 0x6d, offset 0x1b40 + 0x1b40: 0x000a, 0x1b41: 0x000a, 0x1b42: 0x000a, 0x1b43: 0x000a, 0x1b44: 0x000a, 0x1b45: 0x000a, + 0x1b46: 0x000a, 0x1b47: 0x000a, 0x1b48: 0x000a, 0x1b49: 0x000a, 0x1b4a: 0x000a, 0x1b4b: 0x000a, + 0x1b4c: 0x000a, 0x1b4d: 0x000a, 0x1b4e: 0x000a, 0x1b4f: 0x000a, 0x1b50: 0x000a, 0x1b51: 0x000a, + 0x1b52: 0x000a, 0x1b53: 0x000a, 0x1b54: 0x000a, 0x1b55: 0x000a, 0x1b56: 0x000a, 0x1b57: 0x000a, + 0x1b58: 0x000a, 0x1b59: 0x000a, 0x1b5b: 0x000a, 0x1b5c: 0x000a, 0x1b5d: 0x000a, + 0x1b5e: 0x000a, 0x1b5f: 0x000a, 0x1b60: 0x000a, 0x1b61: 0x000a, 0x1b62: 0x000a, 0x1b63: 0x000a, + 0x1b64: 0x000a, 0x1b65: 0x000a, 0x1b66: 0x000a, 0x1b67: 0x000a, 0x1b68: 0x000a, 0x1b69: 0x000a, + 0x1b6a: 0x000a, 0x1b6b: 0x000a, 0x1b6c: 0x000a, 0x1b6d: 0x000a, 0x1b6e: 0x000a, 0x1b6f: 0x000a, + 0x1b70: 0x000a, 0x1b71: 0x000a, 0x1b72: 0x000a, 0x1b73: 0x000a, 0x1b74: 0x000a, 0x1b75: 0x000a, + 0x1b76: 0x000a, 0x1b77: 0x000a, 0x1b78: 0x000a, 0x1b79: 0x000a, 0x1b7a: 0x000a, 0x1b7b: 0x000a, + 0x1b7c: 0x000a, 0x1b7d: 0x000a, 0x1b7e: 0x000a, 0x1b7f: 0x000a, + // Block 0x6e, offset 0x1b80 + 0x1b80: 0x000a, 0x1b81: 0x000a, 0x1b82: 0x000a, 0x1b83: 0x000a, 0x1b84: 0x000a, 0x1b85: 0x000a, + 0x1b86: 0x000a, 0x1b87: 0x000a, 0x1b88: 0x000a, 0x1b89: 0x000a, 0x1b8a: 0x000a, 0x1b8b: 0x000a, + 0x1b8c: 0x000a, 0x1b8d: 0x000a, 0x1b8e: 0x000a, 0x1b8f: 0x000a, 0x1b90: 0x000a, 0x1b91: 0x000a, + 0x1b92: 0x000a, 0x1b93: 0x000a, 0x1b94: 0x000a, 0x1b95: 0x000a, 0x1b96: 0x000a, 0x1b97: 0x000a, + 0x1b98: 0x000a, 0x1b99: 0x000a, 0x1b9a: 0x000a, 0x1b9b: 0x000a, 0x1b9c: 0x000a, 0x1b9d: 0x000a, + 0x1b9e: 0x000a, 0x1b9f: 0x000a, 0x1ba0: 0x000a, 0x1ba1: 0x000a, 0x1ba2: 0x000a, 0x1ba3: 0x000a, + 0x1ba4: 0x000a, 0x1ba5: 0x000a, 0x1ba6: 0x000a, 0x1ba7: 0x000a, 0x1ba8: 0x000a, 0x1ba9: 0x000a, + 0x1baa: 0x000a, 0x1bab: 0x000a, 0x1bac: 0x000a, 0x1bad: 0x000a, 0x1bae: 0x000a, 0x1baf: 0x000a, + 0x1bb0: 0x000a, 0x1bb1: 0x000a, 0x1bb2: 0x000a, 0x1bb3: 0x000a, + // Block 0x6f, offset 0x1bc0 + 0x1bc0: 0x000a, 0x1bc1: 0x000a, 0x1bc2: 0x000a, 0x1bc3: 0x000a, 0x1bc4: 0x000a, 0x1bc5: 0x000a, + 0x1bc6: 0x000a, 0x1bc7: 0x000a, 0x1bc8: 0x000a, 0x1bc9: 0x000a, 0x1bca: 0x000a, 0x1bcb: 0x000a, + 0x1bcc: 0x000a, 0x1bcd: 0x000a, 0x1bce: 0x000a, 0x1bcf: 0x000a, 0x1bd0: 0x000a, 0x1bd1: 0x000a, + 0x1bd2: 0x000a, 0x1bd3: 0x000a, 0x1bd4: 0x000a, 0x1bd5: 0x000a, + 0x1bf0: 0x000a, 0x1bf1: 0x000a, 0x1bf2: 0x000a, 0x1bf3: 0x000a, 0x1bf4: 0x000a, 0x1bf5: 0x000a, + 0x1bf6: 0x000a, 0x1bf7: 0x000a, 0x1bf8: 0x000a, 0x1bf9: 0x000a, 0x1bfa: 0x000a, 0x1bfb: 0x000a, + // Block 0x70, offset 0x1c00 + 0x1c00: 0x0009, 0x1c01: 0x000a, 0x1c02: 0x000a, 0x1c03: 0x000a, 0x1c04: 0x000a, + 0x1c08: 0x003a, 0x1c09: 0x002a, 0x1c0a: 0x003a, 0x1c0b: 0x002a, + 0x1c0c: 0x003a, 0x1c0d: 0x002a, 0x1c0e: 0x003a, 0x1c0f: 0x002a, 0x1c10: 0x003a, 0x1c11: 0x002a, + 0x1c12: 0x000a, 0x1c13: 0x000a, 0x1c14: 0x003a, 0x1c15: 0x002a, 0x1c16: 0x003a, 0x1c17: 0x002a, + 0x1c18: 0x003a, 0x1c19: 0x002a, 0x1c1a: 0x003a, 0x1c1b: 0x002a, 0x1c1c: 0x000a, 0x1c1d: 0x000a, + 0x1c1e: 0x000a, 0x1c1f: 0x000a, 0x1c20: 0x000a, + 0x1c2a: 0x000c, 0x1c2b: 0x000c, 0x1c2c: 0x000c, 0x1c2d: 0x000c, + 0x1c30: 0x000a, + 0x1c36: 0x000a, 0x1c37: 0x000a, + 0x1c3d: 0x000a, 0x1c3e: 0x000a, 0x1c3f: 0x000a, + // Block 0x71, offset 0x1c40 + 0x1c59: 0x000c, 0x1c5a: 0x000c, 0x1c5b: 0x000a, 0x1c5c: 0x000a, + 0x1c60: 0x000a, + // Block 0x72, offset 0x1c80 + 0x1cbb: 0x000a, + // Block 0x73, offset 0x1cc0 + 0x1cc0: 0x000a, 0x1cc1: 0x000a, 0x1cc2: 0x000a, 0x1cc3: 0x000a, 0x1cc4: 0x000a, 0x1cc5: 0x000a, + 0x1cc6: 0x000a, 0x1cc7: 0x000a, 0x1cc8: 0x000a, 0x1cc9: 0x000a, 0x1cca: 0x000a, 0x1ccb: 0x000a, + 0x1ccc: 0x000a, 0x1ccd: 0x000a, 0x1cce: 0x000a, 0x1ccf: 0x000a, 0x1cd0: 0x000a, 0x1cd1: 0x000a, + 0x1cd2: 0x000a, 0x1cd3: 0x000a, 0x1cd4: 0x000a, 0x1cd5: 0x000a, 0x1cd6: 0x000a, 0x1cd7: 0x000a, + 0x1cd8: 0x000a, 0x1cd9: 0x000a, 0x1cda: 0x000a, 0x1cdb: 0x000a, 0x1cdc: 0x000a, 0x1cdd: 0x000a, + 0x1cde: 0x000a, 0x1cdf: 0x000a, 0x1ce0: 0x000a, 0x1ce1: 0x000a, 0x1ce2: 0x000a, 0x1ce3: 0x000a, + // Block 0x74, offset 0x1d00 + 0x1d1d: 0x000a, + 0x1d1e: 0x000a, + // Block 0x75, offset 0x1d40 + 0x1d50: 0x000a, 0x1d51: 0x000a, + 0x1d52: 0x000a, 0x1d53: 0x000a, 0x1d54: 0x000a, 0x1d55: 0x000a, 0x1d56: 0x000a, 0x1d57: 0x000a, + 0x1d58: 0x000a, 0x1d59: 0x000a, 0x1d5a: 0x000a, 0x1d5b: 0x000a, 0x1d5c: 0x000a, 0x1d5d: 0x000a, + 0x1d5e: 0x000a, 0x1d5f: 0x000a, + 0x1d7c: 0x000a, 0x1d7d: 0x000a, 0x1d7e: 0x000a, + // Block 0x76, offset 0x1d80 + 0x1db1: 0x000a, 0x1db2: 0x000a, 0x1db3: 0x000a, 0x1db4: 0x000a, 0x1db5: 0x000a, + 0x1db6: 0x000a, 0x1db7: 0x000a, 0x1db8: 0x000a, 0x1db9: 0x000a, 0x1dba: 0x000a, 0x1dbb: 0x000a, + 0x1dbc: 0x000a, 0x1dbd: 0x000a, 0x1dbe: 0x000a, 0x1dbf: 0x000a, + // Block 0x77, offset 0x1dc0 + 0x1dcc: 0x000a, 0x1dcd: 0x000a, 0x1dce: 0x000a, 0x1dcf: 0x000a, + // Block 0x78, offset 0x1e00 + 0x1e37: 0x000a, 0x1e38: 0x000a, 0x1e39: 0x000a, 0x1e3a: 0x000a, + // Block 0x79, offset 0x1e40 + 0x1e5e: 0x000a, 0x1e5f: 0x000a, + 0x1e7f: 0x000a, + // Block 0x7a, offset 0x1e80 + 0x1e90: 0x000a, 0x1e91: 0x000a, + 0x1e92: 0x000a, 0x1e93: 0x000a, 0x1e94: 0x000a, 0x1e95: 0x000a, 0x1e96: 0x000a, 0x1e97: 0x000a, + 0x1e98: 0x000a, 0x1e99: 0x000a, 0x1e9a: 0x000a, 0x1e9b: 0x000a, 0x1e9c: 0x000a, 0x1e9d: 0x000a, + 0x1e9e: 0x000a, 0x1e9f: 0x000a, 0x1ea0: 0x000a, 0x1ea1: 0x000a, 0x1ea2: 0x000a, 0x1ea3: 0x000a, + 0x1ea4: 0x000a, 0x1ea5: 0x000a, 0x1ea6: 0x000a, 0x1ea7: 0x000a, 0x1ea8: 0x000a, 0x1ea9: 0x000a, + 0x1eaa: 0x000a, 0x1eab: 0x000a, 0x1eac: 0x000a, 0x1ead: 0x000a, 0x1eae: 0x000a, 0x1eaf: 0x000a, + 0x1eb0: 0x000a, 0x1eb1: 0x000a, 0x1eb2: 0x000a, 0x1eb3: 0x000a, 0x1eb4: 0x000a, 0x1eb5: 0x000a, + 0x1eb6: 0x000a, 0x1eb7: 0x000a, 0x1eb8: 0x000a, 0x1eb9: 0x000a, 0x1eba: 0x000a, 0x1ebb: 0x000a, + 0x1ebc: 0x000a, 0x1ebd: 0x000a, 0x1ebe: 0x000a, 0x1ebf: 0x000a, + // Block 0x7b, offset 0x1ec0 + 0x1ec0: 0x000a, 0x1ec1: 0x000a, 0x1ec2: 0x000a, 0x1ec3: 0x000a, 0x1ec4: 0x000a, 0x1ec5: 0x000a, + 0x1ec6: 0x000a, + // Block 0x7c, offset 0x1f00 + 0x1f0d: 0x000a, 0x1f0e: 0x000a, 0x1f0f: 0x000a, + // Block 0x7d, offset 0x1f40 + 0x1f6f: 0x000c, + 0x1f70: 0x000c, 0x1f71: 0x000c, 0x1f72: 0x000c, 0x1f73: 0x000a, 0x1f74: 0x000c, 0x1f75: 0x000c, + 0x1f76: 0x000c, 0x1f77: 0x000c, 0x1f78: 0x000c, 0x1f79: 0x000c, 0x1f7a: 0x000c, 0x1f7b: 0x000c, + 0x1f7c: 0x000c, 0x1f7d: 0x000c, 0x1f7e: 0x000a, 0x1f7f: 0x000a, + // Block 0x7e, offset 0x1f80 + 0x1f9e: 0x000c, 0x1f9f: 0x000c, + // Block 0x7f, offset 0x1fc0 + 0x1ff0: 0x000c, 0x1ff1: 0x000c, + // Block 0x80, offset 0x2000 + 0x2000: 0x000a, 0x2001: 0x000a, 0x2002: 0x000a, 0x2003: 0x000a, 0x2004: 0x000a, 0x2005: 0x000a, + 0x2006: 0x000a, 0x2007: 0x000a, 0x2008: 0x000a, 0x2009: 0x000a, 0x200a: 0x000a, 0x200b: 0x000a, + 0x200c: 0x000a, 0x200d: 0x000a, 0x200e: 0x000a, 0x200f: 0x000a, 0x2010: 0x000a, 0x2011: 0x000a, + 0x2012: 0x000a, 0x2013: 0x000a, 0x2014: 0x000a, 0x2015: 0x000a, 0x2016: 0x000a, 0x2017: 0x000a, + 0x2018: 0x000a, 0x2019: 0x000a, 0x201a: 0x000a, 0x201b: 0x000a, 0x201c: 0x000a, 0x201d: 0x000a, + 0x201e: 0x000a, 0x201f: 0x000a, 0x2020: 0x000a, 0x2021: 0x000a, + // Block 0x81, offset 0x2040 + 0x2048: 0x000a, + // Block 0x82, offset 0x2080 + 0x2082: 0x000c, + 0x2086: 0x000c, 0x208b: 0x000c, + 0x20a5: 0x000c, 0x20a6: 0x000c, 0x20a8: 0x000a, 0x20a9: 0x000a, + 0x20aa: 0x000a, 0x20ab: 0x000a, + 0x20b8: 0x0004, 0x20b9: 0x0004, + // Block 0x83, offset 0x20c0 + 0x20f4: 0x000a, 0x20f5: 0x000a, + 0x20f6: 0x000a, 0x20f7: 0x000a, + // Block 0x84, offset 0x2100 + 0x2104: 0x000c, 0x2105: 0x000c, + 0x2120: 0x000c, 0x2121: 0x000c, 0x2122: 0x000c, 0x2123: 0x000c, + 0x2124: 0x000c, 0x2125: 0x000c, 0x2126: 0x000c, 0x2127: 0x000c, 0x2128: 0x000c, 0x2129: 0x000c, + 0x212a: 0x000c, 0x212b: 0x000c, 0x212c: 0x000c, 0x212d: 0x000c, 0x212e: 0x000c, 0x212f: 0x000c, + 0x2130: 0x000c, 0x2131: 0x000c, + // Block 0x85, offset 0x2140 + 0x2166: 0x000c, 0x2167: 0x000c, 0x2168: 0x000c, 0x2169: 0x000c, + 0x216a: 0x000c, 0x216b: 0x000c, 0x216c: 0x000c, 0x216d: 0x000c, + // Block 0x86, offset 0x2180 + 0x2187: 0x000c, 0x2188: 0x000c, 0x2189: 0x000c, 0x218a: 0x000c, 0x218b: 0x000c, + 0x218c: 0x000c, 0x218d: 0x000c, 0x218e: 0x000c, 0x218f: 0x000c, 0x2190: 0x000c, 0x2191: 0x000c, + // Block 0x87, offset 0x21c0 + 0x21c0: 0x000c, 0x21c1: 0x000c, 0x21c2: 0x000c, + 0x21f3: 0x000c, + 0x21f6: 0x000c, 0x21f7: 0x000c, 0x21f8: 0x000c, 0x21f9: 0x000c, + 0x21fc: 0x000c, + // Block 0x88, offset 0x2200 + 0x2225: 0x000c, + // Block 0x89, offset 0x2240 + 0x2269: 0x000c, + 0x226a: 0x000c, 0x226b: 0x000c, 0x226c: 0x000c, 0x226d: 0x000c, 0x226e: 0x000c, + 0x2271: 0x000c, 0x2272: 0x000c, 0x2275: 0x000c, + 0x2276: 0x000c, + // Block 0x8a, offset 0x2280 + 0x2283: 0x000c, + 0x228c: 0x000c, + 0x22bc: 0x000c, + // Block 0x8b, offset 0x22c0 + 0x22f0: 0x000c, 0x22f2: 0x000c, 0x22f3: 0x000c, 0x22f4: 0x000c, + 0x22f7: 0x000c, 0x22f8: 0x000c, + 0x22fe: 0x000c, 0x22ff: 0x000c, + // Block 0x8c, offset 0x2300 + 0x2301: 0x000c, + 0x232c: 0x000c, 0x232d: 0x000c, + 0x2336: 0x000c, + // Block 0x8d, offset 0x2340 + 0x2365: 0x000c, 0x2368: 0x000c, + 0x236d: 0x000c, + // Block 0x8e, offset 0x2380 + 0x239d: 0x0001, + 0x239e: 0x000c, 0x239f: 0x0001, 0x23a0: 0x0001, 0x23a1: 0x0001, 0x23a2: 0x0001, 0x23a3: 0x0001, + 0x23a4: 0x0001, 0x23a5: 0x0001, 0x23a6: 0x0001, 0x23a7: 0x0001, 0x23a8: 0x0001, 0x23a9: 0x0003, + 0x23aa: 0x0001, 0x23ab: 0x0001, 0x23ac: 0x0001, 0x23ad: 0x0001, 0x23ae: 0x0001, 0x23af: 0x0001, + 0x23b0: 0x0001, 0x23b1: 0x0001, 0x23b2: 0x0001, 0x23b3: 0x0001, 0x23b4: 0x0001, 0x23b5: 0x0001, + 0x23b6: 0x0001, 0x23b7: 0x0001, 0x23b8: 0x0001, 0x23b9: 0x0001, 0x23ba: 0x0001, 0x23bb: 0x0001, + 0x23bc: 0x0001, 0x23bd: 0x0001, 0x23be: 0x0001, 0x23bf: 0x0001, + // Block 0x8f, offset 0x23c0 + 0x23c0: 0x0001, 0x23c1: 0x0001, 0x23c2: 0x0001, 0x23c3: 0x0001, 0x23c4: 0x0001, 0x23c5: 0x0001, + 0x23c6: 0x0001, 0x23c7: 0x0001, 0x23c8: 0x0001, 0x23c9: 0x0001, 0x23ca: 0x0001, 0x23cb: 0x0001, + 0x23cc: 0x0001, 0x23cd: 0x0001, 0x23ce: 0x0001, 0x23cf: 0x0001, 0x23d0: 0x000d, 0x23d1: 0x000d, + 0x23d2: 0x000d, 0x23d3: 0x000d, 0x23d4: 0x000d, 0x23d5: 0x000d, 0x23d6: 0x000d, 0x23d7: 0x000d, + 0x23d8: 0x000d, 0x23d9: 0x000d, 0x23da: 0x000d, 0x23db: 0x000d, 0x23dc: 0x000d, 0x23dd: 0x000d, + 0x23de: 0x000d, 0x23df: 0x000d, 0x23e0: 0x000d, 0x23e1: 0x000d, 0x23e2: 0x000d, 0x23e3: 0x000d, + 0x23e4: 0x000d, 0x23e5: 0x000d, 0x23e6: 0x000d, 0x23e7: 0x000d, 0x23e8: 0x000d, 0x23e9: 0x000d, + 0x23ea: 0x000d, 0x23eb: 0x000d, 0x23ec: 0x000d, 0x23ed: 0x000d, 0x23ee: 0x000d, 0x23ef: 0x000d, + 0x23f0: 0x000d, 0x23f1: 0x000d, 0x23f2: 0x000d, 0x23f3: 0x000d, 0x23f4: 0x000d, 0x23f5: 0x000d, + 0x23f6: 0x000d, 0x23f7: 0x000d, 0x23f8: 0x000d, 0x23f9: 0x000d, 0x23fa: 0x000d, 0x23fb: 0x000d, + 0x23fc: 0x000d, 0x23fd: 0x000d, 0x23fe: 0x000d, 0x23ff: 0x000d, + // Block 0x90, offset 0x2400 + 0x2400: 0x000d, 0x2401: 0x000d, 0x2402: 0x000d, 0x2403: 0x000d, 0x2404: 0x000d, 0x2405: 0x000d, + 0x2406: 0x000d, 0x2407: 0x000d, 0x2408: 0x000d, 0x2409: 0x000d, 0x240a: 0x000d, 0x240b: 0x000d, + 0x240c: 0x000d, 0x240d: 0x000d, 0x240e: 0x000d, 0x240f: 0x000d, 0x2410: 0x000d, 0x2411: 0x000d, + 0x2412: 0x000d, 0x2413: 0x000d, 0x2414: 0x000d, 0x2415: 0x000d, 0x2416: 0x000d, 0x2417: 0x000d, + 0x2418: 0x000d, 0x2419: 0x000d, 0x241a: 0x000d, 0x241b: 0x000d, 0x241c: 0x000d, 0x241d: 0x000d, + 0x241e: 0x000d, 0x241f: 0x000d, 0x2420: 0x000d, 0x2421: 0x000d, 0x2422: 0x000d, 0x2423: 0x000d, + 0x2424: 0x000d, 0x2425: 0x000d, 0x2426: 0x000d, 0x2427: 0x000d, 0x2428: 0x000d, 0x2429: 0x000d, + 0x242a: 0x000d, 0x242b: 0x000d, 0x242c: 0x000d, 0x242d: 0x000d, 0x242e: 0x000d, 0x242f: 0x000d, + 0x2430: 0x000d, 0x2431: 0x000d, 0x2432: 0x000d, 0x2433: 0x000d, 0x2434: 0x000d, 0x2435: 0x000d, + 0x2436: 0x000d, 0x2437: 0x000d, 0x2438: 0x000d, 0x2439: 0x000d, 0x243a: 0x000d, 0x243b: 0x000d, + 0x243c: 0x000d, 0x243d: 0x000d, 0x243e: 0x000a, 0x243f: 0x000a, + // Block 0x91, offset 0x2440 + 0x2440: 0x000d, 0x2441: 0x000d, 0x2442: 0x000d, 0x2443: 0x000d, 0x2444: 0x000d, 0x2445: 0x000d, + 0x2446: 0x000d, 0x2447: 0x000d, 0x2448: 0x000d, 0x2449: 0x000d, 0x244a: 0x000d, 0x244b: 0x000d, + 0x244c: 0x000d, 0x244d: 0x000d, 0x244e: 0x000d, 0x244f: 0x000d, 0x2450: 0x000b, 0x2451: 0x000b, + 0x2452: 0x000b, 0x2453: 0x000b, 0x2454: 0x000b, 0x2455: 0x000b, 0x2456: 0x000b, 0x2457: 0x000b, + 0x2458: 0x000b, 0x2459: 0x000b, 0x245a: 0x000b, 0x245b: 0x000b, 0x245c: 0x000b, 0x245d: 0x000b, + 0x245e: 0x000b, 0x245f: 0x000b, 0x2460: 0x000b, 0x2461: 0x000b, 0x2462: 0x000b, 0x2463: 0x000b, + 0x2464: 0x000b, 0x2465: 0x000b, 0x2466: 0x000b, 0x2467: 0x000b, 0x2468: 0x000b, 0x2469: 0x000b, + 0x246a: 0x000b, 0x246b: 0x000b, 0x246c: 0x000b, 0x246d: 0x000b, 0x246e: 0x000b, 0x246f: 0x000b, + 0x2470: 0x000d, 0x2471: 0x000d, 0x2472: 0x000d, 0x2473: 0x000d, 0x2474: 0x000d, 0x2475: 0x000d, + 0x2476: 0x000d, 0x2477: 0x000d, 0x2478: 0x000d, 0x2479: 0x000d, 0x247a: 0x000d, 0x247b: 0x000d, + 0x247c: 0x000d, 0x247d: 0x000a, 0x247e: 0x000d, 0x247f: 0x000d, + // Block 0x92, offset 0x2480 + 0x2480: 0x000c, 0x2481: 0x000c, 0x2482: 0x000c, 0x2483: 0x000c, 0x2484: 0x000c, 0x2485: 0x000c, + 0x2486: 0x000c, 0x2487: 0x000c, 0x2488: 0x000c, 0x2489: 0x000c, 0x248a: 0x000c, 0x248b: 0x000c, + 0x248c: 0x000c, 0x248d: 0x000c, 0x248e: 0x000c, 0x248f: 0x000c, 0x2490: 0x000a, 0x2491: 0x000a, + 0x2492: 0x000a, 0x2493: 0x000a, 0x2494: 0x000a, 0x2495: 0x000a, 0x2496: 0x000a, 0x2497: 0x000a, + 0x2498: 0x000a, 0x2499: 0x000a, + 0x24a0: 0x000c, 0x24a1: 0x000c, 0x24a2: 0x000c, 0x24a3: 0x000c, + 0x24a4: 0x000c, 0x24a5: 0x000c, 0x24a6: 0x000c, 0x24a7: 0x000c, 0x24a8: 0x000c, 0x24a9: 0x000c, + 0x24aa: 0x000c, 0x24ab: 0x000c, 0x24ac: 0x000c, 0x24ad: 0x000c, 0x24ae: 0x000c, 0x24af: 0x000c, + 0x24b0: 0x000a, 0x24b1: 0x000a, 0x24b2: 0x000a, 0x24b3: 0x000a, 0x24b4: 0x000a, 0x24b5: 0x000a, + 0x24b6: 0x000a, 0x24b7: 0x000a, 0x24b8: 0x000a, 0x24b9: 0x000a, 0x24ba: 0x000a, 0x24bb: 0x000a, + 0x24bc: 0x000a, 0x24bd: 0x000a, 0x24be: 0x000a, 0x24bf: 0x000a, + // Block 0x93, offset 0x24c0 + 0x24c0: 0x000a, 0x24c1: 0x000a, 0x24c2: 0x000a, 0x24c3: 0x000a, 0x24c4: 0x000a, 0x24c5: 0x000a, + 0x24c6: 0x000a, 0x24c7: 0x000a, 0x24c8: 0x000a, 0x24c9: 0x000a, 0x24ca: 0x000a, 0x24cb: 0x000a, + 0x24cc: 0x000a, 0x24cd: 0x000a, 0x24ce: 0x000a, 0x24cf: 0x000a, 0x24d0: 0x0006, 0x24d1: 0x000a, + 0x24d2: 0x0006, 0x24d4: 0x000a, 0x24d5: 0x0006, 0x24d6: 0x000a, 0x24d7: 0x000a, + 0x24d8: 0x000a, 0x24d9: 0x009a, 0x24da: 0x008a, 0x24db: 0x007a, 0x24dc: 0x006a, 0x24dd: 0x009a, + 0x24de: 0x008a, 0x24df: 0x0004, 0x24e0: 0x000a, 0x24e1: 0x000a, 0x24e2: 0x0003, 0x24e3: 0x0003, + 0x24e4: 0x000a, 0x24e5: 0x000a, 0x24e6: 0x000a, 0x24e8: 0x000a, 0x24e9: 0x0004, + 0x24ea: 0x0004, 0x24eb: 0x000a, + 0x24f0: 0x000d, 0x24f1: 0x000d, 0x24f2: 0x000d, 0x24f3: 0x000d, 0x24f4: 0x000d, 0x24f5: 0x000d, + 0x24f6: 0x000d, 0x24f7: 0x000d, 0x24f8: 0x000d, 0x24f9: 0x000d, 0x24fa: 0x000d, 0x24fb: 0x000d, + 0x24fc: 0x000d, 0x24fd: 0x000d, 0x24fe: 0x000d, 0x24ff: 0x000d, + // Block 0x94, offset 0x2500 + 0x2500: 0x000d, 0x2501: 0x000d, 0x2502: 0x000d, 0x2503: 0x000d, 0x2504: 0x000d, 0x2505: 0x000d, + 0x2506: 0x000d, 0x2507: 0x000d, 0x2508: 0x000d, 0x2509: 0x000d, 0x250a: 0x000d, 0x250b: 0x000d, + 0x250c: 0x000d, 0x250d: 0x000d, 0x250e: 0x000d, 0x250f: 0x000d, 0x2510: 0x000d, 0x2511: 0x000d, + 0x2512: 0x000d, 0x2513: 0x000d, 0x2514: 0x000d, 0x2515: 0x000d, 0x2516: 0x000d, 0x2517: 0x000d, + 0x2518: 0x000d, 0x2519: 0x000d, 0x251a: 0x000d, 0x251b: 0x000d, 0x251c: 0x000d, 0x251d: 0x000d, + 0x251e: 0x000d, 0x251f: 0x000d, 0x2520: 0x000d, 0x2521: 0x000d, 0x2522: 0x000d, 0x2523: 0x000d, + 0x2524: 0x000d, 0x2525: 0x000d, 0x2526: 0x000d, 0x2527: 0x000d, 0x2528: 0x000d, 0x2529: 0x000d, + 0x252a: 0x000d, 0x252b: 0x000d, 0x252c: 0x000d, 0x252d: 0x000d, 0x252e: 0x000d, 0x252f: 0x000d, + 0x2530: 0x000d, 0x2531: 0x000d, 0x2532: 0x000d, 0x2533: 0x000d, 0x2534: 0x000d, 0x2535: 0x000d, + 0x2536: 0x000d, 0x2537: 0x000d, 0x2538: 0x000d, 0x2539: 0x000d, 0x253a: 0x000d, 0x253b: 0x000d, + 0x253c: 0x000d, 0x253d: 0x000d, 0x253e: 0x000d, 0x253f: 0x000b, + // Block 0x95, offset 0x2540 + 0x2541: 0x000a, 0x2542: 0x000a, 0x2543: 0x0004, 0x2544: 0x0004, 0x2545: 0x0004, + 0x2546: 0x000a, 0x2547: 0x000a, 0x2548: 0x003a, 0x2549: 0x002a, 0x254a: 0x000a, 0x254b: 0x0003, + 0x254c: 0x0006, 0x254d: 0x0003, 0x254e: 0x0006, 0x254f: 0x0006, 0x2550: 0x0002, 0x2551: 0x0002, + 0x2552: 0x0002, 0x2553: 0x0002, 0x2554: 0x0002, 0x2555: 0x0002, 0x2556: 0x0002, 0x2557: 0x0002, + 0x2558: 0x0002, 0x2559: 0x0002, 0x255a: 0x0006, 0x255b: 0x000a, 0x255c: 0x000a, 0x255d: 0x000a, + 0x255e: 0x000a, 0x255f: 0x000a, 0x2560: 0x000a, + 0x257b: 0x005a, + 0x257c: 0x000a, 0x257d: 0x004a, 0x257e: 0x000a, 0x257f: 0x000a, + // Block 0x96, offset 0x2580 + 0x2580: 0x000a, + 0x259b: 0x005a, 0x259c: 0x000a, 0x259d: 0x004a, + 0x259e: 0x000a, 0x259f: 0x00fa, 0x25a0: 0x00ea, 0x25a1: 0x000a, 0x25a2: 0x003a, 0x25a3: 0x002a, + 0x25a4: 0x000a, 0x25a5: 0x000a, + // Block 0x97, offset 0x25c0 + 0x25e0: 0x0004, 0x25e1: 0x0004, 0x25e2: 0x000a, 0x25e3: 0x000a, + 0x25e4: 0x000a, 0x25e5: 0x0004, 0x25e6: 0x0004, 0x25e8: 0x000a, 0x25e9: 0x000a, + 0x25ea: 0x000a, 0x25eb: 0x000a, 0x25ec: 0x000a, 0x25ed: 0x000a, 0x25ee: 0x000a, + 0x25f0: 0x000b, 0x25f1: 0x000b, 0x25f2: 0x000b, 0x25f3: 0x000b, 0x25f4: 0x000b, 0x25f5: 0x000b, + 0x25f6: 0x000b, 0x25f7: 0x000b, 0x25f8: 0x000b, 0x25f9: 0x000a, 0x25fa: 0x000a, 0x25fb: 0x000a, + 0x25fc: 0x000a, 0x25fd: 0x000a, 0x25fe: 0x000b, 0x25ff: 0x000b, + // Block 0x98, offset 0x2600 + 0x2601: 0x000a, + // Block 0x99, offset 0x2640 + 0x2640: 0x000a, 0x2641: 0x000a, 0x2642: 0x000a, 0x2643: 0x000a, 0x2644: 0x000a, 0x2645: 0x000a, + 0x2646: 0x000a, 0x2647: 0x000a, 0x2648: 0x000a, 0x2649: 0x000a, 0x264a: 0x000a, 0x264b: 0x000a, + 0x264c: 0x000a, 0x2650: 0x000a, 0x2651: 0x000a, + 0x2652: 0x000a, 0x2653: 0x000a, 0x2654: 0x000a, 0x2655: 0x000a, 0x2656: 0x000a, 0x2657: 0x000a, + 0x2658: 0x000a, 0x2659: 0x000a, 0x265a: 0x000a, 0x265b: 0x000a, + 0x2660: 0x000a, + // Block 0x9a, offset 0x2680 + 0x26bd: 0x000c, + // Block 0x9b, offset 0x26c0 + 0x26e0: 0x000c, 0x26e1: 0x0002, 0x26e2: 0x0002, 0x26e3: 0x0002, + 0x26e4: 0x0002, 0x26e5: 0x0002, 0x26e6: 0x0002, 0x26e7: 0x0002, 0x26e8: 0x0002, 0x26e9: 0x0002, + 0x26ea: 0x0002, 0x26eb: 0x0002, 0x26ec: 0x0002, 0x26ed: 0x0002, 0x26ee: 0x0002, 0x26ef: 0x0002, + 0x26f0: 0x0002, 0x26f1: 0x0002, 0x26f2: 0x0002, 0x26f3: 0x0002, 0x26f4: 0x0002, 0x26f5: 0x0002, + 0x26f6: 0x0002, 0x26f7: 0x0002, 0x26f8: 0x0002, 0x26f9: 0x0002, 0x26fa: 0x0002, 0x26fb: 0x0002, + // Block 0x9c, offset 0x2700 + 0x2736: 0x000c, 0x2737: 0x000c, 0x2738: 0x000c, 0x2739: 0x000c, 0x273a: 0x000c, + // Block 0x9d, offset 0x2740 + 0x2740: 0x0001, 0x2741: 0x0001, 0x2742: 0x0001, 0x2743: 0x0001, 0x2744: 0x0001, 0x2745: 0x0001, + 0x2746: 0x0001, 0x2747: 0x0001, 0x2748: 0x0001, 0x2749: 0x0001, 0x274a: 0x0001, 0x274b: 0x0001, + 0x274c: 0x0001, 0x274d: 0x0001, 0x274e: 0x0001, 0x274f: 0x0001, 0x2750: 0x0001, 0x2751: 0x0001, + 0x2752: 0x0001, 0x2753: 0x0001, 0x2754: 0x0001, 0x2755: 0x0001, 0x2756: 0x0001, 0x2757: 0x0001, + 0x2758: 0x0001, 0x2759: 0x0001, 0x275a: 0x0001, 0x275b: 0x0001, 0x275c: 0x0001, 0x275d: 0x0001, + 0x275e: 0x0001, 0x275f: 0x0001, 0x2760: 0x0001, 0x2761: 0x0001, 0x2762: 0x0001, 0x2763: 0x0001, + 0x2764: 0x0001, 0x2765: 0x0001, 0x2766: 0x0001, 0x2767: 0x0001, 0x2768: 0x0001, 0x2769: 0x0001, + 0x276a: 0x0001, 0x276b: 0x0001, 0x276c: 0x0001, 0x276d: 0x0001, 0x276e: 0x0001, 0x276f: 0x0001, + 0x2770: 0x0001, 0x2771: 0x0001, 0x2772: 0x0001, 0x2773: 0x0001, 0x2774: 0x0001, 0x2775: 0x0001, + 0x2776: 0x0001, 0x2777: 0x0001, 0x2778: 0x0001, 0x2779: 0x0001, 0x277a: 0x0001, 0x277b: 0x0001, + 0x277c: 0x0001, 0x277d: 0x0001, 0x277e: 0x0001, 0x277f: 0x0001, + // Block 0x9e, offset 0x2780 + 0x2780: 0x0001, 0x2781: 0x0001, 0x2782: 0x0001, 0x2783: 0x0001, 0x2784: 0x0001, 0x2785: 0x0001, + 0x2786: 0x0001, 0x2787: 0x0001, 0x2788: 0x0001, 0x2789: 0x0001, 0x278a: 0x0001, 0x278b: 0x0001, + 0x278c: 0x0001, 0x278d: 0x0001, 0x278e: 0x0001, 0x278f: 0x0001, 0x2790: 0x0001, 0x2791: 0x0001, + 0x2792: 0x0001, 0x2793: 0x0001, 0x2794: 0x0001, 0x2795: 0x0001, 0x2796: 0x0001, 0x2797: 0x0001, + 0x2798: 0x0001, 0x2799: 0x0001, 0x279a: 0x0001, 0x279b: 0x0001, 0x279c: 0x0001, 0x279d: 0x0001, + 0x279e: 0x0001, 0x279f: 0x000a, 0x27a0: 0x0001, 0x27a1: 0x0001, 0x27a2: 0x0001, 0x27a3: 0x0001, + 0x27a4: 0x0001, 0x27a5: 0x0001, 0x27a6: 0x0001, 0x27a7: 0x0001, 0x27a8: 0x0001, 0x27a9: 0x0001, + 0x27aa: 0x0001, 0x27ab: 0x0001, 0x27ac: 0x0001, 0x27ad: 0x0001, 0x27ae: 0x0001, 0x27af: 0x0001, + 0x27b0: 0x0001, 0x27b1: 0x0001, 0x27b2: 0x0001, 0x27b3: 0x0001, 0x27b4: 0x0001, 0x27b5: 0x0001, + 0x27b6: 0x0001, 0x27b7: 0x0001, 0x27b8: 0x0001, 0x27b9: 0x0001, 0x27ba: 0x0001, 0x27bb: 0x0001, + 0x27bc: 0x0001, 0x27bd: 0x0001, 0x27be: 0x0001, 0x27bf: 0x0001, + // Block 0x9f, offset 0x27c0 + 0x27c0: 0x0001, 0x27c1: 0x000c, 0x27c2: 0x000c, 0x27c3: 0x000c, 0x27c4: 0x0001, 0x27c5: 0x000c, + 0x27c6: 0x000c, 0x27c7: 0x0001, 0x27c8: 0x0001, 0x27c9: 0x0001, 0x27ca: 0x0001, 0x27cb: 0x0001, + 0x27cc: 0x000c, 0x27cd: 0x000c, 0x27ce: 0x000c, 0x27cf: 0x000c, 0x27d0: 0x0001, 0x27d1: 0x0001, + 0x27d2: 0x0001, 0x27d3: 0x0001, 0x27d4: 0x0001, 0x27d5: 0x0001, 0x27d6: 0x0001, 0x27d7: 0x0001, + 0x27d8: 0x0001, 0x27d9: 0x0001, 0x27da: 0x0001, 0x27db: 0x0001, 0x27dc: 0x0001, 0x27dd: 0x0001, + 0x27de: 0x0001, 0x27df: 0x0001, 0x27e0: 0x0001, 0x27e1: 0x0001, 0x27e2: 0x0001, 0x27e3: 0x0001, + 0x27e4: 0x0001, 0x27e5: 0x0001, 0x27e6: 0x0001, 0x27e7: 0x0001, 0x27e8: 0x0001, 0x27e9: 0x0001, + 0x27ea: 0x0001, 0x27eb: 0x0001, 0x27ec: 0x0001, 0x27ed: 0x0001, 0x27ee: 0x0001, 0x27ef: 0x0001, + 0x27f0: 0x0001, 0x27f1: 0x0001, 0x27f2: 0x0001, 0x27f3: 0x0001, 0x27f4: 0x0001, 0x27f5: 0x0001, + 0x27f6: 0x0001, 0x27f7: 0x0001, 0x27f8: 0x000c, 0x27f9: 0x000c, 0x27fa: 0x000c, 0x27fb: 0x0001, + 0x27fc: 0x0001, 0x27fd: 0x0001, 0x27fe: 0x0001, 0x27ff: 0x000c, + // Block 0xa0, offset 0x2800 + 0x2800: 0x0001, 0x2801: 0x0001, 0x2802: 0x0001, 0x2803: 0x0001, 0x2804: 0x0001, 0x2805: 0x0001, + 0x2806: 0x0001, 0x2807: 0x0001, 0x2808: 0x0001, 0x2809: 0x0001, 0x280a: 0x0001, 0x280b: 0x0001, + 0x280c: 0x0001, 0x280d: 0x0001, 0x280e: 0x0001, 0x280f: 0x0001, 0x2810: 0x0001, 0x2811: 0x0001, + 0x2812: 0x0001, 0x2813: 0x0001, 0x2814: 0x0001, 0x2815: 0x0001, 0x2816: 0x0001, 0x2817: 0x0001, + 0x2818: 0x0001, 0x2819: 0x0001, 0x281a: 0x0001, 0x281b: 0x0001, 0x281c: 0x0001, 0x281d: 0x0001, + 0x281e: 0x0001, 0x281f: 0x0001, 0x2820: 0x0001, 0x2821: 0x0001, 0x2822: 0x0001, 0x2823: 0x0001, + 0x2824: 0x0001, 0x2825: 0x000c, 0x2826: 0x000c, 0x2827: 0x0001, 0x2828: 0x0001, 0x2829: 0x0001, + 0x282a: 0x0001, 0x282b: 0x0001, 0x282c: 0x0001, 0x282d: 0x0001, 0x282e: 0x0001, 0x282f: 0x0001, + 0x2830: 0x0001, 0x2831: 0x0001, 0x2832: 0x0001, 0x2833: 0x0001, 0x2834: 0x0001, 0x2835: 0x0001, + 0x2836: 0x0001, 0x2837: 0x0001, 0x2838: 0x0001, 0x2839: 0x0001, 0x283a: 0x0001, 0x283b: 0x0001, + 0x283c: 0x0001, 0x283d: 0x0001, 0x283e: 0x0001, 0x283f: 0x0001, + // Block 0xa1, offset 0x2840 + 0x2840: 0x0001, 0x2841: 0x0001, 0x2842: 0x0001, 0x2843: 0x0001, 0x2844: 0x0001, 0x2845: 0x0001, + 0x2846: 0x0001, 0x2847: 0x0001, 0x2848: 0x0001, 0x2849: 0x0001, 0x284a: 0x0001, 0x284b: 0x0001, + 0x284c: 0x0001, 0x284d: 0x0001, 0x284e: 0x0001, 0x284f: 0x0001, 0x2850: 0x0001, 0x2851: 0x0001, + 0x2852: 0x0001, 0x2853: 0x0001, 0x2854: 0x0001, 0x2855: 0x0001, 0x2856: 0x0001, 0x2857: 0x0001, + 0x2858: 0x0001, 0x2859: 0x0001, 0x285a: 0x0001, 0x285b: 0x0001, 0x285c: 0x0001, 0x285d: 0x0001, + 0x285e: 0x0001, 0x285f: 0x0001, 0x2860: 0x0001, 0x2861: 0x0001, 0x2862: 0x0001, 0x2863: 0x0001, + 0x2864: 0x0001, 0x2865: 0x0001, 0x2866: 0x0001, 0x2867: 0x0001, 0x2868: 0x0001, 0x2869: 0x0001, + 0x286a: 0x0001, 0x286b: 0x0001, 0x286c: 0x0001, 0x286d: 0x0001, 0x286e: 0x0001, 0x286f: 0x0001, + 0x2870: 0x0001, 0x2871: 0x0001, 0x2872: 0x0001, 0x2873: 0x0001, 0x2874: 0x0001, 0x2875: 0x0001, + 0x2876: 0x0001, 0x2877: 0x0001, 0x2878: 0x0001, 0x2879: 0x000a, 0x287a: 0x000a, 0x287b: 0x000a, + 0x287c: 0x000a, 0x287d: 0x000a, 0x287e: 0x000a, 0x287f: 0x000a, + // Block 0xa2, offset 0x2880 + 0x2880: 0x0001, 0x2881: 0x0001, 0x2882: 0x0001, 0x2883: 0x0001, 0x2884: 0x0001, 0x2885: 0x0001, + 0x2886: 0x0001, 0x2887: 0x0001, 0x2888: 0x0001, 0x2889: 0x0001, 0x288a: 0x0001, 0x288b: 0x0001, + 0x288c: 0x0001, 0x288d: 0x0001, 0x288e: 0x0001, 0x288f: 0x0001, 0x2890: 0x0001, 0x2891: 0x0001, + 0x2892: 0x0001, 0x2893: 0x0001, 0x2894: 0x0001, 0x2895: 0x0001, 0x2896: 0x0001, 0x2897: 0x0001, + 0x2898: 0x0001, 0x2899: 0x0001, 0x289a: 0x0001, 0x289b: 0x0001, 0x289c: 0x0001, 0x289d: 0x0001, + 0x289e: 0x0001, 0x289f: 0x0001, 0x28a0: 0x0005, 0x28a1: 0x0005, 0x28a2: 0x0005, 0x28a3: 0x0005, + 0x28a4: 0x0005, 0x28a5: 0x0005, 0x28a6: 0x0005, 0x28a7: 0x0005, 0x28a8: 0x0005, 0x28a9: 0x0005, + 0x28aa: 0x0005, 0x28ab: 0x0005, 0x28ac: 0x0005, 0x28ad: 0x0005, 0x28ae: 0x0005, 0x28af: 0x0005, + 0x28b0: 0x0005, 0x28b1: 0x0005, 0x28b2: 0x0005, 0x28b3: 0x0005, 0x28b4: 0x0005, 0x28b5: 0x0005, + 0x28b6: 0x0005, 0x28b7: 0x0005, 0x28b8: 0x0005, 0x28b9: 0x0005, 0x28ba: 0x0005, 0x28bb: 0x0005, + 0x28bc: 0x0005, 0x28bd: 0x0005, 0x28be: 0x0005, 0x28bf: 0x0001, + // Block 0xa3, offset 0x28c0 + 0x28c1: 0x000c, + 0x28f8: 0x000c, 0x28f9: 0x000c, 0x28fa: 0x000c, 0x28fb: 0x000c, + 0x28fc: 0x000c, 0x28fd: 0x000c, 0x28fe: 0x000c, 0x28ff: 0x000c, + // Block 0xa4, offset 0x2900 + 0x2900: 0x000c, 0x2901: 0x000c, 0x2902: 0x000c, 0x2903: 0x000c, 0x2904: 0x000c, 0x2905: 0x000c, + 0x2906: 0x000c, + 0x2912: 0x000a, 0x2913: 0x000a, 0x2914: 0x000a, 0x2915: 0x000a, 0x2916: 0x000a, 0x2917: 0x000a, + 0x2918: 0x000a, 0x2919: 0x000a, 0x291a: 0x000a, 0x291b: 0x000a, 0x291c: 0x000a, 0x291d: 0x000a, + 0x291e: 0x000a, 0x291f: 0x000a, 0x2920: 0x000a, 0x2921: 0x000a, 0x2922: 0x000a, 0x2923: 0x000a, + 0x2924: 0x000a, 0x2925: 0x000a, + 0x293f: 0x000c, + // Block 0xa5, offset 0x2940 + 0x2940: 0x000c, 0x2941: 0x000c, + 0x2973: 0x000c, 0x2974: 0x000c, 0x2975: 0x000c, + 0x2976: 0x000c, 0x2979: 0x000c, 0x297a: 0x000c, + // Block 0xa6, offset 0x2980 + 0x2980: 0x000c, 0x2981: 0x000c, 0x2982: 0x000c, + 0x29a7: 0x000c, 0x29a8: 0x000c, 0x29a9: 0x000c, + 0x29aa: 0x000c, 0x29ab: 0x000c, 0x29ad: 0x000c, 0x29ae: 0x000c, 0x29af: 0x000c, + 0x29b0: 0x000c, 0x29b1: 0x000c, 0x29b2: 0x000c, 0x29b3: 0x000c, 0x29b4: 0x000c, + // Block 0xa7, offset 0x29c0 + 0x29f3: 0x000c, + // Block 0xa8, offset 0x2a00 + 0x2a00: 0x000c, 0x2a01: 0x000c, + 0x2a36: 0x000c, 0x2a37: 0x000c, 0x2a38: 0x000c, 0x2a39: 0x000c, 0x2a3a: 0x000c, 0x2a3b: 0x000c, + 0x2a3c: 0x000c, 0x2a3d: 0x000c, 0x2a3e: 0x000c, + // Block 0xa9, offset 0x2a40 + 0x2a4a: 0x000c, 0x2a4b: 0x000c, + 0x2a4c: 0x000c, + // Block 0xaa, offset 0x2a80 + 0x2aaf: 0x000c, + 0x2ab0: 0x000c, 0x2ab1: 0x000c, 0x2ab4: 0x000c, + 0x2ab6: 0x000c, 0x2ab7: 0x000c, + 0x2abe: 0x000c, + // Block 0xab, offset 0x2ac0 + 0x2adf: 0x000c, 0x2ae3: 0x000c, + 0x2ae4: 0x000c, 0x2ae5: 0x000c, 0x2ae6: 0x000c, 0x2ae7: 0x000c, 0x2ae8: 0x000c, 0x2ae9: 0x000c, + 0x2aea: 0x000c, + // Block 0xac, offset 0x2b00 + 0x2b00: 0x000c, 0x2b01: 0x000c, + 0x2b3c: 0x000c, + // Block 0xad, offset 0x2b40 + 0x2b40: 0x000c, + 0x2b66: 0x000c, 0x2b67: 0x000c, 0x2b68: 0x000c, 0x2b69: 0x000c, + 0x2b6a: 0x000c, 0x2b6b: 0x000c, 0x2b6c: 0x000c, + 0x2b70: 0x000c, 0x2b71: 0x000c, 0x2b72: 0x000c, 0x2b73: 0x000c, 0x2b74: 0x000c, + // Block 0xae, offset 0x2b80 + 0x2bb8: 0x000c, 0x2bb9: 0x000c, 0x2bba: 0x000c, 0x2bbb: 0x000c, + 0x2bbc: 0x000c, 0x2bbd: 0x000c, 0x2bbe: 0x000c, 0x2bbf: 0x000c, + // Block 0xaf, offset 0x2bc0 + 0x2bc2: 0x000c, 0x2bc3: 0x000c, 0x2bc4: 0x000c, + 0x2bc6: 0x000c, + // Block 0xb0, offset 0x2c00 + 0x2c33: 0x000c, 0x2c34: 0x000c, 0x2c35: 0x000c, + 0x2c36: 0x000c, 0x2c37: 0x000c, 0x2c38: 0x000c, 0x2c3a: 0x000c, + 0x2c3f: 0x000c, + // Block 0xb1, offset 0x2c40 + 0x2c40: 0x000c, 0x2c42: 0x000c, 0x2c43: 0x000c, + // Block 0xb2, offset 0x2c80 + 0x2cb2: 0x000c, 0x2cb3: 0x000c, 0x2cb4: 0x000c, 0x2cb5: 0x000c, + 0x2cbc: 0x000c, 0x2cbd: 0x000c, 0x2cbf: 0x000c, + // Block 0xb3, offset 0x2cc0 + 0x2cc0: 0x000c, + 0x2cdc: 0x000c, 0x2cdd: 0x000c, + // Block 0xb4, offset 0x2d00 + 0x2d33: 0x000c, 0x2d34: 0x000c, 0x2d35: 0x000c, + 0x2d36: 0x000c, 0x2d37: 0x000c, 0x2d38: 0x000c, 0x2d39: 0x000c, 0x2d3a: 0x000c, + 0x2d3d: 0x000c, 0x2d3f: 0x000c, + // Block 0xb5, offset 0x2d40 + 0x2d40: 0x000c, + 0x2d60: 0x000a, 0x2d61: 0x000a, 0x2d62: 0x000a, 0x2d63: 0x000a, + 0x2d64: 0x000a, 0x2d65: 0x000a, 0x2d66: 0x000a, 0x2d67: 0x000a, 0x2d68: 0x000a, 0x2d69: 0x000a, + 0x2d6a: 0x000a, 0x2d6b: 0x000a, 0x2d6c: 0x000a, + // Block 0xb6, offset 0x2d80 + 0x2dab: 0x000c, 0x2dad: 0x000c, + 0x2db0: 0x000c, 0x2db1: 0x000c, 0x2db2: 0x000c, 0x2db3: 0x000c, 0x2db4: 0x000c, 0x2db5: 0x000c, + 0x2db7: 0x000c, + // Block 0xb7, offset 0x2dc0 + 0x2ddd: 0x000c, + 0x2dde: 0x000c, 0x2ddf: 0x000c, 0x2de2: 0x000c, 0x2de3: 0x000c, + 0x2de4: 0x000c, 0x2de5: 0x000c, 0x2de7: 0x000c, 0x2de8: 0x000c, 0x2de9: 0x000c, + 0x2dea: 0x000c, 0x2deb: 0x000c, + // Block 0xb8, offset 0x2e00 + 0x2e30: 0x000c, 0x2e31: 0x000c, 0x2e32: 0x000c, 0x2e33: 0x000c, 0x2e34: 0x000c, 0x2e35: 0x000c, + 0x2e36: 0x000c, 0x2e38: 0x000c, 0x2e39: 0x000c, 0x2e3a: 0x000c, 0x2e3b: 0x000c, + 0x2e3c: 0x000c, 0x2e3d: 0x000c, + // Block 0xb9, offset 0x2e40 + 0x2e52: 0x000c, 0x2e53: 0x000c, 0x2e54: 0x000c, 0x2e55: 0x000c, 0x2e56: 0x000c, 0x2e57: 0x000c, + 0x2e58: 0x000c, 0x2e59: 0x000c, 0x2e5a: 0x000c, 0x2e5b: 0x000c, 0x2e5c: 0x000c, 0x2e5d: 0x000c, + 0x2e5e: 0x000c, 0x2e5f: 0x000c, 0x2e60: 0x000c, 0x2e61: 0x000c, 0x2e62: 0x000c, 0x2e63: 0x000c, + 0x2e64: 0x000c, 0x2e65: 0x000c, 0x2e66: 0x000c, 0x2e67: 0x000c, + 0x2e6a: 0x000c, 0x2e6b: 0x000c, 0x2e6c: 0x000c, 0x2e6d: 0x000c, 0x2e6e: 0x000c, 0x2e6f: 0x000c, + 0x2e70: 0x000c, 0x2e72: 0x000c, 0x2e73: 0x000c, 0x2e75: 0x000c, + 0x2e76: 0x000c, + // Block 0xba, offset 0x2e80 + 0x2eb0: 0x000c, 0x2eb1: 0x000c, 0x2eb2: 0x000c, 0x2eb3: 0x000c, 0x2eb4: 0x000c, + // Block 0xbb, offset 0x2ec0 + 0x2ef0: 0x000c, 0x2ef1: 0x000c, 0x2ef2: 0x000c, 0x2ef3: 0x000c, 0x2ef4: 0x000c, 0x2ef5: 0x000c, + 0x2ef6: 0x000c, + // Block 0xbc, offset 0x2f00 + 0x2f0f: 0x000c, 0x2f10: 0x000c, 0x2f11: 0x000c, + 0x2f12: 0x000c, + // Block 0xbd, offset 0x2f40 + 0x2f5d: 0x000c, + 0x2f5e: 0x000c, 0x2f60: 0x000b, 0x2f61: 0x000b, 0x2f62: 0x000b, 0x2f63: 0x000b, + // Block 0xbe, offset 0x2f80 + 0x2fa7: 0x000c, 0x2fa8: 0x000c, 0x2fa9: 0x000c, + 0x2fb3: 0x000b, 0x2fb4: 0x000b, 0x2fb5: 0x000b, + 0x2fb6: 0x000b, 0x2fb7: 0x000b, 0x2fb8: 0x000b, 0x2fb9: 0x000b, 0x2fba: 0x000b, 0x2fbb: 0x000c, + 0x2fbc: 0x000c, 0x2fbd: 0x000c, 0x2fbe: 0x000c, 0x2fbf: 0x000c, + // Block 0xbf, offset 0x2fc0 + 0x2fc0: 0x000c, 0x2fc1: 0x000c, 0x2fc2: 0x000c, 0x2fc5: 0x000c, + 0x2fc6: 0x000c, 0x2fc7: 0x000c, 0x2fc8: 0x000c, 0x2fc9: 0x000c, 0x2fca: 0x000c, 0x2fcb: 0x000c, + 0x2fea: 0x000c, 0x2feb: 0x000c, 0x2fec: 0x000c, 0x2fed: 0x000c, + // Block 0xc0, offset 0x3000 + 0x3000: 0x000a, 0x3001: 0x000a, 0x3002: 0x000c, 0x3003: 0x000c, 0x3004: 0x000c, 0x3005: 0x000a, + // Block 0xc1, offset 0x3040 + 0x3040: 0x000a, 0x3041: 0x000a, 0x3042: 0x000a, 0x3043: 0x000a, 0x3044: 0x000a, 0x3045: 0x000a, + 0x3046: 0x000a, 0x3047: 0x000a, 0x3048: 0x000a, 0x3049: 0x000a, 0x304a: 0x000a, 0x304b: 0x000a, + 0x304c: 0x000a, 0x304d: 0x000a, 0x304e: 0x000a, 0x304f: 0x000a, 0x3050: 0x000a, 0x3051: 0x000a, + 0x3052: 0x000a, 0x3053: 0x000a, 0x3054: 0x000a, 0x3055: 0x000a, 0x3056: 0x000a, + // Block 0xc2, offset 0x3080 + 0x309b: 0x000a, + // Block 0xc3, offset 0x30c0 + 0x30d5: 0x000a, + // Block 0xc4, offset 0x3100 + 0x310f: 0x000a, + // Block 0xc5, offset 0x3140 + 0x3149: 0x000a, + // Block 0xc6, offset 0x3180 + 0x3183: 0x000a, + 0x318e: 0x0002, 0x318f: 0x0002, 0x3190: 0x0002, 0x3191: 0x0002, + 0x3192: 0x0002, 0x3193: 0x0002, 0x3194: 0x0002, 0x3195: 0x0002, 0x3196: 0x0002, 0x3197: 0x0002, + 0x3198: 0x0002, 0x3199: 0x0002, 0x319a: 0x0002, 0x319b: 0x0002, 0x319c: 0x0002, 0x319d: 0x0002, + 0x319e: 0x0002, 0x319f: 0x0002, 0x31a0: 0x0002, 0x31a1: 0x0002, 0x31a2: 0x0002, 0x31a3: 0x0002, + 0x31a4: 0x0002, 0x31a5: 0x0002, 0x31a6: 0x0002, 0x31a7: 0x0002, 0x31a8: 0x0002, 0x31a9: 0x0002, + 0x31aa: 0x0002, 0x31ab: 0x0002, 0x31ac: 0x0002, 0x31ad: 0x0002, 0x31ae: 0x0002, 0x31af: 0x0002, + 0x31b0: 0x0002, 0x31b1: 0x0002, 0x31b2: 0x0002, 0x31b3: 0x0002, 0x31b4: 0x0002, 0x31b5: 0x0002, + 0x31b6: 0x0002, 0x31b7: 0x0002, 0x31b8: 0x0002, 0x31b9: 0x0002, 0x31ba: 0x0002, 0x31bb: 0x0002, + 0x31bc: 0x0002, 0x31bd: 0x0002, 0x31be: 0x0002, 0x31bf: 0x0002, + // Block 0xc7, offset 0x31c0 + 0x31c0: 0x000c, 0x31c1: 0x000c, 0x31c2: 0x000c, 0x31c3: 0x000c, 0x31c4: 0x000c, 0x31c5: 0x000c, + 0x31c6: 0x000c, 0x31c7: 0x000c, 0x31c8: 0x000c, 0x31c9: 0x000c, 0x31ca: 0x000c, 0x31cb: 0x000c, + 0x31cc: 0x000c, 0x31cd: 0x000c, 0x31ce: 0x000c, 0x31cf: 0x000c, 0x31d0: 0x000c, 0x31d1: 0x000c, + 0x31d2: 0x000c, 0x31d3: 0x000c, 0x31d4: 0x000c, 0x31d5: 0x000c, 0x31d6: 0x000c, 0x31d7: 0x000c, + 0x31d8: 0x000c, 0x31d9: 0x000c, 0x31da: 0x000c, 0x31db: 0x000c, 0x31dc: 0x000c, 0x31dd: 0x000c, + 0x31de: 0x000c, 0x31df: 0x000c, 0x31e0: 0x000c, 0x31e1: 0x000c, 0x31e2: 0x000c, 0x31e3: 0x000c, + 0x31e4: 0x000c, 0x31e5: 0x000c, 0x31e6: 0x000c, 0x31e7: 0x000c, 0x31e8: 0x000c, 0x31e9: 0x000c, + 0x31ea: 0x000c, 0x31eb: 0x000c, 0x31ec: 0x000c, 0x31ed: 0x000c, 0x31ee: 0x000c, 0x31ef: 0x000c, + 0x31f0: 0x000c, 0x31f1: 0x000c, 0x31f2: 0x000c, 0x31f3: 0x000c, 0x31f4: 0x000c, 0x31f5: 0x000c, + 0x31f6: 0x000c, 0x31fb: 0x000c, + 0x31fc: 0x000c, 0x31fd: 0x000c, 0x31fe: 0x000c, 0x31ff: 0x000c, + // Block 0xc8, offset 0x3200 + 0x3200: 0x000c, 0x3201: 0x000c, 0x3202: 0x000c, 0x3203: 0x000c, 0x3204: 0x000c, 0x3205: 0x000c, + 0x3206: 0x000c, 0x3207: 0x000c, 0x3208: 0x000c, 0x3209: 0x000c, 0x320a: 0x000c, 0x320b: 0x000c, + 0x320c: 0x000c, 0x320d: 0x000c, 0x320e: 0x000c, 0x320f: 0x000c, 0x3210: 0x000c, 0x3211: 0x000c, + 0x3212: 0x000c, 0x3213: 0x000c, 0x3214: 0x000c, 0x3215: 0x000c, 0x3216: 0x000c, 0x3217: 0x000c, + 0x3218: 0x000c, 0x3219: 0x000c, 0x321a: 0x000c, 0x321b: 0x000c, 0x321c: 0x000c, 0x321d: 0x000c, + 0x321e: 0x000c, 0x321f: 0x000c, 0x3220: 0x000c, 0x3221: 0x000c, 0x3222: 0x000c, 0x3223: 0x000c, + 0x3224: 0x000c, 0x3225: 0x000c, 0x3226: 0x000c, 0x3227: 0x000c, 0x3228: 0x000c, 0x3229: 0x000c, + 0x322a: 0x000c, 0x322b: 0x000c, 0x322c: 0x000c, + 0x3235: 0x000c, + // Block 0xc9, offset 0x3240 + 0x3244: 0x000c, + 0x325b: 0x000c, 0x325c: 0x000c, 0x325d: 0x000c, + 0x325e: 0x000c, 0x325f: 0x000c, 0x3261: 0x000c, 0x3262: 0x000c, 0x3263: 0x000c, + 0x3264: 0x000c, 0x3265: 0x000c, 0x3266: 0x000c, 0x3267: 0x000c, 0x3268: 0x000c, 0x3269: 0x000c, + 0x326a: 0x000c, 0x326b: 0x000c, 0x326c: 0x000c, 0x326d: 0x000c, 0x326e: 0x000c, 0x326f: 0x000c, + // Block 0xca, offset 0x3280 + 0x3280: 0x000c, 0x3281: 0x000c, 0x3282: 0x000c, 0x3283: 0x000c, 0x3284: 0x000c, 0x3285: 0x000c, + 0x3286: 0x000c, 0x3288: 0x000c, 0x3289: 0x000c, 0x328a: 0x000c, 0x328b: 0x000c, + 0x328c: 0x000c, 0x328d: 0x000c, 0x328e: 0x000c, 0x328f: 0x000c, 0x3290: 0x000c, 0x3291: 0x000c, + 0x3292: 0x000c, 0x3293: 0x000c, 0x3294: 0x000c, 0x3295: 0x000c, 0x3296: 0x000c, 0x3297: 0x000c, + 0x3298: 0x000c, 0x329b: 0x000c, 0x329c: 0x000c, 0x329d: 0x000c, + 0x329e: 0x000c, 0x329f: 0x000c, 0x32a0: 0x000c, 0x32a1: 0x000c, 0x32a3: 0x000c, + 0x32a4: 0x000c, 0x32a6: 0x000c, 0x32a7: 0x000c, 0x32a8: 0x000c, 0x32a9: 0x000c, + 0x32aa: 0x000c, + // Block 0xcb, offset 0x32c0 + 0x32c0: 0x0001, 0x32c1: 0x0001, 0x32c2: 0x0001, 0x32c3: 0x0001, 0x32c4: 0x0001, 0x32c5: 0x0001, + 0x32c6: 0x0001, 0x32c7: 0x0001, 0x32c8: 0x0001, 0x32c9: 0x0001, 0x32ca: 0x0001, 0x32cb: 0x0001, + 0x32cc: 0x0001, 0x32cd: 0x0001, 0x32ce: 0x0001, 0x32cf: 0x0001, 0x32d0: 0x000c, 0x32d1: 0x000c, + 0x32d2: 0x000c, 0x32d3: 0x000c, 0x32d4: 0x000c, 0x32d5: 0x000c, 0x32d6: 0x000c, 0x32d7: 0x0001, + 0x32d8: 0x0001, 0x32d9: 0x0001, 0x32da: 0x0001, 0x32db: 0x0001, 0x32dc: 0x0001, 0x32dd: 0x0001, + 0x32de: 0x0001, 0x32df: 0x0001, 0x32e0: 0x0001, 0x32e1: 0x0001, 0x32e2: 0x0001, 0x32e3: 0x0001, + 0x32e4: 0x0001, 0x32e5: 0x0001, 0x32e6: 0x0001, 0x32e7: 0x0001, 0x32e8: 0x0001, 0x32e9: 0x0001, + 0x32ea: 0x0001, 0x32eb: 0x0001, 0x32ec: 0x0001, 0x32ed: 0x0001, 0x32ee: 0x0001, 0x32ef: 0x0001, + 0x32f0: 0x0001, 0x32f1: 0x0001, 0x32f2: 0x0001, 0x32f3: 0x0001, 0x32f4: 0x0001, 0x32f5: 0x0001, + 0x32f6: 0x0001, 0x32f7: 0x0001, 0x32f8: 0x0001, 0x32f9: 0x0001, 0x32fa: 0x0001, 0x32fb: 0x0001, + 0x32fc: 0x0001, 0x32fd: 0x0001, 0x32fe: 0x0001, 0x32ff: 0x0001, + // Block 0xcc, offset 0x3300 + 0x3300: 0x0001, 0x3301: 0x0001, 0x3302: 0x0001, 0x3303: 0x0001, 0x3304: 0x000c, 0x3305: 0x000c, + 0x3306: 0x000c, 0x3307: 0x000c, 0x3308: 0x000c, 0x3309: 0x000c, 0x330a: 0x000c, 0x330b: 0x0001, + 0x330c: 0x0001, 0x330d: 0x0001, 0x330e: 0x0001, 0x330f: 0x0001, 0x3310: 0x0001, 0x3311: 0x0001, + 0x3312: 0x0001, 0x3313: 0x0001, 0x3314: 0x0001, 0x3315: 0x0001, 0x3316: 0x0001, 0x3317: 0x0001, + 0x3318: 0x0001, 0x3319: 0x0001, 0x331a: 0x0001, 0x331b: 0x0001, 0x331c: 0x0001, 0x331d: 0x0001, + 0x331e: 0x0001, 0x331f: 0x0001, 0x3320: 0x0001, 0x3321: 0x0001, 0x3322: 0x0001, 0x3323: 0x0001, + 0x3324: 0x0001, 0x3325: 0x0001, 0x3326: 0x0001, 0x3327: 0x0001, 0x3328: 0x0001, 0x3329: 0x0001, + 0x332a: 0x0001, 0x332b: 0x0001, 0x332c: 0x0001, 0x332d: 0x0001, 0x332e: 0x0001, 0x332f: 0x0001, + 0x3330: 0x0001, 0x3331: 0x0001, 0x3332: 0x0001, 0x3333: 0x0001, 0x3334: 0x0001, 0x3335: 0x0001, + 0x3336: 0x0001, 0x3337: 0x0001, 0x3338: 0x0001, 0x3339: 0x0001, 0x333a: 0x0001, 0x333b: 0x0001, + 0x333c: 0x0001, 0x333d: 0x0001, 0x333e: 0x0001, 0x333f: 0x0001, + // Block 0xcd, offset 0x3340 + 0x3340: 0x000d, 0x3341: 0x000d, 0x3342: 0x000d, 0x3343: 0x000d, 0x3344: 0x000d, 0x3345: 0x000d, + 0x3346: 0x000d, 0x3347: 0x000d, 0x3348: 0x000d, 0x3349: 0x000d, 0x334a: 0x000d, 0x334b: 0x000d, + 0x334c: 0x000d, 0x334d: 0x000d, 0x334e: 0x000d, 0x334f: 0x000d, 0x3350: 0x000d, 0x3351: 0x000d, + 0x3352: 0x000d, 0x3353: 0x000d, 0x3354: 0x000d, 0x3355: 0x000d, 0x3356: 0x000d, 0x3357: 0x000d, + 0x3358: 0x000d, 0x3359: 0x000d, 0x335a: 0x000d, 0x335b: 0x000d, 0x335c: 0x000d, 0x335d: 0x000d, + 0x335e: 0x000d, 0x335f: 0x000d, 0x3360: 0x000d, 0x3361: 0x000d, 0x3362: 0x000d, 0x3363: 0x000d, + 0x3364: 0x000d, 0x3365: 0x000d, 0x3366: 0x000d, 0x3367: 0x000d, 0x3368: 0x000d, 0x3369: 0x000d, + 0x336a: 0x000d, 0x336b: 0x000d, 0x336c: 0x000d, 0x336d: 0x000d, 0x336e: 0x000d, 0x336f: 0x000d, + 0x3370: 0x000a, 0x3371: 0x000a, 0x3372: 0x000d, 0x3373: 0x000d, 0x3374: 0x000d, 0x3375: 0x000d, + 0x3376: 0x000d, 0x3377: 0x000d, 0x3378: 0x000d, 0x3379: 0x000d, 0x337a: 0x000d, 0x337b: 0x000d, + 0x337c: 0x000d, 0x337d: 0x000d, 0x337e: 0x000d, 0x337f: 0x000d, + // Block 0xce, offset 0x3380 + 0x3380: 0x000a, 0x3381: 0x000a, 0x3382: 0x000a, 0x3383: 0x000a, 0x3384: 0x000a, 0x3385: 0x000a, + 0x3386: 0x000a, 0x3387: 0x000a, 0x3388: 0x000a, 0x3389: 0x000a, 0x338a: 0x000a, 0x338b: 0x000a, + 0x338c: 0x000a, 0x338d: 0x000a, 0x338e: 0x000a, 0x338f: 0x000a, 0x3390: 0x000a, 0x3391: 0x000a, + 0x3392: 0x000a, 0x3393: 0x000a, 0x3394: 0x000a, 0x3395: 0x000a, 0x3396: 0x000a, 0x3397: 0x000a, + 0x3398: 0x000a, 0x3399: 0x000a, 0x339a: 0x000a, 0x339b: 0x000a, 0x339c: 0x000a, 0x339d: 0x000a, + 0x339e: 0x000a, 0x339f: 0x000a, 0x33a0: 0x000a, 0x33a1: 0x000a, 0x33a2: 0x000a, 0x33a3: 0x000a, + 0x33a4: 0x000a, 0x33a5: 0x000a, 0x33a6: 0x000a, 0x33a7: 0x000a, 0x33a8: 0x000a, 0x33a9: 0x000a, + 0x33aa: 0x000a, 0x33ab: 0x000a, + 0x33b0: 0x000a, 0x33b1: 0x000a, 0x33b2: 0x000a, 0x33b3: 0x000a, 0x33b4: 0x000a, 0x33b5: 0x000a, + 0x33b6: 0x000a, 0x33b7: 0x000a, 0x33b8: 0x000a, 0x33b9: 0x000a, 0x33ba: 0x000a, 0x33bb: 0x000a, + 0x33bc: 0x000a, 0x33bd: 0x000a, 0x33be: 0x000a, 0x33bf: 0x000a, + // Block 0xcf, offset 0x33c0 + 0x33c0: 0x000a, 0x33c1: 0x000a, 0x33c2: 0x000a, 0x33c3: 0x000a, 0x33c4: 0x000a, 0x33c5: 0x000a, + 0x33c6: 0x000a, 0x33c7: 0x000a, 0x33c8: 0x000a, 0x33c9: 0x000a, 0x33ca: 0x000a, 0x33cb: 0x000a, + 0x33cc: 0x000a, 0x33cd: 0x000a, 0x33ce: 0x000a, 0x33cf: 0x000a, 0x33d0: 0x000a, 0x33d1: 0x000a, + 0x33d2: 0x000a, 0x33d3: 0x000a, + 0x33e0: 0x000a, 0x33e1: 0x000a, 0x33e2: 0x000a, 0x33e3: 0x000a, + 0x33e4: 0x000a, 0x33e5: 0x000a, 0x33e6: 0x000a, 0x33e7: 0x000a, 0x33e8: 0x000a, 0x33e9: 0x000a, + 0x33ea: 0x000a, 0x33eb: 0x000a, 0x33ec: 0x000a, 0x33ed: 0x000a, 0x33ee: 0x000a, + 0x33f1: 0x000a, 0x33f2: 0x000a, 0x33f3: 0x000a, 0x33f4: 0x000a, 0x33f5: 0x000a, + 0x33f6: 0x000a, 0x33f7: 0x000a, 0x33f8: 0x000a, 0x33f9: 0x000a, 0x33fa: 0x000a, 0x33fb: 0x000a, + 0x33fc: 0x000a, 0x33fd: 0x000a, 0x33fe: 0x000a, 0x33ff: 0x000a, + // Block 0xd0, offset 0x3400 + 0x3401: 0x000a, 0x3402: 0x000a, 0x3403: 0x000a, 0x3404: 0x000a, 0x3405: 0x000a, + 0x3406: 0x000a, 0x3407: 0x000a, 0x3408: 0x000a, 0x3409: 0x000a, 0x340a: 0x000a, 0x340b: 0x000a, + 0x340c: 0x000a, 0x340d: 0x000a, 0x340e: 0x000a, 0x340f: 0x000a, 0x3411: 0x000a, + 0x3412: 0x000a, 0x3413: 0x000a, 0x3414: 0x000a, 0x3415: 0x000a, 0x3416: 0x000a, 0x3417: 0x000a, + 0x3418: 0x000a, 0x3419: 0x000a, 0x341a: 0x000a, 0x341b: 0x000a, 0x341c: 0x000a, 0x341d: 0x000a, + 0x341e: 0x000a, 0x341f: 0x000a, 0x3420: 0x000a, 0x3421: 0x000a, 0x3422: 0x000a, 0x3423: 0x000a, + 0x3424: 0x000a, 0x3425: 0x000a, 0x3426: 0x000a, 0x3427: 0x000a, 0x3428: 0x000a, 0x3429: 0x000a, + 0x342a: 0x000a, 0x342b: 0x000a, 0x342c: 0x000a, 0x342d: 0x000a, 0x342e: 0x000a, 0x342f: 0x000a, + 0x3430: 0x000a, 0x3431: 0x000a, 0x3432: 0x000a, 0x3433: 0x000a, 0x3434: 0x000a, 0x3435: 0x000a, + // Block 0xd1, offset 0x3440 + 0x3440: 0x0002, 0x3441: 0x0002, 0x3442: 0x0002, 0x3443: 0x0002, 0x3444: 0x0002, 0x3445: 0x0002, + 0x3446: 0x0002, 0x3447: 0x0002, 0x3448: 0x0002, 0x3449: 0x0002, 0x344a: 0x0002, 0x344b: 0x000a, + 0x344c: 0x000a, + // Block 0xd2, offset 0x3480 + 0x34aa: 0x000a, 0x34ab: 0x000a, + // Block 0xd3, offset 0x34c0 + 0x34c0: 0x000a, 0x34c1: 0x000a, 0x34c2: 0x000a, 0x34c3: 0x000a, 0x34c4: 0x000a, 0x34c5: 0x000a, + 0x34c6: 0x000a, 0x34c7: 0x000a, 0x34c8: 0x000a, 0x34c9: 0x000a, 0x34ca: 0x000a, 0x34cb: 0x000a, + 0x34cc: 0x000a, 0x34cd: 0x000a, 0x34ce: 0x000a, 0x34cf: 0x000a, 0x34d0: 0x000a, 0x34d1: 0x000a, + 0x34d2: 0x000a, + 0x34e0: 0x000a, 0x34e1: 0x000a, 0x34e2: 0x000a, 0x34e3: 0x000a, + 0x34e4: 0x000a, 0x34e5: 0x000a, 0x34e6: 0x000a, 0x34e7: 0x000a, 0x34e8: 0x000a, 0x34e9: 0x000a, + 0x34ea: 0x000a, 0x34eb: 0x000a, 0x34ec: 0x000a, + 0x34f0: 0x000a, 0x34f1: 0x000a, 0x34f2: 0x000a, 0x34f3: 0x000a, 0x34f4: 0x000a, 0x34f5: 0x000a, + 0x34f6: 0x000a, + // Block 0xd4, offset 0x3500 + 0x3500: 0x000a, 0x3501: 0x000a, 0x3502: 0x000a, 0x3503: 0x000a, 0x3504: 0x000a, 0x3505: 0x000a, + 0x3506: 0x000a, 0x3507: 0x000a, 0x3508: 0x000a, 0x3509: 0x000a, 0x350a: 0x000a, 0x350b: 0x000a, + 0x350c: 0x000a, 0x350d: 0x000a, 0x350e: 0x000a, 0x350f: 0x000a, 0x3510: 0x000a, 0x3511: 0x000a, + 0x3512: 0x000a, 0x3513: 0x000a, 0x3514: 0x000a, + // Block 0xd5, offset 0x3540 + 0x3540: 0x000a, 0x3541: 0x000a, 0x3542: 0x000a, 0x3543: 0x000a, 0x3544: 0x000a, 0x3545: 0x000a, + 0x3546: 0x000a, 0x3547: 0x000a, 0x3548: 0x000a, 0x3549: 0x000a, 0x354a: 0x000a, 0x354b: 0x000a, + 0x3550: 0x000a, 0x3551: 0x000a, + 0x3552: 0x000a, 0x3553: 0x000a, 0x3554: 0x000a, 0x3555: 0x000a, 0x3556: 0x000a, 0x3557: 0x000a, + 0x3558: 0x000a, 0x3559: 0x000a, 0x355a: 0x000a, 0x355b: 0x000a, 0x355c: 0x000a, 0x355d: 0x000a, + 0x355e: 0x000a, 0x355f: 0x000a, 0x3560: 0x000a, 0x3561: 0x000a, 0x3562: 0x000a, 0x3563: 0x000a, + 0x3564: 0x000a, 0x3565: 0x000a, 0x3566: 0x000a, 0x3567: 0x000a, 0x3568: 0x000a, 0x3569: 0x000a, + 0x356a: 0x000a, 0x356b: 0x000a, 0x356c: 0x000a, 0x356d: 0x000a, 0x356e: 0x000a, 0x356f: 0x000a, + 0x3570: 0x000a, 0x3571: 0x000a, 0x3572: 0x000a, 0x3573: 0x000a, 0x3574: 0x000a, 0x3575: 0x000a, + 0x3576: 0x000a, 0x3577: 0x000a, 0x3578: 0x000a, 0x3579: 0x000a, 0x357a: 0x000a, 0x357b: 0x000a, + 0x357c: 0x000a, 0x357d: 0x000a, 0x357e: 0x000a, 0x357f: 0x000a, + // Block 0xd6, offset 0x3580 + 0x3580: 0x000a, 0x3581: 0x000a, 0x3582: 0x000a, 0x3583: 0x000a, 0x3584: 0x000a, 0x3585: 0x000a, + 0x3586: 0x000a, 0x3587: 0x000a, + 0x3590: 0x000a, 0x3591: 0x000a, + 0x3592: 0x000a, 0x3593: 0x000a, 0x3594: 0x000a, 0x3595: 0x000a, 0x3596: 0x000a, 0x3597: 0x000a, + 0x3598: 0x000a, 0x3599: 0x000a, + 0x35a0: 0x000a, 0x35a1: 0x000a, 0x35a2: 0x000a, 0x35a3: 0x000a, + 0x35a4: 0x000a, 0x35a5: 0x000a, 0x35a6: 0x000a, 0x35a7: 0x000a, 0x35a8: 0x000a, 0x35a9: 0x000a, + 0x35aa: 0x000a, 0x35ab: 0x000a, 0x35ac: 0x000a, 0x35ad: 0x000a, 0x35ae: 0x000a, 0x35af: 0x000a, + 0x35b0: 0x000a, 0x35b1: 0x000a, 0x35b2: 0x000a, 0x35b3: 0x000a, 0x35b4: 0x000a, 0x35b5: 0x000a, + 0x35b6: 0x000a, 0x35b7: 0x000a, 0x35b8: 0x000a, 0x35b9: 0x000a, 0x35ba: 0x000a, 0x35bb: 0x000a, + 0x35bc: 0x000a, 0x35bd: 0x000a, 0x35be: 0x000a, 0x35bf: 0x000a, + // Block 0xd7, offset 0x35c0 + 0x35c0: 0x000a, 0x35c1: 0x000a, 0x35c2: 0x000a, 0x35c3: 0x000a, 0x35c4: 0x000a, 0x35c5: 0x000a, + 0x35c6: 0x000a, 0x35c7: 0x000a, + 0x35d0: 0x000a, 0x35d1: 0x000a, + 0x35d2: 0x000a, 0x35d3: 0x000a, 0x35d4: 0x000a, 0x35d5: 0x000a, 0x35d6: 0x000a, 0x35d7: 0x000a, + 0x35d8: 0x000a, 0x35d9: 0x000a, 0x35da: 0x000a, 0x35db: 0x000a, 0x35dc: 0x000a, 0x35dd: 0x000a, + 0x35de: 0x000a, 0x35df: 0x000a, 0x35e0: 0x000a, 0x35e1: 0x000a, 0x35e2: 0x000a, 0x35e3: 0x000a, + 0x35e4: 0x000a, 0x35e5: 0x000a, 0x35e6: 0x000a, 0x35e7: 0x000a, 0x35e8: 0x000a, 0x35e9: 0x000a, + 0x35ea: 0x000a, 0x35eb: 0x000a, 0x35ec: 0x000a, 0x35ed: 0x000a, + // Block 0xd8, offset 0x3600 + 0x3610: 0x000a, 0x3611: 0x000a, + 0x3612: 0x000a, 0x3613: 0x000a, 0x3614: 0x000a, 0x3615: 0x000a, 0x3616: 0x000a, 0x3617: 0x000a, + 0x3618: 0x000a, 0x3619: 0x000a, 0x361a: 0x000a, 0x361b: 0x000a, 0x361c: 0x000a, 0x361d: 0x000a, + 0x361e: 0x000a, 0x3620: 0x000a, 0x3621: 0x000a, 0x3622: 0x000a, 0x3623: 0x000a, + 0x3624: 0x000a, 0x3625: 0x000a, 0x3626: 0x000a, 0x3627: 0x000a, + 0x3630: 0x000a, 0x3633: 0x000a, 0x3634: 0x000a, 0x3635: 0x000a, + 0x3636: 0x000a, 0x3637: 0x000a, 0x3638: 0x000a, 0x3639: 0x000a, 0x363a: 0x000a, 0x363b: 0x000a, + 0x363c: 0x000a, 0x363d: 0x000a, 0x363e: 0x000a, + // Block 0xd9, offset 0x3640 + 0x3640: 0x000a, 0x3641: 0x000a, 0x3642: 0x000a, 0x3643: 0x000a, 0x3644: 0x000a, 0x3645: 0x000a, + 0x3646: 0x000a, 0x3647: 0x000a, 0x3648: 0x000a, 0x3649: 0x000a, 0x364a: 0x000a, 0x364b: 0x000a, + 0x3650: 0x000a, 0x3651: 0x000a, + 0x3652: 0x000a, 0x3653: 0x000a, 0x3654: 0x000a, 0x3655: 0x000a, 0x3656: 0x000a, 0x3657: 0x000a, + 0x3658: 0x000a, 0x3659: 0x000a, 0x365a: 0x000a, 0x365b: 0x000a, 0x365c: 0x000a, 0x365d: 0x000a, + 0x365e: 0x000a, + // Block 0xda, offset 0x3680 + 0x3680: 0x000a, 0x3681: 0x000a, 0x3682: 0x000a, 0x3683: 0x000a, 0x3684: 0x000a, 0x3685: 0x000a, + 0x3686: 0x000a, 0x3687: 0x000a, 0x3688: 0x000a, 0x3689: 0x000a, 0x368a: 0x000a, 0x368b: 0x000a, + 0x368c: 0x000a, 0x368d: 0x000a, 0x368e: 0x000a, 0x368f: 0x000a, 0x3690: 0x000a, 0x3691: 0x000a, + // Block 0xdb, offset 0x36c0 + 0x36fe: 0x000b, 0x36ff: 0x000b, + // Block 0xdc, offset 0x3700 + 0x3700: 0x000b, 0x3701: 0x000b, 0x3702: 0x000b, 0x3703: 0x000b, 0x3704: 0x000b, 0x3705: 0x000b, + 0x3706: 0x000b, 0x3707: 0x000b, 0x3708: 0x000b, 0x3709: 0x000b, 0x370a: 0x000b, 0x370b: 0x000b, + 0x370c: 0x000b, 0x370d: 0x000b, 0x370e: 0x000b, 0x370f: 0x000b, 0x3710: 0x000b, 0x3711: 0x000b, + 0x3712: 0x000b, 0x3713: 0x000b, 0x3714: 0x000b, 0x3715: 0x000b, 0x3716: 0x000b, 0x3717: 0x000b, + 0x3718: 0x000b, 0x3719: 0x000b, 0x371a: 0x000b, 0x371b: 0x000b, 0x371c: 0x000b, 0x371d: 0x000b, + 0x371e: 0x000b, 0x371f: 0x000b, 0x3720: 0x000b, 0x3721: 0x000b, 0x3722: 0x000b, 0x3723: 0x000b, + 0x3724: 0x000b, 0x3725: 0x000b, 0x3726: 0x000b, 0x3727: 0x000b, 0x3728: 0x000b, 0x3729: 0x000b, + 0x372a: 0x000b, 0x372b: 0x000b, 0x372c: 0x000b, 0x372d: 0x000b, 0x372e: 0x000b, 0x372f: 0x000b, + 0x3730: 0x000b, 0x3731: 0x000b, 0x3732: 0x000b, 0x3733: 0x000b, 0x3734: 0x000b, 0x3735: 0x000b, + 0x3736: 0x000b, 0x3737: 0x000b, 0x3738: 0x000b, 0x3739: 0x000b, 0x373a: 0x000b, 0x373b: 0x000b, + 0x373c: 0x000b, 0x373d: 0x000b, 0x373e: 0x000b, 0x373f: 0x000b, + // Block 0xdd, offset 0x3740 + 0x3740: 0x000c, 0x3741: 0x000c, 0x3742: 0x000c, 0x3743: 0x000c, 0x3744: 0x000c, 0x3745: 0x000c, + 0x3746: 0x000c, 0x3747: 0x000c, 0x3748: 0x000c, 0x3749: 0x000c, 0x374a: 0x000c, 0x374b: 0x000c, + 0x374c: 0x000c, 0x374d: 0x000c, 0x374e: 0x000c, 0x374f: 0x000c, 0x3750: 0x000c, 0x3751: 0x000c, + 0x3752: 0x000c, 0x3753: 0x000c, 0x3754: 0x000c, 0x3755: 0x000c, 0x3756: 0x000c, 0x3757: 0x000c, + 0x3758: 0x000c, 0x3759: 0x000c, 0x375a: 0x000c, 0x375b: 0x000c, 0x375c: 0x000c, 0x375d: 0x000c, + 0x375e: 0x000c, 0x375f: 0x000c, 0x3760: 0x000c, 0x3761: 0x000c, 0x3762: 0x000c, 0x3763: 0x000c, + 0x3764: 0x000c, 0x3765: 0x000c, 0x3766: 0x000c, 0x3767: 0x000c, 0x3768: 0x000c, 0x3769: 0x000c, + 0x376a: 0x000c, 0x376b: 0x000c, 0x376c: 0x000c, 0x376d: 0x000c, 0x376e: 0x000c, 0x376f: 0x000c, + 0x3770: 0x000b, 0x3771: 0x000b, 0x3772: 0x000b, 0x3773: 0x000b, 0x3774: 0x000b, 0x3775: 0x000b, + 0x3776: 0x000b, 0x3777: 0x000b, 0x3778: 0x000b, 0x3779: 0x000b, 0x377a: 0x000b, 0x377b: 0x000b, + 0x377c: 0x000b, 0x377d: 0x000b, 0x377e: 0x000b, 0x377f: 0x000b, +} + +// bidiIndex: 24 blocks, 1536 entries, 1536 bytes +// Block 0 is the zero block. +var bidiIndex = [1536]uint8{ + // Block 0x0, offset 0x0 + // Block 0x1, offset 0x40 + // Block 0x2, offset 0x80 + // Block 0x3, offset 0xc0 + 0xc2: 0x01, 0xc3: 0x02, + 0xca: 0x03, 0xcb: 0x04, 0xcc: 0x05, 0xcd: 0x06, 0xce: 0x07, 0xcf: 0x08, + 0xd2: 0x09, 0xd6: 0x0a, 0xd7: 0x0b, + 0xd8: 0x0c, 0xd9: 0x0d, 0xda: 0x0e, 0xdb: 0x0f, 0xdc: 0x10, 0xdd: 0x11, 0xde: 0x12, 0xdf: 0x13, + 0xe0: 0x02, 0xe1: 0x03, 0xe2: 0x04, 0xe3: 0x05, 0xe4: 0x06, + 0xea: 0x07, 0xef: 0x08, + 0xf0: 0x11, 0xf1: 0x12, 0xf2: 0x12, 0xf3: 0x14, 0xf4: 0x15, + // Block 0x4, offset 0x100 + 0x120: 0x14, 0x121: 0x15, 0x122: 0x16, 0x123: 0x17, 0x124: 0x18, 0x125: 0x19, 0x126: 0x1a, 0x127: 0x1b, + 0x128: 0x1c, 0x129: 0x1d, 0x12a: 0x1c, 0x12b: 0x1e, 0x12c: 0x1f, 0x12d: 0x20, 0x12e: 0x21, 0x12f: 0x22, + 0x130: 0x23, 0x131: 0x24, 0x132: 0x1a, 0x133: 0x25, 0x134: 0x26, 0x135: 0x27, 0x137: 0x28, + 0x138: 0x29, 0x139: 0x2a, 0x13a: 0x2b, 0x13b: 0x2c, 0x13c: 0x2d, 0x13d: 0x2e, 0x13e: 0x2f, 0x13f: 0x30, + // Block 0x5, offset 0x140 + 0x140: 0x31, 0x141: 0x32, 0x142: 0x33, + 0x14d: 0x34, 0x14e: 0x35, + 0x150: 0x36, + 0x15a: 0x37, 0x15c: 0x38, 0x15d: 0x39, 0x15e: 0x3a, 0x15f: 0x3b, + 0x160: 0x3c, 0x162: 0x3d, 0x164: 0x3e, 0x165: 0x3f, 0x167: 0x40, + 0x168: 0x41, 0x169: 0x42, 0x16a: 0x43, 0x16c: 0x44, 0x16d: 0x45, 0x16e: 0x46, 0x16f: 0x47, + 0x170: 0x48, 0x173: 0x49, 0x177: 0x4a, + 0x17e: 0x4b, 0x17f: 0x4c, + // Block 0x6, offset 0x180 + 0x180: 0x4d, 0x181: 0x4e, 0x182: 0x4f, 0x183: 0x50, 0x184: 0x51, 0x185: 0x52, 0x186: 0x53, 0x187: 0x54, + 0x188: 0x55, 0x189: 0x54, 0x18a: 0x54, 0x18b: 0x54, 0x18c: 0x56, 0x18d: 0x57, 0x18e: 0x58, 0x18f: 0x59, + 0x190: 0x5a, 0x191: 0x5b, 0x192: 0x5c, 0x193: 0x5d, 0x194: 0x54, 0x195: 0x54, 0x196: 0x54, 0x197: 0x54, + 0x198: 0x54, 0x199: 0x54, 0x19a: 0x5e, 0x19b: 0x54, 0x19c: 0x54, 0x19d: 0x5f, 0x19e: 0x54, 0x19f: 0x60, + 0x1a4: 0x54, 0x1a5: 0x54, 0x1a6: 0x61, 0x1a7: 0x62, + 0x1a8: 0x54, 0x1a9: 0x54, 0x1aa: 0x54, 0x1ab: 0x54, 0x1ac: 0x54, 0x1ad: 0x63, 0x1ae: 0x64, 0x1af: 0x65, + 0x1b3: 0x66, 0x1b5: 0x67, 0x1b7: 0x68, + 0x1b8: 0x69, 0x1b9: 0x6a, 0x1ba: 0x6b, 0x1bb: 0x6c, 0x1bc: 0x54, 0x1bd: 0x54, 0x1be: 0x54, 0x1bf: 0x6d, + // Block 0x7, offset 0x1c0 + 0x1c0: 0x6e, 0x1c2: 0x6f, 0x1c3: 0x70, 0x1c7: 0x71, + 0x1c8: 0x72, 0x1c9: 0x73, 0x1ca: 0x74, 0x1cb: 0x75, 0x1cd: 0x76, 0x1cf: 0x77, + // Block 0x8, offset 0x200 + 0x237: 0x54, + // Block 0x9, offset 0x240 + 0x252: 0x78, 0x253: 0x79, + 0x258: 0x7a, 0x259: 0x7b, 0x25a: 0x7c, 0x25b: 0x7d, 0x25c: 0x7e, 0x25e: 0x7f, + 0x260: 0x80, 0x261: 0x81, 0x263: 0x82, 0x264: 0x83, 0x265: 0x84, 0x266: 0x85, 0x267: 0x86, + 0x268: 0x87, 0x269: 0x88, 0x26a: 0x89, 0x26b: 0x8a, 0x26f: 0x8b, + // Block 0xa, offset 0x280 + 0x2ac: 0x8c, 0x2ad: 0x8d, 0x2ae: 0x0e, 0x2af: 0x0e, + 0x2b0: 0x0e, 0x2b1: 0x0e, 0x2b2: 0x0e, 0x2b3: 0x0e, 0x2b4: 0x8e, 0x2b5: 0x0e, 0x2b6: 0x0e, 0x2b7: 0x8f, + 0x2b8: 0x90, 0x2b9: 0x91, 0x2ba: 0x0e, 0x2bb: 0x92, 0x2bc: 0x93, 0x2bd: 0x94, 0x2bf: 0x95, + // Block 0xb, offset 0x2c0 + 0x2c4: 0x96, 0x2c5: 0x54, 0x2c6: 0x97, 0x2c7: 0x98, + 0x2cb: 0x99, 0x2cd: 0x9a, + 0x2e0: 0x9b, 0x2e1: 0x9b, 0x2e2: 0x9b, 0x2e3: 0x9b, 0x2e4: 0x9c, 0x2e5: 0x9b, 0x2e6: 0x9b, 0x2e7: 0x9b, + 0x2e8: 0x9d, 0x2e9: 0x9b, 0x2ea: 0x9b, 0x2eb: 0x9e, 0x2ec: 0x9f, 0x2ed: 0x9b, 0x2ee: 0x9b, 0x2ef: 0x9b, + 0x2f0: 0x9b, 0x2f1: 0x9b, 0x2f2: 0x9b, 0x2f3: 0x9b, 0x2f4: 0x9b, 0x2f5: 0x9b, 0x2f6: 0x9b, 0x2f7: 0x9b, + 0x2f8: 0x9b, 0x2f9: 0xa0, 0x2fa: 0x9b, 0x2fb: 0x9b, 0x2fc: 0x9b, 0x2fd: 0x9b, 0x2fe: 0x9b, 0x2ff: 0x9b, + // Block 0xc, offset 0x300 + 0x300: 0xa1, 0x301: 0xa2, 0x302: 0xa3, 0x304: 0xa4, 0x305: 0xa5, 0x306: 0xa6, 0x307: 0xa7, + 0x308: 0xa8, 0x30b: 0xa9, 0x30c: 0xaa, 0x30d: 0xab, + 0x310: 0xac, 0x311: 0xad, 0x312: 0xae, 0x313: 0xaf, 0x316: 0xb0, 0x317: 0xb1, + 0x318: 0xb2, 0x319: 0xb3, 0x31a: 0xb4, 0x31c: 0xb5, + 0x330: 0xb6, 0x332: 0xb7, + // Block 0xd, offset 0x340 + 0x36b: 0xb8, 0x36c: 0xb9, + 0x37e: 0xba, + // Block 0xe, offset 0x380 + 0x3b2: 0xbb, + // Block 0xf, offset 0x3c0 + 0x3c5: 0xbc, 0x3c6: 0xbd, + 0x3c8: 0x54, 0x3c9: 0xbe, 0x3cc: 0x54, 0x3cd: 0xbf, + 0x3db: 0xc0, 0x3dc: 0xc1, 0x3dd: 0xc2, 0x3de: 0xc3, 0x3df: 0xc4, + 0x3e8: 0xc5, 0x3e9: 0xc6, 0x3ea: 0xc7, + // Block 0x10, offset 0x400 + 0x400: 0xc8, + 0x420: 0x9b, 0x421: 0x9b, 0x422: 0x9b, 0x423: 0xc9, 0x424: 0x9b, 0x425: 0xca, 0x426: 0x9b, 0x427: 0x9b, + 0x428: 0x9b, 0x429: 0x9b, 0x42a: 0x9b, 0x42b: 0x9b, 0x42c: 0x9b, 0x42d: 0x9b, 0x42e: 0x9b, 0x42f: 0x9b, + 0x430: 0x9b, 0x431: 0x9b, 0x432: 0x9b, 0x433: 0x9b, 0x434: 0x9b, 0x435: 0x9b, 0x436: 0x9b, 0x437: 0x9b, + 0x438: 0x0e, 0x439: 0x0e, 0x43a: 0x0e, 0x43b: 0xcb, 0x43c: 0x9b, 0x43d: 0x9b, 0x43e: 0x9b, 0x43f: 0x9b, + // Block 0x11, offset 0x440 + 0x440: 0xcc, 0x441: 0x54, 0x442: 0xcd, 0x443: 0xce, 0x444: 0xcf, 0x445: 0xd0, + 0x44c: 0x54, 0x44d: 0x54, 0x44e: 0x54, 0x44f: 0x54, + 0x450: 0x54, 0x451: 0x54, 0x452: 0x54, 0x453: 0x54, 0x454: 0x54, 0x455: 0x54, 0x456: 0x54, 0x457: 0x54, + 0x458: 0x54, 0x459: 0x54, 0x45a: 0x54, 0x45b: 0xd1, 0x45c: 0x54, 0x45d: 0x6c, 0x45e: 0x54, 0x45f: 0xd2, + 0x460: 0xd3, 0x461: 0xd4, 0x462: 0xd5, 0x464: 0xd6, 0x465: 0xd7, 0x466: 0xd8, 0x467: 0x36, + 0x47f: 0xd9, + // Block 0x12, offset 0x480 + 0x4bf: 0xd9, + // Block 0x13, offset 0x4c0 + 0x4d0: 0x09, 0x4d1: 0x0a, 0x4d6: 0x0b, + 0x4db: 0x0c, 0x4dd: 0x0d, 0x4de: 0x0e, 0x4df: 0x0f, + 0x4ef: 0x10, + 0x4ff: 0x10, + // Block 0x14, offset 0x500 + 0x50f: 0x10, + 0x51f: 0x10, + 0x52f: 0x10, + 0x53f: 0x10, + // Block 0x15, offset 0x540 + 0x540: 0xda, 0x541: 0xda, 0x542: 0xda, 0x543: 0xda, 0x544: 0x05, 0x545: 0x05, 0x546: 0x05, 0x547: 0xdb, + 0x548: 0xda, 0x549: 0xda, 0x54a: 0xda, 0x54b: 0xda, 0x54c: 0xda, 0x54d: 0xda, 0x54e: 0xda, 0x54f: 0xda, + 0x550: 0xda, 0x551: 0xda, 0x552: 0xda, 0x553: 0xda, 0x554: 0xda, 0x555: 0xda, 0x556: 0xda, 0x557: 0xda, + 0x558: 0xda, 0x559: 0xda, 0x55a: 0xda, 0x55b: 0xda, 0x55c: 0xda, 0x55d: 0xda, 0x55e: 0xda, 0x55f: 0xda, + 0x560: 0xda, 0x561: 0xda, 0x562: 0xda, 0x563: 0xda, 0x564: 0xda, 0x565: 0xda, 0x566: 0xda, 0x567: 0xda, + 0x568: 0xda, 0x569: 0xda, 0x56a: 0xda, 0x56b: 0xda, 0x56c: 0xda, 0x56d: 0xda, 0x56e: 0xda, 0x56f: 0xda, + 0x570: 0xda, 0x571: 0xda, 0x572: 0xda, 0x573: 0xda, 0x574: 0xda, 0x575: 0xda, 0x576: 0xda, 0x577: 0xda, + 0x578: 0xda, 0x579: 0xda, 0x57a: 0xda, 0x57b: 0xda, 0x57c: 0xda, 0x57d: 0xda, 0x57e: 0xda, 0x57f: 0xda, + // Block 0x16, offset 0x580 + 0x58f: 0x10, + 0x59f: 0x10, + 0x5a0: 0x13, + 0x5af: 0x10, + 0x5bf: 0x10, + // Block 0x17, offset 0x5c0 + 0x5cf: 0x10, +} + +// Total table size 15800 bytes (15KiB); checksum: F50EF68C diff --git a/libgo/go/golang_org/x/text/unicode/bidi/trieval.go b/libgo/go/golang_org/x/text/unicode/bidi/trieval.go new file mode 100644 index 00000000000..a825fdea4d0 --- /dev/null +++ b/libgo/go/golang_org/x/text/unicode/bidi/trieval.go @@ -0,0 +1,62 @@ +// Code generated by running "go run gen.go -core" in golang.org/x/text. DO NOT EDIT. + +// Code generated by running "go generate" in golang_org/x/text. DO NOT EDIT. + +package bidi + +// Class is the Unicode BiDi class. Each rune has a single class. +type Class uint + +const ( + L Class = iota // LeftToRight + R // RightToLeft + EN // EuropeanNumber + ES // EuropeanSeparator + ET // EuropeanTerminator + AN // ArabicNumber + CS // CommonSeparator + B // ParagraphSeparator + S // SegmentSeparator + WS // WhiteSpace + ON // OtherNeutral + BN // BoundaryNeutral + NSM // NonspacingMark + AL // ArabicLetter + Control // Control LRO - PDI + + numClass + + LRO // LeftToRightOverride + RLO // RightToLeftOverride + LRE // LeftToRightEmbedding + RLE // RightToLeftEmbedding + PDF // PopDirectionalFormat + LRI // LeftToRightIsolate + RLI // RightToLeftIsolate + FSI // FirstStrongIsolate + PDI // PopDirectionalIsolate + + unknownClass = ^Class(0) +) + +var controlToClass = map[rune]Class{ + 0x202D: LRO, // LeftToRightOverride, + 0x202E: RLO, // RightToLeftOverride, + 0x202A: LRE, // LeftToRightEmbedding, + 0x202B: RLE, // RightToLeftEmbedding, + 0x202C: PDF, // PopDirectionalFormat, + 0x2066: LRI, // LeftToRightIsolate, + 0x2067: RLI, // RightToLeftIsolate, + 0x2068: FSI, // FirstStrongIsolate, + 0x2069: PDI, // PopDirectionalIsolate, +} + +// A trie entry has the following bits: +// 7..5 XOR mask for brackets +// 4 1: Bracket open, 0: Bracket close +// 3..0 Class type + +const ( + openMask = 0x10 + xorMaskShift = 5 +) diff --git a/libgo/go/golang_org/x/text/unicode/doc.go b/libgo/go/golang_org/x/text/unicode/doc.go new file mode 100644 index 00000000000..36b462a3e13 --- /dev/null +++ b/libgo/go/golang_org/x/text/unicode/doc.go @@ -0,0 +1,10 @@ +// Code generated by running "go run gen.go -core" in golang.org/x/text. DO NOT EDIT. + +// Copyright 2015 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// unicode holds packages with implementations of Unicode standards that are +// mostly used as building blocks for other packages in golang_org/x/text, +// layout engines, or are otherwise more low-level in nature. +package unicode diff --git a/libgo/go/golang_org/x/text/unicode/norm/composition.go b/libgo/go/golang_org/x/text/unicode/norm/composition.go index d17b278adc3..7380b0554e4 100644 --- a/libgo/go/golang_org/x/text/unicode/norm/composition.go +++ b/libgo/go/golang_org/x/text/unicode/norm/composition.go @@ -1,3 +1,5 @@ +// Code generated by running "go run gen.go -core" in golang.org/x/text. DO NOT EDIT. + // Copyright 2011 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. diff --git a/libgo/go/golang_org/x/text/unicode/norm/example_iter_test.go b/libgo/go/golang_org/x/text/unicode/norm/example_iter_test.go new file mode 100644 index 00000000000..aed6c16fbb2 --- /dev/null +++ b/libgo/go/golang_org/x/text/unicode/norm/example_iter_test.go @@ -0,0 +1,84 @@ +// Code generated by running "go run gen.go -core" in golang.org/x/text. DO NOT EDIT. + +// Copyright 2012 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package norm_test + +import ( + "bytes" + "fmt" + "unicode/utf8" + + "golang_org/x/text/unicode/norm" +) + +// EqualSimple uses a norm.Iter to compare two non-normalized +// strings for equivalence. +func EqualSimple(a, b string) bool { + var ia, ib norm.Iter + ia.InitString(norm.NFKD, a) + ib.InitString(norm.NFKD, b) + for !ia.Done() && !ib.Done() { + if !bytes.Equal(ia.Next(), ib.Next()) { + return false + } + } + return ia.Done() && ib.Done() +} + +// FindPrefix finds the longest common prefix of ASCII characters +// of a and b. +func FindPrefix(a, b string) int { + i := 0 + for ; i < len(a) && i < len(b) && a[i] < utf8.RuneSelf && a[i] == b[i]; i++ { + } + return i +} + +// EqualOpt is like EqualSimple, but optimizes the special +// case for ASCII characters. +func EqualOpt(a, b string) bool { + n := FindPrefix(a, b) + a, b = a[n:], b[n:] + var ia, ib norm.Iter + ia.InitString(norm.NFKD, a) + ib.InitString(norm.NFKD, b) + for !ia.Done() && !ib.Done() { + if !bytes.Equal(ia.Next(), ib.Next()) { + return false + } + if n := int64(FindPrefix(a[ia.Pos():], b[ib.Pos():])); n != 0 { + ia.Seek(n, 1) + ib.Seek(n, 1) + } + } + return ia.Done() && ib.Done() +} + +var compareTests = []struct{ a, b string }{ + {"aaa", "aaa"}, + {"aaa", "aab"}, + {"a\u0300a", "\u00E0a"}, + {"a\u0300\u0320b", "a\u0320\u0300b"}, + {"\u1E0A\u0323", "\x44\u0323\u0307"}, + // A character that decomposes into multiple segments + // spans several iterations. + {"\u3304", "\u30A4\u30CB\u30F3\u30AF\u3099"}, +} + +func ExampleIter() { + for i, t := range compareTests { + r0 := EqualSimple(t.a, t.b) + r1 := EqualOpt(t.a, t.b) + fmt.Printf("%d: %v %v\n", i, r0, r1) + } + // Output: + // 0: true true + // 1: false false + // 2: true true + // 3: true true + // 4: true true + // 5: true true +} diff --git a/libgo/go/golang_org/x/text/unicode/norm/example_test.go b/libgo/go/golang_org/x/text/unicode/norm/example_test.go new file mode 100644 index 00000000000..72e72c9d34a --- /dev/null +++ b/libgo/go/golang_org/x/text/unicode/norm/example_test.go @@ -0,0 +1,29 @@ +// Code generated by running "go run gen.go -core" in golang.org/x/text. DO NOT EDIT. + +// Copyright 2016 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package norm_test + +import ( + "fmt" + + "golang_org/x/text/unicode/norm" +) + +func ExampleForm_NextBoundary() { + s := norm.NFD.String("Mêlée") + + for i := 0; i < len(s); { + d := norm.NFC.NextBoundaryInString(s[i:], true) + fmt.Printf("%[1]s: %+[1]q\n", s[i:i+d]) + i += d + } + // Output: + // M: "M" + // ê: "e\u0302" + // l: "l" + // é: "e\u0301" + // e: "e" +} diff --git a/libgo/go/golang_org/x/text/unicode/norm/forminfo.go b/libgo/go/golang_org/x/text/unicode/norm/forminfo.go index 15a67c653a3..f3e2930daf2 100644 --- a/libgo/go/golang_org/x/text/unicode/norm/forminfo.go +++ b/libgo/go/golang_org/x/text/unicode/norm/forminfo.go @@ -1,3 +1,5 @@ +// Code generated by running "go run gen.go -core" in golang.org/x/text. DO NOT EDIT. + // Copyright 2011 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. @@ -10,7 +12,7 @@ package norm // and its corresponding decomposing form share the same trie. Each trie maps // a rune to a uint16. The values take two forms. For v >= 0x8000: // bits -// 15: 1 (inverse of NFD_QD bit of qcInfo) +// 15: 1 (inverse of NFD_QC bit of qcInfo) // 13..7: qcInfo (see below). isYesD is always true (no decompostion). // 6..0: ccc (compressed CCC value). // For v < 0x8000, the respective rune has a decomposition and v is an index @@ -56,28 +58,31 @@ type formInfo struct { nextMain iterFunc } -var formTable []*formInfo - -func init() { - formTable = make([]*formInfo, 4) - - for i := range formTable { - f := &formInfo{} - formTable[i] = f - f.form = Form(i) - if Form(i) == NFKD || Form(i) == NFKC { - f.compatibility = true - f.info = lookupInfoNFKC - } else { - f.info = lookupInfoNFC - } - f.nextMain = nextDecomposed - if Form(i) == NFC || Form(i) == NFKC { - f.nextMain = nextComposed - f.composing = true - } - } -} +var formTable = []*formInfo{{ + form: NFC, + composing: true, + compatibility: false, + info: lookupInfoNFC, + nextMain: nextComposed, +}, { + form: NFD, + composing: false, + compatibility: false, + info: lookupInfoNFC, + nextMain: nextDecomposed, +}, { + form: NFKC, + composing: true, + compatibility: true, + info: lookupInfoNFKC, + nextMain: nextComposed, +}, { + form: NFKD, + composing: false, + compatibility: true, + info: lookupInfoNFKC, + nextMain: nextDecomposed, +}} // We do not distinguish between boundaries for NFC, NFD, etc. to avoid // unexpected behavior for the user. For example, in NFD, there is a boundary diff --git a/libgo/go/golang_org/x/text/unicode/norm/input.go b/libgo/go/golang_org/x/text/unicode/norm/input.go index 045d4ccce2e..202bde133ab 100644 --- a/libgo/go/golang_org/x/text/unicode/norm/input.go +++ b/libgo/go/golang_org/x/text/unicode/norm/input.go @@ -1,3 +1,5 @@ +// Code generated by running "go run gen.go -core" in golang.org/x/text. DO NOT EDIT. + // Copyright 2011 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. diff --git a/libgo/go/golang_org/x/text/unicode/norm/iter.go b/libgo/go/golang_org/x/text/unicode/norm/iter.go index 0a42a72de81..c0cf949435e 100644 --- a/libgo/go/golang_org/x/text/unicode/norm/iter.go +++ b/libgo/go/golang_org/x/text/unicode/norm/iter.go @@ -1,3 +1,5 @@ +// Code generated by running "go run gen.go -core" in golang.org/x/text. DO NOT EDIT. + // Copyright 2011 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. diff --git a/libgo/go/golang_org/x/text/unicode/norm/normalize.go b/libgo/go/golang_org/x/text/unicode/norm/normalize.go index 15c962e2e9f..4427ee3177a 100644 --- a/libgo/go/golang_org/x/text/unicode/norm/normalize.go +++ b/libgo/go/golang_org/x/text/unicode/norm/normalize.go @@ -1,12 +1,13 @@ +// Code generated by running "go run gen.go -core" in golang.org/x/text. DO NOT EDIT. + // Copyright 2011 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -//go:generate go run maketables.go triegen.go -//go:generate go run maketables.go triegen.go -test +// Note: the file data_test.go that is generated should not be checked in. // Package norm contains types and functions for normalizing Unicode strings. -package norm // import "golang.org/x/text/unicode/norm" +package norm // import "golang_org/x/text/unicode/norm" import ( "unicode/utf8" diff --git a/libgo/go/golang_org/x/text/unicode/norm/readwriter.go b/libgo/go/golang_org/x/text/unicode/norm/readwriter.go index d926ee903e5..482ac85c745 100644 --- a/libgo/go/golang_org/x/text/unicode/norm/readwriter.go +++ b/libgo/go/golang_org/x/text/unicode/norm/readwriter.go @@ -1,3 +1,5 @@ +// Code generated by running "go run gen.go -core" in golang.org/x/text. DO NOT EDIT. + // Copyright 2011 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. diff --git a/libgo/go/golang_org/x/text/unicode/norm/tables.go b/libgo/go/golang_org/x/text/unicode/norm/tables.go index a56697b571b..ac99519efc5 100644 --- a/libgo/go/golang_org/x/text/unicode/norm/tables.go +++ b/libgo/go/golang_org/x/text/unicode/norm/tables.go @@ -1,4 +1,6 @@ -// This file was generated by go generate; DO NOT EDIT +// Code generated by running "go run gen.go -core" in golang.org/x/text. DO NOT EDIT. + +// Code generated by running "go generate" in golang_org/x/text. DO NOT EDIT. package norm @@ -27,14 +29,14 @@ const ( firstMulti = 0x186D firstCCC = 0x2C9E endMulti = 0x2F60 - firstLeadingCCC = 0x4A44 - firstCCCZeroExcept = 0x4A5A - firstStarterWithNLead = 0x4A81 - lastDecomp = 0x4A83 + firstLeadingCCC = 0x49AE + firstCCCZeroExcept = 0x4A78 + firstStarterWithNLead = 0x4A9F + lastDecomp = 0x4AA1 maxDecomp = 0x8000 ) -// decomps: 19075 bytes +// decomps: 19105 bytes var decomps = [...]byte{ // Bytes 0 - 3f 0x00, 0x41, 0x20, 0x41, 0x21, 0x41, 0x22, 0x41, @@ -2443,283 +2445,287 @@ var decomps = [...]byte{ 0xD9, 0x8F, 0x69, 0x43, 0x20, 0xD9, 0x90, 0x6D, 0x43, 0x20, 0xD9, 0x91, 0x71, 0x43, 0x20, 0xD9, 0x92, 0x75, 0x43, 0x41, 0xCC, 0x8A, 0xC9, 0x43, - 0x73, 0xCC, 0x87, 0xC9, 0x43, 0xE1, 0x85, 0xA1, - 0x01, 0x43, 0xE1, 0x85, 0xA2, 0x01, 0x43, 0xE1, - 0x85, 0xA3, 0x01, 0x43, 0xE1, 0x85, 0xA4, 0x01, - 0x43, 0xE1, 0x85, 0xA5, 0x01, 0x43, 0xE1, 0x85, - 0xA6, 0x01, 0x43, 0xE1, 0x85, 0xA7, 0x01, 0x43, + 0x73, 0xCC, 0x87, 0xC9, 0x44, 0x20, 0xE3, 0x82, + 0x99, 0x0D, 0x44, 0x20, 0xE3, 0x82, 0x9A, 0x0D, + 0x44, 0xC2, 0xA8, 0xCC, 0x81, 0xCA, 0x44, 0xCE, + 0x91, 0xCC, 0x81, 0xC9, 0x44, 0xCE, 0x95, 0xCC, + 0x81, 0xC9, 0x44, 0xCE, 0x97, 0xCC, 0x81, 0xC9, // Bytes 4300 - 433f - 0xE1, 0x85, 0xA8, 0x01, 0x43, 0xE1, 0x85, 0xA9, - 0x01, 0x43, 0xE1, 0x85, 0xAA, 0x01, 0x43, 0xE1, - 0x85, 0xAB, 0x01, 0x43, 0xE1, 0x85, 0xAC, 0x01, - 0x43, 0xE1, 0x85, 0xAD, 0x01, 0x43, 0xE1, 0x85, - 0xAE, 0x01, 0x43, 0xE1, 0x85, 0xAF, 0x01, 0x43, - 0xE1, 0x85, 0xB0, 0x01, 0x43, 0xE1, 0x85, 0xB1, - 0x01, 0x43, 0xE1, 0x85, 0xB2, 0x01, 0x43, 0xE1, - 0x85, 0xB3, 0x01, 0x43, 0xE1, 0x85, 0xB4, 0x01, + 0x44, 0xCE, 0x99, 0xCC, 0x81, 0xC9, 0x44, 0xCE, + 0x9F, 0xCC, 0x81, 0xC9, 0x44, 0xCE, 0xA5, 0xCC, + 0x81, 0xC9, 0x44, 0xCE, 0xA5, 0xCC, 0x88, 0xC9, + 0x44, 0xCE, 0xA9, 0xCC, 0x81, 0xC9, 0x44, 0xCE, + 0xB1, 0xCC, 0x81, 0xC9, 0x44, 0xCE, 0xB5, 0xCC, + 0x81, 0xC9, 0x44, 0xCE, 0xB7, 0xCC, 0x81, 0xC9, + 0x44, 0xCE, 0xB9, 0xCC, 0x81, 0xC9, 0x44, 0xCE, + 0xBF, 0xCC, 0x81, 0xC9, 0x44, 0xCF, 0x85, 0xCC, // Bytes 4340 - 437f - 0x43, 0xE1, 0x85, 0xB5, 0x01, 0x43, 0xE1, 0x86, - 0xAA, 0x01, 0x43, 0xE1, 0x86, 0xAC, 0x01, 0x43, - 0xE1, 0x86, 0xAD, 0x01, 0x43, 0xE1, 0x86, 0xB0, - 0x01, 0x43, 0xE1, 0x86, 0xB1, 0x01, 0x43, 0xE1, - 0x86, 0xB2, 0x01, 0x43, 0xE1, 0x86, 0xB3, 0x01, - 0x43, 0xE1, 0x86, 0xB4, 0x01, 0x43, 0xE1, 0x86, - 0xB5, 0x01, 0x44, 0x20, 0xE3, 0x82, 0x99, 0x0D, - 0x44, 0x20, 0xE3, 0x82, 0x9A, 0x0D, 0x44, 0xC2, + 0x81, 0xC9, 0x44, 0xCF, 0x89, 0xCC, 0x81, 0xC9, + 0x44, 0xD7, 0x90, 0xD6, 0xB7, 0x31, 0x44, 0xD7, + 0x90, 0xD6, 0xB8, 0x35, 0x44, 0xD7, 0x90, 0xD6, + 0xBC, 0x41, 0x44, 0xD7, 0x91, 0xD6, 0xBC, 0x41, + 0x44, 0xD7, 0x91, 0xD6, 0xBF, 0x49, 0x44, 0xD7, + 0x92, 0xD6, 0xBC, 0x41, 0x44, 0xD7, 0x93, 0xD6, + 0xBC, 0x41, 0x44, 0xD7, 0x94, 0xD6, 0xBC, 0x41, + 0x44, 0xD7, 0x95, 0xD6, 0xB9, 0x39, 0x44, 0xD7, // Bytes 4380 - 43bf - 0xA8, 0xCC, 0x81, 0xCA, 0x44, 0xCE, 0x91, 0xCC, - 0x81, 0xC9, 0x44, 0xCE, 0x95, 0xCC, 0x81, 0xC9, - 0x44, 0xCE, 0x97, 0xCC, 0x81, 0xC9, 0x44, 0xCE, - 0x99, 0xCC, 0x81, 0xC9, 0x44, 0xCE, 0x9F, 0xCC, - 0x81, 0xC9, 0x44, 0xCE, 0xA5, 0xCC, 0x81, 0xC9, - 0x44, 0xCE, 0xA5, 0xCC, 0x88, 0xC9, 0x44, 0xCE, - 0xA9, 0xCC, 0x81, 0xC9, 0x44, 0xCE, 0xB1, 0xCC, - 0x81, 0xC9, 0x44, 0xCE, 0xB5, 0xCC, 0x81, 0xC9, + 0x95, 0xD6, 0xBC, 0x41, 0x44, 0xD7, 0x96, 0xD6, + 0xBC, 0x41, 0x44, 0xD7, 0x98, 0xD6, 0xBC, 0x41, + 0x44, 0xD7, 0x99, 0xD6, 0xB4, 0x25, 0x44, 0xD7, + 0x99, 0xD6, 0xBC, 0x41, 0x44, 0xD7, 0x9A, 0xD6, + 0xBC, 0x41, 0x44, 0xD7, 0x9B, 0xD6, 0xBC, 0x41, + 0x44, 0xD7, 0x9B, 0xD6, 0xBF, 0x49, 0x44, 0xD7, + 0x9C, 0xD6, 0xBC, 0x41, 0x44, 0xD7, 0x9E, 0xD6, + 0xBC, 0x41, 0x44, 0xD7, 0xA0, 0xD6, 0xBC, 0x41, // Bytes 43c0 - 43ff - 0x44, 0xCE, 0xB7, 0xCC, 0x81, 0xC9, 0x44, 0xCE, - 0xB9, 0xCC, 0x81, 0xC9, 0x44, 0xCE, 0xBF, 0xCC, - 0x81, 0xC9, 0x44, 0xCF, 0x85, 0xCC, 0x81, 0xC9, - 0x44, 0xCF, 0x89, 0xCC, 0x81, 0xC9, 0x44, 0xD7, - 0x90, 0xD6, 0xB7, 0x31, 0x44, 0xD7, 0x90, 0xD6, - 0xB8, 0x35, 0x44, 0xD7, 0x90, 0xD6, 0xBC, 0x41, - 0x44, 0xD7, 0x91, 0xD6, 0xBC, 0x41, 0x44, 0xD7, - 0x91, 0xD6, 0xBF, 0x49, 0x44, 0xD7, 0x92, 0xD6, + 0x44, 0xD7, 0xA1, 0xD6, 0xBC, 0x41, 0x44, 0xD7, + 0xA3, 0xD6, 0xBC, 0x41, 0x44, 0xD7, 0xA4, 0xD6, + 0xBC, 0x41, 0x44, 0xD7, 0xA4, 0xD6, 0xBF, 0x49, + 0x44, 0xD7, 0xA6, 0xD6, 0xBC, 0x41, 0x44, 0xD7, + 0xA7, 0xD6, 0xBC, 0x41, 0x44, 0xD7, 0xA8, 0xD6, + 0xBC, 0x41, 0x44, 0xD7, 0xA9, 0xD6, 0xBC, 0x41, + 0x44, 0xD7, 0xA9, 0xD7, 0x81, 0x4D, 0x44, 0xD7, + 0xA9, 0xD7, 0x82, 0x51, 0x44, 0xD7, 0xAA, 0xD6, // Bytes 4400 - 443f - 0xBC, 0x41, 0x44, 0xD7, 0x93, 0xD6, 0xBC, 0x41, - 0x44, 0xD7, 0x94, 0xD6, 0xBC, 0x41, 0x44, 0xD7, - 0x95, 0xD6, 0xB9, 0x39, 0x44, 0xD7, 0x95, 0xD6, - 0xBC, 0x41, 0x44, 0xD7, 0x96, 0xD6, 0xBC, 0x41, - 0x44, 0xD7, 0x98, 0xD6, 0xBC, 0x41, 0x44, 0xD7, - 0x99, 0xD6, 0xB4, 0x25, 0x44, 0xD7, 0x99, 0xD6, - 0xBC, 0x41, 0x44, 0xD7, 0x9A, 0xD6, 0xBC, 0x41, - 0x44, 0xD7, 0x9B, 0xD6, 0xBC, 0x41, 0x44, 0xD7, + 0xBC, 0x41, 0x44, 0xD7, 0xB2, 0xD6, 0xB7, 0x31, + 0x44, 0xD8, 0xA7, 0xD9, 0x8B, 0x59, 0x44, 0xD8, + 0xA7, 0xD9, 0x93, 0xC9, 0x44, 0xD8, 0xA7, 0xD9, + 0x94, 0xC9, 0x44, 0xD8, 0xA7, 0xD9, 0x95, 0xB5, + 0x44, 0xD8, 0xB0, 0xD9, 0xB0, 0x79, 0x44, 0xD8, + 0xB1, 0xD9, 0xB0, 0x79, 0x44, 0xD9, 0x80, 0xD9, + 0x8B, 0x59, 0x44, 0xD9, 0x80, 0xD9, 0x8E, 0x65, + 0x44, 0xD9, 0x80, 0xD9, 0x8F, 0x69, 0x44, 0xD9, // Bytes 4440 - 447f - 0x9B, 0xD6, 0xBF, 0x49, 0x44, 0xD7, 0x9C, 0xD6, - 0xBC, 0x41, 0x44, 0xD7, 0x9E, 0xD6, 0xBC, 0x41, - 0x44, 0xD7, 0xA0, 0xD6, 0xBC, 0x41, 0x44, 0xD7, - 0xA1, 0xD6, 0xBC, 0x41, 0x44, 0xD7, 0xA3, 0xD6, - 0xBC, 0x41, 0x44, 0xD7, 0xA4, 0xD6, 0xBC, 0x41, - 0x44, 0xD7, 0xA4, 0xD6, 0xBF, 0x49, 0x44, 0xD7, - 0xA6, 0xD6, 0xBC, 0x41, 0x44, 0xD7, 0xA7, 0xD6, - 0xBC, 0x41, 0x44, 0xD7, 0xA8, 0xD6, 0xBC, 0x41, + 0x80, 0xD9, 0x90, 0x6D, 0x44, 0xD9, 0x80, 0xD9, + 0x91, 0x71, 0x44, 0xD9, 0x80, 0xD9, 0x92, 0x75, + 0x44, 0xD9, 0x87, 0xD9, 0xB0, 0x79, 0x44, 0xD9, + 0x88, 0xD9, 0x94, 0xC9, 0x44, 0xD9, 0x89, 0xD9, + 0xB0, 0x79, 0x44, 0xD9, 0x8A, 0xD9, 0x94, 0xC9, + 0x44, 0xDB, 0x92, 0xD9, 0x94, 0xC9, 0x44, 0xDB, + 0x95, 0xD9, 0x94, 0xC9, 0x45, 0x20, 0xCC, 0x88, + 0xCC, 0x80, 0xCA, 0x45, 0x20, 0xCC, 0x88, 0xCC, // Bytes 4480 - 44bf - 0x44, 0xD7, 0xA9, 0xD6, 0xBC, 0x41, 0x44, 0xD7, - 0xA9, 0xD7, 0x81, 0x4D, 0x44, 0xD7, 0xA9, 0xD7, - 0x82, 0x51, 0x44, 0xD7, 0xAA, 0xD6, 0xBC, 0x41, - 0x44, 0xD7, 0xB2, 0xD6, 0xB7, 0x31, 0x44, 0xD8, - 0xA7, 0xD9, 0x8B, 0x59, 0x44, 0xD8, 0xA7, 0xD9, - 0x93, 0xC9, 0x44, 0xD8, 0xA7, 0xD9, 0x94, 0xC9, - 0x44, 0xD8, 0xA7, 0xD9, 0x95, 0xB5, 0x44, 0xD8, - 0xB0, 0xD9, 0xB0, 0x79, 0x44, 0xD8, 0xB1, 0xD9, + 0x81, 0xCA, 0x45, 0x20, 0xCC, 0x88, 0xCD, 0x82, + 0xCA, 0x45, 0x20, 0xCC, 0x93, 0xCC, 0x80, 0xCA, + 0x45, 0x20, 0xCC, 0x93, 0xCC, 0x81, 0xCA, 0x45, + 0x20, 0xCC, 0x93, 0xCD, 0x82, 0xCA, 0x45, 0x20, + 0xCC, 0x94, 0xCC, 0x80, 0xCA, 0x45, 0x20, 0xCC, + 0x94, 0xCC, 0x81, 0xCA, 0x45, 0x20, 0xCC, 0x94, + 0xCD, 0x82, 0xCA, 0x45, 0x20, 0xD9, 0x8C, 0xD9, + 0x91, 0x72, 0x45, 0x20, 0xD9, 0x8D, 0xD9, 0x91, // Bytes 44c0 - 44ff - 0xB0, 0x79, 0x44, 0xD9, 0x80, 0xD9, 0x8B, 0x59, - 0x44, 0xD9, 0x80, 0xD9, 0x8E, 0x65, 0x44, 0xD9, - 0x80, 0xD9, 0x8F, 0x69, 0x44, 0xD9, 0x80, 0xD9, - 0x90, 0x6D, 0x44, 0xD9, 0x80, 0xD9, 0x91, 0x71, - 0x44, 0xD9, 0x80, 0xD9, 0x92, 0x75, 0x44, 0xD9, - 0x87, 0xD9, 0xB0, 0x79, 0x44, 0xD9, 0x88, 0xD9, - 0x94, 0xC9, 0x44, 0xD9, 0x89, 0xD9, 0xB0, 0x79, - 0x44, 0xD9, 0x8A, 0xD9, 0x94, 0xC9, 0x44, 0xDB, + 0x72, 0x45, 0x20, 0xD9, 0x8E, 0xD9, 0x91, 0x72, + 0x45, 0x20, 0xD9, 0x8F, 0xD9, 0x91, 0x72, 0x45, + 0x20, 0xD9, 0x90, 0xD9, 0x91, 0x72, 0x45, 0x20, + 0xD9, 0x91, 0xD9, 0xB0, 0x7A, 0x45, 0xE2, 0xAB, + 0x9D, 0xCC, 0xB8, 0x05, 0x46, 0xCE, 0xB9, 0xCC, + 0x88, 0xCC, 0x81, 0xCA, 0x46, 0xCF, 0x85, 0xCC, + 0x88, 0xCC, 0x81, 0xCA, 0x46, 0xD7, 0xA9, 0xD6, + 0xBC, 0xD7, 0x81, 0x4E, 0x46, 0xD7, 0xA9, 0xD6, // Bytes 4500 - 453f - 0x92, 0xD9, 0x94, 0xC9, 0x44, 0xDB, 0x95, 0xD9, - 0x94, 0xC9, 0x45, 0x20, 0xCC, 0x88, 0xCC, 0x80, - 0xCA, 0x45, 0x20, 0xCC, 0x88, 0xCC, 0x81, 0xCA, - 0x45, 0x20, 0xCC, 0x88, 0xCD, 0x82, 0xCA, 0x45, - 0x20, 0xCC, 0x93, 0xCC, 0x80, 0xCA, 0x45, 0x20, - 0xCC, 0x93, 0xCC, 0x81, 0xCA, 0x45, 0x20, 0xCC, - 0x93, 0xCD, 0x82, 0xCA, 0x45, 0x20, 0xCC, 0x94, - 0xCC, 0x80, 0xCA, 0x45, 0x20, 0xCC, 0x94, 0xCC, + 0xBC, 0xD7, 0x82, 0x52, 0x46, 0xD9, 0x80, 0xD9, + 0x8E, 0xD9, 0x91, 0x72, 0x46, 0xD9, 0x80, 0xD9, + 0x8F, 0xD9, 0x91, 0x72, 0x46, 0xD9, 0x80, 0xD9, + 0x90, 0xD9, 0x91, 0x72, 0x46, 0xE0, 0xA4, 0x95, + 0xE0, 0xA4, 0xBC, 0x09, 0x46, 0xE0, 0xA4, 0x96, + 0xE0, 0xA4, 0xBC, 0x09, 0x46, 0xE0, 0xA4, 0x97, + 0xE0, 0xA4, 0xBC, 0x09, 0x46, 0xE0, 0xA4, 0x9C, + 0xE0, 0xA4, 0xBC, 0x09, 0x46, 0xE0, 0xA4, 0xA1, // Bytes 4540 - 457f - 0x81, 0xCA, 0x45, 0x20, 0xCC, 0x94, 0xCD, 0x82, - 0xCA, 0x45, 0x20, 0xD9, 0x8C, 0xD9, 0x91, 0x72, - 0x45, 0x20, 0xD9, 0x8D, 0xD9, 0x91, 0x72, 0x45, - 0x20, 0xD9, 0x8E, 0xD9, 0x91, 0x72, 0x45, 0x20, - 0xD9, 0x8F, 0xD9, 0x91, 0x72, 0x45, 0x20, 0xD9, - 0x90, 0xD9, 0x91, 0x72, 0x45, 0x20, 0xD9, 0x91, - 0xD9, 0xB0, 0x7A, 0x45, 0xE2, 0xAB, 0x9D, 0xCC, - 0xB8, 0x05, 0x46, 0xCE, 0xB9, 0xCC, 0x88, 0xCC, + 0xE0, 0xA4, 0xBC, 0x09, 0x46, 0xE0, 0xA4, 0xA2, + 0xE0, 0xA4, 0xBC, 0x09, 0x46, 0xE0, 0xA4, 0xAB, + 0xE0, 0xA4, 0xBC, 0x09, 0x46, 0xE0, 0xA4, 0xAF, + 0xE0, 0xA4, 0xBC, 0x09, 0x46, 0xE0, 0xA6, 0xA1, + 0xE0, 0xA6, 0xBC, 0x09, 0x46, 0xE0, 0xA6, 0xA2, + 0xE0, 0xA6, 0xBC, 0x09, 0x46, 0xE0, 0xA6, 0xAF, + 0xE0, 0xA6, 0xBC, 0x09, 0x46, 0xE0, 0xA8, 0x96, + 0xE0, 0xA8, 0xBC, 0x09, 0x46, 0xE0, 0xA8, 0x97, // Bytes 4580 - 45bf - 0x81, 0xCA, 0x46, 0xCF, 0x85, 0xCC, 0x88, 0xCC, - 0x81, 0xCA, 0x46, 0xD7, 0xA9, 0xD6, 0xBC, 0xD7, - 0x81, 0x4E, 0x46, 0xD7, 0xA9, 0xD6, 0xBC, 0xD7, - 0x82, 0x52, 0x46, 0xD9, 0x80, 0xD9, 0x8E, 0xD9, - 0x91, 0x72, 0x46, 0xD9, 0x80, 0xD9, 0x8F, 0xD9, - 0x91, 0x72, 0x46, 0xD9, 0x80, 0xD9, 0x90, 0xD9, - 0x91, 0x72, 0x46, 0xE0, 0xA4, 0x95, 0xE0, 0xA4, - 0xBC, 0x09, 0x46, 0xE0, 0xA4, 0x96, 0xE0, 0xA4, + 0xE0, 0xA8, 0xBC, 0x09, 0x46, 0xE0, 0xA8, 0x9C, + 0xE0, 0xA8, 0xBC, 0x09, 0x46, 0xE0, 0xA8, 0xAB, + 0xE0, 0xA8, 0xBC, 0x09, 0x46, 0xE0, 0xA8, 0xB2, + 0xE0, 0xA8, 0xBC, 0x09, 0x46, 0xE0, 0xA8, 0xB8, + 0xE0, 0xA8, 0xBC, 0x09, 0x46, 0xE0, 0xAC, 0xA1, + 0xE0, 0xAC, 0xBC, 0x09, 0x46, 0xE0, 0xAC, 0xA2, + 0xE0, 0xAC, 0xBC, 0x09, 0x46, 0xE0, 0xBE, 0xB2, + 0xE0, 0xBE, 0x80, 0x9D, 0x46, 0xE0, 0xBE, 0xB3, // Bytes 45c0 - 45ff - 0xBC, 0x09, 0x46, 0xE0, 0xA4, 0x97, 0xE0, 0xA4, - 0xBC, 0x09, 0x46, 0xE0, 0xA4, 0x9C, 0xE0, 0xA4, - 0xBC, 0x09, 0x46, 0xE0, 0xA4, 0xA1, 0xE0, 0xA4, - 0xBC, 0x09, 0x46, 0xE0, 0xA4, 0xA2, 0xE0, 0xA4, - 0xBC, 0x09, 0x46, 0xE0, 0xA4, 0xAB, 0xE0, 0xA4, - 0xBC, 0x09, 0x46, 0xE0, 0xA4, 0xAF, 0xE0, 0xA4, - 0xBC, 0x09, 0x46, 0xE0, 0xA6, 0xA1, 0xE0, 0xA6, - 0xBC, 0x09, 0x46, 0xE0, 0xA6, 0xA2, 0xE0, 0xA6, + 0xE0, 0xBE, 0x80, 0x9D, 0x46, 0xE3, 0x83, 0x86, + 0xE3, 0x82, 0x99, 0x0D, 0x48, 0xF0, 0x9D, 0x85, + 0x97, 0xF0, 0x9D, 0x85, 0xA5, 0xAD, 0x48, 0xF0, + 0x9D, 0x85, 0x98, 0xF0, 0x9D, 0x85, 0xA5, 0xAD, + 0x48, 0xF0, 0x9D, 0x86, 0xB9, 0xF0, 0x9D, 0x85, + 0xA5, 0xAD, 0x48, 0xF0, 0x9D, 0x86, 0xBA, 0xF0, + 0x9D, 0x85, 0xA5, 0xAD, 0x49, 0xE0, 0xBE, 0xB2, + 0xE0, 0xBD, 0xB1, 0xE0, 0xBE, 0x80, 0x9E, 0x49, // Bytes 4600 - 463f - 0xBC, 0x09, 0x46, 0xE0, 0xA6, 0xAF, 0xE0, 0xA6, - 0xBC, 0x09, 0x46, 0xE0, 0xA8, 0x96, 0xE0, 0xA8, - 0xBC, 0x09, 0x46, 0xE0, 0xA8, 0x97, 0xE0, 0xA8, - 0xBC, 0x09, 0x46, 0xE0, 0xA8, 0x9C, 0xE0, 0xA8, - 0xBC, 0x09, 0x46, 0xE0, 0xA8, 0xAB, 0xE0, 0xA8, - 0xBC, 0x09, 0x46, 0xE0, 0xA8, 0xB2, 0xE0, 0xA8, - 0xBC, 0x09, 0x46, 0xE0, 0xA8, 0xB8, 0xE0, 0xA8, - 0xBC, 0x09, 0x46, 0xE0, 0xAC, 0xA1, 0xE0, 0xAC, - // Bytes 4640 - 467f - 0xBC, 0x09, 0x46, 0xE0, 0xAC, 0xA2, 0xE0, 0xAC, - 0xBC, 0x09, 0x46, 0xE0, 0xBE, 0xB2, 0xE0, 0xBE, - 0x80, 0x9D, 0x46, 0xE0, 0xBE, 0xB3, 0xE0, 0xBE, - 0x80, 0x9D, 0x46, 0xE3, 0x83, 0x86, 0xE3, 0x82, - 0x99, 0x0D, 0x48, 0xF0, 0x9D, 0x85, 0x97, 0xF0, - 0x9D, 0x85, 0xA5, 0xAD, 0x48, 0xF0, 0x9D, 0x85, - 0x98, 0xF0, 0x9D, 0x85, 0xA5, 0xAD, 0x48, 0xF0, - 0x9D, 0x86, 0xB9, 0xF0, 0x9D, 0x85, 0xA5, 0xAD, - // Bytes 4680 - 46bf - 0x48, 0xF0, 0x9D, 0x86, 0xBA, 0xF0, 0x9D, 0x85, - 0xA5, 0xAD, 0x49, 0xE0, 0xBE, 0xB2, 0xE0, 0xBD, - 0xB1, 0xE0, 0xBE, 0x80, 0x9E, 0x49, 0xE0, 0xBE, - 0xB3, 0xE0, 0xBD, 0xB1, 0xE0, 0xBE, 0x80, 0x9E, + 0xE0, 0xBE, 0xB3, 0xE0, 0xBD, 0xB1, 0xE0, 0xBE, + 0x80, 0x9E, 0x4C, 0xF0, 0x9D, 0x85, 0x98, 0xF0, + 0x9D, 0x85, 0xA5, 0xF0, 0x9D, 0x85, 0xAE, 0xAE, 0x4C, 0xF0, 0x9D, 0x85, 0x98, 0xF0, 0x9D, 0x85, - 0xA5, 0xF0, 0x9D, 0x85, 0xAE, 0xAE, 0x4C, 0xF0, + 0xA5, 0xF0, 0x9D, 0x85, 0xAF, 0xAE, 0x4C, 0xF0, 0x9D, 0x85, 0x98, 0xF0, 0x9D, 0x85, 0xA5, 0xF0, - 0x9D, 0x85, 0xAF, 0xAE, 0x4C, 0xF0, 0x9D, 0x85, - // Bytes 46c0 - 46ff + 0x9D, 0x85, 0xB0, 0xAE, 0x4C, 0xF0, 0x9D, 0x85, 0x98, 0xF0, 0x9D, 0x85, 0xA5, 0xF0, 0x9D, 0x85, - 0xB0, 0xAE, 0x4C, 0xF0, 0x9D, 0x85, 0x98, 0xF0, - 0x9D, 0x85, 0xA5, 0xF0, 0x9D, 0x85, 0xB1, 0xAE, - 0x4C, 0xF0, 0x9D, 0x85, 0x98, 0xF0, 0x9D, 0x85, - 0xA5, 0xF0, 0x9D, 0x85, 0xB2, 0xAE, 0x4C, 0xF0, + // Bytes 4640 - 467f + 0xB1, 0xAE, 0x4C, 0xF0, 0x9D, 0x85, 0x98, 0xF0, + 0x9D, 0x85, 0xA5, 0xF0, 0x9D, 0x85, 0xB2, 0xAE, + 0x4C, 0xF0, 0x9D, 0x86, 0xB9, 0xF0, 0x9D, 0x85, + 0xA5, 0xF0, 0x9D, 0x85, 0xAE, 0xAE, 0x4C, 0xF0, 0x9D, 0x86, 0xB9, 0xF0, 0x9D, 0x85, 0xA5, 0xF0, - 0x9D, 0x85, 0xAE, 0xAE, 0x4C, 0xF0, 0x9D, 0x86, - 0xB9, 0xF0, 0x9D, 0x85, 0xA5, 0xF0, 0x9D, 0x85, + 0x9D, 0x85, 0xAF, 0xAE, 0x4C, 0xF0, 0x9D, 0x86, + 0xBA, 0xF0, 0x9D, 0x85, 0xA5, 0xF0, 0x9D, 0x85, + 0xAE, 0xAE, 0x4C, 0xF0, 0x9D, 0x86, 0xBA, 0xF0, + // Bytes 4680 - 46bf + 0x9D, 0x85, 0xA5, 0xF0, 0x9D, 0x85, 0xAF, 0xAE, + 0x83, 0x41, 0xCC, 0x82, 0xC9, 0x83, 0x41, 0xCC, + 0x86, 0xC9, 0x83, 0x41, 0xCC, 0x87, 0xC9, 0x83, + 0x41, 0xCC, 0x88, 0xC9, 0x83, 0x41, 0xCC, 0x8A, + 0xC9, 0x83, 0x41, 0xCC, 0xA3, 0xB5, 0x83, 0x43, + 0xCC, 0xA7, 0xA5, 0x83, 0x45, 0xCC, 0x82, 0xC9, + 0x83, 0x45, 0xCC, 0x84, 0xC9, 0x83, 0x45, 0xCC, + 0xA3, 0xB5, 0x83, 0x45, 0xCC, 0xA7, 0xA5, 0x83, + // Bytes 46c0 - 46ff + 0x49, 0xCC, 0x88, 0xC9, 0x83, 0x4C, 0xCC, 0xA3, + 0xB5, 0x83, 0x4F, 0xCC, 0x82, 0xC9, 0x83, 0x4F, + 0xCC, 0x83, 0xC9, 0x83, 0x4F, 0xCC, 0x84, 0xC9, + 0x83, 0x4F, 0xCC, 0x87, 0xC9, 0x83, 0x4F, 0xCC, + 0x88, 0xC9, 0x83, 0x4F, 0xCC, 0x9B, 0xAD, 0x83, + 0x4F, 0xCC, 0xA3, 0xB5, 0x83, 0x4F, 0xCC, 0xA8, + 0xA5, 0x83, 0x52, 0xCC, 0xA3, 0xB5, 0x83, 0x53, + 0xCC, 0x81, 0xC9, 0x83, 0x53, 0xCC, 0x8C, 0xC9, // Bytes 4700 - 473f - 0xAF, 0xAE, 0x4C, 0xF0, 0x9D, 0x86, 0xBA, 0xF0, - 0x9D, 0x85, 0xA5, 0xF0, 0x9D, 0x85, 0xAE, 0xAE, - 0x4C, 0xF0, 0x9D, 0x86, 0xBA, 0xF0, 0x9D, 0x85, - 0xA5, 0xF0, 0x9D, 0x85, 0xAF, 0xAE, 0x83, 0x41, - 0xCC, 0x82, 0xC9, 0x83, 0x41, 0xCC, 0x86, 0xC9, - 0x83, 0x41, 0xCC, 0x87, 0xC9, 0x83, 0x41, 0xCC, - 0x88, 0xC9, 0x83, 0x41, 0xCC, 0x8A, 0xC9, 0x83, - 0x41, 0xCC, 0xA3, 0xB5, 0x83, 0x43, 0xCC, 0xA7, + 0x83, 0x53, 0xCC, 0xA3, 0xB5, 0x83, 0x55, 0xCC, + 0x83, 0xC9, 0x83, 0x55, 0xCC, 0x84, 0xC9, 0x83, + 0x55, 0xCC, 0x88, 0xC9, 0x83, 0x55, 0xCC, 0x9B, + 0xAD, 0x83, 0x61, 0xCC, 0x82, 0xC9, 0x83, 0x61, + 0xCC, 0x86, 0xC9, 0x83, 0x61, 0xCC, 0x87, 0xC9, + 0x83, 0x61, 0xCC, 0x88, 0xC9, 0x83, 0x61, 0xCC, + 0x8A, 0xC9, 0x83, 0x61, 0xCC, 0xA3, 0xB5, 0x83, + 0x63, 0xCC, 0xA7, 0xA5, 0x83, 0x65, 0xCC, 0x82, // Bytes 4740 - 477f - 0xA5, 0x83, 0x45, 0xCC, 0x82, 0xC9, 0x83, 0x45, - 0xCC, 0x84, 0xC9, 0x83, 0x45, 0xCC, 0xA3, 0xB5, - 0x83, 0x45, 0xCC, 0xA7, 0xA5, 0x83, 0x49, 0xCC, - 0x88, 0xC9, 0x83, 0x4C, 0xCC, 0xA3, 0xB5, 0x83, - 0x4F, 0xCC, 0x82, 0xC9, 0x83, 0x4F, 0xCC, 0x83, - 0xC9, 0x83, 0x4F, 0xCC, 0x84, 0xC9, 0x83, 0x4F, - 0xCC, 0x87, 0xC9, 0x83, 0x4F, 0xCC, 0x88, 0xC9, - 0x83, 0x4F, 0xCC, 0x9B, 0xAD, 0x83, 0x4F, 0xCC, + 0xC9, 0x83, 0x65, 0xCC, 0x84, 0xC9, 0x83, 0x65, + 0xCC, 0xA3, 0xB5, 0x83, 0x65, 0xCC, 0xA7, 0xA5, + 0x83, 0x69, 0xCC, 0x88, 0xC9, 0x83, 0x6C, 0xCC, + 0xA3, 0xB5, 0x83, 0x6F, 0xCC, 0x82, 0xC9, 0x83, + 0x6F, 0xCC, 0x83, 0xC9, 0x83, 0x6F, 0xCC, 0x84, + 0xC9, 0x83, 0x6F, 0xCC, 0x87, 0xC9, 0x83, 0x6F, + 0xCC, 0x88, 0xC9, 0x83, 0x6F, 0xCC, 0x9B, 0xAD, + 0x83, 0x6F, 0xCC, 0xA3, 0xB5, 0x83, 0x6F, 0xCC, // Bytes 4780 - 47bf - 0xA3, 0xB5, 0x83, 0x4F, 0xCC, 0xA8, 0xA5, 0x83, - 0x52, 0xCC, 0xA3, 0xB5, 0x83, 0x53, 0xCC, 0x81, - 0xC9, 0x83, 0x53, 0xCC, 0x8C, 0xC9, 0x83, 0x53, - 0xCC, 0xA3, 0xB5, 0x83, 0x55, 0xCC, 0x83, 0xC9, - 0x83, 0x55, 0xCC, 0x84, 0xC9, 0x83, 0x55, 0xCC, - 0x88, 0xC9, 0x83, 0x55, 0xCC, 0x9B, 0xAD, 0x83, - 0x61, 0xCC, 0x82, 0xC9, 0x83, 0x61, 0xCC, 0x86, - 0xC9, 0x83, 0x61, 0xCC, 0x87, 0xC9, 0x83, 0x61, + 0xA8, 0xA5, 0x83, 0x72, 0xCC, 0xA3, 0xB5, 0x83, + 0x73, 0xCC, 0x81, 0xC9, 0x83, 0x73, 0xCC, 0x8C, + 0xC9, 0x83, 0x73, 0xCC, 0xA3, 0xB5, 0x83, 0x75, + 0xCC, 0x83, 0xC9, 0x83, 0x75, 0xCC, 0x84, 0xC9, + 0x83, 0x75, 0xCC, 0x88, 0xC9, 0x83, 0x75, 0xCC, + 0x9B, 0xAD, 0x84, 0xCE, 0x91, 0xCC, 0x93, 0xC9, + 0x84, 0xCE, 0x91, 0xCC, 0x94, 0xC9, 0x84, 0xCE, + 0x95, 0xCC, 0x93, 0xC9, 0x84, 0xCE, 0x95, 0xCC, // Bytes 47c0 - 47ff - 0xCC, 0x88, 0xC9, 0x83, 0x61, 0xCC, 0x8A, 0xC9, - 0x83, 0x61, 0xCC, 0xA3, 0xB5, 0x83, 0x63, 0xCC, - 0xA7, 0xA5, 0x83, 0x65, 0xCC, 0x82, 0xC9, 0x83, - 0x65, 0xCC, 0x84, 0xC9, 0x83, 0x65, 0xCC, 0xA3, - 0xB5, 0x83, 0x65, 0xCC, 0xA7, 0xA5, 0x83, 0x69, - 0xCC, 0x88, 0xC9, 0x83, 0x6C, 0xCC, 0xA3, 0xB5, - 0x83, 0x6F, 0xCC, 0x82, 0xC9, 0x83, 0x6F, 0xCC, - 0x83, 0xC9, 0x83, 0x6F, 0xCC, 0x84, 0xC9, 0x83, + 0x94, 0xC9, 0x84, 0xCE, 0x97, 0xCC, 0x93, 0xC9, + 0x84, 0xCE, 0x97, 0xCC, 0x94, 0xC9, 0x84, 0xCE, + 0x99, 0xCC, 0x93, 0xC9, 0x84, 0xCE, 0x99, 0xCC, + 0x94, 0xC9, 0x84, 0xCE, 0x9F, 0xCC, 0x93, 0xC9, + 0x84, 0xCE, 0x9F, 0xCC, 0x94, 0xC9, 0x84, 0xCE, + 0xA5, 0xCC, 0x94, 0xC9, 0x84, 0xCE, 0xA9, 0xCC, + 0x93, 0xC9, 0x84, 0xCE, 0xA9, 0xCC, 0x94, 0xC9, + 0x84, 0xCE, 0xB1, 0xCC, 0x80, 0xC9, 0x84, 0xCE, // Bytes 4800 - 483f - 0x6F, 0xCC, 0x87, 0xC9, 0x83, 0x6F, 0xCC, 0x88, - 0xC9, 0x83, 0x6F, 0xCC, 0x9B, 0xAD, 0x83, 0x6F, - 0xCC, 0xA3, 0xB5, 0x83, 0x6F, 0xCC, 0xA8, 0xA5, - 0x83, 0x72, 0xCC, 0xA3, 0xB5, 0x83, 0x73, 0xCC, - 0x81, 0xC9, 0x83, 0x73, 0xCC, 0x8C, 0xC9, 0x83, - 0x73, 0xCC, 0xA3, 0xB5, 0x83, 0x75, 0xCC, 0x83, - 0xC9, 0x83, 0x75, 0xCC, 0x84, 0xC9, 0x83, 0x75, - 0xCC, 0x88, 0xC9, 0x83, 0x75, 0xCC, 0x9B, 0xAD, + 0xB1, 0xCC, 0x81, 0xC9, 0x84, 0xCE, 0xB1, 0xCC, + 0x93, 0xC9, 0x84, 0xCE, 0xB1, 0xCC, 0x94, 0xC9, + 0x84, 0xCE, 0xB1, 0xCD, 0x82, 0xC9, 0x84, 0xCE, + 0xB5, 0xCC, 0x93, 0xC9, 0x84, 0xCE, 0xB5, 0xCC, + 0x94, 0xC9, 0x84, 0xCE, 0xB7, 0xCC, 0x80, 0xC9, + 0x84, 0xCE, 0xB7, 0xCC, 0x81, 0xC9, 0x84, 0xCE, + 0xB7, 0xCC, 0x93, 0xC9, 0x84, 0xCE, 0xB7, 0xCC, + 0x94, 0xC9, 0x84, 0xCE, 0xB7, 0xCD, 0x82, 0xC9, // Bytes 4840 - 487f - 0x84, 0xCE, 0x91, 0xCC, 0x93, 0xC9, 0x84, 0xCE, - 0x91, 0xCC, 0x94, 0xC9, 0x84, 0xCE, 0x95, 0xCC, - 0x93, 0xC9, 0x84, 0xCE, 0x95, 0xCC, 0x94, 0xC9, - 0x84, 0xCE, 0x97, 0xCC, 0x93, 0xC9, 0x84, 0xCE, - 0x97, 0xCC, 0x94, 0xC9, 0x84, 0xCE, 0x99, 0xCC, - 0x93, 0xC9, 0x84, 0xCE, 0x99, 0xCC, 0x94, 0xC9, - 0x84, 0xCE, 0x9F, 0xCC, 0x93, 0xC9, 0x84, 0xCE, - 0x9F, 0xCC, 0x94, 0xC9, 0x84, 0xCE, 0xA5, 0xCC, + 0x84, 0xCE, 0xB9, 0xCC, 0x88, 0xC9, 0x84, 0xCE, + 0xB9, 0xCC, 0x93, 0xC9, 0x84, 0xCE, 0xB9, 0xCC, + 0x94, 0xC9, 0x84, 0xCE, 0xBF, 0xCC, 0x93, 0xC9, + 0x84, 0xCE, 0xBF, 0xCC, 0x94, 0xC9, 0x84, 0xCF, + 0x85, 0xCC, 0x88, 0xC9, 0x84, 0xCF, 0x85, 0xCC, + 0x93, 0xC9, 0x84, 0xCF, 0x85, 0xCC, 0x94, 0xC9, + 0x84, 0xCF, 0x89, 0xCC, 0x80, 0xC9, 0x84, 0xCF, + 0x89, 0xCC, 0x81, 0xC9, 0x84, 0xCF, 0x89, 0xCC, // Bytes 4880 - 48bf - 0x94, 0xC9, 0x84, 0xCE, 0xA9, 0xCC, 0x93, 0xC9, - 0x84, 0xCE, 0xA9, 0xCC, 0x94, 0xC9, 0x84, 0xCE, - 0xB1, 0xCC, 0x80, 0xC9, 0x84, 0xCE, 0xB1, 0xCC, - 0x81, 0xC9, 0x84, 0xCE, 0xB1, 0xCC, 0x93, 0xC9, - 0x84, 0xCE, 0xB1, 0xCC, 0x94, 0xC9, 0x84, 0xCE, - 0xB1, 0xCD, 0x82, 0xC9, 0x84, 0xCE, 0xB5, 0xCC, - 0x93, 0xC9, 0x84, 0xCE, 0xB5, 0xCC, 0x94, 0xC9, - 0x84, 0xCE, 0xB7, 0xCC, 0x80, 0xC9, 0x84, 0xCE, + 0x93, 0xC9, 0x84, 0xCF, 0x89, 0xCC, 0x94, 0xC9, + 0x84, 0xCF, 0x89, 0xCD, 0x82, 0xC9, 0x86, 0xCE, + 0x91, 0xCC, 0x93, 0xCC, 0x80, 0xCA, 0x86, 0xCE, + 0x91, 0xCC, 0x93, 0xCC, 0x81, 0xCA, 0x86, 0xCE, + 0x91, 0xCC, 0x93, 0xCD, 0x82, 0xCA, 0x86, 0xCE, + 0x91, 0xCC, 0x94, 0xCC, 0x80, 0xCA, 0x86, 0xCE, + 0x91, 0xCC, 0x94, 0xCC, 0x81, 0xCA, 0x86, 0xCE, + 0x91, 0xCC, 0x94, 0xCD, 0x82, 0xCA, 0x86, 0xCE, // Bytes 48c0 - 48ff - 0xB7, 0xCC, 0x81, 0xC9, 0x84, 0xCE, 0xB7, 0xCC, - 0x93, 0xC9, 0x84, 0xCE, 0xB7, 0xCC, 0x94, 0xC9, - 0x84, 0xCE, 0xB7, 0xCD, 0x82, 0xC9, 0x84, 0xCE, - 0xB9, 0xCC, 0x88, 0xC9, 0x84, 0xCE, 0xB9, 0xCC, - 0x93, 0xC9, 0x84, 0xCE, 0xB9, 0xCC, 0x94, 0xC9, - 0x84, 0xCE, 0xBF, 0xCC, 0x93, 0xC9, 0x84, 0xCE, - 0xBF, 0xCC, 0x94, 0xC9, 0x84, 0xCF, 0x85, 0xCC, - 0x88, 0xC9, 0x84, 0xCF, 0x85, 0xCC, 0x93, 0xC9, + 0x97, 0xCC, 0x93, 0xCC, 0x80, 0xCA, 0x86, 0xCE, + 0x97, 0xCC, 0x93, 0xCC, 0x81, 0xCA, 0x86, 0xCE, + 0x97, 0xCC, 0x93, 0xCD, 0x82, 0xCA, 0x86, 0xCE, + 0x97, 0xCC, 0x94, 0xCC, 0x80, 0xCA, 0x86, 0xCE, + 0x97, 0xCC, 0x94, 0xCC, 0x81, 0xCA, 0x86, 0xCE, + 0x97, 0xCC, 0x94, 0xCD, 0x82, 0xCA, 0x86, 0xCE, + 0xA9, 0xCC, 0x93, 0xCC, 0x80, 0xCA, 0x86, 0xCE, + 0xA9, 0xCC, 0x93, 0xCC, 0x81, 0xCA, 0x86, 0xCE, // Bytes 4900 - 493f - 0x84, 0xCF, 0x85, 0xCC, 0x94, 0xC9, 0x84, 0xCF, - 0x89, 0xCC, 0x80, 0xC9, 0x84, 0xCF, 0x89, 0xCC, - 0x81, 0xC9, 0x84, 0xCF, 0x89, 0xCC, 0x93, 0xC9, - 0x84, 0xCF, 0x89, 0xCC, 0x94, 0xC9, 0x84, 0xCF, - 0x89, 0xCD, 0x82, 0xC9, 0x86, 0xCE, 0x91, 0xCC, - 0x93, 0xCC, 0x80, 0xCA, 0x86, 0xCE, 0x91, 0xCC, - 0x93, 0xCC, 0x81, 0xCA, 0x86, 0xCE, 0x91, 0xCC, - 0x93, 0xCD, 0x82, 0xCA, 0x86, 0xCE, 0x91, 0xCC, + 0xA9, 0xCC, 0x93, 0xCD, 0x82, 0xCA, 0x86, 0xCE, + 0xA9, 0xCC, 0x94, 0xCC, 0x80, 0xCA, 0x86, 0xCE, + 0xA9, 0xCC, 0x94, 0xCC, 0x81, 0xCA, 0x86, 0xCE, + 0xA9, 0xCC, 0x94, 0xCD, 0x82, 0xCA, 0x86, 0xCE, + 0xB1, 0xCC, 0x93, 0xCC, 0x80, 0xCA, 0x86, 0xCE, + 0xB1, 0xCC, 0x93, 0xCC, 0x81, 0xCA, 0x86, 0xCE, + 0xB1, 0xCC, 0x93, 0xCD, 0x82, 0xCA, 0x86, 0xCE, + 0xB1, 0xCC, 0x94, 0xCC, 0x80, 0xCA, 0x86, 0xCE, // Bytes 4940 - 497f - 0x94, 0xCC, 0x80, 0xCA, 0x86, 0xCE, 0x91, 0xCC, - 0x94, 0xCC, 0x81, 0xCA, 0x86, 0xCE, 0x91, 0xCC, - 0x94, 0xCD, 0x82, 0xCA, 0x86, 0xCE, 0x97, 0xCC, - 0x93, 0xCC, 0x80, 0xCA, 0x86, 0xCE, 0x97, 0xCC, - 0x93, 0xCC, 0x81, 0xCA, 0x86, 0xCE, 0x97, 0xCC, - 0x93, 0xCD, 0x82, 0xCA, 0x86, 0xCE, 0x97, 0xCC, - 0x94, 0xCC, 0x80, 0xCA, 0x86, 0xCE, 0x97, 0xCC, - 0x94, 0xCC, 0x81, 0xCA, 0x86, 0xCE, 0x97, 0xCC, + 0xB1, 0xCC, 0x94, 0xCC, 0x81, 0xCA, 0x86, 0xCE, + 0xB1, 0xCC, 0x94, 0xCD, 0x82, 0xCA, 0x86, 0xCE, + 0xB7, 0xCC, 0x93, 0xCC, 0x80, 0xCA, 0x86, 0xCE, + 0xB7, 0xCC, 0x93, 0xCC, 0x81, 0xCA, 0x86, 0xCE, + 0xB7, 0xCC, 0x93, 0xCD, 0x82, 0xCA, 0x86, 0xCE, + 0xB7, 0xCC, 0x94, 0xCC, 0x80, 0xCA, 0x86, 0xCE, + 0xB7, 0xCC, 0x94, 0xCC, 0x81, 0xCA, 0x86, 0xCE, + 0xB7, 0xCC, 0x94, 0xCD, 0x82, 0xCA, 0x86, 0xCF, // Bytes 4980 - 49bf - 0x94, 0xCD, 0x82, 0xCA, 0x86, 0xCE, 0xA9, 0xCC, - 0x93, 0xCC, 0x80, 0xCA, 0x86, 0xCE, 0xA9, 0xCC, - 0x93, 0xCC, 0x81, 0xCA, 0x86, 0xCE, 0xA9, 0xCC, - 0x93, 0xCD, 0x82, 0xCA, 0x86, 0xCE, 0xA9, 0xCC, - 0x94, 0xCC, 0x80, 0xCA, 0x86, 0xCE, 0xA9, 0xCC, - 0x94, 0xCC, 0x81, 0xCA, 0x86, 0xCE, 0xA9, 0xCC, - 0x94, 0xCD, 0x82, 0xCA, 0x86, 0xCE, 0xB1, 0xCC, - 0x93, 0xCC, 0x80, 0xCA, 0x86, 0xCE, 0xB1, 0xCC, + 0x89, 0xCC, 0x93, 0xCC, 0x80, 0xCA, 0x86, 0xCF, + 0x89, 0xCC, 0x93, 0xCC, 0x81, 0xCA, 0x86, 0xCF, + 0x89, 0xCC, 0x93, 0xCD, 0x82, 0xCA, 0x86, 0xCF, + 0x89, 0xCC, 0x94, 0xCC, 0x80, 0xCA, 0x86, 0xCF, + 0x89, 0xCC, 0x94, 0xCC, 0x81, 0xCA, 0x86, 0xCF, + 0x89, 0xCC, 0x94, 0xCD, 0x82, 0xCA, 0x42, 0xCC, + 0x80, 0xC9, 0x32, 0x42, 0xCC, 0x81, 0xC9, 0x32, + 0x42, 0xCC, 0x93, 0xC9, 0x32, 0x43, 0xE1, 0x85, // Bytes 49c0 - 49ff - 0x93, 0xCC, 0x81, 0xCA, 0x86, 0xCE, 0xB1, 0xCC, - 0x93, 0xCD, 0x82, 0xCA, 0x86, 0xCE, 0xB1, 0xCC, - 0x94, 0xCC, 0x80, 0xCA, 0x86, 0xCE, 0xB1, 0xCC, - 0x94, 0xCC, 0x81, 0xCA, 0x86, 0xCE, 0xB1, 0xCC, - 0x94, 0xCD, 0x82, 0xCA, 0x86, 0xCE, 0xB7, 0xCC, - 0x93, 0xCC, 0x80, 0xCA, 0x86, 0xCE, 0xB7, 0xCC, - 0x93, 0xCC, 0x81, 0xCA, 0x86, 0xCE, 0xB7, 0xCC, - 0x93, 0xCD, 0x82, 0xCA, 0x86, 0xCE, 0xB7, 0xCC, + 0xA1, 0x01, 0x00, 0x43, 0xE1, 0x85, 0xA2, 0x01, + 0x00, 0x43, 0xE1, 0x85, 0xA3, 0x01, 0x00, 0x43, + 0xE1, 0x85, 0xA4, 0x01, 0x00, 0x43, 0xE1, 0x85, + 0xA5, 0x01, 0x00, 0x43, 0xE1, 0x85, 0xA6, 0x01, + 0x00, 0x43, 0xE1, 0x85, 0xA7, 0x01, 0x00, 0x43, + 0xE1, 0x85, 0xA8, 0x01, 0x00, 0x43, 0xE1, 0x85, + 0xA9, 0x01, 0x00, 0x43, 0xE1, 0x85, 0xAA, 0x01, + 0x00, 0x43, 0xE1, 0x85, 0xAB, 0x01, 0x00, 0x43, // Bytes 4a00 - 4a3f - 0x94, 0xCC, 0x80, 0xCA, 0x86, 0xCE, 0xB7, 0xCC, - 0x94, 0xCC, 0x81, 0xCA, 0x86, 0xCE, 0xB7, 0xCC, - 0x94, 0xCD, 0x82, 0xCA, 0x86, 0xCF, 0x89, 0xCC, - 0x93, 0xCC, 0x80, 0xCA, 0x86, 0xCF, 0x89, 0xCC, - 0x93, 0xCC, 0x81, 0xCA, 0x86, 0xCF, 0x89, 0xCC, - 0x93, 0xCD, 0x82, 0xCA, 0x86, 0xCF, 0x89, 0xCC, - 0x94, 0xCC, 0x80, 0xCA, 0x86, 0xCF, 0x89, 0xCC, - 0x94, 0xCC, 0x81, 0xCA, 0x86, 0xCF, 0x89, 0xCC, + 0xE1, 0x85, 0xAC, 0x01, 0x00, 0x43, 0xE1, 0x85, + 0xAD, 0x01, 0x00, 0x43, 0xE1, 0x85, 0xAE, 0x01, + 0x00, 0x43, 0xE1, 0x85, 0xAF, 0x01, 0x00, 0x43, + 0xE1, 0x85, 0xB0, 0x01, 0x00, 0x43, 0xE1, 0x85, + 0xB1, 0x01, 0x00, 0x43, 0xE1, 0x85, 0xB2, 0x01, + 0x00, 0x43, 0xE1, 0x85, 0xB3, 0x01, 0x00, 0x43, + 0xE1, 0x85, 0xB4, 0x01, 0x00, 0x43, 0xE1, 0x85, + 0xB5, 0x01, 0x00, 0x43, 0xE1, 0x86, 0xAA, 0x01, // Bytes 4a40 - 4a7f - 0x94, 0xCD, 0x82, 0xCA, 0x42, 0xCC, 0x80, 0xC9, - 0x32, 0x42, 0xCC, 0x81, 0xC9, 0x32, 0x42, 0xCC, - 0x93, 0xC9, 0x32, 0x44, 0xCC, 0x88, 0xCC, 0x81, - 0xCA, 0x32, 0x43, 0xE3, 0x82, 0x99, 0x0D, 0x03, - 0x43, 0xE3, 0x82, 0x9A, 0x0D, 0x03, 0x46, 0xE0, - 0xBD, 0xB1, 0xE0, 0xBD, 0xB2, 0x9E, 0x26, 0x46, - 0xE0, 0xBD, 0xB1, 0xE0, 0xBD, 0xB4, 0xA2, 0x26, - 0x46, 0xE0, 0xBD, 0xB1, 0xE0, 0xBE, 0x80, 0x9E, + 0x00, 0x43, 0xE1, 0x86, 0xAC, 0x01, 0x00, 0x43, + 0xE1, 0x86, 0xAD, 0x01, 0x00, 0x43, 0xE1, 0x86, + 0xB0, 0x01, 0x00, 0x43, 0xE1, 0x86, 0xB1, 0x01, + 0x00, 0x43, 0xE1, 0x86, 0xB2, 0x01, 0x00, 0x43, + 0xE1, 0x86, 0xB3, 0x01, 0x00, 0x43, 0xE1, 0x86, + 0xB4, 0x01, 0x00, 0x43, 0xE1, 0x86, 0xB5, 0x01, + 0x00, 0x44, 0xCC, 0x88, 0xCC, 0x81, 0xCA, 0x32, + 0x43, 0xE3, 0x82, 0x99, 0x0D, 0x03, 0x43, 0xE3, // Bytes 4a80 - 4abf - 0x26, 0x00, 0x01, + 0x82, 0x9A, 0x0D, 0x03, 0x46, 0xE0, 0xBD, 0xB1, + 0xE0, 0xBD, 0xB2, 0x9E, 0x26, 0x46, 0xE0, 0xBD, + 0xB1, 0xE0, 0xBD, 0xB4, 0xA2, 0x26, 0x46, 0xE0, + 0xBD, 0xB1, 0xE0, 0xBE, 0x80, 0x9E, 0x26, 0x00, + 0x01, } // lookup returns the trie value for the first UTF-8 encoding in s and @@ -2892,7 +2898,7 @@ func (t *nfcTrie) lookupStringUnsafe(s string) uint16 { return 0 } -// nfcTrie. Total size: 10332 bytes (10.09 KiB). Checksum: ad355b768fddb1b6. +// nfcTrie. Total size: 10332 bytes (10.09 KiB). Checksum: 51cc525b297fc970. type nfcTrie struct{} func newNfcTrie(i int) *nfcTrie { @@ -2928,22 +2934,22 @@ var nfcValues = [2944]uint16{ 0x76: 0xa000, 0x77: 0xa000, 0x78: 0xa000, 0x79: 0xa000, 0x7a: 0xa000, // Block 0x2, offset 0x80 // Block 0x3, offset 0xc0 - 0xc0: 0x2f6f, 0xc1: 0x2f74, 0xc2: 0x471e, 0xc3: 0x2f79, 0xc4: 0x472d, 0xc5: 0x4732, - 0xc6: 0xa000, 0xc7: 0x473c, 0xc8: 0x2fe2, 0xc9: 0x2fe7, 0xca: 0x4741, 0xcb: 0x2ffb, - 0xcc: 0x306e, 0xcd: 0x3073, 0xce: 0x3078, 0xcf: 0x4755, 0xd1: 0x3104, - 0xd2: 0x3127, 0xd3: 0x312c, 0xd4: 0x475f, 0xd5: 0x4764, 0xd6: 0x4773, - 0xd8: 0xa000, 0xd9: 0x31b3, 0xda: 0x31b8, 0xdb: 0x31bd, 0xdc: 0x47a5, 0xdd: 0x3235, - 0xe0: 0x327b, 0xe1: 0x3280, 0xe2: 0x47af, 0xe3: 0x3285, - 0xe4: 0x47be, 0xe5: 0x47c3, 0xe6: 0xa000, 0xe7: 0x47cd, 0xe8: 0x32ee, 0xe9: 0x32f3, - 0xea: 0x47d2, 0xeb: 0x3307, 0xec: 0x337f, 0xed: 0x3384, 0xee: 0x3389, 0xef: 0x47e6, - 0xf1: 0x3415, 0xf2: 0x3438, 0xf3: 0x343d, 0xf4: 0x47f0, 0xf5: 0x47f5, - 0xf6: 0x4804, 0xf8: 0xa000, 0xf9: 0x34c9, 0xfa: 0x34ce, 0xfb: 0x34d3, - 0xfc: 0x4836, 0xfd: 0x3550, 0xff: 0x3569, + 0xc0: 0x2f6f, 0xc1: 0x2f74, 0xc2: 0x4688, 0xc3: 0x2f79, 0xc4: 0x4697, 0xc5: 0x469c, + 0xc6: 0xa000, 0xc7: 0x46a6, 0xc8: 0x2fe2, 0xc9: 0x2fe7, 0xca: 0x46ab, 0xcb: 0x2ffb, + 0xcc: 0x306e, 0xcd: 0x3073, 0xce: 0x3078, 0xcf: 0x46bf, 0xd1: 0x3104, + 0xd2: 0x3127, 0xd3: 0x312c, 0xd4: 0x46c9, 0xd5: 0x46ce, 0xd6: 0x46dd, + 0xd8: 0xa000, 0xd9: 0x31b3, 0xda: 0x31b8, 0xdb: 0x31bd, 0xdc: 0x470f, 0xdd: 0x3235, + 0xe0: 0x327b, 0xe1: 0x3280, 0xe2: 0x4719, 0xe3: 0x3285, + 0xe4: 0x4728, 0xe5: 0x472d, 0xe6: 0xa000, 0xe7: 0x4737, 0xe8: 0x32ee, 0xe9: 0x32f3, + 0xea: 0x473c, 0xeb: 0x3307, 0xec: 0x337f, 0xed: 0x3384, 0xee: 0x3389, 0xef: 0x4750, + 0xf1: 0x3415, 0xf2: 0x3438, 0xf3: 0x343d, 0xf4: 0x475a, 0xf5: 0x475f, + 0xf6: 0x476e, 0xf8: 0xa000, 0xf9: 0x34c9, 0xfa: 0x34ce, 0xfb: 0x34d3, + 0xfc: 0x47a0, 0xfd: 0x3550, 0xff: 0x3569, // Block 0x4, offset 0x100 - 0x100: 0x2f7e, 0x101: 0x328a, 0x102: 0x4723, 0x103: 0x47b4, 0x104: 0x2f9c, 0x105: 0x32a8, + 0x100: 0x2f7e, 0x101: 0x328a, 0x102: 0x468d, 0x103: 0x471e, 0x104: 0x2f9c, 0x105: 0x32a8, 0x106: 0x2fb0, 0x107: 0x32bc, 0x108: 0x2fb5, 0x109: 0x32c1, 0x10a: 0x2fba, 0x10b: 0x32c6, 0x10c: 0x2fbf, 0x10d: 0x32cb, 0x10e: 0x2fc9, 0x10f: 0x32d5, - 0x112: 0x4746, 0x113: 0x47d7, 0x114: 0x2ff1, 0x115: 0x32fd, 0x116: 0x2ff6, 0x117: 0x3302, + 0x112: 0x46b0, 0x113: 0x4741, 0x114: 0x2ff1, 0x115: 0x32fd, 0x116: 0x2ff6, 0x117: 0x3302, 0x118: 0x3014, 0x119: 0x3320, 0x11a: 0x3005, 0x11b: 0x3311, 0x11c: 0x302d, 0x11d: 0x3339, 0x11e: 0x3037, 0x11f: 0x3343, 0x120: 0x303c, 0x121: 0x3348, 0x122: 0x3046, 0x123: 0x3352, 0x124: 0x304b, 0x125: 0x3357, 0x128: 0x307d, 0x129: 0x338e, @@ -2954,12 +2960,12 @@ var nfcValues = [2944]uint16{ // Block 0x5, offset 0x140 0x143: 0x30ff, 0x144: 0x3410, 0x145: 0x3118, 0x146: 0x3429, 0x147: 0x310e, 0x148: 0x341f, - 0x14c: 0x4769, 0x14d: 0x47fa, 0x14e: 0x3131, 0x14f: 0x3442, 0x150: 0x313b, 0x151: 0x344c, + 0x14c: 0x46d3, 0x14d: 0x4764, 0x14e: 0x3131, 0x14f: 0x3442, 0x150: 0x313b, 0x151: 0x344c, 0x154: 0x3159, 0x155: 0x346a, 0x156: 0x3172, 0x157: 0x3483, - 0x158: 0x3163, 0x159: 0x3474, 0x15a: 0x478c, 0x15b: 0x481d, 0x15c: 0x317c, 0x15d: 0x348d, - 0x15e: 0x318b, 0x15f: 0x349c, 0x160: 0x4791, 0x161: 0x4822, 0x162: 0x31a4, 0x163: 0x34ba, - 0x164: 0x3195, 0x165: 0x34ab, 0x168: 0x479b, 0x169: 0x482c, - 0x16a: 0x47a0, 0x16b: 0x4831, 0x16c: 0x31c2, 0x16d: 0x34d8, 0x16e: 0x31cc, 0x16f: 0x34e2, + 0x158: 0x3163, 0x159: 0x3474, 0x15a: 0x46f6, 0x15b: 0x4787, 0x15c: 0x317c, 0x15d: 0x348d, + 0x15e: 0x318b, 0x15f: 0x349c, 0x160: 0x46fb, 0x161: 0x478c, 0x162: 0x31a4, 0x163: 0x34ba, + 0x164: 0x3195, 0x165: 0x34ab, 0x168: 0x4705, 0x169: 0x4796, + 0x16a: 0x470a, 0x16b: 0x479b, 0x16c: 0x31c2, 0x16d: 0x34d8, 0x16e: 0x31cc, 0x16f: 0x34e2, 0x170: 0x31d1, 0x171: 0x34e7, 0x172: 0x31ef, 0x173: 0x3505, 0x174: 0x3212, 0x175: 0x3528, 0x176: 0x323a, 0x177: 0x3555, 0x178: 0x324e, 0x179: 0x325d, 0x17a: 0x357d, 0x17b: 0x3267, 0x17c: 0x3587, 0x17d: 0x326c, 0x17e: 0x358c, 0x17f: 0xa000, @@ -2971,7 +2977,7 @@ var nfcValues = [2944]uint16{ 0x198: 0x3b57, 0x199: 0x39d6, 0x19a: 0x3b65, 0x19b: 0x39c1, 0x19c: 0x3b50, 0x19e: 0x38b0, 0x19f: 0x3a3f, 0x1a0: 0x38a9, 0x1a1: 0x3a38, 0x1a2: 0x35b3, 0x1a3: 0x35c5, 0x1a6: 0x3041, 0x1a7: 0x334d, 0x1a8: 0x30be, 0x1a9: 0x33cf, - 0x1aa: 0x4782, 0x1ab: 0x4813, 0x1ac: 0x3990, 0x1ad: 0x3b1f, 0x1ae: 0x35d7, 0x1af: 0x35dd, + 0x1aa: 0x46ec, 0x1ab: 0x477d, 0x1ac: 0x3990, 0x1ad: 0x3b1f, 0x1ae: 0x35d7, 0x1af: 0x35dd, 0x1b0: 0x33c5, 0x1b4: 0x3028, 0x1b5: 0x3334, 0x1b8: 0x30fa, 0x1b9: 0x340b, 0x1ba: 0x38b7, 0x1bb: 0x3a46, 0x1bc: 0x35ad, 0x1bd: 0x35bf, 0x1be: 0x35b9, 0x1bf: 0x35cb, @@ -2982,8 +2988,8 @@ var nfcValues = [2944]uint16{ 0x1d2: 0x316d, 0x1d3: 0x347e, 0x1d4: 0x31db, 0x1d5: 0x34f1, 0x1d6: 0x31e0, 0x1d7: 0x34f6, 0x1d8: 0x3186, 0x1d9: 0x3497, 0x1da: 0x319f, 0x1db: 0x34b5, 0x1de: 0x305a, 0x1df: 0x3366, - 0x1e6: 0x4728, 0x1e7: 0x47b9, 0x1e8: 0x4750, 0x1e9: 0x47e1, - 0x1ea: 0x395f, 0x1eb: 0x3aee, 0x1ec: 0x393c, 0x1ed: 0x3acb, 0x1ee: 0x476e, 0x1ef: 0x47ff, + 0x1e6: 0x4692, 0x1e7: 0x4723, 0x1e8: 0x46ba, 0x1e9: 0x474b, + 0x1ea: 0x395f, 0x1eb: 0x3aee, 0x1ec: 0x393c, 0x1ed: 0x3acb, 0x1ee: 0x46d8, 0x1ef: 0x4769, 0x1f0: 0x3958, 0x1f1: 0x3ae7, 0x1f2: 0x3244, 0x1f3: 0x355f, // Block 0x8, offset 0x200 0x200: 0x9932, 0x201: 0x9932, 0x202: 0x9932, 0x203: 0x9932, 0x204: 0x9932, 0x205: 0x8132, @@ -2998,7 +3004,7 @@ var nfcValues = [2944]uint16{ 0x236: 0x8101, 0x237: 0x8101, 0x238: 0x9901, 0x239: 0x812d, 0x23a: 0x812d, 0x23b: 0x812d, 0x23c: 0x812d, 0x23d: 0x8132, 0x23e: 0x8132, 0x23f: 0x8132, // Block 0x9, offset 0x240 - 0x240: 0x4a44, 0x241: 0x4a49, 0x242: 0x9932, 0x243: 0x4a4e, 0x244: 0x4a53, 0x245: 0x9936, + 0x240: 0x49ae, 0x241: 0x49b3, 0x242: 0x9932, 0x243: 0x49b8, 0x244: 0x4a71, 0x245: 0x9936, 0x246: 0x8132, 0x247: 0x812d, 0x248: 0x812d, 0x249: 0x812d, 0x24a: 0x8132, 0x24b: 0x8132, 0x24c: 0x8132, 0x24d: 0x812d, 0x24e: 0x812d, 0x250: 0x8132, 0x251: 0x8132, 0x252: 0x8132, 0x253: 0x812d, 0x254: 0x812d, 0x255: 0x812d, 0x256: 0x812d, 0x257: 0x8132, @@ -3017,7 +3023,7 @@ var nfcValues = [2944]uint16{ 0x299: 0xa000, 0x29f: 0xa000, 0x2a1: 0xa000, 0x2a5: 0xa000, 0x2a9: 0xa000, - 0x2aa: 0x3637, 0x2ab: 0x3667, 0x2ac: 0x4894, 0x2ad: 0x3697, 0x2ae: 0x48be, 0x2af: 0x36a9, + 0x2aa: 0x3637, 0x2ab: 0x3667, 0x2ac: 0x47fe, 0x2ad: 0x3697, 0x2ae: 0x4828, 0x2af: 0x36a9, 0x2b0: 0x3e70, 0x2b1: 0xa000, 0x2b5: 0xa000, 0x2b7: 0xa000, 0x2b9: 0xa000, 0x2bf: 0xa000, @@ -3078,15 +3084,15 @@ var nfcValues = [2944]uint16{ 0x424: 0x305f, 0x425: 0x336b, 0x426: 0x3055, 0x427: 0x3361, 0x428: 0x3064, 0x429: 0x3370, 0x42a: 0x3069, 0x42b: 0x3375, 0x42c: 0x30af, 0x42d: 0x33bb, 0x42e: 0x390b, 0x42f: 0x3a9a, 0x430: 0x30b9, 0x431: 0x33ca, 0x432: 0x30c3, 0x433: 0x33d4, 0x434: 0x30cd, 0x435: 0x33de, - 0x436: 0x475a, 0x437: 0x47eb, 0x438: 0x3912, 0x439: 0x3aa1, 0x43a: 0x30e6, 0x43b: 0x33f7, + 0x436: 0x46c4, 0x437: 0x4755, 0x438: 0x3912, 0x439: 0x3aa1, 0x43a: 0x30e6, 0x43b: 0x33f7, 0x43c: 0x30e1, 0x43d: 0x33f2, 0x43e: 0x30eb, 0x43f: 0x33fc, // Block 0x11, offset 0x440 0x440: 0x30f0, 0x441: 0x3401, 0x442: 0x30f5, 0x443: 0x3406, 0x444: 0x3109, 0x445: 0x341a, 0x446: 0x3113, 0x447: 0x3424, 0x448: 0x3122, 0x449: 0x3433, 0x44a: 0x311d, 0x44b: 0x342e, 0x44c: 0x3935, 0x44d: 0x3ac4, 0x44e: 0x3943, 0x44f: 0x3ad2, 0x450: 0x394a, 0x451: 0x3ad9, 0x452: 0x3951, 0x453: 0x3ae0, 0x454: 0x314f, 0x455: 0x3460, 0x456: 0x3154, 0x457: 0x3465, - 0x458: 0x315e, 0x459: 0x346f, 0x45a: 0x4787, 0x45b: 0x4818, 0x45c: 0x3997, 0x45d: 0x3b26, - 0x45e: 0x3177, 0x45f: 0x3488, 0x460: 0x3181, 0x461: 0x3492, 0x462: 0x4796, 0x463: 0x4827, + 0x458: 0x315e, 0x459: 0x346f, 0x45a: 0x46f1, 0x45b: 0x4782, 0x45c: 0x3997, 0x45d: 0x3b26, + 0x45e: 0x3177, 0x45f: 0x3488, 0x460: 0x3181, 0x461: 0x3492, 0x462: 0x4700, 0x463: 0x4791, 0x464: 0x399e, 0x465: 0x3b2d, 0x466: 0x39a5, 0x467: 0x3b34, 0x468: 0x39ac, 0x469: 0x3b3b, 0x46a: 0x3190, 0x46b: 0x34a1, 0x46c: 0x319a, 0x46d: 0x34b0, 0x46e: 0x31ae, 0x46f: 0x34c4, 0x470: 0x31a9, 0x471: 0x34bf, 0x472: 0x31ea, 0x473: 0x3500, 0x474: 0x31f9, 0x475: 0x350f, @@ -3098,16 +3104,16 @@ var nfcValues = [2944]uint16{ 0x48c: 0x322b, 0x48d: 0x3546, 0x48e: 0x3249, 0x48f: 0x3564, 0x490: 0x3262, 0x491: 0x3582, 0x492: 0x3271, 0x493: 0x3591, 0x494: 0x3276, 0x495: 0x3596, 0x496: 0x337a, 0x497: 0x34a6, 0x498: 0x3537, 0x499: 0x3573, 0x49b: 0x35d1, - 0x4a0: 0x4737, 0x4a1: 0x47c8, 0x4a2: 0x2f83, 0x4a3: 0x328f, + 0x4a0: 0x46a1, 0x4a1: 0x4732, 0x4a2: 0x2f83, 0x4a3: 0x328f, 0x4a4: 0x3878, 0x4a5: 0x3a07, 0x4a6: 0x3871, 0x4a7: 0x3a00, 0x4a8: 0x3886, 0x4a9: 0x3a15, 0x4aa: 0x387f, 0x4ab: 0x3a0e, 0x4ac: 0x38be, 0x4ad: 0x3a4d, 0x4ae: 0x3894, 0x4af: 0x3a23, 0x4b0: 0x388d, 0x4b1: 0x3a1c, 0x4b2: 0x38a2, 0x4b3: 0x3a31, 0x4b4: 0x389b, 0x4b5: 0x3a2a, - 0x4b6: 0x38c5, 0x4b7: 0x3a54, 0x4b8: 0x474b, 0x4b9: 0x47dc, 0x4ba: 0x3000, 0x4bb: 0x330c, + 0x4b6: 0x38c5, 0x4b7: 0x3a54, 0x4b8: 0x46b5, 0x4b9: 0x4746, 0x4ba: 0x3000, 0x4bb: 0x330c, 0x4bc: 0x2fec, 0x4bd: 0x32f8, 0x4be: 0x38da, 0x4bf: 0x3a69, // Block 0x13, offset 0x4c0 0x4c0: 0x38d3, 0x4c1: 0x3a62, 0x4c2: 0x38e8, 0x4c3: 0x3a77, 0x4c4: 0x38e1, 0x4c5: 0x3a70, 0x4c6: 0x38fd, 0x4c7: 0x3a8c, 0x4c8: 0x3091, 0x4c9: 0x339d, 0x4ca: 0x30a5, 0x4cb: 0x33b1, - 0x4cc: 0x477d, 0x4cd: 0x480e, 0x4ce: 0x3136, 0x4cf: 0x3447, 0x4d0: 0x3920, 0x4d1: 0x3aaf, + 0x4cc: 0x46e7, 0x4cd: 0x4778, 0x4ce: 0x3136, 0x4cf: 0x3447, 0x4d0: 0x3920, 0x4d1: 0x3aaf, 0x4d2: 0x3919, 0x4d3: 0x3aa8, 0x4d4: 0x392e, 0x4d5: 0x3abd, 0x4d6: 0x3927, 0x4d7: 0x3ab6, 0x4d8: 0x3989, 0x4d9: 0x3b18, 0x4da: 0x396d, 0x4db: 0x3afc, 0x4dc: 0x3966, 0x4dd: 0x3af5, 0x4de: 0x397b, 0x4df: 0x3b0a, 0x4e0: 0x3974, 0x4e1: 0x3b03, 0x4e2: 0x3982, 0x4e3: 0x3b11, @@ -3116,29 +3122,29 @@ var nfcValues = [2944]uint16{ 0x4f0: 0x39f9, 0x4f1: 0x3b88, 0x4f2: 0x3230, 0x4f3: 0x354b, 0x4f4: 0x3258, 0x4f5: 0x3578, 0x4f6: 0x3253, 0x4f7: 0x356e, 0x4f8: 0x323f, 0x4f9: 0x355a, // Block 0x14, offset 0x500 - 0x500: 0x489a, 0x501: 0x48a0, 0x502: 0x49b4, 0x503: 0x49cc, 0x504: 0x49bc, 0x505: 0x49d4, - 0x506: 0x49c4, 0x507: 0x49dc, 0x508: 0x4840, 0x509: 0x4846, 0x50a: 0x4924, 0x50b: 0x493c, - 0x50c: 0x492c, 0x50d: 0x4944, 0x50e: 0x4934, 0x50f: 0x494c, 0x510: 0x48ac, 0x511: 0x48b2, + 0x500: 0x4804, 0x501: 0x480a, 0x502: 0x491e, 0x503: 0x4936, 0x504: 0x4926, 0x505: 0x493e, + 0x506: 0x492e, 0x507: 0x4946, 0x508: 0x47aa, 0x509: 0x47b0, 0x50a: 0x488e, 0x50b: 0x48a6, + 0x50c: 0x4896, 0x50d: 0x48ae, 0x50e: 0x489e, 0x50f: 0x48b6, 0x510: 0x4816, 0x511: 0x481c, 0x512: 0x3db8, 0x513: 0x3dc8, 0x514: 0x3dc0, 0x515: 0x3dd0, - 0x518: 0x484c, 0x519: 0x4852, 0x51a: 0x3ce8, 0x51b: 0x3cf8, 0x51c: 0x3cf0, 0x51d: 0x3d00, - 0x520: 0x48c4, 0x521: 0x48ca, 0x522: 0x49e4, 0x523: 0x49fc, - 0x524: 0x49ec, 0x525: 0x4a04, 0x526: 0x49f4, 0x527: 0x4a0c, 0x528: 0x4858, 0x529: 0x485e, - 0x52a: 0x4954, 0x52b: 0x496c, 0x52c: 0x495c, 0x52d: 0x4974, 0x52e: 0x4964, 0x52f: 0x497c, - 0x530: 0x48dc, 0x531: 0x48e2, 0x532: 0x3e18, 0x533: 0x3e30, 0x534: 0x3e20, 0x535: 0x3e38, - 0x536: 0x3e28, 0x537: 0x3e40, 0x538: 0x4864, 0x539: 0x486a, 0x53a: 0x3d18, 0x53b: 0x3d30, + 0x518: 0x47b6, 0x519: 0x47bc, 0x51a: 0x3ce8, 0x51b: 0x3cf8, 0x51c: 0x3cf0, 0x51d: 0x3d00, + 0x520: 0x482e, 0x521: 0x4834, 0x522: 0x494e, 0x523: 0x4966, + 0x524: 0x4956, 0x525: 0x496e, 0x526: 0x495e, 0x527: 0x4976, 0x528: 0x47c2, 0x529: 0x47c8, + 0x52a: 0x48be, 0x52b: 0x48d6, 0x52c: 0x48c6, 0x52d: 0x48de, 0x52e: 0x48ce, 0x52f: 0x48e6, + 0x530: 0x4846, 0x531: 0x484c, 0x532: 0x3e18, 0x533: 0x3e30, 0x534: 0x3e20, 0x535: 0x3e38, + 0x536: 0x3e28, 0x537: 0x3e40, 0x538: 0x47ce, 0x539: 0x47d4, 0x53a: 0x3d18, 0x53b: 0x3d30, 0x53c: 0x3d20, 0x53d: 0x3d38, 0x53e: 0x3d28, 0x53f: 0x3d40, // Block 0x15, offset 0x540 - 0x540: 0x48e8, 0x541: 0x48ee, 0x542: 0x3e48, 0x543: 0x3e58, 0x544: 0x3e50, 0x545: 0x3e60, - 0x548: 0x4870, 0x549: 0x4876, 0x54a: 0x3d48, 0x54b: 0x3d58, - 0x54c: 0x3d50, 0x54d: 0x3d60, 0x550: 0x48fa, 0x551: 0x4900, + 0x540: 0x4852, 0x541: 0x4858, 0x542: 0x3e48, 0x543: 0x3e58, 0x544: 0x3e50, 0x545: 0x3e60, + 0x548: 0x47da, 0x549: 0x47e0, 0x54a: 0x3d48, 0x54b: 0x3d58, + 0x54c: 0x3d50, 0x54d: 0x3d60, 0x550: 0x4864, 0x551: 0x486a, 0x552: 0x3e80, 0x553: 0x3e98, 0x554: 0x3e88, 0x555: 0x3ea0, 0x556: 0x3e90, 0x557: 0x3ea8, - 0x559: 0x487c, 0x55b: 0x3d68, 0x55d: 0x3d70, - 0x55f: 0x3d78, 0x560: 0x4912, 0x561: 0x4918, 0x562: 0x4a14, 0x563: 0x4a2c, - 0x564: 0x4a1c, 0x565: 0x4a34, 0x566: 0x4a24, 0x567: 0x4a3c, 0x568: 0x4882, 0x569: 0x4888, - 0x56a: 0x4984, 0x56b: 0x499c, 0x56c: 0x498c, 0x56d: 0x49a4, 0x56e: 0x4994, 0x56f: 0x49ac, - 0x570: 0x488e, 0x571: 0x43b4, 0x572: 0x3691, 0x573: 0x43ba, 0x574: 0x48b8, 0x575: 0x43c0, - 0x576: 0x36a3, 0x577: 0x43c6, 0x578: 0x36c1, 0x579: 0x43cc, 0x57a: 0x36d9, 0x57b: 0x43d2, - 0x57c: 0x4906, 0x57d: 0x43d8, + 0x559: 0x47e6, 0x55b: 0x3d68, 0x55d: 0x3d70, + 0x55f: 0x3d78, 0x560: 0x487c, 0x561: 0x4882, 0x562: 0x497e, 0x563: 0x4996, + 0x564: 0x4986, 0x565: 0x499e, 0x566: 0x498e, 0x567: 0x49a6, 0x568: 0x47ec, 0x569: 0x47f2, + 0x56a: 0x48ee, 0x56b: 0x4906, 0x56c: 0x48f6, 0x56d: 0x490e, 0x56e: 0x48fe, 0x56f: 0x4916, + 0x570: 0x47f8, 0x571: 0x431e, 0x572: 0x3691, 0x573: 0x4324, 0x574: 0x4822, 0x575: 0x432a, + 0x576: 0x36a3, 0x577: 0x4330, 0x578: 0x36c1, 0x579: 0x4336, 0x57a: 0x36d9, 0x57b: 0x433c, + 0x57c: 0x4870, 0x57d: 0x4342, // Block 0x16, offset 0x580 0x580: 0x3da0, 0x581: 0x3da8, 0x582: 0x4184, 0x583: 0x41a2, 0x584: 0x418e, 0x585: 0x41ac, 0x586: 0x4198, 0x587: 0x41b6, 0x588: 0x3cd8, 0x589: 0x3ce0, 0x58a: 0x40d0, 0x58b: 0x40ee, @@ -3149,19 +3155,19 @@ var nfcValues = [2944]uint16{ 0x5a4: 0x4206, 0x5a5: 0x4224, 0x5a6: 0x4210, 0x5a7: 0x422e, 0x5a8: 0x3d80, 0x5a9: 0x3d88, 0x5aa: 0x4148, 0x5ab: 0x4166, 0x5ac: 0x4152, 0x5ad: 0x4170, 0x5ae: 0x415c, 0x5af: 0x417a, 0x5b0: 0x3685, 0x5b1: 0x367f, 0x5b2: 0x3d90, 0x5b3: 0x368b, 0x5b4: 0x3d98, - 0x5b6: 0x48a6, 0x5b7: 0x3db0, 0x5b8: 0x35f5, 0x5b9: 0x35ef, 0x5ba: 0x35e3, 0x5bb: 0x4384, + 0x5b6: 0x4810, 0x5b7: 0x3db0, 0x5b8: 0x35f5, 0x5b9: 0x35ef, 0x5ba: 0x35e3, 0x5bb: 0x42ee, 0x5bc: 0x35fb, 0x5bd: 0x8100, 0x5be: 0x01d3, 0x5bf: 0xa100, // Block 0x17, offset 0x5c0 0x5c0: 0x8100, 0x5c1: 0x35a7, 0x5c2: 0x3dd8, 0x5c3: 0x369d, 0x5c4: 0x3de0, - 0x5c6: 0x48d0, 0x5c7: 0x3df8, 0x5c8: 0x3601, 0x5c9: 0x438a, 0x5ca: 0x360d, 0x5cb: 0x4390, + 0x5c6: 0x483a, 0x5c7: 0x3df8, 0x5c8: 0x3601, 0x5c9: 0x42f4, 0x5ca: 0x360d, 0x5cb: 0x42fa, 0x5cc: 0x3619, 0x5cd: 0x3b8f, 0x5ce: 0x3b96, 0x5cf: 0x3b9d, 0x5d0: 0x36b5, 0x5d1: 0x36af, - 0x5d2: 0x3e00, 0x5d3: 0x457a, 0x5d6: 0x36bb, 0x5d7: 0x3e10, - 0x5d8: 0x3631, 0x5d9: 0x362b, 0x5da: 0x361f, 0x5db: 0x4396, 0x5dd: 0x3ba4, - 0x5de: 0x3bab, 0x5df: 0x3bb2, 0x5e0: 0x36eb, 0x5e1: 0x36e5, 0x5e2: 0x3e68, 0x5e3: 0x4582, + 0x5d2: 0x3e00, 0x5d3: 0x44e4, 0x5d6: 0x36bb, 0x5d7: 0x3e10, + 0x5d8: 0x3631, 0x5d9: 0x362b, 0x5da: 0x361f, 0x5db: 0x4300, 0x5dd: 0x3ba4, + 0x5de: 0x3bab, 0x5df: 0x3bb2, 0x5e0: 0x36eb, 0x5e1: 0x36e5, 0x5e2: 0x3e68, 0x5e3: 0x44ec, 0x5e4: 0x36cd, 0x5e5: 0x36d3, 0x5e6: 0x36f1, 0x5e7: 0x3e78, 0x5e8: 0x3661, 0x5e9: 0x365b, - 0x5ea: 0x364f, 0x5eb: 0x43a2, 0x5ec: 0x3649, 0x5ed: 0x359b, 0x5ee: 0x437e, 0x5ef: 0x0081, + 0x5ea: 0x364f, 0x5eb: 0x430c, 0x5ec: 0x3649, 0x5ed: 0x359b, 0x5ee: 0x42e8, 0x5ef: 0x0081, 0x5f2: 0x3eb0, 0x5f3: 0x36f7, 0x5f4: 0x3eb8, - 0x5f6: 0x491e, 0x5f7: 0x3ed0, 0x5f8: 0x363d, 0x5f9: 0x439c, 0x5fa: 0x366d, 0x5fb: 0x43ae, + 0x5f6: 0x4888, 0x5f7: 0x3ed0, 0x5f8: 0x363d, 0x5f9: 0x4306, 0x5fa: 0x366d, 0x5fb: 0x4318, 0x5fc: 0x3679, 0x5fd: 0x4256, 0x5fe: 0xa100, // Block 0x18, offset 0x600 0x601: 0x3c06, 0x603: 0xa000, 0x604: 0x3c0d, 0x605: 0xa000, @@ -3519,8 +3525,8 @@ var nfcSparseValues = [688]valueRange{ {value: 0x8100, lo: 0xb8, hi: 0xb8}, // Block 0x1, offset 0x5 {value: 0x0091, lo: 0x03}, - {value: 0x4778, lo: 0xa0, hi: 0xa1}, - {value: 0x47aa, lo: 0xaf, hi: 0xb0}, + {value: 0x46e2, lo: 0xa0, hi: 0xa1}, + {value: 0x4714, lo: 0xaf, hi: 0xb0}, {value: 0xa000, lo: 0xb7, hi: 0xb7}, // Block 0x2, offset 0x9 {value: 0x0000, lo: 0x01}, @@ -3533,11 +3539,11 @@ var nfcSparseValues = [688]valueRange{ {value: 0xa000, lo: 0x81, hi: 0x81}, {value: 0xa000, lo: 0x85, hi: 0x85}, {value: 0xa000, lo: 0x89, hi: 0x89}, - {value: 0x48d6, lo: 0x8a, hi: 0x8a}, - {value: 0x48f4, lo: 0x8b, hi: 0x8b}, + {value: 0x4840, lo: 0x8a, hi: 0x8a}, + {value: 0x485e, lo: 0x8b, hi: 0x8b}, {value: 0x36c7, lo: 0x8c, hi: 0x8c}, {value: 0x36df, lo: 0x8d, hi: 0x8d}, - {value: 0x490c, lo: 0x8e, hi: 0x8e}, + {value: 0x4876, lo: 0x8e, hi: 0x8e}, {value: 0xa000, lo: 0x92, hi: 0x92}, {value: 0x36fd, lo: 0x93, hi: 0x94}, // Block 0x5, offset 0x18 @@ -3665,7 +3671,7 @@ var nfcSparseValues = [688]valueRange{ {value: 0x812d, lo: 0x92, hi: 0x92}, {value: 0x8132, lo: 0x93, hi: 0x93}, {value: 0x8132, lo: 0x94, hi: 0x94}, - {value: 0x45b2, lo: 0x98, hi: 0x9f}, + {value: 0x451c, lo: 0x98, hi: 0x9f}, // Block 0x12, offset 0x89 {value: 0x0000, lo: 0x02}, {value: 0x8102, lo: 0xbc, hi: 0xbc}, @@ -3676,18 +3682,18 @@ var nfcSparseValues = [688]valueRange{ {value: 0x2c9e, lo: 0x8b, hi: 0x8c}, {value: 0x8104, lo: 0x8d, hi: 0x8d}, {value: 0x9900, lo: 0x97, hi: 0x97}, - {value: 0x45f2, lo: 0x9c, hi: 0x9d}, - {value: 0x4602, lo: 0x9f, hi: 0x9f}, + {value: 0x455c, lo: 0x9c, hi: 0x9d}, + {value: 0x456c, lo: 0x9f, hi: 0x9f}, // Block 0x14, offset 0x93 {value: 0x0000, lo: 0x03}, - {value: 0x462a, lo: 0xb3, hi: 0xb3}, - {value: 0x4632, lo: 0xb6, hi: 0xb6}, + {value: 0x4594, lo: 0xb3, hi: 0xb3}, + {value: 0x459c, lo: 0xb6, hi: 0xb6}, {value: 0x8102, lo: 0xbc, hi: 0xbc}, // Block 0x15, offset 0x97 {value: 0x0008, lo: 0x03}, {value: 0x8104, lo: 0x8d, hi: 0x8d}, - {value: 0x460a, lo: 0x99, hi: 0x9b}, - {value: 0x4622, lo: 0x9e, hi: 0x9e}, + {value: 0x4574, lo: 0x99, hi: 0x9b}, + {value: 0x458c, lo: 0x9e, hi: 0x9e}, // Block 0x16, offset 0x9b {value: 0x0000, lo: 0x01}, {value: 0x8102, lo: 0xbc, hi: 0xbc}, @@ -3702,8 +3708,8 @@ var nfcSparseValues = [688]valueRange{ {value: 0x2cbe, lo: 0x8c, hi: 0x8c}, {value: 0x8104, lo: 0x8d, hi: 0x8d}, {value: 0x9900, lo: 0x96, hi: 0x97}, - {value: 0x463a, lo: 0x9c, hi: 0x9c}, - {value: 0x4642, lo: 0x9d, hi: 0x9d}, + {value: 0x45a4, lo: 0x9c, hi: 0x9c}, + {value: 0x45ac, lo: 0x9d, hi: 0x9d}, // Block 0x19, offset 0xa8 {value: 0x0000, lo: 0x03}, {value: 0xa000, lo: 0x92, hi: 0x92}, @@ -3787,18 +3793,18 @@ var nfcSparseValues = [688]valueRange{ {value: 0x263d, lo: 0xa9, hi: 0xa9}, {value: 0x8126, lo: 0xb1, hi: 0xb1}, {value: 0x8127, lo: 0xb2, hi: 0xb2}, - {value: 0x4a66, lo: 0xb3, hi: 0xb3}, + {value: 0x4a84, lo: 0xb3, hi: 0xb3}, {value: 0x8128, lo: 0xb4, hi: 0xb4}, - {value: 0x4a6f, lo: 0xb5, hi: 0xb5}, - {value: 0x464a, lo: 0xb6, hi: 0xb6}, + {value: 0x4a8d, lo: 0xb5, hi: 0xb5}, + {value: 0x45b4, lo: 0xb6, hi: 0xb6}, {value: 0x8200, lo: 0xb7, hi: 0xb7}, - {value: 0x4652, lo: 0xb8, hi: 0xb8}, + {value: 0x45bc, lo: 0xb8, hi: 0xb8}, {value: 0x8200, lo: 0xb9, hi: 0xb9}, {value: 0x8127, lo: 0xba, hi: 0xbd}, // Block 0x27, offset 0xf5 {value: 0x0000, lo: 0x0b}, {value: 0x8127, lo: 0x80, hi: 0x80}, - {value: 0x4a78, lo: 0x81, hi: 0x81}, + {value: 0x4a96, lo: 0x81, hi: 0x81}, {value: 0x8132, lo: 0x82, hi: 0x83}, {value: 0x8104, lo: 0x84, hi: 0x84}, {value: 0x8132, lo: 0x86, hi: 0x87}, @@ -3975,7 +3981,7 @@ var nfcSparseValues = [688]valueRange{ {value: 0x048b, lo: 0xa9, hi: 0xaa}, // Block 0x45, offset 0x189 {value: 0x0000, lo: 0x01}, - {value: 0x4573, lo: 0x9c, hi: 0x9c}, + {value: 0x44dd, lo: 0x9c, hi: 0x9c}, // Block 0x46, offset 0x18b {value: 0x0000, lo: 0x01}, {value: 0x8132, lo: 0xaf, hi: 0xb1}, @@ -3994,12 +4000,12 @@ var nfcSparseValues = [688]valueRange{ {value: 0x812f, lo: 0xae, hi: 0xaf}, // Block 0x4a, offset 0x197 {value: 0x0000, lo: 0x03}, - {value: 0x4a81, lo: 0xb3, hi: 0xb3}, - {value: 0x4a81, lo: 0xb5, hi: 0xb6}, - {value: 0x4a81, lo: 0xba, hi: 0xbf}, + {value: 0x4a9f, lo: 0xb3, hi: 0xb3}, + {value: 0x4a9f, lo: 0xb5, hi: 0xb6}, + {value: 0x4a9f, lo: 0xba, hi: 0xbf}, // Block 0x4b, offset 0x19b {value: 0x0000, lo: 0x01}, - {value: 0x4a81, lo: 0x8f, hi: 0xa3}, + {value: 0x4a9f, lo: 0x8f, hi: 0xa3}, // Block 0x4c, offset 0x19d {value: 0x0000, lo: 0x01}, {value: 0x8100, lo: 0xae, hi: 0xbe}, @@ -4119,29 +4125,29 @@ var nfcSparseValues = [688]valueRange{ {value: 0xc600, lo: 0x89, hi: 0xa3}, // Block 0x63, offset 0x1fb {value: 0x0006, lo: 0x0d}, - {value: 0x4426, lo: 0x9d, hi: 0x9d}, + {value: 0x4390, lo: 0x9d, hi: 0x9d}, {value: 0x8115, lo: 0x9e, hi: 0x9e}, - {value: 0x4498, lo: 0x9f, hi: 0x9f}, - {value: 0x4486, lo: 0xaa, hi: 0xab}, - {value: 0x458a, lo: 0xac, hi: 0xac}, - {value: 0x4592, lo: 0xad, hi: 0xad}, - {value: 0x43de, lo: 0xae, hi: 0xb1}, - {value: 0x43fc, lo: 0xb2, hi: 0xb4}, - {value: 0x4414, lo: 0xb5, hi: 0xb6}, - {value: 0x4420, lo: 0xb8, hi: 0xb8}, - {value: 0x442c, lo: 0xb9, hi: 0xbb}, - {value: 0x4444, lo: 0xbc, hi: 0xbc}, - {value: 0x444a, lo: 0xbe, hi: 0xbe}, + {value: 0x4402, lo: 0x9f, hi: 0x9f}, + {value: 0x43f0, lo: 0xaa, hi: 0xab}, + {value: 0x44f4, lo: 0xac, hi: 0xac}, + {value: 0x44fc, lo: 0xad, hi: 0xad}, + {value: 0x4348, lo: 0xae, hi: 0xb1}, + {value: 0x4366, lo: 0xb2, hi: 0xb4}, + {value: 0x437e, lo: 0xb5, hi: 0xb6}, + {value: 0x438a, lo: 0xb8, hi: 0xb8}, + {value: 0x4396, lo: 0xb9, hi: 0xbb}, + {value: 0x43ae, lo: 0xbc, hi: 0xbc}, + {value: 0x43b4, lo: 0xbe, hi: 0xbe}, // Block 0x64, offset 0x209 {value: 0x0006, lo: 0x08}, - {value: 0x4450, lo: 0x80, hi: 0x81}, - {value: 0x445c, lo: 0x83, hi: 0x84}, - {value: 0x446e, lo: 0x86, hi: 0x89}, - {value: 0x4492, lo: 0x8a, hi: 0x8a}, - {value: 0x440e, lo: 0x8b, hi: 0x8b}, - {value: 0x43f6, lo: 0x8c, hi: 0x8c}, - {value: 0x443e, lo: 0x8d, hi: 0x8d}, - {value: 0x4468, lo: 0x8e, hi: 0x8e}, + {value: 0x43ba, lo: 0x80, hi: 0x81}, + {value: 0x43c6, lo: 0x83, hi: 0x84}, + {value: 0x43d8, lo: 0x86, hi: 0x89}, + {value: 0x43fc, lo: 0x8a, hi: 0x8a}, + {value: 0x4378, lo: 0x8b, hi: 0x8b}, + {value: 0x4360, lo: 0x8c, hi: 0x8c}, + {value: 0x43a8, lo: 0x8d, hi: 0x8d}, + {value: 0x43d2, lo: 0x8e, hi: 0x8e}, // Block 0x65, offset 0x212 {value: 0x0000, lo: 0x02}, {value: 0x8100, lo: 0xa4, hi: 0xa5}, @@ -4179,16 +4185,16 @@ var nfcSparseValues = [688]valueRange{ {value: 0x8100, lo: 0xb5, hi: 0xba}, // Block 0x6e, offset 0x22c {value: 0x0000, lo: 0x04}, - {value: 0x4a81, lo: 0x9e, hi: 0x9f}, - {value: 0x4a81, lo: 0xa3, hi: 0xa3}, - {value: 0x4a81, lo: 0xa5, hi: 0xa6}, - {value: 0x4a81, lo: 0xaa, hi: 0xaf}, + {value: 0x4a9f, lo: 0x9e, hi: 0x9f}, + {value: 0x4a9f, lo: 0xa3, hi: 0xa3}, + {value: 0x4a9f, lo: 0xa5, hi: 0xa6}, + {value: 0x4a9f, lo: 0xaa, hi: 0xaf}, // Block 0x6f, offset 0x231 {value: 0x0000, lo: 0x05}, - {value: 0x4a81, lo: 0x82, hi: 0x87}, - {value: 0x4a81, lo: 0x8a, hi: 0x8f}, - {value: 0x4a81, lo: 0x92, hi: 0x97}, - {value: 0x4a81, lo: 0x9a, hi: 0x9c}, + {value: 0x4a9f, lo: 0x82, hi: 0x87}, + {value: 0x4a9f, lo: 0x8a, hi: 0x8f}, + {value: 0x4a9f, lo: 0x92, hi: 0x97}, + {value: 0x4a9f, lo: 0x9a, hi: 0x9c}, {value: 0x8100, lo: 0xa3, hi: 0xa3}, // Block 0x70, offset 0x237 {value: 0x0000, lo: 0x01}, @@ -4295,13 +4301,13 @@ var nfcSparseValues = [688]valueRange{ {value: 0x8101, lo: 0x9e, hi: 0x9e}, // Block 0x86, offset 0x288 {value: 0x0000, lo: 0x0c}, - {value: 0x4662, lo: 0x9e, hi: 0x9e}, - {value: 0x466c, lo: 0x9f, hi: 0x9f}, - {value: 0x46a0, lo: 0xa0, hi: 0xa0}, - {value: 0x46ae, lo: 0xa1, hi: 0xa1}, - {value: 0x46bc, lo: 0xa2, hi: 0xa2}, - {value: 0x46ca, lo: 0xa3, hi: 0xa3}, - {value: 0x46d8, lo: 0xa4, hi: 0xa4}, + {value: 0x45cc, lo: 0x9e, hi: 0x9e}, + {value: 0x45d6, lo: 0x9f, hi: 0x9f}, + {value: 0x460a, lo: 0xa0, hi: 0xa0}, + {value: 0x4618, lo: 0xa1, hi: 0xa1}, + {value: 0x4626, lo: 0xa2, hi: 0xa2}, + {value: 0x4634, lo: 0xa3, hi: 0xa3}, + {value: 0x4642, lo: 0xa4, hi: 0xa4}, {value: 0x812b, lo: 0xa5, hi: 0xa6}, {value: 0x8101, lo: 0xa7, hi: 0xa9}, {value: 0x8130, lo: 0xad, hi: 0xad}, @@ -4313,14 +4319,14 @@ var nfcSparseValues = [688]valueRange{ {value: 0x8132, lo: 0x85, hi: 0x89}, {value: 0x812d, lo: 0x8a, hi: 0x8b}, {value: 0x8132, lo: 0xaa, hi: 0xad}, - {value: 0x4676, lo: 0xbb, hi: 0xbb}, - {value: 0x4680, lo: 0xbc, hi: 0xbc}, - {value: 0x46e6, lo: 0xbd, hi: 0xbd}, - {value: 0x4702, lo: 0xbe, hi: 0xbe}, - {value: 0x46f4, lo: 0xbf, hi: 0xbf}, + {value: 0x45e0, lo: 0xbb, hi: 0xbb}, + {value: 0x45ea, lo: 0xbc, hi: 0xbc}, + {value: 0x4650, lo: 0xbd, hi: 0xbd}, + {value: 0x466c, lo: 0xbe, hi: 0xbe}, + {value: 0x465e, lo: 0xbf, hi: 0xbf}, // Block 0x88, offset 0x29f {value: 0x0000, lo: 0x01}, - {value: 0x4710, lo: 0x80, hi: 0x80}, + {value: 0x467a, lo: 0x80, hi: 0x80}, // Block 0x89, offset 0x2a1 {value: 0x0000, lo: 0x01}, {value: 0x8132, lo: 0x82, hi: 0x84}, @@ -4513,7 +4519,7 @@ func (t *nfkcTrie) lookupStringUnsafe(s string) uint16 { return 0 } -// nfkcTrie. Total size: 16994 bytes (16.60 KiB). Checksum: 146925fc21092b17. +// nfkcTrie. Total size: 16994 bytes (16.60 KiB). Checksum: c3ed54ee046f3c46. type nfkcTrie struct{} func newNfkcTrie(i int) *nfkcTrie { @@ -4549,22 +4555,22 @@ var nfkcValues = [5888]uint16{ 0x76: 0xa000, 0x77: 0xa000, 0x78: 0xa000, 0x79: 0xa000, 0x7a: 0xa000, // Block 0x2, offset 0x80 // Block 0x3, offset 0xc0 - 0xc0: 0x2f6f, 0xc1: 0x2f74, 0xc2: 0x471e, 0xc3: 0x2f79, 0xc4: 0x472d, 0xc5: 0x4732, - 0xc6: 0xa000, 0xc7: 0x473c, 0xc8: 0x2fe2, 0xc9: 0x2fe7, 0xca: 0x4741, 0xcb: 0x2ffb, - 0xcc: 0x306e, 0xcd: 0x3073, 0xce: 0x3078, 0xcf: 0x4755, 0xd1: 0x3104, - 0xd2: 0x3127, 0xd3: 0x312c, 0xd4: 0x475f, 0xd5: 0x4764, 0xd6: 0x4773, - 0xd8: 0xa000, 0xd9: 0x31b3, 0xda: 0x31b8, 0xdb: 0x31bd, 0xdc: 0x47a5, 0xdd: 0x3235, - 0xe0: 0x327b, 0xe1: 0x3280, 0xe2: 0x47af, 0xe3: 0x3285, - 0xe4: 0x47be, 0xe5: 0x47c3, 0xe6: 0xa000, 0xe7: 0x47cd, 0xe8: 0x32ee, 0xe9: 0x32f3, - 0xea: 0x47d2, 0xeb: 0x3307, 0xec: 0x337f, 0xed: 0x3384, 0xee: 0x3389, 0xef: 0x47e6, - 0xf1: 0x3415, 0xf2: 0x3438, 0xf3: 0x343d, 0xf4: 0x47f0, 0xf5: 0x47f5, - 0xf6: 0x4804, 0xf8: 0xa000, 0xf9: 0x34c9, 0xfa: 0x34ce, 0xfb: 0x34d3, - 0xfc: 0x4836, 0xfd: 0x3550, 0xff: 0x3569, + 0xc0: 0x2f6f, 0xc1: 0x2f74, 0xc2: 0x4688, 0xc3: 0x2f79, 0xc4: 0x4697, 0xc5: 0x469c, + 0xc6: 0xa000, 0xc7: 0x46a6, 0xc8: 0x2fe2, 0xc9: 0x2fe7, 0xca: 0x46ab, 0xcb: 0x2ffb, + 0xcc: 0x306e, 0xcd: 0x3073, 0xce: 0x3078, 0xcf: 0x46bf, 0xd1: 0x3104, + 0xd2: 0x3127, 0xd3: 0x312c, 0xd4: 0x46c9, 0xd5: 0x46ce, 0xd6: 0x46dd, + 0xd8: 0xa000, 0xd9: 0x31b3, 0xda: 0x31b8, 0xdb: 0x31bd, 0xdc: 0x470f, 0xdd: 0x3235, + 0xe0: 0x327b, 0xe1: 0x3280, 0xe2: 0x4719, 0xe3: 0x3285, + 0xe4: 0x4728, 0xe5: 0x472d, 0xe6: 0xa000, 0xe7: 0x4737, 0xe8: 0x32ee, 0xe9: 0x32f3, + 0xea: 0x473c, 0xeb: 0x3307, 0xec: 0x337f, 0xed: 0x3384, 0xee: 0x3389, 0xef: 0x4750, + 0xf1: 0x3415, 0xf2: 0x3438, 0xf3: 0x343d, 0xf4: 0x475a, 0xf5: 0x475f, + 0xf6: 0x476e, 0xf8: 0xa000, 0xf9: 0x34c9, 0xfa: 0x34ce, 0xfb: 0x34d3, + 0xfc: 0x47a0, 0xfd: 0x3550, 0xff: 0x3569, // Block 0x4, offset 0x100 - 0x100: 0x2f7e, 0x101: 0x328a, 0x102: 0x4723, 0x103: 0x47b4, 0x104: 0x2f9c, 0x105: 0x32a8, + 0x100: 0x2f7e, 0x101: 0x328a, 0x102: 0x468d, 0x103: 0x471e, 0x104: 0x2f9c, 0x105: 0x32a8, 0x106: 0x2fb0, 0x107: 0x32bc, 0x108: 0x2fb5, 0x109: 0x32c1, 0x10a: 0x2fba, 0x10b: 0x32c6, 0x10c: 0x2fbf, 0x10d: 0x32cb, 0x10e: 0x2fc9, 0x10f: 0x32d5, - 0x112: 0x4746, 0x113: 0x47d7, 0x114: 0x2ff1, 0x115: 0x32fd, 0x116: 0x2ff6, 0x117: 0x3302, + 0x112: 0x46b0, 0x113: 0x4741, 0x114: 0x2ff1, 0x115: 0x32fd, 0x116: 0x2ff6, 0x117: 0x3302, 0x118: 0x3014, 0x119: 0x3320, 0x11a: 0x3005, 0x11b: 0x3311, 0x11c: 0x302d, 0x11d: 0x3339, 0x11e: 0x3037, 0x11f: 0x3343, 0x120: 0x303c, 0x121: 0x3348, 0x122: 0x3046, 0x123: 0x3352, 0x124: 0x304b, 0x125: 0x3357, 0x128: 0x307d, 0x129: 0x338e, @@ -4575,12 +4581,12 @@ var nfkcValues = [5888]uint16{ // Block 0x5, offset 0x140 0x140: 0x1c34, 0x143: 0x30ff, 0x144: 0x3410, 0x145: 0x3118, 0x146: 0x3429, 0x147: 0x310e, 0x148: 0x341f, 0x149: 0x1c5c, - 0x14c: 0x4769, 0x14d: 0x47fa, 0x14e: 0x3131, 0x14f: 0x3442, 0x150: 0x313b, 0x151: 0x344c, + 0x14c: 0x46d3, 0x14d: 0x4764, 0x14e: 0x3131, 0x14f: 0x3442, 0x150: 0x313b, 0x151: 0x344c, 0x154: 0x3159, 0x155: 0x346a, 0x156: 0x3172, 0x157: 0x3483, - 0x158: 0x3163, 0x159: 0x3474, 0x15a: 0x478c, 0x15b: 0x481d, 0x15c: 0x317c, 0x15d: 0x348d, - 0x15e: 0x318b, 0x15f: 0x349c, 0x160: 0x4791, 0x161: 0x4822, 0x162: 0x31a4, 0x163: 0x34ba, - 0x164: 0x3195, 0x165: 0x34ab, 0x168: 0x479b, 0x169: 0x482c, - 0x16a: 0x47a0, 0x16b: 0x4831, 0x16c: 0x31c2, 0x16d: 0x34d8, 0x16e: 0x31cc, 0x16f: 0x34e2, + 0x158: 0x3163, 0x159: 0x3474, 0x15a: 0x46f6, 0x15b: 0x4787, 0x15c: 0x317c, 0x15d: 0x348d, + 0x15e: 0x318b, 0x15f: 0x349c, 0x160: 0x46fb, 0x161: 0x478c, 0x162: 0x31a4, 0x163: 0x34ba, + 0x164: 0x3195, 0x165: 0x34ab, 0x168: 0x4705, 0x169: 0x4796, + 0x16a: 0x470a, 0x16b: 0x479b, 0x16c: 0x31c2, 0x16d: 0x34d8, 0x16e: 0x31cc, 0x16f: 0x34e2, 0x170: 0x31d1, 0x171: 0x34e7, 0x172: 0x31ef, 0x173: 0x3505, 0x174: 0x3212, 0x175: 0x3528, 0x176: 0x323a, 0x177: 0x3555, 0x178: 0x324e, 0x179: 0x325d, 0x17a: 0x357d, 0x17b: 0x3267, 0x17c: 0x3587, 0x17d: 0x326c, 0x17e: 0x358c, 0x17f: 0x00a7, @@ -4592,7 +4598,7 @@ var nfkcValues = [5888]uint16{ 0x198: 0x3b57, 0x199: 0x39d6, 0x19a: 0x3b65, 0x19b: 0x39c1, 0x19c: 0x3b50, 0x19e: 0x38b0, 0x19f: 0x3a3f, 0x1a0: 0x38a9, 0x1a1: 0x3a38, 0x1a2: 0x35b3, 0x1a3: 0x35c5, 0x1a6: 0x3041, 0x1a7: 0x334d, 0x1a8: 0x30be, 0x1a9: 0x33cf, - 0x1aa: 0x4782, 0x1ab: 0x4813, 0x1ac: 0x3990, 0x1ad: 0x3b1f, 0x1ae: 0x35d7, 0x1af: 0x35dd, + 0x1aa: 0x46ec, 0x1ab: 0x477d, 0x1ac: 0x3990, 0x1ad: 0x3b1f, 0x1ae: 0x35d7, 0x1af: 0x35dd, 0x1b0: 0x33c5, 0x1b1: 0x1942, 0x1b2: 0x1945, 0x1b3: 0x19cf, 0x1b4: 0x3028, 0x1b5: 0x3334, 0x1b8: 0x30fa, 0x1b9: 0x340b, 0x1ba: 0x38b7, 0x1bb: 0x3a46, 0x1bc: 0x35ad, 0x1bd: 0x35bf, 0x1be: 0x35b9, 0x1bf: 0x35cb, @@ -4603,8 +4609,8 @@ var nfkcValues = [5888]uint16{ 0x1d2: 0x316d, 0x1d3: 0x347e, 0x1d4: 0x31db, 0x1d5: 0x34f1, 0x1d6: 0x31e0, 0x1d7: 0x34f6, 0x1d8: 0x3186, 0x1d9: 0x3497, 0x1da: 0x319f, 0x1db: 0x34b5, 0x1de: 0x305a, 0x1df: 0x3366, - 0x1e6: 0x4728, 0x1e7: 0x47b9, 0x1e8: 0x4750, 0x1e9: 0x47e1, - 0x1ea: 0x395f, 0x1eb: 0x3aee, 0x1ec: 0x393c, 0x1ed: 0x3acb, 0x1ee: 0x476e, 0x1ef: 0x47ff, + 0x1e6: 0x4692, 0x1e7: 0x4723, 0x1e8: 0x46ba, 0x1e9: 0x474b, + 0x1ea: 0x395f, 0x1eb: 0x3aee, 0x1ec: 0x393c, 0x1ed: 0x3acb, 0x1ee: 0x46d8, 0x1ef: 0x4769, 0x1f0: 0x3958, 0x1f1: 0x3ae7, 0x1f2: 0x3244, 0x1f3: 0x355f, // Block 0x8, offset 0x200 0x200: 0x9932, 0x201: 0x9932, 0x202: 0x9932, 0x203: 0x9932, 0x204: 0x9932, 0x205: 0x8132, @@ -4619,7 +4625,7 @@ var nfkcValues = [5888]uint16{ 0x236: 0x8101, 0x237: 0x8101, 0x238: 0x9901, 0x239: 0x812d, 0x23a: 0x812d, 0x23b: 0x812d, 0x23c: 0x812d, 0x23d: 0x8132, 0x23e: 0x8132, 0x23f: 0x8132, // Block 0x9, offset 0x240 - 0x240: 0x4a44, 0x241: 0x4a49, 0x242: 0x9932, 0x243: 0x4a4e, 0x244: 0x4a53, 0x245: 0x9936, + 0x240: 0x49ae, 0x241: 0x49b3, 0x242: 0x9932, 0x243: 0x49b8, 0x244: 0x4a71, 0x245: 0x9936, 0x246: 0x8132, 0x247: 0x812d, 0x248: 0x812d, 0x249: 0x812d, 0x24a: 0x8132, 0x24b: 0x8132, 0x24c: 0x8132, 0x24d: 0x812d, 0x24e: 0x812d, 0x250: 0x8132, 0x251: 0x8132, 0x252: 0x8132, 0x253: 0x812d, 0x254: 0x812d, 0x255: 0x812d, 0x256: 0x812d, 0x257: 0x8132, @@ -4631,22 +4637,22 @@ var nfkcValues = [5888]uint16{ 0x27a: 0x42a5, 0x27e: 0x0037, // Block 0xa, offset 0x280 - 0x284: 0x425a, 0x285: 0x4511, + 0x284: 0x425a, 0x285: 0x447b, 0x286: 0x35e9, 0x287: 0x00ce, 0x288: 0x3607, 0x289: 0x3613, 0x28a: 0x3625, 0x28c: 0x3643, 0x28e: 0x3655, 0x28f: 0x3673, 0x290: 0x3e08, 0x291: 0xa000, 0x295: 0xa000, 0x297: 0xa000, 0x299: 0xa000, 0x29f: 0xa000, 0x2a1: 0xa000, 0x2a5: 0xa000, 0x2a9: 0xa000, - 0x2aa: 0x3637, 0x2ab: 0x3667, 0x2ac: 0x4894, 0x2ad: 0x3697, 0x2ae: 0x48be, 0x2af: 0x36a9, + 0x2aa: 0x3637, 0x2ab: 0x3667, 0x2ac: 0x47fe, 0x2ad: 0x3697, 0x2ae: 0x4828, 0x2af: 0x36a9, 0x2b0: 0x3e70, 0x2b1: 0xa000, 0x2b5: 0xa000, 0x2b7: 0xa000, 0x2b9: 0xa000, 0x2bf: 0xa000, // Block 0xb, offset 0x2c0 0x2c1: 0xa000, 0x2c5: 0xa000, - 0x2c9: 0xa000, 0x2ca: 0x48d6, 0x2cb: 0x48f4, - 0x2cc: 0x36c7, 0x2cd: 0x36df, 0x2ce: 0x490c, 0x2d0: 0x01be, 0x2d1: 0x01d0, - 0x2d2: 0x01ac, 0x2d3: 0x43a2, 0x2d4: 0x43a8, 0x2d5: 0x01fa, 0x2d6: 0x01e8, + 0x2c9: 0xa000, 0x2ca: 0x4840, 0x2cb: 0x485e, + 0x2cc: 0x36c7, 0x2cd: 0x36df, 0x2ce: 0x4876, 0x2d0: 0x01be, 0x2d1: 0x01d0, + 0x2d2: 0x01ac, 0x2d3: 0x430c, 0x2d4: 0x4312, 0x2d5: 0x01fa, 0x2d6: 0x01e8, 0x2f0: 0x01d6, 0x2f1: 0x01eb, 0x2f2: 0x01ee, 0x2f4: 0x0188, 0x2f5: 0x01c7, 0x2f9: 0x01a6, // Block 0xc, offset 0x300 @@ -4726,15 +4732,15 @@ var nfkcValues = [5888]uint16{ 0x4e4: 0x305f, 0x4e5: 0x336b, 0x4e6: 0x3055, 0x4e7: 0x3361, 0x4e8: 0x3064, 0x4e9: 0x3370, 0x4ea: 0x3069, 0x4eb: 0x3375, 0x4ec: 0x30af, 0x4ed: 0x33bb, 0x4ee: 0x390b, 0x4ef: 0x3a9a, 0x4f0: 0x30b9, 0x4f1: 0x33ca, 0x4f2: 0x30c3, 0x4f3: 0x33d4, 0x4f4: 0x30cd, 0x4f5: 0x33de, - 0x4f6: 0x475a, 0x4f7: 0x47eb, 0x4f8: 0x3912, 0x4f9: 0x3aa1, 0x4fa: 0x30e6, 0x4fb: 0x33f7, + 0x4f6: 0x46c4, 0x4f7: 0x4755, 0x4f8: 0x3912, 0x4f9: 0x3aa1, 0x4fa: 0x30e6, 0x4fb: 0x33f7, 0x4fc: 0x30e1, 0x4fd: 0x33f2, 0x4fe: 0x30eb, 0x4ff: 0x33fc, // Block 0x14, offset 0x500 0x500: 0x30f0, 0x501: 0x3401, 0x502: 0x30f5, 0x503: 0x3406, 0x504: 0x3109, 0x505: 0x341a, 0x506: 0x3113, 0x507: 0x3424, 0x508: 0x3122, 0x509: 0x3433, 0x50a: 0x311d, 0x50b: 0x342e, 0x50c: 0x3935, 0x50d: 0x3ac4, 0x50e: 0x3943, 0x50f: 0x3ad2, 0x510: 0x394a, 0x511: 0x3ad9, 0x512: 0x3951, 0x513: 0x3ae0, 0x514: 0x314f, 0x515: 0x3460, 0x516: 0x3154, 0x517: 0x3465, - 0x518: 0x315e, 0x519: 0x346f, 0x51a: 0x4787, 0x51b: 0x4818, 0x51c: 0x3997, 0x51d: 0x3b26, - 0x51e: 0x3177, 0x51f: 0x3488, 0x520: 0x3181, 0x521: 0x3492, 0x522: 0x4796, 0x523: 0x4827, + 0x518: 0x315e, 0x519: 0x346f, 0x51a: 0x46f1, 0x51b: 0x4782, 0x51c: 0x3997, 0x51d: 0x3b26, + 0x51e: 0x3177, 0x51f: 0x3488, 0x520: 0x3181, 0x521: 0x3492, 0x522: 0x4700, 0x523: 0x4791, 0x524: 0x399e, 0x525: 0x3b2d, 0x526: 0x39a5, 0x527: 0x3b34, 0x528: 0x39ac, 0x529: 0x3b3b, 0x52a: 0x3190, 0x52b: 0x34a1, 0x52c: 0x319a, 0x52d: 0x34b0, 0x52e: 0x31ae, 0x52f: 0x34c4, 0x530: 0x31a9, 0x531: 0x34bf, 0x532: 0x31ea, 0x533: 0x3500, 0x534: 0x31f9, 0x535: 0x350f, @@ -4746,16 +4752,16 @@ var nfkcValues = [5888]uint16{ 0x54c: 0x322b, 0x54d: 0x3546, 0x54e: 0x3249, 0x54f: 0x3564, 0x550: 0x3262, 0x551: 0x3582, 0x552: 0x3271, 0x553: 0x3591, 0x554: 0x3276, 0x555: 0x3596, 0x556: 0x337a, 0x557: 0x34a6, 0x558: 0x3537, 0x559: 0x3573, 0x55a: 0x1be0, 0x55b: 0x42d7, - 0x560: 0x4737, 0x561: 0x47c8, 0x562: 0x2f83, 0x563: 0x328f, + 0x560: 0x46a1, 0x561: 0x4732, 0x562: 0x2f83, 0x563: 0x328f, 0x564: 0x3878, 0x565: 0x3a07, 0x566: 0x3871, 0x567: 0x3a00, 0x568: 0x3886, 0x569: 0x3a15, 0x56a: 0x387f, 0x56b: 0x3a0e, 0x56c: 0x38be, 0x56d: 0x3a4d, 0x56e: 0x3894, 0x56f: 0x3a23, 0x570: 0x388d, 0x571: 0x3a1c, 0x572: 0x38a2, 0x573: 0x3a31, 0x574: 0x389b, 0x575: 0x3a2a, - 0x576: 0x38c5, 0x577: 0x3a54, 0x578: 0x474b, 0x579: 0x47dc, 0x57a: 0x3000, 0x57b: 0x330c, + 0x576: 0x38c5, 0x577: 0x3a54, 0x578: 0x46b5, 0x579: 0x4746, 0x57a: 0x3000, 0x57b: 0x330c, 0x57c: 0x2fec, 0x57d: 0x32f8, 0x57e: 0x38da, 0x57f: 0x3a69, // Block 0x16, offset 0x580 0x580: 0x38d3, 0x581: 0x3a62, 0x582: 0x38e8, 0x583: 0x3a77, 0x584: 0x38e1, 0x585: 0x3a70, 0x586: 0x38fd, 0x587: 0x3a8c, 0x588: 0x3091, 0x589: 0x339d, 0x58a: 0x30a5, 0x58b: 0x33b1, - 0x58c: 0x477d, 0x58d: 0x480e, 0x58e: 0x3136, 0x58f: 0x3447, 0x590: 0x3920, 0x591: 0x3aaf, + 0x58c: 0x46e7, 0x58d: 0x4778, 0x58e: 0x3136, 0x58f: 0x3447, 0x590: 0x3920, 0x591: 0x3aaf, 0x592: 0x3919, 0x593: 0x3aa8, 0x594: 0x392e, 0x595: 0x3abd, 0x596: 0x3927, 0x597: 0x3ab6, 0x598: 0x3989, 0x599: 0x3b18, 0x59a: 0x396d, 0x59b: 0x3afc, 0x59c: 0x3966, 0x59d: 0x3af5, 0x59e: 0x397b, 0x59f: 0x3b0a, 0x5a0: 0x3974, 0x5a1: 0x3b03, 0x5a2: 0x3982, 0x5a3: 0x3b11, @@ -4764,29 +4770,29 @@ var nfkcValues = [5888]uint16{ 0x5b0: 0x39f9, 0x5b1: 0x3b88, 0x5b2: 0x3230, 0x5b3: 0x354b, 0x5b4: 0x3258, 0x5b5: 0x3578, 0x5b6: 0x3253, 0x5b7: 0x356e, 0x5b8: 0x323f, 0x5b9: 0x355a, // Block 0x17, offset 0x5c0 - 0x5c0: 0x489a, 0x5c1: 0x48a0, 0x5c2: 0x49b4, 0x5c3: 0x49cc, 0x5c4: 0x49bc, 0x5c5: 0x49d4, - 0x5c6: 0x49c4, 0x5c7: 0x49dc, 0x5c8: 0x4840, 0x5c9: 0x4846, 0x5ca: 0x4924, 0x5cb: 0x493c, - 0x5cc: 0x492c, 0x5cd: 0x4944, 0x5ce: 0x4934, 0x5cf: 0x494c, 0x5d0: 0x48ac, 0x5d1: 0x48b2, + 0x5c0: 0x4804, 0x5c1: 0x480a, 0x5c2: 0x491e, 0x5c3: 0x4936, 0x5c4: 0x4926, 0x5c5: 0x493e, + 0x5c6: 0x492e, 0x5c7: 0x4946, 0x5c8: 0x47aa, 0x5c9: 0x47b0, 0x5ca: 0x488e, 0x5cb: 0x48a6, + 0x5cc: 0x4896, 0x5cd: 0x48ae, 0x5ce: 0x489e, 0x5cf: 0x48b6, 0x5d0: 0x4816, 0x5d1: 0x481c, 0x5d2: 0x3db8, 0x5d3: 0x3dc8, 0x5d4: 0x3dc0, 0x5d5: 0x3dd0, - 0x5d8: 0x484c, 0x5d9: 0x4852, 0x5da: 0x3ce8, 0x5db: 0x3cf8, 0x5dc: 0x3cf0, 0x5dd: 0x3d00, - 0x5e0: 0x48c4, 0x5e1: 0x48ca, 0x5e2: 0x49e4, 0x5e3: 0x49fc, - 0x5e4: 0x49ec, 0x5e5: 0x4a04, 0x5e6: 0x49f4, 0x5e7: 0x4a0c, 0x5e8: 0x4858, 0x5e9: 0x485e, - 0x5ea: 0x4954, 0x5eb: 0x496c, 0x5ec: 0x495c, 0x5ed: 0x4974, 0x5ee: 0x4964, 0x5ef: 0x497c, - 0x5f0: 0x48dc, 0x5f1: 0x48e2, 0x5f2: 0x3e18, 0x5f3: 0x3e30, 0x5f4: 0x3e20, 0x5f5: 0x3e38, - 0x5f6: 0x3e28, 0x5f7: 0x3e40, 0x5f8: 0x4864, 0x5f9: 0x486a, 0x5fa: 0x3d18, 0x5fb: 0x3d30, + 0x5d8: 0x47b6, 0x5d9: 0x47bc, 0x5da: 0x3ce8, 0x5db: 0x3cf8, 0x5dc: 0x3cf0, 0x5dd: 0x3d00, + 0x5e0: 0x482e, 0x5e1: 0x4834, 0x5e2: 0x494e, 0x5e3: 0x4966, + 0x5e4: 0x4956, 0x5e5: 0x496e, 0x5e6: 0x495e, 0x5e7: 0x4976, 0x5e8: 0x47c2, 0x5e9: 0x47c8, + 0x5ea: 0x48be, 0x5eb: 0x48d6, 0x5ec: 0x48c6, 0x5ed: 0x48de, 0x5ee: 0x48ce, 0x5ef: 0x48e6, + 0x5f0: 0x4846, 0x5f1: 0x484c, 0x5f2: 0x3e18, 0x5f3: 0x3e30, 0x5f4: 0x3e20, 0x5f5: 0x3e38, + 0x5f6: 0x3e28, 0x5f7: 0x3e40, 0x5f8: 0x47ce, 0x5f9: 0x47d4, 0x5fa: 0x3d18, 0x5fb: 0x3d30, 0x5fc: 0x3d20, 0x5fd: 0x3d38, 0x5fe: 0x3d28, 0x5ff: 0x3d40, // Block 0x18, offset 0x600 - 0x600: 0x48e8, 0x601: 0x48ee, 0x602: 0x3e48, 0x603: 0x3e58, 0x604: 0x3e50, 0x605: 0x3e60, - 0x608: 0x4870, 0x609: 0x4876, 0x60a: 0x3d48, 0x60b: 0x3d58, - 0x60c: 0x3d50, 0x60d: 0x3d60, 0x610: 0x48fa, 0x611: 0x4900, + 0x600: 0x4852, 0x601: 0x4858, 0x602: 0x3e48, 0x603: 0x3e58, 0x604: 0x3e50, 0x605: 0x3e60, + 0x608: 0x47da, 0x609: 0x47e0, 0x60a: 0x3d48, 0x60b: 0x3d58, + 0x60c: 0x3d50, 0x60d: 0x3d60, 0x610: 0x4864, 0x611: 0x486a, 0x612: 0x3e80, 0x613: 0x3e98, 0x614: 0x3e88, 0x615: 0x3ea0, 0x616: 0x3e90, 0x617: 0x3ea8, - 0x619: 0x487c, 0x61b: 0x3d68, 0x61d: 0x3d70, - 0x61f: 0x3d78, 0x620: 0x4912, 0x621: 0x4918, 0x622: 0x4a14, 0x623: 0x4a2c, - 0x624: 0x4a1c, 0x625: 0x4a34, 0x626: 0x4a24, 0x627: 0x4a3c, 0x628: 0x4882, 0x629: 0x4888, - 0x62a: 0x4984, 0x62b: 0x499c, 0x62c: 0x498c, 0x62d: 0x49a4, 0x62e: 0x4994, 0x62f: 0x49ac, - 0x630: 0x488e, 0x631: 0x43b4, 0x632: 0x3691, 0x633: 0x43ba, 0x634: 0x48b8, 0x635: 0x43c0, - 0x636: 0x36a3, 0x637: 0x43c6, 0x638: 0x36c1, 0x639: 0x43cc, 0x63a: 0x36d9, 0x63b: 0x43d2, - 0x63c: 0x4906, 0x63d: 0x43d8, + 0x619: 0x47e6, 0x61b: 0x3d68, 0x61d: 0x3d70, + 0x61f: 0x3d78, 0x620: 0x487c, 0x621: 0x4882, 0x622: 0x497e, 0x623: 0x4996, + 0x624: 0x4986, 0x625: 0x499e, 0x626: 0x498e, 0x627: 0x49a6, 0x628: 0x47ec, 0x629: 0x47f2, + 0x62a: 0x48ee, 0x62b: 0x4906, 0x62c: 0x48f6, 0x62d: 0x490e, 0x62e: 0x48fe, 0x62f: 0x4916, + 0x630: 0x47f8, 0x631: 0x431e, 0x632: 0x3691, 0x633: 0x4324, 0x634: 0x4822, 0x635: 0x432a, + 0x636: 0x36a3, 0x637: 0x4330, 0x638: 0x36c1, 0x639: 0x4336, 0x63a: 0x36d9, 0x63b: 0x433c, + 0x63c: 0x4870, 0x63d: 0x4342, // Block 0x19, offset 0x640 0x640: 0x3da0, 0x641: 0x3da8, 0x642: 0x4184, 0x643: 0x41a2, 0x644: 0x418e, 0x645: 0x41ac, 0x646: 0x4198, 0x647: 0x41b6, 0x648: 0x3cd8, 0x649: 0x3ce0, 0x64a: 0x40d0, 0x64b: 0x40ee, @@ -4797,19 +4803,19 @@ var nfkcValues = [5888]uint16{ 0x664: 0x4206, 0x665: 0x4224, 0x666: 0x4210, 0x667: 0x422e, 0x668: 0x3d80, 0x669: 0x3d88, 0x66a: 0x4148, 0x66b: 0x4166, 0x66c: 0x4152, 0x66d: 0x4170, 0x66e: 0x415c, 0x66f: 0x417a, 0x670: 0x3685, 0x671: 0x367f, 0x672: 0x3d90, 0x673: 0x368b, 0x674: 0x3d98, - 0x676: 0x48a6, 0x677: 0x3db0, 0x678: 0x35f5, 0x679: 0x35ef, 0x67a: 0x35e3, 0x67b: 0x4384, + 0x676: 0x4810, 0x677: 0x3db0, 0x678: 0x35f5, 0x679: 0x35ef, 0x67a: 0x35e3, 0x67b: 0x42ee, 0x67c: 0x35fb, 0x67d: 0x4287, 0x67e: 0x01d3, 0x67f: 0x4287, // Block 0x1a, offset 0x680 - 0x680: 0x42a0, 0x681: 0x4518, 0x682: 0x3dd8, 0x683: 0x369d, 0x684: 0x3de0, - 0x686: 0x48d0, 0x687: 0x3df8, 0x688: 0x3601, 0x689: 0x438a, 0x68a: 0x360d, 0x68b: 0x4390, - 0x68c: 0x3619, 0x68d: 0x451f, 0x68e: 0x4526, 0x68f: 0x452d, 0x690: 0x36b5, 0x691: 0x36af, - 0x692: 0x3e00, 0x693: 0x457a, 0x696: 0x36bb, 0x697: 0x3e10, - 0x698: 0x3631, 0x699: 0x362b, 0x69a: 0x361f, 0x69b: 0x4396, 0x69d: 0x4534, - 0x69e: 0x453b, 0x69f: 0x4542, 0x6a0: 0x36eb, 0x6a1: 0x36e5, 0x6a2: 0x3e68, 0x6a3: 0x4582, + 0x680: 0x42a0, 0x681: 0x4482, 0x682: 0x3dd8, 0x683: 0x369d, 0x684: 0x3de0, + 0x686: 0x483a, 0x687: 0x3df8, 0x688: 0x3601, 0x689: 0x42f4, 0x68a: 0x360d, 0x68b: 0x42fa, + 0x68c: 0x3619, 0x68d: 0x4489, 0x68e: 0x4490, 0x68f: 0x4497, 0x690: 0x36b5, 0x691: 0x36af, + 0x692: 0x3e00, 0x693: 0x44e4, 0x696: 0x36bb, 0x697: 0x3e10, + 0x698: 0x3631, 0x699: 0x362b, 0x69a: 0x361f, 0x69b: 0x4300, 0x69d: 0x449e, + 0x69e: 0x44a5, 0x69f: 0x44ac, 0x6a0: 0x36eb, 0x6a1: 0x36e5, 0x6a2: 0x3e68, 0x6a3: 0x44ec, 0x6a4: 0x36cd, 0x6a5: 0x36d3, 0x6a6: 0x36f1, 0x6a7: 0x3e78, 0x6a8: 0x3661, 0x6a9: 0x365b, - 0x6aa: 0x364f, 0x6ab: 0x43a2, 0x6ac: 0x3649, 0x6ad: 0x450a, 0x6ae: 0x4511, 0x6af: 0x0081, + 0x6aa: 0x364f, 0x6ab: 0x430c, 0x6ac: 0x3649, 0x6ad: 0x4474, 0x6ae: 0x447b, 0x6af: 0x0081, 0x6b2: 0x3eb0, 0x6b3: 0x36f7, 0x6b4: 0x3eb8, - 0x6b6: 0x491e, 0x6b7: 0x3ed0, 0x6b8: 0x363d, 0x6b9: 0x439c, 0x6ba: 0x366d, 0x6bb: 0x43ae, + 0x6b6: 0x4888, 0x6b7: 0x3ed0, 0x6b8: 0x363d, 0x6b9: 0x4306, 0x6ba: 0x366d, 0x6bb: 0x4318, 0x6bc: 0x3679, 0x6bd: 0x425a, 0x6be: 0x428c, // Block 0x1b, offset 0x6c0 0x6c0: 0x1bd8, 0x6c1: 0x1bdc, 0x6c2: 0x0047, 0x6c3: 0x1c54, 0x6c5: 0x1be8, @@ -4922,7 +4928,7 @@ var nfkcValues = [5888]uint16{ 0x93c: 0x3fc0, 0x93d: 0x3fc8, // Block 0x25, offset 0x940 0x954: 0x3f00, - 0x959: 0x9903, 0x95a: 0x9903, 0x95b: 0x4372, 0x95c: 0x4378, 0x95d: 0xa000, + 0x959: 0x9903, 0x95a: 0x9903, 0x95b: 0x42dc, 0x95c: 0x42e2, 0x95d: 0xa000, 0x95e: 0x3fd0, 0x95f: 0x26b4, 0x966: 0xa000, 0x96b: 0xa000, 0x96c: 0x3fe0, 0x96d: 0xa000, 0x96e: 0x3fe8, 0x96f: 0xa000, @@ -4942,10 +4948,10 @@ var nfkcValues = [5888]uint16{ // Block 0x27, offset 0x9c0 0x9c0: 0x0367, 0x9c1: 0x032b, 0x9c2: 0x032f, 0x9c3: 0x0333, 0x9c4: 0x037b, 0x9c5: 0x0337, 0x9c6: 0x033b, 0x9c7: 0x033f, 0x9c8: 0x0343, 0x9c9: 0x0347, 0x9ca: 0x034b, 0x9cb: 0x034f, - 0x9cc: 0x0353, 0x9cd: 0x0357, 0x9ce: 0x035b, 0x9cf: 0x42dc, 0x9d0: 0x42e1, 0x9d1: 0x42e6, - 0x9d2: 0x42eb, 0x9d3: 0x42f0, 0x9d4: 0x42f5, 0x9d5: 0x42fa, 0x9d6: 0x42ff, 0x9d7: 0x4304, - 0x9d8: 0x4309, 0x9d9: 0x430e, 0x9da: 0x4313, 0x9db: 0x4318, 0x9dc: 0x431d, 0x9dd: 0x4322, - 0x9de: 0x4327, 0x9df: 0x432c, 0x9e0: 0x4331, 0x9e1: 0x4336, 0x9e2: 0x433b, 0x9e3: 0x4340, + 0x9cc: 0x0353, 0x9cd: 0x0357, 0x9ce: 0x035b, 0x9cf: 0x49bd, 0x9d0: 0x49c3, 0x9d1: 0x49c9, + 0x9d2: 0x49cf, 0x9d3: 0x49d5, 0x9d4: 0x49db, 0x9d5: 0x49e1, 0x9d6: 0x49e7, 0x9d7: 0x49ed, + 0x9d8: 0x49f3, 0x9d9: 0x49f9, 0x9da: 0x49ff, 0x9db: 0x4a05, 0x9dc: 0x4a0b, 0x9dd: 0x4a11, + 0x9de: 0x4a17, 0x9df: 0x4a1d, 0x9e0: 0x4a23, 0x9e1: 0x4a29, 0x9e2: 0x4a2f, 0x9e3: 0x4a35, 0x9e4: 0x03c3, 0x9e5: 0x035f, 0x9e6: 0x0363, 0x9e7: 0x03e7, 0x9e8: 0x03eb, 0x9e9: 0x03ef, 0x9ea: 0x03f3, 0x9eb: 0x03f7, 0x9ec: 0x03fb, 0x9ed: 0x03ff, 0x9ee: 0x036b, 0x9ef: 0x0403, 0x9f0: 0x0407, 0x9f1: 0x036f, 0x9f2: 0x0373, 0x9f3: 0x0377, 0x9f4: 0x037f, 0x9f5: 0x0383, @@ -5148,17 +5154,17 @@ var nfkcValues = [5888]uint16{ 0xe40: 0x19d5, 0xe41: 0x19d8, 0xe42: 0x19db, 0xe43: 0x1c08, 0xe44: 0x1c0c, 0xe45: 0x1a5f, 0xe46: 0x1a5f, 0xe53: 0x1d75, 0xe54: 0x1d66, 0xe55: 0x1d6b, 0xe56: 0x1d7a, 0xe57: 0x1d70, - 0xe5d: 0x4426, - 0xe5e: 0x8115, 0xe5f: 0x4498, 0xe60: 0x022d, 0xe61: 0x0215, 0xe62: 0x021e, 0xe63: 0x0221, + 0xe5d: 0x4390, + 0xe5e: 0x8115, 0xe5f: 0x4402, 0xe60: 0x022d, 0xe61: 0x0215, 0xe62: 0x021e, 0xe63: 0x0221, 0xe64: 0x0224, 0xe65: 0x0227, 0xe66: 0x022a, 0xe67: 0x0230, 0xe68: 0x0233, 0xe69: 0x0017, - 0xe6a: 0x4486, 0xe6b: 0x448c, 0xe6c: 0x458a, 0xe6d: 0x4592, 0xe6e: 0x43de, 0xe6f: 0x43e4, - 0xe70: 0x43ea, 0xe71: 0x43f0, 0xe72: 0x43fc, 0xe73: 0x4402, 0xe74: 0x4408, 0xe75: 0x4414, - 0xe76: 0x441a, 0xe78: 0x4420, 0xe79: 0x442c, 0xe7a: 0x4432, 0xe7b: 0x4438, - 0xe7c: 0x4444, 0xe7e: 0x444a, + 0xe6a: 0x43f0, 0xe6b: 0x43f6, 0xe6c: 0x44f4, 0xe6d: 0x44fc, 0xe6e: 0x4348, 0xe6f: 0x434e, + 0xe70: 0x4354, 0xe71: 0x435a, 0xe72: 0x4366, 0xe73: 0x436c, 0xe74: 0x4372, 0xe75: 0x437e, + 0xe76: 0x4384, 0xe78: 0x438a, 0xe79: 0x4396, 0xe7a: 0x439c, 0xe7b: 0x43a2, + 0xe7c: 0x43ae, 0xe7e: 0x43b4, // Block 0x3a, offset 0xe80 - 0xe80: 0x4450, 0xe81: 0x4456, 0xe83: 0x445c, 0xe84: 0x4462, - 0xe86: 0x446e, 0xe87: 0x4474, 0xe88: 0x447a, 0xe89: 0x4480, 0xe8a: 0x4492, 0xe8b: 0x440e, - 0xe8c: 0x43f6, 0xe8d: 0x443e, 0xe8e: 0x4468, 0xe8f: 0x1d7f, 0xe90: 0x0299, 0xe91: 0x0299, + 0xe80: 0x43ba, 0xe81: 0x43c0, 0xe83: 0x43c6, 0xe84: 0x43cc, + 0xe86: 0x43d8, 0xe87: 0x43de, 0xe88: 0x43e4, 0xe89: 0x43ea, 0xe8a: 0x43fc, 0xe8b: 0x4378, + 0xe8c: 0x4360, 0xe8d: 0x43a8, 0xe8e: 0x43d2, 0xe8f: 0x1d7f, 0xe90: 0x0299, 0xe91: 0x0299, 0xe92: 0x02a2, 0xe93: 0x02a2, 0xe94: 0x02a2, 0xe95: 0x02a2, 0xe96: 0x02a5, 0xe97: 0x02a5, 0xe98: 0x02a5, 0xe99: 0x02a5, 0xe9a: 0x02ab, 0xe9b: 0x02ab, 0xe9c: 0x02ab, 0xe9d: 0x02ab, 0xe9e: 0x029f, 0xe9f: 0x029f, 0xea0: 0x029f, 0xea1: 0x029f, 0xea2: 0x02a8, 0xea3: 0x02a8, @@ -5174,9 +5180,9 @@ var nfkcValues = [5888]uint16{ 0xed2: 0x02db, 0xed3: 0x02db, 0xed4: 0x02db, 0xed5: 0x02db, 0xed6: 0x02e1, 0xed7: 0x02e1, 0xed8: 0x02e1, 0xed9: 0x02e1, 0xeda: 0x02de, 0xedb: 0x02de, 0xedc: 0x02de, 0xedd: 0x02de, 0xede: 0x02e4, 0xedf: 0x02e4, 0xee0: 0x02e7, 0xee1: 0x02e7, 0xee2: 0x02e7, 0xee3: 0x02e7, - 0xee4: 0x4504, 0xee5: 0x4504, 0xee6: 0x02ed, 0xee7: 0x02ed, 0xee8: 0x02ed, 0xee9: 0x02ed, + 0xee4: 0x446e, 0xee5: 0x446e, 0xee6: 0x02ed, 0xee7: 0x02ed, 0xee8: 0x02ed, 0xee9: 0x02ed, 0xeea: 0x02ea, 0xeeb: 0x02ea, 0xeec: 0x02ea, 0xeed: 0x02ea, 0xeee: 0x0308, 0xeef: 0x0308, - 0xef0: 0x44fe, 0xef1: 0x44fe, + 0xef0: 0x4468, 0xef1: 0x4468, // Block 0x3c, offset 0xf00 0xf13: 0x02d8, 0xf14: 0x02d8, 0xf15: 0x02d8, 0xf16: 0x02d8, 0xf17: 0x02f6, 0xf18: 0x02f6, 0xf19: 0x02f3, 0xf1a: 0x02f3, 0xf1b: 0x02f9, 0xf1c: 0x02f9, 0xf1d: 0x204f, @@ -5203,8 +5209,8 @@ var nfkcValues = [5888]uint16{ 0xf86: 0x1fb4, 0xf87: 0x1fb9, 0xf88: 0x1fbe, 0xf89: 0x1fc3, 0xf8a: 0x1fc8, 0xf8b: 0x1fcd, 0xf8c: 0x1fd2, 0xf8d: 0x1fd7, 0xf8e: 0x1fe6, 0xf8f: 0x1ff5, 0xf90: 0x1ffa, 0xf91: 0x1fff, 0xf92: 0x2004, 0xf93: 0x2009, 0xf94: 0x200e, 0xf95: 0x2018, 0xf96: 0x201d, 0xf97: 0x2022, - 0xf98: 0x2031, 0xf99: 0x2040, 0xf9a: 0x2045, 0xf9b: 0x44b6, 0xf9c: 0x44bc, 0xf9d: 0x44f2, - 0xf9e: 0x4549, 0xf9f: 0x4550, 0xfa0: 0x4557, 0xfa1: 0x455e, 0xfa2: 0x4565, 0xfa3: 0x456c, + 0xf98: 0x2031, 0xf99: 0x2040, 0xf9a: 0x2045, 0xf9b: 0x4420, 0xf9c: 0x4426, 0xf9d: 0x445c, + 0xf9e: 0x44b3, 0xf9f: 0x44ba, 0xfa0: 0x44c1, 0xfa1: 0x44c8, 0xfa2: 0x44cf, 0xfa3: 0x44d6, 0xfa4: 0x25c6, 0xfa5: 0x25cd, 0xfa6: 0x25d4, 0xfa7: 0x25db, 0xfa8: 0x25f0, 0xfa9: 0x25f7, 0xfaa: 0x1d98, 0xfab: 0x1d9d, 0xfac: 0x1da2, 0xfad: 0x1da7, 0xfae: 0x1db1, 0xfaf: 0x1db6, 0xfb0: 0x1dca, 0xfb1: 0x1dcf, 0xfb2: 0x1dd4, 0xfb3: 0x1dd9, 0xfb4: 0x1de3, 0xfb5: 0x1de8, @@ -5213,7 +5219,7 @@ var nfkcValues = [5888]uint16{ // Block 0x3f, offset 0xfc0 0xfc0: 0x1f5a, 0xfc1: 0x1f6e, 0xfc2: 0x1f73, 0xfc3: 0x1f78, 0xfc4: 0x1f7d, 0xfc5: 0x1f96, 0xfc6: 0x1fa0, 0xfc7: 0x1fa5, 0xfc8: 0x1faa, 0xfc9: 0x1fbe, 0xfca: 0x1fdc, 0xfcb: 0x1fe1, - 0xfcc: 0x1fe6, 0xfcd: 0x1feb, 0xfce: 0x1ff5, 0xfcf: 0x1ffa, 0xfd0: 0x44f2, 0xfd1: 0x2027, + 0xfcc: 0x1fe6, 0xfcd: 0x1feb, 0xfce: 0x1ff5, 0xfcf: 0x1ffa, 0xfd0: 0x445c, 0xfd1: 0x2027, 0xfd2: 0x202c, 0xfd3: 0x2031, 0xfd4: 0x2036, 0xfd5: 0x2040, 0xfd6: 0x2045, 0xfd7: 0x25b1, 0xfd8: 0x25b8, 0xfd9: 0x25bf, 0xfda: 0x25d4, 0xfdb: 0x25e2, 0xfdc: 0x1d89, 0xfdd: 0x1d8e, 0xfde: 0x1d93, 0xfdf: 0x1da2, 0xfe0: 0x1dac, 0xfe1: 0x1dbb, 0xfe2: 0x1dc0, 0xfe3: 0x1dc5, @@ -5227,11 +5233,11 @@ var nfkcValues = [5888]uint16{ 0x1006: 0x1f69, 0x1007: 0x1f6e, 0x1008: 0x1f73, 0x1009: 0x1f87, 0x100a: 0x1f8c, 0x100b: 0x1f91, 0x100c: 0x1f96, 0x100d: 0x1f9b, 0x100e: 0x1faf, 0x100f: 0x1fb4, 0x1010: 0x1fb9, 0x1011: 0x1fbe, 0x1012: 0x1fcd, 0x1013: 0x1fd2, 0x1014: 0x1fd7, 0x1015: 0x1fe6, 0x1016: 0x1ff0, 0x1017: 0x1fff, - 0x1018: 0x2004, 0x1019: 0x44e6, 0x101a: 0x2018, 0x101b: 0x201d, 0x101c: 0x2022, 0x101d: 0x2031, + 0x1018: 0x2004, 0x1019: 0x4450, 0x101a: 0x2018, 0x101b: 0x201d, 0x101c: 0x2022, 0x101d: 0x2031, 0x101e: 0x203b, 0x101f: 0x25d4, 0x1020: 0x25e2, 0x1021: 0x1da2, 0x1022: 0x1dac, 0x1023: 0x1dd4, 0x1024: 0x1dde, 0x1025: 0x1dfc, 0x1026: 0x1e06, 0x1027: 0x1e6a, 0x1028: 0x1e6f, 0x1029: 0x1e92, 0x102a: 0x1e97, 0x102b: 0x1f6e, 0x102c: 0x1f73, 0x102d: 0x1f96, 0x102e: 0x1fe6, 0x102f: 0x1ff0, - 0x1030: 0x2031, 0x1031: 0x203b, 0x1032: 0x459a, 0x1033: 0x45a2, 0x1034: 0x45aa, 0x1035: 0x1ef1, + 0x1030: 0x2031, 0x1031: 0x203b, 0x1032: 0x4504, 0x1033: 0x450c, 0x1034: 0x4514, 0x1035: 0x1ef1, 0x1036: 0x1ef6, 0x1037: 0x1f0a, 0x1038: 0x1f0f, 0x1039: 0x1f1e, 0x103a: 0x1f23, 0x103b: 0x1e74, 0x103c: 0x1e79, 0x103d: 0x1e9c, 0x103e: 0x1ea1, 0x103f: 0x1e33, // Block 0x41, offset 0x1040 @@ -5245,7 +5251,7 @@ var nfkcValues = [5888]uint16{ 0x106a: 0x1e65, 0x106b: 0x1eb0, 0x106c: 0x1ed3, 0x106d: 0x1e7e, 0x106e: 0x1e83, 0x106f: 0x1e88, 0x1070: 0x1e92, 0x1071: 0x1e6f, 0x1072: 0x1e97, 0x1073: 0x1eec, 0x1074: 0x1e56, 0x1075: 0x1e5b, 0x1076: 0x1e60, 0x1077: 0x1e7e, 0x1078: 0x1e83, 0x1079: 0x1e88, 0x107a: 0x1eec, 0x107b: 0x1efb, - 0x107c: 0x449e, 0x107d: 0x449e, + 0x107c: 0x4408, 0x107d: 0x4408, // Block 0x42, offset 0x1080 0x1090: 0x2311, 0x1091: 0x2326, 0x1092: 0x2326, 0x1093: 0x232d, 0x1094: 0x2334, 0x1095: 0x2349, 0x1096: 0x2350, 0x1097: 0x2357, @@ -5293,13 +5299,13 @@ var nfkcValues = [5888]uint16{ 0x119e: 0x04bb, 0x119f: 0x0007, 0x11a0: 0x000d, 0x11a1: 0x0015, 0x11a2: 0x0017, 0x11a3: 0x001b, 0x11a4: 0x0039, 0x11a5: 0x003d, 0x11a6: 0x003b, 0x11a8: 0x0079, 0x11a9: 0x0009, 0x11aa: 0x000b, 0x11ab: 0x0041, - 0x11b0: 0x42aa, 0x11b1: 0x44c2, 0x11b2: 0x42af, 0x11b4: 0x42b4, - 0x11b6: 0x42b9, 0x11b7: 0x44c8, 0x11b8: 0x42be, 0x11b9: 0x44ce, 0x11ba: 0x42c3, 0x11bb: 0x44d4, - 0x11bc: 0x42c8, 0x11bd: 0x44da, 0x11be: 0x42cd, 0x11bf: 0x44e0, + 0x11b0: 0x42aa, 0x11b1: 0x442c, 0x11b2: 0x42af, 0x11b4: 0x42b4, + 0x11b6: 0x42b9, 0x11b7: 0x4432, 0x11b8: 0x42be, 0x11b9: 0x4438, 0x11ba: 0x42c3, 0x11bb: 0x443e, + 0x11bc: 0x42c8, 0x11bd: 0x4444, 0x11be: 0x42cd, 0x11bf: 0x444a, // Block 0x47, offset 0x11c0 - 0x11c0: 0x0236, 0x11c1: 0x44a4, 0x11c2: 0x44a4, 0x11c3: 0x44aa, 0x11c4: 0x44aa, 0x11c5: 0x44ec, - 0x11c6: 0x44ec, 0x11c7: 0x44b0, 0x11c8: 0x44b0, 0x11c9: 0x44f8, 0x11ca: 0x44f8, 0x11cb: 0x44f8, - 0x11cc: 0x44f8, 0x11cd: 0x0239, 0x11ce: 0x0239, 0x11cf: 0x023c, 0x11d0: 0x023c, 0x11d1: 0x023c, + 0x11c0: 0x0236, 0x11c1: 0x440e, 0x11c2: 0x440e, 0x11c3: 0x4414, 0x11c4: 0x4414, 0x11c5: 0x4456, + 0x11c6: 0x4456, 0x11c7: 0x441a, 0x11c8: 0x441a, 0x11c9: 0x4462, 0x11ca: 0x4462, 0x11cb: 0x4462, + 0x11cc: 0x4462, 0x11cd: 0x0239, 0x11ce: 0x0239, 0x11cf: 0x023c, 0x11d0: 0x023c, 0x11d1: 0x023c, 0x11d2: 0x023c, 0x11d3: 0x023f, 0x11d4: 0x023f, 0x11d5: 0x0242, 0x11d6: 0x0242, 0x11d7: 0x0242, 0x11d8: 0x0242, 0x11d9: 0x0245, 0x11da: 0x0245, 0x11db: 0x0245, 0x11dc: 0x0245, 0x11dd: 0x0248, 0x11de: 0x0248, 0x11df: 0x0248, 0x11e0: 0x0248, 0x11e1: 0x024b, 0x11e2: 0x024b, 0x11e3: 0x024b, @@ -5338,18 +5344,18 @@ var nfkcValues = [5888]uint16{ 0x128c: 0x054b, 0x128d: 0x054f, 0x128e: 0x0553, 0x128f: 0x0557, 0x1290: 0x055b, 0x1291: 0x055f, 0x1292: 0x0563, 0x1293: 0x0567, 0x1294: 0x056f, 0x1295: 0x0577, 0x1296: 0x057f, 0x1297: 0x0583, 0x1298: 0x0587, 0x1299: 0x058b, 0x129a: 0x058f, 0x129b: 0x0593, 0x129c: 0x0597, 0x129d: 0x05a7, - 0x129e: 0x4a5a, 0x129f: 0x4a60, 0x12a0: 0x03c3, 0x12a1: 0x0313, 0x12a2: 0x0317, 0x12a3: 0x4345, - 0x12a4: 0x031b, 0x12a5: 0x434a, 0x12a6: 0x434f, 0x12a7: 0x031f, 0x12a8: 0x0323, 0x12a9: 0x0327, - 0x12aa: 0x4354, 0x12ab: 0x4359, 0x12ac: 0x435e, 0x12ad: 0x4363, 0x12ae: 0x4368, 0x12af: 0x436d, + 0x129e: 0x4a78, 0x129f: 0x4a7e, 0x12a0: 0x03c3, 0x12a1: 0x0313, 0x12a2: 0x0317, 0x12a3: 0x4a3b, + 0x12a4: 0x031b, 0x12a5: 0x4a41, 0x12a6: 0x4a47, 0x12a7: 0x031f, 0x12a8: 0x0323, 0x12a9: 0x0327, + 0x12aa: 0x4a4d, 0x12ab: 0x4a53, 0x12ac: 0x4a59, 0x12ad: 0x4a5f, 0x12ae: 0x4a65, 0x12af: 0x4a6b, 0x12b0: 0x0367, 0x12b1: 0x032b, 0x12b2: 0x032f, 0x12b3: 0x0333, 0x12b4: 0x037b, 0x12b5: 0x0337, 0x12b6: 0x033b, 0x12b7: 0x033f, 0x12b8: 0x0343, 0x12b9: 0x0347, 0x12ba: 0x034b, 0x12bb: 0x034f, 0x12bc: 0x0353, 0x12bd: 0x0357, 0x12be: 0x035b, // Block 0x4b, offset 0x12c0 - 0x12c2: 0x42dc, 0x12c3: 0x42e1, 0x12c4: 0x42e6, 0x12c5: 0x42eb, - 0x12c6: 0x42f0, 0x12c7: 0x42f5, 0x12ca: 0x42fa, 0x12cb: 0x42ff, - 0x12cc: 0x4304, 0x12cd: 0x4309, 0x12ce: 0x430e, 0x12cf: 0x4313, - 0x12d2: 0x4318, 0x12d3: 0x431d, 0x12d4: 0x4322, 0x12d5: 0x4327, 0x12d6: 0x432c, 0x12d7: 0x4331, - 0x12da: 0x4336, 0x12db: 0x433b, 0x12dc: 0x4340, + 0x12c2: 0x49bd, 0x12c3: 0x49c3, 0x12c4: 0x49c9, 0x12c5: 0x49cf, + 0x12c6: 0x49d5, 0x12c7: 0x49db, 0x12ca: 0x49e1, 0x12cb: 0x49e7, + 0x12cc: 0x49ed, 0x12cd: 0x49f3, 0x12ce: 0x49f9, 0x12cf: 0x49ff, + 0x12d2: 0x4a05, 0x12d3: 0x4a0b, 0x12d4: 0x4a11, 0x12d5: 0x4a17, 0x12d6: 0x4a1d, 0x12d7: 0x4a23, + 0x12da: 0x4a29, 0x12db: 0x4a2f, 0x12dc: 0x4a35, 0x12e0: 0x00bf, 0x12e1: 0x00c2, 0x12e2: 0x00cb, 0x12e3: 0x4264, 0x12e4: 0x00c8, 0x12e5: 0x00c5, 0x12e6: 0x0447, 0x12e8: 0x046b, 0x12e9: 0x044b, 0x12ea: 0x044f, 0x12eb: 0x0453, 0x12ec: 0x0457, 0x12ed: 0x046f, 0x12ee: 0x0473, @@ -5426,7 +5432,7 @@ var nfkcValues = [5888]uint16{ // Block 0x52, offset 0x1480 0x1480: 0x26ad, 0x1481: 0x26c2, 0x1482: 0x0503, 0x1490: 0x0c0f, 0x1491: 0x0a47, - 0x1492: 0x08d3, 0x1493: 0x465a, 0x1494: 0x071b, 0x1495: 0x09ef, 0x1496: 0x132f, 0x1497: 0x09ff, + 0x1492: 0x08d3, 0x1493: 0x45c4, 0x1494: 0x071b, 0x1495: 0x09ef, 0x1496: 0x132f, 0x1497: 0x09ff, 0x1498: 0x0727, 0x1499: 0x0cd7, 0x149a: 0x0eaf, 0x149b: 0x0caf, 0x149c: 0x0827, 0x149d: 0x0b6b, 0x149e: 0x07bf, 0x149f: 0x0cb7, 0x14a0: 0x0813, 0x14a1: 0x1117, 0x14a2: 0x0f83, 0x14a3: 0x138b, 0x14a4: 0x09d3, 0x14a5: 0x090b, 0x14a6: 0x0e63, 0x14a7: 0x0c1b, 0x14a8: 0x0c47, 0x14a9: 0x06bf, @@ -5665,8 +5671,8 @@ var nfkcSparseValues = [875]valueRange{ {value: 0x22b2, lo: 0xbe, hi: 0xbe}, // Block 0x1, offset 0xe {value: 0x0091, lo: 0x03}, - {value: 0x4778, lo: 0xa0, hi: 0xa1}, - {value: 0x47aa, lo: 0xaf, hi: 0xb0}, + {value: 0x46e2, lo: 0xa0, hi: 0xa1}, + {value: 0x4714, lo: 0xaf, hi: 0xb0}, {value: 0xa000, lo: 0xb7, hi: 0xb7}, // Block 0x2, offset 0x12 {value: 0x0003, lo: 0x08}, @@ -5814,7 +5820,7 @@ var nfkcSparseValues = [875]valueRange{ {value: 0x812d, lo: 0x92, hi: 0x92}, {value: 0x8132, lo: 0x93, hi: 0x93}, {value: 0x8132, lo: 0x94, hi: 0x94}, - {value: 0x45b2, lo: 0x98, hi: 0x9f}, + {value: 0x451c, lo: 0x98, hi: 0x9f}, // Block 0x11, offset 0x96 {value: 0x0000, lo: 0x02}, {value: 0x8102, lo: 0xbc, hi: 0xbc}, @@ -5825,18 +5831,18 @@ var nfkcSparseValues = [875]valueRange{ {value: 0x2c9e, lo: 0x8b, hi: 0x8c}, {value: 0x8104, lo: 0x8d, hi: 0x8d}, {value: 0x9900, lo: 0x97, hi: 0x97}, - {value: 0x45f2, lo: 0x9c, hi: 0x9d}, - {value: 0x4602, lo: 0x9f, hi: 0x9f}, + {value: 0x455c, lo: 0x9c, hi: 0x9d}, + {value: 0x456c, lo: 0x9f, hi: 0x9f}, // Block 0x13, offset 0xa0 {value: 0x0000, lo: 0x03}, - {value: 0x462a, lo: 0xb3, hi: 0xb3}, - {value: 0x4632, lo: 0xb6, hi: 0xb6}, + {value: 0x4594, lo: 0xb3, hi: 0xb3}, + {value: 0x459c, lo: 0xb6, hi: 0xb6}, {value: 0x8102, lo: 0xbc, hi: 0xbc}, // Block 0x14, offset 0xa4 {value: 0x0008, lo: 0x03}, {value: 0x8104, lo: 0x8d, hi: 0x8d}, - {value: 0x460a, lo: 0x99, hi: 0x9b}, - {value: 0x4622, lo: 0x9e, hi: 0x9e}, + {value: 0x4574, lo: 0x99, hi: 0x9b}, + {value: 0x458c, lo: 0x9e, hi: 0x9e}, // Block 0x15, offset 0xa8 {value: 0x0000, lo: 0x01}, {value: 0x8102, lo: 0xbc, hi: 0xbc}, @@ -5851,8 +5857,8 @@ var nfkcSparseValues = [875]valueRange{ {value: 0x2cbe, lo: 0x8c, hi: 0x8c}, {value: 0x8104, lo: 0x8d, hi: 0x8d}, {value: 0x9900, lo: 0x96, hi: 0x97}, - {value: 0x463a, lo: 0x9c, hi: 0x9c}, - {value: 0x4642, lo: 0x9d, hi: 0x9d}, + {value: 0x45a4, lo: 0x9c, hi: 0x9c}, + {value: 0x45ac, lo: 0x9d, hi: 0x9d}, // Block 0x18, offset 0xb5 {value: 0x0000, lo: 0x03}, {value: 0xa000, lo: 0x92, hi: 0x92}, @@ -5941,18 +5947,18 @@ var nfkcSparseValues = [875]valueRange{ {value: 0x263d, lo: 0xa9, hi: 0xa9}, {value: 0x8126, lo: 0xb1, hi: 0xb1}, {value: 0x8127, lo: 0xb2, hi: 0xb2}, - {value: 0x4a66, lo: 0xb3, hi: 0xb3}, + {value: 0x4a84, lo: 0xb3, hi: 0xb3}, {value: 0x8128, lo: 0xb4, hi: 0xb4}, - {value: 0x4a6f, lo: 0xb5, hi: 0xb5}, - {value: 0x464a, lo: 0xb6, hi: 0xb6}, - {value: 0x468a, lo: 0xb7, hi: 0xb7}, - {value: 0x4652, lo: 0xb8, hi: 0xb8}, - {value: 0x4695, lo: 0xb9, hi: 0xb9}, + {value: 0x4a8d, lo: 0xb5, hi: 0xb5}, + {value: 0x45b4, lo: 0xb6, hi: 0xb6}, + {value: 0x45f4, lo: 0xb7, hi: 0xb7}, + {value: 0x45bc, lo: 0xb8, hi: 0xb8}, + {value: 0x45ff, lo: 0xb9, hi: 0xb9}, {value: 0x8127, lo: 0xba, hi: 0xbd}, // Block 0x26, offset 0x107 {value: 0x0000, lo: 0x0b}, {value: 0x8127, lo: 0x80, hi: 0x80}, - {value: 0x4a78, lo: 0x81, hi: 0x81}, + {value: 0x4a96, lo: 0x81, hi: 0x81}, {value: 0x8132, lo: 0x82, hi: 0x83}, {value: 0x8104, lo: 0x84, hi: 0x84}, {value: 0x8132, lo: 0x86, hi: 0x87}, @@ -6199,7 +6205,7 @@ var nfkcSparseValues = [875]valueRange{ {value: 0x192d, lo: 0xb5, hi: 0xb6}, // Block 0x4a, offset 0x1db {value: 0x0000, lo: 0x01}, - {value: 0x4573, lo: 0x9c, hi: 0x9c}, + {value: 0x44dd, lo: 0x9c, hi: 0x9c}, // Block 0x4b, offset 0x1dd {value: 0x0000, lo: 0x02}, {value: 0x0095, lo: 0xbc, hi: 0xbc}, @@ -6245,16 +6251,16 @@ var nfkcSparseValues = [875]valueRange{ {value: 0x04b3, lo: 0xb6, hi: 0xb6}, {value: 0x0887, lo: 0xb8, hi: 0xba}, // Block 0x53, offset 0x201 - {value: 0x0005, lo: 0x09}, + {value: 0x0006, lo: 0x09}, {value: 0x0313, lo: 0xb1, hi: 0xb1}, {value: 0x0317, lo: 0xb2, hi: 0xb2}, - {value: 0x4345, lo: 0xb3, hi: 0xb3}, + {value: 0x4a3b, lo: 0xb3, hi: 0xb3}, {value: 0x031b, lo: 0xb4, hi: 0xb4}, - {value: 0x434a, lo: 0xb5, hi: 0xb6}, + {value: 0x4a41, lo: 0xb5, hi: 0xb6}, {value: 0x031f, lo: 0xb7, hi: 0xb7}, {value: 0x0323, lo: 0xb8, hi: 0xb8}, {value: 0x0327, lo: 0xb9, hi: 0xb9}, - {value: 0x4354, lo: 0xba, hi: 0xbf}, + {value: 0x4a4d, lo: 0xba, hi: 0xbf}, // Block 0x54, offset 0x20b {value: 0x0000, lo: 0x02}, {value: 0x8132, lo: 0xaf, hi: 0xaf}, @@ -6479,13 +6485,13 @@ var nfkcSparseValues = [875]valueRange{ {value: 0x8101, lo: 0x9e, hi: 0x9e}, // Block 0x83, offset 0x2ba {value: 0x0000, lo: 0x0c}, - {value: 0x4662, lo: 0x9e, hi: 0x9e}, - {value: 0x466c, lo: 0x9f, hi: 0x9f}, - {value: 0x46a0, lo: 0xa0, hi: 0xa0}, - {value: 0x46ae, lo: 0xa1, hi: 0xa1}, - {value: 0x46bc, lo: 0xa2, hi: 0xa2}, - {value: 0x46ca, lo: 0xa3, hi: 0xa3}, - {value: 0x46d8, lo: 0xa4, hi: 0xa4}, + {value: 0x45cc, lo: 0x9e, hi: 0x9e}, + {value: 0x45d6, lo: 0x9f, hi: 0x9f}, + {value: 0x460a, lo: 0xa0, hi: 0xa0}, + {value: 0x4618, lo: 0xa1, hi: 0xa1}, + {value: 0x4626, lo: 0xa2, hi: 0xa2}, + {value: 0x4634, lo: 0xa3, hi: 0xa3}, + {value: 0x4642, lo: 0xa4, hi: 0xa4}, {value: 0x812b, lo: 0xa5, hi: 0xa6}, {value: 0x8101, lo: 0xa7, hi: 0xa9}, {value: 0x8130, lo: 0xad, hi: 0xad}, @@ -6497,14 +6503,14 @@ var nfkcSparseValues = [875]valueRange{ {value: 0x8132, lo: 0x85, hi: 0x89}, {value: 0x812d, lo: 0x8a, hi: 0x8b}, {value: 0x8132, lo: 0xaa, hi: 0xad}, - {value: 0x4676, lo: 0xbb, hi: 0xbb}, - {value: 0x4680, lo: 0xbc, hi: 0xbc}, - {value: 0x46e6, lo: 0xbd, hi: 0xbd}, - {value: 0x4702, lo: 0xbe, hi: 0xbe}, - {value: 0x46f4, lo: 0xbf, hi: 0xbf}, + {value: 0x45e0, lo: 0xbb, hi: 0xbb}, + {value: 0x45ea, lo: 0xbc, hi: 0xbc}, + {value: 0x4650, lo: 0xbd, hi: 0xbd}, + {value: 0x466c, lo: 0xbe, hi: 0xbe}, + {value: 0x465e, lo: 0xbf, hi: 0xbf}, // Block 0x85, offset 0x2d1 {value: 0x0000, lo: 0x01}, - {value: 0x4710, lo: 0x80, hi: 0x80}, + {value: 0x467a, lo: 0x80, hi: 0x80}, // Block 0x86, offset 0x2d3 {value: 0x0000, lo: 0x01}, {value: 0x8132, lo: 0x82, hi: 0x84}, @@ -7624,4 +7630,4 @@ var recompMap = map[uint32]rune{ 0x15B915AF: 0x115BB, } -// Total size of tables: 53KB (53976 bytes) +// Total size of tables: 53KB (54006 bytes) diff --git a/libgo/go/golang_org/x/text/unicode/norm/transform.go b/libgo/go/golang_org/x/text/unicode/norm/transform.go index b341789b4c6..4dec306adcf 100644 --- a/libgo/go/golang_org/x/text/unicode/norm/transform.go +++ b/libgo/go/golang_org/x/text/unicode/norm/transform.go @@ -1,3 +1,5 @@ +// Code generated by running "go run gen.go -core" in golang.org/x/text. DO NOT EDIT. + // Copyright 2013 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. diff --git a/libgo/go/golang_org/x/text/unicode/norm/trie.go b/libgo/go/golang_org/x/text/unicode/norm/trie.go index 423386bf436..4cbea644a1d 100644 --- a/libgo/go/golang_org/x/text/unicode/norm/trie.go +++ b/libgo/go/golang_org/x/text/unicode/norm/trie.go @@ -1,3 +1,5 @@ +// Code generated by running "go run gen.go -core" in golang.org/x/text. DO NOT EDIT. + // Copyright 2011 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. diff --git a/libgo/go/hash/crc32/crc32_amd64.go b/libgo/go/hash/crc32/crc32_amd64.go index 72844f0037c..b394fc0630a 100644 --- a/libgo/go/hash/crc32/crc32_amd64.go +++ b/libgo/go/hash/crc32/crc32_amd64.go @@ -10,23 +10,20 @@ package crc32 -import "unsafe" +import ( + "internal/cpu" + "unsafe" +) // This file contains the code to call the SSE 4.2 version of the Castagnoli // and IEEE CRC. -// haveSSE41/haveSSE42/haveCLMUL are defined in crc_amd64.s and use -// CPUID to test for SSE 4.1, 4.2 and CLMUL support. -func haveSSE41() bool -func haveSSE42() bool -func haveCLMUL() bool - -// castagnoliSSE42 is defined in crc32_amd64.s and uses the SSE4.2 CRC32 +// castagnoliSSE42 is defined in crc32_amd64.s and uses the SSE 4.2 CRC32 // instruction. //go:noescape func castagnoliSSE42(crc uint32, p []byte) uint32 -// castagnoliSSE42Triple is defined in crc32_amd64.s and uses the SSE4.2 CRC32 +// castagnoliSSE42Triple is defined in crc32_amd64.s and uses the SSE 4.2 CRC32 // instruction. //go:noescape func castagnoliSSE42Triple( @@ -40,9 +37,6 @@ func castagnoliSSE42Triple( //go:noescape func ieeeCLMUL(crc uint32, p []byte) uint32 -var sse42 = haveSSE42() -var useFastIEEE = haveCLMUL() && haveSSE41() - const castagnoliK1 = 168 const castagnoliK2 = 1344 @@ -52,11 +46,11 @@ var castagnoliSSE42TableK1 *sse42Table var castagnoliSSE42TableK2 *sse42Table func archAvailableCastagnoli() bool { - return sse42 + return cpu.X86.HasSSE42 } func archInitCastagnoli() { - if !sse42 { + if !cpu.X86.HasSSE42 { panic("arch-specific Castagnoli not available") } castagnoliSSE42TableK1 = new(sse42Table) @@ -88,7 +82,7 @@ func castagnoliShift(table *sse42Table, crc uint32) uint32 { } func archUpdateCastagnoli(crc uint32, p []byte) uint32 { - if !sse42 { + if !cpu.X86.HasSSE42 { panic("not available") } @@ -199,13 +193,13 @@ func archUpdateCastagnoli(crc uint32, p []byte) uint32 { } func archAvailableIEEE() bool { - return useFastIEEE + return cpu.X86.HasPCLMULQDQ && cpu.X86.HasSSE41 } var archIeeeTable8 *slicing8Table func archInitIEEE() { - if !useFastIEEE { + if !cpu.X86.HasPCLMULQDQ || !cpu.X86.HasSSE41 { panic("not available") } // We still use slicing-by-8 for small buffers. @@ -213,7 +207,7 @@ func archInitIEEE() { } func archUpdateIEEE(crc uint32, p []byte) uint32 { - if !useFastIEEE { + if !cpu.X86.HasPCLMULQDQ || !cpu.X86.HasSSE41 { panic("not available") } diff --git a/libgo/go/hash/crc32/crc32_amd64p32.go b/libgo/go/hash/crc32/crc32_amd64p32.go index 9d728fc8feb..1ec44cb496a 100644 --- a/libgo/go/hash/crc32/crc32_amd64p32.go +++ b/libgo/go/hash/crc32/crc32_amd64p32.go @@ -4,33 +4,29 @@ package crc32 +import "internal/cpu" + // This file contains the code to call the SSE 4.2 version of the Castagnoli // CRC. -// haveSSE42 is defined in crc32_amd64p32.s and uses CPUID to test for SSE 4.2 -// support. -func haveSSE42() bool - // castagnoliSSE42 is defined in crc32_amd64p32.s and uses the SSE4.2 CRC32 // instruction. //go:noescape func castagnoliSSE42(crc uint32, p []byte) uint32 -var sse42 = haveSSE42() - func archAvailableCastagnoli() bool { - return sse42 + return cpu.X86.HasSSE42 } func archInitCastagnoli() { - if !sse42 { + if !cpu.X86.HasSSE42 { panic("not available") } // No initialization necessary. } func archUpdateCastagnoli(crc uint32, p []byte) uint32 { - if !sse42 { + if !cpu.X86.HasSSE42 { panic("not available") } return castagnoliSSE42(crc, p) diff --git a/libgo/go/hash/crc32/crc32_arm64.go b/libgo/go/hash/crc32/crc32_arm64.go new file mode 100644 index 00000000000..4c5cad1181c --- /dev/null +++ b/libgo/go/hash/crc32/crc32_arm64.go @@ -0,0 +1,53 @@ +// Copyright 2017 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build ignore + +// ARM64-specific hardware-assisted CRC32 algorithms. See crc32.go for a +// description of the interface that each architecture-specific file +// implements. + +package crc32 + +func supportsCRC32() bool +func castagnoliUpdate(crc uint32, p []byte) uint32 +func ieeeUpdate(crc uint32, p []byte) uint32 + +var hasCRC32 = supportsCRC32() + +func archAvailableCastagnoli() bool { + return hasCRC32 +} + +func archInitCastagnoli() { + if !hasCRC32 { + panic("arch-specific crc32 instruction for Catagnoli not available") + } +} + +func archUpdateCastagnoli(crc uint32, p []byte) uint32 { + if !hasCRC32 { + panic("arch-specific crc32 instruction for Castagnoli not available") + } + + return ^castagnoliUpdate(^crc, p) +} + +func archAvailableIEEE() bool { + return hasCRC32 +} + +func archInitIEEE() { + if !hasCRC32 { + panic("arch-specific crc32 instruction for IEEE not available") + } +} + +func archUpdateIEEE(crc uint32, p []byte) uint32 { + if !hasCRC32 { + panic("arch-specific crc32 instruction for IEEE not available") + } + + return ^ieeeUpdate(^crc, p) +} diff --git a/libgo/go/hash/crc32/crc32_otherarch.go b/libgo/go/hash/crc32/crc32_otherarch.go index 09c3389ea7a..c3acd254182 100644 --- a/libgo/go/hash/crc32/crc32_otherarch.go +++ b/libgo/go/hash/crc32/crc32_otherarch.go @@ -2,7 +2,7 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -// -build !amd64,!amd64p32,!s390x +// -build !amd64,!amd64p32,!s390x,!ppc64le,!arm64 package crc32 diff --git a/libgo/go/hash/crc32/crc32_ppc64le.go b/libgo/go/hash/crc32/crc32_ppc64le.go new file mode 100644 index 00000000000..4211b5bf3c6 --- /dev/null +++ b/libgo/go/hash/crc32/crc32_ppc64le.go @@ -0,0 +1,89 @@ +// Copyright 2017 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build ignore + +package crc32 + +import ( + "unsafe" +) + +const ( + vecMinLen = 16 + vecAlignMask = 15 // align to 16 bytes + crcIEEE = 1 + crcCast = 2 +) + +//go:noescape +func ppc64SlicingUpdateBy8(crc uint32, table8 *slicing8Table, p []byte) uint32 + +// this function requires the buffer to be 16 byte aligned and > 16 bytes long +//go:noescape +func vectorCrc32(crc uint32, poly uint32, p []byte) uint32 + +var archCastagnoliTable8 *slicing8Table + +func archInitCastagnoli() { + archCastagnoliTable8 = slicingMakeTable(Castagnoli) +} + +func archUpdateCastagnoli(crc uint32, p []byte) uint32 { + if len(p) >= 4*vecMinLen { + // If not aligned then process the initial unaligned bytes + + if uint64(uintptr(unsafe.Pointer(&p[0])))&uint64(vecAlignMask) != 0 { + align := uint64(uintptr(unsafe.Pointer(&p[0]))) & uint64(vecAlignMask) + newlen := vecMinLen - align + crc = ppc64SlicingUpdateBy8(crc, archCastagnoliTable8, p[:newlen]) + p = p[newlen:] + } + // p should be aligned now + aligned := len(p) & ^vecAlignMask + crc = vectorCrc32(crc, crcCast, p[:aligned]) + p = p[aligned:] + } + if len(p) == 0 { + return crc + } + return ppc64SlicingUpdateBy8(crc, archCastagnoliTable8, p) +} + +func archAvailableIEEE() bool { + return true +} +func archAvailableCastagnoli() bool { + return true +} + +var archIeeeTable8 *slicing8Table + +func archInitIEEE() { + // We still use slicing-by-8 for small buffers. + archIeeeTable8 = slicingMakeTable(IEEE) +} + +// archUpdateIEEE calculates the checksum of p using vectorizedIEEE. +func archUpdateIEEE(crc uint32, p []byte) uint32 { + + // Check if vector code should be used. If not aligned, then handle those + // first up to the aligned bytes. + + if len(p) >= 4*vecMinLen { + if uint64(uintptr(unsafe.Pointer(&p[0])))&uint64(vecAlignMask) != 0 { + align := uint64(uintptr(unsafe.Pointer(&p[0]))) & uint64(vecAlignMask) + newlen := vecMinLen - align + crc = ppc64SlicingUpdateBy8(crc, archIeeeTable8, p[:newlen]) + p = p[newlen:] + } + aligned := len(p) & ^vecAlignMask + crc = vectorCrc32(crc, crcIEEE, p[:aligned]) + p = p[aligned:] + } + if len(p) == 0 { + return crc + } + return ppc64SlicingUpdateBy8(crc, archIeeeTable8, p) +} diff --git a/libgo/go/hash/crc32/crc32_test.go b/libgo/go/hash/crc32/crc32_test.go index 1356734d504..0492f46e8ce 100644 --- a/libgo/go/hash/crc32/crc32_test.go +++ b/libgo/go/hash/crc32/crc32_test.go @@ -5,6 +5,7 @@ package crc32 import ( + "fmt" "hash" "math/rand" "testing" @@ -75,8 +76,9 @@ func testCrossCheck(t *testing.T, crcFunc1, crcFunc2 func(crc uint32, b []byte) // The AMD64 implementation has some cutoffs at lengths 168*3=504 and // 1344*3=4032. We should make sure lengths around these values are in the // list. - lengths := []int{0, 1, 2, 3, 4, 5, 10, 16, 50, 100, 128, - 500, 501, 502, 503, 504, 505, 512, 1000, 1024, 2000, + lengths := []int{0, 1, 2, 3, 4, 5, 10, 16, 50, 63, 64, 65, 100, + 127, 128, 129, 255, 256, 257, 300, 312, 384, 416, 448, 480, + 500, 501, 502, 503, 504, 505, 512, 513, 1000, 1024, 2000, 4030, 4031, 4032, 4033, 4036, 4040, 4048, 4096, 5000, 10000} for _, length := range lengths { p := make([]byte, length) @@ -196,68 +198,28 @@ func TestGolden(t *testing.T) { } } -func BenchmarkIEEECrc40B(b *testing.B) { - benchmark(b, NewIEEE(), 40, 0) +func BenchmarkCRC32(b *testing.B) { + b.Run("poly=IEEE", benchmarkAll(NewIEEE())) + b.Run("poly=Castagnoli", benchmarkAll(New(MakeTable(Castagnoli)))) + b.Run("poly=Koopman", benchmarkAll(New(MakeTable(Koopman)))) } -func BenchmarkIEEECrc1KB(b *testing.B) { - benchmark(b, NewIEEE(), 1<<10, 0) -} - -func BenchmarkIEEECrc4KB(b *testing.B) { - benchmark(b, NewIEEE(), 4<<10, 0) -} - -func BenchmarkIEEECrc32KB(b *testing.B) { - benchmark(b, NewIEEE(), 32<<10, 0) -} - -func BenchmarkCastagnoliCrc15B(b *testing.B) { - benchmark(b, New(MakeTable(Castagnoli)), 15, 0) -} - -func BenchmarkCastagnoliCrc15BMisaligned(b *testing.B) { - benchmark(b, New(MakeTable(Castagnoli)), 15, 1) -} - -func BenchmarkCastagnoliCrc40B(b *testing.B) { - benchmark(b, New(MakeTable(Castagnoli)), 40, 0) -} - -func BenchmarkCastagnoliCrc40BMisaligned(b *testing.B) { - benchmark(b, New(MakeTable(Castagnoli)), 40, 1) -} - -func BenchmarkCastagnoliCrc512(b *testing.B) { - benchmark(b, New(MakeTable(Castagnoli)), 512, 0) -} - -func BenchmarkCastagnoliCrc512Misaligned(b *testing.B) { - benchmark(b, New(MakeTable(Castagnoli)), 512, 1) -} - -func BenchmarkCastagnoliCrc1KB(b *testing.B) { - benchmark(b, New(MakeTable(Castagnoli)), 1<<10, 0) -} - -func BenchmarkCastagnoliCrc1KBMisaligned(b *testing.B) { - benchmark(b, New(MakeTable(Castagnoli)), 1<<10, 1) -} - -func BenchmarkCastagnoliCrc4KB(b *testing.B) { - benchmark(b, New(MakeTable(Castagnoli)), 4<<10, 0) -} - -func BenchmarkCastagnoliCrc4KBMisaligned(b *testing.B) { - benchmark(b, New(MakeTable(Castagnoli)), 4<<10, 1) -} - -func BenchmarkCastagnoliCrc32KB(b *testing.B) { - benchmark(b, New(MakeTable(Castagnoli)), 32<<10, 0) -} - -func BenchmarkCastagnoliCrc32KBMisaligned(b *testing.B) { - benchmark(b, New(MakeTable(Castagnoli)), 32<<10, 1) +func benchmarkAll(h hash.Hash32) func(b *testing.B) { + return func(b *testing.B) { + for _, size := range []int{15, 40, 512, 1 << 10, 4 << 10, 32 << 10} { + name := fmt.Sprint(size) + if size >= 1024 { + name = fmt.Sprintf("%dkB", size/1024) + } + b.Run("size="+name, func(b *testing.B) { + for align := 0; align <= 1; align++ { + b.Run(fmt.Sprintf("align=%d", align), func(b *testing.B) { + benchmark(b, h, int64(size), int64(align)) + }) + } + }) + } + } } func benchmark(b *testing.B, h hash.Hash32, n, alignment int64) { diff --git a/libgo/go/hash/crc32/gen_const_ppc64le.go b/libgo/go/hash/crc32/gen_const_ppc64le.go new file mode 100644 index 00000000000..bfb3b3a2278 --- /dev/null +++ b/libgo/go/hash/crc32/gen_const_ppc64le.go @@ -0,0 +1,150 @@ +// Copyright 2017 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build ignore + +// Generate the constant table associated with the poly used by the +// vpmsumd crc32 algorithm. +// +// go run gen_const_ppc64le.go +// +// generates crc32_table_ppc64le.s + +// The following is derived from code written by Anton Blanchard +// found at https://github.com/antonblanchard/crc32-vpmsum. +// The original is dual licensed under GPL and Apache 2. As the copyright holder +// for the work, IBM has contributed this new work under the golang license. + +// This code was written in Go based on the original C implementation. + +// This is a tool needed to generate the appropriate constants needed for +// the vpmsum algorithm. It is included to generate new constant tables if +// new polynomial values are included in the future. + +package main + +import ( + "bytes" + "fmt" + "io/ioutil" +) + +var blocking = 32 * 1024 + +func reflect_bits(b uint64, nr uint) uint64 { + var ref uint64 + + for bit := uint64(0); bit < uint64(nr); bit++ { + if (b & uint64(1)) == 1 { + ref |= (1 << (uint64(nr-1) - bit)) + } + b = (b >> 1) + } + return ref +} + +func get_remainder(poly uint64, deg uint, n uint) uint64 { + + rem, _ := xnmodp(n, poly, deg) + return rem +} + +func get_quotient(poly uint64, bits, n uint) uint64 { + + _, div := xnmodp(n, poly, bits) + return div +} + +// xnmodp returns two values, p and div: +// p is the representation of the binary polynomial x**n mod (x ** deg + "poly") +// That is p is the binary representation of the modulus polynomial except for its highest-order term. +// div is the binary representation of the polynomial x**n / (x ** deg + "poly") +func xnmodp(n uint, poly uint64, deg uint) (uint64, uint64) { + + var mod, mask, high, div uint64 + + if n < deg { + div = 0 + return poly, div + } + mask = 1< deg { + high = (mod >> deg) & 1 + div = (div << 1) | high + mod <<= 1 + if high != 0 { + mod ^= poly + } + n-- + } + return mod & mask, div +} + +func main() { + w := new(bytes.Buffer) + + fmt.Fprintf(w, "// autogenerated: do not edit!\n") + fmt.Fprintf(w, "// generated from crc32/gen_const_ppc64le.go\n") + fmt.Fprintln(w) + fmt.Fprintf(w, "#include \"textflag.h\"\n") + + // These are the polynomials supported in vector now. + // If adding others, include the polynomial and a name + // to identify it. + + genCrc32ConstTable(w, 0xedb88320, "IEEE") + genCrc32ConstTable(w, 0x82f63b78, "Cast") + genCrc32ConstTable(w, 0xeb31d82e, "Koop") + b := w.Bytes() + + err := ioutil.WriteFile("crc32_table_ppc64le.s", b, 0666) + if err != nil { + fmt.Printf("can't write output: %s\n", err) + } +} + +func genCrc32ConstTable(w *bytes.Buffer, poly uint32, polyid string) { + + ref_poly := reflect_bits(uint64(poly), 32) + fmt.Fprintf(w, "\n\t/* Reduce %d kbits to 1024 bits */\n", blocking*8) + j := 0 + for i := (blocking * 8) - 1024; i > 0; i -= 1024 { + a := reflect_bits(get_remainder(ref_poly, 32, uint(i)), 32) << 1 + b := reflect_bits(get_remainder(ref_poly, 32, uint(i+64)), 32) << 1 + + fmt.Fprintf(w, "\t/* x^%d mod p(x)%s, x^%d mod p(x)%s */\n", uint(i+64), "", uint(i), "") + fmt.Fprintf(w, "DATA ·%sConst+%d(SB)/8,$0x%016x\n", polyid, j*8, b) + fmt.Fprintf(w, "DATA ·%sConst+%d(SB)/8,$0x%016x\n", polyid, (j+1)*8, a) + + j += 2 + fmt.Fprintf(w, "\n") + } + + for i := (1024 * 2) - 128; i >= 0; i -= 128 { + a := reflect_bits(get_remainder(ref_poly, 32, uint(i+32)), 32) + b := reflect_bits(get_remainder(ref_poly, 32, uint(i+64)), 32) + c := reflect_bits(get_remainder(ref_poly, 32, uint(i+96)), 32) + d := reflect_bits(get_remainder(ref_poly, 32, uint(i+128)), 32) + + fmt.Fprintf(w, "\t/* x^%d mod p(x)%s, x^%d mod p(x)%s, x^%d mod p(x)%s, x^%d mod p(x)%s */\n", i+128, "", i+96, "", i+64, "", i+32, "") + fmt.Fprintf(w, "DATA ·%sConst+%d(SB)/8,$0x%08x%08x\n", polyid, j*8, c, d) + fmt.Fprintf(w, "DATA ·%sConst+%d(SB)/8,$0x%08x%08x\n", polyid, (j+1)*8, a, b) + + j += 2 + fmt.Fprintf(w, "\n") + } + + fmt.Fprintf(w, "GLOBL ·%sConst(SB),RODATA,$4336\n", polyid) + fmt.Fprintf(w, "\n /* Barrett constant m - (4^32)/n */\n") + fmt.Fprintf(w, "DATA ·%sBarConst(SB)/8,$0x%016x\n", polyid, reflect_bits(get_quotient(ref_poly, 32, 64), 33)) + fmt.Fprintf(w, "DATA ·%sBarConst+8(SB)/8,$0x0000000000000000\n", polyid) + fmt.Fprintf(w, "DATA ·%sBarConst+16(SB)/8,$0x%016x\n", polyid, reflect_bits((uint64(1)<<32)|ref_poly, 33)) // reflected? + fmt.Fprintf(w, "DATA ·%sBarConst+24(SB)/8,$0x0000000000000000\n", polyid) + fmt.Fprintf(w, "GLOBL ·%sBarConst(SB),RODATA,$32\n", polyid) +} diff --git a/libgo/go/hash/fnv/fnv.go b/libgo/go/hash/fnv/fnv.go index f1fbb25bdf5..3d2df73d3dc 100644 --- a/libgo/go/hash/fnv/fnv.go +++ b/libgo/go/hash/fnv/fnv.go @@ -13,17 +13,23 @@ import ( ) type ( - sum32 uint32 - sum32a uint32 - sum64 uint64 - sum64a uint64 + sum32 uint32 + sum32a uint32 + sum64 uint64 + sum64a uint64 + sum128 [2]uint64 + sum128a [2]uint64 ) const ( - offset32 = 2166136261 - offset64 = 14695981039346656037 - prime32 = 16777619 - prime64 = 1099511628211 + offset32 = 2166136261 + offset64 = 14695981039346656037 + offset128Lower = 0x62b821756295c58d + offset128Higher = 0x6c62272e07bb0142 + prime32 = 16777619 + prime64 = 1099511628211 + prime128Lower = 0x13b + prime128Shift = 24 ) // New32 returns a new 32-bit FNV-1 hash.Hash. @@ -54,10 +60,30 @@ func New64a() hash.Hash64 { return &s } -func (s *sum32) Reset() { *s = offset32 } -func (s *sum32a) Reset() { *s = offset32 } -func (s *sum64) Reset() { *s = offset64 } -func (s *sum64a) Reset() { *s = offset64 } +// New128 returns a new 128-bit FNV-1 hash.Hash. +// Its Sum method will lay the value out in big-endian byte order. +func New128() hash.Hash { + var s sum128 + s[0] = offset128Higher + s[1] = offset128Lower + return &s +} + +// New128a returns a new 128-bit FNV-1a hash.Hash. +// Its Sum method will lay the value out in big-endian byte order. +func New128a() hash.Hash { + var s sum128a + s[0] = offset128Higher + s[1] = offset128Lower + return &s +} + +func (s *sum32) Reset() { *s = offset32 } +func (s *sum32a) Reset() { *s = offset32 } +func (s *sum64) Reset() { *s = offset64 } +func (s *sum64a) Reset() { *s = offset64 } +func (s *sum128) Reset() { s[0] = offset128Higher; s[1] = offset128Lower } +func (s *sum128a) Reset() { s[0] = offset128Higher; s[1] = offset128Lower } func (s *sum32) Sum32() uint32 { return uint32(*s) } func (s *sum32a) Sum32() uint32 { return uint32(*s) } @@ -104,15 +130,57 @@ func (s *sum64a) Write(data []byte) (int, error) { return len(data), nil } -func (s *sum32) Size() int { return 4 } -func (s *sum32a) Size() int { return 4 } -func (s *sum64) Size() int { return 8 } -func (s *sum64a) Size() int { return 8 } +func (s *sum128) Write(data []byte) (int, error) { + for _, c := range data { + // Compute the multiplication in 4 parts to simplify carrying + s1l := (s[1] & 0xffffffff) * prime128Lower + s1h := (s[1] >> 32) * prime128Lower + s0l := (s[0]&0xffffffff)*prime128Lower + (s[1]&0xffffffff)<>32)*prime128Lower + (s[1]>>32)<> 32 + s0l += s1h >> 32 + s0h += s0l >> 32 + // Update the values + s[1] = (s1l & 0xffffffff) + (s1h << 32) + s[0] = (s0l & 0xffffffff) + (s0h << 32) + s[1] ^= uint64(c) + } + return len(data), nil +} + +func (s *sum128a) Write(data []byte) (int, error) { + for _, c := range data { + s[1] ^= uint64(c) + // Compute the multiplication in 4 parts to simplify carrying + s1l := (s[1] & 0xffffffff) * prime128Lower + s1h := (s[1] >> 32) * prime128Lower + s0l := (s[0]&0xffffffff)*prime128Lower + (s[1]&0xffffffff)<>32)*prime128Lower + (s[1]>>32)<> 32 + s0l += s1h >> 32 + s0h += s0l >> 32 + // Update the values + s[1] = (s1l & 0xffffffff) + (s1h << 32) + s[0] = (s0l & 0xffffffff) + (s0h << 32) + } + return len(data), nil +} -func (s *sum32) BlockSize() int { return 1 } -func (s *sum32a) BlockSize() int { return 1 } -func (s *sum64) BlockSize() int { return 1 } -func (s *sum64a) BlockSize() int { return 1 } +func (s *sum32) Size() int { return 4 } +func (s *sum32a) Size() int { return 4 } +func (s *sum64) Size() int { return 8 } +func (s *sum64a) Size() int { return 8 } +func (s *sum128) Size() int { return 16 } +func (s *sum128a) Size() int { return 16 } + +func (s *sum32) BlockSize() int { return 1 } +func (s *sum32a) BlockSize() int { return 1 } +func (s *sum64) BlockSize() int { return 1 } +func (s *sum64a) BlockSize() int { return 1 } +func (s *sum128) BlockSize() int { return 1 } +func (s *sum128a) BlockSize() int { return 1 } func (s *sum32) Sum(in []byte) []byte { v := uint32(*s) @@ -133,3 +201,17 @@ func (s *sum64a) Sum(in []byte) []byte { v := uint64(*s) return append(in, byte(v>>56), byte(v>>48), byte(v>>40), byte(v>>32), byte(v>>24), byte(v>>16), byte(v>>8), byte(v)) } + +func (s *sum128) Sum(in []byte) []byte { + return append(in, + byte(s[0]>>56), byte(s[0]>>48), byte(s[0]>>40), byte(s[0]>>32), byte(s[0]>>24), byte(s[0]>>16), byte(s[0]>>8), byte(s[0]), + byte(s[1]>>56), byte(s[1]>>48), byte(s[1]>>40), byte(s[1]>>32), byte(s[1]>>24), byte(s[1]>>16), byte(s[1]>>8), byte(s[1]), + ) +} + +func (s *sum128a) Sum(in []byte) []byte { + return append(in, + byte(s[0]>>56), byte(s[0]>>48), byte(s[0]>>40), byte(s[0]>>32), byte(s[0]>>24), byte(s[0]>>16), byte(s[0]>>8), byte(s[0]), + byte(s[1]>>56), byte(s[1]>>48), byte(s[1]>>40), byte(s[1]>>32), byte(s[1]>>24), byte(s[1]>>16), byte(s[1]>>8), byte(s[1]), + ) +} diff --git a/libgo/go/hash/fnv/fnv_test.go b/libgo/go/hash/fnv/fnv_test.go index 89d39b38ad3..7da15ba9e82 100644 --- a/libgo/go/hash/fnv/fnv_test.go +++ b/libgo/go/hash/fnv/fnv_test.go @@ -44,6 +44,20 @@ var golden64a = []golden{ {[]byte{0xe7, 0x1f, 0xa2, 0x19, 0x05, 0x41, 0x57, 0x4b}, "abc"}, } +var golden128 = []golden{ + {[]byte{0x6c, 0x62, 0x27, 0x2e, 0x07, 0xbb, 0x01, 0x42, 0x62, 0xb8, 0x21, 0x75, 0x62, 0x95, 0xc5, 0x8d}, ""}, + {[]byte{0xd2, 0x28, 0xcb, 0x69, 0x10, 0x1a, 0x8c, 0xaf, 0x78, 0x91, 0x2b, 0x70, 0x4e, 0x4a, 0x14, 0x1e}, "a"}, + {[]byte{0x8, 0x80, 0x94, 0x5a, 0xee, 0xab, 0x1b, 0xe9, 0x5a, 0xa0, 0x73, 0x30, 0x55, 0x26, 0xc0, 0x88}, "ab"}, + {[]byte{0xa6, 0x8b, 0xb2, 0xa4, 0x34, 0x8b, 0x58, 0x22, 0x83, 0x6d, 0xbc, 0x78, 0xc6, 0xae, 0xe7, 0x3b}, "abc"}, +} + +var golden128a = []golden{ + {[]byte{0x6c, 0x62, 0x27, 0x2e, 0x07, 0xbb, 0x01, 0x42, 0x62, 0xb8, 0x21, 0x75, 0x62, 0x95, 0xc5, 0x8d}, ""}, + {[]byte{0xd2, 0x28, 0xcb, 0x69, 0x6f, 0x1a, 0x8c, 0xaf, 0x78, 0x91, 0x2b, 0x70, 0x4e, 0x4a, 0x89, 0x64}, "a"}, + {[]byte{0x08, 0x80, 0x95, 0x44, 0xbb, 0xab, 0x1b, 0xe9, 0x5a, 0xa0, 0x73, 0x30, 0x55, 0xb6, 0x9a, 0x62}, "ab"}, + {[]byte{0xa6, 0x8d, 0x62, 0x2c, 0xec, 0x8b, 0x58, 0x22, 0x83, 0x6d, 0xbc, 0x79, 0x77, 0xaf, 0x7f, 0x3b}, "abc"}, +} + func TestGolden32(t *testing.T) { testGolden(t, New32(), golden32) } @@ -60,6 +74,14 @@ func TestGolden64a(t *testing.T) { testGolden(t, New64a(), golden64a) } +func TestGolden128(t *testing.T) { + testGolden(t, New128(), golden128) +} + +func TestGolden128a(t *testing.T) { + testGolden(t, New128a(), golden128a) +} + func testGolden(t *testing.T, hash hash.Hash, gold []golden) { for _, g := range gold { hash.Reset() @@ -91,6 +113,13 @@ func TestIntegrity64(t *testing.T) { func TestIntegrity64a(t *testing.T) { testIntegrity(t, New64a()) } +func TestIntegrity128(t *testing.T) { + testIntegrity(t, New128()) +} + +func TestIntegrity128a(t *testing.T) { + testIntegrity(t, New128a()) +} func testIntegrity(t *testing.T, h hash.Hash) { data := []byte{'1', '2', 3, 4, 5} @@ -129,6 +158,8 @@ func testIntegrity(t *testing.T, h hash.Hash) { if sum64 != binary.BigEndian.Uint64(sum) { t.Fatalf("Sum()=0x%x, but Sum64()=0x%x", sum, sum64) } + case 16: + // There's no Sum128 function, so we don't need to test anything here. } } @@ -148,6 +179,14 @@ func BenchmarkFnv64aKB(b *testing.B) { benchmarkKB(b, New64a()) } +func BenchmarkFnv128KB(b *testing.B) { + benchmarkKB(b, New128()) +} + +func BenchmarkFnv128aKB(b *testing.B) { + benchmarkKB(b, New128a()) +} + func benchmarkKB(b *testing.B, h hash.Hash) { b.SetBytes(1024) data := make([]byte, 1024) diff --git a/libgo/go/html/template/attr.go b/libgo/go/html/template/attr.go index d65d340073d..7438f51f6a9 100644 --- a/libgo/go/html/template/attr.go +++ b/libgo/go/html/template/attr.go @@ -135,9 +135,8 @@ var attrTypeMap = map[string]contentType{ } // attrType returns a conservative (upper-bound on authority) guess at the -// type of the named attribute. +// type of the lowercase named attribute. func attrType(name string) contentType { - name = strings.ToLower(name) if strings.HasPrefix(name, "data-") { // Strip data- so that custom attribute heuristics below are // widely applied. diff --git a/libgo/go/html/template/doc.go b/libgo/go/html/template/doc.go index cb898127432..35d171c3fca 100644 --- a/libgo/go/html/template/doc.go +++ b/libgo/go/html/template/doc.go @@ -65,8 +65,10 @@ functions to each simple action pipeline, so given the excerpt At parse time each {{.}} is overwritten to add escaping functions as necessary. In this case it becomes - {{. | html}} + {{. | htmlescaper}} +where urlescaper, attrescaper, and htmlescaper are aliases for internal escaping +functions. Errors diff --git a/libgo/go/html/template/error.go b/libgo/go/html/template/error.go index cbcaf92e4ab..0e527063ea6 100644 --- a/libgo/go/html/template/error.go +++ b/libgo/go/html/template/error.go @@ -183,6 +183,34 @@ const ( // Look for missing semicolons inside branches, and maybe add // parentheses to make it clear which interpretation you intend. ErrSlashAmbig + + // ErrPredefinedEscaper: "predefined escaper ... disallowed in template" + // Example: + //
Hello
+ // Discussion: + // Package html/template already contextually escapes all pipelines to + // produce HTML output safe against code injection. Manually escaping + // pipeline output using the predefined escapers "html" or "urlquery" is + // unnecessary, and may affect the correctness or safety of the escaped + // pipeline output in Go 1.8 and earlier. + // + // In most cases, such as the given example, this error can be resolved by + // simply removing the predefined escaper from the pipeline and letting the + // contextual autoescaper handle the escaping of the pipeline. In other + // instances, where the predefined escaper occurs in the middle of a + // pipeline where subsequent commands expect escaped input, e.g. + // {{.X | html | makeALink}} + // where makeALink does + // return `link` + // consider refactoring the surrounding template to make use of the + // contextual autoescaper, i.e. + // link + // + // To ease migration to Go 1.9 and beyond, "html" and "urlquery" will + // continue to be allowed as the last command in a pipeline. However, if the + // pipeline occurs in an unquoted attribute value context, "html" is + // disallowed. Avoid using "html" and "urlquery" entirely in new templates. + ErrPredefinedEscaper ) func (e *Error) Error() string { diff --git a/libgo/go/html/template/escape.go b/libgo/go/html/template/escape.go index 0e7d2be143b..b51a37039bd 100644 --- a/libgo/go/html/template/escape.go +++ b/libgo/go/html/template/escape.go @@ -19,8 +19,7 @@ import ( // been modified. Otherwise the named templates have been rendered // unusable. func escapeTemplate(tmpl *Template, node parse.Node, name string) error { - e := newEscaper(tmpl) - c, _ := e.escapeTree(context{}, node, name, 0) + c, _ := tmpl.esc.escapeTree(context{}, node, name, 0) var err error if c.err != nil { err, c.err.Name = c.err, name @@ -36,7 +35,7 @@ func escapeTemplate(tmpl *Template, node parse.Node, name string) error { } return err } - e.commit() + tmpl.esc.commit() if t := tmpl.set[name]; t != nil { t.escapeErr = escapeOK t.Tree = t.text.Tree @@ -44,6 +43,21 @@ func escapeTemplate(tmpl *Template, node parse.Node, name string) error { return nil } +// evalArgs formats the list of arguments into a string. It is equivalent to +// fmt.Sprint(args...), except that it deferences all pointers. +func evalArgs(args ...interface{}) string { + // Optimization for simple common case of a single string argument. + if len(args) == 1 { + if s, ok := args[0].(string); ok { + return s + } + } + for i, arg := range args { + args[i] = indirectToStringerOrError(arg) + } + return fmt.Sprint(args...) +} + // funcMap maps command names to functions that render their inputs safe. var funcMap = template.FuncMap{ "_html_template_attrescaper": attrEscaper, @@ -60,22 +74,14 @@ var funcMap = template.FuncMap{ "_html_template_urlescaper": urlEscaper, "_html_template_urlfilter": urlFilter, "_html_template_urlnormalizer": urlNormalizer, -} - -// equivEscapers matches contextual escapers to equivalent template builtins. -var equivEscapers = map[string]string{ - "_html_template_attrescaper": "html", - "_html_template_htmlescaper": "html", - "_html_template_nospaceescaper": "html", - "_html_template_rcdataescaper": "html", - "_html_template_urlescaper": "urlquery", - "_html_template_urlnormalizer": "urlquery", + "_eval_args_": evalArgs, } // escaper collects type inferences about templates and changes needed to make // templates injection safe. type escaper struct { - tmpl *Template + // ns is the nameSpace that this escaper is associated with. + ns *nameSpace // output[templateName] is the output context for a templateName that // has been mangled to include its input context. output map[string]context @@ -92,10 +98,10 @@ type escaper struct { textNodeEdits map[*parse.TextNode][]byte } -// newEscaper creates a blank escaper for the given set. -func newEscaper(t *Template) *escaper { - return &escaper{ - t, +// makeEscaper creates a blank escaper for the given set. +func makeEscaper(n *nameSpace) escaper { + return escaper{ + n, map[string]context{}, map[string]*template.Template{}, map[string]bool{}, @@ -140,6 +146,30 @@ func (e *escaper) escapeAction(c context, n *parse.ActionNode) context { return c } c = nudge(c) + // Check for disallowed use of predefined escapers in the pipeline. + for pos, idNode := range n.Pipe.Cmds { + node, ok := idNode.Args[0].(*parse.IdentifierNode) + if !ok { + // A predefined escaper "esc" will never be found as an identifier in a + // Chain or Field node, since: + // - "esc.x ..." is invalid, since predefined escapers return strings, and + // strings do not have methods, keys or fields. + // - "... .esc" is invalid, since predefined escapers are global functions, + // not methods or fields of any types. + // Therefore, it is safe to ignore these two node types. + continue + } + ident := node.Ident + if _, ok := predefinedEscapers[ident]; ok { + if pos < len(n.Pipe.Cmds)-1 || + c.state == stateAttr && c.delim == delimSpaceOrTagEnd && ident == "html" { + return context{ + state: stateError, + err: errorf(ErrPredefinedEscaper, n, n.Line, "predefined escaper %q disallowed in template", ident), + } + } + } + } s := make([]string, 0, 3) switch c.state { case stateError: @@ -204,75 +234,98 @@ func (e *escaper) escapeAction(c context, n *parse.ActionNode) context { return c } -// allIdents returns the names of the identifiers under the Ident field of the node, -// which might be a singleton (Identifier) or a slice (Field or Chain). -func allIdents(node parse.Node) []string { - switch node := node.(type) { - case *parse.IdentifierNode: - return []string{node.Ident} - case *parse.FieldNode: - return node.Ident - case *parse.ChainNode: - return node.Field - } - return nil -} - -// ensurePipelineContains ensures that the pipeline has commands with -// the identifiers in s in order. -// If the pipeline already has some of the sanitizers, do not interfere. -// For example, if p is (.X | html) and s is ["escapeJSVal", "html"] then it -// has one matching, "html", and one to insert, "escapeJSVal", to produce -// (.X | escapeJSVal | html). +// ensurePipelineContains ensures that the pipeline ends with the commands with +// the identifiers in s in order. If the pipeline ends with a predefined escaper +// (i.e. "html" or "urlquery"), merge it with the identifiers in s. func ensurePipelineContains(p *parse.PipeNode, s []string) { if len(s) == 0 { + // Do not rewrite pipeline if we have no escapers to insert. return } - n := len(p.Cmds) - // Find the identifiers at the end of the command chain. - idents := p.Cmds - for i := n - 1; i >= 0; i-- { - if cmd := p.Cmds[i]; len(cmd.Args) != 0 { - if _, ok := cmd.Args[0].(*parse.IdentifierNode); ok { - continue - } - } - idents = p.Cmds[i+1:] - } - dups := 0 - for _, idNode := range idents { - for _, ident := range allIdents(idNode.Args[0]) { - if escFnsEq(s[dups], ident) { - dups++ - if dups == len(s) { - return + // Precondition: p.Cmds contains at most one predefined escaper and the + // escaper will be present at p.Cmds[len(p.Cmds)-1]. This precondition is + // always true because of the checks in escapeAction. + pipelineLen := len(p.Cmds) + if pipelineLen > 0 { + lastCmd := p.Cmds[pipelineLen-1] + if idNode, ok := lastCmd.Args[0].(*parse.IdentifierNode); ok { + if esc := idNode.Ident; predefinedEscapers[esc] { + // Pipeline ends with a predefined escaper. + if len(p.Cmds) == 1 && len(lastCmd.Args) > 1 { + // Special case: pipeline is of the form {{ esc arg1 arg2 ... argN }}, + // where esc is the predefined escaper, and arg1...argN are its arguments. + // Convert this into the equivalent form + // {{ _eval_args_ arg1 arg2 ... argN | esc }}, so that esc can be easily + // merged with the escapers in s. + lastCmd.Args[0] = parse.NewIdentifier("_eval_args_").SetTree(nil).SetPos(lastCmd.Args[0].Position()) + p.Cmds = appendCmd(p.Cmds, newIdentCmd(esc, p.Position())) + pipelineLen++ } - } - } - } - newCmds := make([]*parse.CommandNode, n-len(idents), n+len(s)-dups) - copy(newCmds, p.Cmds) - // Merge existing identifier commands with the sanitizers needed. - for _, idNode := range idents { - pos := idNode.Args[0].Position() - for _, ident := range allIdents(idNode.Args[0]) { - i := indexOfStr(ident, s, escFnsEq) - if i != -1 { - for _, name := range s[:i] { - newCmds = appendCmd(newCmds, newIdentCmd(name, pos)) + // If any of the commands in s that we are about to insert is equivalent + // to the predefined escaper, use the predefined escaper instead. + dup := false + for i, escaper := range s { + if escFnsEq(esc, escaper) { + s[i] = idNode.Ident + dup = true + } + } + if dup { + // The predefined escaper will already be inserted along with the + // escapers in s, so do not copy it to the rewritten pipeline. + pipelineLen-- } - s = s[i+1:] } } - newCmds = appendCmd(newCmds, idNode) } - // Create any remaining sanitizers. + // Rewrite the pipeline, creating the escapers in s at the end of the pipeline. + newCmds := make([]*parse.CommandNode, pipelineLen, pipelineLen+len(s)) + copy(newCmds, p.Cmds) for _, name := range s { newCmds = appendCmd(newCmds, newIdentCmd(name, p.Position())) } p.Cmds = newCmds } +// predefinedEscapers contains template predefined escapers that are equivalent +// to some contextual escapers. Keep in sync with equivEscapers. +var predefinedEscapers = map[string]bool{ + "html": true, + "urlquery": true, +} + +// equivEscapers matches contextual escapers to equivalent predefined +// template escapers. +var equivEscapers = map[string]string{ + // The following pairs of HTML escapers provide equivalent security + // guarantees, since they all escape '\000', '\'', '"', '&', '<', and '>'. + "_html_template_attrescaper": "html", + "_html_template_htmlescaper": "html", + "_html_template_rcdataescaper": "html", + // These two URL escapers produce URLs safe for embedding in a URL query by + // percent-encoding all the reserved characters specified in RFC 3986 Section + // 2.2 + "_html_template_urlescaper": "urlquery", + // These two functions are not actually equivalent; urlquery is stricter as it + // escapes reserved characters (e.g. '#'), while _html_template_urlnormalizer + // does not. It is therefore only safe to replace _html_template_urlnormalizer + // with urlquery (this happens in ensurePipelineContains), but not the otherI've + // way around. We keep this entry around to preserve the behavior of templates + // written before Go 1.9, which might depend on this substitution taking place. + "_html_template_urlnormalizer": "urlquery", +} + +// escFnsEq reports whether the two escaping functions are equivalent. +func escFnsEq(a, b string) bool { + if e := equivEscapers[a]; e != "" { + a = e + } + if e := equivEscapers[b]; e != "" { + b = e + } + return a == b +} + // redundantFuncs[a][b] implies that funcMap[b](funcMap[a](x)) == funcMap[a](x) // for all x. var redundantFuncs = map[string]map[string]bool{ @@ -318,17 +371,6 @@ func indexOfStr(s string, strs []string, eq func(a, b string) bool) int { return -1 } -// escFnsEq reports whether the two escaping functions are equivalent. -func escFnsEq(a, b string) bool { - if e := equivEscapers[a]; e != "" { - a = e - } - if e := equivEscapers[b]; e != "" { - b = e - } - return a == b -} - // newIdentCmd produces a command containing a single identifier node. func newIdentCmd(identifier string, pos parse.Pos) *parse.CommandNode { return &parse.CommandNode{ @@ -449,13 +491,13 @@ func (e *escaper) escapeList(c context, n *parse.ListNode) context { // It returns the best guess at an output context, and the result of the filter // which is the same as whether e was updated. func (e *escaper) escapeListConditionally(c context, n *parse.ListNode, filter func(*escaper, context) bool) (context, bool) { - e1 := newEscaper(e.tmpl) + e1 := makeEscaper(e.ns) // Make type inferences available to f. for k, v := range e.output { e1.output[k] = v } c = e1.escapeList(c, n) - ok := filter != nil && filter(e1, c) + ok := filter != nil && filter(&e1, c) if ok { // Copy inferences and edits from e1 back into e. for k, v := range e1.output { @@ -504,7 +546,7 @@ func (e *escaper) escapeTree(c context, node parse.Node, name string, line int) if t == nil { // Two cases: The template exists but is empty, or has never been mentioned at // all. Distinguish the cases in the error messages. - if e.tmpl.set[name] != nil { + if e.ns.set[name] != nil { return context{ state: stateError, err: errorf(ErrNoSuchTemplate, node, line, "%q is an incomplete or empty template", name), @@ -624,7 +666,7 @@ func (e *escaper) escapeText(c context, n *parse.TextNode) context { // the entire comment is considered to be a // LineTerminator for purposes of parsing by // the syntactic grammar." - if bytes.IndexAny(s[written:i1], "\n\r\u2028\u2029") != -1 { + if bytes.ContainsAny(s[written:i1], "\n\r\u2028\u2029") { b.WriteByte('\n') } else { b.WriteByte(' ') @@ -752,8 +794,11 @@ func (e *escaper) commit() { for name := range e.output { e.template(name).Funcs(funcMap) } + // Any template from the name space associated with this escaper can be used + // to add derived templates to the underlying text/template name space. + tmpl := e.arbitraryTemplate() for _, t := range e.derived { - if _, err := e.tmpl.text.AddParseTree(t.Name(), t.Tree); err != nil { + if _, err := tmpl.text.AddParseTree(t.Name(), t.Tree); err != nil { panic("error adding derived template") } } @@ -766,17 +811,34 @@ func (e *escaper) commit() { for n, s := range e.textNodeEdits { n.Text = s } + // Reset state that is specific to this commit so that the same changes are + // not re-applied to the template on subsequent calls to commit. + e.called = make(map[string]bool) + e.actionNodeEdits = make(map[*parse.ActionNode][]string) + e.templateNodeEdits = make(map[*parse.TemplateNode]string) + e.textNodeEdits = make(map[*parse.TextNode][]byte) } // template returns the named template given a mangled template name. func (e *escaper) template(name string) *template.Template { - t := e.tmpl.text.Lookup(name) + // Any template from the name space associated with this escaper can be used + // to look up templates in the underlying text/template name space. + t := e.arbitraryTemplate().text.Lookup(name) if t == nil { t = e.derived[name] } return t } +// arbitraryTemplate returns an arbitrary template from the name space +// associated with e and panics if no templates are found. +func (e *escaper) arbitraryTemplate() *Template { + for _, t := range e.ns.set { + return t + } + panic("no templates in name space") +} + // Forwarding functions so that clients need only import this package // to reach the general escaping functions of text/template. diff --git a/libgo/go/html/template/escape_test.go b/libgo/go/html/template/escape_test.go index f6ace496e7b..f5a4ce17364 100644 --- a/libgo/go/html/template/escape_test.go +++ b/libgo/go/html/template/escape_test.go @@ -359,7 +359,7 @@ func TestEscape(t *testing.T) { { "styleStrEncodedProtocolEncoded", ``, - // The CSS string 'javascript\\3a alert(1337)' does not contains a colon. + // The CSS string 'javascript\\3a alert(1337)' does not contain a colon. ``, }, { @@ -685,6 +685,40 @@ func TestEscape(t *testing.T) { } } +func TestEscapeMap(t *testing.T) { + data := map[string]string{ + "html": `

Hi!

`, + "urlquery": `http://www.foo.com/index.html?title=main`, + } + for _, test := range [...]struct { + desc, input, output string + }{ + // covering issue 20323 + { + "field with predefined escaper name 1", + `{{.html | print}}`, + `<h1>Hi!</h1>`, + }, + // covering issue 20323 + { + "field with predefined escaper name 2", + `{{.urlquery | print}}`, + `http://www.foo.com/index.html?title=main`, + }, + } { + tmpl := Must(New("").Parse(test.input)) + b := new(bytes.Buffer) + if err := tmpl.Execute(b, data); err != nil { + t.Errorf("%s: template execution failed: %s", test.desc, err) + continue + } + if w, g := test.output, b.String(); w != g { + t.Errorf("%s: escaped output: want\n\t%q\ngot\n\t%q", test.desc, w, g) + continue + } + } +} + func TestEscapeSet(t *testing.T) { type dataItem struct { Children []*dataItem @@ -970,8 +1004,33 @@ func TestErrors(t *testing.T) { ``, `: expected space, attr name, or end of tag, but got "=foo>"`, }, + { + `Hello, {{. | urlquery | print}}!`, + // urlquery is disallowed if it is not the last command in the pipeline. + `predefined escaper "urlquery" disallowed in template`, + }, + { + `Hello, {{. | html | print}}!`, + // html is disallowed if it is not the last command in the pipeline. + `predefined escaper "html" disallowed in template`, + }, + { + `Hello, {{html . | print}}!`, + // A direct call to html is disallowed if it is not the last command in the pipeline. + `predefined escaper "html" disallowed in template`, + }, + { + `
Hello
`, + // html is disallowed in a pipeline that is in an unquoted attribute context, + // even if it is the last command in the pipeline. + `predefined escaper "html" disallowed in template`, + }, + { + `Hello, {{. | urlquery | html}}!`, + // html is allowed since it is the last command in the pipeline, but urlquery is not. + `predefined escaper "urlquery" disallowed in template`, + }, } - for _, test := range tests { buf := new(bytes.Buffer) tmpl, err := New("z").Parse(test.input) @@ -1396,6 +1455,16 @@ func TestEscapeText(t *testing.T) { `", "mime: expected token after slash"}, - {"bogus/bogus", "mime: unexpected content after media subtype"}, + {"bogus ;=========", "bogus", "mime: invalid media parameter"}, + // The following example is from real email delivered by gmail (error: missing semicolon) + // and it is there to check behavior described in #19498 + {"application/pdf; x-mac-type=\"3F3F3F3F\"; x-mac-creator=\"3F3F3F3F\" name=\"a.pdf\";", + "application/pdf", "mime: invalid media parameter"}, + {"bogus/", "", "mime: expected token after slash"}, + {"bogus/bogus", "", "mime: unexpected content after media subtype"}, } func TestParseMediaTypeBogus(t *testing.T) { @@ -275,8 +280,11 @@ func TestParseMediaTypeBogus(t *testing.T) { if params != nil { t.Errorf("ParseMediaType(%q): got non-nil params on error", tt.in) } - if mt != "" { - t.Errorf("ParseMediaType(%q): got non-empty media type string on error", tt.in) + if err != ErrInvalidMediaParameter && mt != "" { + t.Errorf("ParseMediaType(%q): got unexpected non-empty media type string", tt.in) + } + if err == ErrInvalidMediaParameter && mt != tt.mt { + t.Errorf("ParseMediaType(%q): in case of invalid parameters: expected type %q, got %q", tt.in, tt.mt, mt) } } } diff --git a/libgo/go/mime/multipart/formdata.go b/libgo/go/mime/multipart/formdata.go index c9e3188c33a..832d0ad6936 100644 --- a/libgo/go/mime/multipart/formdata.go +++ b/libgo/go/mime/multipart/formdata.go @@ -13,13 +13,20 @@ import ( "os" ) +// ErrMessageTooLarge is returned by ReadForm if the message form +// data is too large to be processed. +var ErrMessageTooLarge = errors.New("multipart: message too large") + // TODO(adg,bradfitz): find a way to unify the DoS-prevention strategy here // with that of the http package's ParseForm. // ReadForm parses an entire multipart message whose parts have // a Content-Disposition of "form-data". -// It stores up to maxMemory bytes of the file parts in memory -// and the remainder on disk in temporary files. +// It stores up to maxMemory bytes + 10MB (reserved for non-file parts) +// in memory. File parts which can't be stored in memory will be stored on +// disk in temporary files. +// It returns ErrMessageTooLarge if all non-file parts can't be stored in +// memory. func (r *Reader) ReadForm(maxMemory int64) (*Form, error) { return r.readForm(maxMemory) } @@ -32,7 +39,8 @@ func (r *Reader) readForm(maxMemory int64) (_ *Form, err error) { } }() - maxValueBytes := int64(10 << 20) // 10 MB is a lot of text. + // Reserve an additional 10 MB for non-file parts. + maxValueBytes := maxMemory + int64(10<<20) for { p, err := r.NextPart() if err == io.EOF { @@ -52,13 +60,13 @@ func (r *Reader) readForm(maxMemory int64) (_ *Form, err error) { if filename == "" { // value, store as string in memory - n, err := io.CopyN(&b, p, maxValueBytes) + n, err := io.CopyN(&b, p, maxValueBytes+1) if err != nil && err != io.EOF { return nil, err } maxValueBytes -= n - if maxValueBytes == 0 { - return nil, errors.New("multipart: message too large") + if maxValueBytes < 0 { + return nil, ErrMessageTooLarge } form.Value[name] = append(form.Value[name], b.String()) continue @@ -79,7 +87,7 @@ func (r *Reader) readForm(maxMemory int64) (_ *Form, err error) { if err != nil { return nil, err } - _, err = io.Copy(file, io.MultiReader(&b, p)) + size, err := io.Copy(file, io.MultiReader(&b, p)) if cerr := file.Close(); err == nil { err = cerr } @@ -88,9 +96,12 @@ func (r *Reader) readForm(maxMemory int64) (_ *Form, err error) { return nil, err } fh.tmpfile = file.Name() + fh.Size = size } else { fh.content = b.Bytes() + fh.Size = int64(len(fh.content)) maxMemory -= n + maxValueBytes -= n } form.File[name] = append(form.File[name], fh) } @@ -128,6 +139,7 @@ func (f *Form) RemoveAll() error { type FileHeader struct { Filename string Header textproto.MIMEHeader + Size int64 content []byte tmpfile string diff --git a/libgo/go/mime/multipart/formdata_test.go b/libgo/go/mime/multipart/formdata_test.go index 1deca0b94df..979ae5c4e10 100644 --- a/libgo/go/mime/multipart/formdata_test.go +++ b/libgo/go/mime/multipart/formdata_test.go @@ -8,14 +8,12 @@ import ( "bytes" "io" "os" - "regexp" "strings" "testing" ) func TestReadForm(t *testing.T) { - testBody := regexp.MustCompile("\n").ReplaceAllString(message, "\r\n") - b := strings.NewReader(testBody) + b := strings.NewReader(strings.Replace(message, "\n", "\r\n", -1)) r := NewReader(b, boundary) f, err := r.ReadForm(25) if err != nil { @@ -44,6 +42,9 @@ func testFile(t *testing.T, fh *FileHeader, efn, econtent string) File { if fh.Filename != efn { t.Errorf("filename = %q, want %q", fh.Filename, efn) } + if fh.Size != int64(len(econtent)) { + t.Errorf("size = %d, want %d", fh.Size, len(econtent)) + } f, err := fh.Open() if err != nil { t.Fatal("opening file:", err) @@ -124,3 +125,44 @@ func (r *failOnReadAfterErrorReader) Read(p []byte) (n int, err error) { r.sawErr = err return } + +// TestReadForm_NonFileMaxMemory asserts that the ReadForm maxMemory limit is applied +// while processing non-file form data as well as file form data. +func TestReadForm_NonFileMaxMemory(t *testing.T) { + largeTextValue := strings.Repeat("1", (10<<20)+25) + message := `--MyBoundary +Content-Disposition: form-data; name="largetext" + +` + largeTextValue + ` +--MyBoundary-- +` + + testBody := strings.Replace(message, "\n", "\r\n", -1) + testCases := []struct { + name string + maxMemory int64 + err error + }{ + {"smaller", 50, nil}, + {"exact-fit", 25, nil}, + {"too-large", 0, ErrMessageTooLarge}, + } + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + b := strings.NewReader(testBody) + r := NewReader(b, boundary) + f, err := r.ReadForm(tc.maxMemory) + if err == nil { + defer f.RemoveAll() + } + if tc.err != err { + t.Fatalf("ReadForm error - got: %v; expected: %v", tc.err, err) + } + if err == nil { + if g := f.Value["largetext"][0]; g != largeTextValue { + t.Errorf("largetext mismatch: got size: %v, expected size: %v", len(g), len(largeTextValue)) + } + } + }) + } +} diff --git a/libgo/go/mime/multipart/writer.go b/libgo/go/mime/multipart/writer.go index f82756d5518..3dd0c8fb136 100644 --- a/libgo/go/mime/multipart/writer.go +++ b/libgo/go/mime/multipart/writer.go @@ -41,22 +41,27 @@ func (w *Writer) Boundary() string { // // SetBoundary must be called before any parts are created, may only // contain certain ASCII characters, and must be non-empty and -// at most 69 bytes long. +// at most 70 bytes long. func (w *Writer) SetBoundary(boundary string) error { if w.lastpart != nil { return errors.New("mime: SetBoundary called after write") } // rfc2046#section-5.1.1 - if len(boundary) < 1 || len(boundary) > 69 { + if len(boundary) < 1 || len(boundary) > 70 { return errors.New("mime: invalid boundary length") } - for _, b := range boundary { + end := len(boundary) - 1 + for i, b := range boundary { if 'A' <= b && b <= 'Z' || 'a' <= b && b <= 'z' || '0' <= b && b <= '9' { continue } switch b { case '\'', '(', ')', '+', '_', ',', '-', '.', '/', ':', '=', '?': continue + case ' ': + if i != end { + continue + } } return errors.New("mime: invalid boundary character") } diff --git a/libgo/go/mime/multipart/writer_test.go b/libgo/go/mime/multipart/writer_test.go index 9670c660a4a..8b1bcd68d87 100644 --- a/libgo/go/mime/multipart/writer_test.go +++ b/libgo/go/mime/multipart/writer_test.go @@ -80,8 +80,6 @@ func TestWriter(t *testing.T) { } func TestWriterSetBoundary(t *testing.T) { - var b bytes.Buffer - w := NewWriter(&b) tests := []struct { b string ok bool @@ -90,12 +88,16 @@ func TestWriterSetBoundary(t *testing.T) { {"", false}, {"ungültig", false}, {"!", false}, - {strings.Repeat("x", 69), true}, - {strings.Repeat("x", 70), false}, + {strings.Repeat("x", 70), true}, + {strings.Repeat("x", 71), false}, {"bad!ascii!", false}, {"my-separator", true}, + {"with space", true}, + {"badspace ", false}, } for i, tt := range tests { + var b bytes.Buffer + w := NewWriter(&b) err := w.SetBoundary(tt.b) got := err == nil if got != tt.ok { @@ -105,12 +107,13 @@ func TestWriterSetBoundary(t *testing.T) { if got != tt.b { t.Errorf("boundary = %q; want %q", got, tt.b) } + w.Close() + wantSub := "\r\n--" + tt.b + "--\r\n" + if got := b.String(); !strings.Contains(got, wantSub) { + t.Errorf("expected %q in output. got: %q", wantSub, got) + } } } - w.Close() - if got := b.String(); !strings.Contains(got, "\r\n--my-separator--\r\n") { - t.Errorf("expected my-separator in output. got: %q", got) - } } func TestWriterBoundaryGoroutines(t *testing.T) { diff --git a/libgo/go/mime/type.go b/libgo/go/mime/type.go index d369259d8b1..78fc6b6714e 100644 --- a/libgo/go/mime/type.go +++ b/libgo/go/mime/type.go @@ -12,24 +12,48 @@ import ( ) var ( - mimeLock sync.RWMutex // guards following 3 maps - mimeTypes map[string]string // ".Z" => "application/x-compress" - mimeTypesLower map[string]string // ".z" => "application/x-compress" + mimeTypes sync.Map // map[string]string; ".Z" => "application/x-compress" + mimeTypesLower sync.Map // map[string]string; ".z" => "application/x-compress" // extensions maps from MIME type to list of lowercase file // extensions: "image/jpeg" => [".jpg", ".jpeg"] - extensions map[string][]string + extensionsMu sync.Mutex // Guards stores (but not loads) on extensions. + extensions sync.Map // map[string][]string; slice values are append-only. ) +func clearSyncMap(m *sync.Map) { + m.Range(func(k, _ interface{}) bool { + m.Delete(k) + return true + }) +} + // setMimeTypes is used by initMime's non-test path, and by tests. -// The two maps must not be the same, or nil. func setMimeTypes(lowerExt, mixExt map[string]string) { - if lowerExt == nil || mixExt == nil { - panic("nil map") + clearSyncMap(&mimeTypes) + clearSyncMap(&mimeTypesLower) + clearSyncMap(&extensions) + + for k, v := range lowerExt { + mimeTypesLower.Store(k, v) + } + for k, v := range mixExt { + mimeTypes.Store(k, v) + } + + extensionsMu.Lock() + defer extensionsMu.Unlock() + for k, v := range lowerExt { + justType, _, err := ParseMediaType(v) + if err != nil { + panic(err) + } + var exts []string + if ei, ok := extensions.Load(k); ok { + exts = ei.([]string) + } + extensions.Store(justType, append(exts, k)) } - mimeTypesLower = lowerExt - mimeTypes = mixExt - extensions = invert(lowerExt) } var builtinTypesLower = map[string]string{ @@ -45,29 +69,6 @@ var builtinTypesLower = map[string]string{ ".xml": "text/xml; charset=utf-8", } -func clone(m map[string]string) map[string]string { - m2 := make(map[string]string, len(m)) - for k, v := range m { - m2[k] = v - if strings.ToLower(k) != k { - panic("keys in builtinTypesLower must be lowercase") - } - } - return m2 -} - -func invert(m map[string]string) map[string][]string { - m2 := make(map[string][]string, len(m)) - for k, v := range m { - justType, _, err := ParseMediaType(v) - if err != nil { - panic(err) - } - m2[justType] = append(m2[justType], k) - } - return m2 -} - var once sync.Once // guards initMime var testInitMime, osInitMime func() @@ -76,7 +77,7 @@ func initMime() { if fn := testInitMime; fn != nil { fn() } else { - setMimeTypes(builtinTypesLower, clone(builtinTypesLower)) + setMimeTypes(builtinTypesLower, builtinTypesLower) osInitMime() } } @@ -100,12 +101,10 @@ func initMime() { // Text types have the charset parameter set to "utf-8" by default. func TypeByExtension(ext string) string { once.Do(initMime) - mimeLock.RLock() - defer mimeLock.RUnlock() // Case-sensitive lookup. - if v := mimeTypes[ext]; v != "" { - return v + if v, ok := mimeTypes.Load(ext); ok { + return v.(string) } // Case-insensitive lookup. @@ -118,7 +117,9 @@ func TypeByExtension(ext string) string { c := ext[i] if c >= utf8RuneSelf { // Slow path. - return mimeTypesLower[strings.ToLower(ext)] + si, _ := mimeTypesLower.Load(strings.ToLower(ext)) + s, _ := si.(string) + return s } if 'A' <= c && c <= 'Z' { lower = append(lower, c+('a'-'A')) @@ -126,9 +127,9 @@ func TypeByExtension(ext string) string { lower = append(lower, c) } } - // The conversion from []byte to string doesn't allocate in - // a map lookup. - return mimeTypesLower[string(lower)] + si, _ := mimeTypesLower.Load(string(lower)) + s, _ := si.(string) + return s } // ExtensionsByType returns the extensions known to be associated with the MIME @@ -142,13 +143,11 @@ func ExtensionsByType(typ string) ([]string, error) { } once.Do(initMime) - mimeLock.RLock() - defer mimeLock.RUnlock() - s, ok := extensions[justType] + s, ok := extensions.Load(justType) if !ok { return nil, nil } - return append([]string{}, s...), nil + return append([]string{}, s.([]string)...), nil } // AddExtensionType sets the MIME type associated with @@ -173,15 +172,20 @@ func setExtensionType(extension, mimeType string) error { } extLower := strings.ToLower(extension) - mimeLock.Lock() - defer mimeLock.Unlock() - mimeTypes[extension] = mimeType - mimeTypesLower[extLower] = mimeType - for _, v := range extensions[justType] { + mimeTypes.Store(extension, mimeType) + mimeTypesLower.Store(extLower, mimeType) + + extensionsMu.Lock() + defer extensionsMu.Unlock() + var exts []string + if ei, ok := extensions.Load(justType); ok { + exts = ei.([]string) + } + for _, v := range exts { if v == extLower { return nil } } - extensions[justType] = append(extensions[justType], extLower) + extensions.Store(justType, append(exts, extLower)) return nil } diff --git a/libgo/go/mime/type_test.go b/libgo/go/mime/type_test.go index 48735ef4470..de5c700faaf 100644 --- a/libgo/go/mime/type_test.go +++ b/libgo/go/mime/type_test.go @@ -149,3 +149,43 @@ func TestLookupMallocs(t *testing.T) { t.Errorf("allocs = %v; want 0", n) } } + +func BenchmarkTypeByExtension(b *testing.B) { + initMime() + b.ResetTimer() + + for _, ext := range []string{ + ".html", + ".HTML", + ".unused", + } { + b.Run(ext, func(b *testing.B) { + b.RunParallel(func(pb *testing.PB) { + for pb.Next() { + TypeByExtension(ext) + } + }) + }) + } +} + +func BenchmarkExtensionsByType(b *testing.B) { + initMime() + b.ResetTimer() + + for _, typ := range []string{ + "text/html", + "text/html; charset=utf-8", + "application/octet-stream", + } { + b.Run(typ, func(b *testing.B) { + b.RunParallel(func(pb *testing.PB) { + for pb.Next() { + if _, err := ExtensionsByType(typ); err != nil { + b.Fatal(err) + } + } + }) + }) + } +} diff --git a/libgo/go/net/cgo_unix.go b/libgo/go/net/cgo_unix.go index 09cfb2a71ae..0de3ff8bebd 100644 --- a/libgo/go/net/cgo_unix.go +++ b/libgo/go/net/cgo_unix.go @@ -220,7 +220,7 @@ func cgoLookupIPCNAME(name string) (addrs []IPAddr, cname string, err error) { addrs = append(addrs, addr) case syscall.AF_INET6: sa := (*syscall.RawSockaddrInet6)(unsafe.Pointer(r.Ai_addr)) - addr := IPAddr{IP: copyIP(sa.Addr[:]), Zone: zoneToString(int(sa.Scope_id))} + addr := IPAddr{IP: copyIP(sa.Addr[:]), Zone: zoneCache.name(int(sa.Scope_id))} addrs = append(addrs, addr) } } @@ -345,7 +345,7 @@ func cgoSockaddr(ip IP, zone string) (*syscall.RawSockaddr, syscall.Socklen_t) { return cgoSockaddrInet4(ip4), syscall.Socklen_t(syscall.SizeofSockaddrInet4) } if ip6 := ip.To16(); ip6 != nil { - return cgoSockaddrInet6(ip6, zoneToInt(zone)), syscall.Socklen_t(syscall.SizeofSockaddrInet6) + return cgoSockaddrInet6(ip6, zoneCache.index(zone)), syscall.Socklen_t(syscall.SizeofSockaddrInet6) } return nil, 0 } diff --git a/libgo/go/net/dial.go b/libgo/go/net/dial.go index 50bba5a49e4..f8b4aa22742 100644 --- a/libgo/go/net/dial.go +++ b/libgo/go/net/dial.go @@ -7,6 +7,7 @@ package net import ( "context" "internal/nettrace" + "internal/poll" "time" ) @@ -22,8 +23,8 @@ type Dialer struct { // // The default is no timeout. // - // When dialing a name with multiple IP addresses, the timeout - // may be divided between them. + // When using TCP and dialing a host name with multiple IP + // addresses, the timeout may be divided between them. // // With or without a timeout, the operating system may impose // its own earlier timeout. For instance, TCP timeouts are @@ -42,10 +43,11 @@ type Dialer struct { // If nil, a local address is automatically chosen. LocalAddr Addr - // DualStack enables RFC 6555-compliant "Happy Eyeballs" dialing - // when the network is "tcp" and the destination is a host name - // with both IPv4 and IPv6 addresses. This allows a client to - // tolerate networks where one address family is silently broken. + // DualStack enables RFC 6555-compliant "Happy Eyeballs" + // dialing when the network is "tcp" and the host in the + // address parameter resolves to both IPv4 and IPv6 addresses. + // This allows a client to tolerate networks where one address + // family is silently broken. DualStack bool // FallbackDelay specifies the length of time to wait before @@ -110,7 +112,7 @@ func partialDeadline(now, deadline time.Time, addrsRemaining int) (time.Time, er } timeRemaining := deadline.Sub(now) if timeRemaining <= 0 { - return time.Time{}, errTimeout + return time.Time{}, poll.ErrTimeout } // Tentatively allocate equal time to each remaining address. timeout := timeRemaining / time.Duration(addrsRemaining) @@ -134,23 +136,26 @@ func (d *Dialer) fallbackDelay() time.Duration { } } -func parseNetwork(ctx context.Context, net string) (afnet string, proto int, err error) { - i := last(net, ':') +func parseNetwork(ctx context.Context, network string, needsProto bool) (afnet string, proto int, err error) { + i := last(network, ':') if i < 0 { // no colon - switch net { + switch network { case "tcp", "tcp4", "tcp6": case "udp", "udp4", "udp6": case "ip", "ip4", "ip6": + if needsProto { + return "", 0, UnknownNetworkError(network) + } case "unix", "unixgram", "unixpacket": default: - return "", 0, UnknownNetworkError(net) + return "", 0, UnknownNetworkError(network) } - return net, 0, nil + return network, 0, nil } - afnet = net[:i] + afnet = network[:i] switch afnet { case "ip", "ip4", "ip6": - protostr := net[i+1:] + protostr := network[i+1:] proto, i, ok := dtoi(protostr) if !ok || i != len(protostr) { proto, err = lookupProtocol(ctx, protostr) @@ -160,14 +165,14 @@ func parseNetwork(ctx context.Context, net string) (afnet string, proto int, err } return afnet, proto, nil } - return "", 0, UnknownNetworkError(net) + return "", 0, UnknownNetworkError(network) } // resolveAddrList resolves addr using hint and returns a list of // addresses. The result contains at least one address when error is // nil. func (r *Resolver) resolveAddrList(ctx context.Context, op, network, addr string, hint Addr) (addrList, error) { - afnet, _, err := parseNetwork(ctx, network) + afnet, _, err := parseNetwork(ctx, network, true) if err != nil { return nil, err } @@ -242,39 +247,60 @@ func (r *Resolver) resolveAddrList(ctx context.Context, op, network, addr string // (IPv4-only), "ip6" (IPv6-only), "unix", "unixgram" and // "unixpacket". // -// For TCP and UDP networks, addresses have the form host:port. -// If host is a literal IPv6 address it must be enclosed -// in square brackets as in "[::1]:80" or "[ipv6-host%zone]:80". -// The functions JoinHostPort and SplitHostPort manipulate addresses -// in this form. -// If the host is empty, as in ":80", the local system is assumed. +// For TCP and UDP networks, the address has the form "host:port". +// The host must be a literal IP address, or a host name that can be +// resolved to IP addresses. +// The port must be a literal port number or a service name. +// If the host is a literal IPv6 address it must be enclosed in square +// brackets, as in "[2001:db8::1]:80" or "[fe80::1%zone]:80". +// The zone specifies the scope of the literal IPv6 address as defined +// in RFC 4007. +// The functions JoinHostPort and SplitHostPort manipulate a pair of +// host and port in this form. +// When using TCP, and the host resolves to multiple IP addresses, +// Dial will try each IP address in order until one succeeds. // // Examples: -// Dial("tcp", "192.0.2.1:80") // Dial("tcp", "golang.org:http") -// Dial("tcp", "[2001:db8::1]:http") -// Dial("tcp", "[fe80::1%lo0]:80") +// Dial("tcp", "192.0.2.1:http") +// Dial("tcp", "198.51.100.1:80") +// Dial("udp", "[2001:db8::1]:domain") +// Dial("udp", "[fe80::1%lo0]:53") // Dial("tcp", ":80") // // For IP networks, the network must be "ip", "ip4" or "ip6" followed -// by a colon and a protocol number or name and the addr must be a -// literal IP address. +// by a colon and a literal protocol number or a protocol name, and +// the address has the form "host". The host must be a literal IP +// address or a literal IPv6 address with zone. +// It depends on each operating system how the operating system +// behaves with a non-well known protocol number such as "0" or "255". // // Examples: // Dial("ip4:1", "192.0.2.1") // Dial("ip6:ipv6-icmp", "2001:db8::1") +// Dial("ip6:58", "fe80::1%lo0") // -// For Unix networks, the address must be a file system path. +// For TCP, UDP and IP networks, if the host is empty or a literal +// unspecified IP address, as in ":80", "0.0.0.0:80" or "[::]:80" for +// TCP and UDP, "", "0.0.0.0" or "::" for IP, the local system is +// assumed. // -// If the host is resolved to multiple addresses, -// Dial will try each address in order until one succeeds. +// For Unix networks, the address must be a file system path. func Dial(network, address string) (Conn, error) { var d Dialer return d.Dial(network, address) } // DialTimeout acts like Dial but takes a timeout. +// // The timeout includes name resolution, if required. +// When using TCP, and the host in the address parameter resolves to +// multiple IP addresses, the timeout is spread over each consecutive +// dial, such that each is given an appropriate fraction of the time +// to connect. +// +// See func Dial for a description of the network and address +// parameters. func DialTimeout(network, address string, timeout time.Duration) (Conn, error) { d := Dialer{Timeout: timeout} return d.Dial(network, address) @@ -537,29 +563,37 @@ func dialSingle(ctx context.Context, dp *dialParam, ra Addr) (c Conn, err error) return c, nil } -// Listen announces on the local network address laddr. -// The network net must be a stream-oriented network: "tcp", "tcp4", -// "tcp6", "unix" or "unixpacket". -// For TCP and UDP, the syntax of laddr is "host:port", like "127.0.0.1:8080". -// If host is omitted, as in ":8080", Listen listens on all available interfaces -// instead of just the interface with the given host address. -// See Dial for more details about address syntax. +// Listen announces on the local network address. +// +// The network must be "tcp", "tcp4", "tcp6", "unix" or "unixpacket". // -// Listening on a hostname is not recommended because this creates a socket -// for at most one of its IP addresses. -func Listen(net, laddr string) (Listener, error) { - addrs, err := DefaultResolver.resolveAddrList(context.Background(), "listen", net, laddr, nil) +// For TCP networks, if the host in the address parameter is empty or +// a literal unspecified IP address, Listen listens on all available +// unicast and anycast IP addresses of the local system. +// To only use IPv4, use network "tcp4". +// The address can use a host name, but this is not recommended, +// because it will create a listener for at most one of the host's IP +// addresses. +// If the port in the address parameter is empty or "0", as in +// "127.0.0.1:" or "[::1]:0", a port number is automatically chosen. +// The Addr method of Listener can be used to discover the chosen +// port. +// +// See func Dial for a description of the network and address +// parameters. +func Listen(network, address string) (Listener, error) { + addrs, err := DefaultResolver.resolveAddrList(context.Background(), "listen", network, address, nil) if err != nil { - return nil, &OpError{Op: "listen", Net: net, Source: nil, Addr: nil, Err: err} + return nil, &OpError{Op: "listen", Net: network, Source: nil, Addr: nil, Err: err} } var l Listener switch la := addrs.first(isIPv4).(type) { case *TCPAddr: - l, err = ListenTCP(net, la) + l, err = ListenTCP(network, la) case *UnixAddr: - l, err = ListenUnix(net, la) + l, err = ListenUnix(network, la) default: - return nil, &OpError{Op: "listen", Net: net, Source: nil, Addr: la, Err: &AddrError{Err: "unexpected address type", Addr: laddr}} + return nil, &OpError{Op: "listen", Net: network, Source: nil, Addr: la, Err: &AddrError{Err: "unexpected address type", Addr: address}} } if err != nil { return nil, err // l is non-nil interface containing nil pointer @@ -567,31 +601,43 @@ func Listen(net, laddr string) (Listener, error) { return l, nil } -// ListenPacket announces on the local network address laddr. -// The network net must be a packet-oriented network: "udp", "udp4", -// "udp6", "ip", "ip4", "ip6" or "unixgram". -// For TCP and UDP, the syntax of laddr is "host:port", like "127.0.0.1:8080". -// If host is omitted, as in ":8080", ListenPacket listens on all available interfaces -// instead of just the interface with the given host address. -// See Dial for the syntax of laddr. +// ListenPacket announces on the local network address. +// +// The network must be "udp", "udp4", "udp6", "unixgram", or an IP +// transport. The IP transports are "ip", "ip4", or "ip6" followed by +// a colon and a literal protocol number or a protocol name, as in +// "ip:1" or "ip:icmp". // -// Listening on a hostname is not recommended because this creates a socket -// for at most one of its IP addresses. -func ListenPacket(net, laddr string) (PacketConn, error) { - addrs, err := DefaultResolver.resolveAddrList(context.Background(), "listen", net, laddr, nil) +// For UDP and IP networks, if the host in the address parameter is +// empty or a literal unspecified IP address, ListenPacket listens on +// all available IP addresses of the local system except multicast IP +// addresses. +// To only use IPv4, use network "udp4" or "ip4:proto". +// The address can use a host name, but this is not recommended, +// because it will create a listener for at most one of the host's IP +// addresses. +// If the port in the address parameter is empty or "0", as in +// "127.0.0.1:" or "[::1]:0", a port number is automatically chosen. +// The LocalAddr method of PacketConn can be used to discover the +// chosen port. +// +// See func Dial for a description of the network and address +// parameters. +func ListenPacket(network, address string) (PacketConn, error) { + addrs, err := DefaultResolver.resolveAddrList(context.Background(), "listen", network, address, nil) if err != nil { - return nil, &OpError{Op: "listen", Net: net, Source: nil, Addr: nil, Err: err} + return nil, &OpError{Op: "listen", Net: network, Source: nil, Addr: nil, Err: err} } var l PacketConn switch la := addrs.first(isIPv4).(type) { case *UDPAddr: - l, err = ListenUDP(net, la) + l, err = ListenUDP(network, la) case *IPAddr: - l, err = ListenIP(net, la) + l, err = ListenIP(network, la) case *UnixAddr: - l, err = ListenUnixgram(net, la) + l, err = ListenUnixgram(network, la) default: - return nil, &OpError{Op: "listen", Net: net, Source: nil, Addr: la, Err: &AddrError{Err: "unexpected address type", Addr: laddr}} + return nil, &OpError{Op: "listen", Net: network, Source: nil, Addr: la, Err: &AddrError{Err: "unexpected address type", Addr: address}} } if err != nil { return nil, err // l is non-nil interface containing nil pointer diff --git a/libgo/go/net/dial_test.go b/libgo/go/net/dial_test.go index 9919d72ce3b..a892bf1e140 100644 --- a/libgo/go/net/dial_test.go +++ b/libgo/go/net/dial_test.go @@ -7,9 +7,9 @@ package net import ( "bufio" "context" + "internal/poll" "internal/testenv" "io" - "net/internal/socktest" "runtime" "sync" "testing" @@ -31,7 +31,7 @@ func TestProhibitionaryDialArg(t *testing.T) { case "plan9": t.Skipf("not supported on %s", runtime.GOOS) } - if !supportsIPv4map { + if !supportsIPv4map() { t.Skip("mapping ipv4 address inside ipv6 address not supported") } @@ -72,70 +72,6 @@ func TestDialLocal(t *testing.T) { c.Close() } -func TestDialTimeoutFDLeak(t *testing.T) { - switch runtime.GOOS { - case "plan9": - t.Skipf("%s does not have full support of socktest", runtime.GOOS) - case "openbsd": - testenv.SkipFlaky(t, 15157) - } - - const T = 100 * time.Millisecond - - switch runtime.GOOS { - case "plan9", "windows": - origTestHookDialChannel := testHookDialChannel - testHookDialChannel = func() { time.Sleep(2 * T) } - defer func() { testHookDialChannel = origTestHookDialChannel }() - if runtime.GOOS == "plan9" { - break - } - fallthrough - default: - sw.Set(socktest.FilterConnect, func(so *socktest.Status) (socktest.AfterFilter, error) { - time.Sleep(2 * T) - return nil, errTimeout - }) - defer sw.Set(socktest.FilterConnect, nil) - } - - // Avoid tracking open-close jitterbugs between netFD and - // socket that leads to confusion of information inside - // socktest.Switch. - // It may happen when the Dial call bumps against TCP - // simultaneous open. See selfConnect in tcpsock_posix.go. - defer func() { sw.Set(socktest.FilterClose, nil) }() - var mu sync.Mutex - var attempts int - sw.Set(socktest.FilterClose, func(so *socktest.Status) (socktest.AfterFilter, error) { - mu.Lock() - attempts++ - mu.Unlock() - return nil, nil - }) - - const N = 100 - var wg sync.WaitGroup - wg.Add(N) - for i := 0; i < N; i++ { - go func() { - defer wg.Done() - // This dial never starts to send any SYN - // segment because of above socket filter and - // test hook. - c, err := DialTimeout("tcp", "127.0.0.1:0", T) - if err == nil { - t.Errorf("unexpectedly established: tcp:%s->%s", c.LocalAddr(), c.RemoteAddr()) - c.Close() - } - }() - } - wg.Wait() - if attempts < N { - t.Errorf("got %d; want >= %d", attempts, N) - } -} - func TestDialerDualStackFDLeak(t *testing.T) { switch runtime.GOOS { case "plan9": @@ -145,7 +81,7 @@ func TestDialerDualStackFDLeak(t *testing.T) { case "openbsd": testenv.SkipFlaky(t, 15157) } - if !supportsIPv4 || !supportsIPv6 { + if !supportsIPv4() || !supportsIPv6() { t.Skip("both IPv4 and IPv6 are required") } @@ -254,7 +190,7 @@ func dialClosedPort() (actual, expected time.Duration) { func TestDialParallel(t *testing.T) { testenv.MustHaveExternalNetwork(t) - if !supportsIPv4 || !supportsIPv6 { + if !supportsIPv4() || !supportsIPv6() { t.Skip("both IPv4 and IPv6 are required") } @@ -425,7 +361,7 @@ func lookupSlowFast(ctx context.Context, fn func(context.Context, string) ([]IPA func TestDialerFallbackDelay(t *testing.T) { testenv.MustHaveExternalNetwork(t) - if !supportsIPv4 || !supportsIPv6 { + if !supportsIPv4() || !supportsIPv6() { t.Skip("both IPv4 and IPv6 are required") } @@ -491,7 +427,7 @@ func TestDialerFallbackDelay(t *testing.T) { } func TestDialParallelSpuriousConnection(t *testing.T) { - if !supportsIPv4 || !supportsIPv6 { + if !supportsIPv4() || !supportsIPv6() { t.Skip("both IPv4 and IPv6 are required") } @@ -585,22 +521,22 @@ func TestDialerPartialDeadline(t *testing.T) { {now, noDeadline, 1, noDeadline, nil}, // Step the clock forward and cross the deadline. {now.Add(-1 * time.Millisecond), now, 1, now, nil}, - {now.Add(0 * time.Millisecond), now, 1, noDeadline, errTimeout}, - {now.Add(1 * time.Millisecond), now, 1, noDeadline, errTimeout}, + {now.Add(0 * time.Millisecond), now, 1, noDeadline, poll.ErrTimeout}, + {now.Add(1 * time.Millisecond), now, 1, noDeadline, poll.ErrTimeout}, } for i, tt := range testCases { deadline, err := partialDeadline(tt.now, tt.deadline, tt.addrs) if err != tt.expectErr { t.Errorf("#%d: got %v; want %v", i, err, tt.expectErr) } - if deadline != tt.expectDeadline { + if !deadline.Equal(tt.expectDeadline) { t.Errorf("#%d: got %v; want %v", i, deadline, tt.expectDeadline) } } } func TestDialerLocalAddr(t *testing.T) { - if !supportsIPv4 || !supportsIPv6 { + if !supportsIPv4() || !supportsIPv6() { t.Skip("both IPv4 and IPv6 are required") } @@ -654,7 +590,7 @@ func TestDialerLocalAddr(t *testing.T) { {"tcp", "::1", &UnixAddr{}, &AddrError{Err: "some error"}}, } - if supportsIPv4map { + if supportsIPv4map() { tests = append(tests, test{ "tcp", "127.0.0.1", &TCPAddr{IP: ParseIP("::")}, nil, }) @@ -714,12 +650,9 @@ func TestDialerLocalAddr(t *testing.T) { } func TestDialerDualStack(t *testing.T) { - // This test is known to be flaky. Don't frighten regular - // users about it; only fail on the build dashboard. - if testenv.Builder() == "" { - testenv.SkipFlaky(t, 13324) - } - if !supportsIPv4 || !supportsIPv6 { + testenv.SkipFlaky(t, 13324) + + if !supportsIPv4() || !supportsIPv6() { t.Skip("both IPv4 and IPv6 are required") } @@ -822,7 +755,7 @@ func TestDialCancel(t *testing.T) { } blackholeIPPort := JoinHostPort(slowDst4, "1234") - if !supportsIPv4 { + if !supportsIPv4() { blackholeIPPort = JoinHostPort(slowDst6, "1234") } @@ -954,3 +887,24 @@ func TestCancelAfterDial(t *testing.T) { try() } } + +// Issue 18806: it should always be possible to net.Dial a +// net.Listener().Addr().String when the listen address was ":n", even +// if the machine has halfway configured IPv6 such that it can bind on +// "::" not connect back to that same address. +func TestDialListenerAddr(t *testing.T) { + if testenv.Builder() == "" { + testenv.MustHaveExternalNetwork(t) + } + ln, err := Listen("tcp", ":0") + if err != nil { + t.Fatal(err) + } + defer ln.Close() + addr := ln.Addr().String() + c, err := Dial("tcp", addr) + if err != nil { + t.Fatalf("for addr %q, dial error: %v", addr, err) + } + c.Close() +} diff --git a/libgo/go/net/dnsclient_unix.go b/libgo/go/net/dnsclient_unix.go index 0647b9c3052..ff6a4f69dcd 100644 --- a/libgo/go/net/dnsclient_unix.go +++ b/libgo/go/net/dnsclient_unix.go @@ -25,13 +25,6 @@ import ( "time" ) -// A dnsDialer provides dialing suitable for DNS queries. -type dnsDialer interface { - dialDNS(ctx context.Context, network, addr string) (dnsConn, error) -} - -var testHookDNSDialer = func() dnsDialer { return &Dialer{} } - // A dnsConn represents a DNS transport endpoint. type dnsConn interface { io.Closer @@ -43,14 +36,14 @@ type dnsConn interface { dnsRoundTrip(query *dnsMsg) (*dnsMsg, error) } -func (c *UDPConn) dnsRoundTrip(query *dnsMsg) (*dnsMsg, error) { - return dnsRoundTripUDP(c, query) +// dnsPacketConn implements the dnsConn interface for RFC 1035's +// "UDP usage" transport mechanism. Conn is a packet-oriented connection, +// such as a *UDPConn. +type dnsPacketConn struct { + Conn } -// dnsRoundTripUDP implements the dnsRoundTrip interface for RFC 1035's -// "UDP usage" transport mechanism. c should be a packet-oriented connection, -// such as a *UDPConn. -func dnsRoundTripUDP(c io.ReadWriter, query *dnsMsg) (*dnsMsg, error) { +func (c *dnsPacketConn) dnsRoundTrip(query *dnsMsg) (*dnsMsg, error) { b, ok := query.Pack() if !ok { return nil, errors.New("cannot marshal DNS message") @@ -76,14 +69,14 @@ func dnsRoundTripUDP(c io.ReadWriter, query *dnsMsg) (*dnsMsg, error) { } } -func (c *TCPConn) dnsRoundTrip(out *dnsMsg) (*dnsMsg, error) { - return dnsRoundTripTCP(c, out) +// dnsStreamConn implements the dnsConn interface for RFC 1035's +// "TCP usage" transport mechanism. Conn is a stream-oriented connection, +// such as a *TCPConn. +type dnsStreamConn struct { + Conn } -// dnsRoundTripTCP implements the dnsRoundTrip interface for RFC 1035's -// "TCP usage" transport mechanism. c should be a stream-oriented connection, -// such as a *TCPConn. -func dnsRoundTripTCP(c io.ReadWriter, query *dnsMsg) (*dnsMsg, error) { +func (c *dnsStreamConn) dnsRoundTrip(query *dnsMsg) (*dnsMsg, error) { b, ok := query.Pack() if !ok { return nil, errors.New("cannot marshal DNS message") @@ -116,33 +109,8 @@ func dnsRoundTripTCP(c io.ReadWriter, query *dnsMsg) (*dnsMsg, error) { return resp, nil } -func (d *Dialer) dialDNS(ctx context.Context, network, server string) (dnsConn, error) { - switch network { - case "tcp", "tcp4", "tcp6", "udp", "udp4", "udp6": - default: - return nil, UnknownNetworkError(network) - } - // Calling Dial here is scary -- we have to be sure not to - // dial a name that will require a DNS lookup, or Dial will - // call back here to translate it. The DNS config parser has - // already checked that all the cfg.servers are IP - // addresses, which Dial will use without a DNS lookup. - c, err := d.DialContext(ctx, network, server) - if err != nil { - return nil, mapErr(err) - } - switch network { - case "tcp", "tcp4", "tcp6": - return c.(*TCPConn), nil - case "udp", "udp4", "udp6": - return c.(*UDPConn), nil - } - panic("unreachable") -} - // exchange sends a query on the connection and hopes for a response. -func exchange(ctx context.Context, server, name string, qtype uint16, timeout time.Duration) (*dnsMsg, error) { - d := testHookDNSDialer() +func (r *Resolver) exchange(ctx context.Context, server, name string, qtype uint16, timeout time.Duration) (*dnsMsg, error) { out := dnsMsg{ dnsMsgHdr: dnsMsgHdr{ recursion_desired: true, @@ -158,7 +126,7 @@ func exchange(ctx context.Context, server, name string, qtype uint16, timeout ti ctx, cancel := context.WithDeadline(ctx, time.Now().Add(timeout)) defer cancel() - c, err := d.dialDNS(ctx, network, server) + c, err := r.dial(ctx, network, server) if err != nil { return nil, err } @@ -181,7 +149,7 @@ func exchange(ctx context.Context, server, name string, qtype uint16, timeout ti // Do a lookup for a single name, which must be rooted // (otherwise answer will not find the answers). -func tryOneName(ctx context.Context, cfg *dnsConfig, name string, qtype uint16) (string, []dnsRR, error) { +func (r *Resolver) tryOneName(ctx context.Context, cfg *dnsConfig, name string, qtype uint16) (string, []dnsRR, error) { var lastErr error serverOffset := cfg.serverOffset() sLen := uint32(len(cfg.servers)) @@ -190,7 +158,7 @@ func tryOneName(ctx context.Context, cfg *dnsConfig, name string, qtype uint16) for j := uint32(0); j < sLen; j++ { server := cfg.servers[(serverOffset+j)%sLen] - msg, err := exchange(ctx, server, name, qtype, cfg.timeout) + msg, err := r.exchange(ctx, server, name, qtype, cfg.timeout) if err != nil { lastErr = &DNSError{ Err: err.Error(), @@ -200,6 +168,11 @@ func tryOneName(ctx context.Context, cfg *dnsConfig, name string, qtype uint16) if nerr, ok := err.(Error); ok && nerr.Timeout() { lastErr.(*DNSError).IsTimeout = true } + // Set IsTemporary for socket-level errors. Note that this flag + // may also be used to indicate a SERVFAIL response. + if _, ok := err.(*OpError); ok { + lastErr.(*DNSError).IsTemporary = true + } continue } // libresolv continues to the next server when it receives @@ -314,7 +287,7 @@ func (conf *resolverConfig) releaseSema() { <-conf.ch } -func lookup(ctx context.Context, name string, qtype uint16) (cname string, rrs []dnsRR, err error) { +func (r *Resolver) lookup(ctx context.Context, name string, qtype uint16) (cname string, rrs []dnsRR, err error) { if !isDomainName(name) { // We used to use "invalid domain name" as the error, // but that is a detail of the specific lookup mechanism. @@ -328,10 +301,15 @@ func lookup(ctx context.Context, name string, qtype uint16) (cname string, rrs [ conf := resolvConf.dnsConfig resolvConf.mu.RUnlock() for _, fqdn := range conf.nameList(name) { - cname, rrs, err = tryOneName(ctx, conf, fqdn, qtype) + cname, rrs, err = r.tryOneName(ctx, conf, fqdn, qtype) if err == nil { break } + if nerr, ok := err.(Error); ok && nerr.Temporary() && r.StrictErrors { + // If we hit a temporary error with StrictErrors enabled, + // stop immediately instead of trying more names. + break + } } if err, ok := err.(*DNSError); ok { // Show original name passed to lookup, not suffixed one. @@ -432,11 +410,11 @@ func (o hostLookupOrder) String() string { // Normally we let cgo use the C library resolver instead of // depending on our lookup code, so that Go and C get the same // answers. -func goLookupHost(ctx context.Context, name string) (addrs []string, err error) { - return goLookupHostOrder(ctx, name, hostLookupFilesDNS) +func (r *Resolver) goLookupHost(ctx context.Context, name string) (addrs []string, err error) { + return r.goLookupHostOrder(ctx, name, hostLookupFilesDNS) } -func goLookupHostOrder(ctx context.Context, name string, order hostLookupOrder) (addrs []string, err error) { +func (r *Resolver) goLookupHostOrder(ctx context.Context, name string, order hostLookupOrder) (addrs []string, err error) { if order == hostLookupFilesDNS || order == hostLookupFiles { // Use entries from /etc/hosts if they match. addrs = lookupStaticHost(name) @@ -444,7 +422,7 @@ func goLookupHostOrder(ctx context.Context, name string, order hostLookupOrder) return } } - ips, _, err := goLookupIPCNAMEOrder(ctx, name, order) + ips, _, err := r.goLookupIPCNAMEOrder(ctx, name, order) if err != nil { return } @@ -470,13 +448,13 @@ func goLookupIPFiles(name string) (addrs []IPAddr) { // goLookupIP is the native Go implementation of LookupIP. // The libc versions are in cgo_*.go. -func goLookupIP(ctx context.Context, host string) (addrs []IPAddr, err error) { +func (r *Resolver) goLookupIP(ctx context.Context, host string) (addrs []IPAddr, err error) { order := systemConf().hostLookupOrder(host) - addrs, _, err = goLookupIPCNAMEOrder(ctx, host, order) + addrs, _, err = r.goLookupIPCNAMEOrder(ctx, host, order) return } -func goLookupIPCNAMEOrder(ctx context.Context, name string, order hostLookupOrder) (addrs []IPAddr, cname string, err error) { +func (r *Resolver) goLookupIPCNAMEOrder(ctx context.Context, name string, order hostLookupOrder) (addrs []IPAddr, cname string, err error) { if order == hostLookupFilesDNS || order == hostLookupFiles { addrs = goLookupIPFiles(name) if len(addrs) > 0 || order == hostLookupFiles { @@ -502,15 +480,20 @@ func goLookupIPCNAMEOrder(ctx context.Context, name string, order hostLookupOrde for _, fqdn := range conf.nameList(name) { for _, qtype := range qtypes { go func(qtype uint16) { - cname, rrs, err := tryOneName(ctx, conf, fqdn, qtype) + cname, rrs, err := r.tryOneName(ctx, conf, fqdn, qtype) lane <- racer{cname, rrs, err} }(qtype) } + hitStrictError := false for range qtypes { racer := <-lane if racer.error != nil { - // Prefer error for original name. - if lastErr == nil || fqdn == name+"." { + if nerr, ok := racer.error.(Error); ok && nerr.Temporary() && r.StrictErrors { + // This error will abort the nameList loop. + hitStrictError = true + lastErr = racer.error + } else if lastErr == nil || fqdn == name+"." { + // Prefer error for original name. lastErr = racer.error } continue @@ -520,6 +503,13 @@ func goLookupIPCNAMEOrder(ctx context.Context, name string, order hostLookupOrde cname = racer.cname } } + if hitStrictError { + // If either family hit an error with StrictErrors enabled, + // discard all addresses. This ensures that network flakiness + // cannot turn a dualstack hostname IPv4/IPv6-only. + addrs = nil + break + } if len(addrs) > 0 { break } @@ -543,9 +533,9 @@ func goLookupIPCNAMEOrder(ctx context.Context, name string, order hostLookupOrde } // goLookupCNAME is the native Go (non-cgo) implementation of LookupCNAME. -func goLookupCNAME(ctx context.Context, host string) (cname string, err error) { +func (r *Resolver) goLookupCNAME(ctx context.Context, host string) (cname string, err error) { order := systemConf().hostLookupOrder(host) - _, cname, err = goLookupIPCNAMEOrder(ctx, host, order) + _, cname, err = r.goLookupIPCNAMEOrder(ctx, host, order) return } @@ -554,7 +544,7 @@ func goLookupCNAME(ctx context.Context, host string) (cname string, err error) { // only if cgoLookupPTR is the stub in cgo_stub.go). // Normally we let cgo use the C library resolver instead of depending // on our lookup code, so that Go and C get the same answers. -func goLookupPTR(ctx context.Context, addr string) ([]string, error) { +func (r *Resolver) goLookupPTR(ctx context.Context, addr string) ([]string, error) { names := lookupStaticAddr(addr) if len(names) > 0 { return names, nil @@ -563,7 +553,7 @@ func goLookupPTR(ctx context.Context, addr string) ([]string, error) { if err != nil { return nil, err } - _, rrs, err := lookup(ctx, arpa, dnsTypePTR) + _, rrs, err := r.lookup(ctx, arpa, dnsTypePTR) if err != nil { return nil, err } diff --git a/libgo/go/net/dnsclient_unix_test.go b/libgo/go/net/dnsclient_unix_test.go index c66d2d196b2..94811c96a6b 100644 --- a/libgo/go/net/dnsclient_unix_test.go +++ b/libgo/go/net/dnsclient_unix_test.go @@ -8,8 +8,9 @@ package net import ( "context" + "errors" "fmt" - "internal/testenv" + "internal/poll" "io/ioutil" "os" "path" @@ -20,9 +21,14 @@ import ( "time" ) +var goResolver = Resolver{PreferGo: true} + // Test address from 192.0.2.0/24 block, reserved by RFC 5737 for documentation. const TestAddr uint32 = 0xc0000201 +// Test address from 2001:db8::/32 block, reserved by RFC 3849 for documentation. +var TestAddr6 = [16]byte{0x20, 0x01, 0x0d, 0xb8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1} + var dnsTransportFallbackTests = []struct { server string name string @@ -37,18 +43,33 @@ var dnsTransportFallbackTests = []struct { } func TestDNSTransportFallback(t *testing.T) { - testenv.MustHaveExternalNetwork(t) - + fake := fakeDNSServer{ + rh: func(n, _ string, q *dnsMsg, _ time.Time) (*dnsMsg, error) { + r := &dnsMsg{ + dnsMsgHdr: dnsMsgHdr{ + id: q.id, + response: true, + rcode: dnsRcodeSuccess, + }, + question: q.question, + } + if n == "udp" { + r.truncated = true + } + return r, nil + }, + } + r := Resolver{PreferGo: true, Dial: fake.DialContext} for _, tt := range dnsTransportFallbackTests { ctx, cancel := context.WithCancel(context.Background()) defer cancel() - msg, err := exchange(ctx, tt.server, tt.name, tt.qtype, time.Second) + msg, err := r.exchange(ctx, tt.server, tt.name, tt.qtype, time.Second) if err != nil { t.Error(err) continue } switch msg.rcode { - case tt.rcode, dnsRcodeServerFailure: + case tt.rcode: default: t.Errorf("got %v from %v; want %v", msg.rcode, tt.server, tt.rcode) continue @@ -78,13 +99,30 @@ var specialDomainNameTests = []struct { } func TestSpecialDomainName(t *testing.T) { - testenv.MustHaveExternalNetwork(t) + fake := fakeDNSServer{func(_, _ string, q *dnsMsg, _ time.Time) (*dnsMsg, error) { + r := &dnsMsg{ + dnsMsgHdr: dnsMsgHdr{ + id: q.id, + response: true, + }, + question: q.question, + } + + switch q.question[0].Name { + case "example.com.": + r.rcode = dnsRcodeSuccess + default: + r.rcode = dnsRcodeNameError + } + return r, nil + }} + r := Resolver{PreferGo: true, Dial: fake.DialContext} server := "8.8.8.8:53" for _, tt := range specialDomainNameTests { ctx, cancel := context.WithCancel(context.Background()) defer cancel() - msg, err := exchange(ctx, server, tt.name, tt.qtype, 3*time.Second) + msg, err := r.exchange(ctx, server, tt.name, tt.qtype, 3*time.Second) if err != nil { t.Error(err) continue @@ -139,15 +177,40 @@ func TestAvoidDNSName(t *testing.T) { } } +var fakeDNSServerSuccessful = fakeDNSServer{func(_, _ string, q *dnsMsg, _ time.Time) (*dnsMsg, error) { + r := &dnsMsg{ + dnsMsgHdr: dnsMsgHdr{ + id: q.id, + response: true, + }, + question: q.question, + } + if len(q.question) == 1 && q.question[0].Qtype == dnsTypeA { + r.answer = []dnsRR{ + &dnsRR_A{ + Hdr: dnsRR_Header{ + Name: q.question[0].Name, + Rrtype: dnsTypeA, + Class: dnsClassINET, + Rdlength: 4, + }, + A: TestAddr, + }, + } + } + return r, nil +}} + // Issue 13705: don't try to resolve onion addresses, etc func TestLookupTorOnion(t *testing.T) { - addrs, err := goLookupIP(context.Background(), "foo.onion") - if len(addrs) > 0 { - t.Errorf("unexpected addresses: %v", addrs) - } + r := Resolver{PreferGo: true, Dial: fakeDNSServerSuccessful.DialContext} + addrs, err := r.LookupIPAddr(context.Background(), "foo.onion") if err != nil { t.Fatalf("lookup = %v; want nil", err) } + if len(addrs) > 0 { + t.Errorf("unexpected addresses: %v", addrs) + } } type resolvConfTest struct { @@ -237,7 +300,7 @@ var updateResolvConfTests = []struct { } func TestUpdateResolvConf(t *testing.T) { - testenv.MustHaveExternalNetwork(t) + r := Resolver{PreferGo: true, Dial: fakeDNSServerSuccessful.DialContext} conf, err := newResolvConfTest() if err != nil { @@ -257,7 +320,7 @@ func TestUpdateResolvConf(t *testing.T) { for j := 0; j < N; j++ { go func(name string) { defer wg.Done() - ips, err := goLookupIP(context.Background(), name) + ips, err := r.LookupIPAddr(context.Background(), name) if err != nil { t.Error(err) return @@ -392,7 +455,60 @@ var goLookupIPWithResolverConfigTests = []struct { } func TestGoLookupIPWithResolverConfig(t *testing.T) { - testenv.MustHaveExternalNetwork(t) + fake := fakeDNSServer{func(n, s string, q *dnsMsg, _ time.Time) (*dnsMsg, error) { + switch s { + case "[2001:4860:4860::8888]:53", "8.8.8.8:53": + break + default: + time.Sleep(10 * time.Millisecond) + return nil, poll.ErrTimeout + } + r := &dnsMsg{ + dnsMsgHdr: dnsMsgHdr{ + id: q.id, + response: true, + }, + question: q.question, + } + for _, question := range q.question { + switch question.Qtype { + case dnsTypeA: + switch question.Name { + case "hostname.as112.net.": + break + case "ipv4.google.com.": + r.answer = append(r.answer, &dnsRR_A{ + Hdr: dnsRR_Header{ + Name: q.question[0].Name, + Rrtype: dnsTypeA, + Class: dnsClassINET, + Rdlength: 4, + }, + A: TestAddr, + }) + default: + + } + case dnsTypeAAAA: + switch question.Name { + case "hostname.as112.net.": + break + case "ipv6.google.com.": + r.answer = append(r.answer, &dnsRR_AAAA{ + Hdr: dnsRR_Header{ + Name: q.question[0].Name, + Rrtype: dnsTypeAAAA, + Class: dnsClassINET, + Rdlength: 16, + }, + AAAA: TestAddr6, + }) + } + } + } + return r, nil + }} + r := Resolver{PreferGo: true, Dial: fake.DialContext} conf, err := newResolvConfTest() if err != nil { @@ -405,14 +521,8 @@ func TestGoLookupIPWithResolverConfig(t *testing.T) { t.Error(err) continue } - addrs, err := goLookupIP(context.Background(), tt.name) + addrs, err := r.LookupIPAddr(context.Background(), tt.name) if err != nil { - // This test uses external network connectivity. - // We need to take care with errors on both - // DNS message exchange layer and DNS - // transport layer because goLookupIP may fail - // when the IP connectivity on node under test - // gets lost during its run. if err, ok := err.(*DNSError); !ok || tt.error != nil && (err.Name != tt.error.(*DNSError).Name || err.Server != tt.error.(*DNSError).Server || err.IsTimeout != tt.error.(*DNSError).IsTimeout) { t.Errorf("got %v; want %v", err, tt.error) } @@ -437,7 +547,17 @@ func TestGoLookupIPWithResolverConfig(t *testing.T) { // Test that goLookupIPOrder falls back to the host file when no DNS servers are available. func TestGoLookupIPOrderFallbackToFile(t *testing.T) { - testenv.MustHaveExternalNetwork(t) + fake := fakeDNSServer{func(n, s string, q *dnsMsg, tm time.Time) (*dnsMsg, error) { + r := &dnsMsg{ + dnsMsgHdr: dnsMsgHdr{ + id: q.id, + response: true, + }, + question: q.question, + } + return r, nil + }} + r := Resolver{PreferGo: true, Dial: fake.DialContext} // Add a config that simulates no dns servers being available. conf, err := newResolvConfTest() @@ -455,14 +575,14 @@ func TestGoLookupIPOrderFallbackToFile(t *testing.T) { name := fmt.Sprintf("order %v", order) // First ensure that we get an error when contacting a non-existent host. - _, _, err := goLookupIPCNAMEOrder(context.Background(), "notarealhost", order) + _, _, err := r.goLookupIPCNAMEOrder(context.Background(), "notarealhost", order) if err == nil { t.Errorf("%s: expected error while looking up name not in hosts file", name) continue } // Now check that we get an address when the name appears in the hosts file. - addrs, _, err := goLookupIPCNAMEOrder(context.Background(), "thor", order) // entry is in "testdata/hosts" + addrs, _, err := r.goLookupIPCNAMEOrder(context.Background(), "thor", order) // entry is in "testdata/hosts" if err != nil { t.Errorf("%s: expected to successfully lookup host entry", name) continue @@ -485,9 +605,6 @@ func TestGoLookupIPOrderFallbackToFile(t *testing.T) { func TestErrorForOriginalNameWhenSearching(t *testing.T) { const fqdn = "doesnotexist.domain" - origTestHookDNSDialer := testHookDNSDialer - defer func() { testHookDNSDialer = origTestHookDNSDialer }() - conf, err := newResolvConfTest() if err != nil { t.Fatal(err) @@ -498,14 +615,13 @@ func TestErrorForOriginalNameWhenSearching(t *testing.T) { t.Fatal(err) } - d := &fakeDNSDialer{} - testHookDNSDialer = func() dnsDialer { return d } - - d.rh = func(s string, q *dnsMsg, _ time.Time) (*dnsMsg, error) { + fake := fakeDNSServer{func(_, _ string, q *dnsMsg, _ time.Time) (*dnsMsg, error) { r := &dnsMsg{ dnsMsgHdr: dnsMsgHdr{ - id: q.id, + id: q.id, + response: true, }, + question: q.question, } switch q.question[0].Name { @@ -516,24 +632,31 @@ func TestErrorForOriginalNameWhenSearching(t *testing.T) { } return r, nil - } + }} - _, err = goLookupIP(context.Background(), fqdn) - if err == nil { - t.Fatal("expected an error") + cases := []struct { + strictErrors bool + wantErr *DNSError + }{ + {true, &DNSError{Name: fqdn, Err: "server misbehaving", IsTemporary: true}}, + {false, &DNSError{Name: fqdn, Err: errNoSuchHost.Error()}}, } + for _, tt := range cases { + r := Resolver{PreferGo: true, StrictErrors: tt.strictErrors, Dial: fake.DialContext} + _, err = r.LookupIPAddr(context.Background(), fqdn) + if err == nil { + t.Fatal("expected an error") + } - want := &DNSError{Name: fqdn, Err: errNoSuchHost.Error()} - if err, ok := err.(*DNSError); !ok || err.Name != want.Name || err.Err != want.Err { - t.Errorf("got %v; want %v", err, want) + want := tt.wantErr + if err, ok := err.(*DNSError); !ok || err.Name != want.Name || err.Err != want.Err || err.IsTemporary != want.IsTemporary { + t.Errorf("got %v; want %v", err, want) + } } } // Issue 15434. If a name server gives a lame referral, continue to the next. func TestIgnoreLameReferrals(t *testing.T) { - origTestHookDNSDialer := testHookDNSDialer - defer func() { testHookDNSDialer = origTestHookDNSDialer }() - conf, err := newResolvConfTest() if err != nil { t.Fatal(err) @@ -545,10 +668,7 @@ func TestIgnoreLameReferrals(t *testing.T) { t.Fatal(err) } - d := &fakeDNSDialer{} - testHookDNSDialer = func() dnsDialer { return d } - - d.rh = func(s string, q *dnsMsg, _ time.Time) (*dnsMsg, error) { + fake := fakeDNSServer{func(_, s string, q *dnsMsg, _ time.Time) (*dnsMsg, error) { t.Log(s, q) r := &dnsMsg{ dnsMsgHdr: dnsMsgHdr{ @@ -576,9 +696,10 @@ func TestIgnoreLameReferrals(t *testing.T) { } return r, nil - } + }} + r := Resolver{PreferGo: true, Dial: fake.DialContext} - addrs, err := goLookupIP(context.Background(), "www.golang.org") + addrs, err := r.LookupIPAddr(context.Background(), "www.golang.org") if err != nil { t.Fatal(err) } @@ -597,7 +718,7 @@ func BenchmarkGoLookupIP(b *testing.B) { ctx := context.Background() for i := 0; i < b.N; i++ { - goLookupIP(ctx, "www.example.com") + goResolver.LookupIPAddr(ctx, "www.example.com") } } @@ -606,7 +727,7 @@ func BenchmarkGoLookupIPNoSuchHost(b *testing.B) { ctx := context.Background() for i := 0; i < b.N; i++ { - goLookupIP(ctx, "some.nonexistent") + goResolver.LookupIPAddr(ctx, "some.nonexistent") } } @@ -629,38 +750,70 @@ func BenchmarkGoLookupIPWithBrokenNameServer(b *testing.B) { ctx := context.Background() for i := 0; i < b.N; i++ { - goLookupIP(ctx, "www.example.com") + goResolver.LookupIPAddr(ctx, "www.example.com") } } -type fakeDNSDialer struct { - // reply handler - rh func(s string, q *dnsMsg, t time.Time) (*dnsMsg, error) +type fakeDNSServer struct { + rh func(n, s string, q *dnsMsg, t time.Time) (*dnsMsg, error) } -func (f *fakeDNSDialer) dialDNS(_ context.Context, n, s string) (dnsConn, error) { - return &fakeDNSConn{f.rh, s, time.Time{}}, nil +func (server *fakeDNSServer) DialContext(_ context.Context, n, s string) (Conn, error) { + return &fakeDNSConn{nil, server, n, s, nil, time.Time{}}, nil } type fakeDNSConn struct { - rh func(s string, q *dnsMsg, t time.Time) (*dnsMsg, error) - s string - t time.Time + Conn + server *fakeDNSServer + n string + s string + q *dnsMsg + t time.Time } func (f *fakeDNSConn) Close() error { return nil } +func (f *fakeDNSConn) Read(b []byte) (int, error) { + resp, err := f.server.rh(f.n, f.s, f.q, f.t) + if err != nil { + return 0, err + } + + bb, ok := resp.Pack() + if !ok { + return 0, errors.New("cannot marshal DNS message") + } + if len(b) < len(bb) { + return 0, errors.New("read would fragment DNS message") + } + + copy(b, bb) + return len(bb), nil +} + +func (f *fakeDNSConn) ReadFrom(b []byte) (int, Addr, error) { + return 0, nil, nil +} + +func (f *fakeDNSConn) Write(b []byte) (int, error) { + f.q = new(dnsMsg) + if !f.q.Unpack(b) { + return 0, errors.New("cannot unmarshal DNS message") + } + return len(b), nil +} + +func (f *fakeDNSConn) WriteTo(b []byte, addr Addr) (int, error) { + return 0, nil +} + func (f *fakeDNSConn) SetDeadline(t time.Time) error { f.t = t return nil } -func (f *fakeDNSConn) dnsRoundTrip(q *dnsMsg) (*dnsMsg, error) { - return f.rh(f.s, q, f.t) -} - // UDP round-tripper algorithm should ignore invalid DNS responses (issue 13281). func TestIgnoreDNSForgeries(t *testing.T) { c, s := Pipe() @@ -723,7 +876,8 @@ func TestIgnoreDNSForgeries(t *testing.T) { }, } - resp, err := dnsRoundTripUDP(c, msg) + dc := &dnsPacketConn{c} + resp, err := dc.dnsRoundTrip(msg) if err != nil { t.Fatalf("dnsRoundTripUDP failed: %v", err) } @@ -735,9 +889,6 @@ func TestIgnoreDNSForgeries(t *testing.T) { // Issue 16865. If a name server times out, continue to the next. func TestRetryTimeout(t *testing.T) { - origTestHookDNSDialer := testHookDNSDialer - defer func() { testHookDNSDialer = origTestHookDNSDialer }() - conf, err := newResolvConfTest() if err != nil { t.Fatal(err) @@ -752,12 +903,9 @@ func TestRetryTimeout(t *testing.T) { t.Fatal(err) } - d := &fakeDNSDialer{} - testHookDNSDialer = func() dnsDialer { return d } - var deadline0 time.Time - d.rh = func(s string, q *dnsMsg, deadline time.Time) (*dnsMsg, error) { + fake := fakeDNSServer{func(_, s string, q *dnsMsg, deadline time.Time) (*dnsMsg, error) { t.Log(s, q, deadline) if deadline.IsZero() { @@ -767,17 +915,18 @@ func TestRetryTimeout(t *testing.T) { if s == "192.0.2.1:53" { deadline0 = deadline time.Sleep(10 * time.Millisecond) - return nil, errTimeout + return nil, poll.ErrTimeout } - if deadline == deadline0 { + if deadline.Equal(deadline0) { t.Error("deadline didn't change") } return mockTXTResponse(q), nil - } + }} + r := &Resolver{PreferGo: true, Dial: fake.DialContext} - _, err = LookupTXT("www.golang.org") + _, err = r.LookupTXT(context.Background(), "www.golang.org") if err != nil { t.Fatal(err) } @@ -796,9 +945,6 @@ func TestRotate(t *testing.T) { } func testRotate(t *testing.T, rotate bool, nameservers, wantServers []string) { - origTestHookDNSDialer := testHookDNSDialer - defer func() { testHookDNSDialer = origTestHookDNSDialer }() - conf, err := newResolvConfTest() if err != nil { t.Fatal(err) @@ -817,18 +963,16 @@ func testRotate(t *testing.T, rotate bool, nameservers, wantServers []string) { t.Fatal(err) } - d := &fakeDNSDialer{} - testHookDNSDialer = func() dnsDialer { return d } - var usedServers []string - d.rh = func(s string, q *dnsMsg, _ time.Time) (*dnsMsg, error) { + fake := fakeDNSServer{func(_, s string, q *dnsMsg, deadline time.Time) (*dnsMsg, error) { usedServers = append(usedServers, s) return mockTXTResponse(q), nil - } + }} + r := Resolver{PreferGo: true, Dial: fake.DialContext} // len(nameservers) + 1 to allow rotation to get back to start for i := 0; i < len(nameservers)+1; i++ { - if _, err := LookupTXT("www.golang.org"); err != nil { + if _, err := r.LookupTXT(context.Background(), "www.golang.org"); err != nil { t.Fatal(err) } } @@ -860,3 +1004,311 @@ func mockTXTResponse(q *dnsMsg) *dnsMsg { return r } + +// Issue 17448. With StrictErrors enabled, temporary errors should make +// LookupIP fail rather than return a partial result. +func TestStrictErrorsLookupIP(t *testing.T) { + conf, err := newResolvConfTest() + if err != nil { + t.Fatal(err) + } + defer conf.teardown() + + confData := []string{ + "nameserver 192.0.2.53", + "search x.golang.org y.golang.org", + } + if err := conf.writeAndUpdate(confData); err != nil { + t.Fatal(err) + } + + const name = "test-issue19592" + const server = "192.0.2.53:53" + const searchX = "test-issue19592.x.golang.org." + const searchY = "test-issue19592.y.golang.org." + const ip4 = "192.0.2.1" + const ip6 = "2001:db8::1" + + type resolveWhichEnum int + const ( + resolveOK resolveWhichEnum = iota + resolveOpError + resolveServfail + resolveTimeout + ) + + makeTempError := func(err string) error { + return &DNSError{ + Err: err, + Name: name, + Server: server, + IsTemporary: true, + } + } + makeTimeout := func() error { + return &DNSError{ + Err: poll.ErrTimeout.Error(), + Name: name, + Server: server, + IsTimeout: true, + } + } + makeNxDomain := func() error { + return &DNSError{ + Err: errNoSuchHost.Error(), + Name: name, + Server: server, + } + } + + cases := []struct { + desc string + resolveWhich func(quest *dnsQuestion) resolveWhichEnum + wantStrictErr error + wantLaxErr error + wantIPs []string + }{ + { + desc: "No errors", + resolveWhich: func(quest *dnsQuestion) resolveWhichEnum { + return resolveOK + }, + wantIPs: []string{ip4, ip6}, + }, + { + desc: "searchX error fails in strict mode", + resolveWhich: func(quest *dnsQuestion) resolveWhichEnum { + if quest.Name == searchX { + return resolveTimeout + } + return resolveOK + }, + wantStrictErr: makeTimeout(), + wantIPs: []string{ip4, ip6}, + }, + { + desc: "searchX IPv4-only timeout fails in strict mode", + resolveWhich: func(quest *dnsQuestion) resolveWhichEnum { + if quest.Name == searchX && quest.Qtype == dnsTypeA { + return resolveTimeout + } + return resolveOK + }, + wantStrictErr: makeTimeout(), + wantIPs: []string{ip4, ip6}, + }, + { + desc: "searchX IPv6-only servfail fails in strict mode", + resolveWhich: func(quest *dnsQuestion) resolveWhichEnum { + if quest.Name == searchX && quest.Qtype == dnsTypeAAAA { + return resolveServfail + } + return resolveOK + }, + wantStrictErr: makeTempError("server misbehaving"), + wantIPs: []string{ip4, ip6}, + }, + { + desc: "searchY error always fails", + resolveWhich: func(quest *dnsQuestion) resolveWhichEnum { + if quest.Name == searchY { + return resolveTimeout + } + return resolveOK + }, + wantStrictErr: makeTimeout(), + wantLaxErr: makeNxDomain(), // This one reaches the "test." FQDN. + }, + { + desc: "searchY IPv4-only socket error fails in strict mode", + resolveWhich: func(quest *dnsQuestion) resolveWhichEnum { + if quest.Name == searchY && quest.Qtype == dnsTypeA { + return resolveOpError + } + return resolveOK + }, + wantStrictErr: makeTempError("write: socket on fire"), + wantIPs: []string{ip6}, + }, + { + desc: "searchY IPv6-only timeout fails in strict mode", + resolveWhich: func(quest *dnsQuestion) resolveWhichEnum { + if quest.Name == searchY && quest.Qtype == dnsTypeAAAA { + return resolveTimeout + } + return resolveOK + }, + wantStrictErr: makeTimeout(), + wantIPs: []string{ip4}, + }, + } + + for i, tt := range cases { + fake := fakeDNSServer{func(_, s string, q *dnsMsg, deadline time.Time) (*dnsMsg, error) { + t.Log(s, q) + + switch tt.resolveWhich(&q.question[0]) { + case resolveOK: + // Handle below. + case resolveOpError: + return nil, &OpError{Op: "write", Err: fmt.Errorf("socket on fire")} + case resolveServfail: + return &dnsMsg{ + dnsMsgHdr: dnsMsgHdr{ + id: q.id, + response: true, + rcode: dnsRcodeServerFailure, + }, + question: q.question, + }, nil + case resolveTimeout: + return nil, poll.ErrTimeout + default: + t.Fatal("Impossible resolveWhich") + } + + switch q.question[0].Name { + case searchX, name + ".": + // Return NXDOMAIN to utilize the search list. + return &dnsMsg{ + dnsMsgHdr: dnsMsgHdr{ + id: q.id, + response: true, + rcode: dnsRcodeNameError, + }, + question: q.question, + }, nil + case searchY: + // Return records below. + default: + return nil, fmt.Errorf("Unexpected Name: %v", q.question[0].Name) + } + + r := &dnsMsg{ + dnsMsgHdr: dnsMsgHdr{ + id: q.id, + response: true, + }, + question: q.question, + } + switch q.question[0].Qtype { + case dnsTypeA: + r.answer = []dnsRR{ + &dnsRR_A{ + Hdr: dnsRR_Header{ + Name: q.question[0].Name, + Rrtype: dnsTypeA, + Class: dnsClassINET, + Rdlength: 4, + }, + A: TestAddr, + }, + } + case dnsTypeAAAA: + r.answer = []dnsRR{ + &dnsRR_AAAA{ + Hdr: dnsRR_Header{ + Name: q.question[0].Name, + Rrtype: dnsTypeAAAA, + Class: dnsClassINET, + Rdlength: 16, + }, + AAAA: TestAddr6, + }, + } + default: + return nil, fmt.Errorf("Unexpected Qtype: %v", q.question[0].Qtype) + } + return r, nil + }} + + for _, strict := range []bool{true, false} { + r := Resolver{PreferGo: true, StrictErrors: strict, Dial: fake.DialContext} + ips, err := r.LookupIPAddr(context.Background(), name) + + var wantErr error + if strict { + wantErr = tt.wantStrictErr + } else { + wantErr = tt.wantLaxErr + } + if !reflect.DeepEqual(err, wantErr) { + t.Errorf("#%d (%s) strict=%v: got err %#v; want %#v", i, tt.desc, strict, err, wantErr) + } + + gotIPs := map[string]struct{}{} + for _, ip := range ips { + gotIPs[ip.String()] = struct{}{} + } + wantIPs := map[string]struct{}{} + if wantErr == nil { + for _, ip := range tt.wantIPs { + wantIPs[ip] = struct{}{} + } + } + if !reflect.DeepEqual(gotIPs, wantIPs) { + t.Errorf("#%d (%s) strict=%v: got ips %v; want %v", i, tt.desc, strict, gotIPs, wantIPs) + } + } + } +} + +// Issue 17448. With StrictErrors enabled, temporary errors should make +// LookupTXT stop walking the search list. +func TestStrictErrorsLookupTXT(t *testing.T) { + conf, err := newResolvConfTest() + if err != nil { + t.Fatal(err) + } + defer conf.teardown() + + confData := []string{ + "nameserver 192.0.2.53", + "search x.golang.org y.golang.org", + } + if err := conf.writeAndUpdate(confData); err != nil { + t.Fatal(err) + } + + const name = "test" + const server = "192.0.2.53:53" + const searchX = "test.x.golang.org." + const searchY = "test.y.golang.org." + const txt = "Hello World" + + fake := fakeDNSServer{func(_, s string, q *dnsMsg, deadline time.Time) (*dnsMsg, error) { + t.Log(s, q) + + switch q.question[0].Name { + case searchX: + return nil, poll.ErrTimeout + case searchY: + return mockTXTResponse(q), nil + default: + return nil, fmt.Errorf("Unexpected Name: %v", q.question[0].Name) + } + }} + + for _, strict := range []bool{true, false} { + r := Resolver{StrictErrors: strict, Dial: fake.DialContext} + _, rrs, err := r.lookup(context.Background(), name, dnsTypeTXT) + var wantErr error + var wantRRs int + if strict { + wantErr = &DNSError{ + Err: poll.ErrTimeout.Error(), + Name: name, + Server: server, + IsTimeout: true, + } + } else { + wantRRs = 1 + } + if !reflect.DeepEqual(err, wantErr) { + t.Errorf("strict=%v: got err %#v; want %#v", strict, err, wantErr) + } + if len(rrs) != wantRRs { + t.Errorf("strict=%v: got %v; want %v", strict, len(rrs), wantRRs) + } + } +} diff --git a/libgo/go/net/error_posix.go b/libgo/go/net/error_posix.go new file mode 100644 index 00000000000..dd9754c841a --- /dev/null +++ b/libgo/go/net/error_posix.go @@ -0,0 +1,21 @@ +// Copyright 2017 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build darwin dragonfly freebsd linux nacl netbsd openbsd solaris windows + +package net + +import ( + "os" + "syscall" +) + +// wrapSyscallError takes an error and a syscall name. If the error is +// a syscall.Errno, it wraps it in a os.SyscallError using the syscall name. +func wrapSyscallError(name string, err error) error { + if _, ok := err.(syscall.Errno); ok { + err = os.NewSyscallError(name, err) + } + return err +} diff --git a/libgo/go/net/error_test.go b/libgo/go/net/error_test.go index c23da49fad9..9791e6fe4de 100644 --- a/libgo/go/net/error_test.go +++ b/libgo/go/net/error_test.go @@ -7,11 +7,13 @@ package net import ( "context" "fmt" + "internal/poll" "io" "io/ioutil" "net/internal/socktest" "os" "runtime" + "strings" "testing" "time" ) @@ -87,7 +89,7 @@ second: return nil } switch err := nestedErr.(type) { - case *AddrError, addrinfoErrno, *DNSError, InvalidAddrError, *ParseError, *timeoutError, UnknownNetworkError: + case *AddrError, addrinfoErrno, *DNSError, InvalidAddrError, *ParseError, *poll.TimeoutError, UnknownNetworkError: return nil case *os.SyscallError: nestedErr = err.Err @@ -97,7 +99,7 @@ second: goto third } switch nestedErr { - case errCanceled, errClosing, errMissingAddress, errNoSuitableAddress, + case errCanceled, poll.ErrNetClosing, errMissingAddress, errNoSuitableAddress, context.DeadlineExceeded, context.Canceled: return nil } @@ -213,7 +215,7 @@ func TestDialAddrError(t *testing.T) { case "nacl", "plan9": t.Skipf("not supported on %s", runtime.GOOS) } - if !supportsIPv4 || !supportsIPv6 { + if !supportsIPv4() || !supportsIPv6() { t.Skip("both IPv4 and IPv6 are required") } @@ -432,7 +434,7 @@ second: goto third } switch nestedErr { - case errClosing, errTimeout: + case poll.ErrNetClosing, poll.ErrTimeout: return nil } return fmt.Errorf("unexpected type on 2nd nested level: %T", nestedErr) @@ -467,14 +469,14 @@ second: return nil } switch err := nestedErr.(type) { - case *AddrError, addrinfoErrno, *DNSError, InvalidAddrError, *ParseError, *timeoutError, UnknownNetworkError: + case *AddrError, addrinfoErrno, *DNSError, InvalidAddrError, *ParseError, *poll.TimeoutError, UnknownNetworkError: return nil case *os.SyscallError: nestedErr = err.Err goto third } switch nestedErr { - case errCanceled, errClosing, errMissingAddress, errTimeout, ErrWriteToConnected, io.ErrUnexpectedEOF: + case errCanceled, poll.ErrNetClosing, errMissingAddress, poll.ErrTimeout, ErrWriteToConnected, io.ErrUnexpectedEOF: return nil } return fmt.Errorf("unexpected type on 2nd nested level: %T", nestedErr) @@ -489,11 +491,21 @@ third: // parseCloseError parses nestedErr and reports whether it is a valid // error value from Close functions. // It returns nil when nestedErr is valid. -func parseCloseError(nestedErr error) error { +func parseCloseError(nestedErr error, isShutdown bool) error { if nestedErr == nil { return nil } + // Because historically we have not exported the error that we + // return for an operation on a closed network connection, + // there are programs that test for the exact error string. + // Verify that string here so that we don't break those + // programs unexpectedly. See issues #4373 and #19252. + want := "use of closed network connection" + if !isShutdown && !strings.Contains(nestedErr.Error(), want) { + return fmt.Errorf("error string %q does not contain expected string %q", nestedErr, want) + } + switch err := nestedErr.(type) { case *OpError: if err := err.isValid(); err != nil { @@ -517,7 +529,7 @@ second: goto third } switch nestedErr { - case errClosing: + case poll.ErrNetClosing: return nil } return fmt.Errorf("unexpected type on 2nd nested level: %T", nestedErr) @@ -547,23 +559,23 @@ func TestCloseError(t *testing.T) { for i := 0; i < 3; i++ { err = c.(*TCPConn).CloseRead() - if perr := parseCloseError(err); perr != nil { + if perr := parseCloseError(err, true); perr != nil { t.Errorf("#%d: %v", i, perr) } } for i := 0; i < 3; i++ { err = c.(*TCPConn).CloseWrite() - if perr := parseCloseError(err); perr != nil { + if perr := parseCloseError(err, true); perr != nil { t.Errorf("#%d: %v", i, perr) } } for i := 0; i < 3; i++ { err = c.Close() - if perr := parseCloseError(err); perr != nil { + if perr := parseCloseError(err, false); perr != nil { t.Errorf("#%d: %v", i, perr) } err = ln.Close() - if perr := parseCloseError(err); perr != nil { + if perr := parseCloseError(err, false); perr != nil { t.Errorf("#%d: %v", i, perr) } } @@ -576,7 +588,7 @@ func TestCloseError(t *testing.T) { for i := 0; i < 3; i++ { err = pc.Close() - if perr := parseCloseError(err); perr != nil { + if perr := parseCloseError(err, false); perr != nil { t.Errorf("#%d: %v", i, perr) } } @@ -613,7 +625,7 @@ second: goto third } switch nestedErr { - case errClosing, errTimeout: + case poll.ErrNetClosing, poll.ErrTimeout: return nil } return fmt.Errorf("unexpected type on 2nd nested level: %T", nestedErr) @@ -692,7 +704,7 @@ second: goto third } switch nestedErr { - case errClosing: + case poll.ErrNetClosing: return nil } return fmt.Errorf("unexpected type on 2nd nested level: %T", nestedErr) diff --git a/libgo/go/net/external_test.go b/libgo/go/net/external_test.go index e18b547cac9..38788efc3d3 100644 --- a/libgo/go/net/external_test.go +++ b/libgo/go/net/external_test.go @@ -15,7 +15,7 @@ import ( func TestResolveGoogle(t *testing.T) { testenv.MustHaveExternalNetwork(t) - if !supportsIPv4 || !supportsIPv6 || !*testIPv4 || !*testIPv6 { + if !supportsIPv4() || !supportsIPv6() || !*testIPv4 || !*testIPv6 { t.Skip("both IPv4 and IPv6 are required") } @@ -62,7 +62,7 @@ var dialGoogleTests = []struct { func TestDialGoogle(t *testing.T) { testenv.MustHaveExternalNetwork(t) - if !supportsIPv4 || !supportsIPv6 || !*testIPv4 || !*testIPv6 { + if !supportsIPv4() || !supportsIPv6() || !*testIPv4 || !*testIPv6 { t.Skip("both IPv4 and IPv6 are required") } diff --git a/libgo/go/net/fd_io_plan9.go b/libgo/go/net/fd_io_plan9.go deleted file mode 100644 index 76da0c546cf..00000000000 --- a/libgo/go/net/fd_io_plan9.go +++ /dev/null @@ -1,93 +0,0 @@ -// Copyright 2016 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package net - -import ( - "os" - "runtime" - "sync" - "syscall" -) - -// asyncIO implements asynchronous cancelable I/O. -// An asyncIO represents a single asynchronous Read or Write -// operation. The result is returned on the result channel. -// The undergoing I/O system call can either complete or be -// interrupted by a note. -type asyncIO struct { - res chan result - - // mu guards the pid field. - mu sync.Mutex - - // pid holds the process id of - // the process running the IO operation. - pid int -} - -// result is the return value of a Read or Write operation. -type result struct { - n int - err error -} - -// newAsyncIO returns a new asyncIO that performs an I/O -// operation by calling fn, which must do one and only one -// interruptible system call. -func newAsyncIO(fn func([]byte) (int, error), b []byte) *asyncIO { - aio := &asyncIO{ - res: make(chan result, 0), - } - aio.mu.Lock() - go func() { - // Lock the current goroutine to its process - // and store the pid in io so that Cancel can - // interrupt it. We ignore the "hangup" signal, - // so the signal does not take down the entire - // Go runtime. - runtime.LockOSThread() - runtime_ignoreHangup() - aio.pid = os.Getpid() - aio.mu.Unlock() - - n, err := fn(b) - - aio.mu.Lock() - aio.pid = -1 - runtime_unignoreHangup() - aio.mu.Unlock() - - aio.res <- result{n, err} - }() - return aio -} - -var hangupNote os.Signal = syscall.Note("hangup") - -// Cancel interrupts the I/O operation, causing -// the Wait function to return. -func (aio *asyncIO) Cancel() { - aio.mu.Lock() - defer aio.mu.Unlock() - if aio.pid == -1 { - return - } - proc, err := os.FindProcess(aio.pid) - if err != nil { - return - } - proc.Signal(hangupNote) -} - -// Wait for the I/O operation to complete. -func (aio *asyncIO) Wait() (int, error) { - res := <-aio.res - return res.n, res.err -} - -// The following functions, provided by the runtime, are used to -// ignore and unignore the "hangup" signal received by the process. -func runtime_ignoreHangup() -func runtime_unignoreHangup() diff --git a/libgo/go/net/fd_mutex.go b/libgo/go/net/fd_mutex.go deleted file mode 100644 index 4591fd1cac8..00000000000 --- a/libgo/go/net/fd_mutex.go +++ /dev/null @@ -1,249 +0,0 @@ -// Copyright 2013 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package net - -import "sync/atomic" - -// fdMutex is a specialized synchronization primitive that manages -// lifetime of an fd and serializes access to Read, Write and Close -// methods on netFD. -type fdMutex struct { - state uint64 - rsema uint32 - wsema uint32 -} - -// fdMutex.state is organized as follows: -// 1 bit - whether netFD is closed, if set all subsequent lock operations will fail. -// 1 bit - lock for read operations. -// 1 bit - lock for write operations. -// 20 bits - total number of references (read+write+misc). -// 20 bits - number of outstanding read waiters. -// 20 bits - number of outstanding write waiters. -const ( - mutexClosed = 1 << 0 - mutexRLock = 1 << 1 - mutexWLock = 1 << 2 - mutexRef = 1 << 3 - mutexRefMask = (1<<20 - 1) << 3 - mutexRWait = 1 << 23 - mutexRMask = (1<<20 - 1) << 23 - mutexWWait = 1 << 43 - mutexWMask = (1<<20 - 1) << 43 -) - -// Read operations must do rwlock(true)/rwunlock(true). -// -// Write operations must do rwlock(false)/rwunlock(false). -// -// Misc operations must do incref/decref. -// Misc operations include functions like setsockopt and setDeadline. -// They need to use incref/decref to ensure that they operate on the -// correct fd in presence of a concurrent close call (otherwise fd can -// be closed under their feet). -// -// Close operations must do increfAndClose/decref. - -// incref adds a reference to mu. -// It reports whether mu is available for reading or writing. -func (mu *fdMutex) incref() bool { - for { - old := atomic.LoadUint64(&mu.state) - if old&mutexClosed != 0 { - return false - } - new := old + mutexRef - if new&mutexRefMask == 0 { - panic("net: inconsistent fdMutex") - } - if atomic.CompareAndSwapUint64(&mu.state, old, new) { - return true - } - } -} - -// increfAndClose sets the state of mu to closed. -// It reports whether there is no remaining reference. -func (mu *fdMutex) increfAndClose() bool { - for { - old := atomic.LoadUint64(&mu.state) - if old&mutexClosed != 0 { - return false - } - // Mark as closed and acquire a reference. - new := (old | mutexClosed) + mutexRef - if new&mutexRefMask == 0 { - panic("net: inconsistent fdMutex") - } - // Remove all read and write waiters. - new &^= mutexRMask | mutexWMask - if atomic.CompareAndSwapUint64(&mu.state, old, new) { - // Wake all read and write waiters, - // they will observe closed flag after wakeup. - for old&mutexRMask != 0 { - old -= mutexRWait - runtime_Semrelease(&mu.rsema) - } - for old&mutexWMask != 0 { - old -= mutexWWait - runtime_Semrelease(&mu.wsema) - } - return true - } - } -} - -// decref removes a reference from mu. -// It reports whether there is no remaining reference. -func (mu *fdMutex) decref() bool { - for { - old := atomic.LoadUint64(&mu.state) - if old&mutexRefMask == 0 { - panic("net: inconsistent fdMutex") - } - new := old - mutexRef - if atomic.CompareAndSwapUint64(&mu.state, old, new) { - return new&(mutexClosed|mutexRefMask) == mutexClosed - } - } -} - -// lock adds a reference to mu and locks mu. -// It reports whether mu is available for reading or writing. -func (mu *fdMutex) rwlock(read bool) bool { - var mutexBit, mutexWait, mutexMask uint64 - var mutexSema *uint32 - if read { - mutexBit = mutexRLock - mutexWait = mutexRWait - mutexMask = mutexRMask - mutexSema = &mu.rsema - } else { - mutexBit = mutexWLock - mutexWait = mutexWWait - mutexMask = mutexWMask - mutexSema = &mu.wsema - } - for { - old := atomic.LoadUint64(&mu.state) - if old&mutexClosed != 0 { - return false - } - var new uint64 - if old&mutexBit == 0 { - // Lock is free, acquire it. - new = (old | mutexBit) + mutexRef - if new&mutexRefMask == 0 { - panic("net: inconsistent fdMutex") - } - } else { - // Wait for lock. - new = old + mutexWait - if new&mutexMask == 0 { - panic("net: inconsistent fdMutex") - } - } - if atomic.CompareAndSwapUint64(&mu.state, old, new) { - if old&mutexBit == 0 { - return true - } - runtime_Semacquire(mutexSema) - // The signaller has subtracted mutexWait. - } - } -} - -// unlock removes a reference from mu and unlocks mu. -// It reports whether there is no remaining reference. -func (mu *fdMutex) rwunlock(read bool) bool { - var mutexBit, mutexWait, mutexMask uint64 - var mutexSema *uint32 - if read { - mutexBit = mutexRLock - mutexWait = mutexRWait - mutexMask = mutexRMask - mutexSema = &mu.rsema - } else { - mutexBit = mutexWLock - mutexWait = mutexWWait - mutexMask = mutexWMask - mutexSema = &mu.wsema - } - for { - old := atomic.LoadUint64(&mu.state) - if old&mutexBit == 0 || old&mutexRefMask == 0 { - panic("net: inconsistent fdMutex") - } - // Drop lock, drop reference and wake read waiter if present. - new := (old &^ mutexBit) - mutexRef - if old&mutexMask != 0 { - new -= mutexWait - } - if atomic.CompareAndSwapUint64(&mu.state, old, new) { - if old&mutexMask != 0 { - runtime_Semrelease(mutexSema) - } - return new&(mutexClosed|mutexRefMask) == mutexClosed - } - } -} - -// Implemented in runtime package. -func runtime_Semacquire(sema *uint32) -func runtime_Semrelease(sema *uint32) - -// incref adds a reference to fd. -// It returns an error when fd cannot be used. -func (fd *netFD) incref() error { - if !fd.fdmu.incref() { - return errClosing - } - return nil -} - -// decref removes a reference from fd. -// It also closes fd when the state of fd is set to closed and there -// is no remaining reference. -func (fd *netFD) decref() { - if fd.fdmu.decref() { - fd.destroy() - } -} - -// readLock adds a reference to fd and locks fd for reading. -// It returns an error when fd cannot be used for reading. -func (fd *netFD) readLock() error { - if !fd.fdmu.rwlock(true) { - return errClosing - } - return nil -} - -// readUnlock removes a reference from fd and unlocks fd for reading. -// It also closes fd when the state of fd is set to closed and there -// is no remaining reference. -func (fd *netFD) readUnlock() { - if fd.fdmu.rwunlock(true) { - fd.destroy() - } -} - -// writeLock adds a reference to fd and locks fd for writing. -// It returns an error when fd cannot be used for writing. -func (fd *netFD) writeLock() error { - if !fd.fdmu.rwlock(false) { - return errClosing - } - return nil -} - -// writeUnlock removes a reference from fd and unlocks fd for writing. -// It also closes fd when the state of fd is set to closed and there -// is no remaining reference. -func (fd *netFD) writeUnlock() { - if fd.fdmu.rwunlock(false) { - fd.destroy() - } -} diff --git a/libgo/go/net/fd_mutex_test.go b/libgo/go/net/fd_mutex_test.go deleted file mode 100644 index 3542c70f9d1..00000000000 --- a/libgo/go/net/fd_mutex_test.go +++ /dev/null @@ -1,195 +0,0 @@ -// Copyright 2013 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package net - -import ( - "math/rand" - "runtime" - "testing" - "time" -) - -func TestMutexLock(t *testing.T) { - var mu fdMutex - - if !mu.incref() { - t.Fatal("broken") - } - if mu.decref() { - t.Fatal("broken") - } - - if !mu.rwlock(true) { - t.Fatal("broken") - } - if mu.rwunlock(true) { - t.Fatal("broken") - } - - if !mu.rwlock(false) { - t.Fatal("broken") - } - if mu.rwunlock(false) { - t.Fatal("broken") - } -} - -func TestMutexClose(t *testing.T) { - var mu fdMutex - if !mu.increfAndClose() { - t.Fatal("broken") - } - - if mu.incref() { - t.Fatal("broken") - } - if mu.rwlock(true) { - t.Fatal("broken") - } - if mu.rwlock(false) { - t.Fatal("broken") - } - if mu.increfAndClose() { - t.Fatal("broken") - } -} - -func TestMutexCloseUnblock(t *testing.T) { - c := make(chan bool) - var mu fdMutex - mu.rwlock(true) - for i := 0; i < 4; i++ { - go func() { - if mu.rwlock(true) { - t.Error("broken") - return - } - c <- true - }() - } - // Concurrent goroutines must not be able to read lock the mutex. - time.Sleep(time.Millisecond) - select { - case <-c: - t.Fatal("broken") - default: - } - mu.increfAndClose() // Must unblock the readers. - for i := 0; i < 4; i++ { - select { - case <-c: - case <-time.After(10 * time.Second): - t.Fatal("broken") - } - } - if mu.decref() { - t.Fatal("broken") - } - if !mu.rwunlock(true) { - t.Fatal("broken") - } -} - -func TestMutexPanic(t *testing.T) { - ensurePanics := func(f func()) { - defer func() { - if recover() == nil { - t.Fatal("does not panic") - } - }() - f() - } - - var mu fdMutex - ensurePanics(func() { mu.decref() }) - ensurePanics(func() { mu.rwunlock(true) }) - ensurePanics(func() { mu.rwunlock(false) }) - - ensurePanics(func() { mu.incref(); mu.decref(); mu.decref() }) - ensurePanics(func() { mu.rwlock(true); mu.rwunlock(true); mu.rwunlock(true) }) - ensurePanics(func() { mu.rwlock(false); mu.rwunlock(false); mu.rwunlock(false) }) - - // ensure that it's still not broken - mu.incref() - mu.decref() - mu.rwlock(true) - mu.rwunlock(true) - mu.rwlock(false) - mu.rwunlock(false) -} - -func TestMutexStress(t *testing.T) { - P := 8 - N := int(1e6) - if testing.Short() { - P = 4 - N = 1e4 - } - defer runtime.GOMAXPROCS(runtime.GOMAXPROCS(P)) - done := make(chan bool) - var mu fdMutex - var readState [2]uint64 - var writeState [2]uint64 - for p := 0; p < P; p++ { - go func() { - r := rand.New(rand.NewSource(rand.Int63())) - for i := 0; i < N; i++ { - switch r.Intn(3) { - case 0: - if !mu.incref() { - t.Error("broken") - return - } - if mu.decref() { - t.Error("broken") - return - } - case 1: - if !mu.rwlock(true) { - t.Error("broken") - return - } - // Ensure that it provides mutual exclusion for readers. - if readState[0] != readState[1] { - t.Error("broken") - return - } - readState[0]++ - readState[1]++ - if mu.rwunlock(true) { - t.Error("broken") - return - } - case 2: - if !mu.rwlock(false) { - t.Error("broken") - return - } - // Ensure that it provides mutual exclusion for writers. - if writeState[0] != writeState[1] { - t.Error("broken") - return - } - writeState[0]++ - writeState[1]++ - if mu.rwunlock(false) { - t.Error("broken") - return - } - } - } - done <- true - }() - } - for p := 0; p < P; p++ { - <-done - } - if !mu.increfAndClose() { - t.Fatal("broken") - } - if !mu.decref() { - t.Fatal("broken") - } -} diff --git a/libgo/go/net/fd_plan9.go b/libgo/go/net/fd_plan9.go index 300d8c4543e..46ee5d97400 100644 --- a/libgo/go/net/fd_plan9.go +++ b/libgo/go/net/fd_plan9.go @@ -5,23 +5,15 @@ package net import ( + "internal/poll" "io" "os" - "sync/atomic" "syscall" - "time" ) -type atomicBool int32 - -func (b *atomicBool) isSet() bool { return atomic.LoadInt32((*int32)(b)) != 0 } -func (b *atomicBool) setFalse() { atomic.StoreInt32((*int32)(b), 0) } -func (b *atomicBool) setTrue() { atomic.StoreInt32((*int32)(b), 1) } - // Network file descriptor. type netFD struct { - // locking/lifetime of sysfd + serialize access to Read and Write methods - fdmu fdMutex + pfd poll.FD // immutable until Close net string @@ -30,26 +22,12 @@ type netFD struct { listen, ctl, data *os.File laddr, raddr Addr isStream bool - - // deadlines - raio *asyncIO - waio *asyncIO - rtimer *time.Timer - wtimer *time.Timer - rtimedout atomicBool // set true when read deadline has been reached - wtimedout atomicBool // set true when write deadline has been reached } -var ( - netdir string // default network -) - -func sysInit() { - netdir = "/net" -} +var netdir = "/net" // default network func newFD(net, name string, listen, ctl, data *os.File, laddr, raddr Addr) (*netFD, error) { - return &netFD{ + ret := &netFD{ net: net, n: name, dir: netdir + "/" + net + "/" + name, @@ -57,7 +35,9 @@ func newFD(net, name string, listen, ctl, data *os.File, laddr, raddr Addr) (*ne ctl: ctl, data: data, laddr: laddr, raddr: raddr, - }, nil + } + ret.pfd.Destroy = ret.destroy + return ret, nil } func (fd *netFD) init() error { @@ -99,28 +79,10 @@ func (fd *netFD) destroy() { } func (fd *netFD) Read(b []byte) (n int, err error) { - if fd.rtimedout.isSet() { - return 0, errTimeout - } if !fd.ok() || fd.data == nil { return 0, syscall.EINVAL } - if err := fd.readLock(); err != nil { - return 0, err - } - defer fd.readUnlock() - if len(b) == 0 { - return 0, nil - } - fd.raio = newAsyncIO(fd.data.Read, b) - n, err = fd.raio.Wait() - fd.raio = nil - if isHangup(err) { - err = io.EOF - } - if isInterrupted(err) { - err = errTimeout - } + n, err = fd.pfd.Read(fd.data.Read, b) if fd.net == "udp" && err == io.EOF { n = 0 err = nil @@ -129,23 +91,10 @@ func (fd *netFD) Read(b []byte) (n int, err error) { } func (fd *netFD) Write(b []byte) (n int, err error) { - if fd.wtimedout.isSet() { - return 0, errTimeout - } if !fd.ok() || fd.data == nil { return 0, syscall.EINVAL } - if err := fd.writeLock(); err != nil { - return 0, err - } - defer fd.writeUnlock() - fd.waio = newAsyncIO(fd.data.Write, b) - n, err = fd.waio.Wait() - fd.waio = nil - if isInterrupted(err) { - err = errTimeout - } - return + return fd.pfd.Write(fd.data.Write, b) } func (fd *netFD) closeRead() error { @@ -163,8 +112,8 @@ func (fd *netFD) closeWrite() error { } func (fd *netFD) Close() error { - if !fd.fdmu.increfAndClose() { - return errClosing + if err := fd.pfd.Close(); err != nil { + return err } if !fd.ok() { return syscall.EINVAL @@ -216,77 +165,6 @@ func (fd *netFD) file(f *os.File, s string) (*os.File, error) { return os.NewFile(uintptr(dfd), s), nil } -func (fd *netFD) setDeadline(t time.Time) error { - return setDeadlineImpl(fd, t, 'r'+'w') -} - -func (fd *netFD) setReadDeadline(t time.Time) error { - return setDeadlineImpl(fd, t, 'r') -} - -func (fd *netFD) setWriteDeadline(t time.Time) error { - return setDeadlineImpl(fd, t, 'w') -} - -func setDeadlineImpl(fd *netFD, t time.Time, mode int) error { - d := t.Sub(time.Now()) - if mode == 'r' || mode == 'r'+'w' { - fd.rtimedout.setFalse() - } - if mode == 'w' || mode == 'r'+'w' { - fd.wtimedout.setFalse() - } - if t.IsZero() || d < 0 { - // Stop timer - if mode == 'r' || mode == 'r'+'w' { - if fd.rtimer != nil { - fd.rtimer.Stop() - } - fd.rtimer = nil - } - if mode == 'w' || mode == 'r'+'w' { - if fd.wtimer != nil { - fd.wtimer.Stop() - } - fd.wtimer = nil - } - } else { - // Interrupt I/O operation once timer has expired - if mode == 'r' || mode == 'r'+'w' { - fd.rtimer = time.AfterFunc(d, func() { - fd.rtimedout.setTrue() - if fd.raio != nil { - fd.raio.Cancel() - } - }) - } - if mode == 'w' || mode == 'r'+'w' { - fd.wtimer = time.AfterFunc(d, func() { - fd.wtimedout.setTrue() - if fd.waio != nil { - fd.waio.Cancel() - } - }) - } - } - if !t.IsZero() && d < 0 { - // Interrupt current I/O operation - if mode == 'r' || mode == 'r'+'w' { - fd.rtimedout.setTrue() - if fd.raio != nil { - fd.raio.Cancel() - } - } - if mode == 'w' || mode == 'r'+'w' { - fd.wtimedout.setTrue() - if fd.waio != nil { - fd.waio.Cancel() - } - } - } - return nil -} - func setReadBuffer(fd *netFD, bytes int) error { return syscall.EPLAN9 } @@ -294,11 +172,3 @@ func setReadBuffer(fd *netFD, bytes int) error { func setWriteBuffer(fd *netFD, bytes int) error { return syscall.EPLAN9 } - -func isHangup(err error) bool { - return err != nil && stringsHasSuffix(err.Error(), "Hangup") -} - -func isInterrupted(err error) bool { - return err != nil && stringsHasSuffix(err.Error(), "interrupted") -} diff --git a/libgo/go/net/fd_poll_nacl.go b/libgo/go/net/fd_poll_nacl.go deleted file mode 100644 index 83987602a58..00000000000 --- a/libgo/go/net/fd_poll_nacl.go +++ /dev/null @@ -1,89 +0,0 @@ -// Copyright 2013 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package net - -import ( - "runtime" - "syscall" - "time" -) - -type pollDesc struct { - fd *netFD - closing bool -} - -func (pd *pollDesc) init(fd *netFD) error { pd.fd = fd; return nil } - -func (pd *pollDesc) close() {} - -func (pd *pollDesc) evict() { - pd.closing = true - if pd.fd != nil { - syscall.StopIO(pd.fd.sysfd) - runtime.KeepAlive(pd.fd) - } -} - -func (pd *pollDesc) prepare(mode int) error { - if pd.closing { - return errClosing - } - return nil -} - -func (pd *pollDesc) prepareRead() error { return pd.prepare('r') } - -func (pd *pollDesc) prepareWrite() error { return pd.prepare('w') } - -func (pd *pollDesc) wait(mode int) error { - if pd.closing { - return errClosing - } - return errTimeout -} - -func (pd *pollDesc) waitRead() error { return pd.wait('r') } - -func (pd *pollDesc) waitWrite() error { return pd.wait('w') } - -func (pd *pollDesc) waitCanceled(mode int) {} - -func (pd *pollDesc) waitCanceledRead() {} - -func (pd *pollDesc) waitCanceledWrite() {} - -func (fd *netFD) setDeadline(t time.Time) error { - return setDeadlineImpl(fd, t, 'r'+'w') -} - -func (fd *netFD) setReadDeadline(t time.Time) error { - return setDeadlineImpl(fd, t, 'r') -} - -func (fd *netFD) setWriteDeadline(t time.Time) error { - return setDeadlineImpl(fd, t, 'w') -} - -func setDeadlineImpl(fd *netFD, t time.Time, mode int) error { - d := t.UnixNano() - if t.IsZero() { - d = 0 - } - if err := fd.incref(); err != nil { - return err - } - switch mode { - case 'r': - syscall.SetReadDeadline(fd.sysfd, d) - case 'w': - syscall.SetWriteDeadline(fd.sysfd, d) - case 'r' + 'w': - syscall.SetReadDeadline(fd.sysfd, d) - syscall.SetWriteDeadline(fd.sysfd, d) - } - fd.decref() - return nil -} diff --git a/libgo/go/net/fd_poll_runtime.go b/libgo/go/net/fd_poll_runtime.go deleted file mode 100644 index 4ea92cb1f87..00000000000 --- a/libgo/go/net/fd_poll_runtime.go +++ /dev/null @@ -1,141 +0,0 @@ -// Copyright 2013 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// +build aix darwin dragonfly freebsd linux netbsd openbsd windows solaris - -package net - -import ( - "runtime" - "sync" - "syscall" - "time" -) - -// runtimeNano returns the current value of the runtime clock in nanoseconds. -func runtimeNano() int64 - -func runtime_pollServerInit() -func runtime_pollOpen(fd uintptr) (uintptr, int) -func runtime_pollClose(ctx uintptr) -func runtime_pollWait(ctx uintptr, mode int) int -func runtime_pollWaitCanceled(ctx uintptr, mode int) int -func runtime_pollReset(ctx uintptr, mode int) int -func runtime_pollSetDeadline(ctx uintptr, d int64, mode int) -func runtime_pollUnblock(ctx uintptr) - -type pollDesc struct { - runtimeCtx uintptr -} - -var serverInit sync.Once - -func (pd *pollDesc) init(fd *netFD) error { - serverInit.Do(runtime_pollServerInit) - ctx, errno := runtime_pollOpen(uintptr(fd.sysfd)) - runtime.KeepAlive(fd) - if errno != 0 { - return syscall.Errno(errno) - } - pd.runtimeCtx = ctx - return nil -} - -func (pd *pollDesc) close() { - if pd.runtimeCtx == 0 { - return - } - runtime_pollClose(pd.runtimeCtx) - pd.runtimeCtx = 0 -} - -// Evict evicts fd from the pending list, unblocking any I/O running on fd. -func (pd *pollDesc) evict() { - if pd.runtimeCtx == 0 { - return - } - runtime_pollUnblock(pd.runtimeCtx) -} - -func (pd *pollDesc) prepare(mode int) error { - res := runtime_pollReset(pd.runtimeCtx, mode) - return convertErr(res) -} - -func (pd *pollDesc) prepareRead() error { - return pd.prepare('r') -} - -func (pd *pollDesc) prepareWrite() error { - return pd.prepare('w') -} - -func (pd *pollDesc) wait(mode int) error { - res := runtime_pollWait(pd.runtimeCtx, mode) - return convertErr(res) -} - -func (pd *pollDesc) waitRead() error { - return pd.wait('r') -} - -func (pd *pollDesc) waitWrite() error { - return pd.wait('w') -} - -func (pd *pollDesc) waitCanceled(mode int) { - runtime_pollWaitCanceled(pd.runtimeCtx, mode) -} - -func (pd *pollDesc) waitCanceledRead() { - pd.waitCanceled('r') -} - -func (pd *pollDesc) waitCanceledWrite() { - pd.waitCanceled('w') -} - -func convertErr(res int) error { - switch res { - case 0: - return nil - case 1: - return errClosing - case 2: - return errTimeout - } - println("unreachable: ", res) - panic("unreachable") -} - -func (fd *netFD) setDeadline(t time.Time) error { - return setDeadlineImpl(fd, t, 'r'+'w') -} - -func (fd *netFD) setReadDeadline(t time.Time) error { - return setDeadlineImpl(fd, t, 'r') -} - -func (fd *netFD) setWriteDeadline(t time.Time) error { - return setDeadlineImpl(fd, t, 'w') -} - -func setDeadlineImpl(fd *netFD, t time.Time, mode int) error { - diff := int64(time.Until(t)) - d := runtimeNano() + diff - if d <= 0 && diff > 0 { - // If the user has a deadline in the future, but the delay calculation - // overflows, then set the deadline to the maximum possible value. - d = 1<<63 - 1 - } - if t.IsZero() { - d = 0 - } - if err := fd.incref(); err != nil { - return err - } - runtime_pollSetDeadline(fd.pd.runtimeCtx, d, mode) - fd.decref() - return nil -} diff --git a/libgo/go/net/fd_posix.go b/libgo/go/net/fd_posix.go deleted file mode 100644 index 72304796e4d..00000000000 --- a/libgo/go/net/fd_posix.go +++ /dev/null @@ -1,21 +0,0 @@ -// Copyright 2009 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// +build aix darwin dragonfly freebsd linux nacl netbsd openbsd solaris windows - -package net - -import ( - "io" - "syscall" -) - -// eofError returns io.EOF when fd is available for reading end of -// file. -func (fd *netFD) eofError(n int, err error) error { - if n == 0 && err == nil && fd.sotype != syscall.SOCK_DGRAM && fd.sotype != syscall.SOCK_RAW { - return io.EOF - } - return err -} diff --git a/libgo/go/net/fd_posix_test.go b/libgo/go/net/fd_posix_test.go deleted file mode 100644 index 85711ef1b70..00000000000 --- a/libgo/go/net/fd_posix_test.go +++ /dev/null @@ -1,57 +0,0 @@ -// Copyright 2012 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// +build darwin dragonfly freebsd linux nacl netbsd openbsd solaris windows - -package net - -import ( - "io" - "syscall" - "testing" -) - -var eofErrorTests = []struct { - n int - err error - fd *netFD - expected error -}{ - {100, nil, &netFD{sotype: syscall.SOCK_STREAM}, nil}, - {100, io.EOF, &netFD{sotype: syscall.SOCK_STREAM}, io.EOF}, - {100, errClosing, &netFD{sotype: syscall.SOCK_STREAM}, errClosing}, - {0, nil, &netFD{sotype: syscall.SOCK_STREAM}, io.EOF}, - {0, io.EOF, &netFD{sotype: syscall.SOCK_STREAM}, io.EOF}, - {0, errClosing, &netFD{sotype: syscall.SOCK_STREAM}, errClosing}, - - {100, nil, &netFD{sotype: syscall.SOCK_DGRAM}, nil}, - {100, io.EOF, &netFD{sotype: syscall.SOCK_DGRAM}, io.EOF}, - {100, errClosing, &netFD{sotype: syscall.SOCK_DGRAM}, errClosing}, - {0, nil, &netFD{sotype: syscall.SOCK_DGRAM}, nil}, - {0, io.EOF, &netFD{sotype: syscall.SOCK_DGRAM}, io.EOF}, - {0, errClosing, &netFD{sotype: syscall.SOCK_DGRAM}, errClosing}, - - {100, nil, &netFD{sotype: syscall.SOCK_SEQPACKET}, nil}, - {100, io.EOF, &netFD{sotype: syscall.SOCK_SEQPACKET}, io.EOF}, - {100, errClosing, &netFD{sotype: syscall.SOCK_SEQPACKET}, errClosing}, - {0, nil, &netFD{sotype: syscall.SOCK_SEQPACKET}, io.EOF}, - {0, io.EOF, &netFD{sotype: syscall.SOCK_SEQPACKET}, io.EOF}, - {0, errClosing, &netFD{sotype: syscall.SOCK_SEQPACKET}, errClosing}, - - {100, nil, &netFD{sotype: syscall.SOCK_RAW}, nil}, - {100, io.EOF, &netFD{sotype: syscall.SOCK_RAW}, io.EOF}, - {100, errClosing, &netFD{sotype: syscall.SOCK_RAW}, errClosing}, - {0, nil, &netFD{sotype: syscall.SOCK_RAW}, nil}, - {0, io.EOF, &netFD{sotype: syscall.SOCK_RAW}, io.EOF}, - {0, errClosing, &netFD{sotype: syscall.SOCK_RAW}, errClosing}, -} - -func TestEOFError(t *testing.T) { - for _, tt := range eofErrorTests { - actual := tt.fd.eofError(tt.n, tt.err) - if actual != tt.expected { - t.Errorf("eofError(%v, %v, %v): expected %v, actual %v", tt.n, tt.err, tt.fd.sotype, tt.expected, actual) - } - } -} diff --git a/libgo/go/net/fd_unix.go b/libgo/go/net/fd_unix.go index b6ee05976dd..e5afd1ae0ae 100644 --- a/libgo/go/net/fd_unix.go +++ b/libgo/go/net/fd_unix.go @@ -8,7 +8,7 @@ package net import ( "context" - "io" + "internal/poll" "os" "runtime" "sync/atomic" @@ -17,38 +17,33 @@ import ( // Network file descriptor. type netFD struct { - // locking/lifetime of sysfd + serialize access to Read and Write methods - fdmu fdMutex + pfd poll.FD // immutable until Close - sysfd int family int sotype int - isStream bool isConnected bool net string laddr Addr raddr Addr - - // writev cache. - iovecs *[]syscall.Iovec - - // wait server - pd pollDesc -} - -func sysInit() { } func newFD(sysfd, family, sotype int, net string) (*netFD, error) { - return &netFD{sysfd: sysfd, family: family, sotype: sotype, net: net, isStream: sotype == syscall.SOCK_STREAM}, nil + ret := &netFD{ + pfd: poll.FD{ + Sysfd: sysfd, + IsStream: sotype == syscall.SOCK_STREAM, + ZeroReadIsEOF: sotype != syscall.SOCK_DGRAM && sotype != syscall.SOCK_RAW, + }, + family: family, + sotype: sotype, + net: net, + } + return ret, nil } func (fd *netFD) init() error { - if err := fd.pd.init(fd); err != nil { - return err - } - return nil + return fd.pfd.Init(fd.net, true) } func (fd *netFD) setAddr(laddr, raddr Addr) { @@ -68,22 +63,23 @@ func (fd *netFD) name() string { return fd.net + ":" + ls + "->" + rs } -func (fd *netFD) connect(ctx context.Context, la, ra syscall.Sockaddr) (ret error) { +func (fd *netFD) connect(ctx context.Context, la, ra syscall.Sockaddr) (rsa syscall.Sockaddr, ret error) { // Do not need to call fd.writeLock here, // because fd is not yet accessible to user, // so no concurrent operations are possible. - switch err := connectFunc(fd.sysfd, ra); err { + switch err := connectFunc(fd.pfd.Sysfd, ra); err { case syscall.EINPROGRESS, syscall.EALREADY, syscall.EINTR: case nil, syscall.EISCONN: select { case <-ctx.Done(): - return mapErr(ctx.Err()) + return nil, mapErr(ctx.Err()) default: } - if err := fd.init(); err != nil { - return err + if err := fd.pfd.Init(fd.net, true); err != nil { + return nil, err } - return nil + runtime.KeepAlive(fd) + return nil, nil case syscall.EINVAL: // On Solaris we can see EINVAL if the socket has // already been accepted and closed by the server. @@ -91,18 +87,18 @@ func (fd *netFD) connect(ctx context.Context, la, ra syscall.Sockaddr) (ret erro // the socket will see EOF. For details and a test // case in C see https://golang.org/issue/6828. if runtime.GOOS == "solaris" { - return nil + return nil, nil } fallthrough default: - return os.NewSyscallError("connect", err) + return nil, os.NewSyscallError("connect", err) } - if err := fd.init(); err != nil { - return err + if err := fd.pfd.Init(fd.net, true); err != nil { + return nil, err } if deadline, _ := ctx.Deadline(); !deadline.IsZero() { - fd.setWriteDeadline(deadline) - defer fd.setWriteDeadline(noDeadline) + fd.pfd.SetWriteDeadline(deadline) + defer fd.pfd.SetWriteDeadline(noDeadline) } // Start the "interrupter" goroutine, if this context might be canceled. @@ -119,7 +115,7 @@ func (fd *netFD) connect(ctx context.Context, la, ra syscall.Sockaddr) (ret erro defer func() { close(done) if ctxErr := <-interruptRes; ctxErr != nil && ret == nil { - // The interrupter goroutine called setWriteDeadline, + // The interrupter goroutine called SetWriteDeadline, // but the connect code below had returned from // waitWrite already and did a successful connect (ret // == nil). Because we've now poisoned the connection @@ -135,7 +131,7 @@ func (fd *netFD) connect(ctx context.Context, la, ra syscall.Sockaddr) (ret erro // Force the runtime's poller to immediately give up // waiting for writability, unblocking waitWrite // below. - fd.setWriteDeadline(aLongTimeAgo) + fd.pfd.SetWriteDeadline(aLongTimeAgo) testHookCanceledDial() interruptRes <- ctx.Err() case <-done: @@ -153,66 +149,45 @@ func (fd *netFD) connect(ctx context.Context, la, ra syscall.Sockaddr) (ret erro // SO_ERROR socket option to see if the connection // succeeded or failed. See issue 7474 for further // details. - if err := fd.pd.waitWrite(); err != nil { + if err := fd.pfd.WaitWrite(); err != nil { select { case <-ctx.Done(): - return mapErr(ctx.Err()) + return nil, mapErr(ctx.Err()) default: } - return err + return nil, err } - nerr, err := getsockoptIntFunc(fd.sysfd, syscall.SOL_SOCKET, syscall.SO_ERROR) + nerr, err := getsockoptIntFunc(fd.pfd.Sysfd, syscall.SOL_SOCKET, syscall.SO_ERROR) if err != nil { - return os.NewSyscallError("getsockopt", err) + return nil, os.NewSyscallError("getsockopt", err) } switch err := syscall.Errno(nerr); err { case syscall.EINPROGRESS, syscall.EALREADY, syscall.EINTR: - case syscall.Errno(0), syscall.EISCONN: - if runtime.GOOS != "darwin" { - return nil - } - // See golang.org/issue/14548. - // On Darwin, multiple connect system calls on - // a non-blocking socket never harm SO_ERROR. - switch err := connectFunc(fd.sysfd, ra); err { - case nil, syscall.EISCONN: - return nil + case syscall.EISCONN: + return nil, nil + case syscall.Errno(0): + // The runtime poller can wake us up spuriously; + // see issues 14548 and 19289. Check that we are + // really connected; if not, wait again. + if rsa, err := syscall.Getpeername(fd.pfd.Sysfd); err == nil { + return rsa, nil } default: - return os.NewSyscallError("getsockopt", err) + return nil, os.NewSyscallError("getsockopt", err) } + runtime.KeepAlive(fd) } } -func (fd *netFD) destroy() { - // Poller may want to unregister fd in readiness notification mechanism, - // so this must be executed before closeFunc. - fd.pd.close() - closeFunc(fd.sysfd) - fd.sysfd = -1 - runtime.SetFinalizer(fd, nil) -} - func (fd *netFD) Close() error { - if !fd.fdmu.increfAndClose() { - return errClosing - } - // Unblock any I/O. Once it all unblocks and returns, - // so that it cannot be referring to fd.sysfd anymore, - // the final decref will close fd.sysfd. This should happen - // fairly quickly, since all the I/O is non-blocking, and any - // attempts to block in the pollDesc will return errClosing. - fd.pd.evict() - fd.decref() - return nil + runtime.SetFinalizer(fd, nil) + return fd.pfd.Close() } func (fd *netFD) shutdown(how int) error { - if err := fd.incref(); err != nil { - return err - } - defer fd.decref() - return os.NewSyscallError("shutdown", syscall.Shutdown(fd.sysfd, how)) + err := fd.pfd.Shutdown(how) + runtime.KeepAlive(fd) + return wrapSyscallError("shutdown", err) } func (fd *netFD) closeRead() error { @@ -224,233 +199,59 @@ func (fd *netFD) closeWrite() error { } func (fd *netFD) Read(p []byte) (n int, err error) { - if err := fd.readLock(); err != nil { - return 0, err - } - defer fd.readUnlock() - if len(p) == 0 { - // If the caller wanted a zero byte read, return immediately - // without trying. (But after acquiring the readLock.) Otherwise - // syscall.Read returns 0, nil and eofError turns that into - // io.EOF. - // TODO(bradfitz): make it wait for readability? (Issue 15735) - return 0, nil - } - if err := fd.pd.prepareRead(); err != nil { - return 0, err - } - if fd.isStream && len(p) > 1<<30 { - p = p[:1<<30] - } - for { - n, err = syscall.Read(fd.sysfd, p) - if err != nil { - n = 0 - if err == syscall.EAGAIN { - if err = fd.pd.waitRead(); err == nil { - continue - } - } - } - err = fd.eofError(n, err) - break - } - if _, ok := err.(syscall.Errno); ok { - err = os.NewSyscallError("read", err) - } - return + n, err = fd.pfd.Read(p) + runtime.KeepAlive(fd) + return n, wrapSyscallError("read", err) } func (fd *netFD) readFrom(p []byte) (n int, sa syscall.Sockaddr, err error) { - if err := fd.readLock(); err != nil { - return 0, nil, err - } - defer fd.readUnlock() - if err := fd.pd.prepareRead(); err != nil { - return 0, nil, err - } - for { - n, sa, err = syscall.Recvfrom(fd.sysfd, p, 0) - if err != nil { - n = 0 - if err == syscall.EAGAIN { - if err = fd.pd.waitRead(); err == nil { - continue - } - } - } - err = fd.eofError(n, err) - break - } - if _, ok := err.(syscall.Errno); ok { - err = os.NewSyscallError("recvfrom", err) - } - return + n, sa, err = fd.pfd.ReadFrom(p) + runtime.KeepAlive(fd) + return n, sa, wrapSyscallError("recvfrom", err) } func (fd *netFD) readMsg(p []byte, oob []byte) (n, oobn, flags int, sa syscall.Sockaddr, err error) { - if err := fd.readLock(); err != nil { - return 0, 0, 0, nil, err - } - defer fd.readUnlock() - if err := fd.pd.prepareRead(); err != nil { - return 0, 0, 0, nil, err - } - for { - n, oobn, flags, sa, err = syscall.Recvmsg(fd.sysfd, p, oob, 0) - if err != nil { - // TODO(dfc) should n and oobn be set to 0 - if err == syscall.EAGAIN { - if err = fd.pd.waitRead(); err == nil { - continue - } - } - } - err = fd.eofError(n, err) - break - } - if _, ok := err.(syscall.Errno); ok { - err = os.NewSyscallError("recvmsg", err) - } - return + n, oobn, flags, sa, err = fd.pfd.ReadMsg(p, oob) + runtime.KeepAlive(fd) + return n, oobn, flags, sa, wrapSyscallError("recvmsg", err) } func (fd *netFD) Write(p []byte) (nn int, err error) { - if err := fd.writeLock(); err != nil { - return 0, err - } - defer fd.writeUnlock() - if err := fd.pd.prepareWrite(); err != nil { - return 0, err - } - for { - var n int - max := len(p) - if fd.isStream && max-nn > 1<<30 { - max = nn + 1<<30 - } - n, err = syscall.Write(fd.sysfd, p[nn:max]) - if n > 0 { - nn += n - } - if nn == len(p) { - break - } - if err == syscall.EAGAIN { - if err = fd.pd.waitWrite(); err == nil { - continue - } - } - if err != nil { - break - } - if n == 0 { - err = io.ErrUnexpectedEOF - break - } - } - if _, ok := err.(syscall.Errno); ok { - err = os.NewSyscallError("write", err) - } - return nn, err + nn, err = fd.pfd.Write(p) + runtime.KeepAlive(fd) + return nn, wrapSyscallError("write", err) } func (fd *netFD) writeTo(p []byte, sa syscall.Sockaddr) (n int, err error) { - if err := fd.writeLock(); err != nil { - return 0, err - } - defer fd.writeUnlock() - if err := fd.pd.prepareWrite(); err != nil { - return 0, err - } - for { - err = syscall.Sendto(fd.sysfd, p, 0, sa) - if err == syscall.EAGAIN { - if err = fd.pd.waitWrite(); err == nil { - continue - } - } - break - } - if err == nil { - n = len(p) - } - if _, ok := err.(syscall.Errno); ok { - err = os.NewSyscallError("sendto", err) - } - return + n, err = fd.pfd.WriteTo(p, sa) + runtime.KeepAlive(fd) + return n, wrapSyscallError("sendto", err) } func (fd *netFD) writeMsg(p []byte, oob []byte, sa syscall.Sockaddr) (n int, oobn int, err error) { - if err := fd.writeLock(); err != nil { - return 0, 0, err - } - defer fd.writeUnlock() - if err := fd.pd.prepareWrite(); err != nil { - return 0, 0, err - } - for { - n, err = syscall.SendmsgN(fd.sysfd, p, oob, sa, 0) - if err == syscall.EAGAIN { - if err = fd.pd.waitWrite(); err == nil { - continue - } - } - break - } - if err == nil { - oobn = len(oob) - } - if _, ok := err.(syscall.Errno); ok { - err = os.NewSyscallError("sendmsg", err) - } - return + n, oobn, err = fd.pfd.WriteMsg(p, oob, sa) + runtime.KeepAlive(fd) + return n, oobn, wrapSyscallError("sendmsg", err) } func (fd *netFD) accept() (netfd *netFD, err error) { - if err := fd.readLock(); err != nil { - return nil, err - } - defer fd.readUnlock() - - var s int - var rsa syscall.Sockaddr - if err = fd.pd.prepareRead(); err != nil { - return nil, err - } - for { - s, rsa, err = accept(fd.sysfd) - if err != nil { - nerr, ok := err.(*os.SyscallError) - if !ok { - return nil, err - } - switch nerr.Err { - case syscall.EAGAIN: - if err = fd.pd.waitRead(); err == nil { - continue - } - case syscall.ECONNABORTED: - // This means that a socket on the - // listen queue was closed before we - // Accept()ed it; it's a silly error, - // so try again. - continue - } - return nil, err + d, rsa, errcall, err := fd.pfd.Accept() + if err != nil { + if errcall != "" { + err = wrapSyscallError(errcall, err) } - break + return nil, err } - if netfd, err = newFD(s, fd.family, fd.sotype, fd.net); err != nil { - closeFunc(s) + if netfd, err = newFD(d, fd.family, fd.sotype, fd.net); err != nil { + poll.CloseFunc(d) return nil, err } if err = netfd.init(); err != nil { fd.Close() return nil, err } - lsa, _ := syscall.Getsockname(netfd.sysfd) + lsa, _ := syscall.Getsockname(netfd.pfd.Sysfd) netfd.setAddr(netfd.addrFunc()(lsa), netfd.addrFunc()(rsa)) return netfd, nil } @@ -511,7 +312,7 @@ func dupCloseOnExecOld(fd int) (newfd int, err error) { } func (fd *netFD) dup() (f *os.File, err error) { - ns, err := dupCloseOnExec(fd.sysfd) + ns, err := dupCloseOnExec(fd.pfd.Sysfd) if err != nil { return nil, err } diff --git a/libgo/go/net/fd_windows.go b/libgo/go/net/fd_windows.go index a976f2ac7f9..c2156b255e5 100644 --- a/libgo/go/net/fd_windows.go +++ b/libgo/go/net/fd_windows.go @@ -6,64 +6,13 @@ package net import ( "context" - "internal/race" + "internal/poll" "os" "runtime" - "sync" "syscall" "unsafe" ) -var ( - initErr error - ioSync uint64 -) - -// CancelIo Windows API cancels all outstanding IO for a particular -// socket on current thread. To overcome that limitation, we run -// special goroutine, locked to OS single thread, that both starts -// and cancels IO. It means, there are 2 unavoidable thread switches -// for every IO. -// Some newer versions of Windows has new CancelIoEx API, that does -// not have that limitation and can be used from any thread. This -// package uses CancelIoEx API, if present, otherwise it fallback -// to CancelIo. - -var ( - canCancelIO bool // determines if CancelIoEx API is present - skipSyncNotif bool - hasLoadSetFileCompletionNotificationModes bool -) - -func sysInit() { - var d syscall.WSAData - e := syscall.WSAStartup(uint32(0x202), &d) - if e != nil { - initErr = os.NewSyscallError("wsastartup", e) - } - canCancelIO = syscall.LoadCancelIoEx() == nil - hasLoadSetFileCompletionNotificationModes = syscall.LoadSetFileCompletionNotificationModes() == nil - if hasLoadSetFileCompletionNotificationModes { - // It's not safe to use FILE_SKIP_COMPLETION_PORT_ON_SUCCESS if non IFS providers are installed: - // http://support.microsoft.com/kb/2568167 - skipSyncNotif = true - protos := [2]int32{syscall.IPPROTO_TCP, 0} - var buf [32]syscall.WSAProtocolInfo - len := uint32(unsafe.Sizeof(buf)) - n, err := syscall.WSAEnumProtocols(&protos[0], &buf[0], &len) - if err != nil { - skipSyncNotif = false - } else { - for i := int32(0); i < n; i++ { - if buf[i].ServiceFlags1&syscall.XP1_IFS_HANDLES == 0 { - skipSyncNotif = false - break - } - } - } - } -} - // canUseConnectEx reports whether we can use the ConnectEx Windows API call // for the given network type. func canUseConnectEx(net string) bool { @@ -75,257 +24,39 @@ func canUseConnectEx(net string) bool { return false } -// operation contains superset of data necessary to perform all async IO. -type operation struct { - // Used by IOCP interface, it must be first field - // of the struct, as our code rely on it. - o syscall.Overlapped - - // fields used by runtime.netpoll - runtimeCtx uintptr - mode int32 - errno int32 - qty uint32 - - // fields used only by net package - fd *netFD - errc chan error - buf syscall.WSABuf - sa syscall.Sockaddr - rsa *syscall.RawSockaddrAny - rsan int32 - handle syscall.Handle - flags uint32 - bufs []syscall.WSABuf -} - -func (o *operation) InitBuf(buf []byte) { - o.buf.Len = uint32(len(buf)) - o.buf.Buf = nil - if len(buf) != 0 { - o.buf.Buf = &buf[0] - } -} - -func (o *operation) InitBufs(buf *Buffers) { - if o.bufs == nil { - o.bufs = make([]syscall.WSABuf, 0, len(*buf)) - } else { - o.bufs = o.bufs[:0] - } - for _, b := range *buf { - var p *byte - if len(b) > 0 { - p = &b[0] - } - o.bufs = append(o.bufs, syscall.WSABuf{Len: uint32(len(b)), Buf: p}) - } -} - -// ClearBufs clears all pointers to Buffers parameter captured -// by InitBufs, so it can be released by garbage collector. -func (o *operation) ClearBufs() { - for i := range o.bufs { - o.bufs[i].Buf = nil - } - o.bufs = o.bufs[:0] -} - -// ioSrv executes net IO requests. -type ioSrv struct { - req chan ioSrvReq -} - -type ioSrvReq struct { - o *operation - submit func(o *operation) error // if nil, cancel the operation -} - -// ProcessRemoteIO will execute submit IO requests on behalf -// of other goroutines, all on a single os thread, so it can -// cancel them later. Results of all operations will be sent -// back to their requesters via channel supplied in request. -// It is used only when the CancelIoEx API is unavailable. -func (s *ioSrv) ProcessRemoteIO() { - runtime.LockOSThread() - defer runtime.UnlockOSThread() - for r := range s.req { - if r.submit != nil { - r.o.errc <- r.submit(r.o) - } else { - r.o.errc <- syscall.CancelIo(r.o.fd.sysfd) - } - } -} - -// ExecIO executes a single IO operation o. It submits and cancels -// IO in the current thread for systems where Windows CancelIoEx API -// is available. Alternatively, it passes the request onto -// runtime netpoll and waits for completion or cancels request. -func (s *ioSrv) ExecIO(o *operation, name string, submit func(o *operation) error) (int, error) { - fd := o.fd - // Notify runtime netpoll about starting IO. - err := fd.pd.prepare(int(o.mode)) - if err != nil { - return 0, err - } - // Start IO. - if canCancelIO { - err = submit(o) - } else { - // Send request to a special dedicated thread, - // so it can stop the IO with CancelIO later. - s.req <- ioSrvReq{o, submit} - err = <-o.errc - } - switch err { - case nil: - // IO completed immediately - if o.fd.skipSyncNotif { - // No completion message will follow, so return immediately. - return int(o.qty), nil - } - // Need to get our completion message anyway. - case syscall.ERROR_IO_PENDING: - // IO started, and we have to wait for its completion. - err = nil - default: - return 0, err - } - // Wait for our request to complete. - err = fd.pd.wait(int(o.mode)) - if err == nil { - // All is good. Extract our IO results and return. - if o.errno != 0 { - err = syscall.Errno(o.errno) - return 0, err - } - return int(o.qty), nil - } - // IO is interrupted by "close" or "timeout" - netpollErr := err - switch netpollErr { - case errClosing, errTimeout: - // will deal with those. - default: - panic("net: unexpected runtime.netpoll error: " + netpollErr.Error()) - } - // Cancel our request. - if canCancelIO { - err := syscall.CancelIoEx(fd.sysfd, &o.o) - // Assuming ERROR_NOT_FOUND is returned, if IO is completed. - if err != nil && err != syscall.ERROR_NOT_FOUND { - // TODO(brainman): maybe do something else, but panic. - panic(err) - } - } else { - s.req <- ioSrvReq{o, nil} - <-o.errc - } - // Wait for cancelation to complete. - fd.pd.waitCanceled(int(o.mode)) - if o.errno != 0 { - err = syscall.Errno(o.errno) - if err == syscall.ERROR_OPERATION_ABORTED { // IO Canceled - err = netpollErr - } - return 0, err - } - // We issued a cancelation request. But, it seems, IO operation succeeded - // before the cancelation request run. We need to treat the IO operation as - // succeeded (the bytes are actually sent/recv from network). - return int(o.qty), nil -} - -// Start helper goroutines. -var rsrv, wsrv *ioSrv -var onceStartServer sync.Once - -func startServer() { - rsrv = new(ioSrv) - wsrv = new(ioSrv) - if !canCancelIO { - // Only CancelIo API is available. Lets start two special goroutines - // locked to an OS thread, that both starts and cancels IO. One will - // process read requests, while other will do writes. - rsrv.req = make(chan ioSrvReq) - go rsrv.ProcessRemoteIO() - wsrv.req = make(chan ioSrvReq) - go wsrv.ProcessRemoteIO() - } -} - // Network file descriptor. type netFD struct { - // locking/lifetime of sysfd + serialize access to Read and Write methods - fdmu fdMutex + pfd poll.FD // immutable until Close - sysfd syscall.Handle - family int - sotype int - isStream bool - isConnected bool - skipSyncNotif bool - net string - laddr Addr - raddr Addr - - rop operation // read operation - wop operation // write operation - - // wait server - pd pollDesc + family int + sotype int + isConnected bool + net string + laddr Addr + raddr Addr } func newFD(sysfd syscall.Handle, family, sotype int, net string) (*netFD, error) { - if initErr != nil { - return nil, initErr + ret := &netFD{ + pfd: poll.FD{ + Sysfd: sysfd, + IsStream: sotype == syscall.SOCK_STREAM, + ZeroReadIsEOF: sotype != syscall.SOCK_DGRAM && sotype != syscall.SOCK_RAW, + }, + family: family, + sotype: sotype, + net: net, } - onceStartServer.Do(startServer) - return &netFD{sysfd: sysfd, family: family, sotype: sotype, net: net, isStream: sotype == syscall.SOCK_STREAM}, nil + return ret, nil } func (fd *netFD) init() error { - if err := fd.pd.init(fd); err != nil { - return err - } - if hasLoadSetFileCompletionNotificationModes { - // We do not use events, so we can skip them always. - flags := uint8(syscall.FILE_SKIP_SET_EVENT_ON_HANDLE) - // It's not safe to skip completion notifications for UDP: - // http://blogs.technet.com/b/winserverperformance/archive/2008/06/26/designing-applications-for-high-performance-part-iii.aspx - if skipSyncNotif && fd.net == "tcp" { - flags |= syscall.FILE_SKIP_COMPLETION_PORT_ON_SUCCESS - } - err := syscall.SetFileCompletionNotificationModes(fd.sysfd, flags) - if err == nil && flags&syscall.FILE_SKIP_COMPLETION_PORT_ON_SUCCESS != 0 { - fd.skipSyncNotif = true - } - } - // Disable SIO_UDP_CONNRESET behavior. - // http://support.microsoft.com/kb/263823 - switch fd.net { - case "udp", "udp4", "udp6": - ret := uint32(0) - flag := uint32(0) - size := uint32(unsafe.Sizeof(flag)) - err := syscall.WSAIoctl(fd.sysfd, syscall.SIO_UDP_CONNRESET, (*byte)(unsafe.Pointer(&flag)), size, nil, 0, &ret, nil, 0) - if err != nil { - return os.NewSyscallError("wsaioctl", err) - } + errcall, err := fd.pfd.Init(fd.net) + if errcall != "" { + err = wrapSyscallError(errcall, err) } - fd.rop.mode = 'r' - fd.wop.mode = 'w' - fd.rop.fd = fd - fd.wop.fd = fd - fd.rop.runtimeCtx = fd.pd.runtimeCtx - fd.wop.runtimeCtx = fd.pd.runtimeCtx - if !canCancelIO { - fd.rop.errc = make(chan error) - fd.wop.errc = make(chan error) - } - return nil + return err } func (fd *netFD) setAddr(laddr, raddr Addr) { @@ -334,20 +65,21 @@ func (fd *netFD) setAddr(laddr, raddr Addr) { runtime.SetFinalizer(fd, (*netFD).Close) } -func (fd *netFD) connect(ctx context.Context, la, ra syscall.Sockaddr) error { +// Always returns nil for connected peer address result. +func (fd *netFD) connect(ctx context.Context, la, ra syscall.Sockaddr) (syscall.Sockaddr, error) { // Do not need to call fd.writeLock here, // because fd is not yet accessible to user, // so no concurrent operations are possible. if err := fd.init(); err != nil { - return err + return nil, err } if deadline, ok := ctx.Deadline(); ok && !deadline.IsZero() { - fd.setWriteDeadline(deadline) - defer fd.setWriteDeadline(noDeadline) + fd.pfd.SetWriteDeadline(deadline) + defer fd.pfd.SetWriteDeadline(noDeadline) } if !canUseConnectEx(fd.net) { - err := connectFunc(fd.sysfd, ra) - return os.NewSyscallError("connect", err) + err := connectFunc(fd.pfd.Sysfd, ra) + return nil, os.NewSyscallError("connect", err) } // ConnectEx windows API requires an unconnected, previously bound socket. if la == nil { @@ -359,13 +91,10 @@ func (fd *netFD) connect(ctx context.Context, la, ra syscall.Sockaddr) error { default: panic("unexpected type in connect") } - if err := syscall.Bind(fd.sysfd, la); err != nil { - return os.NewSyscallError("bind", err) + if err := syscall.Bind(fd.pfd.Sysfd, la); err != nil { + return nil, os.NewSyscallError("bind", err) } } - // Call ConnectEx API. - o := &fd.wop - o.sa = ra // Wait for the goroutine converting context.Done into a write timeout // to exist, otherwise our caller might cancel the context and @@ -377,59 +106,37 @@ func (fd *netFD) connect(ctx context.Context, la, ra syscall.Sockaddr) error { case <-ctx.Done(): // Force the runtime's poller to immediately give // up waiting for writability. - fd.setWriteDeadline(aLongTimeAgo) + fd.pfd.SetWriteDeadline(aLongTimeAgo) <-done case <-done: } }() - _, err := wsrv.ExecIO(o, "ConnectEx", func(o *operation) error { - return connectExFunc(o.fd.sysfd, o.sa, nil, 0, nil, &o.o) - }) - if err != nil { + // Call ConnectEx API. + if err := fd.pfd.ConnectEx(ra); err != nil { select { case <-ctx.Done(): - return mapErr(ctx.Err()) + return nil, mapErr(ctx.Err()) default: if _, ok := err.(syscall.Errno); ok { err = os.NewSyscallError("connectex", err) } - return err + return nil, err } } // Refresh socket properties. - return os.NewSyscallError("setsockopt", syscall.Setsockopt(fd.sysfd, syscall.SOL_SOCKET, syscall.SO_UPDATE_CONNECT_CONTEXT, (*byte)(unsafe.Pointer(&fd.sysfd)), int32(unsafe.Sizeof(fd.sysfd)))) -} - -func (fd *netFD) destroy() { - if fd.sysfd == syscall.InvalidHandle { - return - } - // Poller may want to unregister fd in readiness notification mechanism, - // so this must be executed before closeFunc. - fd.pd.close() - closeFunc(fd.sysfd) - fd.sysfd = syscall.InvalidHandle - // no need for a finalizer anymore - runtime.SetFinalizer(fd, nil) + return nil, os.NewSyscallError("setsockopt", syscall.Setsockopt(fd.pfd.Sysfd, syscall.SOL_SOCKET, syscall.SO_UPDATE_CONNECT_CONTEXT, (*byte)(unsafe.Pointer(&fd.pfd.Sysfd)), int32(unsafe.Sizeof(fd.pfd.Sysfd)))) } func (fd *netFD) Close() error { - if !fd.fdmu.increfAndClose() { - return errClosing - } - // unblock pending reader and writer - fd.pd.evict() - fd.decref() - return nil + runtime.SetFinalizer(fd, nil) + return fd.pfd.Close() } func (fd *netFD) shutdown(how int) error { - if err := fd.incref(); err != nil { - return err - } - defer fd.decref() - return syscall.Shutdown(fd.sysfd, how) + err := fd.pfd.Shutdown(how) + runtime.KeepAlive(fd) + return err } func (fd *netFD) closeRead() error { @@ -441,72 +148,21 @@ func (fd *netFD) closeWrite() error { } func (fd *netFD) Read(buf []byte) (int, error) { - if err := fd.readLock(); err != nil { - return 0, err - } - defer fd.readUnlock() - o := &fd.rop - o.InitBuf(buf) - n, err := rsrv.ExecIO(o, "WSARecv", func(o *operation) error { - return syscall.WSARecv(o.fd.sysfd, &o.buf, 1, &o.qty, &o.flags, &o.o, nil) - }) - if race.Enabled { - race.Acquire(unsafe.Pointer(&ioSync)) - } - if len(buf) != 0 { - err = fd.eofError(n, err) - } - if _, ok := err.(syscall.Errno); ok { - err = os.NewSyscallError("wsarecv", err) - } - return n, err + n, err := fd.pfd.Read(buf) + runtime.KeepAlive(fd) + return n, wrapSyscallError("wsarecv", err) } func (fd *netFD) readFrom(buf []byte) (int, syscall.Sockaddr, error) { - if len(buf) == 0 { - return 0, nil, nil - } - if err := fd.readLock(); err != nil { - return 0, nil, err - } - defer fd.readUnlock() - o := &fd.rop - o.InitBuf(buf) - n, err := rsrv.ExecIO(o, "WSARecvFrom", func(o *operation) error { - if o.rsa == nil { - o.rsa = new(syscall.RawSockaddrAny) - } - o.rsan = int32(unsafe.Sizeof(*o.rsa)) - return syscall.WSARecvFrom(o.fd.sysfd, &o.buf, 1, &o.qty, &o.flags, o.rsa, &o.rsan, &o.o, nil) - }) - err = fd.eofError(n, err) - if _, ok := err.(syscall.Errno); ok { - err = os.NewSyscallError("wsarecvfrom", err) - } - if err != nil { - return n, nil, err - } - sa, _ := o.rsa.Sockaddr() - return n, sa, nil + n, sa, err := fd.pfd.ReadFrom(buf) + runtime.KeepAlive(fd) + return n, sa, wrapSyscallError("wsarecvfrom", err) } func (fd *netFD) Write(buf []byte) (int, error) { - if err := fd.writeLock(); err != nil { - return 0, err - } - defer fd.writeUnlock() - if race.Enabled { - race.ReleaseMerge(unsafe.Pointer(&ioSync)) - } - o := &fd.wop - o.InitBuf(buf) - n, err := wsrv.ExecIO(o, "WSASend", func(o *operation) error { - return syscall.WSASend(o.fd.sysfd, &o.buf, 1, &o.qty, 0, &o.o, nil) - }) - if _, ok := err.(syscall.Errno); ok { - err = os.NewSyscallError("wsasend", err) - } - return n, err + n, err := fd.pfd.Write(buf) + runtime.KeepAlive(fd) + return n, wrapSyscallError("wsasend", err) } func (c *conn) writeBuffers(v *Buffers) (int64, error) { @@ -515,67 +171,39 @@ func (c *conn) writeBuffers(v *Buffers) (int64, error) { } n, err := c.fd.writeBuffers(v) if err != nil { - return n, &OpError{Op: "WSASend", Net: c.fd.net, Source: c.fd.laddr, Addr: c.fd.raddr, Err: err} + return n, &OpError{Op: "wsasend", Net: c.fd.net, Source: c.fd.laddr, Addr: c.fd.raddr, Err: err} } return n, nil } func (fd *netFD) writeBuffers(buf *Buffers) (int64, error) { - if len(*buf) == 0 { - return 0, nil - } - if err := fd.writeLock(); err != nil { - return 0, err - } - defer fd.writeUnlock() - if race.Enabled { - race.ReleaseMerge(unsafe.Pointer(&ioSync)) - } - o := &fd.wop - o.InitBufs(buf) - n, err := wsrv.ExecIO(o, "WSASend", func(o *operation) error { - return syscall.WSASend(o.fd.sysfd, &o.bufs[0], uint32(len(*buf)), &o.qty, 0, &o.o, nil) - }) - o.ClearBufs() - if _, ok := err.(syscall.Errno); ok { - err = os.NewSyscallError("wsasend", err) - } - testHookDidWritev(n) - buf.consume(int64(n)) - return int64(n), err + n, err := fd.pfd.Writev((*[][]byte)(buf)) + runtime.KeepAlive(fd) + return n, wrapSyscallError("wsasend", err) } func (fd *netFD) writeTo(buf []byte, sa syscall.Sockaddr) (int, error) { - if len(buf) == 0 { - return 0, nil - } - if err := fd.writeLock(); err != nil { - return 0, err - } - defer fd.writeUnlock() - o := &fd.wop - o.InitBuf(buf) - o.sa = sa - n, err := wsrv.ExecIO(o, "WSASendto", func(o *operation) error { - return syscall.WSASendto(o.fd.sysfd, &o.buf, 1, &o.qty, 0, o.sa, &o.o, nil) - }) - if _, ok := err.(syscall.Errno); ok { - err = os.NewSyscallError("wsasendto", err) - } - return n, err + n, err := fd.pfd.WriteTo(buf, sa) + runtime.KeepAlive(fd) + return n, wrapSyscallError("wsasendto", err) } -func (fd *netFD) acceptOne(rawsa []syscall.RawSockaddrAny, o *operation) (*netFD, error) { - // Get new socket. - s, err := sysSocket(fd.family, fd.sotype, 0) +func (fd *netFD) accept() (*netFD, error) { + s, rawsa, rsan, errcall, err := fd.pfd.Accept(func() (syscall.Handle, error) { + return sysSocket(fd.family, fd.sotype, 0) + }) + if err != nil { + if errcall != "" { + err = wrapSyscallError(errcall, err) + } return nil, err } // Associate our new socket with IOCP. netfd, err := newFD(s, fd.family, fd.sotype, fd.net) if err != nil { - closeFunc(s) + poll.CloseFunc(s) return nil, err } if err := netfd.init(); err != nil { @@ -583,71 +211,11 @@ func (fd *netFD) acceptOne(rawsa []syscall.RawSockaddrAny, o *operation) (*netFD return nil, err } - // Submit accept request. - o.handle = s - o.rsan = int32(unsafe.Sizeof(rawsa[0])) - _, err = rsrv.ExecIO(o, "AcceptEx", func(o *operation) error { - return acceptFunc(o.fd.sysfd, o.handle, (*byte)(unsafe.Pointer(&rawsa[0])), 0, uint32(o.rsan), uint32(o.rsan), &o.qty, &o.o) - }) - if err != nil { - netfd.Close() - if _, ok := err.(syscall.Errno); ok { - err = os.NewSyscallError("acceptex", err) - } - return nil, err - } - - // Inherit properties of the listening socket. - err = syscall.Setsockopt(s, syscall.SOL_SOCKET, syscall.SO_UPDATE_ACCEPT_CONTEXT, (*byte)(unsafe.Pointer(&fd.sysfd)), int32(unsafe.Sizeof(fd.sysfd))) - if err != nil { - netfd.Close() - return nil, os.NewSyscallError("setsockopt", err) - } - runtime.KeepAlive(fd) - return netfd, nil -} - -func (fd *netFD) accept() (*netFD, error) { - if err := fd.readLock(); err != nil { - return nil, err - } - defer fd.readUnlock() - - o := &fd.rop - var netfd *netFD - var err error - var rawsa [2]syscall.RawSockaddrAny - for { - netfd, err = fd.acceptOne(rawsa[:], o) - if err == nil { - break - } - // Sometimes we see WSAECONNRESET and ERROR_NETNAME_DELETED is - // returned here. These happen if connection reset is received - // before AcceptEx could complete. These errors relate to new - // connection, not to AcceptEx, so ignore broken connection and - // try AcceptEx again for more connections. - nerr, ok := err.(*os.SyscallError) - if !ok { - return nil, err - } - errno, ok := nerr.Err.(syscall.Errno) - if !ok { - return nil, err - } - switch errno { - case syscall.ERROR_NETNAME_DELETED, syscall.WSAECONNRESET: - // ignore these and try again - default: - return nil, err - } - } - // Get local and peer addr out of AcceptEx buffer. var lrsa, rrsa *syscall.RawSockaddrAny var llen, rlen int32 syscall.GetAcceptExSockaddrs((*byte)(unsafe.Pointer(&rawsa[0])), - 0, uint32(o.rsan), uint32(o.rsan), &lrsa, &llen, &rrsa, &rlen) + 0, rsan, rsan, &lrsa, &llen, &rrsa, &rlen) lsa, _ := lrsa.Sockaddr() rsa, _ := rrsa.Sockaddr() diff --git a/libgo/go/net/file_test.go b/libgo/go/net/file_test.go index 6566ce21a1f..abf8b3a6995 100644 --- a/libgo/go/net/file_test.go +++ b/libgo/go/net/file_test.go @@ -90,7 +90,7 @@ func TestFileConn(t *testing.T) { f, err = c1.File() } if err := c1.Close(); err != nil { - if perr := parseCloseError(err); perr != nil { + if perr := parseCloseError(err, false); perr != nil { t.Error(perr) } t.Error(err) @@ -256,7 +256,7 @@ func TestFilePacketConn(t *testing.T) { f, err = c1.File() } if err := c1.Close(); err != nil { - if perr := parseCloseError(err); perr != nil { + if perr := parseCloseError(err, false); perr != nil { t.Error(perr) } t.Error(err) diff --git a/libgo/go/net/file_unix.go b/libgo/go/net/file_unix.go index b47a6143e9a..3655a8930a1 100644 --- a/libgo/go/net/file_unix.go +++ b/libgo/go/net/file_unix.go @@ -7,6 +7,7 @@ package net import ( + "internal/poll" "os" "syscall" ) @@ -17,7 +18,7 @@ func dupSocket(f *os.File) (int, error) { return -1, err } if err := syscall.SetNonblock(s, true); err != nil { - closeFunc(s) + poll.CloseFunc(s) return -1, os.NewSyscallError("setnonblock", err) } return s, nil @@ -31,7 +32,7 @@ func newFileFD(f *os.File) (*netFD, error) { family := syscall.AF_UNSPEC sotype, err := syscall.GetsockoptInt(s, syscall.SOL_SOCKET, syscall.SO_TYPE) if err != nil { - closeFunc(s) + poll.CloseFunc(s) return nil, os.NewSyscallError("getsockopt", err) } lsa, _ := syscall.Getsockname(s) @@ -44,12 +45,12 @@ func newFileFD(f *os.File) (*netFD, error) { case *syscall.SockaddrUnix: family = syscall.AF_UNIX default: - closeFunc(s) + poll.CloseFunc(s) return nil, syscall.EPROTONOSUPPORT } fd, err := newFD(s, family, sotype, "") if err != nil { - closeFunc(s) + poll.CloseFunc(s) return nil, err } laddr := fd.addrFunc()(lsa) diff --git a/libgo/go/net/hook_cloexec.go b/libgo/go/net/hook_cloexec.go deleted file mode 100644 index 870f0d78b12..00000000000 --- a/libgo/go/net/hook_cloexec.go +++ /dev/null @@ -1,14 +0,0 @@ -// Copyright 2015 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// +build freebsd linux - -package net - -import "syscall" - -var ( - // Placeholders for socket system calls. - accept4Func func(int, int) (int, syscall.Sockaddr, error) = syscall.Accept4 -) diff --git a/libgo/go/net/hook_unix.go b/libgo/go/net/hook_unix.go index b2522a2ed23..7d58d0fcf20 100644 --- a/libgo/go/net/hook_unix.go +++ b/libgo/go/net/hook_unix.go @@ -13,10 +13,8 @@ var ( testHookCanceledDial = func() {} // for golang.org/issue/16523 // Placeholders for socket system calls. - socketFunc func(int, int, int) (int, error) = syscall.Socket - closeFunc func(int) error = syscall.Close - connectFunc func(int, syscall.Sockaddr) error = syscall.Connect - listenFunc func(int, int) error = syscall.Listen - acceptFunc func(int) (int, syscall.Sockaddr, error) = syscall.Accept - getsockoptIntFunc func(int, int, int) (int, error) = syscall.GetsockoptInt + socketFunc func(int, int, int) (int, error) = syscall.Socket + connectFunc func(int, syscall.Sockaddr) error = syscall.Connect + listenFunc func(int, int) error = syscall.Listen + getsockoptIntFunc func(int, int, int) (int, error) = syscall.GetsockoptInt ) diff --git a/libgo/go/net/hook_windows.go b/libgo/go/net/hook_windows.go index 63ea35ab8c4..4e64dcef517 100644 --- a/libgo/go/net/hook_windows.go +++ b/libgo/go/net/hook_windows.go @@ -13,10 +13,7 @@ var ( testHookDialChannel = func() { time.Sleep(time.Millisecond) } // see golang.org/issue/5349 // Placeholders for socket system calls. - socketFunc func(int, int, int) (syscall.Handle, error) = syscall.Socket - closeFunc func(syscall.Handle) error = syscall.Closesocket - connectFunc func(syscall.Handle, syscall.Sockaddr) error = syscall.Connect - connectExFunc func(syscall.Handle, syscall.Sockaddr, *byte, uint32, *uint32, *syscall.Overlapped) error = syscall.ConnectEx - listenFunc func(syscall.Handle, int) error = syscall.Listen - acceptFunc func(syscall.Handle, syscall.Handle, *byte, uint32, uint32, uint32, *uint32, *syscall.Overlapped) error = syscall.AcceptEx + socketFunc func(int, int, int) (syscall.Handle, error) = syscall.Socket + connectFunc func(syscall.Handle, syscall.Sockaddr) error = syscall.Connect + listenFunc func(syscall.Handle, int) error = syscall.Listen ) diff --git a/libgo/go/net/http/cgi/host_test.go b/libgo/go/net/http/cgi/host_test.go index f0583729eba..133630013e5 100644 --- a/libgo/go/net/http/cgi/host_test.go +++ b/libgo/go/net/http/cgi/host_test.go @@ -409,7 +409,7 @@ func TestCopyError(t *testing.T) { } childRunning := func() bool { - return isProcessRunning(t, pid) + return isProcessRunning(pid) } if !childRunning() { diff --git a/libgo/go/net/http/cgi/posix_test.go b/libgo/go/net/http/cgi/posix_test.go index 5ff9e7d5eb7..9396ce036af 100644 --- a/libgo/go/net/http/cgi/posix_test.go +++ b/libgo/go/net/http/cgi/posix_test.go @@ -9,10 +9,9 @@ package cgi import ( "os" "syscall" - "testing" ) -func isProcessRunning(t *testing.T, pid int) bool { +func isProcessRunning(pid int) bool { p, err := os.FindProcess(pid) if err != nil { return false diff --git a/libgo/go/net/http/client.go b/libgo/go/net/http/client.go index 0005538e70b..4c9084ae512 100644 --- a/libgo/go/net/http/client.go +++ b/libgo/go/net/http/client.go @@ -38,20 +38,20 @@ import ( // When following redirects, the Client will forward all headers set on the // initial Request except: // -// * when forwarding sensitive headers like "Authorization", -// "WWW-Authenticate", and "Cookie" to untrusted targets. -// These headers will be ignored when following a redirect to a domain -// that is not a subdomain match or exact match of the initial domain. -// For example, a redirect from "foo.com" to either "foo.com" or "sub.foo.com" -// will forward the sensitive headers, but a redirect to "bar.com" will not. -// -// * when forwarding the "Cookie" header with a non-nil cookie Jar. -// Since each redirect may mutate the state of the cookie jar, -// a redirect may possibly alter a cookie set in the initial request. -// When forwarding the "Cookie" header, any mutated cookies will be omitted, -// with the expectation that the Jar will insert those mutated cookies -// with the updated values (assuming the origin matches). -// If Jar is nil, the initial cookies are forwarded without change. +// • when forwarding sensitive headers like "Authorization", +// "WWW-Authenticate", and "Cookie" to untrusted targets. +// These headers will be ignored when following a redirect to a domain +// that is not a subdomain match or exact match of the initial domain. +// For example, a redirect from "foo.com" to either "foo.com" or "sub.foo.com" +// will forward the sensitive headers, but a redirect to "bar.com" will not. +// +// • when forwarding the "Cookie" header with a non-nil cookie Jar. +// Since each redirect may mutate the state of the cookie jar, +// a redirect may possibly alter a cookie set in the initial request. +// When forwarding the "Cookie" header, any mutated cookies will be omitted, +// with the expectation that the Jar will insert those mutated cookies +// with the updated values (assuming the origin matches). +// If Jar is nil, the initial cookies are forwarded without change. // type Client struct { // Transport specifies the mechanism by which individual @@ -494,17 +494,21 @@ func (c *Client) Do(req *Request) (*Response, error) { } var ( - deadline = c.deadline() - reqs []*Request - resp *Response - copyHeaders = c.makeHeadersCopier(req) + deadline = c.deadline() + reqs []*Request + resp *Response + copyHeaders = c.makeHeadersCopier(req) + reqBodyClosed = false // have we closed the current req.Body? // Redirect behavior: redirectMethod string includeBody bool ) uerr := func(err error) error { - req.closeBody() + // the body may have been closed already by c.send() + if !reqBodyClosed { + req.closeBody() + } method := valueOrDefault(reqs[0].Method, "GET") var urlStr string if resp != nil && resp.Request != nil { @@ -524,10 +528,12 @@ func (c *Client) Do(req *Request) (*Response, error) { if len(reqs) > 0 { loc := resp.Header.Get("Location") if loc == "" { + resp.closeBody() return nil, uerr(fmt.Errorf("%d response missing Location header", resp.StatusCode)) } u, err := req.URL.Parse(loc) if err != nil { + resp.closeBody() return nil, uerr(fmt.Errorf("failed to parse Location header %q: %v", loc, err)) } ireq := reqs[0] @@ -542,6 +548,7 @@ func (c *Client) Do(req *Request) (*Response, error) { if includeBody && ireq.GetBody != nil { req.Body, err = ireq.GetBody() if err != nil { + resp.closeBody() return nil, uerr(err) } req.ContentLength = ireq.ContentLength @@ -593,6 +600,8 @@ func (c *Client) Do(req *Request) (*Response, error) { var err error var didTimeout func() bool if resp, didTimeout, err = c.send(req, deadline); err != nil { + // c.send() always closes req.Body + reqBodyClosed = true if !deadline.IsZero() && didTimeout() { err = &httpError{ err: err.Error() + " (Client.Timeout exceeded while awaiting headers)", diff --git a/libgo/go/net/http/client_test.go b/libgo/go/net/http/client_test.go index 4f674dd8d6c..b9a1c31e43a 100644 --- a/libgo/go/net/http/client_test.go +++ b/libgo/go/net/http/client_test.go @@ -10,7 +10,6 @@ import ( "bytes" "context" "crypto/tls" - "crypto/x509" "encoding/base64" "errors" "fmt" @@ -73,7 +72,7 @@ func TestClient(t *testing.T) { ts := httptest.NewServer(robotsTxtHandler) defer ts.Close() - c := &Client{Transport: &Transport{DisableKeepAlives: true}} + c := ts.Client() r, err := c.Get(ts.URL) var b []byte if err == nil { @@ -220,10 +219,7 @@ func TestClientRedirects(t *testing.T) { })) defer ts.Close() - tr := &Transport{} - defer tr.CloseIdleConnections() - - c := &Client{Transport: tr} + c := ts.Client() _, err := c.Get(ts.URL) if e, g := "Get /?n=10: stopped after 10 redirects", fmt.Sprintf("%v", err); e != g { t.Errorf("with default client Get, expected error %q, got %q", e, g) @@ -252,13 +248,10 @@ func TestClientRedirects(t *testing.T) { var checkErr error var lastVia []*Request var lastReq *Request - c = &Client{ - Transport: tr, - CheckRedirect: func(req *Request, via []*Request) error { - lastReq = req - lastVia = via - return checkErr - }, + c.CheckRedirect = func(req *Request, via []*Request) error { + lastReq = req + lastVia = via + return checkErr } res, err := c.Get(ts.URL) if err != nil { @@ -304,6 +297,7 @@ func TestClientRedirects(t *testing.T) { } } +// Tests that Client redirects' contexts are derived from the original request's context. func TestClientRedirectContext(t *testing.T) { setParallel(t) defer afterTest(t) @@ -312,19 +306,16 @@ func TestClientRedirectContext(t *testing.T) { })) defer ts.Close() - tr := &Transport{} - defer tr.CloseIdleConnections() - ctx, cancel := context.WithCancel(context.Background()) - c := &Client{ - Transport: tr, - CheckRedirect: func(req *Request, via []*Request) error { - cancel() - if len(via) > 2 { - return errors.New("too many redirects") - } + c := ts.Client() + c.CheckRedirect = func(req *Request, via []*Request) error { + cancel() + select { + case <-req.Context().Done(): return nil - }, + case <-time.After(5 * time.Second): + return errors.New("redirected request's context never expired after root request canceled") + } } req, _ := NewRequest("GET", ts.URL, nil) req = req.WithContext(ctx) @@ -458,11 +449,12 @@ func testRedirectsByMethod(t *testing.T, method string, table []redirectTest, wa })) defer ts.Close() + c := ts.Client() for _, tt := range table { content := tt.redirectBody req, _ := NewRequest(method, ts.URL+tt.suffix, strings.NewReader(content)) req.GetBody = func() (io.ReadCloser, error) { return ioutil.NopCloser(strings.NewReader(content)), nil } - res, err := DefaultClient.Do(req) + res, err := c.Do(req) if err != nil { t.Fatal(err) @@ -516,17 +508,12 @@ func TestClientRedirectUseResponse(t *testing.T) { })) defer ts.Close() - tr := &Transport{} - defer tr.CloseIdleConnections() - - c := &Client{ - Transport: tr, - CheckRedirect: func(req *Request, via []*Request) error { - if req.Response == nil { - t.Error("expected non-nil Request.Response") - } - return ErrUseLastResponse - }, + c := ts.Client() + c.CheckRedirect = func(req *Request, via []*Request) error { + if req.Response == nil { + t.Error("expected non-nil Request.Response") + } + return ErrUseLastResponse } res, err := c.Get(ts.URL) if err != nil { @@ -555,7 +542,8 @@ func TestClientRedirect308NoLocation(t *testing.T) { w.WriteHeader(308) })) defer ts.Close() - res, err := Get(ts.URL) + c := ts.Client() + res, err := c.Get(ts.URL) if err != nil { t.Fatal(err) } @@ -582,8 +570,9 @@ func TestClientRedirect308NoGetBody(t *testing.T) { if err != nil { t.Fatal(err) } + c := ts.Client() req.GetBody = nil // so it can't rewind. - res, err := DefaultClient.Do(req) + res, err := c.Do(req) if err != nil { t.Fatal(err) } @@ -673,12 +662,8 @@ func TestRedirectCookiesJar(t *testing.T) { var ts *httptest.Server ts = httptest.NewServer(echoCookiesRedirectHandler) defer ts.Close() - tr := &Transport{} - defer tr.CloseIdleConnections() - c := &Client{ - Transport: tr, - Jar: new(TestJar), - } + c := ts.Client() + c.Jar = new(TestJar) u, _ := url.Parse(ts.URL) c.Jar.SetCookies(u, []*Cookie{expectedCookies[0]}) resp, err := c.Get(ts.URL) @@ -722,13 +707,10 @@ func TestJarCalls(t *testing.T) { })) defer ts.Close() jar := new(RecordingJar) - c := &Client{ - Jar: jar, - Transport: &Transport{ - Dial: func(_ string, _ string) (net.Conn, error) { - return net.Dial("tcp", ts.Listener.Addr().String()) - }, - }, + c := ts.Client() + c.Jar = jar + c.Transport.(*Transport).Dial = func(_ string, _ string) (net.Conn, error) { + return net.Dial("tcp", ts.Listener.Addr().String()) } _, err := c.Get("http://firsthost.fake/") if err != nil { @@ -840,7 +822,8 @@ func TestClientWrites(t *testing.T) { } return c, err } - c := &Client{Transport: &Transport{Dial: dialer}} + c := ts.Client() + c.Transport.(*Transport).Dial = dialer _, err := c.Get(ts.URL) if err != nil { @@ -873,14 +856,11 @@ func TestClientInsecureTransport(t *testing.T) { // TODO(bradfitz): add tests for skipping hostname checks too? // would require a new cert for testing, and probably // redundant with these tests. + c := ts.Client() for _, insecure := range []bool{true, false} { - tr := &Transport{ - TLSClientConfig: &tls.Config{ - InsecureSkipVerify: insecure, - }, + c.Transport.(*Transport).TLSClientConfig = &tls.Config{ + InsecureSkipVerify: insecure, } - defer tr.CloseIdleConnections() - c := &Client{Transport: tr} res, err := c.Get(ts.URL) if (err == nil) != insecure { t.Errorf("insecure=%v: got unexpected err=%v", insecure, err) @@ -914,22 +894,6 @@ func TestClientErrorWithRequestURI(t *testing.T) { } } -func newTLSTransport(t *testing.T, ts *httptest.Server) *Transport { - certs := x509.NewCertPool() - for _, c := range ts.TLS.Certificates { - roots, err := x509.ParseCertificates(c.Certificate[len(c.Certificate)-1]) - if err != nil { - t.Fatalf("error parsing server's root cert: %v", err) - } - for _, root := range roots { - certs.AddCert(root) - } - } - return &Transport{ - TLSClientConfig: &tls.Config{RootCAs: certs}, - } -} - func TestClientWithCorrectTLSServerName(t *testing.T) { defer afterTest(t) @@ -941,9 +905,8 @@ func TestClientWithCorrectTLSServerName(t *testing.T) { })) defer ts.Close() - trans := newTLSTransport(t, ts) - trans.TLSClientConfig.ServerName = serverName - c := &Client{Transport: trans} + c := ts.Client() + c.Transport.(*Transport).TLSClientConfig.ServerName = serverName if _, err := c.Get(ts.URL); err != nil { t.Fatalf("expected successful TLS connection, got error: %v", err) } @@ -956,9 +919,8 @@ func TestClientWithIncorrectTLSServerName(t *testing.T) { errc := make(chanWriter, 10) // but only expecting 1 ts.Config.ErrorLog = log.New(errc, "", 0) - trans := newTLSTransport(t, ts) - trans.TLSClientConfig.ServerName = "badserver" - c := &Client{Transport: trans} + c := ts.Client() + c.Transport.(*Transport).TLSClientConfig.ServerName = "badserver" _, err := c.Get(ts.URL) if err == nil { t.Fatalf("expected an error") @@ -992,13 +954,12 @@ func TestTransportUsesTLSConfigServerName(t *testing.T) { })) defer ts.Close() - tr := newTLSTransport(t, ts) + c := ts.Client() + tr := c.Transport.(*Transport) tr.TLSClientConfig.ServerName = "example.com" // one of httptest's Server cert names tr.Dial = func(netw, addr string) (net.Conn, error) { return net.Dial(netw, ts.Listener.Addr().String()) } - defer tr.CloseIdleConnections() - c := &Client{Transport: tr} res, err := c.Get("https://some-other-host.tld/") if err != nil { t.Fatal(err) @@ -1013,13 +974,12 @@ func TestResponseSetsTLSConnectionState(t *testing.T) { })) defer ts.Close() - tr := newTLSTransport(t, ts) + c := ts.Client() + tr := c.Transport.(*Transport) tr.TLSClientConfig.CipherSuites = []uint16{tls.TLS_RSA_WITH_3DES_EDE_CBC_SHA} tr.Dial = func(netw, addr string) (net.Conn, error) { return net.Dial(netw, ts.Listener.Addr().String()) } - defer tr.CloseIdleConnections() - c := &Client{Transport: tr} res, err := c.Get("https://example.com/") if err != nil { t.Fatal(err) @@ -1114,14 +1074,12 @@ func TestEmptyPasswordAuth(t *testing.T) { } })) defer ts.Close() - tr := &Transport{} - defer tr.CloseIdleConnections() - c := &Client{Transport: tr} req, err := NewRequest("GET", ts.URL, nil) if err != nil { t.Fatal(err) } req.URL.User = url.User(gopher) + c := ts.Client() resp, err := c.Do(req) if err != nil { t.Fatal(err) @@ -1498,21 +1456,17 @@ func TestClientCopyHeadersOnRedirect(t *testing.T) { defer ts2.Close() ts2URL = ts2.URL - tr := &Transport{} - defer tr.CloseIdleConnections() - c := &Client{ - Transport: tr, - CheckRedirect: func(r *Request, via []*Request) error { - want := Header{ - "User-Agent": []string{ua}, - "X-Foo": []string{xfoo}, - "Referer": []string{ts2URL}, - } - if !reflect.DeepEqual(r.Header, want) { - t.Errorf("CheckRedirect Request.Header = %#v; want %#v", r.Header, want) - } - return nil - }, + c := ts1.Client() + c.CheckRedirect = func(r *Request, via []*Request) error { + want := Header{ + "User-Agent": []string{ua}, + "X-Foo": []string{xfoo}, + "Referer": []string{ts2URL}, + } + if !reflect.DeepEqual(r.Header, want) { + t.Errorf("CheckRedirect Request.Header = %#v; want %#v", r.Header, want) + } + return nil } req, _ := NewRequest("GET", ts2.URL, nil) @@ -1601,13 +1555,9 @@ func TestClientAltersCookiesOnRedirect(t *testing.T) { })) defer ts.Close() - tr := &Transport{} - defer tr.CloseIdleConnections() jar, _ := cookiejar.New(nil) - c := &Client{ - Transport: tr, - Jar: jar, - } + c := ts.Client() + c.Jar = jar u, _ := url.Parse(ts.URL) req, _ := NewRequest("GET", ts.URL, nil) @@ -1725,9 +1675,7 @@ func TestClientRedirectTypes(t *testing.T) { })) defer ts.Close() - tr := &Transport{} - defer tr.CloseIdleConnections() - + c := ts.Client() for i, tt := range tests { handlerc <- func(w ResponseWriter, r *Request) { w.Header().Set("Location", ts.URL) @@ -1740,7 +1688,6 @@ func TestClientRedirectTypes(t *testing.T) { continue } - c := &Client{Transport: tr} c.CheckRedirect = func(req *Request, via []*Request) error { if got, want := req.Method, tt.wantMethod; got != want { return fmt.Errorf("#%d: got next method %q; want %q", i, got, want) @@ -1780,8 +1727,8 @@ func (b issue18239Body) Close() error { return nil } -// Issue 18239: make sure the Transport doesn't retry requests with bodies. -// (Especially if Request.GetBody is not defined.) +// Issue 18239: make sure the Transport doesn't retry requests with bodies +// if Request.GetBody is not defined. func TestTransportBodyReadError(t *testing.T) { setParallel(t) defer afterTest(t) @@ -1794,9 +1741,8 @@ func TestTransportBodyReadError(t *testing.T) { w.Header().Set("X-Body-Read", fmt.Sprintf("%v, %v", n, err)) })) defer ts.Close() - tr := &Transport{} - defer tr.CloseIdleConnections() - c := &Client{Transport: tr} + c := ts.Client() + tr := c.Transport.(*Transport) // Do one initial successful request to create an idle TCP connection // for the subsequent request to reuse. (The Transport only retries @@ -1816,6 +1762,7 @@ func TestTransportBodyReadError(t *testing.T) { if err != nil { t.Fatal(err) } + req = req.WithT(t) _, err = tr.RoundTrip(req) if err != someErr { t.Errorf("Got error: %v; want Request.Body read error: %v", err, someErr) diff --git a/libgo/go/net/http/clientserver_test.go b/libgo/go/net/http/clientserver_test.go index 580115ca9c0..20feaa70ff6 100644 --- a/libgo/go/net/http/clientserver_test.go +++ b/libgo/go/net/http/clientserver_test.go @@ -1385,3 +1385,30 @@ func testServerUndeclaredTrailers(t *testing.T, h2 bool) { t.Errorf("Trailer = %#v; want %#v", res.Trailer, want) } } + +func TestBadResponseAfterReadingBody(t *testing.T) { + defer afterTest(t) + cst := newClientServerTest(t, false, HandlerFunc(func(w ResponseWriter, r *Request) { + _, err := io.Copy(ioutil.Discard, r.Body) + if err != nil { + t.Fatal(err) + } + c, _, err := w.(Hijacker).Hijack() + if err != nil { + t.Fatal(err) + } + defer c.Close() + fmt.Fprintln(c, "some bogus crap") + })) + defer cst.close() + + closes := 0 + res, err := cst.c.Post(cst.ts.URL, "text/plain", countCloseReader{&closes, strings.NewReader("hello")}) + if err == nil { + res.Body.Close() + t.Fatal("expected an error to be returned from Post") + } + if closes != 1 { + t.Errorf("closes = %d; want 1", closes) + } +} diff --git a/libgo/go/net/http/cookie.go b/libgo/go/net/http/cookie.go index 5a67476cd42..cf522488c15 100644 --- a/libgo/go/net/http/cookie.go +++ b/libgo/go/net/http/cookie.go @@ -328,7 +328,7 @@ func sanitizeCookieValue(v string) string { if len(v) == 0 { return v } - if v[0] == ' ' || v[0] == ',' || v[len(v)-1] == ' ' || v[len(v)-1] == ',' { + if strings.IndexByte(v, ' ') >= 0 || strings.IndexByte(v, ',') >= 0 { return `"` + v + `"` } return v diff --git a/libgo/go/net/http/cookie_test.go b/libgo/go/net/http/cookie_test.go index b3e54f8db32..9d199a3752e 100644 --- a/libgo/go/net/http/cookie_test.go +++ b/libgo/go/net/http/cookie_test.go @@ -69,7 +69,7 @@ var writeSetCookiesTests = []struct { // are disallowed by RFC 6265 but are common in the wild. { &Cookie{Name: "special-1", Value: "a z"}, - `special-1=a z`, + `special-1="a z"`, }, { &Cookie{Name: "special-2", Value: " z"}, @@ -85,7 +85,7 @@ var writeSetCookiesTests = []struct { }, { &Cookie{Name: "special-5", Value: "a,z"}, - `special-5=a,z`, + `special-5="a,z"`, }, { &Cookie{Name: "special-6", Value: ",z"}, @@ -398,9 +398,12 @@ func TestCookieSanitizeValue(t *testing.T) { {"foo\"bar", "foobar"}, {"\x00\x7e\x7f\x80", "\x7e"}, {`"withquotes"`, "withquotes"}, - {"a z", "a z"}, + {"a z", `"a z"`}, {" z", `" z"`}, {"a ", `"a "`}, + {"a,z", `"a,z"`}, + {",z", `",z"`}, + {"a,", `"a,"`}, } for _, tt := range tests { if got := sanitizeCookieValue(tt.in); got != tt.want { diff --git a/libgo/go/net/http/cookiejar/jar.go b/libgo/go/net/http/cookiejar/jar.go index f89abbcd186..ef8c35bf0a1 100644 --- a/libgo/go/net/http/cookiejar/jar.go +++ b/libgo/go/net/http/cookiejar/jar.go @@ -331,7 +331,7 @@ func jarKey(host string, psl PublicSuffixList) string { var i int if psl == nil { i = strings.LastIndex(host, ".") - if i == -1 { + if i <= 0 { return host } } else { @@ -345,6 +345,9 @@ func jarKey(host string, psl PublicSuffixList) string { // Storing cookies under host is a safe stopgap. return host } + // Only len(suffix) is used to determine the jar key from + // here on, so it is okay if psl.PublicSuffix("www.buggy.psl") + // returns "com" as the jar key is generated from host. } prevDot := strings.LastIndex(host[:i-1], ".") return host[prevDot+1:] diff --git a/libgo/go/net/http/cookiejar/jar_test.go b/libgo/go/net/http/cookiejar/jar_test.go index 3aa601586e3..47fb1abdaaf 100644 --- a/libgo/go/net/http/cookiejar/jar_test.go +++ b/libgo/go/net/http/cookiejar/jar_test.go @@ -19,6 +19,9 @@ var tNow = time.Date(2013, 1, 1, 12, 0, 0, 0, time.UTC) // testPSL implements PublicSuffixList with just two rules: "co.uk" // and the default rule "*". +// The implementation has two intentional bugs: +// PublicSuffix("www.buggy.psl") == "xy" +// PublicSuffix("www2.buggy.psl") == "com" type testPSL struct{} func (testPSL) String() string { @@ -28,6 +31,12 @@ func (testPSL) PublicSuffix(d string) string { if d == "co.uk" || strings.HasSuffix(d, ".co.uk") { return "co.uk" } + if d == "www.buggy.psl" { + return "xy" + } + if d == "www2.buggy.psl" { + return "com" + } return d[strings.LastIndex(d, ".")+1:] } @@ -125,6 +134,17 @@ var canonicalHostTests = map[string]string{ "[2001:4860:0:::68]:8080": "2001:4860:0:::68", "www.bücher.de": "www.xn--bcher-kva.de", "www.example.com.": "www.example.com", + // TODO: Fix canonicalHost so that all of the following malformed + // domain names trigger an error. (This list is not exhaustive, e.g. + // malformed internationalized domain names are missing.) + ".": "", + "..": ".", + "...": "..", + ".net": ".net", + ".net.": ".net", + "a..": "a.", + "b.a..": "b.a.", + "weird.stuff...": "weird.stuff..", "[bad.unmatched.bracket:": "error", } @@ -133,7 +153,7 @@ func TestCanonicalHost(t *testing.T) { got, err := canonicalHost(h) if want == "error" { if err == nil { - t.Errorf("%q: got nil error, want non-nil", h) + t.Errorf("%q: got %q and nil error, want non-nil", h, got) } continue } @@ -176,6 +196,17 @@ var jarKeyTests = map[string]string{ "co.uk": "co.uk", "uk": "uk", "192.168.0.5": "192.168.0.5", + "www.buggy.psl": "www.buggy.psl", + "www2.buggy.psl": "buggy.psl", + // The following are actual outputs of canonicalHost for + // malformed inputs to canonicalHost (see above). + "": "", + ".": ".", + "..": ".", + ".net": ".net", + "a.": "a.", + "b.a.": "a.", + "weird.stuff..": ".", } func TestJarKey(t *testing.T) { @@ -197,6 +228,15 @@ var jarKeyNilPSLTests = map[string]string{ "co.uk": "co.uk", "uk": "uk", "192.168.0.5": "192.168.0.5", + // The following are actual outputs of canonicalHost for + // malformed inputs to canonicalHost. + "": "", + ".": ".", + "..": "..", + ".net": ".net", + "a.": "a.", + "b.a.": "a.", + "weird.stuff..": "stuff..", } func TestJarKeyNilPSL(t *testing.T) { @@ -1265,3 +1305,18 @@ func TestDomainHandling(t *testing.T) { test.run(t, jar) } } + +func TestIssue19384(t *testing.T) { + cookies := []*http.Cookie{{Name: "name", Value: "value"}} + for _, host := range []string{"", ".", "..", "..."} { + jar, _ := New(nil) + u := &url.URL{Scheme: "http", Host: host, Path: "/"} + if got := jar.Cookies(u); len(got) != 0 { + t.Errorf("host %q, got %v", host, got) + } + jar.SetCookies(u, cookies) + if got := jar.Cookies(u); len(got) != 1 || got[0].Value != "value" { + t.Errorf("host %q, got %v", host, got) + } + } +} diff --git a/libgo/go/net/http/export_test.go b/libgo/go/net/http/export_test.go index b61f58b2db4..2ef145e5342 100644 --- a/libgo/go/net/http/export_test.go +++ b/libgo/go/net/http/export_test.go @@ -8,24 +8,29 @@ package http import ( + "context" "net" "sort" "sync" + "testing" "time" ) var ( - DefaultUserAgent = defaultUserAgent - NewLoggingConn = newLoggingConn - ExportAppendTime = appendTime - ExportRefererForURL = refererForURL - ExportServerNewConn = (*Server).newConn - ExportCloseWriteAndWait = (*conn).closeWriteAndWait - ExportErrRequestCanceled = errRequestCanceled - ExportErrRequestCanceledConn = errRequestCanceledConn - ExportServeFile = serveFile - ExportScanETag = scanETag - ExportHttp2ConfigureServer = http2ConfigureServer + DefaultUserAgent = defaultUserAgent + NewLoggingConn = newLoggingConn + ExportAppendTime = appendTime + ExportRefererForURL = refererForURL + ExportServerNewConn = (*Server).newConn + ExportCloseWriteAndWait = (*conn).closeWriteAndWait + ExportErrRequestCanceled = errRequestCanceled + ExportErrRequestCanceledConn = errRequestCanceledConn + ExportErrServerClosedIdle = errServerClosedIdle + ExportServeFile = serveFile + ExportScanETag = scanETag + ExportHttp2ConfigureServer = http2ConfigureServer + Export_shouldCopyHeaderOnRedirect = shouldCopyHeaderOnRedirect + Export_writeStatusLine = writeStatusLine ) func init() { @@ -186,8 +191,6 @@ func ExportHttp2ConfigureTransport(t *Transport) error { return nil } -var Export_shouldCopyHeaderOnRedirect = shouldCopyHeaderOnRedirect - func (s *Server) ExportAllConnsIdle() bool { s.mu.Lock() defer s.mu.Unlock() @@ -199,3 +202,7 @@ func (s *Server) ExportAllConnsIdle() bool { } return true } + +func (r *Request) WithT(t *testing.T) *Request { + return r.WithContext(context.WithValue(r.Context(), tLogKey{}, t.Logf)) +} diff --git a/libgo/go/net/http/fcgi/child.go b/libgo/go/net/http/fcgi/child.go index 88704245db8..30a6b2ce2df 100644 --- a/libgo/go/net/http/fcgi/child.go +++ b/libgo/go/net/http/fcgi/child.go @@ -7,6 +7,7 @@ package fcgi // This file implements FastCGI from the perspective of a child process. import ( + "context" "errors" "fmt" "io" @@ -31,6 +32,10 @@ type request struct { keepConn bool } +// envVarsContextKey uniquely identifies a mapping of CGI +// environment variables to their values in a request context +type envVarsContextKey struct{} + func newRequest(reqId uint16, flags uint8) *request { r := &request{ reqId: reqId, @@ -259,6 +264,18 @@ func (c *child) handleRecord(rec *record) error { } } +// filterOutUsedEnvVars returns a new map of env vars without the +// variables in the given envVars map that are read for creating each http.Request +func filterOutUsedEnvVars(envVars map[string]string) map[string]string { + withoutUsedEnvVars := make(map[string]string) + for k, v := range envVars { + if addFastCGIEnvToContext(k) { + withoutUsedEnvVars[k] = v + } + } + return withoutUsedEnvVars +} + func (c *child) serveRequest(req *request, body io.ReadCloser) { r := newResponse(c, req) httpReq, err := cgi.RequestFromMap(req.params) @@ -268,6 +285,9 @@ func (c *child) serveRequest(req *request, body io.ReadCloser) { c.conn.writeRecord(typeStderr, req.reqId, []byte(err.Error())) } else { httpReq.Body = body + withoutUsedEnvVars := filterOutUsedEnvVars(req.params) + envVarCtx := context.WithValue(httpReq.Context(), envVarsContextKey{}, withoutUsedEnvVars) + httpReq = httpReq.WithContext(envVarCtx) c.handler.ServeHTTP(r, httpReq) } r.Close() @@ -329,3 +349,39 @@ func Serve(l net.Listener, handler http.Handler) error { go c.serve() } } + +// ProcessEnv returns FastCGI environment variables associated with the request r +// for which no effort was made to be included in the request itself - the data +// is hidden in the request's context. As an example, if REMOTE_USER is set for a +// request, it will not be found anywhere in r, but it will be included in +// ProcessEnv's response (via r's context). +func ProcessEnv(r *http.Request) map[string]string { + env, _ := r.Context().Value(envVarsContextKey{}).(map[string]string) + return env +} + +// addFastCGIEnvToContext reports whether to include the FastCGI environment variable s +// in the http.Request.Context, accessible via ProcessEnv. +func addFastCGIEnvToContext(s string) bool { + // Exclude things supported by net/http natively: + switch s { + case "CONTENT_LENGTH", "CONTENT_TYPE", "HTTPS", + "PATH_INFO", "QUERY_STRING", "REMOTE_ADDR", + "REMOTE_HOST", "REMOTE_PORT", "REQUEST_METHOD", + "REQUEST_URI", "SCRIPT_NAME", "SERVER_PROTOCOL": + return false + } + if strings.HasPrefix(s, "HTTP_") { + return false + } + // Explicitly include FastCGI-specific things. + // This list is redundant with the default "return true" below. + // Consider this documentation of the sorts of things we expect + // to maybe see. + switch s { + case "REMOTE_USER": + return true + } + // Unknown, so include it to be safe. + return true +} diff --git a/libgo/go/net/http/fcgi/fcgi.go b/libgo/go/net/http/fcgi/fcgi.go index 5057d700981..8f3449a991a 100644 --- a/libgo/go/net/http/fcgi/fcgi.go +++ b/libgo/go/net/http/fcgi/fcgi.go @@ -24,7 +24,7 @@ import ( ) // recType is a record type, as defined by -// http://www.fastcgi.com/devkit/doc/fcgi-spec.html#S8 +// https://web.archive.org/web/20150420080736/http://www.fastcgi.com/drupal/node/6?q=node/22#S8 type recType uint8 const ( diff --git a/libgo/go/net/http/fcgi/fcgi_test.go b/libgo/go/net/http/fcgi/fcgi_test.go index b6013bfdd51..e9d2b34023c 100644 --- a/libgo/go/net/http/fcgi/fcgi_test.go +++ b/libgo/go/net/http/fcgi/fcgi_test.go @@ -278,3 +278,69 @@ func TestMalformedParams(t *testing.T) { c := newChild(rw, http.DefaultServeMux) c.serve() } + +// a series of FastCGI records that start and end a request +var streamFullRequestStdin = bytes.Join([][]byte{ + // set up request + makeRecord(typeBeginRequest, 1, + []byte{0, byte(roleResponder), 0, 0, 0, 0, 0, 0}), + // add required parameters + makeRecord(typeParams, 1, nameValuePair11("REQUEST_METHOD", "GET")), + makeRecord(typeParams, 1, nameValuePair11("SERVER_PROTOCOL", "HTTP/1.1")), + // set optional parameters + makeRecord(typeParams, 1, nameValuePair11("REMOTE_USER", "jane.doe")), + makeRecord(typeParams, 1, nameValuePair11("QUERY_STRING", "/foo/bar")), + makeRecord(typeParams, 1, nil), + // begin sending body of request + makeRecord(typeStdin, 1, []byte("0123456789abcdef")), + // end request + makeRecord(typeEndRequest, 1, nil), +}, + nil) + +var envVarTests = []struct { + input []byte + envVar string + expectedVal string + expectedFilteredOut bool +}{ + { + streamFullRequestStdin, + "REMOTE_USER", + "jane.doe", + false, + }, + { + streamFullRequestStdin, + "QUERY_STRING", + "", + true, + }, +} + +// Test that environment variables set for a request can be +// read by a handler. Ensures that variables not set will not +// be exposed to a handler. +func TestChildServeReadsEnvVars(t *testing.T) { + for _, tt := range envVarTests { + input := make([]byte, len(tt.input)) + copy(input, tt.input) + rc := nopWriteCloser{bytes.NewBuffer(input)} + done := make(chan bool) + c := newChild(rc, http.HandlerFunc(func( + w http.ResponseWriter, + r *http.Request, + ) { + env := ProcessEnv(r) + if _, ok := env[tt.envVar]; ok && tt.expectedFilteredOut { + t.Errorf("Expected environment variable %s to not be set, but set to %s", + tt.envVar, env[tt.envVar]) + } else if env[tt.envVar] != tt.expectedVal { + t.Errorf("Expected %s, got %s", tt.expectedVal, env[tt.envVar]) + } + done <- true + })) + go c.serve() + <-done + } +} diff --git a/libgo/go/net/http/filetransport_test.go b/libgo/go/net/http/filetransport_test.go index 6f1a537e2ed..2a2f32c7694 100644 --- a/libgo/go/net/http/filetransport_test.go +++ b/libgo/go/net/http/filetransport_test.go @@ -49,6 +49,7 @@ func TestFileTransport(t *testing.T) { t.Fatalf("for %s, nil Body", urlstr) } slurp, err := ioutil.ReadAll(res.Body) + res.Body.Close() check("ReadAll "+urlstr, err) if string(slurp) != "Bar" { t.Errorf("for %s, got content %q, want %q", urlstr, string(slurp), "Bar") diff --git a/libgo/go/net/http/fs.go b/libgo/go/net/http/fs.go index bf63bb5441f..5819334b5f4 100644 --- a/libgo/go/net/http/fs.go +++ b/libgo/go/net/http/fs.go @@ -30,21 +30,51 @@ import ( // value is a filename on the native file system, not a URL, so it is separated // by filepath.Separator, which isn't necessarily '/'. // +// Note that Dir will allow access to files and directories starting with a +// period, which could expose sensitive directories like a .git directory or +// sensitive files like .htpasswd. To exclude files with a leading period, +// remove the files/directories from the server or create a custom FileSystem +// implementation. +// // An empty Dir is treated as ".". type Dir string +// mapDirOpenError maps the provided non-nil error from opening name +// to a possibly better non-nil error. In particular, it turns OS-specific errors +// about opening files in non-directories into os.ErrNotExist. See Issue 18984. +func mapDirOpenError(originalErr error, name string) error { + if os.IsNotExist(originalErr) || os.IsPermission(originalErr) { + return originalErr + } + + parts := strings.Split(name, string(filepath.Separator)) + for i := range parts { + if parts[i] == "" { + continue + } + fi, err := os.Stat(strings.Join(parts[:i+1], string(filepath.Separator))) + if err != nil { + return originalErr + } + if !fi.IsDir() { + return os.ErrNotExist + } + } + return originalErr +} + func (d Dir) Open(name string) (File, error) { - if filepath.Separator != '/' && strings.ContainsRune(name, filepath.Separator) || - strings.Contains(name, "\x00") { + if filepath.Separator != '/' && strings.ContainsRune(name, filepath.Separator) { return nil, errors.New("http: invalid character in file path") } dir := string(d) if dir == "" { dir = "." } - f, err := os.Open(filepath.Join(dir, filepath.FromSlash(path.Clean("/"+name)))) + fullName := filepath.Join(dir, filepath.FromSlash(path.Clean("/"+name))) + f, err := os.Open(fullName) if err != nil { - return nil, err + return nil, mapDirOpenError(err, fullName) } return f, nil } @@ -291,7 +321,7 @@ func scanETag(s string) (etag string, remain string) { case c == '"': return string(s[:i+1]), s[i+1:] default: - break + return "", "" } } return "", "" @@ -349,7 +379,7 @@ func checkIfMatch(w ResponseWriter, r *Request) condResult { return condFalse } -func checkIfUnmodifiedSince(w ResponseWriter, r *Request, modtime time.Time) condResult { +func checkIfUnmodifiedSince(r *Request, modtime time.Time) condResult { ius := r.Header.Get("If-Unmodified-Since") if ius == "" || isZeroTime(modtime) { return condNone @@ -394,7 +424,7 @@ func checkIfNoneMatch(w ResponseWriter, r *Request) condResult { return condTrue } -func checkIfModifiedSince(w ResponseWriter, r *Request, modtime time.Time) condResult { +func checkIfModifiedSince(r *Request, modtime time.Time) condResult { if r.Method != "GET" && r.Method != "HEAD" { return condNone } @@ -479,7 +509,7 @@ func checkPreconditions(w ResponseWriter, r *Request, modtime time.Time) (done b // This function carefully follows RFC 7232 section 6. ch := checkIfMatch(w, r) if ch == condNone { - ch = checkIfUnmodifiedSince(w, r, modtime) + ch = checkIfUnmodifiedSince(r, modtime) } if ch == condFalse { w.WriteHeader(StatusPreconditionFailed) @@ -495,7 +525,7 @@ func checkPreconditions(w ResponseWriter, r *Request, modtime time.Time) (done b return true, "" } case condNone: - if checkIfModifiedSince(w, r, modtime) == condFalse { + if checkIfModifiedSince(r, modtime) == condFalse { writeNotModified(w) return true, "" } @@ -580,7 +610,7 @@ func serveFile(w ResponseWriter, r *Request, fs FileSystem, name string, redirec // Still a directory? (we didn't find an index.html file) if d.IsDir() { - if checkIfModifiedSince(w, r, d.ModTime()) == condFalse { + if checkIfModifiedSince(r, d.ModTime()) == condFalse { writeNotModified(w) return } diff --git a/libgo/go/net/http/fs_test.go b/libgo/go/net/http/fs_test.go index bba56821156..f6eab0fcc31 100644 --- a/libgo/go/net/http/fs_test.go +++ b/libgo/go/net/http/fs_test.go @@ -74,6 +74,7 @@ func TestServeFile(t *testing.T) { ServeFile(w, r, "testdata/file") })) defer ts.Close() + c := ts.Client() var err error @@ -91,7 +92,7 @@ func TestServeFile(t *testing.T) { req.Method = "GET" // straight GET - _, body := getBody(t, "straight get", req) + _, body := getBody(t, "straight get", req, c) if !bytes.Equal(body, file) { t.Fatalf("body mismatch: got %q, want %q", body, file) } @@ -102,7 +103,7 @@ Cases: if rt.r != "" { req.Header.Set("Range", rt.r) } - resp, body := getBody(t, fmt.Sprintf("range test %q", rt.r), req) + resp, body := getBody(t, fmt.Sprintf("range test %q", rt.r), req, c) if resp.StatusCode != rt.code { t.Errorf("range=%q: StatusCode=%d, want %d", rt.r, resp.StatusCode, rt.code) } @@ -704,7 +705,8 @@ func TestDirectoryIfNotModified(t *testing.T) { req, _ := NewRequest("GET", ts.URL, nil) req.Header.Set("If-Modified-Since", lastMod) - res, err = DefaultClient.Do(req) + c := ts.Client() + res, err = c.Do(req) if err != nil { t.Fatal(err) } @@ -716,7 +718,7 @@ func TestDirectoryIfNotModified(t *testing.T) { // Advance the index.html file's modtime, but not the directory's. indexFile.modtime = indexFile.modtime.Add(1 * time.Hour) - res, err = DefaultClient.Do(req) + res, err = c.Do(req) if err != nil { t.Fatal(err) } @@ -995,7 +997,9 @@ func TestServeContent(t *testing.T) { for k, v := range tt.reqHeader { req.Header.Set(k, v) } - res, err := DefaultClient.Do(req) + + c := ts.Client() + res, err := c.Do(req) if err != nil { t.Fatal(err) } @@ -1050,8 +1054,9 @@ func TestServeContentErrorMessages(t *testing.T) { } ts := httptest.NewServer(FileServer(fs)) defer ts.Close() + c := ts.Client() for _, code := range []int{403, 404, 500} { - res, err := DefaultClient.Get(fmt.Sprintf("%s/%d", ts.URL, code)) + res, err := c.Get(fmt.Sprintf("%s/%d", ts.URL, code)) if err != nil { t.Errorf("Error fetching /%d: %v", code, err) continue @@ -1090,8 +1095,11 @@ func TestLinuxSendfile(t *testing.T) { // strace on the above platforms doesn't support sendfile64 // and will error out if we specify that with `-e trace='. syscalls = "sendfile" - case "mips64": - t.Skip("TODO: update this test to be robust against various versions of strace on mips64. See golang.org/issue/33430") + } + + // Attempt to run strace, and skip on failure - this test requires SYS_PTRACE. + if err := exec.Command("strace", "-f", "-q", "-e", "trace="+syscalls, os.Args[0], "-test.run=^$").Run(); err != nil { + t.Skipf("skipping; failed to run strace: %v", err) } var buf bytes.Buffer @@ -1125,8 +1133,8 @@ func TestLinuxSendfile(t *testing.T) { } } -func getBody(t *testing.T, testName string, req Request) (*Response, []byte) { - r, err := DefaultClient.Do(&req) +func getBody(t *testing.T, testName string, req Request, client *Client) (*Response, []byte) { + r, err := client.Do(&req) if err != nil { t.Fatalf("%s: for URL %q, send error: %v", testName, req.URL.String(), err) } @@ -1161,6 +1169,50 @@ func TestLinuxSendfileChild(*testing.T) { } } +// Issue 18984: tests that requests for paths beyond files return not-found errors +func TestFileServerNotDirError(t *testing.T) { + defer afterTest(t) + ts := httptest.NewServer(FileServer(Dir("testdata"))) + defer ts.Close() + + res, err := Get(ts.URL + "/index.html/not-a-file") + if err != nil { + t.Fatal(err) + } + res.Body.Close() + if res.StatusCode != 404 { + t.Errorf("StatusCode = %v; want 404", res.StatusCode) + } + + test := func(name string, dir Dir) { + t.Run(name, func(t *testing.T) { + _, err = dir.Open("/index.html/not-a-file") + if err == nil { + t.Fatal("err == nil; want != nil") + } + if !os.IsNotExist(err) { + t.Errorf("err = %v; os.IsNotExist(err) = %v; want true", err, os.IsNotExist(err)) + } + + _, err = dir.Open("/index.html/not-a-dir/not-a-file") + if err == nil { + t.Fatal("err == nil; want != nil") + } + if !os.IsNotExist(err) { + t.Errorf("err = %v; os.IsNotExist(err) = %v; want true", err, os.IsNotExist(err)) + } + }) + } + + absPath, err := filepath.Abs("testdata") + if err != nil { + t.Fatal("get abs path:", err) + } + + test("RelativePath", Dir("testdata")) + test("AbsolutePath", Dir(absPath)) +} + func TestFileServerCleanPath(t *testing.T) { tests := []struct { path string @@ -1210,10 +1262,10 @@ func Test_scanETag(t *testing.T) { {`"etag-2"`, `"etag-2"`, ""}, {`"etag-1", "etag-2"`, `"etag-1"`, `, "etag-2"`}, {"", "", ""}, - {"", "", ""}, {"W/", "", ""}, {`W/"truc`, "", ""}, {`w/"case-sensitive"`, "", ""}, + {`"spaced etag"`, "", ""}, } for _, test := range tests { etag, remain := ExportScanETag(test.in) diff --git a/libgo/go/net/http/h2_bundle.go b/libgo/go/net/http/h2_bundle.go index 6fbbcd0fc76..373f55098a3 100644 --- a/libgo/go/net/http/h2_bundle.go +++ b/libgo/go/net/http/h2_bundle.go @@ -48,6 +48,642 @@ import ( "golang_org/x/net/lex/httplex" ) +// A list of the possible cipher suite ids. Taken from +// http://www.iana.org/assignments/tls-parameters/tls-parameters.txt + +const ( + http2cipher_TLS_NULL_WITH_NULL_NULL uint16 = 0x0000 + http2cipher_TLS_RSA_WITH_NULL_MD5 uint16 = 0x0001 + http2cipher_TLS_RSA_WITH_NULL_SHA uint16 = 0x0002 + http2cipher_TLS_RSA_EXPORT_WITH_RC4_40_MD5 uint16 = 0x0003 + http2cipher_TLS_RSA_WITH_RC4_128_MD5 uint16 = 0x0004 + http2cipher_TLS_RSA_WITH_RC4_128_SHA uint16 = 0x0005 + http2cipher_TLS_RSA_EXPORT_WITH_RC2_CBC_40_MD5 uint16 = 0x0006 + http2cipher_TLS_RSA_WITH_IDEA_CBC_SHA uint16 = 0x0007 + http2cipher_TLS_RSA_EXPORT_WITH_DES40_CBC_SHA uint16 = 0x0008 + http2cipher_TLS_RSA_WITH_DES_CBC_SHA uint16 = 0x0009 + http2cipher_TLS_RSA_WITH_3DES_EDE_CBC_SHA uint16 = 0x000A + http2cipher_TLS_DH_DSS_EXPORT_WITH_DES40_CBC_SHA uint16 = 0x000B + http2cipher_TLS_DH_DSS_WITH_DES_CBC_SHA uint16 = 0x000C + http2cipher_TLS_DH_DSS_WITH_3DES_EDE_CBC_SHA uint16 = 0x000D + http2cipher_TLS_DH_RSA_EXPORT_WITH_DES40_CBC_SHA uint16 = 0x000E + http2cipher_TLS_DH_RSA_WITH_DES_CBC_SHA uint16 = 0x000F + http2cipher_TLS_DH_RSA_WITH_3DES_EDE_CBC_SHA uint16 = 0x0010 + http2cipher_TLS_DHE_DSS_EXPORT_WITH_DES40_CBC_SHA uint16 = 0x0011 + http2cipher_TLS_DHE_DSS_WITH_DES_CBC_SHA uint16 = 0x0012 + http2cipher_TLS_DHE_DSS_WITH_3DES_EDE_CBC_SHA uint16 = 0x0013 + http2cipher_TLS_DHE_RSA_EXPORT_WITH_DES40_CBC_SHA uint16 = 0x0014 + http2cipher_TLS_DHE_RSA_WITH_DES_CBC_SHA uint16 = 0x0015 + http2cipher_TLS_DHE_RSA_WITH_3DES_EDE_CBC_SHA uint16 = 0x0016 + http2cipher_TLS_DH_anon_EXPORT_WITH_RC4_40_MD5 uint16 = 0x0017 + http2cipher_TLS_DH_anon_WITH_RC4_128_MD5 uint16 = 0x0018 + http2cipher_TLS_DH_anon_EXPORT_WITH_DES40_CBC_SHA uint16 = 0x0019 + http2cipher_TLS_DH_anon_WITH_DES_CBC_SHA uint16 = 0x001A + http2cipher_TLS_DH_anon_WITH_3DES_EDE_CBC_SHA uint16 = 0x001B + // Reserved uint16 = 0x001C-1D + http2cipher_TLS_KRB5_WITH_DES_CBC_SHA uint16 = 0x001E + http2cipher_TLS_KRB5_WITH_3DES_EDE_CBC_SHA uint16 = 0x001F + http2cipher_TLS_KRB5_WITH_RC4_128_SHA uint16 = 0x0020 + http2cipher_TLS_KRB5_WITH_IDEA_CBC_SHA uint16 = 0x0021 + http2cipher_TLS_KRB5_WITH_DES_CBC_MD5 uint16 = 0x0022 + http2cipher_TLS_KRB5_WITH_3DES_EDE_CBC_MD5 uint16 = 0x0023 + http2cipher_TLS_KRB5_WITH_RC4_128_MD5 uint16 = 0x0024 + http2cipher_TLS_KRB5_WITH_IDEA_CBC_MD5 uint16 = 0x0025 + http2cipher_TLS_KRB5_EXPORT_WITH_DES_CBC_40_SHA uint16 = 0x0026 + http2cipher_TLS_KRB5_EXPORT_WITH_RC2_CBC_40_SHA uint16 = 0x0027 + http2cipher_TLS_KRB5_EXPORT_WITH_RC4_40_SHA uint16 = 0x0028 + http2cipher_TLS_KRB5_EXPORT_WITH_DES_CBC_40_MD5 uint16 = 0x0029 + http2cipher_TLS_KRB5_EXPORT_WITH_RC2_CBC_40_MD5 uint16 = 0x002A + http2cipher_TLS_KRB5_EXPORT_WITH_RC4_40_MD5 uint16 = 0x002B + http2cipher_TLS_PSK_WITH_NULL_SHA uint16 = 0x002C + http2cipher_TLS_DHE_PSK_WITH_NULL_SHA uint16 = 0x002D + http2cipher_TLS_RSA_PSK_WITH_NULL_SHA uint16 = 0x002E + http2cipher_TLS_RSA_WITH_AES_128_CBC_SHA uint16 = 0x002F + http2cipher_TLS_DH_DSS_WITH_AES_128_CBC_SHA uint16 = 0x0030 + http2cipher_TLS_DH_RSA_WITH_AES_128_CBC_SHA uint16 = 0x0031 + http2cipher_TLS_DHE_DSS_WITH_AES_128_CBC_SHA uint16 = 0x0032 + http2cipher_TLS_DHE_RSA_WITH_AES_128_CBC_SHA uint16 = 0x0033 + http2cipher_TLS_DH_anon_WITH_AES_128_CBC_SHA uint16 = 0x0034 + http2cipher_TLS_RSA_WITH_AES_256_CBC_SHA uint16 = 0x0035 + http2cipher_TLS_DH_DSS_WITH_AES_256_CBC_SHA uint16 = 0x0036 + http2cipher_TLS_DH_RSA_WITH_AES_256_CBC_SHA uint16 = 0x0037 + http2cipher_TLS_DHE_DSS_WITH_AES_256_CBC_SHA uint16 = 0x0038 + http2cipher_TLS_DHE_RSA_WITH_AES_256_CBC_SHA uint16 = 0x0039 + http2cipher_TLS_DH_anon_WITH_AES_256_CBC_SHA uint16 = 0x003A + http2cipher_TLS_RSA_WITH_NULL_SHA256 uint16 = 0x003B + http2cipher_TLS_RSA_WITH_AES_128_CBC_SHA256 uint16 = 0x003C + http2cipher_TLS_RSA_WITH_AES_256_CBC_SHA256 uint16 = 0x003D + http2cipher_TLS_DH_DSS_WITH_AES_128_CBC_SHA256 uint16 = 0x003E + http2cipher_TLS_DH_RSA_WITH_AES_128_CBC_SHA256 uint16 = 0x003F + http2cipher_TLS_DHE_DSS_WITH_AES_128_CBC_SHA256 uint16 = 0x0040 + http2cipher_TLS_RSA_WITH_CAMELLIA_128_CBC_SHA uint16 = 0x0041 + http2cipher_TLS_DH_DSS_WITH_CAMELLIA_128_CBC_SHA uint16 = 0x0042 + http2cipher_TLS_DH_RSA_WITH_CAMELLIA_128_CBC_SHA uint16 = 0x0043 + http2cipher_TLS_DHE_DSS_WITH_CAMELLIA_128_CBC_SHA uint16 = 0x0044 + http2cipher_TLS_DHE_RSA_WITH_CAMELLIA_128_CBC_SHA uint16 = 0x0045 + http2cipher_TLS_DH_anon_WITH_CAMELLIA_128_CBC_SHA uint16 = 0x0046 + // Reserved uint16 = 0x0047-4F + // Reserved uint16 = 0x0050-58 + // Reserved uint16 = 0x0059-5C + // Unassigned uint16 = 0x005D-5F + // Reserved uint16 = 0x0060-66 + http2cipher_TLS_DHE_RSA_WITH_AES_128_CBC_SHA256 uint16 = 0x0067 + http2cipher_TLS_DH_DSS_WITH_AES_256_CBC_SHA256 uint16 = 0x0068 + http2cipher_TLS_DH_RSA_WITH_AES_256_CBC_SHA256 uint16 = 0x0069 + http2cipher_TLS_DHE_DSS_WITH_AES_256_CBC_SHA256 uint16 = 0x006A + http2cipher_TLS_DHE_RSA_WITH_AES_256_CBC_SHA256 uint16 = 0x006B + http2cipher_TLS_DH_anon_WITH_AES_128_CBC_SHA256 uint16 = 0x006C + http2cipher_TLS_DH_anon_WITH_AES_256_CBC_SHA256 uint16 = 0x006D + // Unassigned uint16 = 0x006E-83 + http2cipher_TLS_RSA_WITH_CAMELLIA_256_CBC_SHA uint16 = 0x0084 + http2cipher_TLS_DH_DSS_WITH_CAMELLIA_256_CBC_SHA uint16 = 0x0085 + http2cipher_TLS_DH_RSA_WITH_CAMELLIA_256_CBC_SHA uint16 = 0x0086 + http2cipher_TLS_DHE_DSS_WITH_CAMELLIA_256_CBC_SHA uint16 = 0x0087 + http2cipher_TLS_DHE_RSA_WITH_CAMELLIA_256_CBC_SHA uint16 = 0x0088 + http2cipher_TLS_DH_anon_WITH_CAMELLIA_256_CBC_SHA uint16 = 0x0089 + http2cipher_TLS_PSK_WITH_RC4_128_SHA uint16 = 0x008A + http2cipher_TLS_PSK_WITH_3DES_EDE_CBC_SHA uint16 = 0x008B + http2cipher_TLS_PSK_WITH_AES_128_CBC_SHA uint16 = 0x008C + http2cipher_TLS_PSK_WITH_AES_256_CBC_SHA uint16 = 0x008D + http2cipher_TLS_DHE_PSK_WITH_RC4_128_SHA uint16 = 0x008E + http2cipher_TLS_DHE_PSK_WITH_3DES_EDE_CBC_SHA uint16 = 0x008F + http2cipher_TLS_DHE_PSK_WITH_AES_128_CBC_SHA uint16 = 0x0090 + http2cipher_TLS_DHE_PSK_WITH_AES_256_CBC_SHA uint16 = 0x0091 + http2cipher_TLS_RSA_PSK_WITH_RC4_128_SHA uint16 = 0x0092 + http2cipher_TLS_RSA_PSK_WITH_3DES_EDE_CBC_SHA uint16 = 0x0093 + http2cipher_TLS_RSA_PSK_WITH_AES_128_CBC_SHA uint16 = 0x0094 + http2cipher_TLS_RSA_PSK_WITH_AES_256_CBC_SHA uint16 = 0x0095 + http2cipher_TLS_RSA_WITH_SEED_CBC_SHA uint16 = 0x0096 + http2cipher_TLS_DH_DSS_WITH_SEED_CBC_SHA uint16 = 0x0097 + http2cipher_TLS_DH_RSA_WITH_SEED_CBC_SHA uint16 = 0x0098 + http2cipher_TLS_DHE_DSS_WITH_SEED_CBC_SHA uint16 = 0x0099 + http2cipher_TLS_DHE_RSA_WITH_SEED_CBC_SHA uint16 = 0x009A + http2cipher_TLS_DH_anon_WITH_SEED_CBC_SHA uint16 = 0x009B + http2cipher_TLS_RSA_WITH_AES_128_GCM_SHA256 uint16 = 0x009C + http2cipher_TLS_RSA_WITH_AES_256_GCM_SHA384 uint16 = 0x009D + http2cipher_TLS_DHE_RSA_WITH_AES_128_GCM_SHA256 uint16 = 0x009E + http2cipher_TLS_DHE_RSA_WITH_AES_256_GCM_SHA384 uint16 = 0x009F + http2cipher_TLS_DH_RSA_WITH_AES_128_GCM_SHA256 uint16 = 0x00A0 + http2cipher_TLS_DH_RSA_WITH_AES_256_GCM_SHA384 uint16 = 0x00A1 + http2cipher_TLS_DHE_DSS_WITH_AES_128_GCM_SHA256 uint16 = 0x00A2 + http2cipher_TLS_DHE_DSS_WITH_AES_256_GCM_SHA384 uint16 = 0x00A3 + http2cipher_TLS_DH_DSS_WITH_AES_128_GCM_SHA256 uint16 = 0x00A4 + http2cipher_TLS_DH_DSS_WITH_AES_256_GCM_SHA384 uint16 = 0x00A5 + http2cipher_TLS_DH_anon_WITH_AES_128_GCM_SHA256 uint16 = 0x00A6 + http2cipher_TLS_DH_anon_WITH_AES_256_GCM_SHA384 uint16 = 0x00A7 + http2cipher_TLS_PSK_WITH_AES_128_GCM_SHA256 uint16 = 0x00A8 + http2cipher_TLS_PSK_WITH_AES_256_GCM_SHA384 uint16 = 0x00A9 + http2cipher_TLS_DHE_PSK_WITH_AES_128_GCM_SHA256 uint16 = 0x00AA + http2cipher_TLS_DHE_PSK_WITH_AES_256_GCM_SHA384 uint16 = 0x00AB + http2cipher_TLS_RSA_PSK_WITH_AES_128_GCM_SHA256 uint16 = 0x00AC + http2cipher_TLS_RSA_PSK_WITH_AES_256_GCM_SHA384 uint16 = 0x00AD + http2cipher_TLS_PSK_WITH_AES_128_CBC_SHA256 uint16 = 0x00AE + http2cipher_TLS_PSK_WITH_AES_256_CBC_SHA384 uint16 = 0x00AF + http2cipher_TLS_PSK_WITH_NULL_SHA256 uint16 = 0x00B0 + http2cipher_TLS_PSK_WITH_NULL_SHA384 uint16 = 0x00B1 + http2cipher_TLS_DHE_PSK_WITH_AES_128_CBC_SHA256 uint16 = 0x00B2 + http2cipher_TLS_DHE_PSK_WITH_AES_256_CBC_SHA384 uint16 = 0x00B3 + http2cipher_TLS_DHE_PSK_WITH_NULL_SHA256 uint16 = 0x00B4 + http2cipher_TLS_DHE_PSK_WITH_NULL_SHA384 uint16 = 0x00B5 + http2cipher_TLS_RSA_PSK_WITH_AES_128_CBC_SHA256 uint16 = 0x00B6 + http2cipher_TLS_RSA_PSK_WITH_AES_256_CBC_SHA384 uint16 = 0x00B7 + http2cipher_TLS_RSA_PSK_WITH_NULL_SHA256 uint16 = 0x00B8 + http2cipher_TLS_RSA_PSK_WITH_NULL_SHA384 uint16 = 0x00B9 + http2cipher_TLS_RSA_WITH_CAMELLIA_128_CBC_SHA256 uint16 = 0x00BA + http2cipher_TLS_DH_DSS_WITH_CAMELLIA_128_CBC_SHA256 uint16 = 0x00BB + http2cipher_TLS_DH_RSA_WITH_CAMELLIA_128_CBC_SHA256 uint16 = 0x00BC + http2cipher_TLS_DHE_DSS_WITH_CAMELLIA_128_CBC_SHA256 uint16 = 0x00BD + http2cipher_TLS_DHE_RSA_WITH_CAMELLIA_128_CBC_SHA256 uint16 = 0x00BE + http2cipher_TLS_DH_anon_WITH_CAMELLIA_128_CBC_SHA256 uint16 = 0x00BF + http2cipher_TLS_RSA_WITH_CAMELLIA_256_CBC_SHA256 uint16 = 0x00C0 + http2cipher_TLS_DH_DSS_WITH_CAMELLIA_256_CBC_SHA256 uint16 = 0x00C1 + http2cipher_TLS_DH_RSA_WITH_CAMELLIA_256_CBC_SHA256 uint16 = 0x00C2 + http2cipher_TLS_DHE_DSS_WITH_CAMELLIA_256_CBC_SHA256 uint16 = 0x00C3 + http2cipher_TLS_DHE_RSA_WITH_CAMELLIA_256_CBC_SHA256 uint16 = 0x00C4 + http2cipher_TLS_DH_anon_WITH_CAMELLIA_256_CBC_SHA256 uint16 = 0x00C5 + // Unassigned uint16 = 0x00C6-FE + http2cipher_TLS_EMPTY_RENEGOTIATION_INFO_SCSV uint16 = 0x00FF + // Unassigned uint16 = 0x01-55,* + http2cipher_TLS_FALLBACK_SCSV uint16 = 0x5600 + // Unassigned uint16 = 0x5601 - 0xC000 + http2cipher_TLS_ECDH_ECDSA_WITH_NULL_SHA uint16 = 0xC001 + http2cipher_TLS_ECDH_ECDSA_WITH_RC4_128_SHA uint16 = 0xC002 + http2cipher_TLS_ECDH_ECDSA_WITH_3DES_EDE_CBC_SHA uint16 = 0xC003 + http2cipher_TLS_ECDH_ECDSA_WITH_AES_128_CBC_SHA uint16 = 0xC004 + http2cipher_TLS_ECDH_ECDSA_WITH_AES_256_CBC_SHA uint16 = 0xC005 + http2cipher_TLS_ECDHE_ECDSA_WITH_NULL_SHA uint16 = 0xC006 + http2cipher_TLS_ECDHE_ECDSA_WITH_RC4_128_SHA uint16 = 0xC007 + http2cipher_TLS_ECDHE_ECDSA_WITH_3DES_EDE_CBC_SHA uint16 = 0xC008 + http2cipher_TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA uint16 = 0xC009 + http2cipher_TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA uint16 = 0xC00A + http2cipher_TLS_ECDH_RSA_WITH_NULL_SHA uint16 = 0xC00B + http2cipher_TLS_ECDH_RSA_WITH_RC4_128_SHA uint16 = 0xC00C + http2cipher_TLS_ECDH_RSA_WITH_3DES_EDE_CBC_SHA uint16 = 0xC00D + http2cipher_TLS_ECDH_RSA_WITH_AES_128_CBC_SHA uint16 = 0xC00E + http2cipher_TLS_ECDH_RSA_WITH_AES_256_CBC_SHA uint16 = 0xC00F + http2cipher_TLS_ECDHE_RSA_WITH_NULL_SHA uint16 = 0xC010 + http2cipher_TLS_ECDHE_RSA_WITH_RC4_128_SHA uint16 = 0xC011 + http2cipher_TLS_ECDHE_RSA_WITH_3DES_EDE_CBC_SHA uint16 = 0xC012 + http2cipher_TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA uint16 = 0xC013 + http2cipher_TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA uint16 = 0xC014 + http2cipher_TLS_ECDH_anon_WITH_NULL_SHA uint16 = 0xC015 + http2cipher_TLS_ECDH_anon_WITH_RC4_128_SHA uint16 = 0xC016 + http2cipher_TLS_ECDH_anon_WITH_3DES_EDE_CBC_SHA uint16 = 0xC017 + http2cipher_TLS_ECDH_anon_WITH_AES_128_CBC_SHA uint16 = 0xC018 + http2cipher_TLS_ECDH_anon_WITH_AES_256_CBC_SHA uint16 = 0xC019 + http2cipher_TLS_SRP_SHA_WITH_3DES_EDE_CBC_SHA uint16 = 0xC01A + http2cipher_TLS_SRP_SHA_RSA_WITH_3DES_EDE_CBC_SHA uint16 = 0xC01B + http2cipher_TLS_SRP_SHA_DSS_WITH_3DES_EDE_CBC_SHA uint16 = 0xC01C + http2cipher_TLS_SRP_SHA_WITH_AES_128_CBC_SHA uint16 = 0xC01D + http2cipher_TLS_SRP_SHA_RSA_WITH_AES_128_CBC_SHA uint16 = 0xC01E + http2cipher_TLS_SRP_SHA_DSS_WITH_AES_128_CBC_SHA uint16 = 0xC01F + http2cipher_TLS_SRP_SHA_WITH_AES_256_CBC_SHA uint16 = 0xC020 + http2cipher_TLS_SRP_SHA_RSA_WITH_AES_256_CBC_SHA uint16 = 0xC021 + http2cipher_TLS_SRP_SHA_DSS_WITH_AES_256_CBC_SHA uint16 = 0xC022 + http2cipher_TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA256 uint16 = 0xC023 + http2cipher_TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA384 uint16 = 0xC024 + http2cipher_TLS_ECDH_ECDSA_WITH_AES_128_CBC_SHA256 uint16 = 0xC025 + http2cipher_TLS_ECDH_ECDSA_WITH_AES_256_CBC_SHA384 uint16 = 0xC026 + http2cipher_TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA256 uint16 = 0xC027 + http2cipher_TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA384 uint16 = 0xC028 + http2cipher_TLS_ECDH_RSA_WITH_AES_128_CBC_SHA256 uint16 = 0xC029 + http2cipher_TLS_ECDH_RSA_WITH_AES_256_CBC_SHA384 uint16 = 0xC02A + http2cipher_TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 uint16 = 0xC02B + http2cipher_TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384 uint16 = 0xC02C + http2cipher_TLS_ECDH_ECDSA_WITH_AES_128_GCM_SHA256 uint16 = 0xC02D + http2cipher_TLS_ECDH_ECDSA_WITH_AES_256_GCM_SHA384 uint16 = 0xC02E + http2cipher_TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 uint16 = 0xC02F + http2cipher_TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 uint16 = 0xC030 + http2cipher_TLS_ECDH_RSA_WITH_AES_128_GCM_SHA256 uint16 = 0xC031 + http2cipher_TLS_ECDH_RSA_WITH_AES_256_GCM_SHA384 uint16 = 0xC032 + http2cipher_TLS_ECDHE_PSK_WITH_RC4_128_SHA uint16 = 0xC033 + http2cipher_TLS_ECDHE_PSK_WITH_3DES_EDE_CBC_SHA uint16 = 0xC034 + http2cipher_TLS_ECDHE_PSK_WITH_AES_128_CBC_SHA uint16 = 0xC035 + http2cipher_TLS_ECDHE_PSK_WITH_AES_256_CBC_SHA uint16 = 0xC036 + http2cipher_TLS_ECDHE_PSK_WITH_AES_128_CBC_SHA256 uint16 = 0xC037 + http2cipher_TLS_ECDHE_PSK_WITH_AES_256_CBC_SHA384 uint16 = 0xC038 + http2cipher_TLS_ECDHE_PSK_WITH_NULL_SHA uint16 = 0xC039 + http2cipher_TLS_ECDHE_PSK_WITH_NULL_SHA256 uint16 = 0xC03A + http2cipher_TLS_ECDHE_PSK_WITH_NULL_SHA384 uint16 = 0xC03B + http2cipher_TLS_RSA_WITH_ARIA_128_CBC_SHA256 uint16 = 0xC03C + http2cipher_TLS_RSA_WITH_ARIA_256_CBC_SHA384 uint16 = 0xC03D + http2cipher_TLS_DH_DSS_WITH_ARIA_128_CBC_SHA256 uint16 = 0xC03E + http2cipher_TLS_DH_DSS_WITH_ARIA_256_CBC_SHA384 uint16 = 0xC03F + http2cipher_TLS_DH_RSA_WITH_ARIA_128_CBC_SHA256 uint16 = 0xC040 + http2cipher_TLS_DH_RSA_WITH_ARIA_256_CBC_SHA384 uint16 = 0xC041 + http2cipher_TLS_DHE_DSS_WITH_ARIA_128_CBC_SHA256 uint16 = 0xC042 + http2cipher_TLS_DHE_DSS_WITH_ARIA_256_CBC_SHA384 uint16 = 0xC043 + http2cipher_TLS_DHE_RSA_WITH_ARIA_128_CBC_SHA256 uint16 = 0xC044 + http2cipher_TLS_DHE_RSA_WITH_ARIA_256_CBC_SHA384 uint16 = 0xC045 + http2cipher_TLS_DH_anon_WITH_ARIA_128_CBC_SHA256 uint16 = 0xC046 + http2cipher_TLS_DH_anon_WITH_ARIA_256_CBC_SHA384 uint16 = 0xC047 + http2cipher_TLS_ECDHE_ECDSA_WITH_ARIA_128_CBC_SHA256 uint16 = 0xC048 + http2cipher_TLS_ECDHE_ECDSA_WITH_ARIA_256_CBC_SHA384 uint16 = 0xC049 + http2cipher_TLS_ECDH_ECDSA_WITH_ARIA_128_CBC_SHA256 uint16 = 0xC04A + http2cipher_TLS_ECDH_ECDSA_WITH_ARIA_256_CBC_SHA384 uint16 = 0xC04B + http2cipher_TLS_ECDHE_RSA_WITH_ARIA_128_CBC_SHA256 uint16 = 0xC04C + http2cipher_TLS_ECDHE_RSA_WITH_ARIA_256_CBC_SHA384 uint16 = 0xC04D + http2cipher_TLS_ECDH_RSA_WITH_ARIA_128_CBC_SHA256 uint16 = 0xC04E + http2cipher_TLS_ECDH_RSA_WITH_ARIA_256_CBC_SHA384 uint16 = 0xC04F + http2cipher_TLS_RSA_WITH_ARIA_128_GCM_SHA256 uint16 = 0xC050 + http2cipher_TLS_RSA_WITH_ARIA_256_GCM_SHA384 uint16 = 0xC051 + http2cipher_TLS_DHE_RSA_WITH_ARIA_128_GCM_SHA256 uint16 = 0xC052 + http2cipher_TLS_DHE_RSA_WITH_ARIA_256_GCM_SHA384 uint16 = 0xC053 + http2cipher_TLS_DH_RSA_WITH_ARIA_128_GCM_SHA256 uint16 = 0xC054 + http2cipher_TLS_DH_RSA_WITH_ARIA_256_GCM_SHA384 uint16 = 0xC055 + http2cipher_TLS_DHE_DSS_WITH_ARIA_128_GCM_SHA256 uint16 = 0xC056 + http2cipher_TLS_DHE_DSS_WITH_ARIA_256_GCM_SHA384 uint16 = 0xC057 + http2cipher_TLS_DH_DSS_WITH_ARIA_128_GCM_SHA256 uint16 = 0xC058 + http2cipher_TLS_DH_DSS_WITH_ARIA_256_GCM_SHA384 uint16 = 0xC059 + http2cipher_TLS_DH_anon_WITH_ARIA_128_GCM_SHA256 uint16 = 0xC05A + http2cipher_TLS_DH_anon_WITH_ARIA_256_GCM_SHA384 uint16 = 0xC05B + http2cipher_TLS_ECDHE_ECDSA_WITH_ARIA_128_GCM_SHA256 uint16 = 0xC05C + http2cipher_TLS_ECDHE_ECDSA_WITH_ARIA_256_GCM_SHA384 uint16 = 0xC05D + http2cipher_TLS_ECDH_ECDSA_WITH_ARIA_128_GCM_SHA256 uint16 = 0xC05E + http2cipher_TLS_ECDH_ECDSA_WITH_ARIA_256_GCM_SHA384 uint16 = 0xC05F + http2cipher_TLS_ECDHE_RSA_WITH_ARIA_128_GCM_SHA256 uint16 = 0xC060 + http2cipher_TLS_ECDHE_RSA_WITH_ARIA_256_GCM_SHA384 uint16 = 0xC061 + http2cipher_TLS_ECDH_RSA_WITH_ARIA_128_GCM_SHA256 uint16 = 0xC062 + http2cipher_TLS_ECDH_RSA_WITH_ARIA_256_GCM_SHA384 uint16 = 0xC063 + http2cipher_TLS_PSK_WITH_ARIA_128_CBC_SHA256 uint16 = 0xC064 + http2cipher_TLS_PSK_WITH_ARIA_256_CBC_SHA384 uint16 = 0xC065 + http2cipher_TLS_DHE_PSK_WITH_ARIA_128_CBC_SHA256 uint16 = 0xC066 + http2cipher_TLS_DHE_PSK_WITH_ARIA_256_CBC_SHA384 uint16 = 0xC067 + http2cipher_TLS_RSA_PSK_WITH_ARIA_128_CBC_SHA256 uint16 = 0xC068 + http2cipher_TLS_RSA_PSK_WITH_ARIA_256_CBC_SHA384 uint16 = 0xC069 + http2cipher_TLS_PSK_WITH_ARIA_128_GCM_SHA256 uint16 = 0xC06A + http2cipher_TLS_PSK_WITH_ARIA_256_GCM_SHA384 uint16 = 0xC06B + http2cipher_TLS_DHE_PSK_WITH_ARIA_128_GCM_SHA256 uint16 = 0xC06C + http2cipher_TLS_DHE_PSK_WITH_ARIA_256_GCM_SHA384 uint16 = 0xC06D + http2cipher_TLS_RSA_PSK_WITH_ARIA_128_GCM_SHA256 uint16 = 0xC06E + http2cipher_TLS_RSA_PSK_WITH_ARIA_256_GCM_SHA384 uint16 = 0xC06F + http2cipher_TLS_ECDHE_PSK_WITH_ARIA_128_CBC_SHA256 uint16 = 0xC070 + http2cipher_TLS_ECDHE_PSK_WITH_ARIA_256_CBC_SHA384 uint16 = 0xC071 + http2cipher_TLS_ECDHE_ECDSA_WITH_CAMELLIA_128_CBC_SHA256 uint16 = 0xC072 + http2cipher_TLS_ECDHE_ECDSA_WITH_CAMELLIA_256_CBC_SHA384 uint16 = 0xC073 + http2cipher_TLS_ECDH_ECDSA_WITH_CAMELLIA_128_CBC_SHA256 uint16 = 0xC074 + http2cipher_TLS_ECDH_ECDSA_WITH_CAMELLIA_256_CBC_SHA384 uint16 = 0xC075 + http2cipher_TLS_ECDHE_RSA_WITH_CAMELLIA_128_CBC_SHA256 uint16 = 0xC076 + http2cipher_TLS_ECDHE_RSA_WITH_CAMELLIA_256_CBC_SHA384 uint16 = 0xC077 + http2cipher_TLS_ECDH_RSA_WITH_CAMELLIA_128_CBC_SHA256 uint16 = 0xC078 + http2cipher_TLS_ECDH_RSA_WITH_CAMELLIA_256_CBC_SHA384 uint16 = 0xC079 + http2cipher_TLS_RSA_WITH_CAMELLIA_128_GCM_SHA256 uint16 = 0xC07A + http2cipher_TLS_RSA_WITH_CAMELLIA_256_GCM_SHA384 uint16 = 0xC07B + http2cipher_TLS_DHE_RSA_WITH_CAMELLIA_128_GCM_SHA256 uint16 = 0xC07C + http2cipher_TLS_DHE_RSA_WITH_CAMELLIA_256_GCM_SHA384 uint16 = 0xC07D + http2cipher_TLS_DH_RSA_WITH_CAMELLIA_128_GCM_SHA256 uint16 = 0xC07E + http2cipher_TLS_DH_RSA_WITH_CAMELLIA_256_GCM_SHA384 uint16 = 0xC07F + http2cipher_TLS_DHE_DSS_WITH_CAMELLIA_128_GCM_SHA256 uint16 = 0xC080 + http2cipher_TLS_DHE_DSS_WITH_CAMELLIA_256_GCM_SHA384 uint16 = 0xC081 + http2cipher_TLS_DH_DSS_WITH_CAMELLIA_128_GCM_SHA256 uint16 = 0xC082 + http2cipher_TLS_DH_DSS_WITH_CAMELLIA_256_GCM_SHA384 uint16 = 0xC083 + http2cipher_TLS_DH_anon_WITH_CAMELLIA_128_GCM_SHA256 uint16 = 0xC084 + http2cipher_TLS_DH_anon_WITH_CAMELLIA_256_GCM_SHA384 uint16 = 0xC085 + http2cipher_TLS_ECDHE_ECDSA_WITH_CAMELLIA_128_GCM_SHA256 uint16 = 0xC086 + http2cipher_TLS_ECDHE_ECDSA_WITH_CAMELLIA_256_GCM_SHA384 uint16 = 0xC087 + http2cipher_TLS_ECDH_ECDSA_WITH_CAMELLIA_128_GCM_SHA256 uint16 = 0xC088 + http2cipher_TLS_ECDH_ECDSA_WITH_CAMELLIA_256_GCM_SHA384 uint16 = 0xC089 + http2cipher_TLS_ECDHE_RSA_WITH_CAMELLIA_128_GCM_SHA256 uint16 = 0xC08A + http2cipher_TLS_ECDHE_RSA_WITH_CAMELLIA_256_GCM_SHA384 uint16 = 0xC08B + http2cipher_TLS_ECDH_RSA_WITH_CAMELLIA_128_GCM_SHA256 uint16 = 0xC08C + http2cipher_TLS_ECDH_RSA_WITH_CAMELLIA_256_GCM_SHA384 uint16 = 0xC08D + http2cipher_TLS_PSK_WITH_CAMELLIA_128_GCM_SHA256 uint16 = 0xC08E + http2cipher_TLS_PSK_WITH_CAMELLIA_256_GCM_SHA384 uint16 = 0xC08F + http2cipher_TLS_DHE_PSK_WITH_CAMELLIA_128_GCM_SHA256 uint16 = 0xC090 + http2cipher_TLS_DHE_PSK_WITH_CAMELLIA_256_GCM_SHA384 uint16 = 0xC091 + http2cipher_TLS_RSA_PSK_WITH_CAMELLIA_128_GCM_SHA256 uint16 = 0xC092 + http2cipher_TLS_RSA_PSK_WITH_CAMELLIA_256_GCM_SHA384 uint16 = 0xC093 + http2cipher_TLS_PSK_WITH_CAMELLIA_128_CBC_SHA256 uint16 = 0xC094 + http2cipher_TLS_PSK_WITH_CAMELLIA_256_CBC_SHA384 uint16 = 0xC095 + http2cipher_TLS_DHE_PSK_WITH_CAMELLIA_128_CBC_SHA256 uint16 = 0xC096 + http2cipher_TLS_DHE_PSK_WITH_CAMELLIA_256_CBC_SHA384 uint16 = 0xC097 + http2cipher_TLS_RSA_PSK_WITH_CAMELLIA_128_CBC_SHA256 uint16 = 0xC098 + http2cipher_TLS_RSA_PSK_WITH_CAMELLIA_256_CBC_SHA384 uint16 = 0xC099 + http2cipher_TLS_ECDHE_PSK_WITH_CAMELLIA_128_CBC_SHA256 uint16 = 0xC09A + http2cipher_TLS_ECDHE_PSK_WITH_CAMELLIA_256_CBC_SHA384 uint16 = 0xC09B + http2cipher_TLS_RSA_WITH_AES_128_CCM uint16 = 0xC09C + http2cipher_TLS_RSA_WITH_AES_256_CCM uint16 = 0xC09D + http2cipher_TLS_DHE_RSA_WITH_AES_128_CCM uint16 = 0xC09E + http2cipher_TLS_DHE_RSA_WITH_AES_256_CCM uint16 = 0xC09F + http2cipher_TLS_RSA_WITH_AES_128_CCM_8 uint16 = 0xC0A0 + http2cipher_TLS_RSA_WITH_AES_256_CCM_8 uint16 = 0xC0A1 + http2cipher_TLS_DHE_RSA_WITH_AES_128_CCM_8 uint16 = 0xC0A2 + http2cipher_TLS_DHE_RSA_WITH_AES_256_CCM_8 uint16 = 0xC0A3 + http2cipher_TLS_PSK_WITH_AES_128_CCM uint16 = 0xC0A4 + http2cipher_TLS_PSK_WITH_AES_256_CCM uint16 = 0xC0A5 + http2cipher_TLS_DHE_PSK_WITH_AES_128_CCM uint16 = 0xC0A6 + http2cipher_TLS_DHE_PSK_WITH_AES_256_CCM uint16 = 0xC0A7 + http2cipher_TLS_PSK_WITH_AES_128_CCM_8 uint16 = 0xC0A8 + http2cipher_TLS_PSK_WITH_AES_256_CCM_8 uint16 = 0xC0A9 + http2cipher_TLS_PSK_DHE_WITH_AES_128_CCM_8 uint16 = 0xC0AA + http2cipher_TLS_PSK_DHE_WITH_AES_256_CCM_8 uint16 = 0xC0AB + http2cipher_TLS_ECDHE_ECDSA_WITH_AES_128_CCM uint16 = 0xC0AC + http2cipher_TLS_ECDHE_ECDSA_WITH_AES_256_CCM uint16 = 0xC0AD + http2cipher_TLS_ECDHE_ECDSA_WITH_AES_128_CCM_8 uint16 = 0xC0AE + http2cipher_TLS_ECDHE_ECDSA_WITH_AES_256_CCM_8 uint16 = 0xC0AF + // Unassigned uint16 = 0xC0B0-FF + // Unassigned uint16 = 0xC1-CB,* + // Unassigned uint16 = 0xCC00-A7 + http2cipher_TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256 uint16 = 0xCCA8 + http2cipher_TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256 uint16 = 0xCCA9 + http2cipher_TLS_DHE_RSA_WITH_CHACHA20_POLY1305_SHA256 uint16 = 0xCCAA + http2cipher_TLS_PSK_WITH_CHACHA20_POLY1305_SHA256 uint16 = 0xCCAB + http2cipher_TLS_ECDHE_PSK_WITH_CHACHA20_POLY1305_SHA256 uint16 = 0xCCAC + http2cipher_TLS_DHE_PSK_WITH_CHACHA20_POLY1305_SHA256 uint16 = 0xCCAD + http2cipher_TLS_RSA_PSK_WITH_CHACHA20_POLY1305_SHA256 uint16 = 0xCCAE +) + +// isBadCipher reports whether the cipher is blacklisted by the HTTP/2 spec. +// References: +// https://tools.ietf.org/html/rfc7540#appendix-A +// Reject cipher suites from Appendix A. +// "This list includes those cipher suites that do not +// offer an ephemeral key exchange and those that are +// based on the TLS null, stream or block cipher type" +func http2isBadCipher(cipher uint16) bool { + switch cipher { + case http2cipher_TLS_NULL_WITH_NULL_NULL, + http2cipher_TLS_RSA_WITH_NULL_MD5, + http2cipher_TLS_RSA_WITH_NULL_SHA, + http2cipher_TLS_RSA_EXPORT_WITH_RC4_40_MD5, + http2cipher_TLS_RSA_WITH_RC4_128_MD5, + http2cipher_TLS_RSA_WITH_RC4_128_SHA, + http2cipher_TLS_RSA_EXPORT_WITH_RC2_CBC_40_MD5, + http2cipher_TLS_RSA_WITH_IDEA_CBC_SHA, + http2cipher_TLS_RSA_EXPORT_WITH_DES40_CBC_SHA, + http2cipher_TLS_RSA_WITH_DES_CBC_SHA, + http2cipher_TLS_RSA_WITH_3DES_EDE_CBC_SHA, + http2cipher_TLS_DH_DSS_EXPORT_WITH_DES40_CBC_SHA, + http2cipher_TLS_DH_DSS_WITH_DES_CBC_SHA, + http2cipher_TLS_DH_DSS_WITH_3DES_EDE_CBC_SHA, + http2cipher_TLS_DH_RSA_EXPORT_WITH_DES40_CBC_SHA, + http2cipher_TLS_DH_RSA_WITH_DES_CBC_SHA, + http2cipher_TLS_DH_RSA_WITH_3DES_EDE_CBC_SHA, + http2cipher_TLS_DHE_DSS_EXPORT_WITH_DES40_CBC_SHA, + http2cipher_TLS_DHE_DSS_WITH_DES_CBC_SHA, + http2cipher_TLS_DHE_DSS_WITH_3DES_EDE_CBC_SHA, + http2cipher_TLS_DHE_RSA_EXPORT_WITH_DES40_CBC_SHA, + http2cipher_TLS_DHE_RSA_WITH_DES_CBC_SHA, + http2cipher_TLS_DHE_RSA_WITH_3DES_EDE_CBC_SHA, + http2cipher_TLS_DH_anon_EXPORT_WITH_RC4_40_MD5, + http2cipher_TLS_DH_anon_WITH_RC4_128_MD5, + http2cipher_TLS_DH_anon_EXPORT_WITH_DES40_CBC_SHA, + http2cipher_TLS_DH_anon_WITH_DES_CBC_SHA, + http2cipher_TLS_DH_anon_WITH_3DES_EDE_CBC_SHA, + http2cipher_TLS_KRB5_WITH_DES_CBC_SHA, + http2cipher_TLS_KRB5_WITH_3DES_EDE_CBC_SHA, + http2cipher_TLS_KRB5_WITH_RC4_128_SHA, + http2cipher_TLS_KRB5_WITH_IDEA_CBC_SHA, + http2cipher_TLS_KRB5_WITH_DES_CBC_MD5, + http2cipher_TLS_KRB5_WITH_3DES_EDE_CBC_MD5, + http2cipher_TLS_KRB5_WITH_RC4_128_MD5, + http2cipher_TLS_KRB5_WITH_IDEA_CBC_MD5, + http2cipher_TLS_KRB5_EXPORT_WITH_DES_CBC_40_SHA, + http2cipher_TLS_KRB5_EXPORT_WITH_RC2_CBC_40_SHA, + http2cipher_TLS_KRB5_EXPORT_WITH_RC4_40_SHA, + http2cipher_TLS_KRB5_EXPORT_WITH_DES_CBC_40_MD5, + http2cipher_TLS_KRB5_EXPORT_WITH_RC2_CBC_40_MD5, + http2cipher_TLS_KRB5_EXPORT_WITH_RC4_40_MD5, + http2cipher_TLS_PSK_WITH_NULL_SHA, + http2cipher_TLS_DHE_PSK_WITH_NULL_SHA, + http2cipher_TLS_RSA_PSK_WITH_NULL_SHA, + http2cipher_TLS_RSA_WITH_AES_128_CBC_SHA, + http2cipher_TLS_DH_DSS_WITH_AES_128_CBC_SHA, + http2cipher_TLS_DH_RSA_WITH_AES_128_CBC_SHA, + http2cipher_TLS_DHE_DSS_WITH_AES_128_CBC_SHA, + http2cipher_TLS_DHE_RSA_WITH_AES_128_CBC_SHA, + http2cipher_TLS_DH_anon_WITH_AES_128_CBC_SHA, + http2cipher_TLS_RSA_WITH_AES_256_CBC_SHA, + http2cipher_TLS_DH_DSS_WITH_AES_256_CBC_SHA, + http2cipher_TLS_DH_RSA_WITH_AES_256_CBC_SHA, + http2cipher_TLS_DHE_DSS_WITH_AES_256_CBC_SHA, + http2cipher_TLS_DHE_RSA_WITH_AES_256_CBC_SHA, + http2cipher_TLS_DH_anon_WITH_AES_256_CBC_SHA, + http2cipher_TLS_RSA_WITH_NULL_SHA256, + http2cipher_TLS_RSA_WITH_AES_128_CBC_SHA256, + http2cipher_TLS_RSA_WITH_AES_256_CBC_SHA256, + http2cipher_TLS_DH_DSS_WITH_AES_128_CBC_SHA256, + http2cipher_TLS_DH_RSA_WITH_AES_128_CBC_SHA256, + http2cipher_TLS_DHE_DSS_WITH_AES_128_CBC_SHA256, + http2cipher_TLS_RSA_WITH_CAMELLIA_128_CBC_SHA, + http2cipher_TLS_DH_DSS_WITH_CAMELLIA_128_CBC_SHA, + http2cipher_TLS_DH_RSA_WITH_CAMELLIA_128_CBC_SHA, + http2cipher_TLS_DHE_DSS_WITH_CAMELLIA_128_CBC_SHA, + http2cipher_TLS_DHE_RSA_WITH_CAMELLIA_128_CBC_SHA, + http2cipher_TLS_DH_anon_WITH_CAMELLIA_128_CBC_SHA, + http2cipher_TLS_DHE_RSA_WITH_AES_128_CBC_SHA256, + http2cipher_TLS_DH_DSS_WITH_AES_256_CBC_SHA256, + http2cipher_TLS_DH_RSA_WITH_AES_256_CBC_SHA256, + http2cipher_TLS_DHE_DSS_WITH_AES_256_CBC_SHA256, + http2cipher_TLS_DHE_RSA_WITH_AES_256_CBC_SHA256, + http2cipher_TLS_DH_anon_WITH_AES_128_CBC_SHA256, + http2cipher_TLS_DH_anon_WITH_AES_256_CBC_SHA256, + http2cipher_TLS_RSA_WITH_CAMELLIA_256_CBC_SHA, + http2cipher_TLS_DH_DSS_WITH_CAMELLIA_256_CBC_SHA, + http2cipher_TLS_DH_RSA_WITH_CAMELLIA_256_CBC_SHA, + http2cipher_TLS_DHE_DSS_WITH_CAMELLIA_256_CBC_SHA, + http2cipher_TLS_DHE_RSA_WITH_CAMELLIA_256_CBC_SHA, + http2cipher_TLS_DH_anon_WITH_CAMELLIA_256_CBC_SHA, + http2cipher_TLS_PSK_WITH_RC4_128_SHA, + http2cipher_TLS_PSK_WITH_3DES_EDE_CBC_SHA, + http2cipher_TLS_PSK_WITH_AES_128_CBC_SHA, + http2cipher_TLS_PSK_WITH_AES_256_CBC_SHA, + http2cipher_TLS_DHE_PSK_WITH_RC4_128_SHA, + http2cipher_TLS_DHE_PSK_WITH_3DES_EDE_CBC_SHA, + http2cipher_TLS_DHE_PSK_WITH_AES_128_CBC_SHA, + http2cipher_TLS_DHE_PSK_WITH_AES_256_CBC_SHA, + http2cipher_TLS_RSA_PSK_WITH_RC4_128_SHA, + http2cipher_TLS_RSA_PSK_WITH_3DES_EDE_CBC_SHA, + http2cipher_TLS_RSA_PSK_WITH_AES_128_CBC_SHA, + http2cipher_TLS_RSA_PSK_WITH_AES_256_CBC_SHA, + http2cipher_TLS_RSA_WITH_SEED_CBC_SHA, + http2cipher_TLS_DH_DSS_WITH_SEED_CBC_SHA, + http2cipher_TLS_DH_RSA_WITH_SEED_CBC_SHA, + http2cipher_TLS_DHE_DSS_WITH_SEED_CBC_SHA, + http2cipher_TLS_DHE_RSA_WITH_SEED_CBC_SHA, + http2cipher_TLS_DH_anon_WITH_SEED_CBC_SHA, + http2cipher_TLS_RSA_WITH_AES_128_GCM_SHA256, + http2cipher_TLS_RSA_WITH_AES_256_GCM_SHA384, + http2cipher_TLS_DH_RSA_WITH_AES_128_GCM_SHA256, + http2cipher_TLS_DH_RSA_WITH_AES_256_GCM_SHA384, + http2cipher_TLS_DH_DSS_WITH_AES_128_GCM_SHA256, + http2cipher_TLS_DH_DSS_WITH_AES_256_GCM_SHA384, + http2cipher_TLS_DH_anon_WITH_AES_128_GCM_SHA256, + http2cipher_TLS_DH_anon_WITH_AES_256_GCM_SHA384, + http2cipher_TLS_PSK_WITH_AES_128_GCM_SHA256, + http2cipher_TLS_PSK_WITH_AES_256_GCM_SHA384, + http2cipher_TLS_RSA_PSK_WITH_AES_128_GCM_SHA256, + http2cipher_TLS_RSA_PSK_WITH_AES_256_GCM_SHA384, + http2cipher_TLS_PSK_WITH_AES_128_CBC_SHA256, + http2cipher_TLS_PSK_WITH_AES_256_CBC_SHA384, + http2cipher_TLS_PSK_WITH_NULL_SHA256, + http2cipher_TLS_PSK_WITH_NULL_SHA384, + http2cipher_TLS_DHE_PSK_WITH_AES_128_CBC_SHA256, + http2cipher_TLS_DHE_PSK_WITH_AES_256_CBC_SHA384, + http2cipher_TLS_DHE_PSK_WITH_NULL_SHA256, + http2cipher_TLS_DHE_PSK_WITH_NULL_SHA384, + http2cipher_TLS_RSA_PSK_WITH_AES_128_CBC_SHA256, + http2cipher_TLS_RSA_PSK_WITH_AES_256_CBC_SHA384, + http2cipher_TLS_RSA_PSK_WITH_NULL_SHA256, + http2cipher_TLS_RSA_PSK_WITH_NULL_SHA384, + http2cipher_TLS_RSA_WITH_CAMELLIA_128_CBC_SHA256, + http2cipher_TLS_DH_DSS_WITH_CAMELLIA_128_CBC_SHA256, + http2cipher_TLS_DH_RSA_WITH_CAMELLIA_128_CBC_SHA256, + http2cipher_TLS_DHE_DSS_WITH_CAMELLIA_128_CBC_SHA256, + http2cipher_TLS_DHE_RSA_WITH_CAMELLIA_128_CBC_SHA256, + http2cipher_TLS_DH_anon_WITH_CAMELLIA_128_CBC_SHA256, + http2cipher_TLS_RSA_WITH_CAMELLIA_256_CBC_SHA256, + http2cipher_TLS_DH_DSS_WITH_CAMELLIA_256_CBC_SHA256, + http2cipher_TLS_DH_RSA_WITH_CAMELLIA_256_CBC_SHA256, + http2cipher_TLS_DHE_DSS_WITH_CAMELLIA_256_CBC_SHA256, + http2cipher_TLS_DHE_RSA_WITH_CAMELLIA_256_CBC_SHA256, + http2cipher_TLS_DH_anon_WITH_CAMELLIA_256_CBC_SHA256, + http2cipher_TLS_EMPTY_RENEGOTIATION_INFO_SCSV, + http2cipher_TLS_ECDH_ECDSA_WITH_NULL_SHA, + http2cipher_TLS_ECDH_ECDSA_WITH_RC4_128_SHA, + http2cipher_TLS_ECDH_ECDSA_WITH_3DES_EDE_CBC_SHA, + http2cipher_TLS_ECDH_ECDSA_WITH_AES_128_CBC_SHA, + http2cipher_TLS_ECDH_ECDSA_WITH_AES_256_CBC_SHA, + http2cipher_TLS_ECDHE_ECDSA_WITH_NULL_SHA, + http2cipher_TLS_ECDHE_ECDSA_WITH_RC4_128_SHA, + http2cipher_TLS_ECDHE_ECDSA_WITH_3DES_EDE_CBC_SHA, + http2cipher_TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA, + http2cipher_TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA, + http2cipher_TLS_ECDH_RSA_WITH_NULL_SHA, + http2cipher_TLS_ECDH_RSA_WITH_RC4_128_SHA, + http2cipher_TLS_ECDH_RSA_WITH_3DES_EDE_CBC_SHA, + http2cipher_TLS_ECDH_RSA_WITH_AES_128_CBC_SHA, + http2cipher_TLS_ECDH_RSA_WITH_AES_256_CBC_SHA, + http2cipher_TLS_ECDHE_RSA_WITH_NULL_SHA, + http2cipher_TLS_ECDHE_RSA_WITH_RC4_128_SHA, + http2cipher_TLS_ECDHE_RSA_WITH_3DES_EDE_CBC_SHA, + http2cipher_TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA, + http2cipher_TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA, + http2cipher_TLS_ECDH_anon_WITH_NULL_SHA, + http2cipher_TLS_ECDH_anon_WITH_RC4_128_SHA, + http2cipher_TLS_ECDH_anon_WITH_3DES_EDE_CBC_SHA, + http2cipher_TLS_ECDH_anon_WITH_AES_128_CBC_SHA, + http2cipher_TLS_ECDH_anon_WITH_AES_256_CBC_SHA, + http2cipher_TLS_SRP_SHA_WITH_3DES_EDE_CBC_SHA, + http2cipher_TLS_SRP_SHA_RSA_WITH_3DES_EDE_CBC_SHA, + http2cipher_TLS_SRP_SHA_DSS_WITH_3DES_EDE_CBC_SHA, + http2cipher_TLS_SRP_SHA_WITH_AES_128_CBC_SHA, + http2cipher_TLS_SRP_SHA_RSA_WITH_AES_128_CBC_SHA, + http2cipher_TLS_SRP_SHA_DSS_WITH_AES_128_CBC_SHA, + http2cipher_TLS_SRP_SHA_WITH_AES_256_CBC_SHA, + http2cipher_TLS_SRP_SHA_RSA_WITH_AES_256_CBC_SHA, + http2cipher_TLS_SRP_SHA_DSS_WITH_AES_256_CBC_SHA, + http2cipher_TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA256, + http2cipher_TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA384, + http2cipher_TLS_ECDH_ECDSA_WITH_AES_128_CBC_SHA256, + http2cipher_TLS_ECDH_ECDSA_WITH_AES_256_CBC_SHA384, + http2cipher_TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA256, + http2cipher_TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA384, + http2cipher_TLS_ECDH_RSA_WITH_AES_128_CBC_SHA256, + http2cipher_TLS_ECDH_RSA_WITH_AES_256_CBC_SHA384, + http2cipher_TLS_ECDH_ECDSA_WITH_AES_128_GCM_SHA256, + http2cipher_TLS_ECDH_ECDSA_WITH_AES_256_GCM_SHA384, + http2cipher_TLS_ECDH_RSA_WITH_AES_128_GCM_SHA256, + http2cipher_TLS_ECDH_RSA_WITH_AES_256_GCM_SHA384, + http2cipher_TLS_ECDHE_PSK_WITH_RC4_128_SHA, + http2cipher_TLS_ECDHE_PSK_WITH_3DES_EDE_CBC_SHA, + http2cipher_TLS_ECDHE_PSK_WITH_AES_128_CBC_SHA, + http2cipher_TLS_ECDHE_PSK_WITH_AES_256_CBC_SHA, + http2cipher_TLS_ECDHE_PSK_WITH_AES_128_CBC_SHA256, + http2cipher_TLS_ECDHE_PSK_WITH_AES_256_CBC_SHA384, + http2cipher_TLS_ECDHE_PSK_WITH_NULL_SHA, + http2cipher_TLS_ECDHE_PSK_WITH_NULL_SHA256, + http2cipher_TLS_ECDHE_PSK_WITH_NULL_SHA384, + http2cipher_TLS_RSA_WITH_ARIA_128_CBC_SHA256, + http2cipher_TLS_RSA_WITH_ARIA_256_CBC_SHA384, + http2cipher_TLS_DH_DSS_WITH_ARIA_128_CBC_SHA256, + http2cipher_TLS_DH_DSS_WITH_ARIA_256_CBC_SHA384, + http2cipher_TLS_DH_RSA_WITH_ARIA_128_CBC_SHA256, + http2cipher_TLS_DH_RSA_WITH_ARIA_256_CBC_SHA384, + http2cipher_TLS_DHE_DSS_WITH_ARIA_128_CBC_SHA256, + http2cipher_TLS_DHE_DSS_WITH_ARIA_256_CBC_SHA384, + http2cipher_TLS_DHE_RSA_WITH_ARIA_128_CBC_SHA256, + http2cipher_TLS_DHE_RSA_WITH_ARIA_256_CBC_SHA384, + http2cipher_TLS_DH_anon_WITH_ARIA_128_CBC_SHA256, + http2cipher_TLS_DH_anon_WITH_ARIA_256_CBC_SHA384, + http2cipher_TLS_ECDHE_ECDSA_WITH_ARIA_128_CBC_SHA256, + http2cipher_TLS_ECDHE_ECDSA_WITH_ARIA_256_CBC_SHA384, + http2cipher_TLS_ECDH_ECDSA_WITH_ARIA_128_CBC_SHA256, + http2cipher_TLS_ECDH_ECDSA_WITH_ARIA_256_CBC_SHA384, + http2cipher_TLS_ECDHE_RSA_WITH_ARIA_128_CBC_SHA256, + http2cipher_TLS_ECDHE_RSA_WITH_ARIA_256_CBC_SHA384, + http2cipher_TLS_ECDH_RSA_WITH_ARIA_128_CBC_SHA256, + http2cipher_TLS_ECDH_RSA_WITH_ARIA_256_CBC_SHA384, + http2cipher_TLS_RSA_WITH_ARIA_128_GCM_SHA256, + http2cipher_TLS_RSA_WITH_ARIA_256_GCM_SHA384, + http2cipher_TLS_DH_RSA_WITH_ARIA_128_GCM_SHA256, + http2cipher_TLS_DH_RSA_WITH_ARIA_256_GCM_SHA384, + http2cipher_TLS_DH_DSS_WITH_ARIA_128_GCM_SHA256, + http2cipher_TLS_DH_DSS_WITH_ARIA_256_GCM_SHA384, + http2cipher_TLS_DH_anon_WITH_ARIA_128_GCM_SHA256, + http2cipher_TLS_DH_anon_WITH_ARIA_256_GCM_SHA384, + http2cipher_TLS_ECDH_ECDSA_WITH_ARIA_128_GCM_SHA256, + http2cipher_TLS_ECDH_ECDSA_WITH_ARIA_256_GCM_SHA384, + http2cipher_TLS_ECDH_RSA_WITH_ARIA_128_GCM_SHA256, + http2cipher_TLS_ECDH_RSA_WITH_ARIA_256_GCM_SHA384, + http2cipher_TLS_PSK_WITH_ARIA_128_CBC_SHA256, + http2cipher_TLS_PSK_WITH_ARIA_256_CBC_SHA384, + http2cipher_TLS_DHE_PSK_WITH_ARIA_128_CBC_SHA256, + http2cipher_TLS_DHE_PSK_WITH_ARIA_256_CBC_SHA384, + http2cipher_TLS_RSA_PSK_WITH_ARIA_128_CBC_SHA256, + http2cipher_TLS_RSA_PSK_WITH_ARIA_256_CBC_SHA384, + http2cipher_TLS_PSK_WITH_ARIA_128_GCM_SHA256, + http2cipher_TLS_PSK_WITH_ARIA_256_GCM_SHA384, + http2cipher_TLS_RSA_PSK_WITH_ARIA_128_GCM_SHA256, + http2cipher_TLS_RSA_PSK_WITH_ARIA_256_GCM_SHA384, + http2cipher_TLS_ECDHE_PSK_WITH_ARIA_128_CBC_SHA256, + http2cipher_TLS_ECDHE_PSK_WITH_ARIA_256_CBC_SHA384, + http2cipher_TLS_ECDHE_ECDSA_WITH_CAMELLIA_128_CBC_SHA256, + http2cipher_TLS_ECDHE_ECDSA_WITH_CAMELLIA_256_CBC_SHA384, + http2cipher_TLS_ECDH_ECDSA_WITH_CAMELLIA_128_CBC_SHA256, + http2cipher_TLS_ECDH_ECDSA_WITH_CAMELLIA_256_CBC_SHA384, + http2cipher_TLS_ECDHE_RSA_WITH_CAMELLIA_128_CBC_SHA256, + http2cipher_TLS_ECDHE_RSA_WITH_CAMELLIA_256_CBC_SHA384, + http2cipher_TLS_ECDH_RSA_WITH_CAMELLIA_128_CBC_SHA256, + http2cipher_TLS_ECDH_RSA_WITH_CAMELLIA_256_CBC_SHA384, + http2cipher_TLS_RSA_WITH_CAMELLIA_128_GCM_SHA256, + http2cipher_TLS_RSA_WITH_CAMELLIA_256_GCM_SHA384, + http2cipher_TLS_DH_RSA_WITH_CAMELLIA_128_GCM_SHA256, + http2cipher_TLS_DH_RSA_WITH_CAMELLIA_256_GCM_SHA384, + http2cipher_TLS_DH_DSS_WITH_CAMELLIA_128_GCM_SHA256, + http2cipher_TLS_DH_DSS_WITH_CAMELLIA_256_GCM_SHA384, + http2cipher_TLS_DH_anon_WITH_CAMELLIA_128_GCM_SHA256, + http2cipher_TLS_DH_anon_WITH_CAMELLIA_256_GCM_SHA384, + http2cipher_TLS_ECDH_ECDSA_WITH_CAMELLIA_128_GCM_SHA256, + http2cipher_TLS_ECDH_ECDSA_WITH_CAMELLIA_256_GCM_SHA384, + http2cipher_TLS_ECDH_RSA_WITH_CAMELLIA_128_GCM_SHA256, + http2cipher_TLS_ECDH_RSA_WITH_CAMELLIA_256_GCM_SHA384, + http2cipher_TLS_PSK_WITH_CAMELLIA_128_GCM_SHA256, + http2cipher_TLS_PSK_WITH_CAMELLIA_256_GCM_SHA384, + http2cipher_TLS_RSA_PSK_WITH_CAMELLIA_128_GCM_SHA256, + http2cipher_TLS_RSA_PSK_WITH_CAMELLIA_256_GCM_SHA384, + http2cipher_TLS_PSK_WITH_CAMELLIA_128_CBC_SHA256, + http2cipher_TLS_PSK_WITH_CAMELLIA_256_CBC_SHA384, + http2cipher_TLS_DHE_PSK_WITH_CAMELLIA_128_CBC_SHA256, + http2cipher_TLS_DHE_PSK_WITH_CAMELLIA_256_CBC_SHA384, + http2cipher_TLS_RSA_PSK_WITH_CAMELLIA_128_CBC_SHA256, + http2cipher_TLS_RSA_PSK_WITH_CAMELLIA_256_CBC_SHA384, + http2cipher_TLS_ECDHE_PSK_WITH_CAMELLIA_128_CBC_SHA256, + http2cipher_TLS_ECDHE_PSK_WITH_CAMELLIA_256_CBC_SHA384, + http2cipher_TLS_RSA_WITH_AES_128_CCM, + http2cipher_TLS_RSA_WITH_AES_256_CCM, + http2cipher_TLS_RSA_WITH_AES_128_CCM_8, + http2cipher_TLS_RSA_WITH_AES_256_CCM_8, + http2cipher_TLS_PSK_WITH_AES_128_CCM, + http2cipher_TLS_PSK_WITH_AES_256_CCM, + http2cipher_TLS_PSK_WITH_AES_128_CCM_8, + http2cipher_TLS_PSK_WITH_AES_256_CCM_8: + return true + default: + return false + } +} + // ClientConnPool manages a pool of HTTP/2 client connections. type http2ClientConnPool interface { GetClientConn(req *Request, addr string) (*http2ClientConn, error) @@ -126,7 +762,7 @@ type http2dialCall struct { // requires p.mu is held. func (p *http2clientConnPool) getStartDialLocked(addr string) *http2dialCall { if call, ok := p.dialing[addr]; ok { - + // A dial is already in-flight. Don't start another. return call } call := &http2dialCall{p: p, done: make(chan struct{})} @@ -254,7 +890,12 @@ func (p *http2clientConnPool) MarkDead(cc *http2ClientConn) { func (p *http2clientConnPool) closeIdleConnections() { p.mu.Lock() defer p.mu.Unlock() - + // TODO: don't close a cc if it was just added to the pool + // milliseconds ago and has never been used. There's currently + // a small race window with the HTTP/1 Transport's integration + // where it can add an idle conn just before using it, and + // somebody else can concurrently call CloseIdleConns and + // break some caller's RoundTrip. for _, vv := range p.conns { for _, cc := range vv { cc.closeIfIdle() @@ -269,7 +910,8 @@ func http2filterOutClientConn(in []*http2ClientConn, exclude *http2ClientConn) [ out = append(out, v) } } - + // If we filtered it out, zero out the last item to prevent + // the GC from seeing it. if len(in) != len(out) { in[len(in)-1] = nil } @@ -277,7 +919,7 @@ func http2filterOutClientConn(in []*http2ClientConn, exclude *http2ClientConn) [ } // noDialClientConnPool is an implementation of http2.ClientConnPool -// which never dials. We let the HTTP/1.1 client dial and use its TLS +// which never dials. We let the HTTP/1.1 client dial and use its TLS // connection instead. type http2noDialClientConnPool struct{ *http2clientConnPool } @@ -310,7 +952,10 @@ func http2configureTransport(t1 *Transport) (*http2Transport, error) { go c.Close() return http2erringRoundTripper{err} } else if !used { - + // Turns out we don't need this c. + // For example, two goroutines made requests to the same host + // at the same time, both kicking off TCP dials. (since protocol + // was unknown) go c.Close() } return t2 @@ -326,7 +971,7 @@ func http2configureTransport(t1 *Transport) (*http2Transport, error) { } // registerHTTPSProtocol calls Transport.RegisterProtocol but -// convering panics into errors. +// converting panics into errors. func http2registerHTTPSProtocol(t *Transport, rt RoundTripper) (err error) { defer func() { if e := recover(); e != nil { @@ -349,6 +994,141 @@ func (rt http2noDialH2RoundTripper) RoundTrip(req *Request) (*Response, error) { return res, err } +// Buffer chunks are allocated from a pool to reduce pressure on GC. +// The maximum wasted space per dataBuffer is 2x the largest size class, +// which happens when the dataBuffer has multiple chunks and there is +// one unread byte in both the first and last chunks. We use a few size +// classes to minimize overheads for servers that typically receive very +// small request bodies. +// +// TODO: Benchmark to determine if the pools are necessary. The GC may have +// improved enough that we can instead allocate chunks like this: +// make([]byte, max(16<<10, expectedBytesRemaining)) +var ( + http2dataChunkSizeClasses = []int{ + 1 << 10, + 2 << 10, + 4 << 10, + 8 << 10, + 16 << 10, + } + http2dataChunkPools = [...]sync.Pool{ + {New: func() interface{} { return make([]byte, 1<<10) }}, + {New: func() interface{} { return make([]byte, 2<<10) }}, + {New: func() interface{} { return make([]byte, 4<<10) }}, + {New: func() interface{} { return make([]byte, 8<<10) }}, + {New: func() interface{} { return make([]byte, 16<<10) }}, + } +) + +func http2getDataBufferChunk(size int64) []byte { + i := 0 + for ; i < len(http2dataChunkSizeClasses)-1; i++ { + if size <= int64(http2dataChunkSizeClasses[i]) { + break + } + } + return http2dataChunkPools[i].Get().([]byte) +} + +func http2putDataBufferChunk(p []byte) { + for i, n := range http2dataChunkSizeClasses { + if len(p) == n { + http2dataChunkPools[i].Put(p) + return + } + } + panic(fmt.Sprintf("unexpected buffer len=%v", len(p))) +} + +// dataBuffer is an io.ReadWriter backed by a list of data chunks. +// Each dataBuffer is used to read DATA frames on a single stream. +// The buffer is divided into chunks so the server can limit the +// total memory used by a single connection without limiting the +// request body size on any single stream. +type http2dataBuffer struct { + chunks [][]byte + r int // next byte to read is chunks[0][r] + w int // next byte to write is chunks[len(chunks)-1][w] + size int // total buffered bytes + expected int64 // we expect at least this many bytes in future Write calls (ignored if <= 0) +} + +var http2errReadEmpty = errors.New("read from empty dataBuffer") + +// Read copies bytes from the buffer into p. +// It is an error to read when no data is available. +func (b *http2dataBuffer) Read(p []byte) (int, error) { + if b.size == 0 { + return 0, http2errReadEmpty + } + var ntotal int + for len(p) > 0 && b.size > 0 { + readFrom := b.bytesFromFirstChunk() + n := copy(p, readFrom) + p = p[n:] + ntotal += n + b.r += n + b.size -= n + // If the first chunk has been consumed, advance to the next chunk. + if b.r == len(b.chunks[0]) { + http2putDataBufferChunk(b.chunks[0]) + end := len(b.chunks) - 1 + copy(b.chunks[:end], b.chunks[1:]) + b.chunks[end] = nil + b.chunks = b.chunks[:end] + b.r = 0 + } + } + return ntotal, nil +} + +func (b *http2dataBuffer) bytesFromFirstChunk() []byte { + if len(b.chunks) == 1 { + return b.chunks[0][b.r:b.w] + } + return b.chunks[0][b.r:] +} + +// Len returns the number of bytes of the unread portion of the buffer. +func (b *http2dataBuffer) Len() int { + return b.size +} + +// Write appends p to the buffer. +func (b *http2dataBuffer) Write(p []byte) (int, error) { + ntotal := len(p) + for len(p) > 0 { + // If the last chunk is empty, allocate a new chunk. Try to allocate + // enough to fully copy p plus any additional bytes we expect to + // receive. However, this may allocate less than len(p). + want := int64(len(p)) + if b.expected > want { + want = b.expected + } + chunk := b.lastChunkOrAlloc(want) + n := copy(chunk[b.w:], p) + p = p[n:] + b.w += n + b.size += n + b.expected -= int64(n) + } + return ntotal, nil +} + +func (b *http2dataBuffer) lastChunkOrAlloc(want int64) []byte { + if len(b.chunks) != 0 { + last := b.chunks[len(b.chunks)-1] + if b.w < len(last) { + return last + } + } + chunk := http2getDataBufferChunk(want) + b.chunks = append(b.chunks, chunk) + b.w = 0 + return chunk +} + // An ErrCode is an unsigned 32-bit error code as defined in the HTTP/2 spec. type http2ErrCode uint32 @@ -429,11 +1209,16 @@ type http2goAwayFlowError struct{} func (http2goAwayFlowError) Error() string { return "connection exceeded flow control window size" } +// connError represents an HTTP/2 ConnectionError error code, along +// with a string (for debugging) explaining why. +// // Errors of this type are only returned by the frame parser functions -// and converted into ConnectionError(ErrCodeProtocol). +// and converted into ConnectionError(Code), after stashing away +// the Reason into the Framer's errDetail field, accessible via +// the (*Framer).ErrorDetail method. type http2connError struct { - Code http2ErrCode - Reason string + Code http2ErrCode // the ConnectionError error code + Reason string // additional reason } func (e http2connError) Error() string { @@ -469,56 +1254,6 @@ var ( http2errPseudoAfterRegular = errors.New("pseudo header field after regular") ) -// fixedBuffer is an io.ReadWriter backed by a fixed size buffer. -// It never allocates, but moves old data as new data is written. -type http2fixedBuffer struct { - buf []byte - r, w int -} - -var ( - http2errReadEmpty = errors.New("read from empty fixedBuffer") - http2errWriteFull = errors.New("write on full fixedBuffer") -) - -// Read copies bytes from the buffer into p. -// It is an error to read when no data is available. -func (b *http2fixedBuffer) Read(p []byte) (n int, err error) { - if b.r == b.w { - return 0, http2errReadEmpty - } - n = copy(p, b.buf[b.r:b.w]) - b.r += n - if b.r == b.w { - b.r = 0 - b.w = 0 - } - return n, nil -} - -// Len returns the number of bytes of the unread portion of the buffer. -func (b *http2fixedBuffer) Len() int { - return b.w - b.r -} - -// Write copies bytes from p into the buffer. -// It is an error to write more data than the buffer can hold. -func (b *http2fixedBuffer) Write(p []byte) (n int, err error) { - - if b.r > 0 && len(p) > len(b.buf)-b.w { - copy(b.buf, b.buf[b.r:b.w]) - b.w -= b.r - b.r = 0 - } - - n = copy(b.buf[b.w:], p) - b.w += n - if n < len(p) { - err = http2errWriteFull - } - return n, err -} - // flow is the flow control window's size. type http2flow struct { // n is the number of DATA bytes we're allowed to send. @@ -666,7 +1401,7 @@ var http2flagName = map[http2FrameType]map[http2Flags]string{ // a frameParser parses a frame given its FrameHeader and payload // bytes. The length of payload will always equal fh.Length (which // might be 0). -type http2frameParser func(fh http2FrameHeader, payload []byte) (http2Frame, error) +type http2frameParser func(fc *http2frameCache, fh http2FrameHeader, payload []byte) (http2Frame, error) var http2frameParsers = map[http2FrameType]http2frameParser{ http2FrameData: http2parseDataFrame, @@ -855,25 +1590,33 @@ type http2Framer struct { // If the limit is hit, MetaHeadersFrame.Truncated is set true. MaxHeaderListSize uint32 + // TODO: track which type of frame & with which flags was sent + // last. Then return an error (unless AllowIllegalWrites) if + // we're in the middle of a header block and a + // non-Continuation or Continuation on a different stream is + // attempted to be written. + logReads, logWrites bool debugFramer *http2Framer // only use for logging written writes debugFramerBuf *bytes.Buffer debugReadLoggerf func(string, ...interface{}) debugWriteLoggerf func(string, ...interface{}) + + frameCache *http2frameCache // nil if frames aren't reused (default) } func (fr *http2Framer) maxHeaderListSize() uint32 { if fr.MaxHeaderListSize == 0 { - return 16 << 20 + return 16 << 20 // sane default, per docs } return fr.MaxHeaderListSize } func (f *http2Framer) startWrite(ftype http2FrameType, flags http2Flags, streamID uint32) { - + // Write the FrameHeader. f.wbuf = append(f.wbuf[:0], - 0, + 0, // 3 bytes of length, filled in in endWrite 0, 0, byte(ftype), @@ -885,7 +1628,8 @@ func (f *http2Framer) startWrite(ftype http2FrameType, flags http2Flags, streamI } func (f *http2Framer) endWrite() error { - + // Now that we know the final size, fill in the FrameHeader in + // the space previously reserved for it. Abuse append. length := len(f.wbuf) - http2frameHeaderLen if length >= (1 << 24) { return http2ErrFrameTooLarge @@ -909,8 +1653,9 @@ func (f *http2Framer) logWrite() { if f.debugFramer == nil { f.debugFramerBuf = new(bytes.Buffer) f.debugFramer = http2NewFramer(nil, f.debugFramerBuf) - f.debugFramer.logReads = false - + f.debugFramer.logReads = false // we log it ourselves, saying "wrote" below + // Let us read anything, even if we accidentally wrote it + // in the wrong order: f.debugFramer.AllowIllegalReads = true } f.debugFramerBuf.Write(f.wbuf) @@ -937,6 +1682,27 @@ const ( http2maxFrameSize = 1<<24 - 1 ) +// SetReuseFrames allows the Framer to reuse Frames. +// If called on a Framer, Frames returned by calls to ReadFrame are only +// valid until the next call to ReadFrame. +func (fr *http2Framer) SetReuseFrames() { + if fr.frameCache != nil { + return + } + fr.frameCache = &http2frameCache{} +} + +type http2frameCache struct { + dataFrame http2DataFrame +} + +func (fc *http2frameCache) getDataFrame() *http2DataFrame { + if fc == nil { + return &http2DataFrame{} + } + return &fc.dataFrame +} + // NewFramer returns a Framer that writes frames to w and reads them from r. func http2NewFramer(w io.Writer, r io.Reader) *http2Framer { fr := &http2Framer{ @@ -1016,7 +1782,7 @@ func (fr *http2Framer) ReadFrame() (http2Frame, error) { if _, err := io.ReadFull(fr.r, payload); err != nil { return nil, err } - f, err := http2typeFrameParser(fh.Type)(fh, payload) + f, err := http2typeFrameParser(fh.Type)(fr.frameCache, fh, payload) if err != nil { if ce, ok := err.(http2connError); ok { return nil, fr.connError(ce.Code, ce.Reason) @@ -1104,14 +1870,18 @@ func (f *http2DataFrame) Data() []byte { return f.data } -func http2parseDataFrame(fh http2FrameHeader, payload []byte) (http2Frame, error) { +func http2parseDataFrame(fc *http2frameCache, fh http2FrameHeader, payload []byte) (http2Frame, error) { if fh.StreamID == 0 { - + // DATA frames MUST be associated with a stream. If a + // DATA frame is received whose stream identifier + // field is 0x0, the recipient MUST respond with a + // connection error (Section 5.4.1) of type + // PROTOCOL_ERROR. return nil, http2connError{http2ErrCodeProtocol, "DATA frame with stream ID 0"} } - f := &http2DataFrame{ - http2FrameHeader: fh, - } + f := fc.getDataFrame() + f.http2FrameHeader = fh + var padSize byte if fh.Flags.Has(http2FlagDataPadded) { var err error @@ -1121,7 +1891,10 @@ func http2parseDataFrame(fh http2FrameHeader, payload []byte) (http2Frame, error } } if int(padSize) > len(payload) { - + // If the length of the padding is greater than the + // length of the frame payload, the recipient MUST + // treat this as a connection error. + // Filed: https://github.com/http2/http2-spec/issues/610 return nil, http2connError{http2ErrCodeProtocol, "pad size larger than data payload"} } f.data = payload[:len(payload)-int(padSize)] @@ -1132,6 +1905,7 @@ var ( http2errStreamID = errors.New("invalid stream ID") http2errDepStreamID = errors.New("invalid dependent stream ID") http2errPadLength = errors.New("pad length too large") + http2errPadBytes = errors.New("padding bytes must all be zeros unless AllowIllegalWrites is enabled") ) func http2validStreamIDOrZero(streamID uint32) bool { @@ -1155,6 +1929,7 @@ func (f *http2Framer) WriteData(streamID uint32, endStream bool, data []byte) er // // If pad is nil, the padding bit is not sent. // The length of pad must not exceed 255 bytes. +// The bytes of pad must all be zero, unless f.AllowIllegalWrites is set. // // It will perform exactly one Write to the underlying Writer. // It is the caller's responsibility not to violate the maximum frame size @@ -1163,8 +1938,18 @@ func (f *http2Framer) WriteDataPadded(streamID uint32, endStream bool, data, pad if !http2validStreamID(streamID) && !f.AllowIllegalWrites { return http2errStreamID } - if len(pad) > 255 { - return http2errPadLength + if len(pad) > 0 { + if len(pad) > 255 { + return http2errPadLength + } + if !f.AllowIllegalWrites { + for _, b := range pad { + if b != 0 { + // "Padding octets MUST be set to zero when sending." + return http2errPadBytes + } + } + } } var flags http2Flags if endStream { @@ -1192,22 +1977,35 @@ type http2SettingsFrame struct { p []byte } -func http2parseSettingsFrame(fh http2FrameHeader, p []byte) (http2Frame, error) { +func http2parseSettingsFrame(_ *http2frameCache, fh http2FrameHeader, p []byte) (http2Frame, error) { if fh.Flags.Has(http2FlagSettingsAck) && fh.Length > 0 { - + // When this (ACK 0x1) bit is set, the payload of the + // SETTINGS frame MUST be empty. Receipt of a + // SETTINGS frame with the ACK flag set and a length + // field value other than 0 MUST be treated as a + // connection error (Section 5.4.1) of type + // FRAME_SIZE_ERROR. return nil, http2ConnectionError(http2ErrCodeFrameSize) } if fh.StreamID != 0 { - + // SETTINGS frames always apply to a connection, + // never a single stream. The stream identifier for a + // SETTINGS frame MUST be zero (0x0). If an endpoint + // receives a SETTINGS frame whose stream identifier + // field is anything other than 0x0, the endpoint MUST + // respond with a connection error (Section 5.4.1) of + // type PROTOCOL_ERROR. return nil, http2ConnectionError(http2ErrCodeProtocol) } if len(p)%6 != 0 { - + // Expecting even number of 6 byte settings. return nil, http2ConnectionError(http2ErrCodeFrameSize) } f := &http2SettingsFrame{http2FrameHeader: fh, p: p} if v, ok := f.Value(http2SettingInitialWindowSize); ok && v > (1<<31)-1 { - + // Values above the maximum flow control window size of 2^31 - 1 MUST + // be treated as a connection error (Section 5.4.1) of type + // FLOW_CONTROL_ERROR. return nil, http2ConnectionError(http2ErrCodeFlowControl) } return f, nil @@ -1281,7 +2079,7 @@ type http2PingFrame struct { func (f *http2PingFrame) IsAck() bool { return f.Flags.Has(http2FlagPingAck) } -func http2parsePingFrame(fh http2FrameHeader, payload []byte) (http2Frame, error) { +func http2parsePingFrame(_ *http2frameCache, fh http2FrameHeader, payload []byte) (http2Frame, error) { if len(payload) != 8 { return nil, http2ConnectionError(http2ErrCodeFrameSize) } @@ -1321,7 +2119,7 @@ func (f *http2GoAwayFrame) DebugData() []byte { return f.debugData } -func http2parseGoAwayFrame(fh http2FrameHeader, p []byte) (http2Frame, error) { +func http2parseGoAwayFrame(_ *http2frameCache, fh http2FrameHeader, p []byte) (http2Frame, error) { if fh.StreamID != 0 { return nil, http2ConnectionError(http2ErrCodeProtocol) } @@ -1361,7 +2159,7 @@ func (f *http2UnknownFrame) Payload() []byte { return f.p } -func http2parseUnknownFrame(fh http2FrameHeader, p []byte) (http2Frame, error) { +func http2parseUnknownFrame(_ *http2frameCache, fh http2FrameHeader, p []byte) (http2Frame, error) { return &http2UnknownFrame{fh, p}, nil } @@ -1372,13 +2170,18 @@ type http2WindowUpdateFrame struct { Increment uint32 // never read with high bit set } -func http2parseWindowUpdateFrame(fh http2FrameHeader, p []byte) (http2Frame, error) { +func http2parseWindowUpdateFrame(_ *http2frameCache, fh http2FrameHeader, p []byte) (http2Frame, error) { if len(p) != 4 { return nil, http2ConnectionError(http2ErrCodeFrameSize) } - inc := binary.BigEndian.Uint32(p[:4]) & 0x7fffffff + inc := binary.BigEndian.Uint32(p[:4]) & 0x7fffffff // mask off high reserved bit if inc == 0 { - + // A receiver MUST treat the receipt of a + // WINDOW_UPDATE frame with an flow control window + // increment of 0 as a stream error (Section 5.4.2) of + // type PROTOCOL_ERROR; errors on the connection flow + // control window MUST be treated as a connection + // error (Section 5.4.1). if fh.StreamID == 0 { return nil, http2ConnectionError(http2ErrCodeProtocol) } @@ -1395,7 +2198,7 @@ func http2parseWindowUpdateFrame(fh http2FrameHeader, p []byte) (http2Frame, err // If the Stream ID is zero, the window update applies to the // connection as a whole. func (f *http2Framer) WriteWindowUpdate(streamID, incr uint32) error { - + // "The legal range for the increment to the flow control window is 1 to 2^31-1 (2,147,483,647) octets." if (incr < 1 || incr > 2147483647) && !f.AllowIllegalWrites { return errors.New("illegal window increment value") } @@ -1432,12 +2235,15 @@ func (f *http2HeadersFrame) HasPriority() bool { return f.http2FrameHeader.Flags.Has(http2FlagHeadersPriority) } -func http2parseHeadersFrame(fh http2FrameHeader, p []byte) (_ http2Frame, err error) { +func http2parseHeadersFrame(_ *http2frameCache, fh http2FrameHeader, p []byte) (_ http2Frame, err error) { hf := &http2HeadersFrame{ http2FrameHeader: fh, } if fh.StreamID == 0 { - + // HEADERS frames MUST be associated with a stream. If a HEADERS frame + // is received whose stream identifier field is 0x0, the recipient MUST + // respond with a connection error (Section 5.4.1) of type + // PROTOCOL_ERROR. return nil, http2connError{http2ErrCodeProtocol, "HEADERS frame with stream ID 0"} } var padLength uint8 @@ -1453,7 +2259,7 @@ func http2parseHeadersFrame(fh http2FrameHeader, p []byte) (_ http2Frame, err er return nil, err } hf.Priority.StreamDep = v & 0x7fffffff - hf.Priority.Exclusive = (v != hf.Priority.StreamDep) + hf.Priority.Exclusive = (v != hf.Priority.StreamDep) // high bit was set p, hf.Priority.Weight, err = http2readByte(p) if err != nil { return nil, err @@ -1556,7 +2362,7 @@ type http2PriorityParam struct { Exclusive bool // Weight is the stream's zero-indexed weight. It should be - // set together with StreamDep, or neither should be set. Per + // set together with StreamDep, or neither should be set. Per // the spec, "Add one to the value to obtain a weight between // 1 and 256." Weight uint8 @@ -1566,7 +2372,7 @@ func (p http2PriorityParam) IsZero() bool { return p == http2PriorityParam{} } -func http2parsePriorityFrame(fh http2FrameHeader, payload []byte) (http2Frame, error) { +func http2parsePriorityFrame(_ *http2frameCache, fh http2FrameHeader, payload []byte) (http2Frame, error) { if fh.StreamID == 0 { return nil, http2connError{http2ErrCodeProtocol, "PRIORITY frame with stream ID 0"} } @@ -1574,13 +2380,13 @@ func http2parsePriorityFrame(fh http2FrameHeader, payload []byte) (http2Frame, e return nil, http2connError{http2ErrCodeFrameSize, fmt.Sprintf("PRIORITY frame payload size was %d; want 5", len(payload))} } v := binary.BigEndian.Uint32(payload[:4]) - streamID := v & 0x7fffffff + streamID := v & 0x7fffffff // mask off high bit return &http2PriorityFrame{ http2FrameHeader: fh, http2PriorityParam: http2PriorityParam{ Weight: payload[4], StreamDep: streamID, - Exclusive: streamID != v, + Exclusive: streamID != v, // was high bit set? }, }, nil } @@ -1613,7 +2419,7 @@ type http2RSTStreamFrame struct { ErrCode http2ErrCode } -func http2parseRSTStreamFrame(fh http2FrameHeader, p []byte) (http2Frame, error) { +func http2parseRSTStreamFrame(_ *http2frameCache, fh http2FrameHeader, p []byte) (http2Frame, error) { if len(p) != 4 { return nil, http2ConnectionError(http2ErrCodeFrameSize) } @@ -1643,7 +2449,7 @@ type http2ContinuationFrame struct { headerFragBuf []byte } -func http2parseContinuationFrame(fh http2FrameHeader, p []byte) (http2Frame, error) { +func http2parseContinuationFrame(_ *http2frameCache, fh http2FrameHeader, p []byte) (http2Frame, error) { if fh.StreamID == 0 { return nil, http2connError{http2ErrCodeProtocol, "CONTINUATION frame with stream ID 0"} } @@ -1693,12 +2499,17 @@ func (f *http2PushPromiseFrame) HeadersEnded() bool { return f.http2FrameHeader.Flags.Has(http2FlagPushPromiseEndHeaders) } -func http2parsePushPromise(fh http2FrameHeader, p []byte) (_ http2Frame, err error) { +func http2parsePushPromise(_ *http2frameCache, fh http2FrameHeader, p []byte) (_ http2Frame, err error) { pp := &http2PushPromiseFrame{ http2FrameHeader: fh, } if pp.StreamID == 0 { - + // PUSH_PROMISE frames MUST be associated with an existing, + // peer-initiated stream. The stream identifier of a + // PUSH_PROMISE frame indicates the stream it is associated + // with. If the stream identifier field specifies the value + // 0x0, a recipient MUST respond with a connection error + // (Section 5.4.1) of type PROTOCOL_ERROR. return nil, http2ConnectionError(http2ErrCodeProtocol) } // The PUSH_PROMISE frame includes optional padding. @@ -1717,7 +2528,7 @@ func http2parsePushPromise(fh http2FrameHeader, p []byte) (_ http2Frame, err err pp.PromiseID = pp.PromiseID & (1<<31 - 1) if int(padLength) > len(p) { - + // like the DATA frame, error out if padding is longer than the body. return nil, http2ConnectionError(http2ErrCodeProtocol) } pp.headerFragBuf = p[:len(p)-int(padLength)] @@ -1887,7 +2698,9 @@ func (mh *http2MetaHeadersFrame) checkPseudos() error { default: return http2pseudoHeaderError(hf.Name) } - + // Check for duplicates. + // This would be a bad algorithm, but N is 4. + // And this doesn't allocate. for _, hf2 := range pf[:i] { if hf.Name == hf2.Name { return http2duplicatePseudoHeaderError(hf.Name) @@ -1905,7 +2718,8 @@ func (fr *http2Framer) maxHeaderStringLen() int { if uint32(int(v)) == v { return int(v) } - + // They had a crazy big number for MaxHeaderBytes anyway, + // so give them unlimited header lengths: return 0 } @@ -1960,7 +2774,7 @@ func (fr *http2Framer) readMetaFrame(hf *http2HeadersFrame) (*http2MetaHeadersFr mh.Fields = append(mh.Fields, hf) }) - + // Lose reference to MetaHeadersFrame: defer hdec.SetEmitFunc(func(hf hpack.HeaderField) {}) var hc http2headersOrContinuation = hf @@ -1976,7 +2790,7 @@ func (fr *http2Framer) readMetaFrame(hf *http2HeadersFrame) (*http2MetaHeadersFr if f, err := fr.ReadFrame(); err != nil { return nil, err } else { - hc = f.(*http2ContinuationFrame) + hc = f.(*http2ContinuationFrame) // guaranteed by checkFrameOrder } } @@ -2018,7 +2832,7 @@ func http2summarizeFrame(f http2Frame) string { return nil }) if n > 0 { - buf.Truncate(buf.Len() - 1) + buf.Truncate(buf.Len() - 1) // remove trailing comma } case *http2DataFrame: data := f.Data() @@ -2050,29 +2864,6 @@ func http2transportExpectContinueTimeout(t1 *Transport) time.Duration { return t1.ExpectContinueTimeout } -// isBadCipher reports whether the cipher is blacklisted by the HTTP/2 spec. -func http2isBadCipher(cipher uint16) bool { - switch cipher { - case tls.TLS_RSA_WITH_RC4_128_SHA, - tls.TLS_RSA_WITH_3DES_EDE_CBC_SHA, - tls.TLS_RSA_WITH_AES_128_CBC_SHA, - tls.TLS_RSA_WITH_AES_256_CBC_SHA, - tls.TLS_RSA_WITH_AES_128_GCM_SHA256, - tls.TLS_RSA_WITH_AES_256_GCM_SHA384, - tls.TLS_ECDHE_ECDSA_WITH_RC4_128_SHA, - tls.TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA, - tls.TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA, - tls.TLS_ECDHE_RSA_WITH_RC4_128_SHA, - tls.TLS_ECDHE_RSA_WITH_3DES_EDE_CBC_SHA, - tls.TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA, - tls.TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA: - - return true - default: - return false - } -} - type http2contextContext interface { context.Context } @@ -2164,7 +2955,11 @@ func (cc *http2ClientConn) Ping(ctx context.Context) error { return cc.ping(ctx) } -func http2cloneTLSConfig(c *tls.Config) *tls.Config { return c.Clone() } +func http2cloneTLSConfig(c *tls.Config) *tls.Config { + c2 := c.Clone() + c2.GetClientCertificate = c.GetClientCertificate // golang.org/issue/19264 + return c2 +} var _ Pusher = (*http2responseWriter)(nil) @@ -2201,6 +2996,13 @@ func http2reqBodyIsNoBody(body io.ReadCloser) bool { return body == NoBody } +func http2go18httpNoBody() io.ReadCloser { return NoBody } // for tests only + +func http2configureServer19(s *Server, conf *http2Server) error { + s.RegisterOnShutdown(conf.state.startGracefulShutdown) + return nil +} + var http2DebugGoroutines = os.Getenv("DEBUG_HTTP2_GOROUTINES") == "1" type http2goroutineLock uint64 @@ -2237,7 +3039,7 @@ func http2curGoroutineID() uint64 { defer http2littleBuf.Put(bp) b := *bp b = b[:runtime.Stack(b, false)] - + // Parse the 4707 out of "goroutine 4707 [" b = bytes.TrimPrefix(b, http2goroutineSpace) i := bytes.IndexByte(b, ' ') if i < 0 { @@ -2273,9 +3075,10 @@ func http2parseUintBytes(s []byte, base int, bitSize int) (n uint64, err error) goto Error case 2 <= base && base <= 36: + // valid base; nothing to do case base == 0: - + // Look for octal, hex prefix. switch { case s[0] == '0' && len(s) > 1 && (s[1] == 'x' || s[1] == 'X'): base = 16 @@ -2321,7 +3124,7 @@ func http2parseUintBytes(s []byte, base int, bitSize int) (n uint64, err error) } if n >= cutoff { - + // n*base overflows n = 1<<64 - 1 err = strconv.ErrRange goto Error @@ -2330,7 +3133,7 @@ func http2parseUintBytes(s []byte, base int, bitSize int) (n uint64, err error) n1 := n + uint64(v) if n1 < n || n1 > maxVal { - + // n+v overflows n = 1<<64 - 1 err = strconv.ErrRange goto Error @@ -2514,7 +3317,7 @@ func (s http2Setting) String() string { // Valid reports whether the setting is valid. func (s http2Setting) Valid() error { - + // Limits and error codes from 6.5.2 Defined SETTINGS Parameters switch s.ID { case http2SettingEnablePush: if s.Val != 1 && s.Val != 0 { @@ -2758,7 +3561,8 @@ func (s *http2sorter) Keys(h Header) []string { } func (s *http2sorter) SortStrings(ss []string) { - + // Our sorter works on s.v, which sorter owns, so + // stash it away while we sort the user's buffer. save := s.v s.v = ss sort.Sort(s) @@ -2768,27 +3572,31 @@ func (s *http2sorter) SortStrings(ss []string) { // validPseudoPath reports whether v is a valid :path pseudo-header // value. It must be either: // -// *) a non-empty string starting with '/', but not with with "//", +// *) a non-empty string starting with '/' // *) the string '*', for OPTIONS requests. // // For now this is only used a quick check for deciding when to clean // up Opaque URLs before sending requests from the Transport. // See golang.org/issue/16847 +// +// We used to enforce that the path also didn't start with "//", but +// Google's GFE accepts such paths and Chrome sends them, so ignore +// that part of the spec. See golang.org/issue/19103. func http2validPseudoPath(v string) bool { - return (len(v) > 0 && v[0] == '/' && (len(v) == 1 || v[1] != '/')) || v == "*" + return (len(v) > 0 && v[0] == '/') || v == "*" } -// pipe is a goroutine-safe io.Reader/io.Writer pair. It's like +// pipe is a goroutine-safe io.Reader/io.Writer pair. It's like // io.Pipe except there are no PipeReader/PipeWriter halves, and the // underlying buffer is an interface. (io.Pipe is always unbuffered) type http2pipe struct { mu sync.Mutex - c sync.Cond // c.L lazily initialized to &p.mu - b http2pipeBuffer - err error // read error once empty. non-nil means closed. - breakErr error // immediate read error (caller doesn't see rest of b) - donec chan struct{} // closed on error - readFn func() // optional code to run in Read before error + c sync.Cond // c.L lazily initialized to &p.mu + b http2pipeBuffer // nil when done reading + err error // read error once empty. non-nil means closed. + breakErr error // immediate read error (caller doesn't see rest of b) + donec chan struct{} // closed on error + readFn func() // optional code to run in Read before error } type http2pipeBuffer interface { @@ -2800,6 +3608,9 @@ type http2pipeBuffer interface { func (p *http2pipe) Len() int { p.mu.Lock() defer p.mu.Unlock() + if p.b == nil { + return 0 + } return p.b.Len() } @@ -2815,14 +3626,15 @@ func (p *http2pipe) Read(d []byte) (n int, err error) { if p.breakErr != nil { return 0, p.breakErr } - if p.b.Len() > 0 { + if p.b != nil && p.b.Len() > 0 { return p.b.Read(d) } if p.err != nil { if p.readFn != nil { - p.readFn() - p.readFn = nil + p.readFn() // e.g. copy trailers + p.readFn = nil // not sticky like p.err } + p.b = nil return 0, p.err } p.c.Wait() @@ -2843,6 +3655,9 @@ func (p *http2pipe) Write(d []byte) (n int, err error) { if p.err != nil { return 0, http2errClosedPipeWrite } + if p.breakErr != nil { + return len(d), nil // discard when there is no reader + } return p.b.Write(d) } @@ -2873,10 +3688,13 @@ func (p *http2pipe) closeWithError(dst *error, err error, fn func()) { } defer p.c.Signal() if *dst != nil { - + // Already been done. return } p.readFn = fn + if dst == &p.breakErr { + p.b = nil + } *dst = err p.closeDoneLocked() } @@ -2886,7 +3704,8 @@ func (p *http2pipe) closeDoneLocked() { if p.donec == nil { return } - + // Close if unclosed. This isn't racy since we always + // hold p.mu while closing. select { case <-p.donec: default: @@ -2912,7 +3731,7 @@ func (p *http2pipe) Done() <-chan struct{} { if p.donec == nil { p.donec = make(chan struct{}) if p.err != nil || p.breakErr != nil { - + // Already hit an error. p.closeDoneLocked() } } @@ -2980,9 +3799,41 @@ type http2Server struct { // activity for the purposes of IdleTimeout. IdleTimeout time.Duration + // MaxUploadBufferPerConnection is the size of the initial flow + // control window for each connections. The HTTP/2 spec does not + // allow this to be smaller than 65535 or larger than 2^32-1. + // If the value is outside this range, a default value will be + // used instead. + MaxUploadBufferPerConnection int32 + + // MaxUploadBufferPerStream is the size of the initial flow control + // window for each stream. The HTTP/2 spec does not allow this to + // be larger than 2^32-1. If the value is zero or larger than the + // maximum, a default value will be used instead. + MaxUploadBufferPerStream int32 + // NewWriteScheduler constructs a write scheduler for a connection. // If nil, a default scheduler is chosen. NewWriteScheduler func() http2WriteScheduler + + // Internal state. This is a pointer (rather than embedded directly) + // so that we don't embed a Mutex in this struct, which will make the + // struct non-copyable, which might break some callers. + state *http2serverInternalState +} + +func (s *http2Server) initialConnRecvWindowSize() int32 { + if s.MaxUploadBufferPerConnection > http2initialWindowSize { + return s.MaxUploadBufferPerConnection + } + return 1 << 20 +} + +func (s *http2Server) initialStreamRecvWindowSize() int32 { + if s.MaxUploadBufferPerStream > 0 { + return s.MaxUploadBufferPerStream + } + return 1 << 20 } func (s *http2Server) maxReadFrameSize() uint32 { @@ -2999,6 +3850,40 @@ func (s *http2Server) maxConcurrentStreams() uint32 { return http2defaultMaxStreams } +type http2serverInternalState struct { + mu sync.Mutex + activeConns map[*http2serverConn]struct{} +} + +func (s *http2serverInternalState) registerConn(sc *http2serverConn) { + if s == nil { + return // if the Server was used without calling ConfigureServer + } + s.mu.Lock() + s.activeConns[sc] = struct{}{} + s.mu.Unlock() +} + +func (s *http2serverInternalState) unregisterConn(sc *http2serverConn) { + if s == nil { + return // if the Server was used without calling ConfigureServer + } + s.mu.Lock() + delete(s.activeConns, sc) + s.mu.Unlock() +} + +func (s *http2serverInternalState) startGracefulShutdown() { + if s == nil { + return // if the Server was used without calling ConfigureServer + } + s.mu.Lock() + for sc := range s.activeConns { + sc.startGracefulShutdown() + } + s.mu.Unlock() +} + // ConfigureServer adds HTTP/2 support to a net/http Server. // // The configuration conf may be nil. @@ -3011,9 +3896,13 @@ func http2ConfigureServer(s *Server, conf *http2Server) error { if conf == nil { conf = new(http2Server) } + conf.state = &http2serverInternalState{activeConns: make(map[*http2serverConn]struct{})} if err := http2configureServer18(s, conf); err != nil { return err } + if err := http2configureServer19(s, conf); err != nil { + return err + } if s.TLSConfig == nil { s.TLSConfig = new(tls.Config) @@ -3039,6 +3928,13 @@ func http2ConfigureServer(s *Server, conf *http2Server) error { } } + // Note: not setting MinVersion to tls.VersionTLS12, + // as we don't want to interfere with HTTP/1.1 traffic + // on the user's server. We enforce TLS 1.2 later once + // we accept a connection. Ideally this should be done + // during next-proto selection, but using TLS <1.2 with + // HTTP/2 is still the client's bug. + s.TLSConfig.PreferServerCipherSuites = true haveNPN := false @@ -3118,29 +4014,37 @@ func (s *http2Server) ServeConn(c net.Conn, opts *http2ServeConnOpts) { defer cancel() sc := &http2serverConn{ - srv: s, - hs: opts.baseConfig(), - conn: c, - baseCtx: baseCtx, - remoteAddrStr: c.RemoteAddr().String(), - bw: http2newBufferedWriter(c), - handler: opts.handler(), - streams: make(map[uint32]*http2stream), - readFrameCh: make(chan http2readFrameResult), - wantWriteFrameCh: make(chan http2FrameWriteRequest, 8), - wantStartPushCh: make(chan http2startPushRequest, 8), - wroteFrameCh: make(chan http2frameWriteResult, 1), - bodyReadCh: make(chan http2bodyReadMsg), - doneServing: make(chan struct{}), - clientMaxStreams: math.MaxUint32, - advMaxStreams: s.maxConcurrentStreams(), - initialWindowSize: http2initialWindowSize, - maxFrameSize: http2initialMaxFrameSize, - headerTableSize: http2initialHeaderTableSize, - serveG: http2newGoroutineLock(), - pushEnabled: true, - } - + srv: s, + hs: opts.baseConfig(), + conn: c, + baseCtx: baseCtx, + remoteAddrStr: c.RemoteAddr().String(), + bw: http2newBufferedWriter(c), + handler: opts.handler(), + streams: make(map[uint32]*http2stream), + readFrameCh: make(chan http2readFrameResult), + wantWriteFrameCh: make(chan http2FrameWriteRequest, 8), + serveMsgCh: make(chan interface{}, 8), + wroteFrameCh: make(chan http2frameWriteResult, 1), // buffered; one send in writeFrameAsync + bodyReadCh: make(chan http2bodyReadMsg), // buffering doesn't matter either way + doneServing: make(chan struct{}), + clientMaxStreams: math.MaxUint32, // Section 6.5.2: "Initially, there is no limit to this value" + advMaxStreams: s.maxConcurrentStreams(), + initialStreamSendWindowSize: http2initialWindowSize, + maxFrameSize: http2initialMaxFrameSize, + headerTableSize: http2initialHeaderTableSize, + serveG: http2newGoroutineLock(), + pushEnabled: true, + } + + s.state.registerConn(sc) + defer s.state.unregisterConn(sc) + + // The net/http package sets the write deadline from the + // http.Server.WriteTimeout during the TLS handshake, but then + // passes the connection off to us with the deadline already set. + // Write deadlines are set per stream in serverConn.newStream. + // Disarm the net.Conn write deadline here. if sc.hs.WriteTimeout != 0 { sc.conn.SetWriteDeadline(time.Time{}) } @@ -3151,6 +4055,9 @@ func (s *http2Server) ServeConn(c net.Conn, opts *http2ServeConnOpts) { sc.writeSched = http2NewRandomWriteScheduler() } + // These start at the RFC-specified defaults. If there is a higher + // configured value for inflow, that will be updated when we send a + // WINDOW_UPDATE shortly after sending SETTINGS. sc.flow.add(http2initialWindowSize) sc.inflow.add(http2initialWindowSize) sc.hpackEncoder = hpack.NewEncoder(&sc.headerWriteBuf) @@ -3164,18 +4071,44 @@ func (s *http2Server) ServeConn(c net.Conn, opts *http2ServeConnOpts) { if tc, ok := c.(http2connectionStater); ok { sc.tlsState = new(tls.ConnectionState) *sc.tlsState = tc.ConnectionState() - + // 9.2 Use of TLS Features + // An implementation of HTTP/2 over TLS MUST use TLS + // 1.2 or higher with the restrictions on feature set + // and cipher suite described in this section. Due to + // implementation limitations, it might not be + // possible to fail TLS negotiation. An endpoint MUST + // immediately terminate an HTTP/2 connection that + // does not meet the TLS requirements described in + // this section with a connection error (Section + // 5.4.1) of type INADEQUATE_SECURITY. if sc.tlsState.Version < tls.VersionTLS12 { sc.rejectConn(http2ErrCodeInadequateSecurity, "TLS version too low") return } if sc.tlsState.ServerName == "" { - + // Client must use SNI, but we don't enforce that anymore, + // since it was causing problems when connecting to bare IP + // addresses during development. + // + // TODO: optionally enforce? Or enforce at the time we receive + // a new request, and verify the the ServerName matches the :authority? + // But that precludes proxy situations, perhaps. + // + // So for now, do nothing here again. } if !s.PermitProhibitedCipherSuites && http2isBadCipher(sc.tlsState.CipherSuite) { - + // "Endpoints MAY choose to generate a connection error + // (Section 5.4.1) of type INADEQUATE_SECURITY if one of + // the prohibited cipher suites are negotiated." + // + // We choose that. In my opinion, the spec is weak + // here. It also says both parties must support at least + // TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 so there's no + // excuses here. If we really must, we could allow an + // "AllowInsecureWeakCiphers" option on the server later. + // Let's see how it plays out first. sc.rejectConn(http2ErrCodeInadequateSecurity, fmt.Sprintf("Prohibited TLS 1.2 Cipher Suite: %x", sc.tlsState.CipherSuite)) return } @@ -3189,7 +4122,7 @@ func (s *http2Server) ServeConn(c net.Conn, opts *http2ServeConnOpts) { func (sc *http2serverConn) rejectConn(err http2ErrCode, debug string) { sc.vlogf("http2: server rejecting conn: %v, %s", err, debug) - + // ignoring errors. hanging up anyway. sc.framer.WriteGoAway(0, err, []byte(debug)) sc.bw.Flush() sc.conn.Close() @@ -3207,10 +4140,9 @@ type http2serverConn struct { doneServing chan struct{} // closed when serverConn.serve ends readFrameCh chan http2readFrameResult // written by serverConn.readFrames wantWriteFrameCh chan http2FrameWriteRequest // from handlers -> serve - wantStartPushCh chan http2startPushRequest // from handlers -> serve wroteFrameCh chan http2frameWriteResult // from writeFrameAsync -> serve, tickles more frame writes bodyReadCh chan http2bodyReadMsg // from handlers -> serve - testHookCh chan func(int) // code to run on the serve loop + serveMsgCh chan interface{} // misc messages & code to send to / run on the serve loop flow http2flow // conn-wide (not stream-specific) outbound flow control inflow http2flow // conn-wide inbound flow control tlsState *tls.ConnectionState // shared by all handlers, like net/http @@ -3218,38 +4150,39 @@ type http2serverConn struct { writeSched http2WriteScheduler // Everything following is owned by the serve loop; use serveG.check(): - serveG http2goroutineLock // used to verify funcs are on serve() - pushEnabled bool - sawFirstSettings bool // got the initial SETTINGS frame after the preface - needToSendSettingsAck bool - unackedSettings int // how many SETTINGS have we sent without ACKs? - clientMaxStreams uint32 // SETTINGS_MAX_CONCURRENT_STREAMS from client (our PUSH_PROMISE limit) - advMaxStreams uint32 // our SETTINGS_MAX_CONCURRENT_STREAMS advertised the client - curClientStreams uint32 // number of open streams initiated by the client - curPushedStreams uint32 // number of open streams initiated by server push - maxClientStreamID uint32 // max ever seen from client (odd), or 0 if there have been no client requests - maxPushPromiseID uint32 // ID of the last push promise (even), or 0 if there have been no pushes - streams map[uint32]*http2stream - initialWindowSize int32 - maxFrameSize int32 - headerTableSize uint32 - peerMaxHeaderListSize uint32 // zero means unknown (default) - canonHeader map[string]string // http2-lower-case -> Go-Canonical-Case - writingFrame bool // started writing a frame (on serve goroutine or separate) - writingFrameAsync bool // started a frame on its own goroutine but haven't heard back on wroteFrameCh - needsFrameFlush bool // last frame write wasn't a flush - inGoAway bool // we've started to or sent GOAWAY - inFrameScheduleLoop bool // whether we're in the scheduleFrameWrite loop - needToSendGoAway bool // we need to schedule a GOAWAY frame write - goAwayCode http2ErrCode - shutdownTimerCh <-chan time.Time // nil until used - shutdownTimer *time.Timer // nil until used - idleTimer *time.Timer // nil if unused - idleTimerCh <-chan time.Time // nil if unused + serveG http2goroutineLock // used to verify funcs are on serve() + pushEnabled bool + sawFirstSettings bool // got the initial SETTINGS frame after the preface + needToSendSettingsAck bool + unackedSettings int // how many SETTINGS have we sent without ACKs? + clientMaxStreams uint32 // SETTINGS_MAX_CONCURRENT_STREAMS from client (our PUSH_PROMISE limit) + advMaxStreams uint32 // our SETTINGS_MAX_CONCURRENT_STREAMS advertised the client + curClientStreams uint32 // number of open streams initiated by the client + curPushedStreams uint32 // number of open streams initiated by server push + maxClientStreamID uint32 // max ever seen from client (odd), or 0 if there have been no client requests + maxPushPromiseID uint32 // ID of the last push promise (even), or 0 if there have been no pushes + streams map[uint32]*http2stream + initialStreamSendWindowSize int32 + maxFrameSize int32 + headerTableSize uint32 + peerMaxHeaderListSize uint32 // zero means unknown (default) + canonHeader map[string]string // http2-lower-case -> Go-Canonical-Case + writingFrame bool // started writing a frame (on serve goroutine or separate) + writingFrameAsync bool // started a frame on its own goroutine but haven't heard back on wroteFrameCh + needsFrameFlush bool // last frame write wasn't a flush + inGoAway bool // we've started to or sent GOAWAY + inFrameScheduleLoop bool // whether we're in the scheduleFrameWrite loop + needToSendGoAway bool // we need to schedule a GOAWAY frame write + goAwayCode http2ErrCode + shutdownTimer *time.Timer // nil until used + idleTimer *time.Timer // nil if unused // Owned by the writeFrameAsync goroutine: headerWriteBuf bytes.Buffer hpackEncoder *hpack.Encoder + + // Used by startGracefulShutdown. + shutdownOnce sync.Once } func (sc *http2serverConn) maxHeaderListSize() uint32 { @@ -3294,10 +4227,10 @@ type http2stream struct { numTrailerValues int64 weight uint8 state http2streamState - resetQueued bool // RST_STREAM queued for write; set by sc.resetStream - gotTrailerHeader bool // HEADER frame for trailers was seen - wroteHeaders bool // whether we wrote headers (not status 100) - reqBuf []byte // if non-nil, body pipe buffer to return later at EOF + resetQueued bool // RST_STREAM queued for write; set by sc.resetStream + gotTrailerHeader bool // HEADER frame for trailers was seen + wroteHeaders bool // whether we wrote headers (not status 100) + writeDeadline *time.Timer // nil if unused trailer Header // accumulated trailers reqTrailer Header // handler's Request.Trailer @@ -3315,11 +4248,16 @@ func (sc *http2serverConn) HeaderEncoder() (*hpack.Encoder, *bytes.Buffer) { func (sc *http2serverConn) state(streamID uint32) (http2streamState, *http2stream) { sc.serveG.check() - + // http://tools.ietf.org/html/rfc7540#section-5.1 if st, ok := sc.streams[streamID]; ok { return st.state, st } - + // "The first use of a new stream identifier implicitly closes all + // streams in the "idle" state that might have been initiated by + // that peer with a lower-valued stream identifier. For example, if + // a client sends a HEADERS frame on stream 7 without ever sending a + // frame on stream 5, then stream 5 transitions to the "closed" + // state when the first frame for stream 7 is sent or received." if streamID%2 == 1 { if streamID <= sc.maxClientStreamID { return http2stateClosed, nil @@ -3373,11 +4311,18 @@ func http2isClosedConnError(err error) bool { return false } + // TODO: remove this string search and be more like the Windows + // case below. That might involve modifying the standard library + // to return better error types. str := err.Error() if strings.Contains(str, "use of closed network connection") { return true } + // TODO(bradfitz): x/tools/cmd/bundle doesn't really support + // build tags, so I can't make an http2_windows.go file with + // Windows-specific stuff. Fix that and move this, once we + // have a way to bundle this into std's net/http somehow. if runtime.GOOS == "windows" { if oe, ok := err.(*net.OpError); ok && oe.Op == "read" { if se, ok := oe.Err.(*os.SyscallError); ok && se.Syscall == "wsarecv" { @@ -3397,7 +4342,7 @@ func (sc *http2serverConn) condlogf(err error, format string, args ...interface{ return } if err == io.EOF || err == io.ErrUnexpectedEOF || http2isClosedConnError(err) { - + // Boring, expected errors. sc.vlogf(format, args...) } else { sc.logf(format, args...) @@ -3487,7 +4432,7 @@ func (sc *http2serverConn) stopShutdownTimer() { } func (sc *http2serverConn) notePanic() { - + // Note: this is for serverConn.serve panicking, not http.Handler code. if http2testHookOnPanicMu != nil { http2testHookOnPanicMu.Lock() defer http2testHookOnPanicMu.Unlock() @@ -3507,7 +4452,7 @@ func (sc *http2serverConn) serve() { defer sc.conn.Close() defer sc.closeAllStreamsOnConnClose() defer sc.stopShutdownTimer() - defer close(sc.doneServing) + defer close(sc.doneServing) // unblocks handlers trying to send if http2VerboseLogs { sc.vlogf("http2: server connection from %v on %p", sc.conn.RemoteAddr(), sc.hs) @@ -3518,44 +4463,48 @@ func (sc *http2serverConn) serve() { {http2SettingMaxFrameSize, sc.srv.maxReadFrameSize()}, {http2SettingMaxConcurrentStreams, sc.advMaxStreams}, {http2SettingMaxHeaderListSize, sc.maxHeaderListSize()}, + {http2SettingInitialWindowSize, uint32(sc.srv.initialStreamRecvWindowSize())}, }, }) sc.unackedSettings++ + // Each connection starts with intialWindowSize inflow tokens. + // If a higher value is configured, we add more tokens. + if diff := sc.srv.initialConnRecvWindowSize() - http2initialWindowSize; diff > 0 { + sc.sendWindowUpdate(nil, int(diff)) + } + if err := sc.readPreface(); err != nil { sc.condlogf(err, "http2: server: error reading preface from client %v: %v", sc.conn.RemoteAddr(), err) return } - + // Now that we've got the preface, get us out of the + // "StateNew" state. We can't go directly to idle, though. + // Active means we read some data and anticipate a request. We'll + // do another Active when we get a HEADERS frame. sc.setConnState(StateActive) sc.setConnState(StateIdle) if sc.srv.IdleTimeout != 0 { - sc.idleTimer = time.NewTimer(sc.srv.IdleTimeout) + sc.idleTimer = time.AfterFunc(sc.srv.IdleTimeout, sc.onIdleTimer) defer sc.idleTimer.Stop() - sc.idleTimerCh = sc.idleTimer.C } - var gracefulShutdownCh chan struct{} - if sc.hs != nil { - ch := http2h1ServerShutdownChan(sc.hs) - if ch != nil { - gracefulShutdownCh = make(chan struct{}) - go sc.awaitGracefulShutdown(ch, gracefulShutdownCh) - } - } + go sc.readFrames() // closed by defer sc.conn.Close above - go sc.readFrames() + settingsTimer := time.AfterFunc(http2firstSettingsTimeout, sc.onSettingsTimer) + defer settingsTimer.Stop() - settingsTimer := time.NewTimer(http2firstSettingsTimeout) loopNum := 0 for { loopNum++ select { case wr := <-sc.wantWriteFrameCh: + if se, ok := wr.write.(http2StreamError); ok { + sc.resetStream(se) + break + } sc.writeFrame(wr) - case spr := <-sc.wantStartPushCh: - sc.startPush(spr) case res := <-sc.wroteFrameCh: sc.wroteFrame(res) case res := <-sc.readFrameCh: @@ -3563,26 +4512,37 @@ func (sc *http2serverConn) serve() { return } res.readMore() - if settingsTimer.C != nil { + if settingsTimer != nil { settingsTimer.Stop() - settingsTimer.C = nil + settingsTimer = nil } case m := <-sc.bodyReadCh: sc.noteBodyRead(m.st, m.n) - case <-settingsTimer.C: - sc.logf("timeout waiting for SETTINGS frames from %v", sc.conn.RemoteAddr()) - return - case <-gracefulShutdownCh: - gracefulShutdownCh = nil - sc.startGracefulShutdown() - case <-sc.shutdownTimerCh: - sc.vlogf("GOAWAY close timer fired; closing conn from %v", sc.conn.RemoteAddr()) - return - case <-sc.idleTimerCh: - sc.vlogf("connection is idle") - sc.goAway(http2ErrCodeNo) - case fn := <-sc.testHookCh: - fn(loopNum) + case msg := <-sc.serveMsgCh: + switch v := msg.(type) { + case func(int): + v(loopNum) // for testing + case *http2serverMessage: + switch v { + case http2settingsTimerMsg: + sc.logf("timeout waiting for SETTINGS frames from %v", sc.conn.RemoteAddr()) + return + case http2idleTimerMsg: + sc.vlogf("connection is idle") + sc.goAway(http2ErrCodeNo) + case http2shutdownTimerMsg: + sc.vlogf("GOAWAY close timer fired; closing conn from %v", sc.conn.RemoteAddr()) + return + case http2gracefulShutdownMsg: + sc.startGracefulShutdownInternal() + default: + panic("unknown timer") + } + case *http2startPushRequest: + sc.startPush(v) + default: + panic(fmt.Sprintf("unexpected type %T", v)) + } } if sc.inGoAway && sc.curOpenStreams() == 0 && !sc.needToSendGoAway && !sc.writingFrame { @@ -3599,12 +4559,36 @@ func (sc *http2serverConn) awaitGracefulShutdown(sharedCh <-chan struct{}, priva } } +type http2serverMessage int + +// Message values sent to serveMsgCh. +var ( + http2settingsTimerMsg = new(http2serverMessage) + http2idleTimerMsg = new(http2serverMessage) + http2shutdownTimerMsg = new(http2serverMessage) + http2gracefulShutdownMsg = new(http2serverMessage) +) + +func (sc *http2serverConn) onSettingsTimer() { sc.sendServeMsg(http2settingsTimerMsg) } + +func (sc *http2serverConn) onIdleTimer() { sc.sendServeMsg(http2idleTimerMsg) } + +func (sc *http2serverConn) onShutdownTimer() { sc.sendServeMsg(http2shutdownTimerMsg) } + +func (sc *http2serverConn) sendServeMsg(msg interface{}) { + sc.serveG.checkNotOn() // NOT + select { + case sc.serveMsgCh <- msg: + case <-sc.doneServing: + } +} + // readPreface reads the ClientPreface greeting from the peer // or returns an error on timeout or an invalid greeting. func (sc *http2serverConn) readPreface() error { errc := make(chan error, 1) go func() { - + // Read the client preface buf := make([]byte, len(http2ClientPreface)) if _, err := io.ReadFull(sc.conn, buf); err != nil { errc <- err @@ -3614,7 +4598,7 @@ func (sc *http2serverConn) readPreface() error { errc <- nil } }() - timer := time.NewTimer(http2prefaceTimeout) + timer := time.NewTimer(http2prefaceTimeout) // TODO: configurable on *Server? defer timer.Stop() select { case <-timer.C: @@ -3658,7 +4642,13 @@ func (sc *http2serverConn) writeDataFromHandler(stream *http2stream, data []byte case <-sc.doneServing: return http2errClientDisconnected case <-stream.cw: - + // If both ch and stream.cw were ready (as might + // happen on the final Write after an http.Handler + // ends), prefer the write result. Otherwise this + // might just be us successfully closing the stream. + // The writeFrameAsync and serve goroutines guarantee + // that the ch send will happen before the stream.cw + // close. select { case err = <-ch: frameWriteDone = true @@ -3681,12 +4671,13 @@ func (sc *http2serverConn) writeDataFromHandler(stream *http2stream, data []byte // buffered and is read by serve itself). If you're on the serve // goroutine, call writeFrame instead. func (sc *http2serverConn) writeFrameFromHandler(wr http2FrameWriteRequest) error { - sc.serveG.checkNotOn() + sc.serveG.checkNotOn() // NOT select { case sc.wantWriteFrameCh <- wr: return nil case <-sc.doneServing: - + // Serve loop is gone. + // Client has closed their connection to the server. return http2errClientDisconnected } } @@ -3705,6 +4696,24 @@ func (sc *http2serverConn) writeFrame(wr http2FrameWriteRequest) { // If true, wr will not be written and wr.done will not be signaled. var ignoreWrite bool + // We are not allowed to write frames on closed streams. RFC 7540 Section + // 5.1.1 says: "An endpoint MUST NOT send frames other than PRIORITY on + // a closed stream." Our server never sends PRIORITY, so that exception + // does not apply. + // + // The serverConn might close an open stream while the stream's handler + // is still running. For example, the server might close a stream when it + // receives bad data from the client. If this happens, the handler might + // attempt to write a frame after the stream has been closed (since the + // handler hasn't yet been notified of the close). In this case, we simply + // ignore the frame. The handler will notice that the stream is closed when + // it waits for the frame to be written. + // + // As an exception to this rule, we allow sending RST_STREAM after close. + // This allows us to immediately reject new streams without tracking any + // state for those streams (except for the queued RST_STREAM frame). This + // may result in duplicate RST_STREAMs in some cases, but the client should + // ignore those. if wr.StreamID() != 0 { _, isReset := wr.write.(http2StreamError) if state, _ := sc.state(wr.StreamID()); state == http2stateClosed && !isReset { @@ -3712,12 +4721,15 @@ func (sc *http2serverConn) writeFrame(wr http2FrameWriteRequest) { } } + // Don't send a 100-continue response if we've already sent headers. + // See golang.org/issue/14030. switch wr.write.(type) { case *http2writeResHeaders: wr.stream.wroteHeaders = true case http2write100ContinueHeadersFrame: if wr.stream.wroteHeaders { - + // We do not need to notify wr.done because this frame is + // never written with wr.done != nil. if wr.done != nil { panic("wr.done != nil for write100ContinueHeadersFrame") } @@ -3746,7 +4758,8 @@ func (sc *http2serverConn) startFrameWrite(wr http2FrameWriteRequest) { case http2stateHalfClosedLocal: switch wr.write.(type) { case http2StreamError, http2handlerPanicRST, http2writeWindowUpdate: - + // RFC 7540 Section 5.1 allows sending RST_STREAM, PRIORITY, and WINDOW_UPDATE + // in this state. (We never send PRIORITY from the server, so that is not checked.) default: panic(fmt.Sprintf("internal error: attempt to send frame on a half-closed-local stream: %v", wr)) } @@ -3800,16 +4813,29 @@ func (sc *http2serverConn) wroteFrame(res http2frameWriteResult) { } switch st.state { case http2stateOpen: - + // Here we would go to stateHalfClosedLocal in + // theory, but since our handler is done and + // the net/http package provides no mechanism + // for closing a ResponseWriter while still + // reading data (see possible TODO at top of + // this file), we go into closed state here + // anyway, after telling the peer we're + // hanging up on them. We'll transition to + // stateClosed after the RST_STREAM frame is + // written. st.state = http2stateHalfClosedLocal - sc.resetStream(http2streamError(st.id, http2ErrCodeCancel)) + // Section 8.1: a server MAY request that the client abort + // transmission of a request without error by sending a + // RST_STREAM with an error code of NO_ERROR after sending + // a complete response. + sc.resetStream(http2streamError(st.id, http2ErrCodeNo)) case http2stateHalfClosedRemote: sc.closeStream(st, http2errHandlerComplete) } } else { switch v := wr.write.(type) { case http2StreamError: - + // st may be unknown if the RST_STREAM was generated to reject bad input. if st, ok := sc.streams[v.StreamID]; ok { sc.closeStream(st, v) } @@ -3818,6 +4844,7 @@ func (sc *http2serverConn) wroteFrame(res http2frameWriteResult) { } } + // Reply (if requested) to unblock the ServeHTTP goroutine. wr.replyToWriter(res.err) sc.scheduleFrameWrite() @@ -3865,7 +4892,7 @@ func (sc *http2serverConn) scheduleFrameWrite() { } if sc.needsFrameFlush { sc.startFrameWrite(http2FrameWriteRequest{write: http2flushFrameWriter{}}) - sc.needsFrameFlush = false + sc.needsFrameFlush = false // after startFrameWrite, since it sets this true continue } break @@ -3873,10 +4900,19 @@ func (sc *http2serverConn) scheduleFrameWrite() { sc.inFrameScheduleLoop = false } -// startGracefulShutdown sends a GOAWAY with ErrCodeNo to tell the -// client we're gracefully shutting down. The connection isn't closed -// until all current streams are done. +// startGracefulShutdown gracefully shuts down a connection. This +// sends GOAWAY with ErrCodeNo to tell the client we're gracefully +// shutting down. The connection isn't closed until all current +// streams are done. +// +// startGracefulShutdown returns immediately; it does not wait until +// the connection has shut down. func (sc *http2serverConn) startGracefulShutdown() { + sc.serveG.checkNotOn() // NOT + sc.shutdownOnce.Do(func() { sc.sendServeMsg(http2gracefulShutdownMsg) }) +} + +func (sc *http2serverConn) startGracefulShutdownInternal() { sc.goAwayIn(http2ErrCodeNo, 0) } @@ -3886,7 +4922,7 @@ func (sc *http2serverConn) goAway(code http2ErrCode) { if code != http2ErrCodeNo { forceCloseIn = 250 * time.Millisecond } else { - + // TODO: configurable forceCloseIn = 1 * time.Second } sc.goAwayIn(code, forceCloseIn) @@ -3908,8 +4944,7 @@ func (sc *http2serverConn) goAwayIn(code http2ErrCode, forceCloseIn time.Duratio func (sc *http2serverConn) shutDownIn(d time.Duration) { sc.serveG.check() - sc.shutdownTimer = time.NewTimer(d) - sc.shutdownTimerCh = sc.shutdownTimer.C + sc.shutdownTimer = time.AfterFunc(d, sc.onShutdownTimer) } func (sc *http2serverConn) resetStream(se http2StreamError) { @@ -3929,11 +4964,18 @@ func (sc *http2serverConn) processFrameFromReader(res http2readFrameResult) bool if err != nil { if err == http2ErrFrameTooLarge { sc.goAway(http2ErrCodeFrameSize) - return true + return true // goAway will close the loop } clientGone := err == io.EOF || err == io.ErrUnexpectedEOF || http2isClosedConnError(err) if clientGone { - + // TODO: could we also get into this state if + // the peer does a half close + // (e.g. CloseWrite) because they're done + // sending frames but they're still wanting + // our open replies? Investigate. + // TODO: add CloseWrite to crypto/tls.Conn first + // so we have a way to test this? I suppose + // just for testing we could have a non-TLS mode. return false } } else { @@ -3957,7 +4999,7 @@ func (sc *http2serverConn) processFrameFromReader(res http2readFrameResult) bool case http2ConnectionError: sc.logf("http2: server connection error from %v: %v", sc.conn.RemoteAddr(), ev) sc.goAway(http2ErrCode(ev)) - return true + return true // goAway will handle shutdown default: if res.err != nil { sc.vlogf("http2: server closing client connection; error reading frame from client %s: %v", sc.conn.RemoteAddr(), err) @@ -3971,6 +5013,7 @@ func (sc *http2serverConn) processFrameFromReader(res http2readFrameResult) bool func (sc *http2serverConn) processFrame(f http2Frame) error { sc.serveG.check() + // First frame received must be SETTINGS. if !sc.sawFirstSettings { if _, ok := f.(*http2SettingsFrame); !ok { return http2ConnectionError(http2ErrCodeProtocol) @@ -3996,7 +5039,8 @@ func (sc *http2serverConn) processFrame(f http2Frame) error { case *http2GoAwayFrame: return sc.processGoAway(f) case *http2PushPromiseFrame: - + // A client cannot push. Thus, servers MUST treat the receipt of a PUSH_PROMISE + // frame as a connection error (Section 5.4.1) of type PROTOCOL_ERROR. return http2ConnectionError(http2ErrCodeProtocol) default: sc.vlogf("http2: server ignoring frame: %v", f.Header()) @@ -4007,11 +5051,16 @@ func (sc *http2serverConn) processFrame(f http2Frame) error { func (sc *http2serverConn) processPing(f *http2PingFrame) error { sc.serveG.check() if f.IsAck() { - + // 6.7 PING: " An endpoint MUST NOT respond to PING frames + // containing this flag." return nil } if f.StreamID != 0 { - + // "PING frames are not associated with any individual + // stream. If a PING frame is received with a stream + // identifier field value other than 0x0, the recipient MUST + // respond with a connection error (Section 5.4.1) of type + // PROTOCOL_ERROR." return http2ConnectionError(http2ErrCodeProtocol) } if sc.inGoAway && sc.goAwayCode != http2ErrCodeNo { @@ -4024,20 +5073,27 @@ func (sc *http2serverConn) processPing(f *http2PingFrame) error { func (sc *http2serverConn) processWindowUpdate(f *http2WindowUpdateFrame) error { sc.serveG.check() switch { - case f.StreamID != 0: + case f.StreamID != 0: // stream-level flow control state, st := sc.state(f.StreamID) if state == http2stateIdle { - + // Section 5.1: "Receiving any frame other than HEADERS + // or PRIORITY on a stream in this state MUST be + // treated as a connection error (Section 5.4.1) of + // type PROTOCOL_ERROR." return http2ConnectionError(http2ErrCodeProtocol) } if st == nil { - + // "WINDOW_UPDATE can be sent by a peer that has sent a + // frame bearing the END_STREAM flag. This means that a + // receiver could receive a WINDOW_UPDATE frame on a "half + // closed (remote)" or "closed" stream. A receiver MUST + // NOT treat this as an error, see Section 5.1." return nil } if !st.flow.add(int32(f.Increment)) { return http2streamError(f.StreamID, http2ErrCodeFlowControl) } - default: + default: // connection-level flow control if !sc.flow.add(int32(f.Increment)) { return http2goAwayFlowError{} } @@ -4051,7 +5107,11 @@ func (sc *http2serverConn) processResetStream(f *http2RSTStreamFrame) error { state, st := sc.state(f.StreamID) if state == http2stateIdle { - + // 6.4 "RST_STREAM frames MUST NOT be sent for a + // stream in the "idle" state. If a RST_STREAM frame + // identifying an idle stream is received, the + // recipient MUST treat this as a connection error + // (Section 5.4.1) of type PROTOCOL_ERROR. return http2ConnectionError(http2ErrCodeProtocol) } if st != nil { @@ -4067,6 +5127,9 @@ func (sc *http2serverConn) closeStream(st *http2stream, err error) { panic(fmt.Sprintf("invariant; can't close stream in state %v", st.state)) } st.state = http2stateClosed + if st.writeDeadline != nil { + st.writeDeadline.Stop() + } if st.isPushed() { sc.curPushedStreams-- } else { @@ -4079,16 +5142,17 @@ func (sc *http2serverConn) closeStream(st *http2stream, err error) { sc.idleTimer.Reset(sc.srv.IdleTimeout) } if http2h1ServerKeepAlivesDisabled(sc.hs) { - sc.startGracefulShutdown() + sc.startGracefulShutdownInternal() } } if p := st.body; p != nil { - + // Return any buffered unread bytes worth of conn-level flow control. + // See golang.org/issue/16481 sc.sendWindowUpdate(nil, p.Len()) p.CloseWithError(err) } - st.cw.Close() + st.cw.Close() // signals Handler's CloseNotifier, unblocks writes, etc sc.writeSched.CloseStream(st.id) } @@ -4097,7 +5161,9 @@ func (sc *http2serverConn) processSettings(f *http2SettingsFrame) error { if f.IsAck() { sc.unackedSettings-- if sc.unackedSettings < 0 { - + // Why is the peer ACKing settings we never sent? + // The spec doesn't mention this case, but + // hang up on them anyway. return http2ConnectionError(http2ErrCodeProtocol) } return nil @@ -4129,11 +5195,13 @@ func (sc *http2serverConn) processSetting(s http2Setting) error { case http2SettingInitialWindowSize: return sc.processSettingInitialWindowSize(s.Val) case http2SettingMaxFrameSize: - sc.maxFrameSize = int32(s.Val) + sc.maxFrameSize = int32(s.Val) // the maximum valid s.Val is < 2^31 case http2SettingMaxHeaderListSize: sc.peerMaxHeaderListSize = s.Val default: - + // Unknown setting: "An endpoint that receives a SETTINGS + // frame with any unknown or unsupported identifier MUST + // ignore that setting." if http2VerboseLogs { sc.vlogf("http2: server ignoring unknown setting %v", s) } @@ -4143,13 +5211,26 @@ func (sc *http2serverConn) processSetting(s http2Setting) error { func (sc *http2serverConn) processSettingInitialWindowSize(val uint32) error { sc.serveG.check() - - old := sc.initialWindowSize - sc.initialWindowSize = int32(val) - growth := sc.initialWindowSize - old + // Note: val already validated to be within range by + // processSetting's Valid call. + + // "A SETTINGS frame can alter the initial flow control window + // size for all current streams. When the value of + // SETTINGS_INITIAL_WINDOW_SIZE changes, a receiver MUST + // adjust the size of all stream flow control windows that it + // maintains by the difference between the new value and the + // old value." + old := sc.initialStreamSendWindowSize + sc.initialStreamSendWindowSize = int32(val) + growth := int32(val) - old // may be negative for _, st := range sc.streams { if !st.flow.add(growth) { - + // 6.9.2 Initial Flow Control Window Size + // "An endpoint MUST treat a change to + // SETTINGS_INITIAL_WINDOW_SIZE that causes any flow + // control window to exceed the maximum size as a + // connection error (Section 5.4.1) of type + // FLOW_CONTROL_ERROR." return http2ConnectionError(http2ErrCodeFlowControl) } } @@ -4163,23 +5244,40 @@ func (sc *http2serverConn) processData(f *http2DataFrame) error { } data := f.Data() + // "If a DATA frame is received whose stream is not in "open" + // or "half closed (local)" state, the recipient MUST respond + // with a stream error (Section 5.4.2) of type STREAM_CLOSED." id := f.Header().StreamID state, st := sc.state(id) if id == 0 || state == http2stateIdle { - + // Section 5.1: "Receiving any frame other than HEADERS + // or PRIORITY on a stream in this state MUST be + // treated as a connection error (Section 5.4.1) of + // type PROTOCOL_ERROR." return http2ConnectionError(http2ErrCodeProtocol) } if st == nil || state != http2stateOpen || st.gotTrailerHeader || st.resetQueued { - + // This includes sending a RST_STREAM if the stream is + // in stateHalfClosedLocal (which currently means that + // the http.Handler returned, so it's done reading & + // done writing). Try to stop the client from sending + // more DATA. + + // But still enforce their connection-level flow control, + // and return any flow control bytes since we're not going + // to consume them. if sc.inflow.available() < int32(f.Length) { return http2streamError(id, http2ErrCodeFlowControl) } - + // Deduct the flow control from inflow, since we're + // going to immediately add it back in + // sendWindowUpdate, which also schedules sending the + // frames. sc.inflow.take(int32(f.Length)) - sc.sendWindowUpdate(nil, int(f.Length)) + sc.sendWindowUpdate(nil, int(f.Length)) // conn-level if st != nil && st.resetQueued { - + // Already have a stream error in flight. Don't send another. return nil } return http2streamError(id, http2ErrCodeStreamClosed) @@ -4188,12 +5286,13 @@ func (sc *http2serverConn) processData(f *http2DataFrame) error { panic("internal error: should have a body in this state") } + // Sender sending more than they'd declared? if st.declBodyBytes != -1 && st.bodyBytes+int64(len(data)) > st.declBodyBytes { st.body.CloseWithError(fmt.Errorf("sender tried to send more than declared Content-Length of %d bytes", st.declBodyBytes)) return http2streamError(id, http2ErrCodeStreamClosed) } if f.Length > 0 { - + // Check whether the client has flow control quota. if st.inflow.available() < int32(f.Length) { return http2streamError(id, http2ErrCodeFlowControl) } @@ -4210,6 +5309,8 @@ func (sc *http2serverConn) processData(f *http2DataFrame) error { st.bodyBytes += int64(len(data)) } + // Return any padded flow control now, since we won't + // refund it later on body reads. if pad := int32(f.Length) - int32(len(data)); pad > 0 { sc.sendWindowUpdate32(nil, pad) sc.sendWindowUpdate32(st, pad) @@ -4228,8 +5329,9 @@ func (sc *http2serverConn) processGoAway(f *http2GoAwayFrame) error { } else { sc.vlogf("http2: received GOAWAY %+v, starting graceful shutdown", f) } - sc.startGracefulShutdown() - + sc.startGracefulShutdownInternal() + // http://tools.ietf.org/html/rfc7540#section-6.8 + // We should not create any new streams, which means we should disable push. sc.pushEnabled = false return nil } @@ -4260,32 +5362,51 @@ func (st *http2stream) endStream() { func (st *http2stream) copyTrailersToHandlerRequest() { for k, vv := range st.trailer { if _, ok := st.reqTrailer[k]; ok { - + // Only copy it over it was pre-declared. st.reqTrailer[k] = vv } } } +// onWriteTimeout is run on its own goroutine (from time.AfterFunc) +// when the stream's WriteTimeout has fired. +func (st *http2stream) onWriteTimeout() { + st.sc.writeFrameFromHandler(http2FrameWriteRequest{write: http2streamError(st.id, http2ErrCodeInternal)}) +} + func (sc *http2serverConn) processHeaders(f *http2MetaHeadersFrame) error { sc.serveG.check() id := f.StreamID if sc.inGoAway { - + // Ignore. return nil } - + // http://tools.ietf.org/html/rfc7540#section-5.1.1 + // Streams initiated by a client MUST use odd-numbered stream + // identifiers. [...] An endpoint that receives an unexpected + // stream identifier MUST respond with a connection error + // (Section 5.4.1) of type PROTOCOL_ERROR. if id%2 != 1 { return http2ConnectionError(http2ErrCodeProtocol) } - + // A HEADERS frame can be used to create a new stream or + // send a trailer for an open one. If we already have a stream + // open, let it process its own HEADERS frame (trailers at this + // point, if it's valid). if st := sc.streams[f.StreamID]; st != nil { if st.resetQueued { - + // We're sending RST_STREAM to close the stream, so don't bother + // processing this frame. return nil } return st.processTrailerHeaders(f) } + // [...] The identifier of a newly established stream MUST be + // numerically greater than all streams that the initiating + // endpoint has opened or reserved. [...] An endpoint that + // receives an unexpected stream identifier MUST respond with + // a connection error (Section 5.4.1) of type PROTOCOL_ERROR. if id <= sc.maxClientStreamID { return http2ConnectionError(http2ErrCodeProtocol) } @@ -4295,12 +5416,22 @@ func (sc *http2serverConn) processHeaders(f *http2MetaHeadersFrame) error { sc.idleTimer.Stop() } + // http://tools.ietf.org/html/rfc7540#section-5.1.2 + // [...] Endpoints MUST NOT exceed the limit set by their peer. An + // endpoint that receives a HEADERS frame that causes their + // advertised concurrent stream limit to be exceeded MUST treat + // this as a stream error (Section 5.4.2) of type PROTOCOL_ERROR + // or REFUSED_STREAM. if sc.curClientStreams+1 > sc.advMaxStreams { if sc.unackedSettings == 0 { - + // They should know better. return http2streamError(id, http2ErrCodeProtocol) } - + // Assume it's a network race, where they just haven't + // received our last SETTINGS update. But actually + // this can't happen yet, because we don't yet provide + // a way for users to adjust server parameters at + // runtime. return http2streamError(id, http2ErrCodeRefusedStream) } @@ -4325,17 +5456,24 @@ func (sc *http2serverConn) processHeaders(f *http2MetaHeadersFrame) error { if st.reqTrailer != nil { st.trailer = make(Header) } - st.body = req.Body.(*http2requestBody).pipe + st.body = req.Body.(*http2requestBody).pipe // may be nil st.declBodyBytes = req.ContentLength handler := sc.handler.ServeHTTP if f.Truncated { - + // Their header list was too long. Send a 431 error. handler = http2handleHeaderListTooLong } else if err := http2checkValidHTTP2RequestHeaders(req.Header); err != nil { handler = http2new400Handler(err) } + // The net/http package sets the read deadline from the + // http.Server.ReadTimeout during the TLS handshake, but then + // passes the connection off to us with the deadline already + // set. Disarm it here after the request headers are read, + // similar to how the http1 server works. Here it's + // technically more like the http1 Server's ReadHeaderTimeout + // (in Go 1.8), though. That's a more sane option anyway. if sc.hs.ReadTimeout != 0 { sc.conn.SetReadDeadline(time.Time{}) } @@ -4362,7 +5500,9 @@ func (st *http2stream) processTrailerHeaders(f *http2MetaHeadersFrame) error { for _, hf := range f.RegularFields() { key := sc.canonicalHeader(hf.Name) if !http2ValidTrailerHeader(key) { - + // TODO: send more details to the peer somehow. But http2 has + // no way to send debug data at a stream level. Discuss with + // HTTP folk. return http2streamError(st.id, http2ErrCodeProtocol) } st.trailer[key] = append(st.trailer[key], hf.Value) @@ -4374,7 +5514,10 @@ func (st *http2stream) processTrailerHeaders(f *http2MetaHeadersFrame) error { func http2checkPriority(streamID uint32, p http2PriorityParam) error { if streamID == p.StreamDep { - + // Section 5.3.1: "A stream cannot depend on itself. An endpoint MUST treat + // this as a stream error (Section 5.4.2) of type PROTOCOL_ERROR." + // Section 5.3.3 says that a stream can depend on one of its dependencies, + // so it's only self-dependencies that are forbidden. return http2streamError(streamID, http2ErrCodeProtocol) } return nil @@ -4406,10 +5549,13 @@ func (sc *http2serverConn) newStream(id, pusherID uint32, state http2streamState cancelCtx: cancelCtx, } st.cw.Init() - st.flow.conn = &sc.flow - st.flow.add(sc.initialWindowSize) - st.inflow.conn = &sc.inflow - st.inflow.add(http2initialWindowSize) + st.flow.conn = &sc.flow // link to conn-level counter + st.flow.add(sc.initialStreamSendWindowSize) + st.inflow.conn = &sc.inflow // link to conn-level counter + st.inflow.add(sc.srv.initialStreamRecvWindowSize()) + if sc.hs.WriteTimeout != 0 { + st.writeDeadline = time.AfterFunc(sc.hs.WriteTimeout, st.onWriteTimeout) + } sc.streams[id] = st sc.writeSched.OpenStream(st.id, http2OpenStreamOptions{PusherID: pusherID}) @@ -4441,13 +5587,22 @@ func (sc *http2serverConn) newWriterAndRequest(st *http2stream, f *http2MetaHead return nil, nil, http2streamError(f.StreamID, http2ErrCodeProtocol) } } else if rp.method == "" || rp.path == "" || (rp.scheme != "https" && rp.scheme != "http") { - + // See 8.1.2.6 Malformed Requests and Responses: + // + // Malformed requests or responses that are detected + // MUST be treated as a stream error (Section 5.4.2) + // of type PROTOCOL_ERROR." + // + // 8.1.2.3 Request Pseudo-Header Fields + // "All HTTP/2 requests MUST include exactly one valid + // value for the :method, :scheme, and :path + // pseudo-header fields" return nil, nil, http2streamError(f.StreamID, http2ErrCodeProtocol) } bodyOpen := !f.StreamEnded() if rp.method == "HEAD" && bodyOpen { - + // HEAD requests can't have bodies return nil, nil, http2streamError(f.StreamID, http2ErrCodeProtocol) } @@ -4464,16 +5619,14 @@ func (sc *http2serverConn) newWriterAndRequest(st *http2stream, f *http2MetaHead return nil, nil, err } if bodyOpen { - st.reqBuf = http2getRequestBodyBuf() - req.Body.(*http2requestBody).pipe = &http2pipe{ - b: &http2fixedBuffer{buf: st.reqBuf}, - } - if vv, ok := rp.header["Content-Length"]; ok { req.ContentLength, _ = strconv.ParseInt(vv[0], 10, 64) } else { req.ContentLength = -1 } + req.Body.(*http2requestBody).pipe = &http2pipe{ + b: &http2dataBuffer{expected: req.ContentLength}, + } } return rw, req, nil } @@ -4496,7 +5649,7 @@ func (sc *http2serverConn) newWriterAndRequestNoBody(st *http2stream, rp http2re if needsContinue { rp.header.Del("Expect") } - + // Merge Cookie headers into one "; "-delimited value. if cookies := rp.header["Cookie"]; len(cookies) > 1 { rp.header.Set("Cookie", strings.Join(cookies, "; ")) } @@ -4508,7 +5661,8 @@ func (sc *http2serverConn) newWriterAndRequestNoBody(st *http2stream, rp http2re key = CanonicalHeaderKey(strings.TrimSpace(key)) switch key { case "Transfer-Encoding", "Trailer", "Content-Length": - + // Bogus. (copy of http1 rules) + // Ignore. default: if trailer == nil { trailer = make(Header) @@ -4523,7 +5677,7 @@ func (sc *http2serverConn) newWriterAndRequestNoBody(st *http2stream, rp http2re var requestURI string if rp.method == "CONNECT" { url_ = &url.URL{Host: rp.authority} - requestURI = rp.authority + requestURI = rp.authority // mimic HTTP/1 server behavior } else { var err error url_, err = url.ParseRequestURI(rp.path) @@ -4556,7 +5710,7 @@ func (sc *http2serverConn) newWriterAndRequestNoBody(st *http2stream, rp http2re rws := http2responseWriterStatePool.Get().(*http2responseWriterState) bwSave := rws.bw - *rws = http2responseWriterState{} + *rws = http2responseWriterState{} // zero all the fields rws.conn = sc rws.bw = bwSave rws.bw.Reset(http2chunkWriter{rws}) @@ -4568,24 +5722,6 @@ func (sc *http2serverConn) newWriterAndRequestNoBody(st *http2stream, rp http2re return rw, req, nil } -var http2reqBodyCache = make(chan []byte, 8) - -func http2getRequestBodyBuf() []byte { - select { - case b := <-http2reqBodyCache: - return b - default: - return make([]byte, http2initialWindowSize) - } -} - -func http2putRequestBodyBuf(b []byte) { - select { - case http2reqBodyCache <- b: - default: - } -} - // Run on its own goroutine. func (sc *http2serverConn) runHandler(rw *http2responseWriter, req *Request, handler func(ResponseWriter, *Request)) { didPanic := true @@ -4597,7 +5733,7 @@ func (sc *http2serverConn) runHandler(rw *http2responseWriter, req *Request, han write: http2handlerPanicRST{rw.rws.stream.id}, stream: rw.rws.stream, }) - + // Same as net/http: if http2shouldLogPanic(e) { const size = 64 << 10 buf := make([]byte, size) @@ -4625,10 +5761,13 @@ func http2handleHeaderListTooLong(w ResponseWriter, r *Request) { // called from handler goroutines. // h may be nil. func (sc *http2serverConn) writeHeaders(st *http2stream, headerData *http2writeResHeaders) error { - sc.serveG.checkNotOn() + sc.serveG.checkNotOn() // NOT on var errc chan error if headerData.h != nil { - + // If there's a header map (which we don't own), so we have to block on + // waiting for this frame to be written, so an http.Flush mid-handler + // writes out the correct value of keys, before a handler later potentially + // mutates it. errc = http2errChanPool.Get().(chan error) } if err := sc.writeFrameFromHandler(http2FrameWriteRequest{ @@ -4671,26 +5810,21 @@ type http2bodyReadMsg struct { // Notes that the handler for the given stream ID read n bytes of its body // and schedules flow control tokens to be sent. func (sc *http2serverConn) noteBodyReadFromHandler(st *http2stream, n int, err error) { - sc.serveG.checkNotOn() + sc.serveG.checkNotOn() // NOT on if n > 0 { select { case sc.bodyReadCh <- http2bodyReadMsg{st, n}: case <-sc.doneServing: } } - if err == io.EOF { - if buf := st.reqBuf; buf != nil { - st.reqBuf = nil - http2putRequestBodyBuf(buf) - } - } } func (sc *http2serverConn) noteBodyRead(st *http2stream, n int) { sc.serveG.check() - sc.sendWindowUpdate(nil, n) + sc.sendWindowUpdate(nil, n) // conn-level if st.state != http2stateHalfClosedRemote && st.state != http2stateClosed { - + // Don't send this WINDOW_UPDATE if the stream is closed + // remotely. sc.sendWindowUpdate(st, n) } } @@ -4777,8 +5911,8 @@ func (b *http2requestBody) Read(p []byte) (n int, err error) { return } -// responseWriter is the http.ResponseWriter implementation. It's -// intentionally small (1 pointer wide) to minimize garbage. The +// responseWriter is the http.ResponseWriter implementation. It's +// intentionally small (1 pointer wide) to minimize garbage. The // responseWriterState pointer inside is zeroed at the end of a // request (in handlerDone) and calls on the responseWriter thereafter // simply crash (caller's mistake), but the much larger responseWriterState @@ -4812,6 +5946,7 @@ type http2responseWriterState struct { wroteHeader bool // WriteHeader called (explicitly or implicitly). Not necessarily sent to user yet. sentHeader bool // have we sent the header frame? handlerDone bool // handler has finished + dirty bool // a Write failed; don't reuse this responseWriterState sentContentLen int64 // non-zero if handler set a Content-Length header wroteBytes int64 @@ -4832,7 +5967,7 @@ func (rws *http2responseWriterState) hasTrailers() bool { return len(rws.trailer func (rws *http2responseWriterState) declareTrailer(k string) { k = CanonicalHeaderKey(k) if !http2ValidTrailerHeader(k) { - + // Forbidden by RFC 2616 14.40. rws.conn.logf("ignoring invalid trailer %q", k) return } @@ -4874,7 +6009,7 @@ func (rws *http2responseWriterState) writeChunk(p []byte) (n int, err error) { } var date string if _, ok := rws.snapHeader["Date"]; !ok { - + // TODO(bradfitz): be faster here, like net/http? measure. date = time.Now().UTC().Format(TimeFormat) } @@ -4893,6 +6028,7 @@ func (rws *http2responseWriterState) writeChunk(p []byte) (n int, err error) { date: date, }) if err != nil { + rws.dirty = true return 0, err } if endStream { @@ -4912,8 +6048,9 @@ func (rws *http2responseWriterState) writeChunk(p []byte) (n int, err error) { endStream := rws.handlerDone && !rws.hasTrailers() if len(p) > 0 || endStream { - + // only send a 0 byte DATA frame if we're ending the stream. if err := rws.conn.writeDataFromHandler(rws.stream, p, endStream); err != nil { + rws.dirty = true return 0, err } } @@ -4925,6 +6062,9 @@ func (rws *http2responseWriterState) writeChunk(p []byte) (n int, err error) { trailers: rws.trailers, endStream: true, }) + if err != nil { + rws.dirty = true + } return len(p), err } return len(p), nil @@ -4952,7 +6092,7 @@ const http2TrailerPrefix = "Trailer:" // says you SHOULD (but not must) predeclare any trailers in the // header, the official ResponseWriter rules said trailers in Go must // be predeclared, and then we reuse the same ResponseWriter.Header() -// map to mean both Headers and Trailers. When it's time to write the +// map to mean both Headers and Trailers. When it's time to write the // Trailers, we pick out the fields of Headers that were declared as // trailers. That worked for a while, until we found the first major // user of Trailers in the wild: gRPC (using them only over http2), @@ -4989,11 +6129,14 @@ func (w *http2responseWriter) Flush() { } if rws.bw.Buffered() > 0 { if err := rws.bw.Flush(); err != nil { - + // Ignore the error. The frame writer already knows. return } } else { - + // The bufio.Writer won't call chunkWriter.Write + // (writeChunk with zero bytes, so we have to do it + // ourselves to force the HTTP response header and/or + // final DATA frame (with END_STREAM) to be sent. rws.writeChunk(nil) } } @@ -5010,7 +6153,7 @@ func (w *http2responseWriter) CloseNotify() <-chan bool { rws.closeNotifierCh = ch cw := rws.stream.cw go func() { - cw.Wait() + cw.Wait() // wait for close ch <- true }() } @@ -5061,7 +6204,7 @@ func http2cloneHeader(h Header) Header { // // * Handler calls w.Write or w.WriteString -> // * -> rws.bw (*bufio.Writer) -> -// * (Handler migth call Flush) +// * (Handler might call Flush) // * -> chunkWriter{rws} // * -> responseWriterState.writeChunk(p []byte) // * -> responseWriterState.writeChunk (most of the magic; see comment there) @@ -5085,9 +6228,9 @@ func (w *http2responseWriter) write(lenData int, dataB []byte, dataS string) (n if !http2bodyAllowedForStatus(rws.status) { return 0, ErrBodyNotAllowed } - rws.wroteBytes += int64(len(dataB)) + int64(len(dataS)) + rws.wroteBytes += int64(len(dataB)) + int64(len(dataS)) // only one can be set if rws.sentContentLen != 0 && rws.wroteBytes > rws.sentContentLen { - + // TODO: send a RST_STREAM return 0, errors.New("http2: handler wrote more than declared Content-Length") } @@ -5100,10 +6243,19 @@ func (w *http2responseWriter) write(lenData int, dataB []byte, dataS string) (n func (w *http2responseWriter) handlerDone() { rws := w.rws + dirty := rws.dirty rws.handlerDone = true w.Flush() w.rws = nil - http2responseWriterStatePool.Put(rws) + if !dirty { + // Only recycle the pool if all prior Write calls to + // the serverConn goroutine completed successfully. If + // they returned earlier due to resets from the peer + // there might still be write goroutines outstanding + // from the serverConn referencing the rws memory. See + // issue 20704. + http2responseWriterStatePool.Put(rws) + } } // Push errors. @@ -5124,10 +6276,13 @@ func (w *http2responseWriter) push(target string, opts http2pushOptions) error { sc := st.sc sc.serveG.checkNotOn() + // No recursive pushes: "PUSH_PROMISE frames MUST only be sent on a peer-initiated stream." + // http://tools.ietf.org/html/rfc7540#section-6.6 if st.isPushed() { return http2ErrRecursivePush } + // Default options. if opts.Method == "" { opts.Method = "GET" } @@ -5139,6 +6294,7 @@ func (w *http2responseWriter) push(target string, opts http2pushOptions) error { wantScheme = "https" } + // Validate the request. u, err := url.Parse(target) if err != nil { return err @@ -5161,7 +6317,10 @@ func (w *http2responseWriter) push(target string, opts http2pushOptions) error { if strings.HasPrefix(k, ":") { return fmt.Errorf("promised request headers cannot include pseudo header %q", k) } - + // These headers are meaningful only if the request has a body, + // but PUSH_PROMISE requests cannot have a body. + // http://tools.ietf.org/html/rfc7540#section-8.2 + // Also disallow Host, since the promised URL must be absolute. switch strings.ToLower(k) { case "content-length", "content-encoding", "trailer", "te", "expect", "host": return fmt.Errorf("promised request headers cannot include %q", k) @@ -5171,11 +6330,14 @@ func (w *http2responseWriter) push(target string, opts http2pushOptions) error { return err } + // The RFC effectively limits promised requests to GET and HEAD: + // "Promised requests MUST be cacheable [GET, HEAD, or POST], and MUST be safe [GET or HEAD]" + // http://tools.ietf.org/html/rfc7540#section-8.2 if opts.Method != "GET" && opts.Method != "HEAD" { return fmt.Errorf("method %q must be GET or HEAD", opts.Method) } - msg := http2startPushRequest{ + msg := &http2startPushRequest{ parent: st, method: opts.Method, url: u, @@ -5188,7 +6350,7 @@ func (w *http2responseWriter) push(target string, opts http2pushOptions) error { return http2errClientDisconnected case <-st.cw: return http2errStreamClosed - case sc.wantStartPushCh <- msg: + case sc.serveMsgCh <- msg: } select { @@ -5210,48 +6372,66 @@ type http2startPushRequest struct { done chan error } -func (sc *http2serverConn) startPush(msg http2startPushRequest) { +func (sc *http2serverConn) startPush(msg *http2startPushRequest) { sc.serveG.check() + // http://tools.ietf.org/html/rfc7540#section-6.6. + // PUSH_PROMISE frames MUST only be sent on a peer-initiated stream that + // is in either the "open" or "half-closed (remote)" state. if msg.parent.state != http2stateOpen && msg.parent.state != http2stateHalfClosedRemote { - + // responseWriter.Push checks that the stream is peer-initiaed. msg.done <- http2errStreamClosed return } + // http://tools.ietf.org/html/rfc7540#section-6.6. if !sc.pushEnabled { msg.done <- ErrNotSupported return } + // PUSH_PROMISE frames must be sent in increasing order by stream ID, so + // we allocate an ID for the promised stream lazily, when the PUSH_PROMISE + // is written. Once the ID is allocated, we start the request handler. allocatePromisedID := func() (uint32, error) { sc.serveG.check() + // Check this again, just in case. Technically, we might have received + // an updated SETTINGS by the time we got around to writing this frame. if !sc.pushEnabled { return 0, ErrNotSupported } - + // http://tools.ietf.org/html/rfc7540#section-6.5.2. if sc.curPushedStreams+1 > sc.clientMaxStreams { return 0, http2ErrPushLimitReached } + // http://tools.ietf.org/html/rfc7540#section-5.1.1. + // Streams initiated by the server MUST use even-numbered identifiers. + // A server that is unable to establish a new stream identifier can send a GOAWAY + // frame so that the client is forced to open a new connection for new streams. if sc.maxPushPromiseID+2 >= 1<<31 { - sc.startGracefulShutdown() + sc.startGracefulShutdownInternal() return 0, http2ErrPushLimitReached } sc.maxPushPromiseID += 2 promisedID := sc.maxPushPromiseID + // http://tools.ietf.org/html/rfc7540#section-8.2. + // Strictly speaking, the new stream should start in "reserved (local)", then + // transition to "half closed (remote)" after sending the initial HEADERS, but + // we start in "half closed (remote)" for simplicity. + // See further comments at the definition of stateHalfClosedRemote. promised := sc.newStream(promisedID, msg.parent.id, http2stateHalfClosedRemote) rw, req, err := sc.newWriterAndRequestNoBody(promised, http2requestParam{ method: msg.method, scheme: msg.url.Scheme, authority: msg.url.Host, path: msg.url.RequestURI(), - header: http2cloneHeader(msg.header), + header: http2cloneHeader(msg.header), // clone since handler runs concurrently with writing the PUSH_PROMISE }) if err != nil { - + // Should not happen, since we've already validated msg.url. panic(fmt.Sprintf("newWriterAndRequestNoBody(%+v): %v", msg.url, err)) } @@ -5356,31 +6536,6 @@ var http2badTrailer = map[string]bool{ "Www-Authenticate": true, } -// h1ServerShutdownChan returns a channel that will be closed when the -// provided *http.Server wants to shut down. -// -// This is a somewhat hacky way to get at http1 innards. It works -// when the http2 code is bundled into the net/http package in the -// standard library. The alternatives ended up making the cmd/go tool -// depend on http Servers. This is the lightest option for now. -// This is tested via the TestServeShutdown* tests in net/http. -func http2h1ServerShutdownChan(hs *Server) <-chan struct{} { - if fn := http2testh1ServerShutdownChan; fn != nil { - return fn(hs) - } - var x interface{} = hs - type I interface { - getDoneChan() <-chan struct{} - } - if hs, ok := x.(I); ok { - return hs.getDoneChan() - } - return nil -} - -// optional test hook for h1ServerShutdownChan. -var http2testh1ServerShutdownChan func(hs *Server) <-chan struct{} - // h1ServerKeepAlivesDisabled reports whether hs has its keep-alives // disabled. See comments on h1ServerShutdownChan above for why // the code is written this way. @@ -5486,7 +6641,7 @@ var http2errTransportVersion = errors.New("http2: ConfigureTransport is only sup // It requires Go 1.6 or later and returns an error if the net/http package is too old // or if t1 has already been HTTP/2-enabled. func http2ConfigureTransport(t1 *Transport) error { - _, err := http2configureTransport(t1) + _, err := http2configureTransport(t1) // in configure_transport.go (go1.6) or not_go16.go return err } @@ -5669,7 +6824,7 @@ func (t *http2Transport) RoundTrip(req *Request) (*Response, error) { // and returns a host:port. The port 443 is added if needed. func http2authorityAddr(scheme string, authority string) (addr string) { host, port, err := net.SplitHostPort(authority) - if err != nil { + if err != nil { // authority didn't have a port port = "443" if scheme == "http" { port = "80" @@ -5679,7 +6834,7 @@ func http2authorityAddr(scheme string, authority string) (addr string) { if a, err := idna.ToASCII(host); err == nil { host = a } - + // IPv6 address literal, without a port: if strings.HasPrefix(host, "[") && strings.HasSuffix(host, "]") { return host + ":" + port } @@ -5742,12 +6897,14 @@ func http2shouldRetryRequest(req *Request, err error) (*Request, error) { case http2errClientConnUnusable, http2errClientConnGotGoAway: return req, nil case http2errClientConnGotGoAwayAfterSomeReqBody: - + // If the Body is nil (or http.NoBody), it's safe to reuse + // this request and its Body. if req.Body == nil || http2reqBodyIsNoBody(req.Body) { return req, nil } - - getBody := http2reqGetBody(req) + // Otherwise we depend on the Request having its GetBody + // func defined. + getBody := http2reqGetBody(req) // Go 1.8: getBody = req.GetBody if getBody == nil { return nil, errors.New("http2: Transport: peer server initiated graceful shutdown after some of Request.Body was written; define Request.GetBody to avoid this error") } @@ -5840,9 +6997,9 @@ func (t *http2Transport) newClientConn(c net.Conn, singleUse bool) (*http2Client tconn: c, readerDone: make(chan struct{}), nextStreamID: 1, - maxFrameSize: 16 << 10, - initialWindowSize: 65535, - maxConcurrentStreams: 1000, + maxFrameSize: 16 << 10, // spec default + initialWindowSize: 65535, // spec default + maxConcurrentStreams: 1000, // "infinite", per spec. 1000 seems good enough. streams: make(map[uint32]*http2clientStream), singleUse: singleUse, wantSettingsAck: true, @@ -5859,12 +7016,16 @@ func (t *http2Transport) newClientConn(c net.Conn, singleUse bool) (*http2Client cc.cond = sync.NewCond(&cc.mu) cc.flow.add(int32(http2initialWindowSize)) + // TODO: adjust this writer size to account for frame size + + // MTU + crypto/tls record padding. cc.bw = bufio.NewWriter(http2stickyErrWriter{c, &cc.werr}) cc.br = bufio.NewReader(c) cc.fr = http2NewFramer(cc.bw, cc.br) cc.fr.ReadMetaHeaders = hpack.NewDecoder(http2initialHeaderTableSize, nil) cc.fr.MaxHeaderListSize = t.maxHeaderListSize() + // TODO: SetMaxDynamicTableSize, SetMaxDynamicTableSizeLimit on + // henc in response to SETTINGS frames? cc.henc = hpack.NewEncoder(&cc.hbuf) if cs, ok := c.(http2connectionStater); ok { @@ -5900,6 +7061,7 @@ func (cc *http2ClientConn) setGoAway(f *http2GoAwayFrame) { old := cc.goAway cc.goAway = f + // Merge the previous and current GoAway error frames. if cc.goAwayDebug == "" { cc.goAwayDebug = string(f.DebugData()) } @@ -5932,7 +7094,7 @@ func (cc *http2ClientConn) canTakeNewRequestLocked() bool { cc.nextStreamID < math.MaxInt32 } -// onIdleTimeout is called from a time.AfterFunc goroutine. It will +// onIdleTimeout is called from a time.AfterFunc goroutine. It will // only be called when we're idle, but because we're coming from a new // goroutine, there could be a new request coming in at the same time, // so this simply calls the synchronized closeIfIdle to shut down this @@ -5950,7 +7112,7 @@ func (cc *http2ClientConn) closeIfIdle() { } cc.closed = true nextID := cc.nextStreamID - + // TODO: do clients send GOAWAY too? maybe? Just Close: cc.mu.Unlock() if http2VerboseLogs { @@ -5996,7 +7158,7 @@ func (cc *http2ClientConn) putFrameScratchBuffer(buf []byte) { return } } - + // forget about it. } // errRequestCanceled is a copy of net/http's errRequestCanceled because it's not @@ -6024,7 +7186,10 @@ func (cc *http2ClientConn) responseHeaderTimeout() time.Duration { if cc.t.t1 != nil { return cc.t.t1.ResponseHeaderTimeout } - + // No way to do this (yet?) with just an http2.Transport. Probably + // no need. Request.Cancel this is the new way. We only need to support + // this for compatibility with the old http.Transport fields when + // we're doing transparent http2. return 0 } @@ -6048,7 +7213,7 @@ func http2checkConnHeaders(req *Request) error { // req.ContentLength, where 0 actually means zero (not unknown) and -1 // means unknown. func http2actualContentLength(req *Request) int64 { - if req.Body == nil { + if req.Body == nil || http2reqBodyIsNoBody(req.Body) { return 0 } if req.ContentLength != 0 { @@ -6079,8 +7244,8 @@ func (cc *http2ClientConn) RoundTrip(req *Request) (*Response, error) { } body := req.Body - hasBody := body != nil contentLen := http2actualContentLength(req) + hasBody := contentLen != 0 // TODO(bradfitz): this is a copy of the logic in net/http. Unify somewhere? var requestedGzip bool @@ -6088,10 +7253,24 @@ func (cc *http2ClientConn) RoundTrip(req *Request) (*Response, error) { req.Header.Get("Accept-Encoding") == "" && req.Header.Get("Range") == "" && req.Method != "HEAD" { - + // Request gzip only, not deflate. Deflate is ambiguous and + // not as universally supported anyway. + // See: http://www.gzip.org/zlib/zlib_faq.html#faq38 + // + // Note that we don't request this for HEAD requests, + // due to a bug in nginx: + // http://trac.nginx.org/nginx/ticket/358 + // https://golang.org/issue/5522 + // + // We don't request gzip if the request is for a range, since + // auto-decoding a portion of a gzipped document will just fail + // anyway. See https://golang.org/issue/8923 requestedGzip = true } + // we send: HEADERS{1}, CONTINUATION{0,} + DATA{0,} (DATA is + // sent by writeRequestBody below, along with any Trailers, + // again in form HEADERS{1}, CONTINUATION{0,}) hdrs, err := cc.encodeHeaders(req, requestedGzip, trailers, contentLen) if err != nil { cc.mu.Unlock() @@ -6114,11 +7293,12 @@ func (cc *http2ClientConn) RoundTrip(req *Request) (*Response, error) { if werr != nil { if hasBody { - req.Body.Close() + req.Body.Close() // per RoundTripper contract bodyWriter.cancel() } cc.forgetStreamID(cs.ID) - + // Don't bother sending a RST_STREAM (our write already failed; + // no need to keep writing) http2traceWroteRequest(cs.trace, werr) return nil, werr } @@ -6142,7 +7322,15 @@ func (cc *http2ClientConn) RoundTrip(req *Request) (*Response, error) { handleReadLoopResponse := func(re http2resAndError) (*Response, error) { res := re.res if re.err != nil || res.StatusCode > 299 { - + // On error or status code 3xx, 4xx, 5xx, etc abort any + // ongoing write, assuming that the server doesn't care + // about our request body. If the server replied with 1xx or + // 2xx, however, then assume the server DOES potentially + // want our body (e.g. full-duplex streaming: + // golang.org/issue/13444). If it turns out the server + // doesn't, they'll RST_STREAM us soon enough. This is a + // heuristic to avoid adding knobs to Transport. Hopefully + // we can keep it. bodyWriter.cancel() cs.abortRequestBodyWrite(http2errStopReqBodyWrite) } @@ -6209,9 +7397,12 @@ func (cc *http2ClientConn) RoundTrip(req *Request) (*Response, error) { return handleReadLoopResponse(re) default: } + // processResetStream already removed the + // stream from the streams map; no need for + // forgetStreamID. return nil, cs.resetErr case err := <-bodyWriter.resc: - + // Prefer the read loop's response, if available. Issue 16102. select { case re := <-readLoopResCh: return handleReadLoopResponse(re) @@ -6232,7 +7423,7 @@ func (cc *http2ClientConn) RoundTrip(req *Request) (*Response, error) { // requires cc.wmu be held func (cc *http2ClientConn) writeHeaders(streamID uint32, endStream bool, hdrs []byte) error { - first := true + first := true // first frame written (HEADERS is first, then CONTINUATION) frameSize := int(cc.maxFrameSize) for len(hdrs) > 0 && cc.werr == nil { chunk := hdrs @@ -6253,7 +7444,10 @@ func (cc *http2ClientConn) writeHeaders(streamID uint32, endStream bool, hdrs [] cc.fr.WriteContinuation(streamID, endHeaders, chunk) } } - + // TODO(bradfitz): this Flush could potentially block (as + // could the WriteHeaders call(s) above), which means they + // wouldn't respond to Request.Cancel being readable. That's + // rare, but this should probably be in a goroutine. cc.bw.Flush() return cc.werr } @@ -6269,13 +7463,16 @@ var ( func (cs *http2clientStream) writeRequestBody(body io.Reader, bodyCloser io.Closer) (err error) { cc := cs.cc - sentEnd := false + sentEnd := false // whether we sent the final DATA frame w/ END_STREAM buf := cc.frameScratchBuffer() defer cc.putFrameScratchBuffer(buf) defer func() { http2traceWroteRequest(cs.trace, err) - + // TODO: write h12Compare test showing whether + // Request.Body is closed by the Transport, + // and in multiple cases: server replies <=299 and >299 + // while still writing request body cerr := bodyCloser.Close() if err == nil { err = cerr @@ -6314,7 +7511,12 @@ func (cs *http2clientStream) writeRequestBody(body io.Reader, bodyCloser io.Clos sentEnd = sawEOF && len(remain) == 0 && !hasTrailers err = cc.fr.WriteData(cs.ID, sentEnd, data) if err == nil { - + // TODO(bradfitz): this flush is for latency, not bandwidth. + // Most requests won't need this. Make this opt-in or + // opt-out? Use some heuristic on the body type? Nagel-like + // timers? Based on 'n'? Only last chunk of this for loop, + // unless flow control tokens are low? For now, always. + // If we change this, see comment below. err = cc.bw.Flush() } cc.wmu.Unlock() @@ -6325,7 +7527,9 @@ func (cs *http2clientStream) writeRequestBody(body io.Reader, bodyCloser io.Clos } if sentEnd { - + // Already sent END_STREAM (which implies we have no + // trailers) and flushed, because currently all + // WriteData frames above get a flush. So we're done. return nil } @@ -6339,6 +7543,8 @@ func (cs *http2clientStream) writeRequestBody(body io.Reader, bodyCloser io.Clos cc.wmu.Lock() defer cc.wmu.Unlock() + // Two ways to send END_STREAM: either with trailers, or + // with an empty DATA frame. if len(trls) > 0 { err = cc.writeHeaders(cs.ID, true, trls) } else { @@ -6372,7 +7578,7 @@ func (cs *http2clientStream) awaitFlowControl(maxBytes int) (taken int32, err er take := a if int(take) > maxBytes { - take = int32(maxBytes) + take = int32(maxBytes) // can't truncate int; take is int32 } if take > int32(cc.maxFrameSize) { take = int32(cc.maxFrameSize) @@ -6420,6 +7626,9 @@ func (cc *http2ClientConn) encodeHeaders(req *Request, addGzipHeader bool, trail } } + // Check for any invalid headers and return an error before we + // potentially pollute our hpack state. (We want to be able to + // continue to reuse the hpack encoder for future requests) for k, vv := range req.Header { if !httplex.ValidHeaderFieldName(k) { return nil, fmt.Errorf("invalid HTTP header name %q", k) @@ -6431,6 +7640,11 @@ func (cc *http2ClientConn) encodeHeaders(req *Request, addGzipHeader bool, trail } } + // 8.1.2.3 Request Pseudo-Header Fields + // The :path pseudo-header field includes the path and query parts of the + // target URI (the path-absolute production and optionally a '?' character + // followed by the query production (see Sections 3.3 and 3.4 of + // [RFC3986]). cc.writeHeader(":authority", host) cc.writeHeader(":method", req.Method) if req.Method != "CONNECT" { @@ -6446,13 +7660,20 @@ func (cc *http2ClientConn) encodeHeaders(req *Request, addGzipHeader bool, trail lowKey := strings.ToLower(k) switch lowKey { case "host", "content-length": - + // Host is :authority, already sent. + // Content-Length is automatic, set below. continue case "connection", "proxy-connection", "transfer-encoding", "upgrade", "keep-alive": - + // Per 8.1.2.2 Connection-Specific Header + // Fields, don't send connection-specific + // fields. We have already checked if any + // are error-worthy so just ignore the rest. continue case "user-agent": - + // Match Go's http1 behavior: at most one + // User-Agent. If set to nil or empty string, + // then omit it. Otherwise if not mentioned, + // include the default (below). didUA = true if len(vv) < 1 { continue @@ -6490,7 +7711,8 @@ func http2shouldSendReqContentLength(method string, contentLength int64) bool { if contentLength < 0 { return false } - + // For zero bodies, whether we send a content-length depends on the method. + // It also kinda doesn't matter for http2 either way, with END_STREAM. switch method { case "POST", "PUT", "PATCH": return true @@ -6503,7 +7725,8 @@ func http2shouldSendReqContentLength(method string, contentLength int64) bool { func (cc *http2ClientConn) encodeTrailers(req *Request) []byte { cc.hbuf.Reset() for k, vv := range req.Trailer { - + // Transfer-Encoding, etc.. have already been filter at the + // start of RoundTrip lowKey := strings.ToLower(k) for _, v := range vv { cc.writeHeader(lowKey, v) @@ -6557,7 +7780,7 @@ func (cc *http2ClientConn) streamByID(id uint32, andRemove bool) *http2clientStr cc.idleTimer.Reset(cc.idleTimeout) } close(cs.done) - cc.cond.Broadcast() + cc.cond.Broadcast() // wake up checkResetOrDone via clientStream.awaitFlowControl } return cs } @@ -6616,6 +7839,9 @@ func (rl *http2clientConnReadLoop) cleanup() { cc.idleTimer.Stop() } + // Close any response bodies if the server closes prematurely. + // TODO: also do this if we've written the headers but not + // gotten a response yet. err := cc.readerErr cc.mu.Lock() if cc.goAway != nil && http2isEOFOrNetReadError(err) { @@ -6645,7 +7871,7 @@ func (rl *http2clientConnReadLoop) cleanup() { func (rl *http2clientConnReadLoop) run() error { cc := rl.cc rl.closeWhenIdle = cc.t.disableKeepAlives() || cc.singleUse - gotReply := false + gotReply := false // ever saw a HEADERS reply gotSettings := false for { f, err := cc.fr.ReadFrame() @@ -6653,7 +7879,7 @@ func (rl *http2clientConnReadLoop) run() error { cc.vlogf("http2: Transport readFrame error on conn %p: (%T) %v", cc, err, err) } if se, ok := err.(http2StreamError); ok { - if cs := cc.streamByID(se.StreamID, true); cs != nil { + if cs := cc.streamByID(se.StreamID, true /*ended; remove it*/); cs != nil { cs.cc.writeStreamReset(cs.ID, se.Code, err) if se.Cause == nil { se.Cause = cc.fr.errDetail @@ -6674,7 +7900,7 @@ func (rl *http2clientConnReadLoop) run() error { } gotSettings = true } - maybeIdle := false + maybeIdle := false // whether frame might transition us to idle switch f := f.(type) { case *http2MetaHeadersFrame: @@ -6717,12 +7943,17 @@ func (rl *http2clientConnReadLoop) processHeaders(f *http2MetaHeadersFrame) erro cc := rl.cc cs := cc.streamByID(f.StreamID, f.StreamEnded()) if cs == nil { - + // We'd get here if we canceled a request while the + // server had its response still in flight. So if this + // was just something we canceled, ignore it. return nil } if !cs.firstByte { if cs.trace != nil { - + // TODO(bradfitz): move first response byte earlier, + // when we first read the 9 byte header, not waiting + // until all the HEADERS+CONTINUATION frames have been + // merged. This works for now. http2traceFirstResponseByte(cs.trace) } cs.firstByte = true @@ -6738,13 +7969,13 @@ func (rl *http2clientConnReadLoop) processHeaders(f *http2MetaHeadersFrame) erro if _, ok := err.(http2ConnectionError); ok { return err } - + // Any other error type is a stream error. cs.cc.writeStreamReset(f.StreamID, http2ErrCodeProtocol, err) cs.resc <- http2resAndError{err: err} - return nil + return nil // return nil from process* funcs to keep conn alive } if res == nil { - + // (nil, nil) special case. See handleResponse docs. return nil } if res.Body != http2noBody { @@ -6779,9 +8010,9 @@ func (rl *http2clientConnReadLoop) handleResponse(cs *http2clientStream, f *http if statusCode == 100 { http2traceGot100Continue(cs.trace) if cs.on100 != nil { - cs.on100() + cs.on100() // forces any write delay timer to fire } - cs.pastHeaders = false + cs.pastHeaders = false // do it all again return nil, nil } @@ -6817,10 +8048,12 @@ func (rl *http2clientConnReadLoop) handleResponse(cs *http2clientStream, f *http if clen64, err := strconv.ParseInt(clens[0], 10, 64); err == nil { res.ContentLength = clen64 } else { - + // TODO: care? unlike http/1, it won't mess up our framing, so it's + // more safe smuggling-wise to ignore. } } else if len(clens) > 1 { - + // TODO: care? unlike http/1, it won't mess up our framing, so it's + // more safe smuggling-wise to ignore. } } @@ -6829,8 +8062,7 @@ func (rl *http2clientConnReadLoop) handleResponse(cs *http2clientStream, f *http return res, nil } - buf := new(bytes.Buffer) - cs.bufPipe = http2pipe{b: buf} + cs.bufPipe = http2pipe{b: &http2dataBuffer{expected: res.ContentLength}} cs.bytesRemain = res.ContentLength res.Body = http2transportResponseBody{cs} go cs.awaitRequestCancel(cs.req) @@ -6847,16 +8079,18 @@ func (rl *http2clientConnReadLoop) handleResponse(cs *http2clientStream, f *http func (rl *http2clientConnReadLoop) processTrailers(cs *http2clientStream, f *http2MetaHeadersFrame) error { if cs.pastTrailers { - + // Too many HEADERS frames for this stream. return http2ConnectionError(http2ErrCodeProtocol) } cs.pastTrailers = true if !f.StreamEnded() { - + // We expect that any headers for trailers also + // has END_STREAM. return http2ConnectionError(http2ErrCodeProtocol) } if len(f.PseudoFields()) > 0 { - + // No pseudo header fields are defined for trailers. + // TODO: ConnectionError might be overly harsh? Check. return http2ConnectionError(http2ErrCodeProtocol) } @@ -6904,7 +8138,7 @@ func (b http2transportResponseBody) Read(p []byte) (n int, err error) { } } if n == 0 { - + // No flow control tokens to send back. return } @@ -6912,13 +8146,15 @@ func (b http2transportResponseBody) Read(p []byte) (n int, err error) { defer cc.mu.Unlock() var connAdd, streamAdd int32 - + // Check the conn-level first, before the stream-level. if v := cc.inflow.available(); v < http2transportDefaultConnFlow/2 { connAdd = http2transportDefaultConnFlow - v cc.inflow.add(connAdd) } - if err == nil { - + if err == nil { // No need to refresh if the stream is over or failed. + // Consider any buffered body data (read from the conn but not + // consumed by the client) when computing flow control for this + // stream. v := int(cs.inflow.available()) + cs.bufPipe.Len() if v < http2transportDefaultStreamFlow-http2transportDefaultStreamMinRefresh { streamAdd = int32(http2transportDefaultStreamFlow - v) @@ -6953,8 +8189,9 @@ func (b http2transportResponseBody) Close() error { cc.wmu.Lock() if !serverSentStreamEnd { cc.fr.WriteRSTStream(cs.ID, http2ErrCodeCancel) + cs.didReset = true } - + // Return connection-level flow control. if unread > 0 { cc.inflow.add(int32(unread)) cc.fr.WriteWindowUpdate(0, uint32(unread)) @@ -6977,11 +8214,16 @@ func (rl *http2clientConnReadLoop) processData(f *http2DataFrame) error { neverSent := cc.nextStreamID cc.mu.Unlock() if f.StreamID >= neverSent { - + // We never asked for this. cc.logf("http2: Transport received unsolicited DATA frame; closing connection") return http2ConnectionError(http2ErrCodeProtocol) } + // We probably did ask for this, but canceled. Just ignore it. + // TODO: be stricter here? only silently ignore things which + // we canceled, but not things which were closed normally + // by the peer? Tough without accumulating too much state. + // But at least return their flow control: if f.Length > 0 { cc.mu.Lock() cc.inflow.add(int32(f.Length)) @@ -6995,12 +8237,7 @@ func (rl *http2clientConnReadLoop) processData(f *http2DataFrame) error { return nil } if f.Length > 0 { - if len(data) > 0 && cs.bufPipe.b == nil { - - cc.logf("http2: Transport received DATA frame for closed stream; closing connection") - return http2ConnectionError(http2ErrCodeProtocol) - } - + // Check connection-level flow control. cc.mu.Lock() if cs.inflow.available() >= int32(f.Length) { cs.inflow.take(int32(f.Length)) @@ -7008,17 +8245,29 @@ func (rl *http2clientConnReadLoop) processData(f *http2DataFrame) error { cc.mu.Unlock() return http2ConnectionError(http2ErrCodeFlowControl) } - - if pad := int32(f.Length) - int32(len(data)); pad > 0 { - cs.inflow.add(pad) - cc.inflow.add(pad) + // Return any padded flow control now, since we won't + // refund it later on body reads. + var refund int + if pad := int(f.Length) - len(data); pad > 0 { + refund += pad + } + // Return len(data) now if the stream is already closed, + // since data will never be read. + didReset := cs.didReset + if didReset { + refund += len(data) + } + if refund > 0 { + cc.inflow.add(int32(refund)) cc.wmu.Lock() - cc.fr.WriteWindowUpdate(0, uint32(pad)) - cc.fr.WriteWindowUpdate(cs.ID, uint32(pad)) + cc.fr.WriteWindowUpdate(0, uint32(refund)) + if !didReset { + cs.inflow.add(int32(refund)) + cc.fr.WriteWindowUpdate(cs.ID, uint32(refund)) + } cc.bw.Flush() cc.wmu.Unlock() } - didReset := cs.didReset cc.mu.Unlock() if len(data) > 0 && !didReset { @@ -7038,7 +8287,8 @@ func (rl *http2clientConnReadLoop) processData(f *http2DataFrame) error { var http2errInvalidTrailers = errors.New("http2: invalid trailers") func (rl *http2clientConnReadLoop) endStream(cs *http2clientStream) { - + // TODO: check that any declared content-length matches, like + // server.go's (*stream).endStream method. rl.endStreamError(cs, nil) } @@ -7074,7 +8324,7 @@ func (rl *http2clientConnReadLoop) processGoAway(f *http2GoAwayFrame) error { cc := rl.cc cc.t.connPool().MarkDead(cc) if f.ErrCode != 0 { - + // TODO: deal with GOAWAY more. particularly the error code cc.vlogf("transport got GOAWAY with error code = %v", f.ErrCode) } cc.setGoAway(f) @@ -7101,11 +8351,17 @@ func (rl *http2clientConnReadLoop) processSettings(f *http2SettingsFrame) error case http2SettingMaxConcurrentStreams: cc.maxConcurrentStreams = s.Val case http2SettingInitialWindowSize: - + // Values above the maximum flow-control + // window size of 2^31-1 MUST be treated as a + // connection error (Section 5.4.1) of type + // FLOW_CONTROL_ERROR. if s.Val > math.MaxInt32 { return http2ConnectionError(http2ErrCodeFlowControl) } + // Adjust flow control of currently-open + // frames by the difference of the old initial + // window size and this one. delta := int32(s.Val) - int32(cc.initialWindowSize) for _, cs := range cc.streams { cs.flow.add(delta) @@ -7114,7 +8370,7 @@ func (rl *http2clientConnReadLoop) processSettings(f *http2SettingsFrame) error cc.initialWindowSize = s.Val default: - + // TODO(bradfitz): handle more settings? SETTINGS_HEADER_TABLE_SIZE probably. cc.vlogf("Unhandled Setting: %v", s) } return nil @@ -7155,18 +8411,21 @@ func (rl *http2clientConnReadLoop) processWindowUpdate(f *http2WindowUpdateFrame func (rl *http2clientConnReadLoop) processResetStream(f *http2RSTStreamFrame) error { cs := rl.cc.streamByID(f.StreamID, true) if cs == nil { - + // TODO: return error if server tries to RST_STEAM an idle stream return nil } select { case <-cs.peerReset: - + // Already reset. + // This is the only goroutine + // which closes this, so there + // isn't a race. default: err := http2streamError(cs.ID, f.ErrCode) cs.resetErr = err close(cs.peerReset) cs.bufPipe.CloseWithError(err) - cs.cc.cond.Broadcast() + cs.cc.cond.Broadcast() // wake up checkResetOrDone via clientStream.awaitFlowControl } delete(rl.activeRes, cs.ID) return nil @@ -7183,7 +8442,7 @@ func (cc *http2ClientConn) ping(ctx http2contextContext) error { return err } cc.mu.Lock() - + // check for dup before insert if _, found := cc.pings[p]; !found { cc.pings[p] = c cc.mu.Unlock() @@ -7207,7 +8466,7 @@ func (cc *http2ClientConn) ping(ctx http2contextContext) error { case <-ctx.Done(): return ctx.Err() case <-cc.readerDone: - + // connection closed return cc.readerErr } } @@ -7217,7 +8476,7 @@ func (rl *http2clientConnReadLoop) processPing(f *http2PingFrame) error { cc := rl.cc cc.mu.Lock() defer cc.mu.Unlock() - + // If ack, notify listener if any if c, ok := cc.pings[f.Data]; ok { close(c) delete(cc.pings, f.Data) @@ -7234,12 +8493,21 @@ func (rl *http2clientConnReadLoop) processPing(f *http2PingFrame) error { } func (rl *http2clientConnReadLoop) processPushPromise(f *http2PushPromiseFrame) error { - + // We told the peer we don't want them. + // Spec says: + // "PUSH_PROMISE MUST NOT be sent if the SETTINGS_ENABLE_PUSH + // setting of the peer endpoint is set to 0. An endpoint that + // has set this setting and has received acknowledgement MUST + // treat the receipt of a PUSH_PROMISE frame as a connection + // error (Section 5.4.1) of type PROTOCOL_ERROR." return http2ConnectionError(http2ErrCodeProtocol) } func (cc *http2ClientConn) writeStreamReset(streamID uint32, code http2ErrCode, err error) { - + // TODO: map err to more interesting error codes, once the + // HTTP community comes up with some. But currently for + // RST_STREAM there's no equivalent to GOAWAY frame's debug + // data, and the error codes are all pretty vague ("cancel"). cc.wmu.Lock() cc.fr.WriteRSTStream(streamID, code) cc.bw.Flush() @@ -7368,7 +8636,8 @@ func (s http2bodyWriterState) cancel() { func (s http2bodyWriterState) on100() { if s.timer == nil { - + // If we didn't do a delayed write, ignore the server's + // bogus 100 continue response. return } s.timer.Stop() @@ -7380,7 +8649,9 @@ func (s http2bodyWriterState) on100() { // called until after the headers have been written. func (s http2bodyWriterState) scheduleBodyWrite() { if s.timer == nil { - + // We're not doing a delayed write (see + // getBodyWriterState), so just start the writing + // goroutine immediately. go s.fn() return } @@ -7435,7 +8706,9 @@ func http2writeEndsStream(w http2writeFramer) bool { case *http2writeResHeaders: return v.endStream case nil: - + // This can only happen if the caller reuses w after it's + // been intentionally nil'ed out to prevent use. Keep this + // here to catch future refactoring breaking it. panic("writeEndsStream called on nil writeFramer") } return false @@ -7469,14 +8742,14 @@ type http2writeGoAway struct { func (p *http2writeGoAway) writeFrame(ctx http2writeContext) error { err := ctx.Framer().WriteGoAway(p.maxStreamID, p.code, nil) if p.code != 0 { - ctx.Flush() + ctx.Flush() // ignore error: we're hanging up on them anyway time.Sleep(50 * time.Millisecond) ctx.CloseConn() } return err } -func (*http2writeGoAway) staysWithinBuffer(max int) bool { return false } +func (*http2writeGoAway) staysWithinBuffer(max int) bool { return false } // flushes type http2writeData struct { streamID uint32 @@ -7581,7 +8854,13 @@ func http2encKV(enc *hpack.Encoder, k, v string) { } func (w *http2writeResHeaders) staysWithinBuffer(max int) bool { - + // TODO: this is a common one. It'd be nice to return true + // here and get into the fast path if we could be clever and + // calculate the size fast enough, or at least a conservative + // uppper bound that usually fires. (Maybe if w.h and + // w.trailers are nil, so we don't need to enumerate it.) + // Otherwise I'm afraid that just calculating the length to + // answer this question would be slower than the ~2µs benefit. return false } @@ -7640,7 +8919,7 @@ type http2writePushPromise struct { } func (w *http2writePushPromise) staysWithinBuffer(max int) bool { - + // TODO: see writeResHeaders.staysWithinBuffer return false } @@ -7692,7 +8971,7 @@ func (w http2write100ContinueHeadersFrame) writeFrame(ctx http2writeContext) err } func (w http2write100ContinueHeadersFrame) staysWithinBuffer(max int) bool { - + // Sloppy but conservative: return 9+2*(len(":status")+len("100")) <= max } @@ -7712,7 +8991,9 @@ func (wu http2writeWindowUpdate) writeFrame(ctx http2writeContext) error { func http2encodeHeaders(enc *hpack.Encoder, h Header, keys []string) { if keys == nil { sorter := http2sorterPool.Get().(*http2sorter) - + // Using defer here, since the returned keys from the + // sorter.Keys method is only valid until the sorter + // is returned: defer http2sorterPool.Put(sorter) keys = sorter.Keys(h) } @@ -7720,16 +9001,19 @@ func http2encodeHeaders(enc *hpack.Encoder, h Header, keys []string) { vv := h[k] k = http2lowerHeader(k) if !http2validWireHeaderFieldName(k) { - + // Skip it as backup paranoia. Per + // golang.org/issue/14048, these should + // already be rejected at a higher level. continue } isTE := k == "transfer-encoding" for _, v := range vv { if !httplex.ValidHeaderFieldValue(v) { - + // TODO: return an error? golang.org/issue/14048 + // For now just omit it. continue } - + // TODO: more of "8.1.2.2 Connection-Specific Header Fields" if isTE && v != "trailers" { continue } @@ -7797,7 +9081,10 @@ type http2FrameWriteRequest struct { func (wr http2FrameWriteRequest) StreamID() uint32 { if wr.stream == nil { if se, ok := wr.write.(http2StreamError); ok { - + // (*serverConn).resetStream doesn't set + // stream because it doesn't necessarily have + // one. So special case this type of write + // message. return se.StreamID } return 0 @@ -7827,11 +9114,13 @@ func (wr http2FrameWriteRequest) DataSize() int { func (wr http2FrameWriteRequest) Consume(n int32) (http2FrameWriteRequest, http2FrameWriteRequest, int) { var empty http2FrameWriteRequest + // Non-DATA frames are always consumed whole. wd, ok := wr.write.(*http2writeData) if !ok || len(wd.p) == 0 { return wr, empty, 1 } + // Might need to split after applying limits. allowed := wr.stream.flow.available() if n < allowed { allowed = n @@ -7849,10 +9138,13 @@ func (wr http2FrameWriteRequest) Consume(n int32) (http2FrameWriteRequest, http2 write: &http2writeData{ streamID: wd.streamID, p: wd.p[:allowed], - + // Even if the original had endStream set, there + // are bytes remaining because len(wd.p) > allowed, + // so we know endStream is false. endStream: false, }, - + // Our caller is blocking on the final DATA frame, not + // this intermediate frame, so no need to wait. done: nil, } rest := http2FrameWriteRequest{ @@ -7867,6 +9159,8 @@ func (wr http2FrameWriteRequest) Consume(n int32) (http2FrameWriteRequest, http2 return consumed, rest, 2 } + // The frame is consumed whole. + // NB: This cast cannot overflow because allowed is <= math.MaxInt32. wr.stream.flow.take(int32(len(wd.p))) return wr, empty, 1 } @@ -7893,7 +9187,7 @@ func (wr *http2FrameWriteRequest) replyToWriter(err error) { default: panic(fmt.Sprintf("unbuffered done channel passed in for type %T", wr.write)) } - wr.write = nil + wr.write = nil // prevent use (assume it's tainted after wr.done send) } // writeQueue is used by implementations of WriteScheduler. @@ -7912,7 +9206,7 @@ func (q *http2writeQueue) shift() http2FrameWriteRequest { panic("invalid use of queue") } wr := q.s[0] - + // TODO: less copy-happy queue. copy(q.s, q.s[1:]) q.s[len(q.s)-1] = http2FrameWriteRequest{} q.s = q.s[:len(q.s)-1] @@ -7941,6 +9235,8 @@ func (q *http2writeQueue) consume(n int32) (http2FrameWriteRequest, bool) { type http2writeQueuePool []*http2writeQueue +// put inserts an unused writeQueue into the pool. + // put inserts an unused writeQueue into the pool. func (p *http2writeQueuePool) put(q *http2writeQueue) { for i := range q.s { @@ -8006,11 +9302,12 @@ type http2PriorityWriteSchedulerConfig struct { } // NewPriorityWriteScheduler constructs a WriteScheduler that schedules -// frames by following HTTP/2 priorities as described in RFC 7340 Section 5.3. +// frames by following HTTP/2 priorities as described in RFC 7540 Section 5.3. // If cfg is nil, default options are used. func http2NewPriorityWriteScheduler(cfg *http2PriorityWriteSchedulerConfig) http2WriteScheduler { if cfg == nil { - + // For justification of these defaults, see: + // https://docs.google.com/document/d/1oLhNg1skaWD4_DtaoCxdSRN5erEXrH-KnLrMwEpOtFY cfg = &http2PriorityWriteSchedulerConfig{ MaxClosedNodesInTree: 10, MaxIdleNodesInTree: 10, @@ -8065,7 +9362,7 @@ func (n *http2priorityNode) setParent(parent *http2priorityNode) { if n.parent == parent { return } - + // Unlink from current parent. if parent := n.parent; parent != nil { if n.prev == nil { parent.kids = n.next @@ -8076,7 +9373,9 @@ func (n *http2priorityNode) setParent(parent *http2priorityNode) { n.next.prev = n.prev } } - + // Link to new parent. + // If parent=nil, remove n from the tree. + // Always insert at the head of parent.kids (this is assumed by walkReadyInOrder). n.parent = parent if parent == nil { n.next = nil @@ -8112,10 +9411,15 @@ func (n *http2priorityNode) walkReadyInOrder(openParent bool, tmp *[]*http2prior return false } + // Don't consider the root "open" when updating openParent since + // we can't send data frames on the root stream (only control frames). if n.id != 0 { openParent = openParent || (n.state == http2priorityNodeOpen) } + // Common case: only one kid or all kids have the same weight. + // Some clients don't use weights; other clients (like web browsers) + // use mostly-linear priority trees. w := n.kids.weight needSort := false for k := n.kids.next; k != nil; k = k.next { @@ -8133,6 +9437,8 @@ func (n *http2priorityNode) walkReadyInOrder(openParent bool, tmp *[]*http2prior return false } + // Uncommon case: sort the child nodes. We remove the kids from the parent, + // then re-insert after sorting so we can reuse tmp for future sort calls. *tmp = (*tmp)[:0] for n.kids != nil { *tmp = append(*tmp, n.kids) @@ -8140,7 +9446,7 @@ func (n *http2priorityNode) walkReadyInOrder(openParent bool, tmp *[]*http2prior } sort.Sort(http2sortPriorityNodeSiblings(*tmp)) for i := len(*tmp) - 1; i >= 0; i-- { - (*tmp)[i].setParent(n) + (*tmp)[i].setParent(n) // setParent inserts at the head of n.kids } for k := n.kids; k != nil; k = k.next { if k.walkReadyInOrder(openParent, tmp, f) { @@ -8157,7 +9463,8 @@ func (z http2sortPriorityNodeSiblings) Len() int { return len(z) } func (z http2sortPriorityNodeSiblings) Swap(i, k int) { z[i], z[k] = z[k], z[i] } func (z http2sortPriorityNodeSiblings) Less(i, k int) bool { - + // Prefer the subtree that has sent fewer bytes relative to its weight. + // See sections 5.3.2 and 5.3.4. wi, bi := float64(z[i].weight+1), float64(z[i].subtreeBytes) wk, bk := float64(z[k].weight+1), float64(z[k].subtreeBytes) if bi == 0 && bk == 0 { @@ -8199,7 +9506,7 @@ type http2priorityWriteScheduler struct { } func (ws *http2priorityWriteScheduler) OpenStream(streamID uint32, options http2OpenStreamOptions) { - + // The stream may be currently idle but cannot be opened or closed. if curr := ws.nodes[streamID]; curr != nil { if curr.state != http2priorityNodeIdle { panic(fmt.Sprintf("stream %d already opened", streamID)) @@ -8208,6 +9515,10 @@ func (ws *http2priorityWriteScheduler) OpenStream(streamID uint32, options http2 return } + // RFC 7540, Section 5.3.5: + // "All streams are initially assigned a non-exclusive dependency on stream 0x0. + // Pushed streams initially depend on their associated stream. In both cases, + // streams are assigned a default weight of 16." parent := ws.nodes[options.PusherID] if parent == nil { parent = &ws.root @@ -8255,6 +9566,9 @@ func (ws *http2priorityWriteScheduler) AdjustStream(streamID uint32, priority ht panic("adjustPriority on root") } + // If streamID does not exist, there are two cases: + // - A closed stream that has been removed (this will have ID <= maxID) + // - An idle stream that is being used for "grouping" (this will have ID > maxID) n := ws.nodes[streamID] if n == nil { if streamID <= ws.maxID || ws.maxIdleNodesInTree == 0 { @@ -8272,6 +9586,8 @@ func (ws *http2priorityWriteScheduler) AdjustStream(streamID uint32, priority ht ws.addClosedOrIdleNode(&ws.idleNodes, ws.maxIdleNodesInTree, n) } + // Section 5.3.1: A dependency on a stream that is not currently in the tree + // results in that stream being given a default priority (Section 5.3.5). parent := ws.nodes[priority.StreamDep] if parent == nil { n.setParent(&ws.root) @@ -8279,10 +9595,18 @@ func (ws *http2priorityWriteScheduler) AdjustStream(streamID uint32, priority ht return } + // Ignore if the client tries to make a node its own parent. if n == parent { return } + // Section 5.3.3: + // "If a stream is made dependent on one of its own dependencies, the + // formerly dependent stream is first moved to be dependent on the + // reprioritized stream's previous parent. The moved dependency retains + // its weight." + // + // That is: if parent depends on n, move parent to depend on n.parent. for x := parent.parent; x != nil; x = x.parent { if x == n { parent.setParent(n.parent) @@ -8290,6 +9614,9 @@ func (ws *http2priorityWriteScheduler) AdjustStream(streamID uint32, priority ht } } + // Section 5.3.3: The exclusive flag causes the stream to become the sole + // dependency of its parent stream, causing other dependencies to become + // dependent on the exclusive stream. if priority.Exclusive { k := parent.kids for k != nil { @@ -8312,7 +9639,11 @@ func (ws *http2priorityWriteScheduler) Push(wr http2FrameWriteRequest) { } else { n = ws.nodes[id] if n == nil { - + // id is an idle or closed stream. wr should not be a HEADERS or + // DATA frame. However, wr can be a RST_STREAM. In this case, we + // push wr onto the root, rather than creating a new priorityNode, + // since RST_STREAM is tiny and the stream's priority is unknown + // anyway. See issue #17919. if wr.DataSize() > 0 { panic("add DATA on non-open stream") } @@ -8333,7 +9664,9 @@ func (ws *http2priorityWriteScheduler) Pop() (wr http2FrameWriteRequest, ok bool return false } n.addBytes(int64(wr.DataSize())) - + // If B depends on A and B continuously has data available but A + // does not, gradually increase the throttling limit to allow B to + // steal more and more bandwidth from A. if openParent { ws.writeThrottleLimit += 1024 if ws.writeThrottleLimit < 0 { @@ -8352,7 +9685,7 @@ func (ws *http2priorityWriteScheduler) addClosedOrIdleNode(list *[]*http2priorit return } if len(*list) == maxSize { - + // Remove the oldest node, then shift left. ws.removeNode((*list)[0]) x := (*list)[1:] copy(*list, x) @@ -8390,7 +9723,7 @@ type http2randomWriteScheduler struct { } func (ws *http2randomWriteScheduler) OpenStream(streamID uint32, options http2OpenStreamOptions) { - + // no-op: idle streams are not tracked } func (ws *http2randomWriteScheduler) CloseStream(streamID uint32) { @@ -8403,7 +9736,7 @@ func (ws *http2randomWriteScheduler) CloseStream(streamID uint32) { } func (ws *http2randomWriteScheduler) AdjustStream(streamID uint32, priority http2PriorityParam) { - + // no-op: priorities are ignored } func (ws *http2randomWriteScheduler) Push(wr http2FrameWriteRequest) { @@ -8421,11 +9754,11 @@ func (ws *http2randomWriteScheduler) Push(wr http2FrameWriteRequest) { } func (ws *http2randomWriteScheduler) Pop() (http2FrameWriteRequest, bool) { - + // Control frames first. if !ws.zero.empty() { return ws.zero.shift(), true } - + // Iterate over all non-idle streams until finding one that can be consumed. for _, q := range ws.sq { if wr, ok := q.consume(math.MaxInt32); ok { return wr, true diff --git a/libgo/go/net/http/httptest/recorder.go b/libgo/go/net/http/httptest/recorder.go index 5f1aa6af479..741f076b363 100644 --- a/libgo/go/net/http/httptest/recorder.go +++ b/libgo/go/net/http/httptest/recorder.go @@ -6,6 +6,7 @@ package httptest import ( "bytes" + "fmt" "io/ioutil" "net/http" "strconv" @@ -176,7 +177,7 @@ func (rw *ResponseRecorder) Result() *http.Response { if res.StatusCode == 0 { res.StatusCode = 200 } - res.Status = http.StatusText(res.StatusCode) + res.Status = fmt.Sprintf("%03d %s", res.StatusCode, http.StatusText(res.StatusCode)) if rw.Body != nil { res.Body = ioutil.NopCloser(bytes.NewReader(rw.Body.Bytes())) } diff --git a/libgo/go/net/http/httptest/recorder_test.go b/libgo/go/net/http/httptest/recorder_test.go index 9afba4e556a..a6259ebac71 100644 --- a/libgo/go/net/http/httptest/recorder_test.go +++ b/libgo/go/net/http/httptest/recorder_test.go @@ -23,7 +23,15 @@ func TestRecorder(t *testing.T) { return nil } } - hasResultStatus := func(wantCode int) checkFunc { + hasResultStatus := func(want string) checkFunc { + return func(rec *ResponseRecorder) error { + if rec.Result().Status != want { + return fmt.Errorf("Result().Status = %q; want %q", rec.Result().Status, want) + } + return nil + } + } + hasResultStatusCode := func(wantCode int) checkFunc { return func(rec *ResponseRecorder) error { if rec.Result().StatusCode != wantCode { return fmt.Errorf("Result().StatusCode = %d; want %d", rec.Result().StatusCode, wantCode) @@ -235,7 +243,8 @@ func TestRecorder(t *testing.T) { hasOldHeader("X-Foo", "1"), hasStatus(0), hasHeader("X-Foo", "1"), - hasResultStatus(200), + hasResultStatus("200 OK"), + hasResultStatusCode(200), ), }, { diff --git a/libgo/go/net/http/httptest/server.go b/libgo/go/net/http/httptest/server.go index 5e9ace591f3..e543672b1e8 100644 --- a/libgo/go/net/http/httptest/server.go +++ b/libgo/go/net/http/httptest/server.go @@ -9,6 +9,7 @@ package httptest import ( "bytes" "crypto/tls" + "crypto/x509" "flag" "fmt" "log" @@ -35,6 +36,9 @@ type Server struct { // before Start or StartTLS. Config *http.Server + // certificate is a parsed version of the TLS config certificate, if present. + certificate *x509.Certificate + // wg counts the number of outstanding HTTP requests on this server. // Close blocks until all requests are finished. wg sync.WaitGroup @@ -42,6 +46,10 @@ type Server struct { mu sync.Mutex // guards closed and conns closed bool conns map[net.Conn]http.ConnState // except terminal states + + // client is configured for use with the server. + // Its transport is automatically closed when Close is called. + client *http.Client } func newLocalListener() net.Listener { @@ -93,6 +101,9 @@ func (s *Server) Start() { if s.URL != "" { panic("Server already started") } + if s.client == nil { + s.client = &http.Client{Transport: &http.Transport{}} + } s.URL = "http://" + s.Listener.Addr().String() s.wrap() s.goServe() @@ -107,6 +118,9 @@ func (s *Server) StartTLS() { if s.URL != "" { panic("Server already started") } + if s.client == nil { + s.client = &http.Client{Transport: &http.Transport{}} + } cert, err := tls.X509KeyPair(internal.LocalhostCert, internal.LocalhostKey) if err != nil { panic(fmt.Sprintf("httptest: NewTLSServer: %v", err)) @@ -124,6 +138,17 @@ func (s *Server) StartTLS() { if len(s.TLS.Certificates) == 0 { s.TLS.Certificates = []tls.Certificate{cert} } + s.certificate, err = x509.ParseCertificate(s.TLS.Certificates[0].Certificate[0]) + if err != nil { + panic(fmt.Sprintf("httptest: NewTLSServer: %v", err)) + } + certpool := x509.NewCertPool() + certpool.AddCert(s.certificate) + s.client.Transport = &http.Transport{ + TLSClientConfig: &tls.Config{ + RootCAs: certpool, + }, + } s.Listener = tls.NewListener(s.Listener, s.TLS) s.URL = "https://" + s.Listener.Addr().String() s.wrap() @@ -186,6 +211,13 @@ func (s *Server) Close() { t.CloseIdleConnections() } + // Also close the client idle connections. + if s.client != nil { + if t, ok := s.client.Transport.(closeIdleTransport); ok { + t.CloseIdleConnections() + } + } + s.wg.Wait() } @@ -206,7 +238,7 @@ func (s *Server) CloseClientConnections() { nconn := len(s.conns) ch := make(chan struct{}, nconn) for c := range s.conns { - s.closeConnChan(c, ch) + go s.closeConnChan(c, ch) } s.mu.Unlock() @@ -228,6 +260,19 @@ func (s *Server) CloseClientConnections() { } } +// Certificate returns the certificate used by the server, or nil if +// the server doesn't use TLS. +func (s *Server) Certificate() *x509.Certificate { + return s.certificate +} + +// Client returns an HTTP client configured for making requests to the server. +// It is configured to trust the server's TLS test certificate and will +// close its idle connections on Server.Close. +func (s *Server) Client() *http.Client { + return s.client +} + func (s *Server) goServe() { s.wg.Add(1) go func() { diff --git a/libgo/go/net/http/httptest/server_test.go b/libgo/go/net/http/httptest/server_test.go index d032c5983b7..8ab50cdb0ab 100644 --- a/libgo/go/net/http/httptest/server_test.go +++ b/libgo/go/net/http/httptest/server_test.go @@ -12,8 +12,48 @@ import ( "testing" ) +type newServerFunc func(http.Handler) *Server + +var newServers = map[string]newServerFunc{ + "NewServer": NewServer, + "NewTLSServer": NewTLSServer, + + // The manual variants of newServer create a Server manually by only filling + // in the exported fields of Server. + "NewServerManual": func(h http.Handler) *Server { + ts := &Server{Listener: newLocalListener(), Config: &http.Server{Handler: h}} + ts.Start() + return ts + }, + "NewTLSServerManual": func(h http.Handler) *Server { + ts := &Server{Listener: newLocalListener(), Config: &http.Server{Handler: h}} + ts.StartTLS() + return ts + }, +} + func TestServer(t *testing.T) { - ts := NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + for _, name := range []string{"NewServer", "NewServerManual"} { + t.Run(name, func(t *testing.T) { + newServer := newServers[name] + t.Run("Server", func(t *testing.T) { testServer(t, newServer) }) + t.Run("GetAfterClose", func(t *testing.T) { testGetAfterClose(t, newServer) }) + t.Run("ServerCloseBlocking", func(t *testing.T) { testServerCloseBlocking(t, newServer) }) + t.Run("ServerCloseClientConnections", func(t *testing.T) { testServerCloseClientConnections(t, newServer) }) + t.Run("ServerClientTransportType", func(t *testing.T) { testServerClientTransportType(t, newServer) }) + }) + } + for _, name := range []string{"NewTLSServer", "NewTLSServerManual"} { + t.Run(name, func(t *testing.T) { + newServer := newServers[name] + t.Run("ServerClient", func(t *testing.T) { testServerClient(t, newServer) }) + t.Run("TLSServerClientTransportType", func(t *testing.T) { testTLSServerClientTransportType(t, newServer) }) + }) + } +} + +func testServer(t *testing.T, newServer newServerFunc) { + ts := newServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.Write([]byte("hello")) })) defer ts.Close() @@ -22,6 +62,7 @@ func TestServer(t *testing.T) { t.Fatal(err) } got, err := ioutil.ReadAll(res.Body) + res.Body.Close() if err != nil { t.Fatal(err) } @@ -31,8 +72,8 @@ func TestServer(t *testing.T) { } // Issue 12781 -func TestGetAfterClose(t *testing.T) { - ts := NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { +func testGetAfterClose(t *testing.T, newServer newServerFunc) { + ts := newServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.Write([]byte("hello")) })) @@ -57,8 +98,8 @@ func TestGetAfterClose(t *testing.T) { } } -func TestServerCloseBlocking(t *testing.T) { - ts := NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { +func testServerCloseBlocking(t *testing.T, newServer newServerFunc) { + ts := newServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.Write([]byte("hello")) })) dial := func() net.Conn { @@ -86,9 +127,9 @@ func TestServerCloseBlocking(t *testing.T) { } // Issue 14290 -func TestServerCloseClientConnections(t *testing.T) { +func testServerCloseClientConnections(t *testing.T, newServer newServerFunc) { var s *Server - s = NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + s = newServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { s.CloseClientConnections() })) defer s.Close() @@ -98,3 +139,66 @@ func TestServerCloseClientConnections(t *testing.T) { t.Fatalf("Unexpected response: %#v", res) } } + +// Tests that the Server.Client method works and returns an http.Client that can hit +// NewTLSServer without cert warnings. +func testServerClient(t *testing.T, newTLSServer newServerFunc) { + ts := newTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Write([]byte("hello")) + })) + defer ts.Close() + client := ts.Client() + res, err := client.Get(ts.URL) + if err != nil { + t.Fatal(err) + } + got, err := ioutil.ReadAll(res.Body) + res.Body.Close() + if err != nil { + t.Fatal(err) + } + if string(got) != "hello" { + t.Errorf("got %q, want hello", string(got)) + } +} + +// Tests that the Server.Client.Transport interface is implemented +// by a *http.Transport. +func testServerClientTransportType(t *testing.T, newServer newServerFunc) { + ts := newServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + })) + defer ts.Close() + client := ts.Client() + if _, ok := client.Transport.(*http.Transport); !ok { + t.Errorf("got %T, want *http.Transport", client.Transport) + } +} + +// Tests that the TLS Server.Client.Transport interface is implemented +// by a *http.Transport. +func testTLSServerClientTransportType(t *testing.T, newTLSServer newServerFunc) { + ts := newTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + })) + defer ts.Close() + client := ts.Client() + if _, ok := client.Transport.(*http.Transport); !ok { + t.Errorf("got %T, want *http.Transport", client.Transport) + } +} + +type onlyCloseListener struct { + net.Listener +} + +func (onlyCloseListener) Close() error { return nil } + +// Issue 19729: panic in Server.Close for values created directly +// without a constructor (so the unexported client field is nil). +func TestServerZeroValueClose(t *testing.T) { + ts := &Server{ + Listener: onlyCloseListener{}, + Config: &http.Server{}, + } + + ts.Close() // tests that it doesn't panic +} diff --git a/libgo/go/net/http/httputil/reverseproxy.go b/libgo/go/net/http/httputil/reverseproxy.go index 79c8fe27702..0d514f529ba 100644 --- a/libgo/go/net/http/httputil/reverseproxy.go +++ b/libgo/go/net/http/httputil/reverseproxy.go @@ -114,6 +114,16 @@ func copyHeader(dst, src http.Header) { } } +func cloneHeader(h http.Header) http.Header { + h2 := make(http.Header, len(h)) + for k, vv := range h { + vv2 := make([]string, len(vv)) + copy(vv2, vv) + h2[k] = vv2 + } + return h2 +} + // Hop-by-hop headers. These are removed when sent to the backend. // http://www.w3.org/Protocols/rfc2616/rfc2616-sec13.html var hopHeaders = []string{ @@ -149,30 +159,21 @@ func (p *ReverseProxy) ServeHTTP(rw http.ResponseWriter, req *http.Request) { }() } - outreq := new(http.Request) - *outreq = *req // includes shallow copies of maps, but okay + outreq := req.WithContext(ctx) // includes shallow copies of maps, but okay if req.ContentLength == 0 { outreq.Body = nil // Issue 16036: nil Body for http.Transport retries } - outreq = outreq.WithContext(ctx) + + outreq.Header = cloneHeader(req.Header) p.Director(outreq) outreq.Close = false - // We are modifying the same underlying map from req (shallow - // copied above) so we only copy it if necessary. - copiedHeaders := false - // Remove hop-by-hop headers listed in the "Connection" header. // See RFC 2616, section 14.10. if c := outreq.Header.Get("Connection"); c != "" { for _, f := range strings.Split(c, ",") { if f = strings.TrimSpace(f); f != "" { - if !copiedHeaders { - outreq.Header = make(http.Header) - copyHeader(outreq.Header, req.Header) - copiedHeaders = true - } outreq.Header.Del(f) } } @@ -183,11 +184,6 @@ func (p *ReverseProxy) ServeHTTP(rw http.ResponseWriter, req *http.Request) { // connection, regardless of what the client sent to us. for _, h := range hopHeaders { if outreq.Header.Get(h) != "" { - if !copiedHeaders { - outreq.Header = make(http.Header) - copyHeader(outreq.Header, req.Header) - copiedHeaders = true - } outreq.Header.Del(h) } } @@ -235,7 +231,8 @@ func (p *ReverseProxy) ServeHTTP(rw http.ResponseWriter, req *http.Request) { // The "Trailer" header isn't included in the Transport's response, // at least for *http.Transport. Build it up from Trailer. - if len(res.Trailer) > 0 { + announcedTrailers := len(res.Trailer) + if announcedTrailers > 0 { trailerKeys := make([]string, 0, len(res.Trailer)) for k := range res.Trailer { trailerKeys = append(trailerKeys, k) @@ -254,7 +251,18 @@ func (p *ReverseProxy) ServeHTTP(rw http.ResponseWriter, req *http.Request) { } p.copyResponse(rw, res.Body) res.Body.Close() // close now, instead of defer, to populate res.Trailer - copyHeader(rw.Header(), res.Trailer) + + if len(res.Trailer) == announcedTrailers { + copyHeader(rw.Header(), res.Trailer) + return + } + + for k, vv := range res.Trailer { + k = http.TrailerPrefix + k + for _, v := range vv { + rw.Header().Add(k, v) + } + } } func (p *ReverseProxy) copyResponse(dst io.Writer, src io.Reader) { @@ -288,7 +296,7 @@ func (p *ReverseProxy) copyBuffer(dst io.Writer, src io.Reader, buf []byte) (int var written int64 for { nr, rerr := src.Read(buf) - if rerr != nil && rerr != io.EOF { + if rerr != nil && rerr != io.EOF && rerr != context.Canceled { p.logf("httputil: ReverseProxy read error during body copy: %v", rerr) } if nr > 0 { diff --git a/libgo/go/net/http/httputil/reverseproxy_test.go b/libgo/go/net/http/httputil/reverseproxy_test.go index 20c4e16bcb8..37a9992375d 100644 --- a/libgo/go/net/http/httputil/reverseproxy_test.go +++ b/libgo/go/net/http/httputil/reverseproxy_test.go @@ -69,6 +69,7 @@ func TestReverseProxy(t *testing.T) { w.WriteHeader(backendStatus) w.Write([]byte(backendResponse)) w.Header().Set("X-Trailer", "trailer_value") + w.Header().Set(http.TrailerPrefix+"X-Unannounced-Trailer", "unannounced_trailer_value") })) defer backend.Close() backendURL, err := url.Parse(backend.URL) @@ -79,6 +80,7 @@ func TestReverseProxy(t *testing.T) { proxyHandler.ErrorLog = log.New(ioutil.Discard, "", 0) // quiet for tests frontend := httptest.NewServer(proxyHandler) defer frontend.Close() + frontendClient := frontend.Client() getReq, _ := http.NewRequest("GET", frontend.URL, nil) getReq.Host = "some-name" @@ -86,7 +88,7 @@ func TestReverseProxy(t *testing.T) { getReq.Header.Set("Proxy-Connection", "should be deleted") getReq.Header.Set("Upgrade", "foo") getReq.Close = true - res, err := http.DefaultClient.Do(getReq) + res, err := frontendClient.Do(getReq) if err != nil { t.Fatalf("Get: %v", err) } @@ -121,12 +123,15 @@ func TestReverseProxy(t *testing.T) { if g, e := res.Trailer.Get("X-Trailer"), "trailer_value"; g != e { t.Errorf("Trailer(X-Trailer) = %q ; want %q", g, e) } + if g, e := res.Trailer.Get("X-Unannounced-Trailer"), "unannounced_trailer_value"; g != e { + t.Errorf("Trailer(X-Unannounced-Trailer) = %q ; want %q", g, e) + } // Test that a backend failing to be reached or one which doesn't return // a response results in a StatusBadGateway. getReq, _ = http.NewRequest("GET", frontend.URL+"/?mode=hangup", nil) getReq.Close = true - res, err = http.DefaultClient.Do(getReq) + res, err = frontendClient.Do(getReq) if err != nil { t.Fatal(err) } @@ -172,7 +177,7 @@ func TestReverseProxyStripHeadersPresentInConnection(t *testing.T) { getReq.Header.Set("Connection", "Upgrade, "+fakeConnectionToken) getReq.Header.Set("Upgrade", "original value") getReq.Header.Set(fakeConnectionToken, "should be deleted") - res, err := http.DefaultClient.Do(getReq) + res, err := frontend.Client().Do(getReq) if err != nil { t.Fatalf("Get: %v", err) } @@ -220,7 +225,7 @@ func TestXForwardedFor(t *testing.T) { getReq.Header.Set("Connection", "close") getReq.Header.Set("X-Forwarded-For", prevForwardedFor) getReq.Close = true - res, err := http.DefaultClient.Do(getReq) + res, err := frontend.Client().Do(getReq) if err != nil { t.Fatalf("Get: %v", err) } @@ -259,7 +264,7 @@ func TestReverseProxyQuery(t *testing.T) { frontend := httptest.NewServer(NewSingleHostReverseProxy(backendURL)) req, _ := http.NewRequest("GET", frontend.URL+tt.reqSuffix, nil) req.Close = true - res, err := http.DefaultClient.Do(req) + res, err := frontend.Client().Do(req) if err != nil { t.Fatalf("%d. Get: %v", i, err) } @@ -295,7 +300,7 @@ func TestReverseProxyFlushInterval(t *testing.T) { req, _ := http.NewRequest("GET", frontend.URL, nil) req.Close = true - res, err := http.DefaultClient.Do(req) + res, err := frontend.Client().Do(req) if err != nil { t.Fatalf("Get: %v", err) } @@ -349,13 +354,14 @@ func TestReverseProxyCancelation(t *testing.T) { frontend := httptest.NewServer(proxyHandler) defer frontend.Close() + frontendClient := frontend.Client() getReq, _ := http.NewRequest("GET", frontend.URL, nil) go func() { <-reqInFlight - http.DefaultTransport.(*http.Transport).CancelRequest(getReq) + frontendClient.Transport.(*http.Transport).CancelRequest(getReq) }() - res, err := http.DefaultClient.Do(getReq) + res, err := frontendClient.Do(getReq) if res != nil { t.Errorf("got response %v; want nil", res.Status) } @@ -363,7 +369,7 @@ func TestReverseProxyCancelation(t *testing.T) { // This should be an error like: // Get http://127.0.0.1:58079: read tcp 127.0.0.1:58079: // use of closed network connection - t.Error("DefaultClient.Do() returned nil error; want non-nil error") + t.Error("Server.Client().Do() returned nil error; want non-nil error") } } @@ -428,11 +434,12 @@ func TestUserAgentHeader(t *testing.T) { proxyHandler.ErrorLog = log.New(ioutil.Discard, "", 0) // quiet for tests frontend := httptest.NewServer(proxyHandler) defer frontend.Close() + frontendClient := frontend.Client() getReq, _ := http.NewRequest("GET", frontend.URL, nil) getReq.Header.Set("User-Agent", explicitUA) getReq.Close = true - res, err := http.DefaultClient.Do(getReq) + res, err := frontendClient.Do(getReq) if err != nil { t.Fatalf("Get: %v", err) } @@ -441,7 +448,7 @@ func TestUserAgentHeader(t *testing.T) { getReq, _ = http.NewRequest("GET", frontend.URL+"/noua", nil) getReq.Header.Set("User-Agent", "") getReq.Close = true - res, err = http.DefaultClient.Do(getReq) + res, err = frontendClient.Do(getReq) if err != nil { t.Fatalf("Get: %v", err) } @@ -493,7 +500,7 @@ func TestReverseProxyGetPutBuffer(t *testing.T) { req, _ := http.NewRequest("GET", frontend.URL, nil) req.Close = true - res, err := http.DefaultClient.Do(req) + res, err := frontend.Client().Do(req) if err != nil { t.Fatalf("Get: %v", err) } @@ -540,7 +547,7 @@ func TestReverseProxy_Post(t *testing.T) { defer frontend.Close() postReq, _ := http.NewRequest("POST", frontend.URL, bytes.NewReader(requestBody)) - res, err := http.DefaultClient.Do(postReq) + res, err := frontend.Client().Do(postReq) if err != nil { t.Fatalf("Do: %v", err) } @@ -573,7 +580,7 @@ func TestReverseProxy_NilBody(t *testing.T) { frontend := httptest.NewServer(proxyHandler) defer frontend.Close() - res, err := http.DefaultClient.Get(frontend.URL) + res, err := frontend.Client().Get(frontend.URL) if err != nil { t.Fatal(err) } @@ -664,3 +671,101 @@ func TestReverseProxy_CopyBuffer(t *testing.T) { } } } + +type staticTransport struct { + res *http.Response +} + +func (t *staticTransport) RoundTrip(r *http.Request) (*http.Response, error) { + return t.res, nil +} + +func BenchmarkServeHTTP(b *testing.B) { + res := &http.Response{ + StatusCode: 200, + Body: ioutil.NopCloser(strings.NewReader("")), + } + proxy := &ReverseProxy{ + Director: func(*http.Request) {}, + Transport: &staticTransport{res}, + } + + w := httptest.NewRecorder() + r := httptest.NewRequest("GET", "/", nil) + + b.ReportAllocs() + for i := 0; i < b.N; i++ { + proxy.ServeHTTP(w, r) + } +} + +func TestServeHTTPDeepCopy(t *testing.T) { + backend := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Write([]byte("Hello Gopher!")) + })) + defer backend.Close() + backendURL, err := url.Parse(backend.URL) + if err != nil { + t.Fatal(err) + } + + type result struct { + before, after string + } + + resultChan := make(chan result, 1) + proxyHandler := NewSingleHostReverseProxy(backendURL) + frontend := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + before := r.URL.String() + proxyHandler.ServeHTTP(w, r) + after := r.URL.String() + resultChan <- result{before: before, after: after} + })) + defer frontend.Close() + + want := result{before: "/", after: "/"} + + res, err := frontend.Client().Get(frontend.URL) + if err != nil { + t.Fatalf("Do: %v", err) + } + res.Body.Close() + + got := <-resultChan + if got != want { + t.Errorf("got = %+v; want = %+v", got, want) + } +} + +// Issue 18327: verify we always do a deep copy of the Request.Header map +// before any mutations. +func TestClonesRequestHeaders(t *testing.T) { + req, _ := http.NewRequest("GET", "http://foo.tld/", nil) + req.RemoteAddr = "1.2.3.4:56789" + rp := &ReverseProxy{ + Director: func(req *http.Request) { + req.Header.Set("From-Director", "1") + }, + Transport: roundTripperFunc(func(req *http.Request) (*http.Response, error) { + if v := req.Header.Get("From-Director"); v != "1" { + t.Errorf("From-Directory value = %q; want 1", v) + } + return nil, io.EOF + }), + } + rp.ServeHTTP(httptest.NewRecorder(), req) + + if req.Header.Get("From-Director") == "1" { + t.Error("Director header mutation modified caller's request") + } + if req.Header.Get("X-Forwarded-For") != "" { + t.Error("X-Forward-For header mutation modified caller's request") + } + +} + +type roundTripperFunc func(req *http.Request) (*http.Response, error) + +func (fn roundTripperFunc) RoundTrip(req *http.Request) (*http.Response, error) { + return fn(req) +} diff --git a/libgo/go/net/http/main_test.go b/libgo/go/net/http/main_test.go index 438bd2e58fd..21c850566cc 100644 --- a/libgo/go/net/http/main_test.go +++ b/libgo/go/net/http/main_test.go @@ -37,6 +37,8 @@ func interestingGoroutines() (gs []string) { } stack := strings.TrimSpace(sl[1]) if stack == "" || + strings.Contains(stack, "testing.(*M).before.func1") || + strings.Contains(stack, "os/signal.signal_recv") || strings.Contains(stack, "created by net.startServer") || strings.Contains(stack, "created by testing.RunTests") || strings.Contains(stack, "closeWriteAndWait") || @@ -56,8 +58,9 @@ func interestingGoroutines() (gs []string) { // Verify the other tests didn't leave any goroutines running. func goroutineLeaked() bool { - if testing.Short() { - // not counting goroutines for leakage in -short mode + if testing.Short() || runningBenchmarks() { + // Don't worry about goroutine leaks in -short mode or in + // benchmark mode. Too distracting when there are false positives. return false } @@ -92,6 +95,18 @@ func setParallel(t *testing.T) { } } +func runningBenchmarks() bool { + for i, arg := range os.Args { + if strings.HasPrefix(arg, "-test.bench=") && !strings.HasSuffix(arg, "=") { + return true + } + if arg == "-test.bench" && i < len(os.Args)-1 && os.Args[i+1] != "" { + return true + } + } + return false +} + func afterTest(t testing.TB) { http.DefaultTransport.(*http.Transport).CloseIdleConnections() if testing.Short() { @@ -151,7 +166,3 @@ func waitErrCondition(waitFor, checkEvery time.Duration, fn func() error) error } return err } - -func closeClient(c *http.Client) { - c.Transport.(*http.Transport).CloseIdleConnections() -} diff --git a/libgo/go/net/http/npn_test.go b/libgo/go/net/http/npn_test.go index 4c1f6b573df..618bdbe54a6 100644 --- a/libgo/go/net/http/npn_test.go +++ b/libgo/go/net/http/npn_test.go @@ -8,6 +8,7 @@ import ( "bufio" "bytes" "crypto/tls" + "crypto/x509" "fmt" "io" "io/ioutil" @@ -43,10 +44,7 @@ func TestNextProtoUpgrade(t *testing.T) { // Normal request, without NPN. { - tr := newTLSTransport(t, ts) - defer tr.CloseIdleConnections() - c := &Client{Transport: tr} - + c := ts.Client() res, err := c.Get(ts.URL) if err != nil { t.Fatal(err) @@ -63,11 +61,18 @@ func TestNextProtoUpgrade(t *testing.T) { // Request to an advertised but unhandled NPN protocol. // Server will hang up. { - tr := newTLSTransport(t, ts) - tr.TLSClientConfig.NextProtos = []string{"unhandled-proto"} + certPool := x509.NewCertPool() + certPool.AddCert(ts.Certificate()) + tr := &Transport{ + TLSClientConfig: &tls.Config{ + RootCAs: certPool, + NextProtos: []string{"unhandled-proto"}, + }, + } defer tr.CloseIdleConnections() - c := &Client{Transport: tr} - + c := &Client{ + Transport: tr, + } res, err := c.Get(ts.URL) if err == nil { defer res.Body.Close() @@ -80,7 +85,8 @@ func TestNextProtoUpgrade(t *testing.T) { // Request using the "tls-0.9" protocol, which we register here. // It is HTTP/0.9 over TLS. { - tlsConfig := newTLSTransport(t, ts).TLSClientConfig + c := ts.Client() + tlsConfig := c.Transport.(*Transport).TLSClientConfig tlsConfig.NextProtos = []string{"tls-0.9"} conn, err := tls.Dial("tcp", ts.Listener.Addr().String(), tlsConfig) if err != nil { diff --git a/libgo/go/net/http/pprof/pprof.go b/libgo/go/net/http/pprof/pprof.go index 05d0890fdf3..12c7599ab0f 100644 --- a/libgo/go/net/http/pprof/pprof.go +++ b/libgo/go/net/http/pprof/pprof.go @@ -37,6 +37,11 @@ // // wget http://localhost:6060/debug/pprof/trace?seconds=5 // +// Or to look at the holders of contended mutexes, after calling +// runtime.SetMutexProfileFraction in your program: +// +// go tool pprof http://localhost:6060/debug/pprof/mutex +// // To view all available profiles, open http://localhost:6060/debug/pprof/ // in your browser. // @@ -57,6 +62,7 @@ import ( "os" "runtime" "runtime/pprof" + "runtime/trace" "strconv" "strings" "time" @@ -89,6 +95,11 @@ func sleep(w http.ResponseWriter, d time.Duration) { } } +func durationExceedsWriteTimeout(r *http.Request, seconds float64) bool { + srv, ok := r.Context().Value(http.ServerContextKey).(*http.Server) + return ok && srv.WriteTimeout != 0 && seconds >= srv.WriteTimeout.Seconds() +} + // Profile responds with the pprof-formatted cpu profile. // The package initialization registers it as /debug/pprof/profile. func Profile(w http.ResponseWriter, r *http.Request) { @@ -97,6 +108,14 @@ func Profile(w http.ResponseWriter, r *http.Request) { sec = 30 } + if durationExceedsWriteTimeout(r, float64(sec)) { + w.Header().Set("Content-Type", "text/plain; charset=utf-8") + w.Header().Set("X-Go-Pprof", "1") + w.WriteHeader(http.StatusBadRequest) + fmt.Fprintln(w, "profile duration exceeds server's WriteTimeout") + return + } + // Set Content Type assuming StartCPUProfile will work, // because if it does it starts writing. w.Header().Set("Content-Type", "application/octet-stream") @@ -105,6 +124,7 @@ func Profile(w http.ResponseWriter, r *http.Request) { // Can change header back to text content // and send error code. w.Header().Set("Content-Type", "text/plain; charset=utf-8") + w.Header().Set("X-Go-Pprof", "1") w.WriteHeader(http.StatusInternalServerError) fmt.Fprintf(w, "Could not enable CPU profiling: %s\n", err) return @@ -122,20 +142,28 @@ func Trace(w http.ResponseWriter, r *http.Request) { sec = 1 } + if durationExceedsWriteTimeout(r, sec) { + w.Header().Set("Content-Type", "text/plain; charset=utf-8") + w.Header().Set("X-Go-Pprof", "1") + w.WriteHeader(http.StatusBadRequest) + fmt.Fprintln(w, "profile duration exceeds server's WriteTimeout") + return + } + // Set Content Type assuming trace.Start will work, // because if it does it starts writing. w.Header().Set("Content-Type", "application/octet-stream") - w.Write([]byte("tracing not yet supported with gccgo")) - // if err := trace.Start(w); err != nil { - // // trace.Start failed, so no writes yet. - // // Can change header back to text content and send error code. - // w.Header().Set("Content-Type", "text/plain; charset=utf-8") - // w.WriteHeader(http.StatusInternalServerError) - // fmt.Fprintf(w, "Could not enable tracing: %s\n", err) - // return - // } - // sleep(w, time.Duration(sec*float64(time.Second))) - // trace.Stop() + if err := trace.Start(w); err != nil { + // trace.Start failed, so no writes yet. + // Can change header back to text content and send error code. + w.Header().Set("Content-Type", "text/plain; charset=utf-8") + w.Header().Set("X-Go-Pprof", "1") + w.WriteHeader(http.StatusInternalServerError) + fmt.Fprintf(w, "Could not enable tracing: %s\n", err) + return + } + sleep(w, time.Duration(sec*float64(time.Second))) + trace.Stop() } // Symbol looks up the program counters listed in the request, @@ -207,7 +235,6 @@ func (name handler) ServeHTTP(w http.ResponseWriter, r *http.Request) { runtime.GC() } p.WriteTo(w, debug) - return } // Index responds with the pprof-formatted profile named by the request. diff --git a/libgo/go/net/http/proxy_test.go b/libgo/go/net/http/proxy_test.go index 823d1447ee9..f59a551f0ac 100644 --- a/libgo/go/net/http/proxy_test.go +++ b/libgo/go/net/http/proxy_test.go @@ -75,7 +75,13 @@ func TestCacheKeys(t *testing.T) { func ResetProxyEnv() { for _, v := range []string{"HTTP_PROXY", "http_proxy", "NO_PROXY", "no_proxy"} { - os.Setenv(v, "") + os.Unsetenv(v) } ResetCachedEnvironment() } + +func TestInvalidNoProxy(t *testing.T) { + ResetProxyEnv() + os.Setenv("NO_PROXY", ":1") + useProxy("example.com:80") // should not panic +} diff --git a/libgo/go/net/http/request.go b/libgo/go/net/http/request.go index fb6bb0aab58..13f367c1a8f 100644 --- a/libgo/go/net/http/request.go +++ b/libgo/go/net/http/request.go @@ -27,8 +27,6 @@ import ( "sync" "golang_org/x/net/idna" - "golang_org/x/text/unicode/norm" - "golang_org/x/text/width" ) const ( @@ -331,6 +329,16 @@ func (r *Request) WithContext(ctx context.Context) *Request { r2 := new(Request) *r2 = *r r2.ctx = ctx + + // Deep copy the URL because it isn't + // a map and the URL is mutable by users + // of WithContext. + if r.URL != nil { + r2URL := new(url.URL) + *r2URL = *r.URL + r2.URL = r2URL + } + return r2 } @@ -341,18 +349,6 @@ func (r *Request) ProtoAtLeast(major, minor int) bool { r.ProtoMajor == major && r.ProtoMinor >= minor } -// protoAtLeastOutgoing is like ProtoAtLeast, but is for outgoing -// requests (see issue 18407) where these fields aren't supposed to -// matter. As a minor fix for Go 1.8, at least treat (0, 0) as -// matching HTTP/1.1 or HTTP/1.0. Only HTTP/1.1 is used. -// TODO(bradfitz): ideally remove this whole method. It shouldn't be used. -func (r *Request) protoAtLeastOutgoing(major, minor int) bool { - if r.ProtoMajor == 0 && r.ProtoMinor == 0 && major == 1 && minor <= 1 { - return true - } - return r.ProtoAtLeast(major, minor) -} - // UserAgent returns the client's User-Agent, if sent in the request. func (r *Request) UserAgent() string { return r.Header.Get("User-Agent") @@ -621,6 +617,9 @@ func (req *Request) write(w io.Writer, usingProxy bool, extraHeaders Header, wai // Write body and trailer err = tw.WriteBody(w) if err != nil { + if tw.bodyReadError == err { + err = requestBodyReadError{err} + } return err } @@ -630,17 +629,25 @@ func (req *Request) write(w io.Writer, usingProxy bool, extraHeaders Header, wai return nil } +// requestBodyReadError wraps an error from (*Request).write to indicate +// that the error came from a Read call on the Request.Body. +// This error type should not escape the net/http package to users. +type requestBodyReadError struct{ error } + func idnaASCII(v string) (string, error) { + // TODO: Consider removing this check after verifying performance is okay. + // Right now punycode verification, length checks, context checks, and the + // permissible character tests are all omitted. It also prevents the ToASCII + // call from salvaging an invalid IDN, when possible. As a result it may be + // possible to have two IDNs that appear identical to the user where the + // ASCII-only version causes an error downstream whereas the non-ASCII + // version does not. + // Note that for correct ASCII IDNs ToASCII will only do considerably more + // work, but it will not cause an allocation. if isASCII(v) { return v, nil } - // The idna package doesn't do everything from - // https://tools.ietf.org/html/rfc5895 so we do it here. - // TODO(bradfitz): should the idna package do this instead? - v = strings.ToLower(v) - v = width.Fold.String(v) - v = norm.NFC.String(v) - return idna.ToASCII(v) + return idna.Lookup.ToASCII(v) } // cleanHost cleans up the host sent in request's Host header. @@ -755,7 +762,7 @@ func validMethod(method string) bool { // exact value (instead of -1), GetBody is populated (so 307 and 308 // redirects can replay the body), and Body is set to NoBody if the // ContentLength is 0. -func NewRequest(method, urlStr string, body io.Reader) (*Request, error) { +func NewRequest(method, url string, body io.Reader) (*Request, error) { if method == "" { // We document that "" means "GET" for Request.Method, and people have // relied on that from NewRequest, so keep that working. @@ -765,7 +772,7 @@ func NewRequest(method, urlStr string, body io.Reader) (*Request, error) { if !validMethod(method) { return nil, fmt.Errorf("net/http: invalid method %q", method) } - u, err := url.Parse(urlStr) + u, err := parseURL(url) // Just url.Parse (url is shadowed for godoc). if err != nil { return nil, err } @@ -930,6 +937,9 @@ func readRequest(b *bufio.Reader, deleteHostHeader bool) (req *Request, err erro if !ok { return nil, &badStringError{"malformed HTTP request", s} } + if !validMethod(req.Method) { + return nil, &badStringError{"invalid method", req.Method} + } rawurl := req.RequestURI if req.ProtoMajor, req.ProtoMinor, ok = ParseHTTPVersion(req.Proto); !ok { return nil, &badStringError{"malformed HTTP version", req.Proto} @@ -1021,11 +1031,6 @@ type maxBytesReader struct { err error // sticky error } -func (l *maxBytesReader) tooLarge() (n int, err error) { - l.err = errors.New("http: request body too large") - return 0, l.err -} - func (l *maxBytesReader) Read(p []byte) (n int, err error) { if l.err != nil { return 0, l.err @@ -1297,7 +1302,7 @@ func (r *Request) closeBody() { } func (r *Request) isReplayable() bool { - if r.Body == nil { + if r.Body == nil || r.Body == NoBody || r.GetBody != nil { switch valueOrDefault(r.Method, "GET") { case "GET", "HEAD", "OPTIONS", "TRACE": return true diff --git a/libgo/go/net/http/request_test.go b/libgo/go/net/http/request_test.go index e6748375b58..967156bac9b 100644 --- a/libgo/go/net/http/request_test.go +++ b/libgo/go/net/http/request_test.go @@ -7,6 +7,7 @@ package http_test import ( "bufio" "bytes" + "context" "encoding/base64" "fmt" "io" @@ -785,6 +786,28 @@ func TestMaxBytesReaderStickyError(t *testing.T) { } } +func TestWithContextDeepCopiesURL(t *testing.T) { + req, err := NewRequest("POST", "https://golang.org/", nil) + if err != nil { + t.Fatal(err) + } + + reqCopy := req.WithContext(context.Background()) + reqCopy.URL.Scheme = "http" + + firstURL, secondURL := req.URL.String(), reqCopy.URL.String() + if firstURL == secondURL { + t.Errorf("unexpected change to original request's URL") + } + + // And also check we don't crash on nil (Issue 20601) + req.URL = nil + reqCopy = req.WithContext(context.Background()) + if reqCopy.URL != nil { + t.Error("expected nil URL in cloned request") + } +} + // verify that NewRequest sets Request.GetBody and that it works func TestNewRequestGetBody(t *testing.T) { tests := []struct { diff --git a/libgo/go/net/http/response.go b/libgo/go/net/http/response.go index ae118fb386d..0357b605023 100644 --- a/libgo/go/net/http/response.go +++ b/libgo/go/net/http/response.go @@ -37,9 +37,10 @@ type Response struct { // Header maps header keys to values. If the response had multiple // headers with the same key, they may be concatenated, with comma // delimiters. (Section 4.2 of RFC 2616 requires that multiple headers - // be semantically equivalent to a comma-delimited sequence.) Values - // duplicated by other fields in this struct (e.g., ContentLength) are - // omitted from Header. + // be semantically equivalent to a comma-delimited sequence.) When + // Header values are duplicated by other fields in this struct (e.g., + // ContentLength, TransferEncoding, Trailer), the field values are + // authoritative. // // Keys in the map are canonicalized (see CanonicalHeaderKey). Header Header @@ -152,23 +153,23 @@ func ReadResponse(r *bufio.Reader, req *Request) (*Response, error) { } return nil, err } - f := strings.SplitN(line, " ", 3) - if len(f) < 2 { + if i := strings.IndexByte(line, ' '); i == -1 { return nil, &badStringError{"malformed HTTP response", line} + } else { + resp.Proto = line[:i] + resp.Status = strings.TrimLeft(line[i+1:], " ") } - reasonPhrase := "" - if len(f) > 2 { - reasonPhrase = f[2] + statusCode := resp.Status + if i := strings.IndexByte(resp.Status, ' '); i != -1 { + statusCode = resp.Status[:i] } - if len(f[1]) != 3 { - return nil, &badStringError{"malformed HTTP status code", f[1]} + if len(statusCode) != 3 { + return nil, &badStringError{"malformed HTTP status code", statusCode} } - resp.StatusCode, err = strconv.Atoi(f[1]) + resp.StatusCode, err = strconv.Atoi(statusCode) if err != nil || resp.StatusCode < 0 { - return nil, &badStringError{"malformed HTTP status code", f[1]} + return nil, &badStringError{"malformed HTTP status code", statusCode} } - resp.Status = f[1] + " " + reasonPhrase - resp.Proto = f[0] var ok bool if resp.ProtoMajor, resp.ProtoMinor, ok = ParseHTTPVersion(resp.Proto); !ok { return nil, &badStringError{"malformed HTTP version", resp.Proto} @@ -320,3 +321,9 @@ func (r *Response) Write(w io.Writer) error { // Success return nil } + +func (r *Response) closeBody() { + if r.Body != nil { + r.Body.Close() + } +} diff --git a/libgo/go/net/http/response_test.go b/libgo/go/net/http/response_test.go index 660d51791b7..f1a50bd5989 100644 --- a/libgo/go/net/http/response_test.go +++ b/libgo/go/net/http/response_test.go @@ -318,7 +318,7 @@ var respTests = []respTest{ { "HTTP/1.0 303\r\n\r\n", Response{ - Status: "303 ", + Status: "303", StatusCode: 303, Proto: "HTTP/1.0", ProtoMajor: 1, @@ -532,6 +532,29 @@ some body`, }, "\x1f\x8b\b\x00\x00\x00\x00\x00\x00\x00s\xf3\xf7\a\x00\xab'\xd4\x1a\x03\x00\x00\x00", }, + + // Issue 19989: two spaces between HTTP version and status. + { + "HTTP/1.0 401 Unauthorized\r\n" + + "Content-type: text/html\r\n" + + "WWW-Authenticate: Basic realm=\"\"\r\n\r\n" + + "Your Authentication failed.\r\n", + Response{ + Status: "401 Unauthorized", + StatusCode: 401, + Proto: "HTTP/1.0", + ProtoMajor: 1, + ProtoMinor: 0, + Request: dummyReq("GET"), + Header: Header{ + "Content-Type": {"text/html"}, + "Www-Authenticate": {`Basic realm=""`}, + }, + Close: true, + ContentLength: -1, + }, + "Your Authentication failed.\r\n", + }, } // tests successful calls to ReadResponse, and inspects the returned Response. @@ -926,3 +949,29 @@ func TestNeedsSniff(t *testing.T) { t.Errorf("needsSniff empty Content-Type = %t; want %t", got, want) } } + +// A response should only write out single Connection: close header. Tests #19499. +func TestResponseWritesOnlySingleConnectionClose(t *testing.T) { + const connectionCloseHeader = "Connection: close" + + res, err := ReadResponse(bufio.NewReader(strings.NewReader("HTTP/1.0 200 OK\r\n\r\nAAAA")), nil) + if err != nil { + t.Fatalf("ReadResponse failed %v", err) + } + + var buf1 bytes.Buffer + if err = res.Write(&buf1); err != nil { + t.Fatalf("Write failed %v", err) + } + if res, err = ReadResponse(bufio.NewReader(&buf1), nil); err != nil { + t.Fatalf("ReadResponse failed %v", err) + } + + var buf2 bytes.Buffer + if err = res.Write(&buf2); err != nil { + t.Fatalf("Write failed %v", err) + } + if count := strings.Count(buf2.String(), connectionCloseHeader); count != 1 { + t.Errorf("Found %d %q header", count, connectionCloseHeader) + } +} diff --git a/libgo/go/net/http/serve_test.go b/libgo/go/net/http/serve_test.go index 73dd56e8c42..7137599c42e 100644 --- a/libgo/go/net/http/serve_test.go +++ b/libgo/go/net/http/serve_test.go @@ -337,6 +337,7 @@ var serveMuxTests = []struct { {"GET", "codesearch.google.com", "/search/", 203, "codesearch.google.com/"}, {"GET", "codesearch.google.com", "/search/foo", 203, "codesearch.google.com/"}, {"GET", "codesearch.google.com", "/", 203, "codesearch.google.com/"}, + {"GET", "codesearch.google.com:443", "/", 203, "codesearch.google.com/"}, {"GET", "images.google.com", "/search", 201, "/search"}, {"GET", "images.google.com", "/search/", 404, ""}, {"GET", "images.google.com", "/search/foo", 404, ""}, @@ -460,31 +461,86 @@ func TestMuxRedirectLeadingSlashes(t *testing.T) { } } +func BenchmarkServeMux(b *testing.B) { + + type test struct { + path string + code int + req *Request + } + + // Build example handlers and requests + var tests []test + endpoints := []string{"search", "dir", "file", "change", "count", "s"} + for _, e := range endpoints { + for i := 200; i < 230; i++ { + p := fmt.Sprintf("/%s/%d/", e, i) + tests = append(tests, test{ + path: p, + code: i, + req: &Request{Method: "GET", Host: "localhost", URL: &url.URL{Path: p}}, + }) + } + } + mux := NewServeMux() + for _, tt := range tests { + mux.Handle(tt.path, serve(tt.code)) + } + + rw := httptest.NewRecorder() + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + for _, tt := range tests { + *rw = httptest.ResponseRecorder{} + h, pattern := mux.Handler(tt.req) + h.ServeHTTP(rw, tt.req) + if pattern != tt.path || rw.Code != tt.code { + b.Fatalf("got %d, %q, want %d, %q", rw.Code, pattern, tt.code, tt.path) + } + } + } +} + func TestServerTimeouts(t *testing.T) { setParallel(t) defer afterTest(t) + // Try three times, with increasing timeouts. + tries := []time.Duration{250 * time.Millisecond, 500 * time.Millisecond, 1 * time.Second} + for i, timeout := range tries { + err := testServerTimeouts(timeout) + if err == nil { + return + } + t.Logf("failed at %v: %v", timeout, err) + if i != len(tries)-1 { + t.Logf("retrying at %v ...", tries[i+1]) + } + } + t.Fatal("all attempts failed") +} + +func testServerTimeouts(timeout time.Duration) error { reqNum := 0 ts := httptest.NewUnstartedServer(HandlerFunc(func(res ResponseWriter, req *Request) { reqNum++ fmt.Fprintf(res, "req=%d", reqNum) })) - ts.Config.ReadTimeout = 250 * time.Millisecond - ts.Config.WriteTimeout = 250 * time.Millisecond + ts.Config.ReadTimeout = timeout + ts.Config.WriteTimeout = timeout ts.Start() defer ts.Close() // Hit the HTTP server successfully. - tr := &Transport{DisableKeepAlives: true} // they interfere with this test - defer tr.CloseIdleConnections() - c := &Client{Transport: tr} + c := ts.Client() r, err := c.Get(ts.URL) if err != nil { - t.Fatalf("http Get #1: %v", err) + return fmt.Errorf("http Get #1: %v", err) } got, err := ioutil.ReadAll(r.Body) expected := "req=1" if string(got) != expected || err != nil { - t.Errorf("Unexpected response for request #1; got %q ,%v; expected %q, nil", + return fmt.Errorf("Unexpected response for request #1; got %q ,%v; expected %q, nil", string(got), err, expected) } @@ -492,17 +548,18 @@ func TestServerTimeouts(t *testing.T) { t1 := time.Now() conn, err := net.Dial("tcp", ts.Listener.Addr().String()) if err != nil { - t.Fatalf("Dial: %v", err) + return fmt.Errorf("Dial: %v", err) } buf := make([]byte, 1) n, err := conn.Read(buf) conn.Close() latency := time.Since(t1) if n != 0 || err != io.EOF { - t.Errorf("Read = %v, %v, wanted %v, %v", n, err, 0, io.EOF) + return fmt.Errorf("Read = %v, %v, wanted %v, %v", n, err, 0, io.EOF) } - if latency < 200*time.Millisecond /* fudge from 250 ms above */ { - t.Errorf("got EOF after %s, want >= %s", latency, 200*time.Millisecond) + minLatency := timeout / 5 * 4 + if latency < minLatency { + return fmt.Errorf("got EOF after %s, want >= %s", latency, minLatency) } // Hit the HTTP server successfully again, verifying that the @@ -510,29 +567,31 @@ func TestServerTimeouts(t *testing.T) { // get "req=2", not "req=3") r, err = c.Get(ts.URL) if err != nil { - t.Fatalf("http Get #2: %v", err) + return fmt.Errorf("http Get #2: %v", err) } got, err = ioutil.ReadAll(r.Body) + r.Body.Close() expected = "req=2" if string(got) != expected || err != nil { - t.Errorf("Get #2 got %q, %v, want %q, nil", string(got), err, expected) + return fmt.Errorf("Get #2 got %q, %v, want %q, nil", string(got), err, expected) } if !testing.Short() { conn, err := net.Dial("tcp", ts.Listener.Addr().String()) if err != nil { - t.Fatalf("Dial: %v", err) + return fmt.Errorf("long Dial: %v", err) } defer conn.Close() go io.Copy(ioutil.Discard, conn) for i := 0; i < 5; i++ { _, err := conn.Write([]byte("GET / HTTP/1.1\r\nHost: foo\r\n\r\n")) if err != nil { - t.Fatalf("on write %d: %v", i, err) + return fmt.Errorf("on write %d: %v", i, err) } - time.Sleep(ts.Config.ReadTimeout / 2) + time.Sleep(timeout / 2) } } + return nil } // Test that the HTTP/2 server handles Server.WriteTimeout (Issue 18437) @@ -548,12 +607,10 @@ func TestHTTP2WriteDeadlineExtendedOnNewRequest(t *testing.T) { ts.StartTLS() defer ts.Close() - tr := newTLSTransport(t, ts) - defer tr.CloseIdleConnections() - if err := ExportHttp2ConfigureTransport(tr); err != nil { + c := ts.Client() + if err := ExportHttp2ConfigureTransport(c.Transport.(*Transport)); err != nil { t.Fatal(err) } - c := &Client{Transport: tr} for i := 1; i <= 3; i++ { req, err := NewRequest("GET", ts.URL, nil) @@ -585,13 +642,139 @@ func TestHTTP2WriteDeadlineExtendedOnNewRequest(t *testing.T) { } } +// tryTimeouts runs testFunc with increasing timeouts. Test passes on first success, +// and fails if all timeouts fail. +func tryTimeouts(t *testing.T, testFunc func(timeout time.Duration) error) { + tries := []time.Duration{250 * time.Millisecond, 500 * time.Millisecond, 1 * time.Second} + for i, timeout := range tries { + err := testFunc(timeout) + if err == nil { + return + } + t.Logf("failed at %v: %v", timeout, err) + if i != len(tries)-1 { + t.Logf("retrying at %v ...", tries[i+1]) + } + } + t.Fatal("all attempts failed") +} + +// Test that the HTTP/2 server RSTs stream on slow write. +func TestHTTP2WriteDeadlineEnforcedPerStream(t *testing.T) { + if testing.Short() { + t.Skip("skipping in short mode") + } + setParallel(t) + defer afterTest(t) + tryTimeouts(t, testHTTP2WriteDeadlineEnforcedPerStream) +} + +func testHTTP2WriteDeadlineEnforcedPerStream(timeout time.Duration) error { + reqNum := 0 + ts := httptest.NewUnstartedServer(HandlerFunc(func(res ResponseWriter, req *Request) { + reqNum++ + if reqNum == 1 { + return // first request succeeds + } + time.Sleep(timeout) // second request times out + })) + ts.Config.WriteTimeout = timeout / 2 + ts.TLS = &tls.Config{NextProtos: []string{"h2"}} + ts.StartTLS() + defer ts.Close() + + c := ts.Client() + if err := ExportHttp2ConfigureTransport(c.Transport.(*Transport)); err != nil { + return fmt.Errorf("ExportHttp2ConfigureTransport: %v", err) + } + + req, err := NewRequest("GET", ts.URL, nil) + if err != nil { + return fmt.Errorf("NewRequest: %v", err) + } + r, err := c.Do(req) + if err != nil { + return fmt.Errorf("http2 Get #1: %v", err) + } + r.Body.Close() + if r.ProtoMajor != 2 { + return fmt.Errorf("http2 Get expected HTTP/2.0, got %q", r.Proto) + } + + req, err = NewRequest("GET", ts.URL, nil) + if err != nil { + return fmt.Errorf("NewRequest: %v", err) + } + r, err = c.Do(req) + if err == nil { + r.Body.Close() + if r.ProtoMajor != 2 { + return fmt.Errorf("http2 Get expected HTTP/2.0, got %q", r.Proto) + } + return fmt.Errorf("http2 Get #2 expected error, got nil") + } + expected := "stream ID 3; INTERNAL_ERROR" // client IDs are odd, second stream should be 3 + if !strings.Contains(err.Error(), expected) { + return fmt.Errorf("http2 Get #2: expected error to contain %q, got %q", expected, err) + } + return nil +} + +// Test that the HTTP/2 server does not send RST when WriteDeadline not set. +func TestHTTP2NoWriteDeadline(t *testing.T) { + if testing.Short() { + t.Skip("skipping in short mode") + } + setParallel(t) + defer afterTest(t) + tryTimeouts(t, testHTTP2NoWriteDeadline) +} + +func testHTTP2NoWriteDeadline(timeout time.Duration) error { + reqNum := 0 + ts := httptest.NewUnstartedServer(HandlerFunc(func(res ResponseWriter, req *Request) { + reqNum++ + if reqNum == 1 { + return // first request succeeds + } + time.Sleep(timeout) // second request timesout + })) + ts.TLS = &tls.Config{NextProtos: []string{"h2"}} + ts.StartTLS() + defer ts.Close() + + c := ts.Client() + if err := ExportHttp2ConfigureTransport(c.Transport.(*Transport)); err != nil { + return fmt.Errorf("ExportHttp2ConfigureTransport: %v", err) + } + + for i := 0; i < 2; i++ { + req, err := NewRequest("GET", ts.URL, nil) + if err != nil { + return fmt.Errorf("NewRequest: %v", err) + } + r, err := c.Do(req) + if err != nil { + return fmt.Errorf("http2 Get #%d: %v", i, err) + } + r.Body.Close() + if r.ProtoMajor != 2 { + return fmt.Errorf("http2 Get expected HTTP/2.0, got %q", r.Proto) + } + } + return nil +} + // golang.org/issue/4741 -- setting only a write timeout that triggers // shouldn't cause a handler to block forever on reads (next HTTP // request) that will never happen. func TestOnlyWriteTimeout(t *testing.T) { setParallel(t) defer afterTest(t) - var conn net.Conn + var ( + mu sync.RWMutex + conn net.Conn + ) var afterTimeoutErrc = make(chan error, 1) ts := httptest.NewUnstartedServer(HandlerFunc(func(w ResponseWriter, req *Request) { buf := make([]byte, 512<<10) @@ -600,17 +783,21 @@ func TestOnlyWriteTimeout(t *testing.T) { t.Errorf("handler Write error: %v", err) return } + mu.RLock() + defer mu.RUnlock() + if conn == nil { + t.Error("no established connection found") + return + } conn.SetWriteDeadline(time.Now().Add(-30 * time.Second)) _, err = w.Write(buf) afterTimeoutErrc <- err })) - ts.Listener = trackLastConnListener{ts.Listener, &conn} + ts.Listener = trackLastConnListener{ts.Listener, &mu, &conn} ts.Start() defer ts.Close() - tr := &Transport{DisableKeepAlives: false} - defer tr.CloseIdleConnections() - c := &Client{Transport: tr} + c := ts.Client() errc := make(chan error) go func() { @@ -620,6 +807,7 @@ func TestOnlyWriteTimeout(t *testing.T) { return } _, err = io.Copy(ioutil.Discard, res.Body) + res.Body.Close() errc <- err }() select { @@ -638,12 +826,18 @@ func TestOnlyWriteTimeout(t *testing.T) { // trackLastConnListener tracks the last net.Conn that was accepted. type trackLastConnListener struct { net.Listener + + mu *sync.RWMutex last *net.Conn // destination } func (l trackLastConnListener) Accept() (c net.Conn, err error) { c, err = l.Listener.Accept() - *l.last = c + if err == nil { + l.mu.Lock() + *l.last = c + l.mu.Unlock() + } return } @@ -671,8 +865,7 @@ func TestIdentityResponse(t *testing.T) { ts := httptest.NewServer(handler) defer ts.Close() - c := &Client{Transport: new(Transport)} - defer closeClient(c) + c := ts.Client() // Note: this relies on the assumption (which is true) that // Get sends HTTP/1.1 or greater requests. Otherwise the @@ -936,7 +1129,6 @@ func (c *blockingRemoteAddrConn) RemoteAddr() net.Addr { // Issue 12943 func TestServerAllowsBlockingRemoteAddr(t *testing.T) { - setParallel(t) defer afterTest(t) ts := httptest.NewUnstartedServer(HandlerFunc(func(w ResponseWriter, r *Request) { fmt.Fprintf(w, "RA:%s", r.RemoteAddr) @@ -949,21 +1141,22 @@ func TestServerAllowsBlockingRemoteAddr(t *testing.T) { ts.Start() defer ts.Close() - tr := &Transport{DisableKeepAlives: true} - defer tr.CloseIdleConnections() - c := &Client{Transport: tr, Timeout: time.Second} + c := ts.Client() + c.Timeout = time.Second + // Force separate connection for each: + c.Transport.(*Transport).DisableKeepAlives = true - fetch := func(response chan string) { + fetch := func(num int, response chan<- string) { resp, err := c.Get(ts.URL) if err != nil { - t.Error(err) + t.Errorf("Request %d: %v", num, err) response <- "" return } defer resp.Body.Close() body, err := ioutil.ReadAll(resp.Body) if err != nil { - t.Error(err) + t.Errorf("Request %d: %v", num, err) response <- "" return } @@ -972,14 +1165,14 @@ func TestServerAllowsBlockingRemoteAddr(t *testing.T) { // Start a request. The server will block on getting conn.RemoteAddr. response1c := make(chan string, 1) - go fetch(response1c) + go fetch(1, response1c) // Wait for the server to accept it; grab the connection. conn1 := <-conns // Start another request and grab its connection response2c := make(chan string, 1) - go fetch(response2c) + go fetch(2, response2c) var conn2 net.Conn select { @@ -1022,9 +1215,7 @@ func TestIdentityResponseHeaders(t *testing.T) { })) defer ts.Close() - c := &Client{Transport: new(Transport)} - defer closeClient(c) - + c := ts.Client() res, err := c.Get(ts.URL) if err != nil { t.Fatalf("Get error: %v", err) @@ -1145,12 +1336,7 @@ func TestTLSServer(t *testing.T) { t.Errorf("expected test TLS server to start with https://, got %q", ts.URL) return } - noVerifyTransport := &Transport{ - TLSClientConfig: &tls.Config{ - InsecureSkipVerify: true, - }, - } - client := &Client{Transport: noVerifyTransport} + client := ts.Client() res, err := client.Get(ts.URL) if err != nil { t.Error(err) @@ -1171,6 +1357,59 @@ func TestTLSServer(t *testing.T) { }) } +func TestServeTLS(t *testing.T) { + // Not parallel: uses global test hooks. + defer afterTest(t) + defer SetTestHookServerServe(nil) + + cert, err := tls.X509KeyPair(internal.LocalhostCert, internal.LocalhostKey) + if err != nil { + t.Fatal(err) + } + tlsConf := &tls.Config{ + Certificates: []tls.Certificate{cert}, + } + + ln := newLocalListener(t) + defer ln.Close() + addr := ln.Addr().String() + + serving := make(chan bool, 1) + SetTestHookServerServe(func(s *Server, ln net.Listener) { + serving <- true + }) + handler := HandlerFunc(func(w ResponseWriter, r *Request) {}) + s := &Server{ + Addr: addr, + TLSConfig: tlsConf, + Handler: handler, + } + errc := make(chan error, 1) + go func() { errc <- s.ServeTLS(ln, "", "") }() + select { + case err := <-errc: + t.Fatalf("ServeTLS: %v", err) + case <-serving: + case <-time.After(5 * time.Second): + t.Fatal("timeout") + } + + c, err := tls.Dial("tcp", ln.Addr().String(), &tls.Config{ + InsecureSkipVerify: true, + NextProtos: []string{"h2", "http/1.1"}, + }) + if err != nil { + t.Fatal(err) + } + defer c.Close() + if got, want := c.ConnectionState().NegotiatedProtocol, "h2"; got != want { + t.Errorf("NegotiatedProtocol = %q; want %q", got, want) + } + if got, want := c.ConnectionState().NegotiatedProtocolIsMutual, true; got != want { + t.Errorf("NegotiatedProtocolIsMutual = %v; want %v", got, want) + } +} + // Issue 15908 func TestAutomaticHTTP2_Serve_NoTLSConfig(t *testing.T) { testAutomaticHTTP2_Serve(t, nil, true) @@ -1967,8 +2206,7 @@ func TestTimeoutHandlerRace(t *testing.T) { ts := httptest.NewServer(TimeoutHandler(delayHi, 20*time.Millisecond, "")) defer ts.Close() - c := &Client{Transport: new(Transport)} - defer closeClient(c) + c := ts.Client() var wg sync.WaitGroup gate := make(chan bool, 10) @@ -2011,8 +2249,8 @@ func TestTimeoutHandlerRaceHeader(t *testing.T) { if testing.Short() { n = 10 } - c := &Client{Transport: new(Transport)} - defer closeClient(c) + + c := ts.Client() for i := 0; i < n; i++ { gate <- true wg.Add(1) @@ -2099,8 +2337,7 @@ func TestTimeoutHandlerStartTimerWhenServing(t *testing.T) { ts := httptest.NewServer(TimeoutHandler(handler, timeout, "")) defer ts.Close() - c := &Client{Transport: new(Transport)} - defer closeClient(c) + c := ts.Client() // Issue was caused by the timeout handler starting the timer when // was created, not when the request. So wait for more than the timeout @@ -2127,8 +2364,7 @@ func TestTimeoutHandlerEmptyResponse(t *testing.T) { ts := httptest.NewServer(TimeoutHandler(handler, timeout, "")) defer ts.Close() - c := &Client{Transport: new(Transport)} - defer closeClient(c) + c := ts.Client() res, err := c.Get(ts.URL) if err != nil { @@ -2364,9 +2600,7 @@ func TestServerWriteHijackZeroBytes(t *testing.T) { ts.Start() defer ts.Close() - tr := &Transport{} - defer tr.CloseIdleConnections() - c := &Client{Transport: tr} + c := ts.Client() res, err := c.Get(ts.URL) if err != nil { t.Fatal(err) @@ -2411,8 +2645,7 @@ func TestStripPrefix(t *testing.T) { ts := httptest.NewServer(StripPrefix("/foo", h)) defer ts.Close() - c := &Client{Transport: new(Transport)} - defer closeClient(c) + c := ts.Client() res, err := c.Get(ts.URL + "/foo/bar") if err != nil { @@ -2433,6 +2666,16 @@ func TestStripPrefix(t *testing.T) { res.Body.Close() } +// https://golang.org/issue/18952. +func TestStripPrefix_notModifyRequest(t *testing.T) { + h := StripPrefix("/foo", NotFoundHandler()) + req := httptest.NewRequest("GET", "/foo/bar", nil) + h.ServeHTTP(httptest.NewRecorder(), req) + if req.URL.Path != "/foo/bar" { + t.Errorf("StripPrefix should not modify the provided Request, but it did") + } +} + func TestRequestLimit_h1(t *testing.T) { testRequestLimit(t, h1Mode) } func TestRequestLimit_h2(t *testing.T) { testRequestLimit(t, h2Mode) } func testRequestLimit(t *testing.T, h2 bool) { @@ -3512,8 +3755,8 @@ func testTransportAndServerSharedBodyRace(t *testing.T, h2 bool) { // Test that a hanging Request.Body.Read from another goroutine can't // cause the Handler goroutine's Request.Body.Close to block. +// See issue 7121. func TestRequestBodyCloseDoesntBlock(t *testing.T) { - t.Skipf("Skipping known issue; see golang.org/issue/7121") if testing.Short() { t.Skip("skipping in -short mode") } @@ -3644,9 +3887,7 @@ func TestServerConnState(t *testing.T) { } ts.Start() - tr := &Transport{} - defer tr.CloseIdleConnections() - c := &Client{Transport: tr} + c := ts.Client() mustGet := func(url string, headers ...string) { req, err := NewRequest("GET", url, nil) @@ -4170,6 +4411,9 @@ func TestServerValidatesHostHeader(t *testing.T) { // Make an exception for HTTP upgrade requests: {"PRI * HTTP/2.0", "", 200}, + // Also an exception for CONNECT requests: (Issue 18215) + {"CONNECT golang.org:443 HTTP/1.1", "", 200}, + // But not other HTTP/2 stuff: {"PRI / HTTP/2.0", "", 400}, {"GET / HTTP/2.0", "", 400}, @@ -4373,13 +4617,6 @@ func testServerContext_ServerContextKey(t *testing.T, h2 bool) { if _, ok := got.(*Server); !ok { t.Errorf("context value = %T; want *http.Server", got) } - - got = ctx.Value(LocalAddrContextKey) - if addr, ok := got.(net.Addr); !ok { - t.Errorf("local addr value = %T; want net.Addr", got) - } else if fmt.Sprint(addr) != r.Host { - t.Errorf("local addr = %v; want %v", addr, r.Host) - } })) defer cst.close() res, err := cst.c.Get(cst.ts.URL) @@ -4389,6 +4626,37 @@ func testServerContext_ServerContextKey(t *testing.T, h2 bool) { res.Body.Close() } +func TestServerContext_LocalAddrContextKey_h1(t *testing.T) { + testServerContext_LocalAddrContextKey(t, h1Mode) +} +func TestServerContext_LocalAddrContextKey_h2(t *testing.T) { + testServerContext_LocalAddrContextKey(t, h2Mode) +} +func testServerContext_LocalAddrContextKey(t *testing.T, h2 bool) { + setParallel(t) + defer afterTest(t) + ch := make(chan interface{}, 1) + cst := newClientServerTest(t, h2, HandlerFunc(func(w ResponseWriter, r *Request) { + ch <- r.Context().Value(LocalAddrContextKey) + })) + defer cst.close() + if _, err := cst.c.Head(cst.ts.URL); err != nil { + t.Fatal(err) + } + + host := cst.ts.Listener.Addr().String() + select { + case got := <-ch: + if addr, ok := got.(net.Addr); !ok { + t.Errorf("local addr value = %T; want net.Addr", got) + } else if fmt.Sprint(addr) != host { + t.Errorf("local addr = %v; want %v", addr, host) + } + case <-time.After(5 * time.Second): + t.Error("timed out") + } +} + // https://golang.org/issue/15960 func TestHandlerSetTransferEncodingChunked(t *testing.T) { setParallel(t) @@ -4481,15 +4749,9 @@ func benchmarkClientServerParallel(b *testing.B, parallelism int, useTLS bool) { b.ResetTimer() b.SetParallelism(parallelism) b.RunParallel(func(pb *testing.PB) { - noVerifyTransport := &Transport{ - TLSClientConfig: &tls.Config{ - InsecureSkipVerify: true, - }, - } - defer noVerifyTransport.CloseIdleConnections() - client := &Client{Transport: noVerifyTransport} + c := ts.Client() for pb.Next() { - res, err := client.Get(ts.URL) + res, err := c.Get(ts.URL) if err != nil { b.Logf("Get: %v", err) continue @@ -4924,10 +5186,7 @@ func TestServerIdleTimeout(t *testing.T) { ts.Config.IdleTimeout = 2 * time.Second ts.Start() defer ts.Close() - - tr := &Transport{} - defer tr.CloseIdleConnections() - c := &Client{Transport: tr} + c := ts.Client() get := func() string { res, err := c.Get(ts.URL) @@ -4988,9 +5247,8 @@ func TestServerSetKeepAlivesEnabledClosesConns(t *testing.T) { })) defer ts.Close() - tr := &Transport{} - defer tr.CloseIdleConnections() - c := &Client{Transport: tr} + c := ts.Client() + tr := c.Transport.(*Transport) get := func() string { return get(t, c, ts.URL) } @@ -5030,7 +5288,8 @@ func testServerShutdown(t *testing.T, h2 bool) { defer afterTest(t) var doShutdown func() // set later var shutdownRes = make(chan error, 1) - cst := newClientServerTest(t, h2, HandlerFunc(func(w ResponseWriter, r *Request) { + var gotOnShutdown = make(chan struct{}, 1) + handler := HandlerFunc(func(w ResponseWriter, r *Request) { go doShutdown() // Shutdown is graceful, so it should not interrupt // this in-flight response. Add a tiny sleep here to @@ -5038,7 +5297,10 @@ func testServerShutdown(t *testing.T, h2 bool) { // bugs. time.Sleep(20 * time.Millisecond) io.WriteString(w, r.RemoteAddr) - })) + }) + cst := newClientServerTest(t, h2, handler, func(srv *httptest.Server) { + srv.Config.RegisterOnShutdown(func() { gotOnShutdown <- struct{}{} }) + }) defer cst.close() doShutdown = func() { @@ -5049,6 +5311,11 @@ func testServerShutdown(t *testing.T, h2 bool) { if err := <-shutdownRes; err != nil { t.Fatalf("Shutdown: %v", err) } + select { + case <-gotOnShutdown: + case <-time.After(5 * time.Second): + t.Errorf("onShutdown callback not called, RegisterOnShutdown broken?") + } res, err := cst.c.Get(cst.ts.URL) if err == nil { @@ -5109,9 +5376,7 @@ func TestServerCancelsReadTimeoutWhenIdle(t *testing.T) { ts.Start() defer ts.Close() - tr := &Transport{} - defer tr.CloseIdleConnections() - c := &Client{Transport: tr} + c := ts.Client() res, err := c.Get(ts.URL) if err != nil { @@ -5312,3 +5577,41 @@ func TestServerHijackGetsBackgroundByte_big(t *testing.T) { t.Error("timeout") } } + +// Issue 18319: test that the Server validates the request method. +func TestServerValidatesMethod(t *testing.T) { + tests := []struct { + method string + want int + }{ + {"GET", 200}, + {"GE(T", 400}, + } + for _, tt := range tests { + conn := &testConn{closec: make(chan bool, 1)} + io.WriteString(&conn.readBuf, tt.method+" / HTTP/1.1\r\nHost: foo.example\r\n\r\n") + + ln := &oneConnListener{conn} + go Serve(ln, serve(200)) + <-conn.closec + res, err := ReadResponse(bufio.NewReader(&conn.writeBuf), nil) + if err != nil { + t.Errorf("For %s, ReadResponse: %v", tt.method, res) + continue + } + if res.StatusCode != tt.want { + t.Errorf("For %s, Status = %d; want %d", tt.method, res.StatusCode, tt.want) + } + } +} + +func BenchmarkResponseStatusLine(b *testing.B) { + b.ReportAllocs() + b.RunParallel(func(pb *testing.PB) { + bw := bufio.NewWriter(ioutil.Discard) + var buf3 [3]byte + for pb.Next() { + Export_writeStatusLine(bw, true, 200, buf3[:]) + } + }) +} diff --git a/libgo/go/net/http/server.go b/libgo/go/net/http/server.go index df70a15193b..2fa8ab23d8a 100644 --- a/libgo/go/net/http/server.go +++ b/libgo/go/net/http/server.go @@ -75,9 +75,10 @@ var ( // If ServeHTTP panics, the server (the caller of ServeHTTP) assumes // that the effect of the panic was isolated to the active request. // It recovers the panic, logs a stack trace to the server error log, -// and hangs up the connection. To abort a handler so the client sees -// an interrupted response but the server doesn't log an error, panic -// with the value ErrAbortHandler. +// and either closes the network connection or sends an HTTP/2 +// RST_STREAM, depending on the HTTP protocol. To abort a handler so +// the client sees an interrupted response but the server doesn't log +// an error, panic with the value ErrAbortHandler. type Handler interface { ServeHTTP(ResponseWriter, *Request) } @@ -177,6 +178,9 @@ type Hijacker interface { // // The returned bufio.Reader may contain unprocessed buffered // data from the client. + // + // After a call to Hijack, the original Request.Body should + // not be used. Hijack() (net.Conn, *bufio.ReadWriter, error) } @@ -439,9 +443,10 @@ type response struct { handlerDone atomicBool // set true when the handler exits - // Buffers for Date and Content-Length - dateBuf [len(TimeFormat)]byte - clenBuf [10]byte + // Buffers for Date, Content-Length, and status code + dateBuf [len(TimeFormat)]byte + clenBuf [10]byte + statusBuf [3]byte // closeNotifyCh is the channel returned by CloseNotify. // TODO(bradfitz): this is currently (for Go 1.8) always @@ -622,7 +627,6 @@ type connReader struct { mu sync.Mutex // guards following hasByte bool byteBuf [1]byte - bgErr error // non-nil means error happened on background read cond *sync.Cond inRead bool aborted bool // set true before conn.rwc deadline is set to past @@ -731,11 +735,6 @@ func (cr *connReader) Read(p []byte) (n int, err error) { cr.unlock() return 0, io.EOF } - if cr.bgErr != nil { - err = cr.bgErr - cr.unlock() - return 0, err - } if len(p) == 0 { cr.unlock() return 0, nil @@ -839,7 +838,7 @@ func (srv *Server) initialReadLimitSize() int64 { return int64(srv.maxHeaderBytes()) + 4096 // bufio slop } -// wrapper around io.ReaderCloser which on first read, sends an +// wrapper around io.ReadCloser which on first read, sends an // HTTP/1.1 100 Continue header type expectContinueReader struct { resp *response @@ -948,7 +947,7 @@ func (c *conn) readRequest(ctx context.Context) (w *response, err error) { hosts, haveHost := req.Header["Host"] isH2Upgrade := req.isH2Upgrade() - if req.ProtoAtLeast(1, 1) && (!haveHost || len(hosts) == 0) && !isH2Upgrade { + if req.ProtoAtLeast(1, 1) && (!haveHost || len(hosts) == 0) && !isH2Upgrade && req.Method != "CONNECT" { return nil, badRequestError("missing required Host header") } if len(hosts) > 1 { @@ -1379,7 +1378,7 @@ func (cw *chunkWriter) writeHeader(p []byte) { } } - w.conn.bufw.WriteString(statusLine(w.req, code)) + writeStatusLine(w.conn.bufw, w.req.ProtoAtLeast(1, 1), code, w.statusBuf[:]) cw.header.WriteSubset(w.conn.bufw, excludeHeader) setHeader.Write(w.conn.bufw) w.conn.bufw.Write(crlf) @@ -1403,49 +1402,25 @@ func foreachHeaderElement(v string, fn func(string)) { } } -// statusLines is a cache of Status-Line strings, keyed by code (for -// HTTP/1.1) or negative code (for HTTP/1.0). This is faster than a -// map keyed by struct of two fields. This map's max size is bounded -// by 2*len(statusText), two protocol types for each known official -// status code in the statusText map. -var ( - statusMu sync.RWMutex - statusLines = make(map[int]string) -) - -// statusLine returns a response Status-Line (RFC 2616 Section 6.1) -// for the given request and response status code. -func statusLine(req *Request, code int) string { - // Fast path: - key := code - proto11 := req.ProtoAtLeast(1, 1) - if !proto11 { - key = -key - } - statusMu.RLock() - line, ok := statusLines[key] - statusMu.RUnlock() - if ok { - return line - } - - // Slow path: - proto := "HTTP/1.0" - if proto11 { - proto = "HTTP/1.1" - } - codestring := fmt.Sprintf("%03d", code) - text, ok := statusText[code] - if !ok { - text = "status code " + codestring +// writeStatusLine writes an HTTP/1.x Status-Line (RFC 2616 Section 6.1) +// to bw. is11 is whether the HTTP request is HTTP/1.1. false means HTTP/1.0. +// code is the response status code. +// scratch is an optional scratch buffer. If it has at least capacity 3, it's used. +func writeStatusLine(bw *bufio.Writer, is11 bool, code int, scratch []byte) { + if is11 { + bw.WriteString("HTTP/1.1 ") + } else { + bw.WriteString("HTTP/1.0 ") } - line = proto + " " + codestring + " " + text + "\r\n" - if ok { - statusMu.Lock() - defer statusMu.Unlock() - statusLines[key] = line + if text, ok := statusText[code]; ok { + bw.Write(strconv.AppendInt(scratch[:0], int64(code), 10)) + bw.WriteByte(' ') + bw.WriteString(text) + bw.WriteString("\r\n") + } else { + // don't worry about performance + fmt.Fprintf(bw, "%03d status code %d\r\n", code, code) } - return line } // bodyAllowed reports whether a Write is allowed for this response type. @@ -1714,6 +1689,7 @@ func isCommonNetReadError(err error) bool { // Serve a new connection. func (c *conn) serve(ctx context.Context) { c.remoteAddr = c.rwc.RemoteAddr().String() + ctx = context.WithValue(ctx, LocalAddrContextKey, c.rwc.LocalAddr()) defer func() { if err := recover(); err != nil && err != ErrAbortHandler { const size = 64 << 10 @@ -1973,8 +1949,12 @@ func StripPrefix(prefix string, h Handler) Handler { } return HandlerFunc(func(w ResponseWriter, r *Request) { if p := strings.TrimPrefix(r.URL.Path, prefix); len(p) < len(r.URL.Path) { - r.URL.Path = p - h.ServeHTTP(w, r) + r2 := new(Request) + *r2 = *r + r2.URL = new(url.URL) + *r2.URL = *r.URL + r2.URL.Path = p + h.ServeHTTP(w, r2) } else { NotFound(w, r) } @@ -1986,8 +1966,9 @@ func StripPrefix(prefix string, h Handler) Handler { // // The provided code should be in the 3xx range and is usually // StatusMovedPermanently, StatusFound or StatusSeeOther. -func Redirect(w ResponseWriter, r *Request, urlStr string, code int) { - if u, err := url.Parse(urlStr); err == nil { +func Redirect(w ResponseWriter, r *Request, url string, code int) { + // parseURL is just url.Parse (url is shadowed for godoc). + if u, err := parseURL(url); err == nil { // If url was relative, make absolute by // combining with request path. // The browser would probably do this for us, @@ -2011,39 +1992,43 @@ func Redirect(w ResponseWriter, r *Request, urlStr string, code int) { } // no leading http://server - if urlStr == "" || urlStr[0] != '/' { + if url == "" || url[0] != '/' { // make relative path absolute olddir, _ := path.Split(oldpath) - urlStr = olddir + urlStr + url = olddir + url } var query string - if i := strings.Index(urlStr, "?"); i != -1 { - urlStr, query = urlStr[:i], urlStr[i:] + if i := strings.Index(url, "?"); i != -1 { + url, query = url[:i], url[i:] } // clean up but preserve trailing slash - trailing := strings.HasSuffix(urlStr, "/") - urlStr = path.Clean(urlStr) - if trailing && !strings.HasSuffix(urlStr, "/") { - urlStr += "/" + trailing := strings.HasSuffix(url, "/") + url = path.Clean(url) + if trailing && !strings.HasSuffix(url, "/") { + url += "/" } - urlStr += query + url += query } } - w.Header().Set("Location", hexEscapeNonASCII(urlStr)) + w.Header().Set("Location", hexEscapeNonASCII(url)) w.WriteHeader(code) // RFC 2616 recommends that a short note "SHOULD" be included in the // response because older user agents may not understand 301/307. // Shouldn't send the response for POST or HEAD; that leaves GET. if r.Method == "GET" { - note := "" + statusText[code] + ".\n" + note := "" + statusText[code] + ".\n" fmt.Fprintln(w, note) } } +// parseURL is just url.Parse. It exists only so that url.Parse can be called +// in places where url is shadowed for godoc. See https://golang.org/cl/49930. +var parseURL = url.Parse + var htmlReplacer = strings.NewReplacer( "&", "&", "<", "<", @@ -2163,9 +2148,29 @@ func cleanPath(p string) string { return np } -// Find a handler on a handler map given a path string -// Most-specific (longest) pattern wins +// stripHostPort returns h without any trailing ":". +func stripHostPort(h string) string { + // If no port on host, return unchanged + if strings.IndexByte(h, ':') == -1 { + return h + } + host, _, err := net.SplitHostPort(h) + if err != nil { + return h // on error, return unchanged + } + return host +} + +// Find a handler on a handler map given a path string. +// Most-specific (longest) pattern wins. func (mux *ServeMux) match(path string) (h Handler, pattern string) { + // Check for exact match first. + v, ok := mux.m[path] + if ok { + return v.h, v.pattern + } + + // Check for longest valid match. var n = 0 for k, v := range mux.m { if !pathMatch(k, path) { @@ -2184,7 +2189,10 @@ func (mux *ServeMux) match(path string) (h Handler, pattern string) { // consulting r.Method, r.Host, and r.URL.Path. It always returns // a non-nil handler. If the path is not in its canonical form, the // handler will be an internally-generated handler that redirects -// to the canonical path. +// to the canonical path. If the host contains a port, it is ignored +// when matching handlers. +// +// The path and host are used unchanged for CONNECT requests. // // Handler also returns the registered pattern that matches the // request or, in the case of internally-generated redirects, @@ -2193,16 +2201,24 @@ func (mux *ServeMux) match(path string) (h Handler, pattern string) { // If there is no registered handler that applies to the request, // Handler returns a ``page not found'' handler and an empty pattern. func (mux *ServeMux) Handler(r *Request) (h Handler, pattern string) { - if r.Method != "CONNECT" { - if p := cleanPath(r.URL.Path); p != r.URL.Path { - _, pattern = mux.handler(r.Host, p) - url := *r.URL - url.Path = p - return RedirectHandler(url.String(), StatusMovedPermanently), pattern - } + + // CONNECT requests are not canonicalized. + if r.Method == "CONNECT" { + return mux.handler(r.Host, r.URL.Path) } - return mux.handler(r.Host, r.URL.Path) + // All other requests have any port stripped and path cleaned + // before passing to mux.handler. + host := stripHostPort(r.Host) + path := cleanPath(r.URL.Path) + if path != r.URL.Path { + _, pattern = mux.handler(host, path) + url := *r.URL + url.Path = path + return RedirectHandler(url.String(), StatusMovedPermanently), pattern + } + + return mux.handler(host, r.URL.Path) } // handler is the main implementation of Handler. @@ -2307,12 +2323,27 @@ func Serve(l net.Listener, handler Handler) error { return srv.Serve(l) } +// Serve accepts incoming HTTPS connections on the listener l, +// creating a new service goroutine for each. The service goroutines +// read requests and then call handler to reply to them. +// +// Handler is typically nil, in which case the DefaultServeMux is used. +// +// Additionally, files containing a certificate and matching private key +// for the server must be provided. If the certificate is signed by a +// certificate authority, the certFile should be the concatenation +// of the server's certificate, any intermediates, and the CA's certificate. +func ServeTLS(l net.Listener, handler Handler, certFile, keyFile string) error { + srv := &Server{Handler: handler} + return srv.ServeTLS(l, certFile, keyFile) +} + // A Server defines parameters for running an HTTP server. // The zero value for Server is a valid configuration. type Server struct { Addr string // TCP address to listen on, ":http" if empty Handler Handler // handler to invoke, http.DefaultServeMux if nil - TLSConfig *tls.Config // optional TLS config, used by ListenAndServeTLS + TLSConfig *tls.Config // optional TLS config, used by ServeTLS and ListenAndServeTLS // ReadTimeout is the maximum duration for reading the entire // request, including the body. @@ -2338,7 +2369,7 @@ type Server struct { // IdleTimeout is the maximum amount of time to wait for the // next request when keep-alives are enabled. If IdleTimeout // is zero, the value of ReadTimeout is used. If both are - // zero, there is no timeout. + // zero, ReadHeaderTimeout is used. IdleTimeout time.Duration // MaxHeaderBytes controls the maximum number of bytes the @@ -2379,6 +2410,7 @@ type Server struct { listeners map[net.Listener]struct{} activeConn map[*conn]struct{} doneChan chan struct{} + onShutdown []func() } func (s *Server) getDoneChan() <-chan struct{} { @@ -2441,7 +2473,12 @@ var shutdownPollInterval = 500 * time.Millisecond // listeners, then closing all idle connections, and then waiting // indefinitely for connections to return to idle and then shut down. // If the provided context expires before the shutdown is complete, -// then the context's error is returned. +// Shutdown returns the context's error, otherwise it returns any +// error returned from closing the Server's underlying Listener(s). +// +// When Shutdown is called, Serve, ListenAndServe, and +// ListenAndServeTLS immediately return ErrServerClosed. Make sure the +// program doesn't exit and waits instead for Shutdown to return. // // Shutdown does not attempt to close nor wait for hijacked // connections such as WebSockets. The caller of Shutdown should @@ -2454,6 +2491,9 @@ func (srv *Server) Shutdown(ctx context.Context) error { srv.mu.Lock() lnerr := srv.closeListenersLocked() srv.closeDoneChanLocked() + for _, f := range srv.onShutdown { + go f() + } srv.mu.Unlock() ticker := time.NewTicker(shutdownPollInterval) @@ -2470,6 +2510,17 @@ func (srv *Server) Shutdown(ctx context.Context) error { } } +// RegisterOnShutdown registers a function to call on Shutdown. +// This can be used to gracefully shutdown connections that have +// undergone NPN/ALPN protocol upgrade or that have been hijacked. +// This function should start protocol-specific graceful shutdown, +// but should not wait for shutdown to complete. +func (srv *Server) RegisterOnShutdown(f func()) { + srv.mu.Lock() + srv.onShutdown = append(srv.onShutdown, f) + srv.mu.Unlock() +} + // closeIdleConns closes all idle connections and reports whether the // server is quiescent. func (s *Server) closeIdleConns() bool { @@ -2609,6 +2660,8 @@ func (srv *Server) shouldConfigureHTTP2ForServe() bool { return strSliceContains(srv.TLSConfig.NextProtos, http2NextProtoTLS) } +// ErrServerClosed is returned by the Server's Serve, ServeTLS, ListenAndServe, +// and ListenAndServeTLS methods after a call to Shutdown or Close. var ErrServerClosed = errors.New("http: Server closed") // Serve accepts incoming connections on the Listener l, creating a @@ -2638,7 +2691,6 @@ func (srv *Server) Serve(l net.Listener) error { baseCtx := context.Background() // base is always background, per Issue 16220 ctx := context.WithValue(baseCtx, ServerContextKey, srv) - ctx = context.WithValue(ctx, LocalAddrContextKey, l.Addr()) for { rw, e := l.Accept() if e != nil { @@ -2669,6 +2721,49 @@ func (srv *Server) Serve(l net.Listener) error { } } +// ServeTLS accepts incoming connections on the Listener l, creating a +// new service goroutine for each. The service goroutines read requests and +// then call srv.Handler to reply to them. +// +// Additionally, files containing a certificate and matching private key for +// the server must be provided if neither the Server's TLSConfig.Certificates +// nor TLSConfig.GetCertificate are populated.. If the certificate is signed by +// a certificate authority, the certFile should be the concatenation of the +// server's certificate, any intermediates, and the CA's certificate. +// +// For HTTP/2 support, srv.TLSConfig should be initialized to the +// provided listener's TLS Config before calling Serve. If +// srv.TLSConfig is non-nil and doesn't include the string "h2" in +// Config.NextProtos, HTTP/2 support is not enabled. +// +// ServeTLS always returns a non-nil error. After Shutdown or Close, the +// returned error is ErrServerClosed. +func (srv *Server) ServeTLS(l net.Listener, certFile, keyFile string) error { + // Setup HTTP/2 before srv.Serve, to initialize srv.TLSConfig + // before we clone it and create the TLS Listener. + if err := srv.setupHTTP2_ServeTLS(); err != nil { + return err + } + + config := cloneTLSConfig(srv.TLSConfig) + if !strSliceContains(config.NextProtos, "http/1.1") { + config.NextProtos = append(config.NextProtos, "http/1.1") + } + + configHasCert := len(config.Certificates) > 0 || config.GetCertificate != nil + if !configHasCert || certFile != "" || keyFile != "" { + var err error + config.Certificates = make([]tls.Certificate, 1) + config.Certificates[0], err = tls.LoadX509KeyPair(certFile, keyFile) + if err != nil { + return err + } + } + + tlsListener := tls.NewListener(l, config) + return srv.Serve(tlsListener) +} + func (s *Server) trackListener(ln net.Listener, add bool) { s.mu.Lock() defer s.mu.Unlock() @@ -2840,47 +2935,25 @@ func (srv *Server) ListenAndServeTLS(certFile, keyFile string) error { addr = ":https" } - // Setup HTTP/2 before srv.Serve, to initialize srv.TLSConfig - // before we clone it and create the TLS Listener. - if err := srv.setupHTTP2_ListenAndServeTLS(); err != nil { - return err - } - - config := cloneTLSConfig(srv.TLSConfig) - if !strSliceContains(config.NextProtos, "http/1.1") { - config.NextProtos = append(config.NextProtos, "http/1.1") - } - - configHasCert := len(config.Certificates) > 0 || config.GetCertificate != nil - if !configHasCert || certFile != "" || keyFile != "" { - var err error - config.Certificates = make([]tls.Certificate, 1) - config.Certificates[0], err = tls.LoadX509KeyPair(certFile, keyFile) - if err != nil { - return err - } - } - ln, err := net.Listen("tcp", addr) if err != nil { return err } - tlsListener := tls.NewListener(tcpKeepAliveListener{ln.(*net.TCPListener)}, config) - return srv.Serve(tlsListener) + return srv.ServeTLS(tcpKeepAliveListener{ln.(*net.TCPListener)}, certFile, keyFile) } -// setupHTTP2_ListenAndServeTLS conditionally configures HTTP/2 on +// setupHTTP2_ServeTLS conditionally configures HTTP/2 on // srv and returns whether there was an error setting it up. If it is // not configured for policy reasons, nil is returned. -func (srv *Server) setupHTTP2_ListenAndServeTLS() error { +func (srv *Server) setupHTTP2_ServeTLS() error { srv.nextProtoOnce.Do(srv.onceSetNextProtoDefaults) return srv.nextProtoErr } // setupHTTP2_Serve is called from (*Server).Serve and conditionally // configures HTTP/2 on srv using a more conservative policy than -// setupHTTP2_ListenAndServeTLS because Serve may be called +// setupHTTP2_ServeTLS because Serve may be called // concurrently. // // The tests named TestTransportAutomaticHTTP2* and @@ -2907,7 +2980,10 @@ func (srv *Server) onceSetNextProtoDefaults() { // Enable HTTP/2 by default if the user hasn't otherwise // configured their TLSNextProto map. if srv.TLSNextProto == nil { - srv.nextProtoErr = http2ConfigureServer(srv, nil) + conf := &http2Server{ + NewWriteScheduler: func() http2WriteScheduler { return http2NewPriorityWriteScheduler(nil) }, + } + srv.nextProtoErr = http2ConfigureServer(srv, conf) } } diff --git a/libgo/go/net/http/sniff.go b/libgo/go/net/http/sniff.go index 0d21b44a560..ecc65e4de64 100644 --- a/libgo/go/net/http/sniff.go +++ b/libgo/go/net/http/sniff.go @@ -107,8 +107,8 @@ var sniffSignatures = []sniffSig{ ct: "audio/basic", }, &maskedSig{ - mask: []byte("OggS\x00"), - pat: []byte("\x4F\x67\x67\x53\x00"), + mask: []byte("\xFF\xFF\xFF\xFF\xFF"), + pat: []byte("OggS\x00"), ct: "application/ogg", }, &maskedSig{ diff --git a/libgo/go/net/http/sniff_test.go b/libgo/go/net/http/sniff_test.go index 38f3f8197e9..24f1298e5d9 100644 --- a/libgo/go/net/http/sniff_test.go +++ b/libgo/go/net/http/sniff_test.go @@ -45,7 +45,11 @@ var sniffTests = []struct { {"WAV audio #1", []byte("RIFFb\xb8\x00\x00WAVEfmt \x12\x00\x00\x00\x06"), "audio/wave"}, {"WAV audio #2", []byte("RIFF,\x00\x00\x00WAVEfmt \x12\x00\x00\x00\x06"), "audio/wave"}, {"AIFF audio #1", []byte("FORM\x00\x00\x00\x00AIFFCOMM\x00\x00\x00\x12\x00\x01\x00\x00\x57\x55\x00\x10\x40\x0d\xf3\x34"), "audio/aiff"}, + {"OGG audio", []byte("OggS\x00\x02\x00\x00\x00\x00\x00\x00\x00\x00\x7e\x46\x00\x00\x00\x00\x00\x00\x1f\xf6\xb4\xfc\x01\x1e\x01\x76\x6f\x72"), "application/ogg"}, + {"Must not match OGG", []byte("owow\x00"), "application/octet-stream"}, + {"Must not match OGG", []byte("oooS\x00"), "application/octet-stream"}, + {"Must not match OGG", []byte("oggS\x00"), "application/octet-stream"}, // Video types. {"MP4 video", []byte("\x00\x00\x00\x18ftypmp42\x00\x00\x00\x00mp42isom<\x06t\xbfmdat"), "video/mp4"}, diff --git a/libgo/go/net/http/transfer.go b/libgo/go/net/http/transfer.go index 4f47637aa76..8faff2d74a6 100644 --- a/libgo/go/net/http/transfer.go +++ b/libgo/go/net/http/transfer.go @@ -51,6 +51,19 @@ func (br *byteReader) Read(p []byte) (n int, err error) { return 1, io.EOF } +// transferBodyReader is an io.Reader that reads from tw.Body +// and records any non-EOF error in tw.bodyReadError. +// It is exactly 1 pointer wide to avoid allocations into interfaces. +type transferBodyReader struct{ tw *transferWriter } + +func (br transferBodyReader) Read(p []byte) (n int, err error) { + n, err = br.tw.Body.Read(p) + if err != nil && err != io.EOF { + br.tw.bodyReadError = err + } + return +} + // transferWriter inspects the fields of a user-supplied Request or Response, // sanitizes them without changing the user object and provides methods for // writing the respective header, body and trailer in wire format. @@ -62,8 +75,10 @@ type transferWriter struct { ContentLength int64 // -1 means unknown, 0 means exactly none Close bool TransferEncoding []string + Header Header Trailer Header IsResponse bool + bodyReadError error // any non-EOF error from reading Body FlushHeaders bool // flush headers to network before body ByteReadCh chan readResult // non-nil if probeRequestBody called @@ -82,14 +97,15 @@ func newTransferWriter(r interface{}) (t *transferWriter, err error) { t.Method = valueOrDefault(rr.Method, "GET") t.Close = rr.Close t.TransferEncoding = rr.TransferEncoding + t.Header = rr.Header t.Trailer = rr.Trailer - atLeastHTTP11 = rr.protoAtLeastOutgoing(1, 1) t.Body = rr.Body t.BodyCloser = rr.Body t.ContentLength = rr.outgoingLength() - if t.ContentLength < 0 && len(t.TransferEncoding) == 0 && atLeastHTTP11 && t.shouldSendChunkedRequestBody() { + if t.ContentLength < 0 && len(t.TransferEncoding) == 0 && t.shouldSendChunkedRequestBody() { t.TransferEncoding = []string{"chunked"} } + atLeastHTTP11 = true // Transport requests are always 1.1 or 2.0 case *Response: t.IsResponse = true if rr.Request != nil { @@ -100,6 +116,7 @@ func newTransferWriter(r interface{}) (t *transferWriter, err error) { t.ContentLength = rr.ContentLength t.Close = rr.Close t.TransferEncoding = rr.TransferEncoding + t.Header = rr.Header t.Trailer = rr.Trailer atLeastHTTP11 = rr.ProtoAtLeast(1, 1) t.ResponseToHEAD = noResponseBodyExpected(t.Method) @@ -252,7 +269,7 @@ func (t *transferWriter) shouldSendContentLength() bool { } func (t *transferWriter) WriteHeader(w io.Writer) error { - if t.Close { + if t.Close && !hasToken(t.Header.get("Connection"), "close") { if _, err := io.WriteString(w, "Connection: close\r\n"); err != nil { return err } @@ -304,24 +321,25 @@ func (t *transferWriter) WriteBody(w io.Writer) error { // Write body if t.Body != nil { + var body = transferBodyReader{t} if chunked(t.TransferEncoding) { if bw, ok := w.(*bufio.Writer); ok && !t.IsResponse { w = &internal.FlushAfterChunkWriter{Writer: bw} } cw := internal.NewChunkedWriter(w) - _, err = io.Copy(cw, t.Body) + _, err = io.Copy(cw, body) if err == nil { err = cw.Close() } } else if t.ContentLength == -1 { - ncopy, err = io.Copy(w, t.Body) + ncopy, err = io.Copy(w, body) } else { - ncopy, err = io.Copy(w, io.LimitReader(t.Body, t.ContentLength)) + ncopy, err = io.Copy(w, io.LimitReader(body, t.ContentLength)) if err != nil { return err } var nextra int64 - nextra, err = io.Copy(ioutil.Discard, t.Body) + nextra, err = io.Copy(ioutil.Discard, body) ncopy += nextra } if err != nil { diff --git a/libgo/go/net/http/transport.go b/libgo/go/net/http/transport.go index 571943d6e5c..6a89392a996 100644 --- a/libgo/go/net/http/transport.go +++ b/libgo/go/net/http/transport.go @@ -29,6 +29,7 @@ import ( "time" "golang_org/x/net/lex/httplex" + "golang_org/x/net/proxy" ) // DefaultTransport is the default implementation of Transport and is @@ -88,6 +89,11 @@ type Transport struct { // Proxy specifies a function to return a proxy for a given // Request. If the function returns a non-nil error, the // request is aborted with the provided error. + // + // The proxy type is determined by the URL scheme. "http" + // and "socks5" are supported. If the scheme is empty, + // "http" is assumed. + // // If Proxy is nil or returns a nil *URL, no proxy is used. Proxy func(*Request) (*url.URL, error) @@ -275,13 +281,17 @@ func ProxyFromEnvironment(req *Request) (*url.URL, error) { return nil, nil } proxyURL, err := url.Parse(proxy) - if err != nil || !strings.HasPrefix(proxyURL.Scheme, "http") { + if err != nil || + (proxyURL.Scheme != "http" && + proxyURL.Scheme != "https" && + proxyURL.Scheme != "socks5") { // proxy was bogus. Try prepending "http://" to it and // see if that parses correctly. If not, we fall // through and complain about the original one. if proxyURL, err := url.Parse("http://" + proxy); err == nil { return proxyURL, nil } + } if err != nil { return nil, fmt.Errorf("invalid proxy address %q: %v", proxy, err) @@ -298,11 +308,15 @@ func ProxyURL(fixedURL *url.URL) func(*Request) (*url.URL, error) { } // transportRequest is a wrapper around a *Request that adds -// optional extra headers to write. +// optional extra headers to write and stores any error to return +// from roundTrip. type transportRequest struct { *Request // original request, not to be mutated extra Header // extra headers to write, or nil trace *httptrace.ClientTrace // optional + + mu sync.Mutex // guards err + err error // first setError value for mapRoundTripError to consider } func (tr *transportRequest) extraHeaders() Header { @@ -312,6 +326,14 @@ func (tr *transportRequest) extraHeaders() Header { return tr.extra } +func (tr *transportRequest) setError(err error) { + tr.mu.Lock() + if tr.err == nil { + tr.err = err + } + tr.mu.Unlock() +} + // RoundTrip implements the RoundTripper interface. // // For higher-level HTTP client support (such as handling of cookies @@ -402,6 +424,18 @@ func (t *Transport) RoundTrip(req *Request) (*Response, error) { return nil, err } testHookRoundTripRetried() + + // Rewind the body if we're able to. (HTTP/2 does this itself so we only + // need to do it for HTTP/1.1 connections.) + if req.GetBody != nil && pconn.alt == nil { + newReq := *req + var err error + newReq.Body, err = req.GetBody() + if err != nil { + return nil, err + } + req = &newReq + } } } @@ -433,8 +467,9 @@ func (pc *persistConn) shouldRetryRequest(req *Request, err error) bool { return false } if _, ok := err.(nothingWrittenError); ok { - // We never wrote anything, so it's safe to retry. - return true + // We never wrote anything, so it's safe to retry, if there's no body or we + // can "rewind" the body with GetBody. + return req.outgoingLength() == 0 || req.GetBody != nil } if !req.isReplayable() { // Don't retry non-idempotent requests. @@ -788,7 +823,7 @@ func (t *Transport) removeIdleConnLocked(pconn *persistConn) { } t.idleLRU.remove(pconn) key := pconn.cacheKey - pconns, _ := t.idleConn[key] + pconns := t.idleConn[key] switch len(pconns) { case 0: // Nothing @@ -964,6 +999,23 @@ func (t *Transport) getConn(treq *transportRequest, cm connectMethod) (*persistC } } +type oneConnDialer <-chan net.Conn + +func newOneConnDialer(c net.Conn) proxy.Dialer { + ch := make(chan net.Conn, 1) + ch <- c + return oneConnDialer(ch) +} + +func (d oneConnDialer) Dial(network, addr string) (net.Conn, error) { + select { + case c := <-d: + return c, nil + default: + return nil, io.EOF + } +} + func (t *Transport) dialConn(ctx context.Context, cm connectMethod) (*persistConn, error) { pconn := &persistConn{ t: t, @@ -1020,6 +1072,23 @@ func (t *Transport) dialConn(ctx context.Context, cm connectMethod) (*persistCon switch { case cm.proxyURL == nil: // Do nothing. Not using a proxy. + case cm.proxyURL.Scheme == "socks5": + conn := pconn.conn + var auth *proxy.Auth + if u := cm.proxyURL.User; u != nil { + auth = &proxy.Auth{} + auth.User = u.Username() + auth.Password, _ = u.Password() + } + p, err := proxy.SOCKS5("", cm.addr(), auth, newOneConnDialer(conn)) + if err != nil { + conn.Close() + return nil, err + } + if _, err := p.Dial("tcp", cm.targetAddr); err != nil { + conn.Close() + return nil, err + } case cm.targetScheme == "http": pconn.isProxy = true if pa := cm.proxyAuth(); pa != "" { @@ -1176,6 +1245,10 @@ func useProxy(addr string) bool { if addr == p { return false } + if len(p) == 0 { + // There is no host part, likely the entry is malformed; ignore. + continue + } if p[0] == '.' && (strings.HasSuffix(addr, p) || addr == p[1:]) { // no_proxy ".foo.com" matches "bar.foo.com" or "foo.com" return false @@ -1193,19 +1266,21 @@ func useProxy(addr string) bool { // // A connect method may be of the following types: // -// Cache key form Description -// ----------------- ------------------------- -// |http|foo.com http directly to server, no proxy -// |https|foo.com https directly to server, no proxy -// http://proxy.com|https|foo.com http to proxy, then CONNECT to foo.com -// http://proxy.com|http http to proxy, http to anywhere after that +// Cache key form Description +// ----------------- ------------------------- +// |http|foo.com http directly to server, no proxy +// |https|foo.com https directly to server, no proxy +// http://proxy.com|https|foo.com http to proxy, then CONNECT to foo.com +// http://proxy.com|http http to proxy, http to anywhere after that +// socks5://proxy.com|http|foo.com socks5 to proxy, then http to foo.com +// socks5://proxy.com|https|foo.com socks5 to proxy, then https to foo.com // // Note: no support to https to the proxy yet. // type connectMethod struct { proxyURL *url.URL // nil for no proxy, else full proxy URL targetScheme string // "http" or "https" - targetAddr string // Not used if proxy + http targetScheme (4th example in table) + targetAddr string // Not used if http proxy + http targetScheme (4th example in table) } func (cm *connectMethod) key() connectMethodKey { @@ -1213,7 +1288,7 @@ func (cm *connectMethod) key() connectMethodKey { targetAddr := cm.targetAddr if cm.proxyURL != nil { proxyStr = cm.proxyURL.String() - if cm.targetScheme == "http" { + if strings.HasPrefix(cm.proxyURL.Scheme, "http") && cm.targetScheme == "http" { targetAddr = "" } } @@ -1379,63 +1454,53 @@ func (pc *persistConn) closeConnIfStillIdle() { pc.close(errIdleConnTimeout) } -// mapRoundTripErrorFromReadLoop maps the provided readLoop error into -// the error value that should be returned from persistConn.roundTrip. +// mapRoundTripError returns the appropriate error value for +// persistConn.roundTrip. +// +// The provided err is the first error that (*persistConn).roundTrip +// happened to receive from its select statement. // // The startBytesWritten value should be the value of pc.nwrite before the roundTrip // started writing the request. -func (pc *persistConn) mapRoundTripErrorFromReadLoop(req *Request, startBytesWritten int64, err error) (out error) { +func (pc *persistConn) mapRoundTripError(req *transportRequest, startBytesWritten int64, err error) error { if err == nil { return nil } - if err := pc.canceled(); err != nil { - return err + + // If the request was canceled, that's better than network + // failures that were likely the result of tearing down the + // connection. + if cerr := pc.canceled(); cerr != nil { + return cerr + } + + // See if an error was set explicitly. + req.mu.Lock() + reqErr := req.err + req.mu.Unlock() + if reqErr != nil { + return reqErr } + if err == errServerClosedIdle { + // Don't decorate return err } + if _, ok := err.(transportReadFromServerError); ok { + // Don't decorate return err } if pc.isBroken() { <-pc.writeLoopDone - if pc.nwrite == startBytesWritten && req.outgoingLength() == 0 { + if pc.nwrite == startBytesWritten { return nothingWrittenError{err} } + return fmt.Errorf("net/http: HTTP/1.x transport connection broken: %v", err) } return err } -// mapRoundTripErrorAfterClosed returns the error value to be propagated -// up to Transport.RoundTrip method when persistConn.roundTrip sees -// its pc.closech channel close, indicating the persistConn is dead. -// (after closech is closed, pc.closed is valid). -func (pc *persistConn) mapRoundTripErrorAfterClosed(req *Request, startBytesWritten int64) error { - if err := pc.canceled(); err != nil { - return err - } - err := pc.closed - if err == errServerClosedIdle { - // Don't decorate - return err - } - if _, ok := err.(transportReadFromServerError); ok { - // Don't decorate - return err - } - - // Wait for the writeLoop goroutine to terminated, and then - // see if we actually managed to write anything. If not, we - // can retry the request. - <-pc.writeLoopDone - if pc.nwrite == startBytesWritten && req.outgoingLength() == 0 { - return nothingWrittenError{err} - } - - return fmt.Errorf("net/http: HTTP/1.x transport connection broken: %v", err) - -} - func (pc *persistConn) readLoop() { closeErr := errReadLoopExiting // default value, if not changed below defer func() { @@ -1497,16 +1562,6 @@ func (pc *persistConn) readLoop() { err = fmt.Errorf("net/http: server response headers exceeded %d bytes; aborted", pc.maxHeaderResponseSize()) } - // If we won't be able to retry this request later (from the - // roundTrip goroutine), mark it as done now. - // BEFORE the send on rc.ch, as the client might re-use the - // same *Request pointer, and we don't want to set call - // t.setReqCanceler from this persistConn while the Transport - // potentially spins up a different persistConn for the - // caller's subsequent request. - if !pc.shouldRetryRequest(rc.req, err) { - pc.t.setReqCanceler(rc.req, nil) - } select { case rc.ch <- responseAndError{err: err}: case <-rc.callerGone: @@ -1579,7 +1634,7 @@ func (pc *persistConn) readLoop() { } resp.Body = body - if rc.addedGzip && resp.Header.Get("Content-Encoding") == "gzip" { + if rc.addedGzip && strings.EqualFold(resp.Header.Get("Content-Encoding"), "gzip") { resp.Body = &gzipReader{body: body} resp.Header.Del("Content-Encoding") resp.Header.Del("Content-Length") @@ -1705,12 +1760,23 @@ func (pc *persistConn) writeLoop() { case wr := <-pc.writech: startBytesWritten := pc.nwrite err := wr.req.Request.write(pc.bw, pc.isProxy, wr.req.extra, pc.waitForContinue(wr.continueCh)) + if bre, ok := err.(requestBodyReadError); ok { + err = bre.error + // Errors reading from the user's + // Request.Body are high priority. + // Set it here before sending on the + // channels below or calling + // pc.close() which tears town + // connections and causes other + // errors. + wr.req.setError(err) + } if err == nil { err = pc.bw.Flush() } if err != nil { wr.req.Request.closeBody() - if pc.nwrite == startBytesWritten && wr.req.outgoingLength() == 0 { + if pc.nwrite == startBytesWritten { err = nothingWrittenError{err} } } @@ -1872,6 +1938,14 @@ func (pc *persistConn) roundTrip(req *transportRequest) (resp *Response, err err gone := make(chan struct{}) defer close(gone) + defer func() { + if err != nil { + pc.t.setReqCanceler(req.Request, nil) + } + }() + + const debugRoundTrip = false + // Write the request concurrently with waiting for a response, // in case the server decides to reply before reading our full // request body. @@ -1888,38 +1962,50 @@ func (pc *persistConn) roundTrip(req *transportRequest) (resp *Response, err err callerGone: gone, } - var re responseAndError var respHeaderTimer <-chan time.Time cancelChan := req.Request.Cancel ctxDoneChan := req.Context().Done() -WaitResponse: for { testHookWaitResLoop() select { case err := <-writeErrCh: + if debugRoundTrip { + req.logf("writeErrCh resv: %T/%#v", err, err) + } if err != nil { - if cerr := pc.canceled(); cerr != nil { - err = cerr - } - re = responseAndError{err: err} pc.close(fmt.Errorf("write error: %v", err)) - break WaitResponse + return nil, pc.mapRoundTripError(req, startBytesWritten, err) } if d := pc.t.ResponseHeaderTimeout; d > 0 { + if debugRoundTrip { + req.logf("starting timer for %v", d) + } timer := time.NewTimer(d) defer timer.Stop() // prevent leaks respHeaderTimer = timer.C } case <-pc.closech: - re = responseAndError{err: pc.mapRoundTripErrorAfterClosed(req.Request, startBytesWritten)} - break WaitResponse + if debugRoundTrip { + req.logf("closech recv: %T %#v", pc.closed, pc.closed) + } + return nil, pc.mapRoundTripError(req, startBytesWritten, pc.closed) case <-respHeaderTimer: + if debugRoundTrip { + req.logf("timeout waiting for response headers.") + } pc.close(errTimeout) - re = responseAndError{err: errTimeout} - break WaitResponse - case re = <-resc: - re.err = pc.mapRoundTripErrorFromReadLoop(req.Request, startBytesWritten, re.err) - break WaitResponse + return nil, errTimeout + case re := <-resc: + if (re.res == nil) == (re.err == nil) { + panic(fmt.Sprintf("internal error: exactly one of res or err should be set; nil=%v", re.res == nil)) + } + if debugRoundTrip { + req.logf("resc recv: %p, %T/%#v", re.res, re.err, re.err) + } + if re.err != nil { + return nil, pc.mapRoundTripError(req, startBytesWritten, re.err) + } + return re.res, nil case <-cancelChan: pc.t.CancelRequest(req.Request) cancelChan = nil @@ -1929,14 +2015,16 @@ WaitResponse: ctxDoneChan = nil } } +} - if re.err != nil { - pc.t.setReqCanceler(req.Request, nil) - } - if (re.res == nil) == (re.err == nil) { - panic("internal error: exactly one of res or err should be set") +// tLogKey is a context WithValue key for test debugging contexts containing +// a t.Logf func. See export_test.go's Request.WithT method. +type tLogKey struct{} + +func (r *transportRequest) logf(format string, args ...interface{}) { + if logf, ok := r.Request.Context().Value(tLogKey{}).(func(string, ...interface{})); ok { + logf(time.Now().Format(time.RFC3339Nano)+": "+format, args...) } - return re.res, re.err } // markReused marks this connection as having been successfully used for a @@ -1982,8 +2070,9 @@ func (pc *persistConn) closeLocked(err error) { } var portMap = map[string]string{ - "http": "80", - "https": "443", + "http": "80", + "https": "443", + "socks5": "1080", } // canonicalAddr returns url.Host but always with a ":port" suffix diff --git a/libgo/go/net/http/transport_internal_test.go b/libgo/go/net/http/transport_internal_test.go index 3d24fc127d4..594bf6e2c83 100644 --- a/libgo/go/net/http/transport_internal_test.go +++ b/libgo/go/net/http/transport_internal_test.go @@ -9,6 +9,7 @@ package http import ( "errors" "net" + "strings" "testing" ) @@ -30,6 +31,7 @@ func TestTransportPersistConnReadLoopEOF(t *testing.T) { tr := new(Transport) req, _ := NewRequest("GET", "http://"+ln.Addr().String(), nil) + req = req.WithT(t) treq := &transportRequest{Request: req} cm := connectMethod{targetScheme: "http", targetAddr: ln.Addr().String()} pc, err := tr.getConn(treq, cm) @@ -47,13 +49,13 @@ func TestTransportPersistConnReadLoopEOF(t *testing.T) { _, err = pc.roundTrip(treq) if !isTransportReadFromServerError(err) && err != errServerClosedIdle { - t.Fatalf("roundTrip = %#v, %v; want errServerClosedConn or errServerClosedIdle", err, err) + t.Errorf("roundTrip = %#v, %v; want errServerClosedIdle or transportReadFromServerError", err, err) } <-pc.closech err = pc.closed if !isTransportReadFromServerError(err) && err != errServerClosedIdle { - t.Fatalf("pc.closed = %#v, %v; want errServerClosedConn or errServerClosedIdle", err, err) + t.Errorf("pc.closed = %#v, %v; want errServerClosedIdle or transportReadFromServerError", err, err) } } @@ -80,6 +82,19 @@ func dummyRequest(method string) *Request { } return req } +func dummyRequestWithBody(method string) *Request { + req, err := NewRequest(method, "http://fake.tld/", strings.NewReader("foo")) + if err != nil { + panic(err) + } + return req +} + +func dummyRequestWithBodyNoGetBody(method string) *Request { + req := dummyRequestWithBody(method) + req.GetBody = nil + return req +} func TestTransportShouldRetryRequest(t *testing.T) { tests := []struct { @@ -131,6 +146,18 @@ func TestTransportShouldRetryRequest(t *testing.T) { err: errServerClosedIdle, want: true, }, + 7: { + pc: &persistConn{reused: true}, + req: dummyRequestWithBody("POST"), + err: nothingWrittenError{}, + want: true, + }, + 8: { + pc: &persistConn{reused: true}, + req: dummyRequestWithBodyNoGetBody("POST"), + err: nothingWrittenError{}, + want: false, + }, } for i, tt := range tests { got := tt.pc.shouldRetryRequest(tt.req, tt.err) diff --git a/libgo/go/net/http/transport_test.go b/libgo/go/net/http/transport_test.go index a58b1839cc6..27b55dca2f3 100644 --- a/libgo/go/net/http/transport_test.go +++ b/libgo/go/net/http/transport_test.go @@ -16,6 +16,7 @@ import ( "context" "crypto/rand" "crypto/tls" + "encoding/binary" "errors" "fmt" "internal/nettrace" @@ -130,11 +131,9 @@ func TestTransportKeepAlives(t *testing.T) { ts := httptest.NewServer(hostPortHandler) defer ts.Close() + c := ts.Client() for _, disableKeepAlive := range []bool{false, true} { - tr := &Transport{DisableKeepAlives: disableKeepAlive} - defer tr.CloseIdleConnections() - c := &Client{Transport: tr} - + c.Transport.(*Transport).DisableKeepAlives = disableKeepAlive fetch := func(n int) string { res, err := c.Get(ts.URL) if err != nil { @@ -165,12 +164,11 @@ func TestTransportConnectionCloseOnResponse(t *testing.T) { connSet, testDial := makeTestDial(t) - for _, connectionClose := range []bool{false, true} { - tr := &Transport{ - Dial: testDial, - } - c := &Client{Transport: tr} + c := ts.Client() + tr := c.Transport.(*Transport) + tr.Dial = testDial + for _, connectionClose := range []bool{false, true} { fetch := func(n int) string { req := new(Request) var err error @@ -216,12 +214,10 @@ func TestTransportConnectionCloseOnRequest(t *testing.T) { connSet, testDial := makeTestDial(t) + c := ts.Client() + tr := c.Transport.(*Transport) + tr.Dial = testDial for _, connectionClose := range []bool{false, true} { - tr := &Transport{ - Dial: testDial, - } - c := &Client{Transport: tr} - fetch := func(n int) string { req := new(Request) var err error @@ -272,10 +268,9 @@ func TestTransportConnectionCloseOnRequestDisableKeepAlive(t *testing.T) { ts := httptest.NewServer(hostPortHandler) defer ts.Close() - tr := &Transport{ - DisableKeepAlives: true, - } - c := &Client{Transport: tr} + c := ts.Client() + c.Transport.(*Transport).DisableKeepAlives = true + res, err := c.Get(ts.URL) if err != nil { t.Fatal(err) @@ -290,9 +285,8 @@ func TestTransportIdleCacheKeys(t *testing.T) { defer afterTest(t) ts := httptest.NewServer(hostPortHandler) defer ts.Close() - - tr := &Transport{DisableKeepAlives: false} - c := &Client{Transport: tr} + c := ts.Client() + tr := c.Transport.(*Transport) if e, g := 0, len(tr.IdleConnKeysForTesting()); e != g { t.Errorf("After CloseIdleConnections expected %d idle conn cache keys; got %d", e, g) @@ -384,9 +378,11 @@ func TestTransportMaxPerHostIdleConns(t *testing.T) { } })) defer ts.Close() + + c := ts.Client() + tr := c.Transport.(*Transport) maxIdleConnsPerHost := 2 - tr := &Transport{DisableKeepAlives: false, MaxIdleConnsPerHost: maxIdleConnsPerHost} - c := &Client{Transport: tr} + tr.MaxIdleConnsPerHost = maxIdleConnsPerHost // Start 3 outstanding requests and wait for the server to get them. // Their responses will hang until we write to resch, though. @@ -449,9 +445,8 @@ func TestTransportRemovesDeadIdleConnections(t *testing.T) { })) defer ts.Close() - tr := &Transport{} - defer tr.CloseIdleConnections() - c := &Client{Transport: tr} + c := ts.Client() + tr := c.Transport.(*Transport) doReq := func(name string) string { // Do a POST instead of a GET to prevent the Transport's @@ -495,9 +490,7 @@ func TestTransportServerClosingUnexpectedly(t *testing.T) { defer afterTest(t) ts := httptest.NewServer(hostPortHandler) defer ts.Close() - - tr := &Transport{} - c := &Client{Transport: tr} + c := ts.Client() fetch := func(n, retries int) string { condFatalf := func(format string, arg ...interface{}) { @@ -563,10 +556,7 @@ func TestStressSurpriseServerCloses(t *testing.T) { conn.Close() })) defer ts.Close() - - tr := &Transport{DisableKeepAlives: false} - c := &Client{Transport: tr} - defer tr.CloseIdleConnections() + c := ts.Client() // Do a bunch of traffic from different goroutines. Send to activityc // after each request completes, regardless of whether it failed. @@ -619,9 +609,8 @@ func TestTransportHeadResponses(t *testing.T) { w.WriteHeader(200) })) defer ts.Close() + c := ts.Client() - tr := &Transport{DisableKeepAlives: false} - c := &Client{Transport: tr} for i := 0; i < 2; i++ { res, err := c.Head(ts.URL) if err != nil { @@ -655,10 +644,7 @@ func TestTransportHeadChunkedResponse(t *testing.T) { w.WriteHeader(200) })) defer ts.Close() - - tr := &Transport{DisableKeepAlives: false} - c := &Client{Transport: tr} - defer tr.CloseIdleConnections() + c := ts.Client() // Ensure that we wait for the readLoop to complete before // calling Head again @@ -719,6 +705,7 @@ func TestRoundTripGzip(t *testing.T) { } })) defer ts.Close() + tr := ts.Client().Transport.(*Transport) for i, test := range roundTripTests { // Test basic request (no accept-encoding) @@ -726,7 +713,7 @@ func TestRoundTripGzip(t *testing.T) { if test.accept != "" { req.Header.Set("Accept-Encoding", test.accept) } - res, err := DefaultTransport.RoundTrip(req) + res, err := tr.RoundTrip(req) var body []byte if test.compressed { var r *gzip.Reader @@ -791,10 +778,9 @@ func TestTransportGzip(t *testing.T) { gz.Close() })) defer ts.Close() + c := ts.Client() for _, chunked := range []string{"1", "0"} { - c := &Client{Transport: &Transport{}} - // First fetch something large, but only read some of it. res, err := c.Get(ts.URL + "/?body=large&chunked=" + chunked) if err != nil { @@ -844,7 +830,6 @@ func TestTransportGzip(t *testing.T) { } // And a HEAD request too, because they're always weird. - c := &Client{Transport: &Transport{}} res, err := c.Head(ts.URL) if err != nil { t.Fatalf("Head: %v", err) @@ -914,11 +899,13 @@ func TestTransportExpect100Continue(t *testing.T) { {path: "/timeout", body: []byte("hello"), sent: 5, status: 200}, // Timeout exceeded and entire body is sent. } + c := ts.Client() for i, v := range tests { - tr := &Transport{ExpectContinueTimeout: 2 * time.Second} + tr := &Transport{ + ExpectContinueTimeout: 2 * time.Second, + } defer tr.CloseIdleConnections() - c := &Client{Transport: tr} - + c.Transport = tr body := bytes.NewReader(v.body) req, err := NewRequest("PUT", ts.URL+v.path, body) if err != nil { @@ -943,6 +930,99 @@ func TestTransportExpect100Continue(t *testing.T) { } } +func TestSocks5Proxy(t *testing.T) { + defer afterTest(t) + ch := make(chan string, 1) + ts := httptest.NewServer(HandlerFunc(func(w ResponseWriter, r *Request) { + ch <- "real server" + })) + defer ts.Close() + l := newLocalListener(t) + defer l.Close() + go func() { + defer close(ch) + s, err := l.Accept() + if err != nil { + t.Errorf("socks5 proxy Accept(): %v", err) + return + } + defer s.Close() + var buf [22]byte + if _, err := io.ReadFull(s, buf[:3]); err != nil { + t.Errorf("socks5 proxy initial read: %v", err) + return + } + if want := []byte{5, 1, 0}; !bytes.Equal(buf[:3], want) { + t.Errorf("socks5 proxy initial read: got %v, want %v", buf[:3], want) + return + } + if _, err := s.Write([]byte{5, 0}); err != nil { + t.Errorf("socks5 proxy initial write: %v", err) + return + } + if _, err := io.ReadFull(s, buf[:4]); err != nil { + t.Errorf("socks5 proxy second read: %v", err) + return + } + if want := []byte{5, 1, 0}; !bytes.Equal(buf[:3], want) { + t.Errorf("socks5 proxy second read: got %v, want %v", buf[:3], want) + return + } + var ipLen int + switch buf[3] { + case 1: + ipLen = 4 + case 4: + ipLen = 16 + default: + t.Fatalf("socks5 proxy second read: unexpected address type %v", buf[4]) + } + if _, err := io.ReadFull(s, buf[4:ipLen+6]); err != nil { + t.Errorf("socks5 proxy address read: %v", err) + return + } + ip := net.IP(buf[4 : ipLen+4]) + port := binary.BigEndian.Uint16(buf[ipLen+4 : ipLen+6]) + copy(buf[:3], []byte{5, 0, 0}) + if _, err := s.Write(buf[:ipLen+6]); err != nil { + t.Errorf("socks5 proxy connect write: %v", err) + return + } + done := make(chan struct{}) + srv := &Server{Handler: HandlerFunc(func(w ResponseWriter, r *Request) { + done <- struct{}{} + })} + srv.Serve(&oneConnListener{conn: s}) + <-done + srv.Shutdown(context.Background()) + ch <- fmt.Sprintf("proxy for %s:%d", ip, port) + }() + + pu, err := url.Parse("socks5://" + l.Addr().String()) + if err != nil { + t.Fatal(err) + } + c := ts.Client() + c.Transport.(*Transport).Proxy = ProxyURL(pu) + if _, err := c.Head(ts.URL); err != nil { + t.Error(err) + } + var got string + select { + case got = <-ch: + case <-time.After(5 * time.Second): + t.Fatal("timeout connecting to socks5 proxy") + } + tsu, err := url.Parse(ts.URL) + if err != nil { + t.Fatal(err) + } + want := "proxy for " + tsu.Host + if got != want { + t.Errorf("got %q, want %q", got, want) + } +} + func TestTransportProxy(t *testing.T) { defer afterTest(t) ch := make(chan string, 1) @@ -959,12 +1039,20 @@ func TestTransportProxy(t *testing.T) { if err != nil { t.Fatal(err) } - c := &Client{Transport: &Transport{Proxy: ProxyURL(pu)}} - c.Head(ts.URL) - got := <-ch + c := ts.Client() + c.Transport.(*Transport).Proxy = ProxyURL(pu) + if _, err := c.Head(ts.URL); err != nil { + t.Error(err) + } + var got string + select { + case got = <-ch: + case <-time.After(5 * time.Second): + t.Fatal("timeout connecting to http proxy") + } want := "proxy for " + ts.URL + "/" if got != want { - t.Errorf("want %q, got %q", want, got) + t.Errorf("got %q, want %q", got, want) } } @@ -1022,9 +1110,7 @@ func TestTransportGzipRecursive(t *testing.T) { })) defer ts.Close() - tr := &Transport{} - defer tr.CloseIdleConnections() - c := &Client{Transport: tr} + c := ts.Client() res, err := c.Get(ts.URL) if err != nil { t.Fatal(err) @@ -1052,9 +1138,7 @@ func TestTransportGzipShort(t *testing.T) { })) defer ts.Close() - tr := &Transport{} - defer tr.CloseIdleConnections() - c := &Client{Transport: tr} + c := ts.Client() res, err := c.Get(ts.URL) if err != nil { t.Fatal(err) @@ -1095,9 +1179,8 @@ func TestTransportPersistConnLeak(t *testing.T) { w.WriteHeader(204) })) defer ts.Close() - - tr := &Transport{} - c := &Client{Transport: tr} + c := ts.Client() + tr := c.Transport.(*Transport) n0 := runtime.NumGoroutine() @@ -1160,9 +1243,8 @@ func TestTransportPersistConnLeakShortBody(t *testing.T) { ts := httptest.NewServer(HandlerFunc(func(w ResponseWriter, r *Request) { })) defer ts.Close() - - tr := &Transport{} - c := &Client{Transport: tr} + c := ts.Client() + tr := c.Transport.(*Transport) n0 := runtime.NumGoroutine() body := []byte("Hello") @@ -1194,8 +1276,7 @@ func TestTransportPersistConnLeakShortBody(t *testing.T) { // This used to crash; https://golang.org/issue/3266 func TestTransportIdleConnCrash(t *testing.T) { defer afterTest(t) - tr := &Transport{} - c := &Client{Transport: tr} + var tr *Transport unblockCh := make(chan bool, 1) ts := httptest.NewServer(HandlerFunc(func(w ResponseWriter, r *Request) { @@ -1203,6 +1284,8 @@ func TestTransportIdleConnCrash(t *testing.T) { tr.CloseIdleConnections() })) defer ts.Close() + c := ts.Client() + tr = c.Transport.(*Transport) didreq := make(chan bool) go func() { @@ -1232,8 +1315,7 @@ func TestIssue3644(t *testing.T) { } })) defer ts.Close() - tr := &Transport{} - c := &Client{Transport: tr} + c := ts.Client() res, err := c.Get(ts.URL) if err != nil { t.Fatal(err) @@ -1258,8 +1340,7 @@ func TestIssue3595(t *testing.T) { Error(w, deniedMsg, StatusUnauthorized) })) defer ts.Close() - tr := &Transport{} - c := &Client{Transport: tr} + c := ts.Client() res, err := c.Post(ts.URL, "application/octet-stream", neverEnding('a')) if err != nil { t.Errorf("Post: %v", err) @@ -1283,8 +1364,8 @@ func TestChunkedNoContent(t *testing.T) { })) defer ts.Close() + c := ts.Client() for _, closeBody := range []bool{true, false} { - c := &Client{Transport: &Transport{}} const n = 4 for i := 1; i <= n; i++ { res, err := c.Get(ts.URL) @@ -1324,10 +1405,7 @@ func TestTransportConcurrency(t *testing.T) { SetPendingDialHooks(func() { wg.Add(1) }, wg.Done) defer SetPendingDialHooks(nil, nil) - tr := &Transport{} - defer tr.CloseIdleConnections() - - c := &Client{Transport: tr} + c := ts.Client() reqs := make(chan string) defer close(reqs) @@ -1369,23 +1447,20 @@ func TestIssue4191_InfiniteGetTimeout(t *testing.T) { io.Copy(w, neverEnding('a')) }) ts := httptest.NewServer(mux) + defer ts.Close() timeout := 100 * time.Millisecond - client := &Client{ - Transport: &Transport{ - Dial: func(n, addr string) (net.Conn, error) { - conn, err := net.Dial(n, addr) - if err != nil { - return nil, err - } - conn.SetDeadline(time.Now().Add(timeout)) - if debug { - conn = NewLoggingConn("client", conn) - } - return conn, nil - }, - DisableKeepAlives: true, - }, + c := ts.Client() + c.Transport.(*Transport).Dial = func(n, addr string) (net.Conn, error) { + conn, err := net.Dial(n, addr) + if err != nil { + return nil, err + } + conn.SetDeadline(time.Now().Add(timeout)) + if debug { + conn = NewLoggingConn("client", conn) + } + return conn, nil } getFailed := false @@ -1397,7 +1472,7 @@ func TestIssue4191_InfiniteGetTimeout(t *testing.T) { if debug { println("run", i+1, "of", nRuns) } - sres, err := client.Get(ts.URL + "/get") + sres, err := c.Get(ts.URL + "/get") if err != nil { if !getFailed { // Make the timeout longer, once. @@ -1419,7 +1494,6 @@ func TestIssue4191_InfiniteGetTimeout(t *testing.T) { if debug { println("tests complete; waiting for handlers to finish") } - ts.Close() } func TestIssue4191_InfiniteGetToPutTimeout(t *testing.T) { @@ -1437,21 +1511,17 @@ func TestIssue4191_InfiniteGetToPutTimeout(t *testing.T) { ts := httptest.NewServer(mux) timeout := 100 * time.Millisecond - client := &Client{ - Transport: &Transport{ - Dial: func(n, addr string) (net.Conn, error) { - conn, err := net.Dial(n, addr) - if err != nil { - return nil, err - } - conn.SetDeadline(time.Now().Add(timeout)) - if debug { - conn = NewLoggingConn("client", conn) - } - return conn, nil - }, - DisableKeepAlives: true, - }, + c := ts.Client() + c.Transport.(*Transport).Dial = func(n, addr string) (net.Conn, error) { + conn, err := net.Dial(n, addr) + if err != nil { + return nil, err + } + conn.SetDeadline(time.Now().Add(timeout)) + if debug { + conn = NewLoggingConn("client", conn) + } + return conn, nil } getFailed := false @@ -1463,7 +1533,7 @@ func TestIssue4191_InfiniteGetToPutTimeout(t *testing.T) { if debug { println("run", i+1, "of", nRuns) } - sres, err := client.Get(ts.URL + "/get") + sres, err := c.Get(ts.URL + "/get") if err != nil { if !getFailed { // Make the timeout longer, once. @@ -1477,7 +1547,7 @@ func TestIssue4191_InfiniteGetToPutTimeout(t *testing.T) { break } req, _ := NewRequest("PUT", ts.URL+"/put", sres.Body) - _, err = client.Do(req) + _, err = c.Do(req) if err == nil { sres.Body.Close() t.Errorf("Unexpected successful PUT") @@ -1509,11 +1579,8 @@ func TestTransportResponseHeaderTimeout(t *testing.T) { ts := httptest.NewServer(mux) defer ts.Close() - tr := &Transport{ - ResponseHeaderTimeout: 500 * time.Millisecond, - } - defer tr.CloseIdleConnections() - c := &Client{Transport: tr} + c := ts.Client() + c.Transport.(*Transport).ResponseHeaderTimeout = 500 * time.Millisecond tests := []struct { path string @@ -1525,7 +1592,9 @@ func TestTransportResponseHeaderTimeout(t *testing.T) { {path: "/fast", want: 200}, } for i, tt := range tests { - res, err := c.Get(ts.URL + tt.path) + req, _ := NewRequest("GET", ts.URL+tt.path, nil) + req = req.WithT(t) + res, err := c.Do(req) select { case <-inHandler: case <-time.After(5 * time.Second): @@ -1578,9 +1647,8 @@ func TestTransportCancelRequest(t *testing.T) { defer ts.Close() defer close(unblockc) - tr := &Transport{} - defer tr.CloseIdleConnections() - c := &Client{Transport: tr} + c := ts.Client() + tr := c.Transport.(*Transport) req, _ := NewRequest("GET", ts.URL, nil) res, err := c.Do(req) @@ -1688,9 +1756,8 @@ func TestCancelRequestWithChannel(t *testing.T) { defer ts.Close() defer close(unblockc) - tr := &Transport{} - defer tr.CloseIdleConnections() - c := &Client{Transport: tr} + c := ts.Client() + tr := c.Transport.(*Transport) req, _ := NewRequest("GET", ts.URL, nil) ch := make(chan struct{}) @@ -1747,9 +1814,7 @@ func testCancelRequestWithChannelBeforeDo(t *testing.T, withCtx bool) { defer ts.Close() defer close(unblockc) - tr := &Transport{} - defer tr.CloseIdleConnections() - c := &Client{Transport: tr} + c := ts.Client() req, _ := NewRequest("GET", ts.URL, nil) if withCtx { @@ -1837,9 +1902,8 @@ func TestTransportCloseResponseBody(t *testing.T) { })) defer ts.Close() - tr := &Transport{} - defer tr.CloseIdleConnections() - c := &Client{Transport: tr} + c := ts.Client() + tr := c.Transport.(*Transport) req, _ := NewRequest("GET", ts.URL, nil) defer tr.CancelRequest(req) @@ -1959,18 +2023,12 @@ func TestTransportSocketLateBinding(t *testing.T) { defer ts.Close() dialGate := make(chan bool, 1) - tr := &Transport{ - Dial: func(n, addr string) (net.Conn, error) { - if <-dialGate { - return net.Dial(n, addr) - } - return nil, errors.New("manually closed") - }, - DisableKeepAlives: false, - } - defer tr.CloseIdleConnections() - c := &Client{ - Transport: tr, + c := ts.Client() + c.Transport.(*Transport).Dial = func(n, addr string) (net.Conn, error) { + if <-dialGate { + return net.Dial(n, addr) + } + return nil, errors.New("manually closed") } dialGate <- true // only allow one dial @@ -2160,6 +2218,7 @@ var proxyFromEnvTests = []proxyFromEnvTest{ {env: "https://cache.corp.example.com", want: "https://cache.corp.example.com"}, {env: "http://127.0.0.1:8080", want: "http://127.0.0.1:8080"}, {env: "https://127.0.0.1:8080", want: "https://127.0.0.1:8080"}, + {env: "socks5://127.0.0.1", want: "socks5://127.0.0.1"}, // Don't use secure for http {req: "http://insecure.tld/", env: "http.proxy.tld", httpsenv: "secure.proxy.tld", want: "http://http.proxy.tld"}, @@ -2184,6 +2243,7 @@ var proxyFromEnvTests = []proxyFromEnvTest{ func TestProxyFromEnvironment(t *testing.T) { ResetProxyEnv() + defer ResetProxyEnv() for _, tt := range proxyFromEnvTests { os.Setenv("HTTP_PROXY", tt.env) os.Setenv("HTTPS_PROXY", tt.httpsenv) @@ -2223,14 +2283,11 @@ func TestIdleConnChannelLeak(t *testing.T) { SetReadLoopBeforeNextReadHook(func() { didRead <- true }) defer SetReadLoopBeforeNextReadHook(nil) - tr := &Transport{ - Dial: func(netw, addr string) (net.Conn, error) { - return net.Dial(netw, ts.Listener.Addr().String()) - }, + c := ts.Client() + tr := c.Transport.(*Transport) + tr.Dial = func(netw, addr string) (net.Conn, error) { + return net.Dial(netw, ts.Listener.Addr().String()) } - defer tr.CloseIdleConnections() - - c := &Client{Transport: tr} // First, without keep-alives. for _, disableKeep := range []bool{true, false} { @@ -2273,13 +2330,11 @@ func TestTransportClosesRequestBody(t *testing.T) { })) defer ts.Close() - tr := &Transport{} - defer tr.CloseIdleConnections() - cl := &Client{Transport: tr} + c := ts.Client() closes := 0 - res, err := cl.Post(ts.URL, "text/plain", countCloseReader{&closes, strings.NewReader("hello")}) + res, err := c.Post(ts.URL, "text/plain", countCloseReader{&closes, strings.NewReader("hello")}) if err != nil { t.Fatal(err) } @@ -2365,20 +2420,16 @@ func TestTLSServerClosesConnection(t *testing.T) { fmt.Fprintf(w, "hello") })) defer ts.Close() - tr := &Transport{ - TLSClientConfig: &tls.Config{ - InsecureSkipVerify: true, - }, - } - defer tr.CloseIdleConnections() - client := &Client{Transport: tr} + + c := ts.Client() + tr := c.Transport.(*Transport) var nSuccess = 0 var errs []error const trials = 20 for i := 0; i < trials; i++ { tr.CloseIdleConnections() - res, err := client.Get(ts.URL + "/keep-alive-then-die") + res, err := c.Get(ts.URL + "/keep-alive-then-die") if err != nil { t.Fatal(err) } @@ -2393,7 +2444,7 @@ func TestTLSServerClosesConnection(t *testing.T) { // Now try again and see if we successfully // pick a new connection. - res, err = client.Get(ts.URL + "/") + res, err = c.Get(ts.URL + "/") if err != nil { errs = append(errs, err) continue @@ -2472,22 +2523,20 @@ func TestTransportNoReuseAfterEarlyResponse(t *testing.T) { go io.Copy(ioutil.Discard, conn) })) defer ts.Close() - tr := &Transport{} - defer tr.CloseIdleConnections() - client := &Client{Transport: tr} + c := ts.Client() const bodySize = 256 << 10 finalBit := make(byteFromChanReader, 1) req, _ := NewRequest("POST", ts.URL, io.MultiReader(io.LimitReader(neverEnding('x'), bodySize-1), finalBit)) req.ContentLength = bodySize - res, err := client.Do(req) + res, err := c.Do(req) if err := wantBody(res, err, "foo"); err != nil { t.Errorf("POST response: %v", err) } donec := make(chan bool) go func() { defer close(donec) - res, err = client.Get(ts.URL) + res, err = c.Get(ts.URL) if err := wantBody(res, err, "bar"); err != nil { t.Errorf("GET response: %v", err) return @@ -2519,10 +2568,9 @@ func TestTransportIssue10457(t *testing.T) { conn.Close() })) defer ts.Close() - tr := &Transport{} - defer tr.CloseIdleConnections() - cl := &Client{Transport: tr} - res, err := cl.Get(ts.URL) + c := ts.Client() + + res, err := c.Get(ts.URL) if err != nil { t.Fatalf("Get: %v", err) } @@ -2553,89 +2601,160 @@ type writerFuncConn struct { func (c writerFuncConn) Write(p []byte) (n int, err error) { return c.write(p) } -// Issue 4677. If we try to reuse a connection that the server is in the -// process of closing, we may end up successfully writing out our request (or a -// portion of our request) only to find a connection error when we try to read -// from (or finish writing to) the socket. +// Issues 4677, 18241, and 17844. If we try to reuse a connection that the +// server is in the process of closing, we may end up successfully writing out +// our request (or a portion of our request) only to find a connection error +// when we try to read from (or finish writing to) the socket. // -// NOTE: we resend a request only if the request is idempotent, we reused a -// keep-alive connection, and we haven't yet received any header data. This -// automatically prevents an infinite resend loop because we'll run out of the -// cached keep-alive connections eventually. -func TestRetryIdempotentRequestsOnError(t *testing.T) { - defer afterTest(t) +// NOTE: we resend a request only if: +// - we reused a keep-alive connection +// - we haven't yet received any header data +// - either we wrote no bytes to the server, or the request is idempotent +// This automatically prevents an infinite resend loop because we'll run out of +// the cached keep-alive connections eventually. +func TestRetryRequestsOnError(t *testing.T) { + newRequest := func(method, urlStr string, body io.Reader) *Request { + req, err := NewRequest(method, urlStr, body) + if err != nil { + t.Fatal(err) + } + return req + } - var ( - mu sync.Mutex - logbuf bytes.Buffer - ) - logf := func(format string, args ...interface{}) { - mu.Lock() - defer mu.Unlock() - fmt.Fprintf(&logbuf, format, args...) - logbuf.WriteByte('\n') + testCases := []struct { + name string + failureN int + failureErr error + // Note that we can't just re-use the Request object across calls to c.Do + // because we need to rewind Body between calls. (GetBody is only used to + // rewind Body on failure and redirects, not just because it's done.) + req func() *Request + reqString string + }{ + { + name: "IdempotentNoBodySomeWritten", + // Believe that we've written some bytes to the server, so we know we're + // not just in the "retry when no bytes sent" case". + failureN: 1, + // Use the specific error that shouldRetryRequest looks for with idempotent requests. + failureErr: ExportErrServerClosedIdle, + req: func() *Request { + return newRequest("GET", "http://fake.golang", nil) + }, + reqString: `GET / HTTP/1.1\r\nHost: fake.golang\r\nUser-Agent: Go-http-client/1.1\r\nAccept-Encoding: gzip\r\n\r\n`, + }, + { + name: "IdempotentGetBodySomeWritten", + // Believe that we've written some bytes to the server, so we know we're + // not just in the "retry when no bytes sent" case". + failureN: 1, + // Use the specific error that shouldRetryRequest looks for with idempotent requests. + failureErr: ExportErrServerClosedIdle, + req: func() *Request { + return newRequest("GET", "http://fake.golang", strings.NewReader("foo\n")) + }, + reqString: `GET / HTTP/1.1\r\nHost: fake.golang\r\nUser-Agent: Go-http-client/1.1\r\nContent-Length: 4\r\nAccept-Encoding: gzip\r\n\r\nfoo\n`, + }, + { + name: "NothingWrittenNoBody", + // It's key that we return 0 here -- that's what enables Transport to know + // that nothing was written, even though this is a non-idempotent request. + failureN: 0, + failureErr: errors.New("second write fails"), + req: func() *Request { + return newRequest("DELETE", "http://fake.golang", nil) + }, + reqString: `DELETE / HTTP/1.1\r\nHost: fake.golang\r\nUser-Agent: Go-http-client/1.1\r\nAccept-Encoding: gzip\r\n\r\n`, + }, + { + name: "NothingWrittenGetBody", + // It's key that we return 0 here -- that's what enables Transport to know + // that nothing was written, even though this is a non-idempotent request. + failureN: 0, + failureErr: errors.New("second write fails"), + // Note that NewRequest will set up GetBody for strings.Reader, which is + // required for the retry to occur + req: func() *Request { + return newRequest("POST", "http://fake.golang", strings.NewReader("foo\n")) + }, + reqString: `POST / HTTP/1.1\r\nHost: fake.golang\r\nUser-Agent: Go-http-client/1.1\r\nContent-Length: 4\r\nAccept-Encoding: gzip\r\n\r\nfoo\n`, + }, } - ts := httptest.NewServer(HandlerFunc(func(w ResponseWriter, r *Request) { - logf("Handler") - w.Header().Set("X-Status", "ok") - })) - defer ts.Close() + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + defer afterTest(t) - var writeNumAtomic int32 - tr := &Transport{ - Dial: func(network, addr string) (net.Conn, error) { - logf("Dial") - c, err := net.Dial(network, ts.Listener.Addr().String()) - if err != nil { - logf("Dial error: %v", err) - return nil, err + var ( + mu sync.Mutex + logbuf bytes.Buffer + ) + logf := func(format string, args ...interface{}) { + mu.Lock() + defer mu.Unlock() + fmt.Fprintf(&logbuf, format, args...) + logbuf.WriteByte('\n') } - return &writerFuncConn{ - Conn: c, - write: func(p []byte) (n int, err error) { - if atomic.AddInt32(&writeNumAtomic, 1) == 2 { - logf("intentional write failure") - return 0, errors.New("second write fails") - } - logf("Write(%q)", p) - return c.Write(p) - }, - }, nil - }, - } - defer tr.CloseIdleConnections() - c := &Client{Transport: tr} - SetRoundTripRetried(func() { - logf("Retried.") - }) - defer SetRoundTripRetried(nil) + ts := httptest.NewServer(HandlerFunc(func(w ResponseWriter, r *Request) { + logf("Handler") + w.Header().Set("X-Status", "ok") + })) + defer ts.Close() + + var writeNumAtomic int32 + c := ts.Client() + c.Transport.(*Transport).Dial = func(network, addr string) (net.Conn, error) { + logf("Dial") + c, err := net.Dial(network, ts.Listener.Addr().String()) + if err != nil { + logf("Dial error: %v", err) + return nil, err + } + return &writerFuncConn{ + Conn: c, + write: func(p []byte) (n int, err error) { + if atomic.AddInt32(&writeNumAtomic, 1) == 2 { + logf("intentional write failure") + return tc.failureN, tc.failureErr + } + logf("Write(%q)", p) + return c.Write(p) + }, + }, nil + } - for i := 0; i < 3; i++ { - res, err := c.Get("http://fake.golang/") - if err != nil { - t.Fatalf("i=%d: Get = %v", i, err) - } - res.Body.Close() - } + SetRoundTripRetried(func() { + logf("Retried.") + }) + defer SetRoundTripRetried(nil) - mu.Lock() - got := logbuf.String() - mu.Unlock() - const want = `Dial -Write("GET / HTTP/1.1\r\nHost: fake.golang\r\nUser-Agent: Go-http-client/1.1\r\nAccept-Encoding: gzip\r\n\r\n") + for i := 0; i < 3; i++ { + res, err := c.Do(tc.req()) + if err != nil { + t.Fatalf("i=%d: Do = %v", i, err) + } + res.Body.Close() + } + + mu.Lock() + got := logbuf.String() + mu.Unlock() + want := fmt.Sprintf(`Dial +Write("%s") Handler intentional write failure Retried. Dial -Write("GET / HTTP/1.1\r\nHost: fake.golang\r\nUser-Agent: Go-http-client/1.1\r\nAccept-Encoding: gzip\r\n\r\n") +Write("%s") Handler -Write("GET / HTTP/1.1\r\nHost: fake.golang\r\nUser-Agent: Go-http-client/1.1\r\nAccept-Encoding: gzip\r\n\r\n") +Write("%s") Handler -` - if got != want { - t.Errorf("Log of events differs. Got:\n%s\nWant:\n%s", got, want) +`, tc.reqString, tc.reqString, tc.reqString) + if got != want { + t.Errorf("Log of events differs. Got:\n%s\nWant:\n%s", got, want) + } + }) } } @@ -2649,6 +2768,7 @@ func TestTransportClosesBodyOnError(t *testing.T) { readBody <- err })) defer ts.Close() + c := ts.Client() fakeErr := errors.New("fake error") didClose := make(chan bool, 1) req, _ := NewRequest("POST", ts.URL, struct { @@ -2664,7 +2784,7 @@ func TestTransportClosesBodyOnError(t *testing.T) { return nil }), }) - res, err := DefaultClient.Do(req) + res, err := c.Do(req) if res != nil { defer res.Body.Close() } @@ -2698,23 +2818,19 @@ func TestTransportDialTLS(t *testing.T) { mu.Unlock() })) defer ts.Close() - tr := &Transport{ - DialTLS: func(netw, addr string) (net.Conn, error) { - mu.Lock() - didDial = true - mu.Unlock() - c, err := tls.Dial(netw, addr, &tls.Config{ - InsecureSkipVerify: true, - }) - if err != nil { - return nil, err - } - return c, c.Handshake() - }, + c := ts.Client() + c.Transport.(*Transport).DialTLS = func(netw, addr string) (net.Conn, error) { + mu.Lock() + didDial = true + mu.Unlock() + c, err := tls.Dial(netw, addr, c.Transport.(*Transport).TLSClientConfig) + if err != nil { + return nil, err + } + return c, c.Handshake() } - defer tr.CloseIdleConnections() - client := &Client{Transport: tr} - res, err := client.Get(ts.URL) + + res, err := c.Get(ts.URL) if err != nil { t.Fatal(err) } @@ -2796,10 +2912,11 @@ func TestTransportRangeAndGzip(t *testing.T) { reqc <- r })) defer ts.Close() + c := ts.Client() req, _ := NewRequest("GET", ts.URL, nil) req.Header.Set("Range", "bytes=7-11") - res, err := DefaultClient.Do(req) + res, err := c.Do(req) if err != nil { t.Fatal(err) } @@ -2828,9 +2945,7 @@ func TestTransportResponseCancelRace(t *testing.T) { w.Write(b[:]) })) defer ts.Close() - - tr := &Transport{} - defer tr.CloseIdleConnections() + tr := ts.Client().Transport.(*Transport) req, err := NewRequest("GET", ts.URL, nil) if err != nil { @@ -2859,14 +2974,46 @@ func TestTransportResponseCancelRace(t *testing.T) { res.Body.Close() } +// Test for issue 19248: Content-Encoding's value is case insensitive. +func TestTransportContentEncodingCaseInsensitive(t *testing.T) { + setParallel(t) + defer afterTest(t) + for _, ce := range []string{"gzip", "GZIP"} { + ce := ce + t.Run(ce, func(t *testing.T) { + const encodedString = "Hello Gopher" + ts := httptest.NewServer(HandlerFunc(func(w ResponseWriter, r *Request) { + w.Header().Set("Content-Encoding", ce) + gz := gzip.NewWriter(w) + gz.Write([]byte(encodedString)) + gz.Close() + })) + defer ts.Close() + + res, err := ts.Client().Get(ts.URL) + if err != nil { + t.Fatal(err) + } + + body, err := ioutil.ReadAll(res.Body) + res.Body.Close() + if err != nil { + t.Fatal(err) + } + + if string(body) != encodedString { + t.Fatalf("Expected body %q, got: %q\n", encodedString, string(body)) + } + }) + } +} + func TestTransportDialCancelRace(t *testing.T) { defer afterTest(t) ts := httptest.NewServer(HandlerFunc(func(w ResponseWriter, r *Request) {})) defer ts.Close() - - tr := &Transport{} - defer tr.CloseIdleConnections() + tr := ts.Client().Transport.(*Transport) req, err := NewRequest("GET", ts.URL, nil) if err != nil { @@ -2993,6 +3140,7 @@ func TestTransportPrefersResponseOverWriteError(t *testing.T) { w.WriteHeader(StatusOK) })) defer ts.Close() + c := ts.Client() fail := 0 count := 100 @@ -3002,10 +3150,7 @@ func TestTransportPrefersResponseOverWriteError(t *testing.T) { if err != nil { t.Fatal(err) } - tr := new(Transport) - defer tr.CloseIdleConnections() - client := &Client{Transport: tr} - resp, err := client.Do(req) + resp, err := c.Do(req) if err != nil { fail++ t.Logf("%d = %#v", i, err) @@ -3218,10 +3363,8 @@ func testTransportReuseConnection_Gzip(t *testing.T, chunked bool) { w.Write(rgz) // arbitrary gzip response })) defer ts.Close() + c := ts.Client() - tr := &Transport{} - defer tr.CloseIdleConnections() - c := &Client{Transport: tr} for i := 0; i < 2; i++ { res, err := c.Get(ts.URL) if err != nil { @@ -3250,12 +3393,9 @@ func TestTransportResponseHeaderLength(t *testing.T) { } })) defer ts.Close() + c := ts.Client() + c.Transport.(*Transport).MaxResponseHeaderBytes = 512 << 10 - tr := &Transport{ - MaxResponseHeaderBytes: 512 << 10, - } - defer tr.CloseIdleConnections() - c := &Client{Transport: tr} if res, err := c.Get(ts.URL); err != nil { t.Fatal(err) } else { @@ -3426,16 +3566,26 @@ func testTransportEventTrace(t *testing.T, h2 bool, noHooks bool) { } } -func TestTransportEventTraceRealDNS(t *testing.T) { - if testing.Short() && testenv.Builder() == "" { - // Skip this test in short mode (the default for - // all.bash), in case the user is using a shady/ISP - // DNS server hijacking queries. - // See issues 16732, 16716. - // Our builders use 8.8.8.8, though, which correctly - // returns NXDOMAIN, so still run this test there. - t.Skip("skipping in short mode") +var ( + isDNSHijackedOnce sync.Once + isDNSHijacked bool +) + +func skipIfDNSHijacked(t *testing.T) { + // Skip this test if the user is using a shady/ISP + // DNS server hijacking queries. + // See issues 16732, 16716. + isDNSHijackedOnce.Do(func() { + addrs, _ := net.LookupHost("dns-should-not-resolve.golang") + isDNSHijacked = len(addrs) != 0 + }) + if isDNSHijacked { + t.Skip("skipping; test requires non-hijacking DNS server") } +} + +func TestTransportEventTraceRealDNS(t *testing.T) { + skipIfDNSHijacked(t) defer afterTest(t) tr := &Transport{} defer tr.CloseIdleConnections() @@ -3506,8 +3656,8 @@ func TestTransportRejectsAlphaPort(t *testing.T) { // connections. The http2 test is done in TestTransportEventTrace_h2 func TestTLSHandshakeTrace(t *testing.T) { defer afterTest(t) - s := httptest.NewTLSServer(HandlerFunc(func(w ResponseWriter, r *Request) {})) - defer s.Close() + ts := httptest.NewTLSServer(HandlerFunc(func(w ResponseWriter, r *Request) {})) + defer ts.Close() var mu sync.Mutex var start, done bool @@ -3527,10 +3677,8 @@ func TestTLSHandshakeTrace(t *testing.T) { }, } - tr := &Transport{TLSClientConfig: &tls.Config{InsecureSkipVerify: true}} - defer tr.CloseIdleConnections() - c := &Client{Transport: tr} - req, err := NewRequest("GET", s.URL, nil) + c := ts.Client() + req, err := NewRequest("GET", ts.URL, nil) if err != nil { t.Fatal("Unable to construct test request:", err) } @@ -3557,16 +3705,14 @@ func TestTransportMaxIdleConns(t *testing.T) { // No body for convenience. })) defer ts.Close() - tr := &Transport{ - MaxIdleConns: 4, - } - defer tr.CloseIdleConnections() + c := ts.Client() + tr := c.Transport.(*Transport) + tr.MaxIdleConns = 4 ip, port, err := net.SplitHostPort(ts.Listener.Addr().String()) if err != nil { t.Fatal(err) } - c := &Client{Transport: tr} ctx := context.WithValue(context.Background(), nettrace.LookupIPAltResolverKey{}, func(ctx context.Context, host string) ([]net.IPAddr, error) { return []net.IPAddr{{IP: net.ParseIP(ip)}}, nil }) @@ -3862,17 +4008,16 @@ func TestTransportProxyConnectHeader(t *testing.T) { c.Close() })) defer ts.Close() - tr := &Transport{ - ProxyConnectHeader: Header{ - "User-Agent": {"foo"}, - "Other": {"bar"}, - }, - Proxy: func(r *Request) (*url.URL, error) { - return url.Parse(ts.URL) - }, + + c := ts.Client() + c.Transport.(*Transport).Proxy = func(r *Request) (*url.URL, error) { + return url.Parse(ts.URL) } - defer tr.CloseIdleConnections() - c := &Client{Transport: tr} + c.Transport.(*Transport).ProxyConnectHeader = Header{ + "User-Agent": {"foo"}, + "Other": {"bar"}, + } + res, err := c.Get("https://dummy.tld/") // https to force a CONNECT if err == nil { res.Body.Close() diff --git a/libgo/go/net/interface.go b/libgo/go/net/interface.go index b3297f249d6..4036a7f4750 100644 --- a/libgo/go/net/interface.go +++ b/libgo/go/net/interface.go @@ -211,30 +211,30 @@ func (zc *ipv6ZoneCache) update(ift []Interface) { } } -func zoneToString(zone int) string { - if zone == 0 { +func (zc *ipv6ZoneCache) name(index int) string { + if index == 0 { return "" } zoneCache.update(nil) zoneCache.RLock() defer zoneCache.RUnlock() - name, ok := zoneCache.toName[zone] + name, ok := zoneCache.toName[index] if !ok { - name = uitoa(uint(zone)) + name = uitoa(uint(index)) } return name } -func zoneToInt(zone string) int { - if zone == "" { +func (zc *ipv6ZoneCache) index(name string) int { + if name == "" { return 0 } zoneCache.update(nil) zoneCache.RLock() defer zoneCache.RUnlock() - index, ok := zoneCache.toIndex[zone] + index, ok := zoneCache.toIndex[name] if !ok { - index, _, _ = dtoi(zone) + index, _, _ = dtoi(name) } return index } diff --git a/libgo/go/net/interface_linux.go b/libgo/go/net/interface_linux.go index 5e391b28b0f..441ab2f8805 100644 --- a/libgo/go/net/interface_linux.go +++ b/libgo/go/net/interface_linux.go @@ -162,7 +162,7 @@ loop: if err != nil { return nil, os.NewSyscallError("parsenetlinkrouteattr", err) } - ifa := newAddr(ifi, ifam, attrs) + ifa := newAddr(ifam, attrs) if ifa != nil { ifat = append(ifat, ifa) } @@ -172,7 +172,7 @@ loop: return ifat, nil } -func newAddr(ifi *Interface, ifam *syscall.IfAddrmsg, attrs []syscall.NetlinkRouteAttr) Addr { +func newAddr(ifam *syscall.IfAddrmsg, attrs []syscall.NetlinkRouteAttr) Addr { var ipPointToPoint bool // Seems like we need to make sure whether the IP interface // stack consists of IP point-to-point numbered or unnumbered diff --git a/libgo/go/net/interface_test.go b/libgo/go/net/interface_test.go index 38a2ca46565..534137a9133 100644 --- a/libgo/go/net/interface_test.go +++ b/libgo/go/net/interface_test.go @@ -262,13 +262,13 @@ func validateInterfaceMulticastAddrs(ifat []Addr) (*routeStats, error) { func checkUnicastStats(ifStats *ifStats, uniStats *routeStats) error { // Test the existence of connected unicast routes for IPv4. - if supportsIPv4 && ifStats.loop+ifStats.other > 0 && uniStats.ipv4 == 0 { + if supportsIPv4() && ifStats.loop+ifStats.other > 0 && uniStats.ipv4 == 0 { return fmt.Errorf("num IPv4 unicast routes = 0; want >0; summary: %+v, %+v", ifStats, uniStats) } // Test the existence of connected unicast routes for IPv6. // We can assume the existence of ::1/128 when at least one // loopback interface is installed. - if supportsIPv6 && ifStats.loop > 0 && uniStats.ipv6 == 0 { + if supportsIPv6() && ifStats.loop > 0 && uniStats.ipv6 == 0 { return fmt.Errorf("num IPv6 unicast routes = 0; want >0; summary: %+v, %+v", ifStats, uniStats) } return nil @@ -290,7 +290,7 @@ func checkMulticastStats(ifStats *ifStats, uniStats, multiStats *routeStats) err // We can assume the existence of connected multicast // route clones when at least two connected unicast // routes, ::1/128 and other, are installed. - if supportsIPv6 && ifStats.loop > 0 && uniStats.ipv6 > 1 && multiStats.ipv6 == 0 { + if supportsIPv6() && ifStats.loop > 0 && uniStats.ipv6 > 1 && multiStats.ipv6 == 0 { return fmt.Errorf("num IPv6 multicast route clones = 0; want >0; summary: %+v, %+v, %+v", ifStats, uniStats, multiStats) } } diff --git a/libgo/go/net/interface_windows.go b/libgo/go/net/interface_windows.go index 8b976e585f3..b08d1582d8a 100644 --- a/libgo/go/net/interface_windows.go +++ b/libgo/go/net/interface_windows.go @@ -24,10 +24,7 @@ func probeWindowsIPStack() (supportsVistaIP bool) { if err != nil { return true // Windows 10 and above will deprecate this API } - if byte(v) < 6 { // major version of Windows Vista is 6 - return false - } - return true + return byte(v) >= 6 // major version of Windows Vista is 6 } // adapterAddresses returns a list of IP adapter and address diff --git a/libgo/go/net/internal/socktest/sys_cloexec.go b/libgo/go/net/internal/socktest/sys_cloexec.go index 340ff071e7e..d1b8f4f3749 100644 --- a/libgo/go/net/internal/socktest/sys_cloexec.go +++ b/libgo/go/net/internal/socktest/sys_cloexec.go @@ -2,7 +2,7 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -// +build freebsd linux +// +build dragonfly freebsd linux package socktest @@ -15,7 +15,7 @@ func (sw *Switch) Accept4(s, flags int) (ns int, sa syscall.Sockaddr, err error) return syscall.Accept4(s, flags) } sw.fmu.RLock() - f, _ := sw.fltab[FilterAccept] + f := sw.fltab[FilterAccept] sw.fmu.RUnlock() af, err := f.apply(so) diff --git a/libgo/go/net/internal/socktest/sys_unix.go b/libgo/go/net/internal/socktest/sys_unix.go index a3d1282c2a4..397c524fb83 100644 --- a/libgo/go/net/internal/socktest/sys_unix.go +++ b/libgo/go/net/internal/socktest/sys_unix.go @@ -14,7 +14,7 @@ func (sw *Switch) Socket(family, sotype, proto int) (s int, err error) { so := &Status{Cookie: cookie(family, sotype, proto)} sw.fmu.RLock() - f, _ := sw.fltab[FilterSocket] + f := sw.fltab[FilterSocket] sw.fmu.RUnlock() af, err := f.apply(so) @@ -47,7 +47,7 @@ func (sw *Switch) Close(s int) (err error) { return syscall.Close(s) } sw.fmu.RLock() - f, _ := sw.fltab[FilterClose] + f := sw.fltab[FilterClose] sw.fmu.RUnlock() af, err := f.apply(so) @@ -77,7 +77,7 @@ func (sw *Switch) Connect(s int, sa syscall.Sockaddr) (err error) { return syscall.Connect(s, sa) } sw.fmu.RLock() - f, _ := sw.fltab[FilterConnect] + f := sw.fltab[FilterConnect] sw.fmu.RUnlock() af, err := f.apply(so) @@ -106,7 +106,7 @@ func (sw *Switch) Listen(s, backlog int) (err error) { return syscall.Listen(s, backlog) } sw.fmu.RLock() - f, _ := sw.fltab[FilterListen] + f := sw.fltab[FilterListen] sw.fmu.RUnlock() af, err := f.apply(so) @@ -135,7 +135,7 @@ func (sw *Switch) Accept(s int) (ns int, sa syscall.Sockaddr, err error) { return syscall.Accept(s) } sw.fmu.RLock() - f, _ := sw.fltab[FilterAccept] + f := sw.fltab[FilterAccept] sw.fmu.RUnlock() af, err := f.apply(so) @@ -168,7 +168,7 @@ func (sw *Switch) GetsockoptInt(s, level, opt int) (soerr int, err error) { return syscall.GetsockoptInt(s, level, opt) } sw.fmu.RLock() - f, _ := sw.fltab[FilterGetsockoptInt] + f := sw.fltab[FilterGetsockoptInt] sw.fmu.RUnlock() af, err := f.apply(so) diff --git a/libgo/go/net/ip.go b/libgo/go/net/ip.go index db3364c1b31..6b7ba4c23ea 100644 --- a/libgo/go/net/ip.go +++ b/libgo/go/net/ip.go @@ -12,6 +12,8 @@ package net +import _ "unsafe" // for go:linkname + // IP address lengths (bytes). const ( IPv4len = 4 @@ -106,7 +108,8 @@ var ( IPv6linklocalallrouters = IP{0xff, 0x02, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0x02} ) -// IsUnspecified reports whether ip is an unspecified address. +// IsUnspecified reports whether ip is an unspecified address, either +// the IPv4 address "0.0.0.0" or the IPv6 address "::". func (ip IP) IsUnspecified() bool { return ip.Equal(IPv4zero) || ip.Equal(IPv6unspecified) } @@ -338,7 +341,8 @@ func ipEmptyString(ip IP) string { } // MarshalText implements the encoding.TextMarshaler interface. -// The encoding is the same as returned by String. +// The encoding is the same as returned by String, with one exception: +// When len(ip) is zero, it returns an empty slice. func (ip IP) MarshalText() ([]byte, error) { if len(ip) == 0 { return []byte(""), nil @@ -381,17 +385,9 @@ func (ip IP) Equal(x IP) bool { return false } -func bytesEqual(x, y []byte) bool { - if len(x) != len(y) { - return false - } - for i, b := range x { - if y[i] != b { - return false - } - } - return true -} +// bytes.Equal is implemented in runtime/asm_$goarch.s +//go:linkname bytesEqual bytes.Equal +func bytesEqual(x, y []byte) bool func (ip IP) matchAddrFamily(x IP) bool { return ip.To4() != nil && x.To4() != nil || ip.To16() != nil && ip.To4() == nil && x.To16() != nil && x.To4() == nil @@ -667,7 +663,7 @@ func ParseIP(s string) IP { // It returns the IP address and the network implied by the IP and // prefix length. // For example, ParseCIDR("192.0.2.1/24") returns the IP address -// 198.0.2.1 and the network 198.0.2.0/24. +// 192.0.2.1 and the network 192.0.2.0/24. func ParseCIDR(s string) (IP, *IPNet, error) { i := byteIndex(s, '/') if i < 0 { diff --git a/libgo/go/net/ip_test.go b/libgo/go/net/ip_test.go index 46551633ce2..ad13388dd27 100644 --- a/libgo/go/net/ip_test.go +++ b/libgo/go/net/ip_test.go @@ -6,6 +6,7 @@ package net import ( "bytes" + "math/rand" "reflect" "runtime" "testing" @@ -468,61 +469,77 @@ func TestNetworkNumberAndMask(t *testing.T) { } } -var splitJoinTests = []struct { - host string - port string - join string -}{ - {"www.google.com", "80", "www.google.com:80"}, - {"127.0.0.1", "1234", "127.0.0.1:1234"}, - {"::1", "80", "[::1]:80"}, - {"fe80::1%lo0", "80", "[fe80::1%lo0]:80"}, - {"localhost%lo0", "80", "[localhost%lo0]:80"}, - {"", "0", ":0"}, - - {"google.com", "https%foo", "google.com:https%foo"}, // Go 1.0 behavior - {"127.0.0.1", "", "127.0.0.1:"}, // Go 1.0 behavior - {"www.google.com", "", "www.google.com:"}, // Go 1.0 behavior -} - -var splitFailureTests = []struct { - hostPort string - err string -}{ - {"www.google.com", "missing port in address"}, - {"127.0.0.1", "missing port in address"}, - {"[::1]", "missing port in address"}, - {"[fe80::1%lo0]", "missing port in address"}, - {"[localhost%lo0]", "missing port in address"}, - {"localhost%lo0", "missing port in address"}, - - {"::1", "too many colons in address"}, - {"fe80::1%lo0", "too many colons in address"}, - {"fe80::1%lo0:80", "too many colons in address"}, +func TestSplitHostPort(t *testing.T) { + for _, tt := range []struct { + hostPort string + host string + port string + }{ + // Host name + {"localhost:http", "localhost", "http"}, + {"localhost:80", "localhost", "80"}, + + // Go-specific host name with zone identifier + {"localhost%lo0:http", "localhost%lo0", "http"}, + {"localhost%lo0:80", "localhost%lo0", "80"}, + {"[localhost%lo0]:http", "localhost%lo0", "http"}, // Go 1 behavior + {"[localhost%lo0]:80", "localhost%lo0", "80"}, // Go 1 behavior + + // IP literal + {"127.0.0.1:http", "127.0.0.1", "http"}, + {"127.0.0.1:80", "127.0.0.1", "80"}, + {"[::1]:http", "::1", "http"}, + {"[::1]:80", "::1", "80"}, + + // IP literal with zone identifier + {"[::1%lo0]:http", "::1%lo0", "http"}, + {"[::1%lo0]:80", "::1%lo0", "80"}, + + // Go-specific wildcard for host name + {":http", "", "http"}, // Go 1 behavior + {":80", "", "80"}, // Go 1 behavior + + // Go-specific wildcard for service name or transport port number + {"golang.org:", "golang.org", ""}, // Go 1 behavior + {"127.0.0.1:", "127.0.0.1", ""}, // Go 1 behavior + {"[::1]:", "::1", ""}, // Go 1 behavior + + // Opaque service name + {"golang.org:https%foo", "golang.org", "https%foo"}, // Go 1 behavior + } { + if host, port, err := SplitHostPort(tt.hostPort); host != tt.host || port != tt.port || err != nil { + t.Errorf("SplitHostPort(%q) = %q, %q, %v; want %q, %q, nil", tt.hostPort, host, port, err, tt.host, tt.port) + } + } - {"localhost%lo0:80", "missing brackets in address"}, + for _, tt := range []struct { + hostPort string + err string + }{ + {"golang.org", "missing port in address"}, + {"127.0.0.1", "missing port in address"}, + {"[::1]", "missing port in address"}, + {"[fe80::1%lo0]", "missing port in address"}, + {"[localhost%lo0]", "missing port in address"}, + {"localhost%lo0", "missing port in address"}, - // Test cases that didn't fail in Go 1.0 + {"::1", "too many colons in address"}, + {"fe80::1%lo0", "too many colons in address"}, + {"fe80::1%lo0:80", "too many colons in address"}, - {"[foo:bar]", "missing port in address"}, - {"[foo:bar]baz", "missing port in address"}, - {"[foo]bar:baz", "missing port in address"}, + // Test cases that didn't fail in Go 1 - {"[foo]:[bar]:baz", "too many colons in address"}, + {"[foo:bar]", "missing port in address"}, + {"[foo:bar]baz", "missing port in address"}, + {"[foo]bar:baz", "missing port in address"}, - {"[foo]:[bar]baz", "unexpected '[' in address"}, - {"foo[bar]:baz", "unexpected '[' in address"}, + {"[foo]:[bar]:baz", "too many colons in address"}, - {"foo]bar:baz", "unexpected ']' in address"}, -} + {"[foo]:[bar]baz", "unexpected '[' in address"}, + {"foo[bar]:baz", "unexpected '[' in address"}, -func TestSplitHostPort(t *testing.T) { - for _, tt := range splitJoinTests { - if host, port, err := SplitHostPort(tt.join); host != tt.host || port != tt.port || err != nil { - t.Errorf("SplitHostPort(%q) = %q, %q, %v; want %q, %q, nil", tt.join, host, port, err, tt.host, tt.port) - } - } - for _, tt := range splitFailureTests { + {"foo]bar:baz", "unexpected ']' in address"}, + } { if host, port, err := SplitHostPort(tt.hostPort); err == nil { t.Errorf("SplitHostPort(%q) should have failed", tt.hostPort) } else { @@ -538,9 +555,43 @@ func TestSplitHostPort(t *testing.T) { } func TestJoinHostPort(t *testing.T) { - for _, tt := range splitJoinTests { - if join := JoinHostPort(tt.host, tt.port); join != tt.join { - t.Errorf("JoinHostPort(%q, %q) = %q; want %q", tt.host, tt.port, join, tt.join) + for _, tt := range []struct { + host string + port string + hostPort string + }{ + // Host name + {"localhost", "http", "localhost:http"}, + {"localhost", "80", "localhost:80"}, + + // Go-specific host name with zone identifier + {"localhost%lo0", "http", "localhost%lo0:http"}, + {"localhost%lo0", "80", "localhost%lo0:80"}, + + // IP literal + {"127.0.0.1", "http", "127.0.0.1:http"}, + {"127.0.0.1", "80", "127.0.0.1:80"}, + {"::1", "http", "[::1]:http"}, + {"::1", "80", "[::1]:80"}, + + // IP literal with zone identifier + {"::1%lo0", "http", "[::1%lo0]:http"}, + {"::1%lo0", "80", "[::1%lo0]:80"}, + + // Go-specific wildcard for host name + {"", "http", ":http"}, // Go 1 behavior + {"", "80", ":80"}, // Go 1 behavior + + // Go-specific wildcard for service name or transport port number + {"golang.org", "", "golang.org:"}, // Go 1 behavior + {"127.0.0.1", "", "127.0.0.1:"}, // Go 1 behavior + {"::1", "", "[::1]:"}, // Go 1 behavior + + // Opaque service name + {"golang.org", "https%foo", "golang.org:https%foo"}, // Go 1 behavior + } { + if hostPort := JoinHostPort(tt.host, tt.port); hostPort != tt.hostPort { + t.Errorf("JoinHostPort(%q, %q) = %q; want %q", tt.host, tt.port, hostPort, tt.hostPort) } } } @@ -645,3 +696,32 @@ func TestIPAddrScope(t *testing.T) { } } } + +func BenchmarkIPEqual(b *testing.B) { + b.Run("IPv4", func(b *testing.B) { + benchmarkIPEqual(b, IPv4len) + }) + b.Run("IPv6", func(b *testing.B) { + benchmarkIPEqual(b, IPv6len) + }) +} + +func benchmarkIPEqual(b *testing.B, size int) { + ips := make([]IP, 1000) + for i := range ips { + ips[i] = make(IP, size) + rand.Read(ips[i]) + } + // Half of the N are equal. + for i := 0; i < b.N/2; i++ { + x := ips[i%len(ips)] + y := ips[i%len(ips)] + x.Equal(y) + } + // The other half are not equal. + for i := 0; i < b.N/2; i++ { + x := ips[i%len(ips)] + y := ips[(i+1)%len(ips)] + x.Equal(y) + } +} diff --git a/libgo/go/net/iprawsock.go b/libgo/go/net/iprawsock.go index d994fc67c65..c4b54f00c4e 100644 --- a/libgo/go/net/iprawsock.go +++ b/libgo/go/net/iprawsock.go @@ -61,30 +61,37 @@ func (a *IPAddr) opAddr() Addr { return a } -// ResolveIPAddr parses addr as an IP address of the form "host" or -// "ipv6-host%zone" and resolves the domain name on the network net, -// which must be "ip", "ip4" or "ip6". +// ResolveIPAddr returns an address of IP end point. // -// Resolving a hostname is not recommended because this returns at most -// one of its IP addresses. -func ResolveIPAddr(net, addr string) (*IPAddr, error) { - if net == "" { // a hint wildcard for Go 1.0 undocumented behavior - net = "ip" +// The network must be an IP network name. +// +// If the host in the address parameter is not a literal IP address, +// ResolveIPAddr resolves the address to an address of IP end point. +// Otherwise, it parses the address as a literal IP address. +// The address parameter can use a host name, but this is not +// recommended, because it will return at most one of the host name's +// IP addresses. +// +// See func Dial for a description of the network and address +// parameters. +func ResolveIPAddr(network, address string) (*IPAddr, error) { + if network == "" { // a hint wildcard for Go 1.0 undocumented behavior + network = "ip" } - afnet, _, err := parseNetwork(context.Background(), net) + afnet, _, err := parseNetwork(context.Background(), network, false) if err != nil { return nil, err } switch afnet { case "ip", "ip4", "ip6": default: - return nil, UnknownNetworkError(net) + return nil, UnknownNetworkError(network) } - addrs, err := DefaultResolver.internetAddrList(context.Background(), afnet, addr) + addrs, err := DefaultResolver.internetAddrList(context.Background(), afnet, address) if err != nil { return nil, err } - return addrs.first(isIPv4).(*IPAddr), nil + return addrs.forResolve(network, address).(*IPAddr), nil } // IPConn is the implementation of the Conn and PacketConn interfaces @@ -93,13 +100,16 @@ type IPConn struct { conn } -// ReadFromIP reads an IP packet from c, copying the payload into b. -// It returns the number of bytes copied into b and the return address -// that was on the packet. -// -// ReadFromIP can be made to time out and return an error with -// Timeout() == true after a fixed time limit; see SetDeadline and -// SetReadDeadline. +// SyscallConn returns a raw network connection. +// This implements the syscall.Conn interface. +func (c *IPConn) SyscallConn() (syscall.RawConn, error) { + if !c.ok() { + return nil, syscall.EINVAL + } + return newRawConn(c.fd) +} + +// ReadFromIP acts like ReadFrom but returns an IPAddr. func (c *IPConn) ReadFromIP(b []byte) (int, *IPAddr, error) { if !c.ok() { return 0, nil, syscall.EINVAL @@ -126,10 +136,13 @@ func (c *IPConn) ReadFrom(b []byte) (int, Addr, error) { return n, addr, err } -// ReadMsgIP reads a packet from c, copying the payload into b and the -// associated out-of-band data into oob. It returns the number of +// ReadMsgIP reads a message from c, copying the payload into b and +// the associated out-of-band data into oob. It returns the number of // bytes copied into b, the number of bytes copied into oob, the flags -// that were set on the packet and the source address of the packet. +// that were set on the message and the source address of the message. +// +// The packages golang.org/x/net/ipv4 and golang.org/x/net/ipv6 can be +// used to manipulate IP-level socket options in oob. func (c *IPConn) ReadMsgIP(b, oob []byte) (n, oobn, flags int, addr *IPAddr, err error) { if !c.ok() { return 0, 0, 0, nil, syscall.EINVAL @@ -141,13 +154,7 @@ func (c *IPConn) ReadMsgIP(b, oob []byte) (n, oobn, flags int, addr *IPAddr, err return } -// WriteToIP writes an IP packet to addr via c, copying the payload -// from b. -// -// WriteToIP can be made to time out and return an error with -// Timeout() == true after a fixed time limit; see SetDeadline and -// SetWriteDeadline. On packet-oriented connections, write timeouts -// are rare. +// WriteToIP acts like WriteTo but takes an IPAddr. func (c *IPConn) WriteToIP(b []byte, addr *IPAddr) (int, error) { if !c.ok() { return 0, syscall.EINVAL @@ -175,9 +182,12 @@ func (c *IPConn) WriteTo(b []byte, addr Addr) (int, error) { return n, err } -// WriteMsgIP writes a packet to addr via c, copying the payload from +// WriteMsgIP writes a message to addr via c, copying the payload from // b and the associated out-of-band data from oob. It returns the // number of payload and out-of-band bytes written. +// +// The packages golang.org/x/net/ipv4 and golang.org/x/net/ipv6 can be +// used to manipulate IP-level socket options in oob. func (c *IPConn) WriteMsgIP(b, oob []byte, addr *IPAddr) (n, oobn int, err error) { if !c.ok() { return 0, 0, syscall.EINVAL @@ -191,25 +201,32 @@ func (c *IPConn) WriteMsgIP(b, oob []byte, addr *IPAddr) (n, oobn int, err error func newIPConn(fd *netFD) *IPConn { return &IPConn{conn{fd}} } -// DialIP connects to the remote address raddr on the network protocol -// netProto, which must be "ip", "ip4", or "ip6" followed by a colon -// and a protocol number or name. -func DialIP(netProto string, laddr, raddr *IPAddr) (*IPConn, error) { - c, err := dialIP(context.Background(), netProto, laddr, raddr) +// DialIP acts like Dial for IP networks. +// +// The network must be an IP network name; see func Dial for details. +// +// If laddr is nil, a local address is automatically chosen. +// If the IP field of raddr is nil or an unspecified IP address, the +// local system is assumed. +func DialIP(network string, laddr, raddr *IPAddr) (*IPConn, error) { + c, err := dialIP(context.Background(), network, laddr, raddr) if err != nil { - return nil, &OpError{Op: "dial", Net: netProto, Source: laddr.opAddr(), Addr: raddr.opAddr(), Err: err} + return nil, &OpError{Op: "dial", Net: network, Source: laddr.opAddr(), Addr: raddr.opAddr(), Err: err} } return c, nil } -// ListenIP listens for incoming IP packets addressed to the local -// address laddr. The returned connection's ReadFrom and WriteTo -// methods can be used to receive and send IP packets with per-packet -// addressing. -func ListenIP(netProto string, laddr *IPAddr) (*IPConn, error) { - c, err := listenIP(context.Background(), netProto, laddr) +// ListenIP acts like ListenPacket for IP networks. +// +// The network must be an IP network name; see func Dial for details. +// +// If the IP field of laddr is nil or an unspecified IP address, +// ListenIP listens on all available IP addresses of the local system +// except multicast IP addresses. +func ListenIP(network string, laddr *IPAddr) (*IPConn, error) { + c, err := listenIP(context.Background(), network, laddr) if err != nil { - return nil, &OpError{Op: "listen", Net: netProto, Source: nil, Addr: laddr.opAddr(), Err: err} + return nil, &OpError{Op: "listen", Net: network, Source: nil, Addr: laddr.opAddr(), Err: err} } return c, nil } diff --git a/libgo/go/net/iprawsock_posix.go b/libgo/go/net/iprawsock_posix.go index 16e65dcc2a8..d613e6fb8e5 100644 --- a/libgo/go/net/iprawsock_posix.go +++ b/libgo/go/net/iprawsock_posix.go @@ -16,7 +16,7 @@ func sockaddrToIP(sa syscall.Sockaddr) Addr { case *syscall.SockaddrInet4: return &IPAddr{IP: sa.Addr[0:]} case *syscall.SockaddrInet6: - return &IPAddr{IP: sa.Addr[0:], Zone: zoneToString(int(sa.ZoneId))} + return &IPAddr{IP: sa.Addr[0:], Zone: zoneCache.name(int(sa.ZoneId))} } return nil } @@ -52,7 +52,7 @@ func (c *IPConn) readFrom(b []byte) (int, *IPAddr, error) { addr = &IPAddr{IP: sa.Addr[0:]} n = stripIPv4Header(n, b) case *syscall.SockaddrInet6: - addr = &IPAddr{IP: sa.Addr[0:], Zone: zoneToString(int(sa.ZoneId))} + addr = &IPAddr{IP: sa.Addr[0:], Zone: zoneCache.name(int(sa.ZoneId))} } return n, addr, err } @@ -79,7 +79,7 @@ func (c *IPConn) readMsg(b, oob []byte) (n, oobn, flags int, addr *IPAddr, err e case *syscall.SockaddrInet4: addr = &IPAddr{IP: sa.Addr[0:]} case *syscall.SockaddrInet6: - addr = &IPAddr{IP: sa.Addr[0:], Zone: zoneToString(int(sa.ZoneId))} + addr = &IPAddr{IP: sa.Addr[0:], Zone: zoneCache.name(int(sa.ZoneId))} } return } @@ -113,7 +113,7 @@ func (c *IPConn) writeMsg(b, oob []byte, addr *IPAddr) (n, oobn int, err error) } func dialIP(ctx context.Context, netProto string, laddr, raddr *IPAddr) (*IPConn, error) { - network, proto, err := parseNetwork(ctx, netProto) + network, proto, err := parseNetwork(ctx, netProto, true) if err != nil { return nil, err } @@ -133,7 +133,7 @@ func dialIP(ctx context.Context, netProto string, laddr, raddr *IPAddr) (*IPConn } func listenIP(ctx context.Context, netProto string, laddr *IPAddr) (*IPConn, error) { - network, proto, err := parseNetwork(ctx, netProto) + network, proto, err := parseNetwork(ctx, netProto, true) if err != nil { return nil, err } diff --git a/libgo/go/net/iprawsock_test.go b/libgo/go/net/iprawsock_test.go index 5d33b26a91e..8972051f5d5 100644 --- a/libgo/go/net/iprawsock_test.go +++ b/libgo/go/net/iprawsock_test.go @@ -117,3 +117,75 @@ func TestIPConnRemoteName(t *testing.T) { t.Fatalf("got %#v; want %#v", c.RemoteAddr(), raddr) } } + +func TestDialListenIPArgs(t *testing.T) { + type test struct { + argLists [][2]string + shouldFail bool + } + tests := []test{ + { + argLists: [][2]string{ + {"ip", "127.0.0.1"}, + {"ip:", "127.0.0.1"}, + {"ip::", "127.0.0.1"}, + {"ip", "::1"}, + {"ip:", "::1"}, + {"ip::", "::1"}, + {"ip4", "127.0.0.1"}, + {"ip4:", "127.0.0.1"}, + {"ip4::", "127.0.0.1"}, + {"ip6", "::1"}, + {"ip6:", "::1"}, + {"ip6::", "::1"}, + }, + shouldFail: true, + }, + } + if testableNetwork("ip") { + priv := test{shouldFail: false} + for _, tt := range []struct { + network, address string + args [2]string + }{ + {"ip4:47", "127.0.0.1", [2]string{"ip4:47", "127.0.0.1"}}, + {"ip6:47", "::1", [2]string{"ip6:47", "::1"}}, + } { + c, err := ListenPacket(tt.network, tt.address) + if err != nil { + continue + } + c.Close() + priv.argLists = append(priv.argLists, tt.args) + } + if len(priv.argLists) > 0 { + tests = append(tests, priv) + } + } + + for _, tt := range tests { + for _, args := range tt.argLists { + _, err := Dial(args[0], args[1]) + if tt.shouldFail != (err != nil) { + t.Errorf("Dial(%q, %q) = %v; want (err != nil) is %t", args[0], args[1], err, tt.shouldFail) + } + _, err = ListenPacket(args[0], args[1]) + if tt.shouldFail != (err != nil) { + t.Errorf("ListenPacket(%q, %q) = %v; want (err != nil) is %t", args[0], args[1], err, tt.shouldFail) + } + a, err := ResolveIPAddr("ip", args[1]) + if err != nil { + t.Errorf("ResolveIPAddr(\"ip\", %q) = %v", args[1], err) + continue + } + _, err = DialIP(args[0], nil, a) + if tt.shouldFail != (err != nil) { + t.Errorf("DialIP(%q, %v) = %v; want (err != nil) is %t", args[0], a, err, tt.shouldFail) + } + _, err = ListenIP(args[0], a) + if tt.shouldFail != (err != nil) { + t.Errorf("ListenIP(%q, %v) = %v; want (err != nil) is %t", args[0], a, err, tt.shouldFail) + } + } + } +} diff --git a/libgo/go/net/ipsock.go b/libgo/go/net/ipsock.go index f1394a7ed87..947bdf34897 100644 --- a/libgo/go/net/ipsock.go +++ b/libgo/go/net/ipsock.go @@ -2,12 +2,11 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -// Internet protocol family sockets - package net import ( "context" + "sync" ) // BUG(rsc,mikio): On DragonFly BSD and OpenBSD, listening on the @@ -17,25 +16,41 @@ import ( // both address families are to be supported. // See inet6(4) for details. -var ( - // supportsIPv4 reports whether the platform supports IPv4 - // networking functionality. - supportsIPv4 bool +type ipStackCapabilities struct { + sync.Once // guards following + ipv4Enabled bool + ipv6Enabled bool + ipv4MappedIPv6Enabled bool +} - // supportsIPv6 reports whether the platform supports IPv6 - // networking functionality. - supportsIPv6 bool +var ipStackCaps ipStackCapabilities - // supportsIPv4map reports whether the platform supports - // mapping an IPv4 address inside an IPv6 address at transport - // layer protocols. See RFC 4291, RFC 4038 and RFC 3493. - supportsIPv4map bool -) +// supportsIPv4 reports whether the platform supports IPv4 networking +// functionality. +func supportsIPv4() bool { + ipStackCaps.Once.Do(ipStackCaps.probe) + return ipStackCaps.ipv4Enabled +} + +// supportsIPv6 reports whether the platform supports IPv6 networking +// functionality. +func supportsIPv6() bool { + ipStackCaps.Once.Do(ipStackCaps.probe) + return ipStackCaps.ipv6Enabled +} + +// supportsIPv4map reports whether the platform supports mapping an +// IPv4 address inside an IPv6 address at transport layer +// protocols. See RFC 4291, RFC 4038 and RFC 3493. +func supportsIPv4map() bool { + ipStackCaps.Once.Do(ipStackCaps.probe) + return ipStackCaps.ipv4MappedIPv6Enabled +} // An addrList represents a list of network endpoint addresses. type addrList []Addr -// isIPv4 returns true if the Addr contains an IPv4 address. +// isIPv4 reports whether addr contains an IPv4 address. func isIPv4(addr Addr) bool { switch addr := addr.(type) { case *TCPAddr: @@ -48,6 +63,28 @@ func isIPv4(addr Addr) bool { return false } +// isNotIPv4 reports whether addr does not contain an IPv4 address. +func isNotIPv4(addr Addr) bool { return !isIPv4(addr) } + +// forResolve returns the most appropriate address in address for +// a call to ResolveTCPAddr, ResolveUDPAddr, or ResolveIPAddr. +// IPv4 is preferred, unless addr contains an IPv6 literal. +func (addrs addrList) forResolve(network, addr string) Addr { + var want6 bool + switch network { + case "ip": + // IPv6 literal (addr does NOT contain a port) + want6 = count(addr, ':') > 0 + case "tcp", "udp": + // IPv6 literal. (addr contains a port, so look for '[') + want6 = count(addr, '[') > 0 + } + if want6 { + return addrs.first(isNotIPv4) + } + return addrs.first(isIPv4) +} + // first returns the first address which satisfies strategy, or if // none do, then the first address of any kind. func (addrs addrList) first(strategy func(Addr) bool) Addr { @@ -107,10 +144,14 @@ func ipv6only(addr IPAddr) bool { } // SplitHostPort splits a network address of the form "host:port", -// "[host]:port" or "[ipv6-host%zone]:port" into host or -// ipv6-host%zone and port. A literal address or host name for IPv6 -// must be enclosed in square brackets, as in "[::1]:80", -// "[ipv6-host]:http" or "[ipv6-host%zone]:80". +// "host%zone:port", "[host]:port" or "[host%zone]:port" into host or +// host%zone and port. +// +// A literal IPv6 address in hostport must be enclosed in square +// brackets, as in "[::1]:80", "[::1%lo0]:80". +// +// See func Dial for a description of the hostport parameter, and host +// and port results. func SplitHostPort(hostport string) (host, port string, err error) { const ( missingPort = "missing port in address" @@ -154,9 +195,6 @@ func SplitHostPort(hostport string) (host, port string, err error) { if byteIndex(host, ':') >= 0 { return addrErr(hostport, tooManyColons) } - if byteIndex(host, '%') >= 0 { - return addrErr(hostport, "missing brackets in address") - } } if byteIndex(hostport[j:], '[') >= 0 { return addrErr(hostport, "unexpected '[' in address") @@ -181,11 +219,14 @@ func splitHostZone(s string) (host, zone string) { } // JoinHostPort combines host and port into a network address of the -// form "host:port" or, if host contains a colon or a percent sign, -// "[host]:port". +// form "host:port". If host contains a colon, as found in literal +// IPv6 addresses, then JoinHostPort returns "[host]:port". +// +// See func Dial for a description of the host and port parameters. func JoinHostPort(host, port string) string { - // If host has colons or a percent sign, have to bracket it. - if byteIndex(host, ':') >= 0 || byteIndex(host, '%') >= 0 { + // We assume that host is a literal IPv6 address if host has + // colons. + if byteIndex(host, ':') >= 0 { return "[" + host + "]:" + port } return host + ":" + port @@ -240,6 +281,13 @@ func (r *Resolver) internetAddrList(ctx context.Context, net, addr string) (addr ips = []IPAddr{{IP: ip}} } else if ip, zone := parseIPv6(host, true); ip != nil { ips = []IPAddr{{IP: ip, Zone: zone}} + // Issue 18806: if the machine has halfway configured + // IPv6 such that it can bind on "::" (IPv6unspecified) + // but not connect back to that same address, fall + // back to dialing 0.0.0.0. + if ip.Equal(IPv6unspecified) { + ips = append(ips, IPAddr{IP: IPv4zero}) + } } else { // Try as a DNS name. ips, err = r.LookupIPAddr(ctx, host) diff --git a/libgo/go/net/ipsock_plan9.go b/libgo/go/net/ipsock_plan9.go index b7fd344c8ad..312e4adb47d 100644 --- a/libgo/go/net/ipsock_plan9.go +++ b/libgo/go/net/ipsock_plan9.go @@ -2,8 +2,6 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -// Internet protocol family sockets for Plan 9 - package net import ( @@ -12,12 +10,25 @@ import ( "syscall" ) +// Probe probes IPv4, IPv6 and IPv4-mapped IPv6 communication +// capabilities. +// +// Plan 9 uses IPv6 natively, see ip(3). +func (p *ipStackCapabilities) probe() { + p.ipv4Enabled = probe(netdir+"/iproute", "4i") + p.ipv6Enabled = probe(netdir+"/iproute", "6i") + if p.ipv4Enabled && p.ipv6Enabled { + p.ipv4MappedIPv6Enabled = true + } +} + func probe(filename, query string) bool { var file *file var err error if file, err = open(filename); err != nil { return false } + defer file.close() r := false for line, ok := file.readLine(); ok && !r; line, ok = file.readLine() { @@ -32,27 +43,9 @@ func probe(filename, query string) bool { } } } - file.close() return r } -func probeIPv4Stack() bool { - return probe(netdir+"/iproute", "4i") -} - -// probeIPv6Stack returns two boolean values. If the first boolean -// value is true, kernel supports basic IPv6 functionality. If the -// second boolean value is true, kernel supports IPv6 IPv4-mapping. -func probeIPv6Stack() (supportsIPv6, supportsIPv4map bool) { - // Plan 9 uses IPv6 natively, see ip(3). - r := probe(netdir+"/iproute", "6i") - v := false - if r { - v = probe(netdir+"/iproute", "4i") - } - return r, v -} - // parsePlan9Addr parses address of the form [ip!]port (e.g. 127.0.0.1!80). func parsePlan9Addr(s string) (ip IP, iport int, err error) { addr := IPv4zero // address contains port only @@ -249,10 +242,10 @@ func (fd *netFD) netFD() (*netFD, error) { func (fd *netFD) acceptPlan9() (nfd *netFD, err error) { defer func() { fixErr(err) }() - if err := fd.readLock(); err != nil { + if err := fd.pfd.ReadLock(); err != nil { return nil, err } - defer fd.readUnlock() + defer fd.pfd.ReadUnlock() listen, err := os.Open(fd.dir + "/listen") if err != nil { return nil, err diff --git a/libgo/go/net/ipsock_posix.go b/libgo/go/net/ipsock_posix.go index 05bf93956bf..4b4363a47a1 100644 --- a/libgo/go/net/ipsock_posix.go +++ b/libgo/go/net/ipsock_posix.go @@ -8,35 +8,29 @@ package net import ( "context" + "internal/poll" "runtime" "syscall" ) -func probeIPv4Stack() bool { +// Probe probes IPv4, IPv6 and IPv4-mapped IPv6 communication +// capabilities which are controlled by the IPV6_V6ONLY socket option +// and kernel configuration. +// +// Should we try to use the IPv4 socket interface if we're only +// dealing with IPv4 sockets? As long as the host system understands +// IPv4-mapped IPv6, it's okay to pass IPv4-mapeed IPv6 addresses to +// the IPv6 interface. That simplifies our code and is most +// general. Unfortunately, we need to run on kernels built without +// IPv6 support too. So probe the kernel to figure it out. +func (p *ipStackCapabilities) probe() { s, err := socketFunc(syscall.AF_INET, syscall.SOCK_STREAM, syscall.IPPROTO_TCP) switch err { case syscall.EAFNOSUPPORT, syscall.EPROTONOSUPPORT: - return false case nil: - closeFunc(s) + poll.CloseFunc(s) + p.ipv4Enabled = true } - return true -} - -// Should we try to use the IPv4 socket interface if we're -// only dealing with IPv4 sockets? As long as the host system -// understands IPv6, it's okay to pass IPv4 addresses to the IPv6 -// interface. That simplifies our code and is most general. -// Unfortunately, we need to run on kernels built without IPv6 -// support too. So probe the kernel to figure it out. -// -// probeIPv6Stack probes both basic IPv6 capability and IPv6 IPv4- -// mapping capability which is controlled by IPV6_V6ONLY socket -// option and/or kernel state "net.inet6.ip6.v6only". -// It returns two boolean values. If the first boolean value is -// true, kernel supports basic IPv6 functionality. If the second -// boolean value is true, kernel supports IPv6 IPv4-mapping. -func probeIPv6Stack() (supportsIPv6, supportsIPv4map bool) { var probes = []struct { laddr TCPAddr value int @@ -46,29 +40,19 @@ func probeIPv6Stack() (supportsIPv6, supportsIPv4map bool) { // IPv4-mapped IPv6 address communication capability {laddr: TCPAddr{IP: IPv4(127, 0, 0, 1)}, value: 0}, } - var supps [2]bool switch runtime.GOOS { case "dragonfly", "openbsd": - // Some released versions of DragonFly BSD pretend to - // accept IPV6_V6ONLY=0 successfully, but the state - // still stays IPV6_V6ONLY=1. Eventually DragonFly BSD - // stops pretending, but the transition period would - // cause unpredictable behavior and we need to avoid - // it. - // - // OpenBSD also doesn't support IPV6_V6ONLY=0 but it - // never pretends to accept IPV6_V6OLY=0. It always - // returns an error and we don't need to probe the - // capability. + // The latest DragonFly BSD and OpenBSD kernels don't + // support IPV6_V6ONLY=0. They always return an error + // and we don't need to probe the capability. probes = probes[:1] } - for i := range probes { s, err := socketFunc(syscall.AF_INET6, syscall.SOCK_STREAM, syscall.IPPROTO_TCP) if err != nil { continue } - defer closeFunc(s) + defer poll.CloseFunc(s) syscall.SetsockoptInt(s, syscall.IPPROTO_IPV6, syscall.IPV6_V6ONLY, probes[i].value) sa, err := probes[i].laddr.sockaddr(syscall.AF_INET6) if err != nil { @@ -77,51 +61,55 @@ func probeIPv6Stack() (supportsIPv6, supportsIPv4map bool) { if err := syscall.Bind(s, sa); err != nil { continue } - supps[i] = true + if i == 0 { + p.ipv6Enabled = true + } else { + p.ipv4MappedIPv6Enabled = true + } } - - return supps[0], supps[1] } -// favoriteAddrFamily returns the appropriate address family to -// the given net, laddr, raddr and mode. At first it figures -// address family out from the net. If mode indicates "listen" -// and laddr is a wildcard, it assumes that the user wants to -// make a passive connection with a wildcard address family, both -// AF_INET and AF_INET6, and a wildcard address like following: +// favoriteAddrFamily returns the appropriate address family for the +// given network, laddr, raddr and mode. +// +// If mode indicates "listen" and laddr is a wildcard, we assume that +// the user wants to make a passive-open connection with a wildcard +// address family, both AF_INET and AF_INET6, and a wildcard address +// like the following: // -// 1. A wild-wild listen, "tcp" + "" -// If the platform supports both IPv6 and IPv6 IPv4-mapping -// capabilities, or does not support IPv4, we assume that -// the user wants to listen on both IPv4 and IPv6 wildcard -// addresses over an AF_INET6 socket with IPV6_V6ONLY=0. -// Otherwise we prefer an IPv4 wildcard address listen over -// an AF_INET socket. +// - A listen for a wildcard communication domain, "tcp" or +// "udp", with a wildcard address: If the platform supports +// both IPv6 and IPv4-mapped IPv6 communication capabilities, +// or does not support IPv4, we use a dual stack, AF_INET6 and +// IPV6_V6ONLY=0, wildcard address listen. The dual stack +// wildcard address listen may fall back to an IPv6-only, +// AF_INET6 and IPV6_V6ONLY=1, wildcard address listen. +// Otherwise we prefer an IPv4-only, AF_INET, wildcard address +// listen. // -// 2. A wild-ipv4wild listen, "tcp" + "0.0.0.0" -// Same as 1. +// - A listen for a wildcard communication domain, "tcp" or +// "udp", with an IPv4 wildcard address: same as above. // -// 3. A wild-ipv6wild listen, "tcp" + "[::]" -// Almost same as 1 but we prefer an IPv6 wildcard address -// listen over an AF_INET6 socket with IPV6_V6ONLY=0 when -// the platform supports IPv6 capability but not IPv6 IPv4- -// mapping capability. +// - A listen for a wildcard communication domain, "tcp" or +// "udp", with an IPv6 wildcard address: same as above. // -// 4. A ipv4-ipv4wild listen, "tcp4" + "" or "0.0.0.0" -// We use an IPv4 (AF_INET) wildcard address listen. +// - A listen for an IPv4 communication domain, "tcp4" or "udp4", +// with an IPv4 wildcard address: We use an IPv4-only, AF_INET, +// wildcard address listen. // -// 5. A ipv6-ipv6wild listen, "tcp6" + "" or "[::]" -// We use an IPv6 (AF_INET6, IPV6_V6ONLY=1) wildcard address -// listen. +// - A listen for an IPv6 communication domain, "tcp6" or "udp6", +// with an IPv6 wildcard address: We use an IPv6-only, AF_INET6 +// and IPV6_V6ONLY=1, wildcard address listen. // -// Otherwise guess: if the addresses are IPv4 then returns AF_INET, -// or else returns AF_INET6. It also returns a boolean value what +// Otherwise guess: If the addresses are IPv4 then returns AF_INET, +// or else returns AF_INET6. It also returns a boolean value what // designates IPV6_V6ONLY option. // -// Note that OpenBSD allows neither "net.inet6.ip6.v6only=1" change -// nor IPPROTO_IPV6 level IPV6_V6ONLY socket option setting. -func favoriteAddrFamily(net string, laddr, raddr sockaddr, mode string) (family int, ipv6only bool) { - switch net[len(net)-1] { +// Note that the latest DragonFly BSD and OpenBSD kernels allow +// neither "net.inet6.ip6.v6only=1" change nor IPPROTO_IPV6 level +// IPV6_V6ONLY socket option setting. +func favoriteAddrFamily(network string, laddr, raddr sockaddr, mode string) (family int, ipv6only bool) { + switch network[len(network)-1] { case '4': return syscall.AF_INET, false case '6': @@ -129,7 +117,7 @@ func favoriteAddrFamily(net string, laddr, raddr sockaddr, mode string) (family } if mode == "listen" && (laddr == nil || laddr.isWildcard()) { - if supportsIPv4map || !supportsIPv4 { + if supportsIPv4map() || !supportsIPv4() { return syscall.AF_INET6, false } if laddr == nil { @@ -145,7 +133,6 @@ func favoriteAddrFamily(net string, laddr, raddr sockaddr, mode string) (family return syscall.AF_INET6, false } -// Internet sockets (TCP, UDP, IP) func internetSocket(ctx context.Context, net string, laddr, raddr sockaddr, sotype, proto int, mode string) (fd *netFD, err error) { if (runtime.GOOS == "windows" || runtime.GOOS == "openbsd" || runtime.GOOS == "nacl") && mode == "dial" && raddr.isWildcard() { raddr = raddr.toLocal(net) @@ -187,7 +174,7 @@ func ipToSockaddr(family int, ip IP, port int, zone string) (syscall.Sockaddr, e if ip6 == nil { return nil, &AddrError{Err: "non-IPv6 address", Addr: ip.String()} } - sa := &syscall.SockaddrInet6{Port: port, ZoneId: uint32(zoneToInt(zone))} + sa := &syscall.SockaddrInet6{Port: port, ZoneId: uint32(zoneCache.index(zone))} copy(sa.Addr[:], ip6) return sa, nil } diff --git a/libgo/go/net/ipsock_test.go b/libgo/go/net/ipsock_test.go index 1d0f00ff5ee..aede3548447 100644 --- a/libgo/go/net/ipsock_test.go +++ b/libgo/go/net/ipsock_test.go @@ -215,7 +215,7 @@ var addrListTests = []struct { } func TestAddrList(t *testing.T) { - if !supportsIPv4 || !supportsIPv6 { + if !supportsIPv4() || !supportsIPv6() { t.Skip("both IPv4 and IPv6 are required") } diff --git a/libgo/go/net/listen_test.go b/libgo/go/net/listen_test.go index 6037f3600df..21ad4462f68 100644 --- a/libgo/go/net/listen_test.go +++ b/libgo/go/net/listen_test.go @@ -225,7 +225,7 @@ func TestDualStackTCPListener(t *testing.T) { case "nacl", "plan9": t.Skipf("not supported on %s", runtime.GOOS) } - if !supportsIPv4 || !supportsIPv6 { + if !supportsIPv4() || !supportsIPv6() { t.Skip("both IPv4 and IPv6 are required") } @@ -235,7 +235,7 @@ func TestDualStackTCPListener(t *testing.T) { continue } - if !supportsIPv4map && differentWildcardAddr(tt.address1, tt.address2) { + if !supportsIPv4map() && differentWildcardAddr(tt.address1, tt.address2) { tt.xerr = nil } var firstErr, secondErr error @@ -315,7 +315,7 @@ func TestDualStackUDPListener(t *testing.T) { case "nacl", "plan9": t.Skipf("not supported on %s", runtime.GOOS) } - if !supportsIPv4 || !supportsIPv6 { + if !supportsIPv4() || !supportsIPv6() { t.Skip("both IPv4 and IPv6 are required") } @@ -325,7 +325,7 @@ func TestDualStackUDPListener(t *testing.T) { continue } - if !supportsIPv4map && differentWildcardAddr(tt.address1, tt.address2) { + if !supportsIPv4map() && differentWildcardAddr(tt.address1, tt.address2) { tt.xerr = nil } var firstErr, secondErr error @@ -454,7 +454,7 @@ func checkDualStackAddrFamily(fd *netFD) error { // and IPv6 IPv4-mapping capability, we can assume // that the node listens on a wildcard address with an // AF_INET6 socket. - if supportsIPv4map && fd.laddr.(*TCPAddr).isWildcard() { + if supportsIPv4map() && fd.laddr.(*TCPAddr).isWildcard() { if fd.family != syscall.AF_INET6 { return fmt.Errorf("Listen(%s, %v) returns %v; want %v", fd.net, fd.laddr, fd.family, syscall.AF_INET6) } @@ -468,7 +468,7 @@ func checkDualStackAddrFamily(fd *netFD) error { // and IPv6 IPv4-mapping capability, we can assume // that the node listens on a wildcard address with an // AF_INET6 socket. - if supportsIPv4map && fd.laddr.(*UDPAddr).isWildcard() { + if supportsIPv4map() && fd.laddr.(*UDPAddr).isWildcard() { if fd.family != syscall.AF_INET6 { return fmt.Errorf("ListenPacket(%s, %v) returns %v; want %v", fd.net, fd.laddr, fd.family, syscall.AF_INET6) } @@ -535,7 +535,7 @@ func TestIPv4MulticastListener(t *testing.T) { case "solaris": t.Skipf("not supported on solaris, see golang.org/issue/7399") } - if !supportsIPv4 { + if !supportsIPv4() { t.Skip("IPv4 is not supported") } @@ -610,7 +610,7 @@ func TestIPv6MulticastListener(t *testing.T) { case "solaris": t.Skipf("not supported on solaris, see issue 7399") } - if !supportsIPv6 { + if !supportsIPv6() { t.Skip("IPv6 is not supported") } if os.Getuid() != 0 { diff --git a/libgo/go/net/lookup.go b/libgo/go/net/lookup.go index cc2013e4325..c9f327050af 100644 --- a/libgo/go/net/lookup.go +++ b/libgo/go/net/lookup.go @@ -28,6 +28,9 @@ var protocols = map[string]int{ // services contains minimal mappings between services names and port // numbers for platforms that don't have a complete list of port numbers // (some Solaris distros, nacl, etc). +// +// See https://www.iana.org/assignments/service-names-port-numbers +// // On Unix, this map is augmented by readServices via goLookupPort. var services = map[string]map[string]int{ "udp": { @@ -63,7 +66,12 @@ func lookupProtocolMap(name string) (int, error) { return proto, nil } -const maxServiceLength = len("mobility-header") + 10 // with room to grow +// maxPortBufSize is the longest reasonable name of a service +// (non-numeric port). +// Currently the longest known IANA-unregistered name is +// "mobility-header", so we use that length, plus some slop in case +// something longer is added in the future. +const maxPortBufSize = len("mobility-header") + 10 func lookupPortMap(network, service string) (port int, error error) { switch network { @@ -74,7 +82,7 @@ func lookupPortMap(network, service string) (port int, error error) { } if m, ok := services[network]; ok { - var lowerService [maxServiceLength]byte + var lowerService [maxPortBufSize]byte n := copy(lowerService[:], service) lowerASCIIBytes(lowerService[:n]) if port, ok := m[string(lowerService[:n])]; ok && n == len(service) { @@ -97,6 +105,29 @@ type Resolver struct { // GODEBUG=netdns=go, but scoped to just this resolver. PreferGo bool + // StrictErrors controls the behavior of temporary errors + // (including timeout, socket errors, and SERVFAIL) when using + // Go's built-in resolver. For a query composed of multiple + // sub-queries (such as an A+AAAA address lookup, or walking the + // DNS search list), this option causes such errors to abort the + // whole query instead of returning a partial result. This is + // not enabled by default because it may affect compatibility + // with resolvers that process AAAA queries incorrectly. + StrictErrors bool + + // Dial optionally specifies an alternate dialer for use by + // Go's built-in DNS resolver to make TCP and UDP connections + // to DNS services. The host in the address parameter will + // always be a literal IP address and not a host name, and the + // port in the address parameter will be a literal port number + // and not a service name. + // If the Conn returned is also a PacketConn, sent and received DNS + // messages must adhere to RFC 1035 section 4.2.1, "UDP usage". + // Otherwise, DNS messages transmitted over Conn must adhere + // to RFC 7766 section 5, "Transport Protocol Selection". + // If nil, the default dialer is used. + Dial func(ctx context.Context, network, address string) (Conn, error) + // TODO(bradfitz): optional interface impl override hook // TODO(bradfitz): Timeout time.Duration? } @@ -164,12 +195,15 @@ func (r *Resolver) LookupIPAddr(ctx context.Context, host string) ([]IPAddr, err select { case <-ctx.Done(): - // The DNS lookup timed out for some reason. Force + // If the DNS lookup timed out for some reason, force // future requests to start the DNS lookup again // rather than waiting for the current lookup to // complete. See issue 8602. - err := mapErr(ctx.Err()) - lookupGroup.Forget(host) + ctxErr := ctx.Err() + if ctxErr == context.DeadlineExceeded { + lookupGroup.Forget(host) + } + err := mapErr(ctxErr) if trace != nil && trace.DNSDone != nil { trace.DNSDone(nil, false, err) } diff --git a/libgo/go/net/lookup_test.go b/libgo/go/net/lookup_test.go index 36db56acd03..68a7abe95df 100644 --- a/libgo/go/net/lookup_test.go +++ b/libgo/go/net/lookup_test.go @@ -63,7 +63,7 @@ func TestLookupGoogleSRV(t *testing.T) { testenv.MustHaveExternalNetwork(t) } - if !supportsIPv4 || !*testIPv4 { + if !supportsIPv4() || !*testIPv4 { t.Skip("IPv4 is required") } @@ -99,7 +99,7 @@ func TestLookupGmailMX(t *testing.T) { testenv.MustHaveExternalNetwork(t) } - if !supportsIPv4 || !*testIPv4 { + if !supportsIPv4() || !*testIPv4 { t.Skip("IPv4 is required") } @@ -131,7 +131,7 @@ func TestLookupGmailNS(t *testing.T) { testenv.MustHaveExternalNetwork(t) } - if !supportsIPv4 || !*testIPv4 { + if !supportsIPv4() || !*testIPv4 { t.Skip("IPv4 is required") } @@ -164,7 +164,7 @@ func TestLookupGmailTXT(t *testing.T) { testenv.MustHaveExternalNetwork(t) } - if !supportsIPv4 || !*testIPv4 { + if !supportsIPv4() || !*testIPv4 { t.Skip("IPv4 is required") } @@ -199,7 +199,7 @@ func TestLookupGooglePublicDNSAddr(t *testing.T) { testenv.MustHaveExternalNetwork(t) } - if !supportsIPv4 || !supportsIPv6 || !*testIPv4 || !*testIPv6 { + if !supportsIPv4() || !supportsIPv6() || !*testIPv4 || !*testIPv6 { t.Skip("both IPv4 and IPv6 are required") } @@ -220,7 +220,7 @@ func TestLookupGooglePublicDNSAddr(t *testing.T) { } func TestLookupIPv6LinkLocalAddr(t *testing.T) { - if !supportsIPv6 || !*testIPv6 { + if !supportsIPv6() || !*testIPv6 { t.Skip("IPv6 is required") } @@ -256,7 +256,7 @@ func TestLookupCNAME(t *testing.T) { testenv.MustHaveExternalNetwork(t) } - if !supportsIPv4 || !*testIPv4 { + if !supportsIPv4() || !*testIPv4 { t.Skip("IPv4 is required") } @@ -283,7 +283,7 @@ func TestLookupGoogleHost(t *testing.T) { testenv.MustHaveExternalNetwork(t) } - if !supportsIPv4 || !*testIPv4 { + if !supportsIPv4() || !*testIPv4 { t.Skip("IPv4 is required") } @@ -315,7 +315,7 @@ func TestLookupGoogleIP(t *testing.T) { testenv.MustHaveExternalNetwork(t) } - if !supportsIPv4 || !*testIPv4 { + if !supportsIPv4() || !*testIPv4 { t.Skip("IPv4 is required") } @@ -450,7 +450,7 @@ func TestDNSFlood(t *testing.T) { } func TestLookupDotsWithLocalSource(t *testing.T) { - if !supportsIPv4 || !*testIPv4 { + if !supportsIPv4() || !*testIPv4 { t.Skip("IPv4 is required") } @@ -499,7 +499,7 @@ func TestLookupDotsWithRemoteSource(t *testing.T) { testenv.MustHaveExternalNetwork(t) } - if !supportsIPv4 || !*testIPv4 { + if !supportsIPv4() || !*testIPv4 { t.Skip("IPv4 is required") } diff --git a/libgo/go/net/lookup_unix.go b/libgo/go/net/lookup_unix.go index f96c8beb614..2813f14be74 100644 --- a/libgo/go/net/lookup_unix.go +++ b/libgo/go/net/lookup_unix.go @@ -16,28 +16,31 @@ var onceReadProtocols sync.Once // readProtocols loads contents of /etc/protocols into protocols map // for quick access. func readProtocols() { - if file, err := open("/etc/protocols"); err == nil { - for line, ok := file.readLine(); ok; line, ok = file.readLine() { - // tcp 6 TCP # transmission control protocol - if i := byteIndex(line, '#'); i >= 0 { - line = line[0:i] - } - f := getFields(line) - if len(f) < 2 { - continue + file, err := open("/etc/protocols") + if err != nil { + return + } + defer file.close() + + for line, ok := file.readLine(); ok; line, ok = file.readLine() { + // tcp 6 TCP # transmission control protocol + if i := byteIndex(line, '#'); i >= 0 { + line = line[0:i] + } + f := getFields(line) + if len(f) < 2 { + continue + } + if proto, _, ok := dtoi(f[1]); ok { + if _, ok := protocols[f[0]]; !ok { + protocols[f[0]] = proto } - if proto, _, ok := dtoi(f[1]); ok { - if _, ok := protocols[f[0]]; !ok { - protocols[f[0]] = proto - } - for _, alias := range f[2:] { - if _, ok := protocols[alias]; !ok { - protocols[alias] = proto - } + for _, alias := range f[2:] { + if _, ok := protocols[alias]; !ok { + protocols[alias] = proto } } } - file.close() } } @@ -48,6 +51,29 @@ func lookupProtocol(_ context.Context, name string) (int, error) { return lookupProtocolMap(name) } +func (r *Resolver) dial(ctx context.Context, network, server string) (dnsConn, error) { + // Calling Dial here is scary -- we have to be sure not to + // dial a name that will require a DNS lookup, or Dial will + // call back here to translate it. The DNS config parser has + // already checked that all the cfg.servers are IP + // addresses, which Dial will use without a DNS lookup. + var c Conn + var err error + if r.Dial != nil { + c, err = r.Dial(ctx, network, server) + } else { + var d Dialer + c, err = d.DialContext(ctx, network, server) + } + if err != nil { + return nil, mapErr(err) + } + if _, ok := c.(PacketConn); ok { + return &dnsPacketConn{c}, nil + } + return &dnsStreamConn{c}, nil +} + func (r *Resolver) lookupHost(ctx context.Context, host string) (addrs []string, err error) { order := systemConf().hostLookupOrder(host) if !r.PreferGo && order == hostLookupCgo { @@ -57,12 +83,12 @@ func (r *Resolver) lookupHost(ctx context.Context, host string) (addrs []string, // cgo not available (or netgo); fall back to Go's DNS resolver order = hostLookupFilesDNS } - return goLookupHostOrder(ctx, host, order) + return r.goLookupHostOrder(ctx, host, order) } func (r *Resolver) lookupIP(ctx context.Context, host string) (addrs []IPAddr, err error) { if r.PreferGo { - return goLookupIP(ctx, host) + return r.goLookupIP(ctx, host) } order := systemConf().hostLookupOrder(host) if order == hostLookupCgo { @@ -72,7 +98,7 @@ func (r *Resolver) lookupIP(ctx context.Context, host string) (addrs []IPAddr, e // cgo not available (or netgo); fall back to Go's DNS resolver order = hostLookupFilesDNS } - addrs, _, err = goLookupIPCNAMEOrder(ctx, host, order) + addrs, _, err = r.goLookupIPCNAMEOrder(ctx, host, order) return } @@ -98,17 +124,17 @@ func (r *Resolver) lookupCNAME(ctx context.Context, name string) (string, error) return cname, err } } - return goLookupCNAME(ctx, name) + return r.goLookupCNAME(ctx, name) } -func (*Resolver) lookupSRV(ctx context.Context, service, proto, name string) (string, []*SRV, error) { +func (r *Resolver) lookupSRV(ctx context.Context, service, proto, name string) (string, []*SRV, error) { var target string if service == "" && proto == "" { target = name } else { target = "_" + service + "._" + proto + "." + name } - cname, rrs, err := lookup(ctx, target, dnsTypeSRV) + cname, rrs, err := r.lookup(ctx, target, dnsTypeSRV) if err != nil { return "", nil, err } @@ -121,8 +147,8 @@ func (*Resolver) lookupSRV(ctx context.Context, service, proto, name string) (st return cname, srvs, nil } -func (*Resolver) lookupMX(ctx context.Context, name string) ([]*MX, error) { - _, rrs, err := lookup(ctx, name, dnsTypeMX) +func (r *Resolver) lookupMX(ctx context.Context, name string) ([]*MX, error) { + _, rrs, err := r.lookup(ctx, name, dnsTypeMX) if err != nil { return nil, err } @@ -135,8 +161,8 @@ func (*Resolver) lookupMX(ctx context.Context, name string) ([]*MX, error) { return mxs, nil } -func (*Resolver) lookupNS(ctx context.Context, name string) ([]*NS, error) { - _, rrs, err := lookup(ctx, name, dnsTypeNS) +func (r *Resolver) lookupNS(ctx context.Context, name string) ([]*NS, error) { + _, rrs, err := r.lookup(ctx, name, dnsTypeNS) if err != nil { return nil, err } @@ -148,7 +174,7 @@ func (*Resolver) lookupNS(ctx context.Context, name string) ([]*NS, error) { } func (r *Resolver) lookupTXT(ctx context.Context, name string) ([]string, error) { - _, rrs, err := lookup(ctx, name, dnsTypeTXT) + _, rrs, err := r.lookup(ctx, name, dnsTypeTXT) if err != nil { return nil, err } @@ -165,5 +191,5 @@ func (r *Resolver) lookupAddr(ctx context.Context, addr string) ([]string, error return ptrs, err } } - return goLookupPTR(ctx, addr) + return r.goLookupPTR(ctx, addr) } diff --git a/libgo/go/net/lookup_windows.go b/libgo/go/net/lookup_windows.go index 5808293058a..0036d89d150 100644 --- a/libgo/go/net/lookup_windows.go +++ b/libgo/go/net/lookup_windows.go @@ -107,7 +107,7 @@ func (r *Resolver) lookupIP(ctx context.Context, name string) ([]IPAddr, error) addrs = append(addrs, IPAddr{IP: IPv4(a[0], a[1], a[2], a[3])}) case syscall.AF_INET6: a := (*syscall.RawSockaddrInet6)(addr).Addr - zone := zoneToString(int((*syscall.RawSockaddrInet6)(addr).Scope_id)) + zone := zoneCache.name(int((*syscall.RawSockaddrInet6)(addr).Scope_id)) addrs = append(addrs, IPAddr{IP: IP{a[0], a[1], a[2], a[3], a[4], a[5], a[6], a[7], a[8], a[9], a[10], a[11], a[12], a[13], a[14], a[15]}, Zone: zone}) default: ch <- ret{err: &DNSError{Err: syscall.EWINDOWS.Error(), Name: name}} diff --git a/libgo/go/net/mail/message.go b/libgo/go/net/mail/message.go index 702b765c349..45a995ec720 100644 --- a/libgo/go/net/mail/message.go +++ b/libgo/go/net/mail/message.go @@ -49,7 +49,7 @@ type Message struct { // ReadMessage reads a message from r. // The headers are parsed, and the body of the message will be available -// for reading from r. +// for reading from msg.Body. func ReadMessage(r io.Reader) (msg *Message, err error) { tp := textproto.NewReader(bufio.NewReader(r)) @@ -387,13 +387,15 @@ func (p *addrParser) consumePhrase() (phrase string, err error) { debug.Printf("consumePhrase: [%s]", p.s) // phrase = 1*word var words []string + var isPrevEncoded bool for { // word = atom / quoted-string var word string p.skipSpace() if p.empty() { - return "", errors.New("mail: missing phrase") + break } + isEncoded := false if p.peek() == '"' { // quoted-string word, err = p.consumeQuotedString() @@ -403,7 +405,7 @@ func (p *addrParser) consumePhrase() (phrase string, err error) { // than what RFC 5322 specifies. word, err = p.consumeAtom(true, true) if err == nil { - word, err = p.decodeRFC2047Word(word) + word, isEncoded, err = p.decodeRFC2047Word(word) } } @@ -411,7 +413,12 @@ func (p *addrParser) consumePhrase() (phrase string, err error) { break } debug.Printf("consumePhrase: consumed %q", word) - words = append(words, word) + if isPrevEncoded && isEncoded { + words[len(words)-1] += word + } else { + words = append(words, word) + } + isPrevEncoded = isEncoded } // Ignore any error if we got at least one word. if err != nil && len(words) == 0 { @@ -540,22 +547,23 @@ func (p *addrParser) len() int { return len(p.s) } -func (p *addrParser) decodeRFC2047Word(s string) (string, error) { +func (p *addrParser) decodeRFC2047Word(s string) (word string, isEncoded bool, err error) { if p.dec != nil { - return p.dec.DecodeHeader(s) + word, err = p.dec.Decode(s) + } else { + word, err = rfc2047Decoder.Decode(s) } - dec, err := rfc2047Decoder.Decode(s) if err == nil { - return dec, nil + return word, true, nil } if _, ok := err.(charsetError); ok { - return s, err + return s, true, err } // Ignore invalid RFC 2047 encoded-word errors. - return s, nil + return s, false, nil } var rfc2047Decoder = mime.WordDecoder{ diff --git a/libgo/go/net/mail/message_test.go b/libgo/go/net/mail/message_test.go index f0761ab09fb..2106a0b97d6 100644 --- a/libgo/go/net/mail/message_test.go +++ b/libgo/go/net/mail/message_test.go @@ -136,6 +136,7 @@ func TestAddressParsingError(t *testing.T) { 4: {"\"\\" + string([]byte{0x80}) + "\" ", "invalid utf-8 in quoted-string"}, 5: {"\"\x00\" ", "bad character in quoted-string"}, 6: {"\"\\\x00\" ", "bad character in quoted-string"}, + 7: {"John Doe", "no angle-addr"}, } for i, tc := range mustErrTestCases { @@ -235,6 +236,16 @@ func TestAddressParsing(t *testing.T) { }, }, }, + // RFC 2047 "Q"-encoded UTF-8 address with multiple encoded-words. + { + `=?utf-8?q?J=C3=B6rg?= =?utf-8?q?Doe?= `, + []*Address{ + { + Name: `JörgDoe`, + Address: "joerg@example.com", + }, + }, + }, // RFC 2047, Section 8. { `=?ISO-8859-1?Q?Andr=E9?= Pirard `, diff --git a/libgo/go/net/main_cloexec_test.go b/libgo/go/net/main_cloexec_test.go index 79038195859..fa1ed020572 100644 --- a/libgo/go/net/main_cloexec_test.go +++ b/libgo/go/net/main_cloexec_test.go @@ -2,10 +2,12 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -// +build freebsd linux +// +build dragonfly freebsd linux package net +import "internal/poll" + func init() { extraTestHookInstallers = append(extraTestHookInstallers, installAccept4TestHook) extraTestHookUninstallers = append(extraTestHookUninstallers, uninstallAccept4TestHook) @@ -13,13 +15,13 @@ func init() { var ( // Placeholders for saving original socket system calls. - origAccept4 = accept4Func + origAccept4 = poll.Accept4Func ) func installAccept4TestHook() { - accept4Func = sw.Accept4 + poll.Accept4Func = sw.Accept4 } func uninstallAccept4TestHook() { - accept4Func = origAccept4 + poll.Accept4Func = origAccept4 } diff --git a/libgo/go/net/main_test.go b/libgo/go/net/main_test.go index 28a8ff66d65..3e7a85ad2d9 100644 --- a/libgo/go/net/main_test.go +++ b/libgo/go/net/main_test.go @@ -70,7 +70,7 @@ var ( ) func setupTestData() { - if supportsIPv4 { + if supportsIPv4() { resolveTCPAddrTests = append(resolveTCPAddrTests, []resolveTCPAddrTest{ {"tcp", "localhost:1", &TCPAddr{IP: IPv4(127, 0, 0, 1), Port: 1}, nil}, {"tcp4", "localhost:2", &TCPAddr{IP: IPv4(127, 0, 0, 1), Port: 2}, nil}, @@ -85,25 +85,31 @@ func setupTestData() { }...) } - if supportsIPv6 { + if supportsIPv6() { resolveTCPAddrTests = append(resolveTCPAddrTests, resolveTCPAddrTest{"tcp6", "localhost:3", &TCPAddr{IP: IPv6loopback, Port: 3}, nil}) resolveUDPAddrTests = append(resolveUDPAddrTests, resolveUDPAddrTest{"udp6", "localhost:3", &UDPAddr{IP: IPv6loopback, Port: 3}, nil}) resolveIPAddrTests = append(resolveIPAddrTests, resolveIPAddrTest{"ip6", "localhost", &IPAddr{IP: IPv6loopback}, nil}) + + // Issue 20911: don't return IPv4 addresses for + // Resolve*Addr calls of the IPv6 unspecified address. + resolveTCPAddrTests = append(resolveTCPAddrTests, resolveTCPAddrTest{"tcp", "[::]:4", &TCPAddr{IP: IPv6unspecified, Port: 4}, nil}) + resolveUDPAddrTests = append(resolveUDPAddrTests, resolveUDPAddrTest{"udp", "[::]:4", &UDPAddr{IP: IPv6unspecified, Port: 4}, nil}) + resolveIPAddrTests = append(resolveIPAddrTests, resolveIPAddrTest{"ip", "::", &IPAddr{IP: IPv6unspecified}, nil}) } ifi := loopbackInterface() if ifi != nil { index := fmt.Sprintf("%v", ifi.Index) resolveTCPAddrTests = append(resolveTCPAddrTests, []resolveTCPAddrTest{ - {"tcp6", "[fe80::1%" + ifi.Name + "]:1", &TCPAddr{IP: ParseIP("fe80::1"), Port: 1, Zone: zoneToString(ifi.Index)}, nil}, + {"tcp6", "[fe80::1%" + ifi.Name + "]:1", &TCPAddr{IP: ParseIP("fe80::1"), Port: 1, Zone: zoneCache.name(ifi.Index)}, nil}, {"tcp6", "[fe80::1%" + index + "]:2", &TCPAddr{IP: ParseIP("fe80::1"), Port: 2, Zone: index}, nil}, }...) resolveUDPAddrTests = append(resolveUDPAddrTests, []resolveUDPAddrTest{ - {"udp6", "[fe80::1%" + ifi.Name + "]:1", &UDPAddr{IP: ParseIP("fe80::1"), Port: 1, Zone: zoneToString(ifi.Index)}, nil}, + {"udp6", "[fe80::1%" + ifi.Name + "]:1", &UDPAddr{IP: ParseIP("fe80::1"), Port: 1, Zone: zoneCache.name(ifi.Index)}, nil}, {"udp6", "[fe80::1%" + index + "]:2", &UDPAddr{IP: ParseIP("fe80::1"), Port: 2, Zone: index}, nil}, }...) resolveIPAddrTests = append(resolveIPAddrTests, []resolveIPAddrTest{ - {"ip6", "fe80::1%" + ifi.Name, &IPAddr{IP: ParseIP("fe80::1"), Zone: zoneToString(ifi.Index)}, nil}, + {"ip6", "fe80::1%" + ifi.Name, &IPAddr{IP: ParseIP("fe80::1"), Zone: zoneCache.name(ifi.Index)}, nil}, {"ip6", "fe80::1%" + index, &IPAddr{IP: ParseIP("fe80::1"), Zone: index}, nil}, }...) } diff --git a/libgo/go/net/main_unix_test.go b/libgo/go/net/main_unix_test.go index 8c8f94479de..34a8a104e82 100644 --- a/libgo/go/net/main_unix_test.go +++ b/libgo/go/net/main_unix_test.go @@ -6,13 +6,15 @@ package net +import "internal/poll" + var ( // Placeholders for saving original socket system calls. origSocket = socketFunc - origClose = closeFunc + origClose = poll.CloseFunc origConnect = connectFunc origListen = listenFunc - origAccept = acceptFunc + origAccept = poll.AcceptFunc origGetsockoptInt = getsockoptIntFunc extraTestHookInstallers []func() @@ -21,10 +23,10 @@ var ( func installTestHooks() { socketFunc = sw.Socket - closeFunc = sw.Close + poll.CloseFunc = sw.Close connectFunc = sw.Connect listenFunc = sw.Listen - acceptFunc = sw.Accept + poll.AcceptFunc = sw.Accept getsockoptIntFunc = sw.GetsockoptInt for _, fn := range extraTestHookInstallers { @@ -34,10 +36,10 @@ func installTestHooks() { func uninstallTestHooks() { socketFunc = origSocket - closeFunc = origClose + poll.CloseFunc = origClose connectFunc = origConnect listenFunc = origListen - acceptFunc = origAccept + poll.AcceptFunc = origAccept getsockoptIntFunc = origGetsockoptInt for _, fn := range extraTestHookUninstallers { @@ -48,6 +50,6 @@ func uninstallTestHooks() { // forceCloseSockets must be called only from TestMain. func forceCloseSockets() { for s := range sw.Sockets() { - closeFunc(s) + poll.CloseFunc(s) } } diff --git a/libgo/go/net/main_windows_test.go b/libgo/go/net/main_windows_test.go index 6ea318c2a5f..f38a3a0d668 100644 --- a/libgo/go/net/main_windows_test.go +++ b/libgo/go/net/main_windows_test.go @@ -4,37 +4,39 @@ package net +import "internal/poll" + var ( // Placeholders for saving original socket system calls. origSocket = socketFunc - origClosesocket = closeFunc + origClosesocket = poll.CloseFunc origConnect = connectFunc - origConnectEx = connectExFunc + origConnectEx = poll.ConnectExFunc origListen = listenFunc - origAccept = acceptFunc + origAccept = poll.AcceptFunc ) func installTestHooks() { socketFunc = sw.Socket - closeFunc = sw.Closesocket + poll.CloseFunc = sw.Closesocket connectFunc = sw.Connect - connectExFunc = sw.ConnectEx + poll.ConnectExFunc = sw.ConnectEx listenFunc = sw.Listen - acceptFunc = sw.AcceptEx + poll.AcceptFunc = sw.AcceptEx } func uninstallTestHooks() { socketFunc = origSocket - closeFunc = origClosesocket + poll.CloseFunc = origClosesocket connectFunc = origConnect - connectExFunc = origConnectEx + poll.ConnectExFunc = origConnectEx listenFunc = origListen - acceptFunc = origAccept + poll.AcceptFunc = origAccept } // forceCloseSockets must be called only from TestMain. func forceCloseSockets() { for s := range sw.Sockets() { - closeFunc(s) + poll.CloseFunc(s) } } diff --git a/libgo/go/net/mockserver_test.go b/libgo/go/net/mockserver_test.go index 766de6a815b..44581d90498 100644 --- a/libgo/go/net/mockserver_test.go +++ b/libgo/go/net/mockserver_test.go @@ -31,20 +31,20 @@ func testUnixAddr() string { func newLocalListener(network string) (Listener, error) { switch network { case "tcp": - if supportsIPv4 { + if supportsIPv4() { if ln, err := Listen("tcp4", "127.0.0.1:0"); err == nil { return ln, nil } } - if supportsIPv6 { + if supportsIPv6() { return Listen("tcp6", "[::1]:0") } case "tcp4": - if supportsIPv4 { + if supportsIPv4() { return Listen("tcp4", "127.0.0.1:0") } case "tcp6": - if supportsIPv6 { + if supportsIPv6() { return Listen("tcp6", "[::1]:0") } case "unix", "unixpacket": @@ -333,18 +333,18 @@ func timeoutTransmitter(c Conn, d, min, max time.Duration, ch chan<- error) { func newLocalPacketListener(network string) (PacketConn, error) { switch network { case "udp": - if supportsIPv4 { + if supportsIPv4() { return ListenPacket("udp4", "127.0.0.1:0") } - if supportsIPv6 { + if supportsIPv6() { return ListenPacket("udp6", "[::1]:0") } case "udp4": - if supportsIPv4 { + if supportsIPv4() { return ListenPacket("udp4", "127.0.0.1:0") } case "udp6": - if supportsIPv6 { + if supportsIPv6() { return ListenPacket("udp6", "[::1]:0") } case "unixgram": diff --git a/libgo/go/net/net.go b/libgo/go/net/net.go index a8b57361e6c..91ec048e0be 100644 --- a/libgo/go/net/net.go +++ b/libgo/go/net/net.go @@ -81,6 +81,7 @@ package net import ( "context" "errors" + "internal/poll" "io" "os" "syscall" @@ -95,12 +96,6 @@ var ( netCgo bool // set true in conf_netcgo.go for build tag "netcgo" ) -func init() { - sysInit() - supportsIPv4 = probeIPv4Stack() - supportsIPv6, supportsIPv4map = probeIPv6Stack() -} - // Addr represents a network end point address. // // The two methods Network and String conventionally return strings @@ -234,7 +229,7 @@ func (c *conn) SetDeadline(t time.Time) error { if !c.ok() { return syscall.EINVAL } - if err := c.fd.setDeadline(t); err != nil { + if err := c.fd.pfd.SetDeadline(t); err != nil { return &OpError{Op: "set", Net: c.fd.net, Source: nil, Addr: c.fd.laddr, Err: err} } return nil @@ -245,7 +240,7 @@ func (c *conn) SetReadDeadline(t time.Time) error { if !c.ok() { return syscall.EINVAL } - if err := c.fd.setReadDeadline(t); err != nil { + if err := c.fd.pfd.SetReadDeadline(t); err != nil { return &OpError{Op: "set", Net: c.fd.net, Source: nil, Addr: c.fd.laddr, Err: err} } return nil @@ -256,7 +251,7 @@ func (c *conn) SetWriteDeadline(t time.Time) error { if !c.ok() { return syscall.EINVAL } - if err := c.fd.setWriteDeadline(t); err != nil { + if err := c.fd.pfd.SetWriteDeadline(t); err != nil { return &OpError{Op: "set", Net: c.fd.net, Source: nil, Addr: c.fd.laddr, Err: err} } return nil @@ -391,10 +386,8 @@ var ( errMissingAddress = errors.New("missing address") // For both read and write operations. - errTimeout error = &timeoutError{} - errCanceled = errors.New("operation was canceled") - errClosing = errors.New("use of closed network connection") - ErrWriteToConnected = errors.New("use of WriteTo with pre-connected connection") + errCanceled = errors.New("operation was canceled") + ErrWriteToConnected = errors.New("use of WriteTo with pre-connected connection") ) // mapErr maps from the context errors to the historical internal net @@ -407,7 +400,7 @@ func mapErr(err error) error { case context.Canceled: return errCanceled case context.DeadlineExceeded: - return errTimeout + return poll.ErrTimeout default: return err } @@ -502,12 +495,6 @@ func (e *OpError) Temporary() bool { return ok && t.Temporary() } -type timeoutError struct{} - -func (e *timeoutError) Error() string { return "i/o timeout" } -func (e *timeoutError) Timeout() bool { return true } -func (e *timeoutError) Temporary() bool { return true } - // A ParseError is the error type of literal network address parsers. type ParseError struct { // Type is the type of string that was expected, such as @@ -632,8 +619,6 @@ type buffersWriter interface { writeBuffers(*Buffers) (int64, error) } -var testHookDidWritev = func(wrote int) {} - // Buffers contains zero or more runs of bytes to write. // // On certain machines, for certain types of connections, this is diff --git a/libgo/go/net/net_test.go b/libgo/go/net/net_test.go index 9a9a7e552c4..024505e7c62 100644 --- a/libgo/go/net/net_test.go +++ b/libgo/go/net/net_test.go @@ -54,7 +54,7 @@ func TestCloseRead(t *testing.T) { err = c.CloseRead() } if err != nil { - if perr := parseCloseError(err); perr != nil { + if perr := parseCloseError(err, true); perr != nil { t.Error(perr) } t.Fatal(err) @@ -94,7 +94,7 @@ func TestCloseWrite(t *testing.T) { err = c.CloseWrite() } if err != nil { - if perr := parseCloseError(err); perr != nil { + if perr := parseCloseError(err, true); perr != nil { t.Error(perr) } t.Error(err) @@ -139,7 +139,7 @@ func TestCloseWrite(t *testing.T) { err = c.CloseWrite() } if err != nil { - if perr := parseCloseError(err); perr != nil { + if perr := parseCloseError(err, true); perr != nil { t.Error(perr) } t.Fatal(err) @@ -184,7 +184,7 @@ func TestConnClose(t *testing.T) { defer c.Close() if err := c.Close(); err != nil { - if perr := parseCloseError(err); perr != nil { + if perr := parseCloseError(err, false); perr != nil { t.Error(perr) } t.Fatal(err) @@ -215,7 +215,7 @@ func TestListenerClose(t *testing.T) { dst := ln.Addr().String() if err := ln.Close(); err != nil { - if perr := parseCloseError(err); perr != nil { + if perr := parseCloseError(err, false); perr != nil { t.Error(perr) } t.Fatal(err) @@ -269,7 +269,7 @@ func TestPacketConnClose(t *testing.T) { defer c.Close() if err := c.Close(); err != nil { - if perr := parseCloseError(err); perr != nil { + if perr := parseCloseError(err, false); perr != nil { t.Error(perr) } t.Fatal(err) @@ -292,7 +292,7 @@ func TestListenCloseListen(t *testing.T) { } addr := ln.Addr().String() if err := ln.Close(); err != nil { - if perr := parseCloseError(err); perr != nil { + if perr := parseCloseError(err, false); perr != nil { t.Error(perr) } t.Fatal(err) diff --git a/libgo/go/net/platform_test.go b/libgo/go/net/platform_test.go index 2a14095cc28..5841ca35a00 100644 --- a/libgo/go/net/platform_test.go +++ b/libgo/go/net/platform_test.go @@ -50,11 +50,11 @@ func testableNetwork(network string) bool { } switch ss[0] { case "tcp4", "udp4", "ip4": - if !supportsIPv4 { + if !supportsIPv4() { return false } case "tcp6", "udp6", "ip6": - if !supportsIPv6 { + if !supportsIPv6() { return false } } @@ -117,25 +117,25 @@ func testableListenArgs(network, address, client string) bool { // Test functionality of IPv4 communication using AF_INET and // IPv6 communication using AF_INET6 sockets. - if !supportsIPv4 && ip.To4() != nil { + if !supportsIPv4() && ip.To4() != nil { return false } - if !supportsIPv6 && ip.To16() != nil && ip.To4() == nil { + if !supportsIPv6() && ip.To16() != nil && ip.To4() == nil { return false } cip := ParseIP(client) if cip != nil { - if !supportsIPv4 && cip.To4() != nil { + if !supportsIPv4() && cip.To4() != nil { return false } - if !supportsIPv6 && cip.To16() != nil && cip.To4() == nil { + if !supportsIPv6() && cip.To16() != nil && cip.To4() == nil { return false } } // Test functionality of IPv4 communication using AF_INET6 // sockets. - if !supportsIPv4map && supportsIPv4 && (network == "tcp" || network == "udp" || network == "ip") && wildcard { + if !supportsIPv4map() && supportsIPv4() && (network == "tcp" || network == "udp" || network == "ip") && wildcard { // At this point, we prefer IPv4 when ip is nil. // See favoriteAddrFamily for further information. if ip.To16() != nil && ip.To4() == nil && cip.To4() != nil { // a pair of IPv6 server and IPv4 client diff --git a/libgo/go/net/port_unix.go b/libgo/go/net/port_unix.go index 3120ba1c7e2..8dd1c32f95d 100644 --- a/libgo/go/net/port_unix.go +++ b/libgo/go/net/port_unix.go @@ -17,6 +17,8 @@ func readServices() { if err != nil { return } + defer file.close() + for line, ok := file.readLine(); ok; line, ok = file.readLine() { // "http 80/tcp www www-http # World Wide Web HTTP" if i := byteIndex(line, '#'); i >= 0 { @@ -43,7 +45,6 @@ func readServices() { } } } - file.close() } // goLookupPort is the native Go implementation of LookupPort. diff --git a/libgo/go/net/rawconn.go b/libgo/go/net/rawconn.go new file mode 100644 index 00000000000..d67be644a34 --- /dev/null +++ b/libgo/go/net/rawconn.go @@ -0,0 +1,62 @@ +// Copyright 2017 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package net + +import ( + "runtime" + "syscall" +) + +// BUG(mikio): On Windows, the Read and Write methods of +// syscall.RawConn are not implemented. + +// BUG(mikio): On NaCl and Plan 9, the Control, Read and Write methods +// of syscall.RawConn are not implemented. + +type rawConn struct { + fd *netFD +} + +func (c *rawConn) ok() bool { return c != nil && c.fd != nil } + +func (c *rawConn) Control(f func(uintptr)) error { + if !c.ok() { + return syscall.EINVAL + } + err := c.fd.pfd.RawControl(f) + runtime.KeepAlive(c.fd) + if err != nil { + err = &OpError{Op: "raw-control", Net: c.fd.net, Source: nil, Addr: c.fd.laddr, Err: err} + } + return err +} + +func (c *rawConn) Read(f func(uintptr) bool) error { + if !c.ok() { + return syscall.EINVAL + } + err := c.fd.pfd.RawRead(f) + runtime.KeepAlive(c.fd) + if err != nil { + err = &OpError{Op: "raw-read", Net: c.fd.net, Source: c.fd.laddr, Addr: c.fd.raddr, Err: err} + } + return err +} + +func (c *rawConn) Write(f func(uintptr) bool) error { + if !c.ok() { + return syscall.EINVAL + } + err := c.fd.pfd.RawWrite(f) + runtime.KeepAlive(c.fd) + if err != nil { + err = &OpError{Op: "raw-write", Net: c.fd.net, Source: c.fd.laddr, Addr: c.fd.raddr, Err: err} + } + return err +} + +func newRawConn(fd *netFD) (*rawConn, error) { + return &rawConn{fd: fd}, nil +} diff --git a/libgo/go/net/rawconn_unix_test.go b/libgo/go/net/rawconn_unix_test.go new file mode 100644 index 00000000000..294249ba5d1 --- /dev/null +++ b/libgo/go/net/rawconn_unix_test.go @@ -0,0 +1,94 @@ +// Copyright 2017 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build darwin dragonfly freebsd linux netbsd openbsd solaris + +package net + +import ( + "bytes" + "syscall" + "testing" +) + +func TestRawConn(t *testing.T) { + handler := func(ls *localServer, ln Listener) { + c, err := ln.Accept() + if err != nil { + t.Error(err) + return + } + defer c.Close() + var b [32]byte + n, err := c.Read(b[:]) + if err != nil { + t.Error(err) + return + } + if _, err := c.Write(b[:n]); err != nil { + t.Error(err) + return + } + } + ls, err := newLocalServer("tcp") + if err != nil { + t.Fatal(err) + } + defer ls.teardown() + if err := ls.buildup(handler); err != nil { + t.Fatal(err) + } + + c, err := Dial(ls.Listener.Addr().Network(), ls.Listener.Addr().String()) + if err != nil { + t.Fatal(err) + } + defer c.Close() + cc, err := c.(*TCPConn).SyscallConn() + if err != nil { + t.Fatal(err) + } + + var operr error + data := []byte("HELLO-R-U-THERE") + err = cc.Write(func(s uintptr) bool { + _, operr = syscall.Write(int(s), data) + if operr == syscall.EAGAIN { + return false + } + return true + }) + if err != nil || operr != nil { + t.Fatal(err, operr) + } + + var nr int + var b [32]byte + err = cc.Read(func(s uintptr) bool { + nr, operr = syscall.Read(int(s), b[:]) + if operr == syscall.EAGAIN { + return false + } + return true + }) + if err != nil || operr != nil { + t.Fatal(err, operr) + } + if bytes.Compare(b[:nr], data) != 0 { + t.Fatalf("got %#v; want %#v", b[:nr], data) + } + + fn := func(s uintptr) { + operr = syscall.SetsockoptInt(int(s), syscall.SOL_SOCKET, syscall.SO_REUSEADDR, 1) + } + err = cc.Control(fn) + if err != nil || operr != nil { + t.Fatal(err, operr) + } + c.Close() + err = cc.Control(fn) + if err == nil { + t.Fatal("should fail") + } +} diff --git a/libgo/go/net/rawconn_windows_test.go b/libgo/go/net/rawconn_windows_test.go new file mode 100644 index 00000000000..5fb6de75393 --- /dev/null +++ b/libgo/go/net/rawconn_windows_test.go @@ -0,0 +1,36 @@ +// Copyright 2017 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package net + +import ( + "syscall" + "testing" +) + +func TestRawConn(t *testing.T) { + c, err := newLocalPacketListener("udp") + if err != nil { + t.Fatal(err) + } + defer c.Close() + cc, err := c.(*UDPConn).SyscallConn() + if err != nil { + t.Fatal(err) + } + + var operr error + fn := func(s uintptr) { + operr = syscall.SetsockoptInt(syscall.Handle(s), syscall.SOL_SOCKET, syscall.SO_REUSEADDR, 1) + } + err = cc.Control(fn) + if err != nil || operr != nil { + t.Fatal(err, operr) + } + c.Close() + err = cc.Control(fn) + if err == nil { + t.Fatal("should fail") + } +} diff --git a/libgo/go/net/rpc/debug.go b/libgo/go/net/rpc/debug.go index 98b2c1c6c4a..a1d799ff19a 100644 --- a/libgo/go/net/rpc/debug.go +++ b/libgo/go/net/rpc/debug.go @@ -71,20 +71,17 @@ type debugHTTP struct { // Runs at /debug/rpc func (server debugHTTP) ServeHTTP(w http.ResponseWriter, req *http.Request) { // Build a sorted version of the data. - var services = make(serviceArray, len(server.serviceMap)) - i := 0 - server.mu.Lock() - for sname, service := range server.serviceMap { - services[i] = debugService{service, sname, make(methodArray, len(service.method))} - j := 0 - for mname, method := range service.method { - services[i].Method[j] = debugMethod{method, mname} - j++ + var services serviceArray + server.serviceMap.Range(func(snamei, svci interface{}) bool { + svc := svci.(*service) + ds := debugService{svc, snamei.(string), make(methodArray, 0, len(svc.method))} + for mname, method := range svc.method { + ds.Method = append(ds.Method, debugMethod{method, mname}) } - sort.Sort(services[i].Method) - i++ - } - server.mu.Unlock() + sort.Sort(ds.Method) + services = append(services, ds) + return true + }) sort.Sort(services) err := debug.Execute(w, services) if err != nil { diff --git a/libgo/go/net/rpc/jsonrpc/all_test.go b/libgo/go/net/rpc/jsonrpc/all_test.go index b811d3c0c7c..bbb8eb02918 100644 --- a/libgo/go/net/rpc/jsonrpc/all_test.go +++ b/libgo/go/net/rpc/jsonrpc/all_test.go @@ -13,6 +13,7 @@ import ( "io/ioutil" "net" "net/rpc" + "reflect" "strings" "testing" ) @@ -55,8 +56,26 @@ func (t *Arith) Error(args *Args, reply *Reply) error { panic("ERROR") } +type BuiltinTypes struct{} + +func (BuiltinTypes) Map(i int, reply *map[int]int) error { + (*reply)[i] = i + return nil +} + +func (BuiltinTypes) Slice(i int, reply *[]int) error { + *reply = append(*reply, i) + return nil +} + +func (BuiltinTypes) Array(i int, reply *[1]int) error { + (*reply)[0] = i + return nil +} + func init() { rpc.Register(new(Arith)) + rpc.Register(BuiltinTypes{}) } func TestServerNoParams(t *testing.T) { @@ -182,6 +201,45 @@ func TestClient(t *testing.T) { } } +func TestBuiltinTypes(t *testing.T) { + cli, srv := net.Pipe() + go ServeConn(srv) + + client := NewClient(cli) + defer client.Close() + + // Map + arg := 7 + replyMap := map[int]int{} + err := client.Call("BuiltinTypes.Map", arg, &replyMap) + if err != nil { + t.Errorf("Map: expected no error but got string %q", err.Error()) + } + if replyMap[arg] != arg { + t.Errorf("Map: expected %d got %d", arg, replyMap[arg]) + } + + // Slice + replySlice := []int{} + err = client.Call("BuiltinTypes.Slice", arg, &replySlice) + if err != nil { + t.Errorf("Slice: expected no error but got string %q", err.Error()) + } + if e := []int{arg}; !reflect.DeepEqual(replySlice, e) { + t.Errorf("Slice: expected %v got %v", e, replySlice) + } + + // Array + replyArray := [1]int{} + err = client.Call("BuiltinTypes.Array", arg, &replyArray) + if err != nil { + t.Errorf("Array: expected no error but got string %q", err.Error()) + } + if e := [1]int{arg}; !reflect.DeepEqual(replyArray, e) { + t.Errorf("Array: expected %v got %v", e, replyArray) + } +} + func TestMalformedInput(t *testing.T) { cli, srv := net.Pipe() go cli.Write([]byte(`{id:1}`)) // invalid json diff --git a/libgo/go/net/rpc/jsonrpc/client.go b/libgo/go/net/rpc/jsonrpc/client.go index da1b8165fc7..e6359bed598 100644 --- a/libgo/go/net/rpc/jsonrpc/client.go +++ b/libgo/go/net/rpc/jsonrpc/client.go @@ -2,8 +2,9 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -// Package jsonrpc implements a JSON-RPC ClientCodec and ServerCodec +// Package jsonrpc implements a JSON-RPC 1.0 ClientCodec and ServerCodec // for the rpc package. +// For JSON-RPC 2.0 support, see https://godoc.org/?q=json-rpc+2.0 package jsonrpc import ( diff --git a/libgo/go/net/rpc/server.go b/libgo/go/net/rpc/server.go index 18ea629b0d6..29aae7ee7ff 100644 --- a/libgo/go/net/rpc/server.go +++ b/libgo/go/net/rpc/server.go @@ -187,8 +187,7 @@ type Response struct { // Server represents an RPC Server. type Server struct { - mu sync.RWMutex // protects the serviceMap - serviceMap map[string]*service + serviceMap sync.Map // map[string]*service reqLock sync.Mutex // protects freeReq freeReq *Request respLock sync.Mutex // protects freeResp @@ -197,7 +196,7 @@ type Server struct { // NewServer returns a new Server. func NewServer() *Server { - return &Server{serviceMap: make(map[string]*service)} + return &Server{} } // DefaultServer is the default instance of *Server. @@ -240,11 +239,6 @@ func (server *Server) RegisterName(name string, rcvr interface{}) error { } func (server *Server) register(rcvr interface{}, name string, useName bool) error { - server.mu.Lock() - defer server.mu.Unlock() - if server.serviceMap == nil { - server.serviceMap = make(map[string]*service) - } s := new(service) s.typ = reflect.TypeOf(rcvr) s.rcvr = reflect.ValueOf(rcvr) @@ -262,9 +256,6 @@ func (server *Server) register(rcvr interface{}, name string, useName bool) erro log.Print(s) return errors.New(s) } - if _, present := server.serviceMap[sname]; present { - return errors.New("rpc: service already defined: " + sname) - } s.name = sname // Install the methods @@ -283,7 +274,10 @@ func (server *Server) register(rcvr interface{}, name string, useName bool) erro log.Print(str) return errors.New(str) } - server.serviceMap[s.name] = s + + if _, dup := server.serviceMap.LoadOrStore(sname, s); dup { + return errors.New("rpc: service already defined: " + sname) + } return nil } @@ -571,10 +565,17 @@ func (server *Server) readRequest(codec ServerCodec) (service *service, mtype *m } replyv = reflect.New(mtype.ReplyType.Elem()) + + switch mtype.ReplyType.Elem().Kind() { + case reflect.Map: + replyv.Elem().Set(reflect.MakeMap(mtype.ReplyType.Elem())) + case reflect.Slice: + replyv.Elem().Set(reflect.MakeSlice(mtype.ReplyType.Elem(), 0, 0)) + } return } -func (server *Server) readRequestHeader(codec ServerCodec) (service *service, mtype *methodType, req *Request, keepReading bool, err error) { +func (server *Server) readRequestHeader(codec ServerCodec) (svc *service, mtype *methodType, req *Request, keepReading bool, err error) { // Grab the request header. req = server.getRequest() err = codec.ReadRequestHeader(req) @@ -600,14 +601,13 @@ func (server *Server) readRequestHeader(codec ServerCodec) (service *service, mt methodName := req.ServiceMethod[dot+1:] // Look up the request. - server.mu.RLock() - service = server.serviceMap[serviceName] - server.mu.RUnlock() - if service == nil { + svci, ok := server.serviceMap.Load(serviceName) + if !ok { err = errors.New("rpc: can't find service " + req.ServiceMethod) return } - mtype = service.method[methodName] + svc = svci.(*service) + mtype = svc.method[methodName] if mtype == nil { err = errors.New("rpc: can't find method " + req.ServiceMethod) } diff --git a/libgo/go/net/rpc/server_test.go b/libgo/go/net/rpc/server_test.go index 8369c9dec7b..fb97f82a2f7 100644 --- a/libgo/go/net/rpc/server_test.go +++ b/libgo/go/net/rpc/server_test.go @@ -11,6 +11,7 @@ import ( "log" "net" "net/http/httptest" + "reflect" "runtime" "strings" "sync" @@ -85,6 +86,24 @@ type Embed struct { hidden } +type BuiltinTypes struct{} + +func (BuiltinTypes) Map(args *Args, reply *map[int]int) error { + (*reply)[args.A] = args.B + return nil +} + +func (BuiltinTypes) Slice(args *Args, reply *[]int) error { + *reply = append(*reply, args.A, args.B) + return nil +} + +func (BuiltinTypes) Array(args *Args, reply *[2]int) error { + (*reply)[0] = args.A + (*reply)[1] = args.B + return nil +} + func listenTCP() (net.Listener, string) { l, e := net.Listen("tcp", "127.0.0.1:0") // any available address if e != nil { @@ -97,6 +116,7 @@ func startServer() { Register(new(Arith)) Register(new(Embed)) RegisterName("net.rpc.Arith", new(Arith)) + Register(BuiltinTypes{}) var l net.Listener l, serverAddr = listenTCP() @@ -326,6 +346,49 @@ func testHTTPRPC(t *testing.T, path string) { } } +func TestBuiltinTypes(t *testing.T) { + once.Do(startServer) + + client, err := DialHTTP("tcp", httpServerAddr) + if err != nil { + t.Fatal("dialing", err) + } + defer client.Close() + + // Map + args := &Args{7, 8} + replyMap := map[int]int{} + err = client.Call("BuiltinTypes.Map", args, &replyMap) + if err != nil { + t.Errorf("Map: expected no error but got string %q", err.Error()) + } + if replyMap[args.A] != args.B { + t.Errorf("Map: expected %d got %d", args.B, replyMap[args.A]) + } + + // Slice + args = &Args{7, 8} + replySlice := []int{} + err = client.Call("BuiltinTypes.Slice", args, &replySlice) + if err != nil { + t.Errorf("Slice: expected no error but got string %q", err.Error()) + } + if e := []int{args.A, args.B}; !reflect.DeepEqual(replySlice, e) { + t.Errorf("Slice: expected %v got %v", e, replySlice) + } + + // Array + args = &Args{7, 8} + replyArray := [2]int{} + err = client.Call("BuiltinTypes.Array", args, &replyArray) + if err != nil { + t.Errorf("Array: expected no error but got string %q", err.Error()) + } + if e := [2]int{args.A, args.B}; !reflect.DeepEqual(replyArray, e) { + t.Errorf("Array: expected %v got %v", e, replyArray) + } +} + // CodecEmulator provides a client-like api and a ServerCodec interface. // Can be used to test ServeRequest. type CodecEmulator struct { @@ -619,13 +682,13 @@ func TestErrorAfterClientClose(t *testing.T) { // Tests the fix to issue 11221. Without the fix, this loops forever or crashes. func TestAcceptExitAfterListenerClose(t *testing.T) { - newServer = NewServer() + newServer := NewServer() newServer.Register(new(Arith)) newServer.RegisterName("net.rpc.Arith", new(Arith)) newServer.RegisterName("newServer.Arith", new(Arith)) var l net.Listener - l, newServerAddr = listenTCP() + l, _ = listenTCP() l.Close() newServer.Accept(l) } diff --git a/libgo/go/net/sendfile_bsd.go b/libgo/go/net/sendfile_bsd.go new file mode 100644 index 00000000000..7a2b48c6cfa --- /dev/null +++ b/libgo/go/net/sendfile_bsd.go @@ -0,0 +1,67 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build dragonfly freebsd + +package net + +import ( + "internal/poll" + "io" + "os" +) + +// sendFile copies the contents of r to c using the sendfile +// system call to minimize copies. +// +// if handled == true, sendFile returns the number of bytes copied and any +// non-EOF error. +// +// if handled == false, sendFile performed no work. +func sendFile(c *netFD, r io.Reader) (written int64, err error, handled bool) { + // FreeBSD and DragonFly use 0 as the "until EOF" value. + // If you pass in more bytes than the file contains, it will + // loop back to the beginning ad nauseam until it's sent + // exactly the number of bytes told to. As such, we need to + // know exactly how many bytes to send. + var remain int64 = 0 + + lr, ok := r.(*io.LimitedReader) + if ok { + remain, r = lr.N, lr.R + if remain <= 0 { + return 0, nil, true + } + } + f, ok := r.(*os.File) + if !ok { + return 0, nil, false + } + + if remain == 0 { + fi, err := f.Stat() + if err != nil { + return 0, err, false + } + + remain = fi.Size() + } + + // The other quirk with FreeBSD/DragonFly's sendfile + // implementation is that it doesn't use the current position + // of the file -- if you pass it offset 0, it starts from + // offset 0. There's no way to tell it "start from current + // position", so we have to manage that explicitly. + pos, err := f.Seek(0, io.SeekCurrent) + if err != nil { + return 0, err, false + } + + written, err = poll.SendFile(&c.pfd, int(f.Fd()), pos, remain) + + if lr != nil { + lr.N = remain - written + } + return written, wrapSyscallError("sendfile", err), written > 0 +} diff --git a/libgo/go/net/sendfile_dragonfly.go b/libgo/go/net/sendfile_dragonfly.go deleted file mode 100644 index d4b825c3705..00000000000 --- a/libgo/go/net/sendfile_dragonfly.go +++ /dev/null @@ -1,106 +0,0 @@ -// Copyright 2011 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package net - -import ( - "io" - "os" - "syscall" -) - -// maxSendfileSize is the largest chunk size we ask the kernel to copy -// at a time. -const maxSendfileSize int = 4 << 20 - -// sendFile copies the contents of r to c using the sendfile -// system call to minimize copies. -// -// if handled == true, sendFile returns the number of bytes copied and any -// non-EOF error. -// -// if handled == false, sendFile performed no work. -func sendFile(c *netFD, r io.Reader) (written int64, err error, handled bool) { - // DragonFly uses 0 as the "until EOF" value. If you pass in more bytes than the - // file contains, it will loop back to the beginning ad nauseam until it's sent - // exactly the number of bytes told to. As such, we need to know exactly how many - // bytes to send. - var remain int64 = 0 - - lr, ok := r.(*io.LimitedReader) - if ok { - remain, r = lr.N, lr.R - if remain <= 0 { - return 0, nil, true - } - } - f, ok := r.(*os.File) - if !ok { - return 0, nil, false - } - - if remain == 0 { - fi, err := f.Stat() - if err != nil { - return 0, err, false - } - - remain = fi.Size() - } - - // The other quirk with DragonFly's sendfile implementation is that it doesn't - // use the current position of the file -- if you pass it offset 0, it starts - // from offset 0. There's no way to tell it "start from current position", so - // we have to manage that explicitly. - pos, err := f.Seek(0, io.SeekCurrent) - if err != nil { - return 0, err, false - } - - if err := c.writeLock(); err != nil { - return 0, err, true - } - defer c.writeUnlock() - - dst := c.sysfd - src := int(f.Fd()) - for remain > 0 { - n := maxSendfileSize - if int64(n) > remain { - n = int(remain) - } - pos1 := pos - n, err1 := syscall.Sendfile(dst, src, &pos1, n) - if n > 0 { - pos += int64(n) - written += int64(n) - remain -= int64(n) - } - if n == 0 && err1 == nil { - break - } - if err1 == syscall.EAGAIN { - if err1 = c.pd.waitWrite(); err1 == nil { - continue - } - } - if err1 == syscall.EINTR { - continue - } - if err1 != nil { - // This includes syscall.ENOSYS (no kernel - // support) and syscall.EINVAL (fd types which - // don't implement sendfile) - err = err1 - break - } - } - if lr != nil { - lr.N = remain - } - if err != nil { - err = os.NewSyscallError("sendfile", err) - } - return written, err, written > 0 -} diff --git a/libgo/go/net/sendfile_freebsd.go b/libgo/go/net/sendfile_freebsd.go deleted file mode 100644 index 18cbb27b533..00000000000 --- a/libgo/go/net/sendfile_freebsd.go +++ /dev/null @@ -1,106 +0,0 @@ -// Copyright 2011 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package net - -import ( - "io" - "os" - "syscall" -) - -// maxSendfileSize is the largest chunk size we ask the kernel to copy -// at a time. -const maxSendfileSize int = 4 << 20 - -// sendFile copies the contents of r to c using the sendfile -// system call to minimize copies. -// -// if handled == true, sendFile returns the number of bytes copied and any -// non-EOF error. -// -// if handled == false, sendFile performed no work. -func sendFile(c *netFD, r io.Reader) (written int64, err error, handled bool) { - // FreeBSD uses 0 as the "until EOF" value. If you pass in more bytes than the - // file contains, it will loop back to the beginning ad nauseam until it's sent - // exactly the number of bytes told to. As such, we need to know exactly how many - // bytes to send. - var remain int64 = 0 - - lr, ok := r.(*io.LimitedReader) - if ok { - remain, r = lr.N, lr.R - if remain <= 0 { - return 0, nil, true - } - } - f, ok := r.(*os.File) - if !ok { - return 0, nil, false - } - - if remain == 0 { - fi, err := f.Stat() - if err != nil { - return 0, err, false - } - - remain = fi.Size() - } - - // The other quirk with FreeBSD's sendfile implementation is that it doesn't - // use the current position of the file -- if you pass it offset 0, it starts - // from offset 0. There's no way to tell it "start from current position", so - // we have to manage that explicitly. - pos, err := f.Seek(0, io.SeekCurrent) - if err != nil { - return 0, err, false - } - - if err := c.writeLock(); err != nil { - return 0, err, true - } - defer c.writeUnlock() - - dst := c.sysfd - src := int(f.Fd()) - for remain > 0 { - n := maxSendfileSize - if int64(n) > remain { - n = int(remain) - } - pos1 := pos - n, err1 := syscall.Sendfile(dst, src, &pos1, n) - if n > 0 { - pos += int64(n) - written += int64(n) - remain -= int64(n) - } - if n == 0 && err1 == nil { - break - } - if err1 == syscall.EAGAIN { - if err1 = c.pd.waitWrite(); err1 == nil { - continue - } - } - if err1 == syscall.EINTR { - continue - } - if err1 != nil { - // This includes syscall.ENOSYS (no kernel - // support) and syscall.EINVAL (fd types which - // don't implement sendfile) - err = err1 - break - } - } - if lr != nil { - lr.N = remain - } - if err != nil { - err = os.NewSyscallError("sendfile", err) - } - return written, err, written > 0 -} diff --git a/libgo/go/net/sendfile_linux.go b/libgo/go/net/sendfile_linux.go index 7e741f97941..c537ea68b2b 100644 --- a/libgo/go/net/sendfile_linux.go +++ b/libgo/go/net/sendfile_linux.go @@ -5,15 +5,11 @@ package net import ( + "internal/poll" "io" "os" - "syscall" ) -// maxSendfileSize is the largest chunk size we ask the kernel to copy -// at a time. -const maxSendfileSize int = 4 << 20 - // sendFile copies the contents of r to c using the sendfile // system call to minimize copies. // @@ -36,44 +32,10 @@ func sendFile(c *netFD, r io.Reader) (written int64, err error, handled bool) { return 0, nil, false } - if err := c.writeLock(); err != nil { - return 0, err, true - } - defer c.writeUnlock() + written, err = poll.SendFile(&c.pfd, int(f.Fd()), remain) - dst := c.sysfd - src := int(f.Fd()) - for remain > 0 { - n := maxSendfileSize - if int64(n) > remain { - n = int(remain) - } - n, err1 := syscall.Sendfile(dst, src, nil, n) - if n > 0 { - written += int64(n) - remain -= int64(n) - } - if n == 0 && err1 == nil { - break - } - if err1 == syscall.EAGAIN { - if err1 = c.pd.waitWrite(); err1 == nil { - continue - } - } - if err1 != nil { - // This includes syscall.ENOSYS (no kernel - // support) and syscall.EINVAL (fd types which - // don't implement sendfile) - err = err1 - break - } - } if lr != nil { - lr.N = remain - } - if err != nil { - err = os.NewSyscallError("sendfile", err) + lr.N = remain - written } - return written, err, written > 0 + return written, wrapSyscallError("sendfile", err), written > 0 } diff --git a/libgo/go/net/sendfile_solaris.go b/libgo/go/net/sendfile_solaris.go index add70c3147e..63ca9d47b8a 100644 --- a/libgo/go/net/sendfile_solaris.go +++ b/libgo/go/net/sendfile_solaris.go @@ -5,19 +5,11 @@ package net import ( + "internal/poll" "io" "os" - "syscall" ) -// Not strictly needed, but very helpful for debugging, see issue #10221. -//go:cgo_import_dynamic _ _ "libsendfile.so" -//go:cgo_import_dynamic _ _ "libsocket.so" - -// maxSendfileSize is the largest chunk size we ask the kernel to copy -// at a time. -const maxSendfileSize int = 4 << 20 - // sendFile copies the contents of r to c using the sendfile // system call to minimize copies. // @@ -62,56 +54,10 @@ func sendFile(c *netFD, r io.Reader) (written int64, err error, handled bool) { return 0, err, false } - if err := c.writeLock(); err != nil { - return 0, err, true - } - defer c.writeUnlock() + written, err = poll.SendFile(&c.pfd, int(f.Fd()), pos, remain) - dst := c.sysfd - src := int(f.Fd()) - for remain > 0 { - n := maxSendfileSize - if int64(n) > remain { - n = int(remain) - } - pos1 := pos - n, err1 := syscall.Sendfile(dst, src, &pos1, n) - if err1 == syscall.EAGAIN || err1 == syscall.EINTR { - // partial write may have occurred - if n = int(pos1 - pos); n == 0 { - // nothing more to write - err1 = nil - } - } - if n > 0 { - pos += int64(n) - written += int64(n) - remain -= int64(n) - } - if n == 0 && err1 == nil { - break - } - if err1 == syscall.EAGAIN { - if err1 = c.pd.waitWrite(); err1 == nil { - continue - } - } - if err1 == syscall.EINTR { - continue - } - if err1 != nil { - // This includes syscall.ENOSYS (no kernel - // support) and syscall.EINVAL (fd types which - // don't implement sendfile) - err = err1 - break - } - } if lr != nil { - lr.N = remain - } - if err != nil { - err = os.NewSyscallError("sendfile", err) + lr.N = remain - written } - return written, err, written > 0 + return written, wrapSyscallError("sendfile", err), written > 0 } diff --git a/libgo/go/net/sendfile_windows.go b/libgo/go/net/sendfile_windows.go index bc0b7fb5b25..bccd8b149f7 100644 --- a/libgo/go/net/sendfile_windows.go +++ b/libgo/go/net/sendfile_windows.go @@ -5,6 +5,7 @@ package net import ( + "internal/poll" "io" "os" "syscall" @@ -34,19 +35,10 @@ func sendFile(fd *netFD, r io.Reader) (written int64, err error, handled bool) { return 0, nil, false } - if err := fd.writeLock(); err != nil { - return 0, err, true - } - defer fd.writeUnlock() + done, err := poll.SendFile(&fd.pfd, syscall.Handle(f.Fd()), n) - o := &fd.wop - o.qty = uint32(n) - o.handle = syscall.Handle(f.Fd()) - done, err := wsrv.ExecIO(o, "TransmitFile", func(o *operation) error { - return syscall.TransmitFile(o.fd.sysfd, o.handle, o.qty, 0, &o.o, nil, syscall.TF_WRITE_BEHIND) - }) if err != nil { - return 0, os.NewSyscallError("transmitfile", err), false + return 0, wrapSyscallError("transmitfile", err), false } if lr != nil { lr.N -= int64(done) diff --git a/libgo/go/net/smtp/smtp.go b/libgo/go/net/smtp/smtp.go index a408fa53363..28472e447b5 100644 --- a/libgo/go/net/smtp/smtp.go +++ b/libgo/go/net/smtp/smtp.go @@ -298,7 +298,7 @@ var testHookStartTLS func(*tls.Config) // nil, except for tests // messages is accomplished by including an email address in the to // parameter but not including it in the msg headers. // -// The SendMail function and the the net/smtp package are low-level +// The SendMail function and the net/smtp package are low-level // mechanisms and provide no support for DKIM signing, MIME // attachments (see the mime/multipart package), or other mail // functionality. Higher-level packages exist outside of the standard diff --git a/libgo/go/net/smtp/smtp_test.go b/libgo/go/net/smtp/smtp_test.go index c48fae6d5ac..9dbe3eb9ecb 100644 --- a/libgo/go/net/smtp/smtp_test.go +++ b/libgo/go/net/smtp/smtp_test.go @@ -9,9 +9,11 @@ import ( "bytes" "crypto/tls" "crypto/x509" + "internal/testenv" "io" "net" "net/textproto" + "runtime" "strings" "testing" "time" @@ -592,6 +594,9 @@ QUIT ` func TestTLSClient(t *testing.T) { + if runtime.GOOS == "freebsd" && runtime.GOARCH == "amd64" { + testenv.SkipFlaky(t, 19229) + } ln := newLocalListener(t) defer ln.Close() errc := make(chan error) diff --git a/libgo/go/net/sock_cloexec.go b/libgo/go/net/sock_cloexec.go index 616a101eacb..06ff10d834a 100644 --- a/libgo/go/net/sock_cloexec.go +++ b/libgo/go/net/sock_cloexec.go @@ -5,11 +5,12 @@ // This file implements sysSocket and accept for platforms that // provide a fast path for setting SetNonblock and CloseOnExec. -// +build freebsd linux +// +build dragonfly freebsd linux package net import ( + "internal/poll" "os" "syscall" ) @@ -42,46 +43,8 @@ func sysSocket(family, sotype, proto int) (int, error) { return -1, os.NewSyscallError("socket", err) } if err = syscall.SetNonblock(s, true); err != nil { - closeFunc(s) + poll.CloseFunc(s) return -1, os.NewSyscallError("setnonblock", err) } return s, nil } - -// Wrapper around the accept system call that marks the returned file -// descriptor as nonblocking and close-on-exec. -func accept(s int) (int, syscall.Sockaddr, error) { - ns, sa, err := accept4Func(s, syscall.SOCK_NONBLOCK|syscall.SOCK_CLOEXEC) - // On Linux the accept4 system call was introduced in 2.6.28 - // kernel and on FreeBSD it was introduced in 10 kernel. If we - // get an ENOSYS error on both Linux and FreeBSD, or EINVAL - // error on Linux, fall back to using accept. - switch err { - case nil: - return ns, sa, nil - default: // errors other than the ones listed - return -1, sa, os.NewSyscallError("accept4", err) - case syscall.ENOSYS: // syscall missing - case syscall.EINVAL: // some Linux use this instead of ENOSYS - case syscall.EACCES: // some Linux use this instead of ENOSYS - case syscall.EFAULT: // some Linux use this instead of ENOSYS - } - - // See ../syscall/exec_unix.go for description of ForkLock. - // It is probably okay to hold the lock across syscall.Accept - // because we have put fd.sysfd into non-blocking mode. - // However, a call to the File method will put it back into - // blocking mode. We can't take that risk, so no use of ForkLock here. - ns, sa, err = acceptFunc(s) - if err == nil { - syscall.CloseOnExec(ns) - } - if err != nil { - return -1, nil, os.NewSyscallError("accept", err) - } - if err = syscall.SetNonblock(ns, true); err != nil { - closeFunc(ns) - return -1, nil, os.NewSyscallError("setnonblock", err) - } - return ns, sa, nil -} diff --git a/libgo/go/net/sock_posix.go b/libgo/go/net/sock_posix.go index 6bbfd1208ee..a30efe2336a 100644 --- a/libgo/go/net/sock_posix.go +++ b/libgo/go/net/sock_posix.go @@ -8,6 +8,7 @@ package net import ( "context" + "internal/poll" "os" "syscall" ) @@ -43,11 +44,11 @@ func socket(ctx context.Context, net string, family, sotype, proto int, ipv6only return nil, err } if err = setDefaultSockopts(s, family, sotype, ipv6only); err != nil { - closeFunc(s) + poll.CloseFunc(s) return nil, err } if fd, err = newFD(s, family, sotype, net); err != nil { - closeFunc(s) + poll.CloseFunc(s) return nil, err } @@ -127,17 +128,18 @@ func (fd *netFD) dial(ctx context.Context, laddr, raddr sockaddr) error { if lsa, err = laddr.sockaddr(fd.family); err != nil { return err } else if lsa != nil { - if err := syscall.Bind(fd.sysfd, lsa); err != nil { + if err := syscall.Bind(fd.pfd.Sysfd, lsa); err != nil { return os.NewSyscallError("bind", err) } } } - var rsa syscall.Sockaddr + var rsa syscall.Sockaddr // remote address from the user + var crsa syscall.Sockaddr // remote address we actually connected to if raddr != nil { if rsa, err = raddr.sockaddr(fd.family); err != nil { return err } - if err := fd.connect(ctx, lsa, rsa); err != nil { + if crsa, err = fd.connect(ctx, lsa, rsa); err != nil { return err } fd.isConnected = true @@ -146,8 +148,16 @@ func (fd *netFD) dial(ctx context.Context, laddr, raddr sockaddr) error { return err } } - lsa, _ = syscall.Getsockname(fd.sysfd) - if rsa, _ = syscall.Getpeername(fd.sysfd); rsa != nil { + // Record the local and remote addresses from the actual socket. + // Get the local address by calling Getsockname. + // For the remote address, use + // 1) the one returned by the connect method, if any; or + // 2) the one from Getpeername, if it succeeds; or + // 3) the one passed to us as the raddr parameter. + lsa, _ = syscall.Getsockname(fd.pfd.Sysfd) + if crsa != nil { + fd.setAddr(fd.addrFunc()(lsa), fd.addrFunc()(crsa)) + } else if rsa, _ = syscall.Getpeername(fd.pfd.Sysfd); rsa != nil { fd.setAddr(fd.addrFunc()(lsa), fd.addrFunc()(rsa)) } else { fd.setAddr(fd.addrFunc()(lsa), raddr) @@ -156,23 +166,23 @@ func (fd *netFD) dial(ctx context.Context, laddr, raddr sockaddr) error { } func (fd *netFD) listenStream(laddr sockaddr, backlog int) error { - if err := setDefaultListenerSockopts(fd.sysfd); err != nil { + if err := setDefaultListenerSockopts(fd.pfd.Sysfd); err != nil { return err } if lsa, err := laddr.sockaddr(fd.family); err != nil { return err } else if lsa != nil { - if err := syscall.Bind(fd.sysfd, lsa); err != nil { + if err := syscall.Bind(fd.pfd.Sysfd, lsa); err != nil { return os.NewSyscallError("bind", err) } } - if err := listenFunc(fd.sysfd, backlog); err != nil { + if err := listenFunc(fd.pfd.Sysfd, backlog); err != nil { return os.NewSyscallError("listen", err) } if err := fd.init(); err != nil { return err } - lsa, _ := syscall.Getsockname(fd.sysfd) + lsa, _ := syscall.Getsockname(fd.pfd.Sysfd) fd.setAddr(fd.addrFunc()(lsa), nil) return nil } @@ -188,7 +198,7 @@ func (fd *netFD) listenDatagram(laddr sockaddr) error { // multiple UDP listeners that listen on the same UDP // port to join the same group address. if addr.IP != nil && addr.IP.IsMulticast() { - if err := setDefaultMulticastSockopts(fd.sysfd); err != nil { + if err := setDefaultMulticastSockopts(fd.pfd.Sysfd); err != nil { return err } addr := *addr @@ -204,14 +214,14 @@ func (fd *netFD) listenDatagram(laddr sockaddr) error { if lsa, err := laddr.sockaddr(fd.family); err != nil { return err } else if lsa != nil { - if err := syscall.Bind(fd.sysfd, lsa); err != nil { + if err := syscall.Bind(fd.pfd.Sysfd, lsa); err != nil { return os.NewSyscallError("bind", err) } } if err := fd.init(); err != nil { return err } - lsa, _ := syscall.Getsockname(fd.sysfd) + lsa, _ := syscall.Getsockname(fd.pfd.Sysfd) fd.setAddr(fd.addrFunc()(lsa), nil) return nil } diff --git a/libgo/go/net/sockopt_bsd.go b/libgo/go/net/sockopt_bsd.go index 734a1093d68..1aae88a3e8d 100644 --- a/libgo/go/net/sockopt_bsd.go +++ b/libgo/go/net/sockopt_bsd.go @@ -25,7 +25,7 @@ func setDefaultSockopts(s, family, sotype int, ipv6only bool) error { syscall.SetsockoptInt(s, syscall.IPPROTO_IPV6, syscall.IPV6_PORTRANGE, syscall.IPV6_PORTRANGE_HIGH) } } - if supportsIPv4map && family == syscall.AF_INET6 && sotype != syscall.SOCK_RAW { + if supportsIPv4map() && family == syscall.AF_INET6 && sotype != syscall.SOCK_RAW { // Allow both IP versions even if the OS default // is otherwise. Note that some operating systems // never admit this option. diff --git a/libgo/go/net/sockopt_posix.go b/libgo/go/net/sockopt_posix.go index cacd04889d9..29edddbf374 100644 --- a/libgo/go/net/sockopt_posix.go +++ b/libgo/go/net/sockopt_posix.go @@ -7,7 +7,7 @@ package net import ( - "os" + "runtime" "syscall" ) @@ -101,27 +101,21 @@ done: } func setReadBuffer(fd *netFD, bytes int) error { - if err := fd.incref(); err != nil { - return err - } - defer fd.decref() - return os.NewSyscallError("setsockopt", syscall.SetsockoptInt(fd.sysfd, syscall.SOL_SOCKET, syscall.SO_RCVBUF, bytes)) + err := fd.pfd.SetsockoptInt(syscall.SOL_SOCKET, syscall.SO_RCVBUF, bytes) + runtime.KeepAlive(fd) + return wrapSyscallError("setsockopt", err) } func setWriteBuffer(fd *netFD, bytes int) error { - if err := fd.incref(); err != nil { - return err - } - defer fd.decref() - return os.NewSyscallError("setsockopt", syscall.SetsockoptInt(fd.sysfd, syscall.SOL_SOCKET, syscall.SO_SNDBUF, bytes)) + err := fd.pfd.SetsockoptInt(syscall.SOL_SOCKET, syscall.SO_SNDBUF, bytes) + runtime.KeepAlive(fd) + return wrapSyscallError("setsockopt", err) } func setKeepAlive(fd *netFD, keepalive bool) error { - if err := fd.incref(); err != nil { - return err - } - defer fd.decref() - return os.NewSyscallError("setsockopt", syscall.SetsockoptInt(fd.sysfd, syscall.SOL_SOCKET, syscall.SO_KEEPALIVE, boolint(keepalive))) + err := fd.pfd.SetsockoptInt(syscall.SOL_SOCKET, syscall.SO_KEEPALIVE, boolint(keepalive)) + runtime.KeepAlive(fd) + return wrapSyscallError("setsockopt", err) } func setLinger(fd *netFD, sec int) error { @@ -133,9 +127,7 @@ func setLinger(fd *netFD, sec int) error { l.Onoff = 0 l.Linger = 0 } - if err := fd.incref(); err != nil { - return err - } - defer fd.decref() - return os.NewSyscallError("setsockopt", syscall.SetsockoptLinger(fd.sysfd, syscall.SOL_SOCKET, syscall.SO_LINGER, &l)) + err := fd.pfd.SetsockoptLinger(syscall.SOL_SOCKET, syscall.SO_LINGER, &l) + runtime.KeepAlive(fd) + return wrapSyscallError("setsockopt", err) } diff --git a/libgo/go/net/sockoptip_bsd.go b/libgo/go/net/sockoptip_bsd.go index b15c6396ba1..b11f3a4edbe 100644 --- a/libgo/go/net/sockoptip_bsd.go +++ b/libgo/go/net/sockoptip_bsd.go @@ -7,28 +7,24 @@ package net import ( - "os" + "runtime" "syscall" ) func setIPv4MulticastInterface(fd *netFD, ifi *Interface) error { ip, err := interfaceToIPv4Addr(ifi) if err != nil { - return os.NewSyscallError("setsockopt", err) + return wrapSyscallError("setsockopt", err) } var a [4]byte copy(a[:], ip.To4()) - if err := fd.incref(); err != nil { - return err - } - defer fd.decref() - return os.NewSyscallError("setsockopt", syscall.SetsockoptInet4Addr(fd.sysfd, syscall.IPPROTO_IP, syscall.IP_MULTICAST_IF, a)) + err = fd.pfd.SetsockoptInet4Addr(syscall.IPPROTO_IP, syscall.IP_MULTICAST_IF, a) + runtime.KeepAlive(fd) + return wrapSyscallError("setsockopt", err) } func setIPv4MulticastLoopback(fd *netFD, v bool) error { - if err := fd.incref(); err != nil { - return err - } - defer fd.decref() - return os.NewSyscallError("setsockopt", syscall.SetsockoptByte(fd.sysfd, syscall.IPPROTO_IP, syscall.IP_MULTICAST_LOOP, byte(boolint(v)))) + err := fd.pfd.SetsockoptByte(syscall.IPPROTO_IP, syscall.IP_MULTICAST_LOOP, byte(boolint(v))) + runtime.KeepAlive(fd) + return wrapSyscallError("setsockopt", err) } diff --git a/libgo/go/net/sockoptip_linux.go b/libgo/go/net/sockoptip_linux.go index c1dcc911c73..bd7d8344258 100644 --- a/libgo/go/net/sockoptip_linux.go +++ b/libgo/go/net/sockoptip_linux.go @@ -5,7 +5,7 @@ package net import ( - "os" + "runtime" "syscall" ) @@ -15,17 +15,13 @@ func setIPv4MulticastInterface(fd *netFD, ifi *Interface) error { v = int32(ifi.Index) } mreq := &syscall.IPMreqn{Ifindex: v} - if err := fd.incref(); err != nil { - return err - } - defer fd.decref() - return os.NewSyscallError("setsockopt", syscall.SetsockoptIPMreqn(fd.sysfd, syscall.IPPROTO_IP, syscall.IP_MULTICAST_IF, mreq)) + err := fd.pfd.SetsockoptIPMreqn(syscall.IPPROTO_IP, syscall.IP_MULTICAST_IF, mreq) + runtime.KeepAlive(fd) + return wrapSyscallError("setsockopt", err) } func setIPv4MulticastLoopback(fd *netFD, v bool) error { - if err := fd.incref(); err != nil { - return err - } - defer fd.decref() - return os.NewSyscallError("setsockopt", syscall.SetsockoptInt(fd.sysfd, syscall.IPPROTO_IP, syscall.IP_MULTICAST_LOOP, boolint(v))) + err := fd.pfd.SetsockoptInt(syscall.IPPROTO_IP, syscall.IP_MULTICAST_LOOP, boolint(v)) + runtime.KeepAlive(fd) + return wrapSyscallError("setsockopt", err) } diff --git a/libgo/go/net/sockoptip_posix.go b/libgo/go/net/sockoptip_posix.go index 4afd4c8ea3f..92af7646ef9 100644 --- a/libgo/go/net/sockoptip_posix.go +++ b/libgo/go/net/sockoptip_posix.go @@ -7,7 +7,7 @@ package net import ( - "os" + "runtime" "syscall" ) @@ -16,11 +16,9 @@ func joinIPv4Group(fd *netFD, ifi *Interface, ip IP) error { if err := setIPv4MreqToInterface(mreq, ifi); err != nil { return err } - if err := fd.incref(); err != nil { - return err - } - defer fd.decref() - return os.NewSyscallError("setsockopt", syscall.SetsockoptIPMreq(fd.sysfd, syscall.IPPROTO_IP, syscall.IP_ADD_MEMBERSHIP, mreq)) + err := fd.pfd.SetsockoptIPMreq(syscall.IPPROTO_IP, syscall.IP_ADD_MEMBERSHIP, mreq) + runtime.KeepAlive(fd) + return wrapSyscallError("setsockopt", err) } func setIPv6MulticastInterface(fd *netFD, ifi *Interface) error { @@ -28,19 +26,15 @@ func setIPv6MulticastInterface(fd *netFD, ifi *Interface) error { if ifi != nil { v = ifi.Index } - if err := fd.incref(); err != nil { - return err - } - defer fd.decref() - return os.NewSyscallError("setsockopt", syscall.SetsockoptInt(fd.sysfd, syscall.IPPROTO_IPV6, syscall.IPV6_MULTICAST_IF, v)) + err := fd.pfd.SetsockoptInt(syscall.IPPROTO_IPV6, syscall.IPV6_MULTICAST_IF, v) + runtime.KeepAlive(fd) + return wrapSyscallError("setsockopt", err) } func setIPv6MulticastLoopback(fd *netFD, v bool) error { - if err := fd.incref(); err != nil { - return err - } - defer fd.decref() - return os.NewSyscallError("setsockopt", syscall.SetsockoptInt(fd.sysfd, syscall.IPPROTO_IPV6, syscall.IPV6_MULTICAST_LOOP, boolint(v))) + err := fd.pfd.SetsockoptInt(syscall.IPPROTO_IPV6, syscall.IPV6_MULTICAST_LOOP, boolint(v)) + runtime.KeepAlive(fd) + return wrapSyscallError("setsockopt", err) } func joinIPv6Group(fd *netFD, ifi *Interface, ip IP) error { @@ -49,9 +43,7 @@ func joinIPv6Group(fd *netFD, ifi *Interface, ip IP) error { if ifi != nil { mreq.Interface = uint32(ifi.Index) } - if err := fd.incref(); err != nil { - return err - } - defer fd.decref() - return os.NewSyscallError("setsockopt", syscall.SetsockoptIPv6Mreq(fd.sysfd, syscall.IPPROTO_IPV6, syscall.IPV6_JOIN_GROUP, mreq)) + err := fd.pfd.SetsockoptIPv6Mreq(syscall.IPPROTO_IPV6, syscall.IPV6_JOIN_GROUP, mreq) + runtime.KeepAlive(fd) + return wrapSyscallError("setsockopt", err) } diff --git a/libgo/go/net/sockoptip_windows.go b/libgo/go/net/sockoptip_windows.go index 916debebc6f..62676039a3b 100644 --- a/libgo/go/net/sockoptip_windows.go +++ b/libgo/go/net/sockoptip_windows.go @@ -6,6 +6,7 @@ package net import ( "os" + "runtime" "syscall" "unsafe" ) @@ -17,17 +18,13 @@ func setIPv4MulticastInterface(fd *netFD, ifi *Interface) error { } var a [4]byte copy(a[:], ip.To4()) - if err := fd.incref(); err != nil { - return err - } - defer fd.decref() - return os.NewSyscallError("setsockopt", syscall.Setsockopt(fd.sysfd, syscall.IPPROTO_IP, syscall.IP_MULTICAST_IF, (*byte)(unsafe.Pointer(&a[0])), 4)) + err = fd.pfd.Setsockopt(syscall.IPPROTO_IP, syscall.IP_MULTICAST_IF, (*byte)(unsafe.Pointer(&a[0])), 4) + runtime.KeepAlive(fd) + return wrapSyscallError("setsockopt", err) } func setIPv4MulticastLoopback(fd *netFD, v bool) error { - if err := fd.incref(); err != nil { - return err - } - defer fd.decref() - return os.NewSyscallError("setsockopt", syscall.SetsockoptInt(fd.sysfd, syscall.IPPROTO_IP, syscall.IP_MULTICAST_LOOP, boolint(v))) + err := fd.pfd.SetsockoptInt(syscall.IPPROTO_IP, syscall.IP_MULTICAST_LOOP, boolint(v)) + runtime.KeepAlive(fd) + return wrapSyscallError("setsockopt", err) } diff --git a/libgo/go/net/sys_cloexec.go b/libgo/go/net/sys_cloexec.go index f2ea8425493..def05cb5a7d 100644 --- a/libgo/go/net/sys_cloexec.go +++ b/libgo/go/net/sys_cloexec.go @@ -5,11 +5,12 @@ // This file implements sysSocket and accept for platforms that do not // provide a fast path for setting SetNonblock and CloseOnExec. -// +build aix darwin dragonfly nacl netbsd openbsd solaris +// +build aix darwin nacl netbsd openbsd solaris package net import ( + "internal/poll" "os" "syscall" ) @@ -28,30 +29,8 @@ func sysSocket(family, sotype, proto int) (int, error) { return -1, os.NewSyscallError("socket", err) } if err = syscall.SetNonblock(s, true); err != nil { - closeFunc(s) + poll.CloseFunc(s) return -1, os.NewSyscallError("setnonblock", err) } return s, nil } - -// Wrapper around the accept system call that marks the returned file -// descriptor as nonblocking and close-on-exec. -func accept(s int) (int, syscall.Sockaddr, error) { - // See ../syscall/exec_unix.go for description of ForkLock. - // It is probably okay to hold the lock across syscall.Accept - // because we have put fd.sysfd into non-blocking mode. - // However, a call to the File method will put it back into - // blocking mode. We can't take that risk, so no use of ForkLock here. - ns, sa, err := acceptFunc(s) - if err == nil { - syscall.CloseOnExec(ns) - } - if err != nil { - return -1, nil, os.NewSyscallError("accept", err) - } - if err = syscall.SetNonblock(ns, true); err != nil { - closeFunc(ns) - return -1, nil, os.NewSyscallError("setnonblock", err) - } - return ns, sa, nil -} diff --git a/libgo/go/net/tcpsock.go b/libgo/go/net/tcpsock.go index 69731ebc914..e957aa3005a 100644 --- a/libgo/go/net/tcpsock.go +++ b/libgo/go/net/tcpsock.go @@ -50,28 +50,34 @@ func (a *TCPAddr) opAddr() Addr { return a } -// ResolveTCPAddr parses addr as a TCP address of the form "host:port" -// or "[ipv6-host%zone]:port" and resolves a pair of domain name and -// port name on the network net, which must be "tcp", "tcp4" or -// "tcp6". A literal address or host name for IPv6 must be enclosed -// in square brackets, as in "[::1]:80", "[ipv6-host]:http" or -// "[ipv6-host%zone]:80". +// ResolveTCPAddr returns an address of TCP end point. // -// Resolving a hostname is not recommended because this returns at most -// one of its IP addresses. -func ResolveTCPAddr(net, addr string) (*TCPAddr, error) { - switch net { +// The network must be a TCP network name. +// +// If the host in the address parameter is not a literal IP address or +// the port is not a literal port number, ResolveTCPAddr resolves the +// address to an address of TCP end point. +// Otherwise, it parses the address as a pair of literal IP address +// and port number. +// The address parameter can use a host name, but this is not +// recommended, because it will return at most one of the host name's +// IP addresses. +// +// See func Dial for a description of the network and address +// parameters. +func ResolveTCPAddr(network, address string) (*TCPAddr, error) { + switch network { case "tcp", "tcp4", "tcp6": case "": // a hint wildcard for Go 1.0 undocumented behavior - net = "tcp" + network = "tcp" default: - return nil, UnknownNetworkError(net) + return nil, UnknownNetworkError(network) } - addrs, err := DefaultResolver.internetAddrList(context.Background(), net, addr) + addrs, err := DefaultResolver.internetAddrList(context.Background(), network, address) if err != nil { return nil, err } - return addrs.first(isIPv4).(*TCPAddr), nil + return addrs.forResolve(network, address).(*TCPAddr), nil } // TCPConn is an implementation of the Conn interface for TCP network @@ -80,6 +86,15 @@ type TCPConn struct { conn } +// SyscallConn returns a raw network connection. +// This implements the syscall.Conn interface. +func (c *TCPConn) SyscallConn() (syscall.RawConn, error) { + if !c.ok() { + return nil, syscall.EINVAL + } + return newRawConn(c.fd) +} + // ReadFrom implements the io.ReaderFrom ReadFrom method. func (c *TCPConn) ReadFrom(r io.Reader) (int64, error) { if !c.ok() { @@ -181,21 +196,25 @@ func newTCPConn(fd *netFD) *TCPConn { return c } -// DialTCP connects to the remote address raddr on the network net, -// which must be "tcp", "tcp4", or "tcp6". If laddr is not nil, it is -// used as the local address for the connection. -func DialTCP(net string, laddr, raddr *TCPAddr) (*TCPConn, error) { - switch net { +// DialTCP acts like Dial for TCP networks. +// +// The network must be a TCP network name; see func Dial for details. +// +// If laddr is nil, a local address is automatically chosen. +// If the IP field of raddr is nil or an unspecified IP address, the +// local system is assumed. +func DialTCP(network string, laddr, raddr *TCPAddr) (*TCPConn, error) { + switch network { case "tcp", "tcp4", "tcp6": default: - return nil, &OpError{Op: "dial", Net: net, Source: laddr.opAddr(), Addr: raddr.opAddr(), Err: UnknownNetworkError(net)} + return nil, &OpError{Op: "dial", Net: network, Source: laddr.opAddr(), Addr: raddr.opAddr(), Err: UnknownNetworkError(network)} } if raddr == nil { - return nil, &OpError{Op: "dial", Net: net, Source: laddr.opAddr(), Addr: nil, Err: errMissingAddress} + return nil, &OpError{Op: "dial", Net: network, Source: laddr.opAddr(), Addr: nil, Err: errMissingAddress} } - c, err := dialTCP(context.Background(), net, laddr, raddr) + c, err := dialTCP(context.Background(), network, laddr, raddr) if err != nil { - return nil, &OpError{Op: "dial", Net: net, Source: laddr.opAddr(), Addr: raddr.opAddr(), Err: err} + return nil, &OpError{Op: "dial", Net: network, Source: laddr.opAddr(), Addr: raddr.opAddr(), Err: err} } return c, nil } @@ -255,7 +274,7 @@ func (l *TCPListener) SetDeadline(t time.Time) error { if !l.ok() { return syscall.EINVAL } - if err := l.fd.setDeadline(t); err != nil { + if err := l.fd.pfd.SetDeadline(t); err != nil { return &OpError{Op: "set", Net: l.fd.net, Source: nil, Addr: l.fd.laddr, Err: err} } return nil @@ -279,22 +298,27 @@ func (l *TCPListener) File() (f *os.File, err error) { return } -// ListenTCP announces on the TCP address laddr and returns a TCP -// listener. Net must be "tcp", "tcp4", or "tcp6". If laddr has a -// port of 0, ListenTCP will choose an available port. The caller can -// use the Addr method of TCPListener to retrieve the chosen address. -func ListenTCP(net string, laddr *TCPAddr) (*TCPListener, error) { - switch net { +// ListenTCP acts like Listen for TCP networks. +// +// The network must be a TCP network name; see func Dial for details. +// +// If the IP field of laddr is nil or an unspecified IP address, +// ListenTCP listens on all available unicast and anycast IP addresses +// of the local system. +// If the Port field of laddr is 0, a port number is automatically +// chosen. +func ListenTCP(network string, laddr *TCPAddr) (*TCPListener, error) { + switch network { case "tcp", "tcp4", "tcp6": default: - return nil, &OpError{Op: "listen", Net: net, Source: nil, Addr: laddr.opAddr(), Err: UnknownNetworkError(net)} + return nil, &OpError{Op: "listen", Net: network, Source: nil, Addr: laddr.opAddr(), Err: UnknownNetworkError(network)} } if laddr == nil { laddr = &TCPAddr{} } - ln, err := listenTCP(context.Background(), net, laddr) + ln, err := listenTCP(context.Background(), network, laddr) if err != nil { - return nil, &OpError{Op: "listen", Net: net, Source: nil, Addr: laddr.opAddr(), Err: err} + return nil, &OpError{Op: "listen", Net: network, Source: nil, Addr: laddr.opAddr(), Err: err} } return ln, nil } diff --git a/libgo/go/net/tcpsock_plan9.go b/libgo/go/net/tcpsock_plan9.go index d2860607f8b..e37f0657c07 100644 --- a/libgo/go/net/tcpsock_plan9.go +++ b/libgo/go/net/tcpsock_plan9.go @@ -48,6 +48,9 @@ func (ln *TCPListener) accept() (*TCPConn, error) { } func (ln *TCPListener) close() error { + if err := ln.fd.pfd.Close(); err != nil { + return err + } if _, err := ln.fd.ctl.WriteString("hangup"); err != nil { ln.fd.ctl.Close() return err diff --git a/libgo/go/net/tcpsock_posix.go b/libgo/go/net/tcpsock_posix.go index 7533c24d122..9ba199dfb02 100644 --- a/libgo/go/net/tcpsock_posix.go +++ b/libgo/go/net/tcpsock_posix.go @@ -18,7 +18,7 @@ func sockaddrToTCP(sa syscall.Sockaddr) Addr { case *syscall.SockaddrInet4: return &TCPAddr{IP: sa.Addr[0:], Port: sa.Port} case *syscall.SockaddrInet6: - return &TCPAddr{IP: sa.Addr[0:], Port: sa.Port, Zone: zoneToString(int(sa.ZoneId))} + return &TCPAddr{IP: sa.Addr[0:], Port: sa.Port, Zone: zoneCache.name(int(sa.ZoneId))} } return nil } diff --git a/libgo/go/net/tcpsock_test.go b/libgo/go/net/tcpsock_test.go index 51154221d0a..660f4249d40 100644 --- a/libgo/go/net/tcpsock_test.go +++ b/libgo/go/net/tcpsock_test.go @@ -32,28 +32,28 @@ func BenchmarkTCP4PersistentTimeout(b *testing.B) { } func BenchmarkTCP6OneShot(b *testing.B) { - if !supportsIPv6 { + if !supportsIPv6() { b.Skip("ipv6 is not supported") } benchmarkTCP(b, false, false, "[::1]:0") } func BenchmarkTCP6OneShotTimeout(b *testing.B) { - if !supportsIPv6 { + if !supportsIPv6() { b.Skip("ipv6 is not supported") } benchmarkTCP(b, false, true, "[::1]:0") } func BenchmarkTCP6Persistent(b *testing.B) { - if !supportsIPv6 { + if !supportsIPv6() { b.Skip("ipv6 is not supported") } benchmarkTCP(b, true, false, "[::1]:0") } func BenchmarkTCP6PersistentTimeout(b *testing.B) { - if !supportsIPv6 { + if !supportsIPv6() { b.Skip("ipv6 is not supported") } benchmarkTCP(b, true, true, "[::1]:0") @@ -163,7 +163,7 @@ func BenchmarkTCP4ConcurrentReadWrite(b *testing.B) { } func BenchmarkTCP6ConcurrentReadWrite(b *testing.B) { - if !supportsIPv6 { + if !supportsIPv6() { b.Skip("ipv6 is not supported") } benchmarkTCPConcurrentReadWrite(b, "[::1]:0") @@ -372,7 +372,7 @@ func TestTCPListenerName(t *testing.T) { func TestIPv6LinkLocalUnicastTCP(t *testing.T) { testenv.MustHaveExternalNetwork(t) - if !supportsIPv6 { + if !supportsIPv6() { t.Skip("IPv6 is not supported") } diff --git a/libgo/go/net/tcpsock_unix_test.go b/libgo/go/net/tcpsock_unix_test.go index 2375fe24dc4..3af1834455b 100644 --- a/libgo/go/net/tcpsock_unix_test.go +++ b/libgo/go/net/tcpsock_unix_test.go @@ -2,11 +2,14 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -// +build darwin +// +build !plan9,!windows package net import ( + "context" + "internal/testenv" + "math/rand" "runtime" "sync" "syscall" @@ -77,3 +80,37 @@ func TestTCPSpuriousConnSetupCompletion(t *testing.T) { ln.Close() wg.Wait() } + +// Issue 19289. +// Test that a canceled Dial does not cause a subsequent Dial to succeed. +func TestTCPSpuriousConnSetupCompletionWithCancel(t *testing.T) { + if testenv.Builder() == "" { + testenv.MustHaveExternalNetwork(t) + } + t.Parallel() + const tries = 10000 + var wg sync.WaitGroup + wg.Add(tries * 2) + sem := make(chan bool, 5) + for i := 0; i < tries; i++ { + sem <- true + ctx, cancel := context.WithCancel(context.Background()) + go func() { + defer wg.Done() + time.Sleep(time.Duration(rand.Int63n(int64(5 * time.Millisecond)))) + cancel() + }() + go func(i int) { + defer wg.Done() + var dialer Dialer + // Try to connect to a real host on a port + // that it is not listening on. + _, err := dialer.DialContext(ctx, "tcp", "golang.org:3") + if err == nil { + t.Errorf("Dial to unbound port succeeded on attempt %d", i) + } + <-sem + }(i) + } + wg.Wait() +} diff --git a/libgo/go/net/tcpsockopt_darwin.go b/libgo/go/net/tcpsockopt_darwin.go index 0d1310eaf9e..7415c763c50 100644 --- a/libgo/go/net/tcpsockopt_darwin.go +++ b/libgo/go/net/tcpsockopt_darwin.go @@ -5,7 +5,7 @@ package net import ( - "os" + "runtime" "syscall" "time" ) @@ -13,17 +13,15 @@ import ( const sysTCP_KEEPINTVL = 0x101 func setKeepAlivePeriod(fd *netFD, d time.Duration) error { - if err := fd.incref(); err != nil { - return err - } - defer fd.decref() // The kernel expects seconds so round to next highest second. d += (time.Second - time.Nanosecond) secs := int(d.Seconds()) - switch err := syscall.SetsockoptInt(fd.sysfd, syscall.IPPROTO_TCP, sysTCP_KEEPINTVL, secs); err { + switch err := fd.pfd.SetsockoptInt(syscall.IPPROTO_TCP, sysTCP_KEEPINTVL, secs); err { case nil, syscall.ENOPROTOOPT: // OS X 10.7 and earlier don't support this option default: - return os.NewSyscallError("setsockopt", err) + return wrapSyscallError("setsockopt", err) } - return os.NewSyscallError("setsockopt", syscall.SetsockoptInt(fd.sysfd, syscall.IPPROTO_TCP, syscall.TCP_KEEPALIVE, secs)) + err := fd.pfd.SetsockoptInt(syscall.IPPROTO_TCP, syscall.TCP_KEEPALIVE, secs) + runtime.KeepAlive(fd) + return wrapSyscallError("setsockopt", err) } diff --git a/libgo/go/net/tcpsockopt_dragonfly.go b/libgo/go/net/tcpsockopt_dragonfly.go index 7cc716bad10..2b018f2bb2b 100644 --- a/libgo/go/net/tcpsockopt_dragonfly.go +++ b/libgo/go/net/tcpsockopt_dragonfly.go @@ -5,22 +5,20 @@ package net import ( - "os" + "runtime" "syscall" "time" ) func setKeepAlivePeriod(fd *netFD, d time.Duration) error { - if err := fd.incref(); err != nil { - return err - } - defer fd.decref() // The kernel expects milliseconds so round to next highest // millisecond. d += (time.Millisecond - time.Nanosecond) msecs := int(d / time.Millisecond) - if err := syscall.SetsockoptInt(fd.sysfd, syscall.IPPROTO_TCP, syscall.TCP_KEEPINTVL, msecs); err != nil { - return os.NewSyscallError("setsockopt", err) + if err := fd.pfd.SetsockoptInt(syscall.IPPROTO_TCP, syscall.TCP_KEEPINTVL, msecs); err != nil { + return wrapSyscallError("setsockopt", err) } - return os.NewSyscallError("setsockopt", syscall.SetsockoptInt(fd.sysfd, syscall.IPPROTO_TCP, syscall.TCP_KEEPIDLE, msecs)) + err := fd.pfd.SetsockoptInt(syscall.IPPROTO_TCP, syscall.TCP_KEEPIDLE, msecs) + runtime.KeepAlive(fd) + return wrapSyscallError("setsockopt", err) } diff --git a/libgo/go/net/tcpsockopt_posix.go b/libgo/go/net/tcpsockopt_posix.go index 36866ac0418..5e00ba15647 100644 --- a/libgo/go/net/tcpsockopt_posix.go +++ b/libgo/go/net/tcpsockopt_posix.go @@ -7,14 +7,12 @@ package net import ( - "os" + "runtime" "syscall" ) func setNoDelay(fd *netFD, noDelay bool) error { - if err := fd.incref(); err != nil { - return err - } - defer fd.decref() - return os.NewSyscallError("setsockopt", syscall.SetsockoptInt(fd.sysfd, syscall.IPPROTO_TCP, syscall.TCP_NODELAY, boolint(noDelay))) + err := fd.pfd.SetsockoptInt(syscall.IPPROTO_TCP, syscall.TCP_NODELAY, boolint(noDelay)) + runtime.KeepAlive(fd) + return wrapSyscallError("setsockopt", err) } diff --git a/libgo/go/net/tcpsockopt_solaris.go b/libgo/go/net/tcpsockopt_solaris.go index 347c17d7e27..aa86a29063e 100644 --- a/libgo/go/net/tcpsockopt_solaris.go +++ b/libgo/go/net/tcpsockopt_solaris.go @@ -2,26 +2,20 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -// TCP socket options for solaris - package net import ( - "os" + "runtime" "syscall" "time" ) -// Set keep alive period. func setKeepAlivePeriod(fd *netFD, d time.Duration) error { - if err := fd.incref(); err != nil { - return err - } - defer fd.decref() - // The kernel expects seconds so round to next highest second. d += (time.Second - time.Nanosecond) secs := int(d.Seconds()) - return os.NewSyscallError("setsockopt", syscall.SetsockoptInt(fd.sysfd, syscall.IPPROTO_TCP, syscall.SO_KEEPALIVE, secs)) + err := fd.pfd.SetsockoptInt(syscall.IPPROTO_TCP, syscall.SO_KEEPALIVE, secs) + runtime.KeepAlive(fd) + return wrapSyscallError("setsockopt", err) } diff --git a/libgo/go/net/tcpsockopt_unix.go b/libgo/go/net/tcpsockopt_unix.go index 46e5e6d23fe..d5892588fea 100644 --- a/libgo/go/net/tcpsockopt_unix.go +++ b/libgo/go/net/tcpsockopt_unix.go @@ -7,21 +7,19 @@ package net import ( - "os" + "runtime" "syscall" "time" ) func setKeepAlivePeriod(fd *netFD, d time.Duration) error { - if err := fd.incref(); err != nil { - return err - } - defer fd.decref() // The kernel expects seconds so round to next highest second. d += (time.Second - time.Nanosecond) secs := int(d.Seconds()) - if err := syscall.SetsockoptInt(fd.sysfd, syscall.IPPROTO_TCP, syscall.TCP_KEEPINTVL, secs); err != nil { - return os.NewSyscallError("setsockopt", err) + if err := fd.pfd.SetsockoptInt(syscall.IPPROTO_TCP, syscall.TCP_KEEPINTVL, secs); err != nil { + return wrapSyscallError("setsockopt", err) } - return os.NewSyscallError("setsockopt", syscall.SetsockoptInt(fd.sysfd, syscall.IPPROTO_TCP, syscall.TCP_KEEPIDLE, secs)) + err := fd.pfd.SetsockoptInt(syscall.IPPROTO_TCP, syscall.TCP_KEEPIDLE, secs) + runtime.KeepAlive(fd) + return wrapSyscallError("setsockopt", err) } diff --git a/libgo/go/net/tcpsockopt_windows.go b/libgo/go/net/tcpsockopt_windows.go index 45a4dca5257..73dead11d00 100644 --- a/libgo/go/net/tcpsockopt_windows.go +++ b/libgo/go/net/tcpsockopt_windows.go @@ -6,16 +6,13 @@ package net import ( "os" + "runtime" "syscall" "time" "unsafe" ) func setKeepAlivePeriod(fd *netFD, d time.Duration) error { - if err := fd.incref(); err != nil { - return err - } - defer fd.decref() // The kernel expects milliseconds so round to next highest // millisecond. d += (time.Millisecond - time.Nanosecond) @@ -27,6 +24,7 @@ func setKeepAlivePeriod(fd *netFD, d time.Duration) error { } ret := uint32(0) size := uint32(unsafe.Sizeof(ka)) - err := syscall.WSAIoctl(fd.sysfd, syscall.SIO_KEEPALIVE_VALS, (*byte)(unsafe.Pointer(&ka)), size, nil, 0, &ret, nil, 0) + err := fd.pfd.WSAIoctl(syscall.SIO_KEEPALIVE_VALS, (*byte)(unsafe.Pointer(&ka)), size, nil, 0, &ret, nil, 0) + runtime.KeepAlive(fd) return os.NewSyscallError("wsaioctl", err) } diff --git a/libgo/go/net/timeout_test.go b/libgo/go/net/timeout_test.go index 55bbf4402d9..9de7801ad10 100644 --- a/libgo/go/net/timeout_test.go +++ b/libgo/go/net/timeout_test.go @@ -6,6 +6,7 @@ package net import ( "fmt" + "internal/poll" "internal/testenv" "io" "io/ioutil" @@ -145,9 +146,9 @@ var acceptTimeoutTests = []struct { }{ // Tests that accept deadlines in the past work, even if // there's incoming connections available. - {-5 * time.Second, [2]error{errTimeout, errTimeout}}, + {-5 * time.Second, [2]error{poll.ErrTimeout, poll.ErrTimeout}}, - {50 * time.Millisecond, [2]error{nil, errTimeout}}, + {50 * time.Millisecond, [2]error{nil, poll.ErrTimeout}}, } func TestAcceptTimeout(t *testing.T) { @@ -299,9 +300,9 @@ var readTimeoutTests = []struct { }{ // Tests that read deadlines work, even if there's data ready // to be read. - {-5 * time.Second, [2]error{errTimeout, errTimeout}}, + {-5 * time.Second, [2]error{poll.ErrTimeout, poll.ErrTimeout}}, - {50 * time.Millisecond, [2]error{nil, errTimeout}}, + {50 * time.Millisecond, [2]error{nil, poll.ErrTimeout}}, } func TestReadTimeout(t *testing.T) { @@ -423,9 +424,9 @@ var readFromTimeoutTests = []struct { }{ // Tests that read deadlines work, even if there's data ready // to be read. - {-5 * time.Second, [2]error{errTimeout, errTimeout}}, + {-5 * time.Second, [2]error{poll.ErrTimeout, poll.ErrTimeout}}, - {50 * time.Millisecond, [2]error{nil, errTimeout}}, + {50 * time.Millisecond, [2]error{nil, poll.ErrTimeout}}, } func TestReadFromTimeout(t *testing.T) { @@ -496,9 +497,9 @@ var writeTimeoutTests = []struct { }{ // Tests that write deadlines work, even if there's buffer // space available to write. - {-5 * time.Second, [2]error{errTimeout, errTimeout}}, + {-5 * time.Second, [2]error{poll.ErrTimeout, poll.ErrTimeout}}, - {10 * time.Millisecond, [2]error{nil, errTimeout}}, + {10 * time.Millisecond, [2]error{nil, poll.ErrTimeout}}, } func TestWriteTimeout(t *testing.T) { @@ -610,9 +611,9 @@ var writeToTimeoutTests = []struct { }{ // Tests that write deadlines work, even if there's buffer // space available to write. - {-5 * time.Second, [2]error{errTimeout, errTimeout}}, + {-5 * time.Second, [2]error{poll.ErrTimeout, poll.ErrTimeout}}, - {10 * time.Millisecond, [2]error{nil, errTimeout}}, + {10 * time.Millisecond, [2]error{nil, poll.ErrTimeout}}, } func TestWriteToTimeout(t *testing.T) { diff --git a/libgo/go/net/udpsock.go b/libgo/go/net/udpsock.go index 841ef533590..2c0f74fdabd 100644 --- a/libgo/go/net/udpsock.go +++ b/libgo/go/net/udpsock.go @@ -53,28 +53,34 @@ func (a *UDPAddr) opAddr() Addr { return a } -// ResolveUDPAddr parses addr as a UDP address of the form "host:port" -// or "[ipv6-host%zone]:port" and resolves a pair of domain name and -// port name on the network net, which must be "udp", "udp4" or -// "udp6". A literal address or host name for IPv6 must be enclosed -// in square brackets, as in "[::1]:80", "[ipv6-host]:http" or -// "[ipv6-host%zone]:80". +// ResolveUDPAddr returns an address of UDP end point. // -// Resolving a hostname is not recommended because this returns at most -// one of its IP addresses. -func ResolveUDPAddr(net, addr string) (*UDPAddr, error) { - switch net { +// The network must be a UDP network name. +// +// If the host in the address parameter is not a literal IP address or +// the port is not a literal port number, ResolveUDPAddr resolves the +// address to an address of UDP end point. +// Otherwise, it parses the address as a pair of literal IP address +// and port number. +// The address parameter can use a host name, but this is not +// recommended, because it will return at most one of the host name's +// IP addresses. +// +// See func Dial for a description of the network and address +// parameters. +func ResolveUDPAddr(network, address string) (*UDPAddr, error) { + switch network { case "udp", "udp4", "udp6": case "": // a hint wildcard for Go 1.0 undocumented behavior - net = "udp" + network = "udp" default: - return nil, UnknownNetworkError(net) + return nil, UnknownNetworkError(network) } - addrs, err := DefaultResolver.internetAddrList(context.Background(), net, addr) + addrs, err := DefaultResolver.internetAddrList(context.Background(), network, address) if err != nil { return nil, err } - return addrs.first(isIPv4).(*UDPAddr), nil + return addrs.forResolve(network, address).(*UDPAddr), nil } // UDPConn is the implementation of the Conn and PacketConn interfaces @@ -83,13 +89,16 @@ type UDPConn struct { conn } -// ReadFromUDP reads a UDP packet from c, copying the payload into b. -// It returns the number of bytes copied into b and the return address -// that was on the packet. -// -// ReadFromUDP can be made to time out and return an error with -// Timeout() == true after a fixed time limit; see SetDeadline and -// SetReadDeadline. +// SyscallConn returns a raw network connection. +// This implements the syscall.Conn interface. +func (c *UDPConn) SyscallConn() (syscall.RawConn, error) { + if !c.ok() { + return nil, syscall.EINVAL + } + return newRawConn(c.fd) +} + +// ReadFromUDP acts like ReadFrom but returns a UDPAddr. func (c *UDPConn) ReadFromUDP(b []byte) (int, *UDPAddr, error) { if !c.ok() { return 0, nil, syscall.EINVAL @@ -116,11 +125,13 @@ func (c *UDPConn) ReadFrom(b []byte) (int, Addr, error) { return n, addr, err } -// ReadMsgUDP reads a packet from c, copying the payload into b and -// the associated out-of-band data into oob. It returns the number -// of bytes copied into b, the number of bytes copied into oob, the -// flags that were set on the packet and the source address of the -// packet. +// ReadMsgUDP reads a message from c, copying the payload into b and +// the associated out-of-band data into oob. It returns the number of +// bytes copied into b, the number of bytes copied into oob, the flags +// that were set on the message and the source address of the message. +// +// The packages golang.org/x/net/ipv4 and golang.org/x/net/ipv6 can be +// used to manipulate IP-level socket options in oob. func (c *UDPConn) ReadMsgUDP(b, oob []byte) (n, oobn, flags int, addr *UDPAddr, err error) { if !c.ok() { return 0, 0, 0, nil, syscall.EINVAL @@ -132,13 +143,7 @@ func (c *UDPConn) ReadMsgUDP(b, oob []byte) (n, oobn, flags int, addr *UDPAddr, return } -// WriteToUDP writes a UDP packet to addr via c, copying the payload -// from b. -// -// WriteToUDP can be made to time out and return an error with -// Timeout() == true after a fixed time limit; see SetDeadline and -// SetWriteDeadline. On packet-oriented connections, write timeouts -// are rare. +// WriteToUDP acts like WriteTo but takes a UDPAddr. func (c *UDPConn) WriteToUDP(b []byte, addr *UDPAddr) (int, error) { if !c.ok() { return 0, syscall.EINVAL @@ -166,11 +171,14 @@ func (c *UDPConn) WriteTo(b []byte, addr Addr) (int, error) { return n, err } -// WriteMsgUDP writes a packet to addr via c if c isn't connected, or -// to c's remote destination address if c is connected (in which case -// addr must be nil). The payload is copied from b and the associated -// out-of-band data is copied from oob. It returns the number of -// payload and out-of-band bytes written. +// WriteMsgUDP writes a message to addr via c if c isn't connected, or +// to c's remote address if c is connected (in which case addr must be +// nil). The payload is copied from b and the associated out-of-band +// data is copied from oob. It returns the number of payload and +// out-of-band bytes written. +// +// The packages golang.org/x/net/ipv4 and golang.org/x/net/ipv6 can be +// used to manipulate IP-level socket options in oob. func (c *UDPConn) WriteMsgUDP(b, oob []byte, addr *UDPAddr) (n, oobn int, err error) { if !c.ok() { return 0, 0, syscall.EINVAL @@ -184,55 +192,67 @@ func (c *UDPConn) WriteMsgUDP(b, oob []byte, addr *UDPAddr) (n, oobn int, err er func newUDPConn(fd *netFD) *UDPConn { return &UDPConn{conn{fd}} } -// DialUDP connects to the remote address raddr on the network net, -// which must be "udp", "udp4", or "udp6". If laddr is not nil, it is -// used as the local address for the connection. -func DialUDP(net string, laddr, raddr *UDPAddr) (*UDPConn, error) { - switch net { +// DialUDP acts like Dial for UDP networks. +// +// The network must be a UDP network name; see func Dial for details. +// +// If laddr is nil, a local address is automatically chosen. +// If the IP field of raddr is nil or an unspecified IP address, the +// local system is assumed. +func DialUDP(network string, laddr, raddr *UDPAddr) (*UDPConn, error) { + switch network { case "udp", "udp4", "udp6": default: - return nil, &OpError{Op: "dial", Net: net, Source: laddr.opAddr(), Addr: raddr.opAddr(), Err: UnknownNetworkError(net)} + return nil, &OpError{Op: "dial", Net: network, Source: laddr.opAddr(), Addr: raddr.opAddr(), Err: UnknownNetworkError(network)} } if raddr == nil { - return nil, &OpError{Op: "dial", Net: net, Source: laddr.opAddr(), Addr: nil, Err: errMissingAddress} + return nil, &OpError{Op: "dial", Net: network, Source: laddr.opAddr(), Addr: nil, Err: errMissingAddress} } - c, err := dialUDP(context.Background(), net, laddr, raddr) + c, err := dialUDP(context.Background(), network, laddr, raddr) if err != nil { - return nil, &OpError{Op: "dial", Net: net, Source: laddr.opAddr(), Addr: raddr.opAddr(), Err: err} + return nil, &OpError{Op: "dial", Net: network, Source: laddr.opAddr(), Addr: raddr.opAddr(), Err: err} } return c, nil } -// ListenUDP listens for incoming UDP packets addressed to the local -// address laddr. Net must be "udp", "udp4", or "udp6". If laddr has -// a port of 0, ListenUDP will choose an available port. -// The LocalAddr method of the returned UDPConn can be used to -// discover the port. The returned connection's ReadFrom and WriteTo -// methods can be used to receive and send UDP packets with per-packet -// addressing. -func ListenUDP(net string, laddr *UDPAddr) (*UDPConn, error) { - switch net { +// ListenUDP acts like ListenPacket for UDP networks. +// +// The network must be a UDP network name; see func Dial for details. +// +// If the IP field of laddr is nil or an unspecified IP address, +// ListenUDP listens on all available IP addresses of the local system +// except multicast IP addresses. +// If the Port field of laddr is 0, a port number is automatically +// chosen. +func ListenUDP(network string, laddr *UDPAddr) (*UDPConn, error) { + switch network { case "udp", "udp4", "udp6": default: - return nil, &OpError{Op: "listen", Net: net, Source: nil, Addr: laddr.opAddr(), Err: UnknownNetworkError(net)} + return nil, &OpError{Op: "listen", Net: network, Source: nil, Addr: laddr.opAddr(), Err: UnknownNetworkError(network)} } if laddr == nil { laddr = &UDPAddr{} } - c, err := listenUDP(context.Background(), net, laddr) + c, err := listenUDP(context.Background(), network, laddr) if err != nil { - return nil, &OpError{Op: "listen", Net: net, Source: nil, Addr: laddr.opAddr(), Err: err} + return nil, &OpError{Op: "listen", Net: network, Source: nil, Addr: laddr.opAddr(), Err: err} } return c, nil } -// ListenMulticastUDP listens for incoming multicast UDP packets -// addressed to the group address gaddr on the interface ifi. -// Network must be "udp", "udp4" or "udp6". -// ListenMulticastUDP uses the system-assigned multicast interface -// when ifi is nil, although this is not recommended because the +// ListenMulticastUDP acts like ListenPacket for UDP networks but +// takes a group address on a specific network interface. +// +// The network must be a UDP network name; see func Dial for details. +// +// ListenMulticastUDP listens on all available IP addresses of the +// local system including the group, multicast IP address. +// If ifi is nil, ListenMulticastUDP uses the system-assigned +// multicast interface, although this is not recommended because the // assignment depends on platforms and sometimes it might require // routing configuration. +// If the Port field of gaddr is 0, a port number is automatically +// chosen. // // ListenMulticastUDP is just for convenience of simple, small // applications. There are golang.org/x/net/ipv4 and diff --git a/libgo/go/net/udpsock_posix.go b/libgo/go/net/udpsock_posix.go index 0c905afafcc..fe552ba929f 100644 --- a/libgo/go/net/udpsock_posix.go +++ b/libgo/go/net/udpsock_posix.go @@ -16,7 +16,7 @@ func sockaddrToUDP(sa syscall.Sockaddr) Addr { case *syscall.SockaddrInet4: return &UDPAddr{IP: sa.Addr[0:], Port: sa.Port} case *syscall.SockaddrInet6: - return &UDPAddr{IP: sa.Addr[0:], Port: sa.Port, Zone: zoneToString(int(sa.ZoneId))} + return &UDPAddr{IP: sa.Addr[0:], Port: sa.Port, Zone: zoneCache.name(int(sa.ZoneId))} } return nil } @@ -49,7 +49,7 @@ func (c *UDPConn) readFrom(b []byte) (int, *UDPAddr, error) { case *syscall.SockaddrInet4: addr = &UDPAddr{IP: sa.Addr[0:], Port: sa.Port} case *syscall.SockaddrInet6: - addr = &UDPAddr{IP: sa.Addr[0:], Port: sa.Port, Zone: zoneToString(int(sa.ZoneId))} + addr = &UDPAddr{IP: sa.Addr[0:], Port: sa.Port, Zone: zoneCache.name(int(sa.ZoneId))} } return n, addr, err } @@ -61,7 +61,7 @@ func (c *UDPConn) readMsg(b, oob []byte) (n, oobn, flags int, addr *UDPAddr, err case *syscall.SockaddrInet4: addr = &UDPAddr{IP: sa.Addr[0:], Port: sa.Port} case *syscall.SockaddrInet6: - addr = &UDPAddr{IP: sa.Addr[0:], Port: sa.Port, Zone: zoneToString(int(sa.ZoneId))} + addr = &UDPAddr{IP: sa.Addr[0:], Port: sa.Port, Zone: zoneCache.name(int(sa.ZoneId))} } return } diff --git a/libgo/go/net/udpsock_test.go b/libgo/go/net/udpsock_test.go index 708cc101209..6d4974e3e49 100644 --- a/libgo/go/net/udpsock_test.go +++ b/libgo/go/net/udpsock_test.go @@ -15,7 +15,7 @@ import ( func BenchmarkUDP6LinkLocalUnicast(b *testing.B) { testHookUninstaller.Do(uninstallTestHooks) - if !supportsIPv6 { + if !supportsIPv6() { b.Skip("IPv6 is not supported") } ifi := loopbackInterface() @@ -279,7 +279,7 @@ func TestUDPConnLocalAndRemoteNames(t *testing.T) { func TestIPv6LinkLocalUnicastUDP(t *testing.T) { testenv.MustHaveExternalNetwork(t) - if !supportsIPv6 { + if !supportsIPv6() { t.Skip("IPv6 is not supported") } diff --git a/libgo/go/net/unixsock.go b/libgo/go/net/unixsock.go index b25d492f591..057940acf65 100644 --- a/libgo/go/net/unixsock.go +++ b/libgo/go/net/unixsock.go @@ -42,15 +42,18 @@ func (a *UnixAddr) opAddr() Addr { return a } -// ResolveUnixAddr parses addr as a Unix domain socket address. -// The string net gives the network name, "unix", "unixgram" or -// "unixpacket". -func ResolveUnixAddr(net, addr string) (*UnixAddr, error) { - switch net { +// ResolveUnixAddr returns an address of Unix domain socket end point. +// +// The network must be a Unix network name. +// +// See func Dial for a description of the network and address +// parameters. +func ResolveUnixAddr(network, address string) (*UnixAddr, error) { + switch network { case "unix", "unixgram", "unixpacket": - return &UnixAddr{Name: addr, Net: net}, nil + return &UnixAddr{Name: address, Net: network}, nil default: - return nil, UnknownNetworkError(net) + return nil, UnknownNetworkError(network) } } @@ -60,6 +63,15 @@ type UnixConn struct { conn } +// SyscallConn returns a raw network connection. +// This implements the syscall.Conn interface. +func (c *UnixConn) SyscallConn() (syscall.RawConn, error) { + if !c.ok() { + return nil, syscall.EINVAL + } + return newRawConn(c.fd) +} + // CloseRead shuts down the reading side of the Unix domain connection. // Most callers should just use Close. func (c *UnixConn) CloseRead() error { @@ -84,13 +96,7 @@ func (c *UnixConn) CloseWrite() error { return nil } -// ReadFromUnix reads a packet from c, copying the payload into b. It -// returns the number of bytes copied into b and the source address of -// the packet. -// -// ReadFromUnix can be made to time out and return an error with -// Timeout() == true after a fixed time limit; see SetDeadline and -// SetReadDeadline. +// ReadFromUnix acts like ReadFrom but returns a UnixAddr. func (c *UnixConn) ReadFromUnix(b []byte) (int, *UnixAddr, error) { if !c.ok() { return 0, nil, syscall.EINVAL @@ -117,10 +123,10 @@ func (c *UnixConn) ReadFrom(b []byte) (int, Addr, error) { return n, addr, err } -// ReadMsgUnix reads a packet from c, copying the payload into b and +// ReadMsgUnix reads a message from c, copying the payload into b and // the associated out-of-band data into oob. It returns the number of // bytes copied into b, the number of bytes copied into oob, the flags -// that were set on the packet, and the source address of the packet. +// that were set on the message and the source address of the message. // // Note that if len(b) == 0 and len(oob) > 0, this function will still // read (and discard) 1 byte from the connection. @@ -135,12 +141,7 @@ func (c *UnixConn) ReadMsgUnix(b, oob []byte) (n, oobn, flags int, addr *UnixAdd return } -// WriteToUnix writes a packet to addr via c, copying the payload from b. -// -// WriteToUnix can be made to time out and return an error with -// Timeout() == true after a fixed time limit; see SetDeadline and -// SetWriteDeadline. On packet-oriented connections, write timeouts -// are rare. +// WriteToUnix acts like WriteTo but takes a UnixAddr. func (c *UnixConn) WriteToUnix(b []byte, addr *UnixAddr) (int, error) { if !c.ok() { return 0, syscall.EINVAL @@ -168,9 +169,9 @@ func (c *UnixConn) WriteTo(b []byte, addr Addr) (int, error) { return n, err } -// WriteMsgUnix writes a packet to addr via c, copying the payload -// from b and the associated out-of-band data from oob. It returns -// the number of payload and out-of-band bytes written. +// WriteMsgUnix writes a message to addr via c, copying the payload +// from b and the associated out-of-band data from oob. It returns the +// number of payload and out-of-band bytes written. // // Note that if len(b) == 0 and len(oob) > 0, this function will still // write 1 byte to the connection. @@ -187,18 +188,21 @@ func (c *UnixConn) WriteMsgUnix(b, oob []byte, addr *UnixAddr) (n, oobn int, err func newUnixConn(fd *netFD) *UnixConn { return &UnixConn{conn{fd}} } -// DialUnix connects to the remote address raddr on the network net, -// which must be "unix", "unixgram" or "unixpacket". If laddr is not -// nil, it is used as the local address for the connection. -func DialUnix(net string, laddr, raddr *UnixAddr) (*UnixConn, error) { - switch net { +// DialUnix acts like Dial for Unix networks. +// +// The network must be a Unix network name; see func Dial for details. +// +// If laddr is non-nil, it is used as the local address for the +// connection. +func DialUnix(network string, laddr, raddr *UnixAddr) (*UnixConn, error) { + switch network { case "unix", "unixgram", "unixpacket": default: - return nil, &OpError{Op: "dial", Net: net, Source: laddr.opAddr(), Addr: raddr.opAddr(), Err: UnknownNetworkError(net)} + return nil, &OpError{Op: "dial", Net: network, Source: laddr.opAddr(), Addr: raddr.opAddr(), Err: UnknownNetworkError(network)} } - c, err := dialUnix(context.Background(), net, laddr, raddr) + c, err := dialUnix(context.Background(), network, laddr, raddr) if err != nil { - return nil, &OpError{Op: "dial", Net: net, Source: laddr.opAddr(), Addr: raddr.opAddr(), Err: err} + return nil, &OpError{Op: "dial", Net: network, Source: laddr.opAddr(), Addr: raddr.opAddr(), Err: err} } return c, nil } @@ -264,7 +268,7 @@ func (l *UnixListener) SetDeadline(t time.Time) error { if !l.ok() { return syscall.EINVAL } - if err := l.fd.setDeadline(t); err != nil { + if err := l.fd.pfd.SetDeadline(t); err != nil { return &OpError{Op: "set", Net: l.fd.net, Source: nil, Addr: l.fd.laddr, Err: err} } return nil @@ -288,40 +292,40 @@ func (l *UnixListener) File() (f *os.File, err error) { return } -// ListenUnix announces on the Unix domain socket laddr and returns a -// Unix listener. The network net must be "unix" or "unixpacket". -func ListenUnix(net string, laddr *UnixAddr) (*UnixListener, error) { - switch net { +// ListenUnix acts like Listen for Unix networks. +// +// The network must be "unix" or "unixpacket". +func ListenUnix(network string, laddr *UnixAddr) (*UnixListener, error) { + switch network { case "unix", "unixpacket": default: - return nil, &OpError{Op: "listen", Net: net, Source: nil, Addr: laddr.opAddr(), Err: UnknownNetworkError(net)} + return nil, &OpError{Op: "listen", Net: network, Source: nil, Addr: laddr.opAddr(), Err: UnknownNetworkError(network)} } if laddr == nil { - return nil, &OpError{Op: "listen", Net: net, Source: nil, Addr: laddr.opAddr(), Err: errMissingAddress} + return nil, &OpError{Op: "listen", Net: network, Source: nil, Addr: laddr.opAddr(), Err: errMissingAddress} } - ln, err := listenUnix(context.Background(), net, laddr) + ln, err := listenUnix(context.Background(), network, laddr) if err != nil { - return nil, &OpError{Op: "listen", Net: net, Source: nil, Addr: laddr.opAddr(), Err: err} + return nil, &OpError{Op: "listen", Net: network, Source: nil, Addr: laddr.opAddr(), Err: err} } return ln, nil } -// ListenUnixgram listens for incoming Unix datagram packets addressed -// to the local address laddr. The network net must be "unixgram". -// The returned connection's ReadFrom and WriteTo methods can be used -// to receive and send packets with per-packet addressing. -func ListenUnixgram(net string, laddr *UnixAddr) (*UnixConn, error) { - switch net { +// ListenUnixgram acts like ListenPacket for Unix networks. +// +// The network must be "unixgram". +func ListenUnixgram(network string, laddr *UnixAddr) (*UnixConn, error) { + switch network { case "unixgram": default: - return nil, &OpError{Op: "listen", Net: net, Source: nil, Addr: laddr.opAddr(), Err: UnknownNetworkError(net)} + return nil, &OpError{Op: "listen", Net: network, Source: nil, Addr: laddr.opAddr(), Err: UnknownNetworkError(network)} } if laddr == nil { - return nil, &OpError{Op: "listen", Net: net, Source: nil, Addr: nil, Err: errMissingAddress} + return nil, &OpError{Op: "listen", Net: network, Source: nil, Addr: nil, Err: errMissingAddress} } - c, err := listenUnixgram(context.Background(), net, laddr) + c, err := listenUnixgram(context.Background(), network, laddr) if err != nil { - return nil, &OpError{Op: "listen", Net: net, Source: nil, Addr: laddr.opAddr(), Err: err} + return nil, &OpError{Op: "listen", Net: network, Source: nil, Addr: laddr.opAddr(), Err: err} } return c, nil } diff --git a/libgo/go/net/url/url.go b/libgo/go/net/url/url.go index 42a514bbc12..2ac24725692 100644 --- a/libgo/go/net/url/url.go +++ b/libgo/go/net/url/url.go @@ -309,9 +309,10 @@ func escape(s string, mode encoding) string { } // A URL represents a parsed URL (technically, a URI reference). +// // The general form represented is: // -// scheme://[userinfo@]host/path[?query][#fragment] +// [scheme:][//[userinfo@]host][/]path[?query][#fragment] // // URLs that do not start with a slash after the scheme are interpreted as: // @@ -321,26 +322,19 @@ func escape(s string, mode encoding) string { // A consequence is that it is impossible to tell which slashes in the Path were // slashes in the raw URL and which were %2f. This distinction is rarely important, // but when it is, code must not use Path directly. -// -// Go 1.5 introduced the RawPath field to hold the encoded form of Path. // The Parse function sets both Path and RawPath in the URL it returns, // and URL's String method uses RawPath if it is a valid encoding of Path, // by calling the EscapedPath method. -// -// In earlier versions of Go, the more indirect workarounds were that an -// HTTP server could consult req.RequestURI and an HTTP client could -// construct a URL struct directly and set the Opaque field instead of Path. -// These still work as well. type URL struct { Scheme string Opaque string // encoded opaque data User *Userinfo // username and password information Host string // host or host:port - Path string - RawPath string // encoded path hint (Go 1.5 and later only; see EscapedPath method) - ForceQuery bool // append a query ('?') even if RawQuery is empty - RawQuery string // encoded query values, without '?' - Fragment string // fragment for references, without '#' + Path string // path (relative paths may omit leading slash) + RawPath string // encoded path hint (see EscapedPath method) + ForceQuery bool // append a query ('?') even if RawQuery is empty + RawQuery string // encoded query values, without '?' + Fragment string // fragment for references, without '#' } // User returns a Userinfo containing the provided username @@ -351,6 +345,7 @@ func User(username string) *Userinfo { // UserPassword returns a Userinfo containing the provided username // and password. +// // This functionality should only be used with legacy web sites. // RFC 2396 warns that interpreting Userinfo this way // ``is NOT RECOMMENDED, because the passing of authentication @@ -974,6 +969,8 @@ func (u *URL) ResolveReference(ref *URL) *URL { } // Query parses RawQuery and returns the corresponding values. +// It silently discards malformed value pairs. +// To check errors use ParseQuery. func (u *URL) Query() Values { v, _ := ParseQuery(u.RawQuery) return v diff --git a/libgo/go/net/writev_test.go b/libgo/go/net/writev_test.go index 7160d28c3a0..4c05be473d9 100644 --- a/libgo/go/net/writev_test.go +++ b/libgo/go/net/writev_test.go @@ -7,6 +7,7 @@ package net import ( "bytes" "fmt" + "internal/poll" "io" "io/ioutil" "reflect" @@ -99,13 +100,13 @@ func TestBuffers_WriteTo(t *testing.T) { } func testBuffer_writeTo(t *testing.T, chunks int, useCopy bool) { - oldHook := testHookDidWritev - defer func() { testHookDidWritev = oldHook }() + oldHook := poll.TestHookDidWritev + defer func() { poll.TestHookDidWritev = oldHook }() var writeLog struct { sync.Mutex log []int } - testHookDidWritev = func(size int) { + poll.TestHookDidWritev = func(size int) { writeLog.Lock() writeLog.log = append(writeLog.log, size) writeLog.Unlock() diff --git a/libgo/go/net/writev_unix.go b/libgo/go/net/writev_unix.go index 174e6bc51e3..bf0fbf8a136 100644 --- a/libgo/go/net/writev_unix.go +++ b/libgo/go/net/writev_unix.go @@ -7,10 +7,8 @@ package net import ( - "io" - "os" + "runtime" "syscall" - "unsafe" ) func (c *conn) writeBuffers(v *Buffers) (int64, error) { @@ -25,71 +23,7 @@ func (c *conn) writeBuffers(v *Buffers) (int64, error) { } func (fd *netFD) writeBuffers(v *Buffers) (n int64, err error) { - if err := fd.writeLock(); err != nil { - return 0, err - } - defer fd.writeUnlock() - if err := fd.pd.prepareWrite(); err != nil { - return 0, err - } - - var iovecs []syscall.Iovec - if fd.iovecs != nil { - iovecs = *fd.iovecs - } - // TODO: read from sysconf(_SC_IOV_MAX)? The Linux default is - // 1024 and this seems conservative enough for now. Darwin's - // UIO_MAXIOV also seems to be 1024. - maxVec := 1024 - - for len(*v) > 0 { - iovecs = iovecs[:0] - for _, chunk := range *v { - if len(chunk) == 0 { - continue - } - iovecs = append(iovecs, syscall.Iovec{Base: &chunk[0]}) - if fd.isStream && len(chunk) > 1<<30 { - iovecs[len(iovecs)-1].SetLen(1 << 30) - break // continue chunk on next writev - } - iovecs[len(iovecs)-1].SetLen(len(chunk)) - if len(iovecs) == maxVec { - break - } - } - if len(iovecs) == 0 { - break - } - fd.iovecs = &iovecs // cache - - wrote, _, e0 := syscall.Syscall(syscall.SYS_WRITEV, - uintptr(fd.sysfd), - uintptr(unsafe.Pointer(&iovecs[0])), - uintptr(len(iovecs))) - if wrote == ^uintptr(0) { - wrote = 0 - } - testHookDidWritev(int(wrote)) - n += int64(wrote) - v.consume(int64(wrote)) - if e0 == syscall.EAGAIN { - if err = fd.pd.waitWrite(); err == nil { - continue - } - } else if e0 != 0 { - err = syscall.Errno(e0) - } - if err != nil { - break - } - if n == 0 { - err = io.ErrUnexpectedEOF - break - } - } - if _, ok := err.(syscall.Errno); ok { - err = os.NewSyscallError("writev", err) - } - return n, err + n, err = fd.pfd.Writev((*[][]byte)(v)) + runtime.KeepAlive(fd) + return n, wrapSyscallError("writev", err) } diff --git a/libgo/go/os/error_posix.go b/libgo/go/os/error_posix.go new file mode 100644 index 00000000000..2049e448e8f --- /dev/null +++ b/libgo/go/os/error_posix.go @@ -0,0 +1,18 @@ +// Copyright 2017 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build darwin dragonfly freebsd linux nacl netbsd openbsd solaris windows + +package os + +import "syscall" + +// wrapSyscallError takes an error and a syscall name. If the error is +// a syscall.Errno, it wraps it in a os.SyscallError using the syscall name. +func wrapSyscallError(name string, err error) error { + if _, ok := err.(syscall.Errno); ok { + err = NewSyscallError(name, err) + } + return err +} diff --git a/libgo/go/os/example_test.go b/libgo/go/os/example_test.go index 07f9c769590..5749194871b 100644 --- a/libgo/go/os/example_test.go +++ b/libgo/go/os/example_test.go @@ -21,6 +21,20 @@ func ExampleOpenFile() { } } +func ExampleOpenFile_append() { + // If the file doesn't exist, create it, or append to the file + f, err := os.OpenFile("access.log", os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644) + if err != nil { + log.Fatal(err) + } + if _, err := f.Write([]byte("appended some data\n")); err != nil { + log.Fatal(err) + } + if err := f.Close(); err != nil { + log.Fatal(err) + } +} + func ExampleChmod() { if err := os.Chmod("some-filename", 0644); err != nil { log.Fatal(err) @@ -36,7 +50,7 @@ func ExampleChtimes() { } func ExampleFileMode() { - fi, err := os.Stat("some-filename") + fi, err := os.Lstat("some-filename") if err != nil { log.Fatal(err) } diff --git a/libgo/go/os/exec/env_test.go b/libgo/go/os/exec/env_test.go new file mode 100644 index 00000000000..b5ac398c274 --- /dev/null +++ b/libgo/go/os/exec/env_test.go @@ -0,0 +1,39 @@ +// Copyright 2017 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package exec + +import ( + "reflect" + "testing" +) + +func TestDedupEnv(t *testing.T) { + tests := []struct { + noCase bool + in []string + want []string + }{ + { + noCase: true, + in: []string{"k1=v1", "k2=v2", "K1=v3"}, + want: []string{"K1=v3", "k2=v2"}, + }, + { + noCase: false, + in: []string{"k1=v1", "K1=V2", "k1=v3"}, + want: []string{"k1=v3", "K1=V2"}, + }, + { + in: []string{"=a", "=b", "foo", "bar"}, + want: []string{"=b", "foo", "bar"}, + }, + } + for _, tt := range tests { + got := dedupEnvCase(tt.noCase, tt.in) + if !reflect.DeepEqual(got, tt.want) { + t.Errorf("Dedup(%v, %q) = %q; want %q", tt.noCase, tt.in, got, tt.want) + } + } +} diff --git a/libgo/go/os/exec/exec.go b/libgo/go/os/exec/exec.go index c4c5168b980..893d8ee99a8 100644 --- a/libgo/go/os/exec/exec.go +++ b/libgo/go/os/exec/exec.go @@ -6,6 +6,15 @@ // easier to remap stdin and stdout, connect I/O with pipes, and do other // adjustments. // +// Unlike the "system" library call from C and other languages, the +// os/exec package intentionally does not invoke the system shell and +// does not expand any glob patterns or handle other expansions, +// pipelines, or redirections typically done by shells. The package +// behaves more like C's "exec" family of functions. To expand glob +// patterns, either call the shell directly, taking care to escape any +// dangerous input, or use the path/filepath package's Glob function. +// To expand environment variables, use package os's ExpandEnv. +// // Note that the examples in this package assume a Unix system. // They may not run on Windows, and they do not run in the Go Playground // used by golang.org and godoc.org. @@ -55,7 +64,11 @@ type Cmd struct { Args []string // Env specifies the environment of the process. - // If Env is nil, Run uses the current process's environment. + // Each entry is of the form "key=value". + // If Env is nil, the new process uses the current process's + // environment. + // If Env contains duplicate environment keys, only the last + // value in the slice for each duplicate key is used. Env []string // Dir specifies the working directory of the command. @@ -79,17 +92,14 @@ type Cmd struct { // If either is nil, Run connects the corresponding file descriptor // to the null device (os.DevNull). // - // If Stdout and Stderr are the same writer, at most one - // goroutine at a time will call Write. + // If Stdout and Stderr are the same writer, and have a type that can be compared with ==, + // at most one goroutine at a time will call Write. Stdout io.Writer Stderr io.Writer // ExtraFiles specifies additional open files to be inherited by the // new process. It does not include standard input, standard output, or // standard error. If non-nil, entry i becomes file descriptor 3+i. - // - // BUG(rsc): On OS X 10.6, child processes may sometimes inherit unwanted fds. - // https://golang.org/issue/2603 ExtraFiles []*os.File // SysProcAttr holds optional, operating system-specific attributes. @@ -270,9 +280,8 @@ func (c *Cmd) closeDescriptors(closers []io.Closer) { // copying stdin, stdout, and stderr, and exits with a zero exit // status. // -// If the command fails to run or doesn't complete successfully, the -// error is of type *ExitError. Other error types may be -// returned for I/O problems. +// If the command starts but does not complete successfully, the error is of +// type *ExitError. Other error types may be returned for other situations. func (c *Cmd) Run() error { if err := c.Start(); err != nil { return err @@ -354,7 +363,7 @@ func (c *Cmd) Start() error { c.Process, err = os.StartProcess(c.Path, c.argv(), &os.ProcAttr{ Dir: c.Dir, Files: c.childFiles, - Env: c.envv(), + Env: dedupEnv(c.envv()), Sys: c.SysProcAttr, }) if err != nil { @@ -407,8 +416,10 @@ func (e *ExitError) Error() string { return e.ProcessState.String() } -// Wait waits for the command to exit. -// It must have been started by Start. +// Wait waits for the command to exit and waits for any copying to +// stdin or copying from stdout or stderr to complete. +// +// The command must have been started by Start. // // The returned error is nil if the command runs, has no problems // copying stdin, stdout, and stderr, and exits with a zero exit @@ -712,3 +723,35 @@ func minInt(a, b int) int { } return b } + +// dedupEnv returns a copy of env with any duplicates removed, in favor of +// later values. +// Items not of the normal environment "key=value" form are preserved unchanged. +func dedupEnv(env []string) []string { + return dedupEnvCase(runtime.GOOS == "windows", env) +} + +// dedupEnvCase is dedupEnv with a case option for testing. +// If caseInsensitive is true, the case of keys is ignored. +func dedupEnvCase(caseInsensitive bool, env []string) []string { + out := make([]string, 0, len(env)) + saw := map[string]int{} // key => index into out + for _, kv := range env { + eq := strings.Index(kv, "=") + if eq < 0 { + out = append(out, kv) + continue + } + k := kv[:eq] + if caseInsensitive { + k = strings.ToLower(k) + } + if dupIdx, isDup := saw[k]; isDup { + out[dupIdx] = kv + continue + } + saw[k] = len(out) + out = append(out, kv) + } + return out +} diff --git a/libgo/go/os/exec/exec_posix.go b/libgo/go/os/exec/exec_posix.go deleted file mode 100644 index 5e1113748cd..00000000000 --- a/libgo/go/os/exec/exec_posix.go +++ /dev/null @@ -1,24 +0,0 @@ -// Copyright 2015 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// +build !plan9 - -package exec - -import ( - "os" - "syscall" -) - -func init() { - skipStdinCopyError = func(err error) bool { - // Ignore EPIPE errors copying to stdin if the program - // completed successfully otherwise. - // See Issue 9173. - pe, ok := err.(*os.PathError) - return ok && - pe.Op == "write" && pe.Path == "|1" && - pe.Err == syscall.EPIPE - } -} diff --git a/libgo/go/os/exec/exec_posix_test.go b/libgo/go/os/exec/exec_posix_test.go new file mode 100644 index 00000000000..865b6c3ced2 --- /dev/null +++ b/libgo/go/os/exec/exec_posix_test.go @@ -0,0 +1,83 @@ +// Copyright 2017 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build darwin dragonfly freebsd linux netbsd openbsd solaris + +package exec_test + +import ( + "os/user" + "strconv" + "syscall" + "testing" + "time" +) + +func TestCredentialNoSetGroups(t *testing.T) { + u, err := user.Current() + if err != nil { + t.Fatalf("error getting current user: %v", err) + } + + uid, err := strconv.Atoi(u.Uid) + if err != nil { + t.Fatalf("error converting Uid=%s to integer: %v", u.Uid, err) + } + + gid, err := strconv.Atoi(u.Gid) + if err != nil { + t.Fatalf("error converting Gid=%s to integer: %v", u.Gid, err) + } + + // If NoSetGroups is true, setgroups isn't called and cmd.Run should succeed + cmd := helperCommand(t, "echo", "foo") + cmd.SysProcAttr = &syscall.SysProcAttr{ + Credential: &syscall.Credential{ + Uid: uint32(uid), + Gid: uint32(gid), + NoSetGroups: true, + }, + } + + if err = cmd.Run(); err != nil { + t.Errorf("Failed to run command: %v", err) + } +} + +// For issue #19314: make sure that SIGSTOP does not cause the process +// to appear done. +func TestWaitid(t *testing.T) { + t.Parallel() + + cmd := helperCommand(t, "sleep") + if err := cmd.Start(); err != nil { + t.Fatal(err) + } + + // The sleeps here are unnecessary in the sense that the test + // should still pass, but they are useful to make it more + // likely that we are testing the expected state of the child. + time.Sleep(100 * time.Millisecond) + + if err := cmd.Process.Signal(syscall.SIGSTOP); err != nil { + cmd.Process.Kill() + t.Fatal(err) + } + + ch := make(chan error) + go func() { + ch <- cmd.Wait() + }() + + time.Sleep(100 * time.Millisecond) + + if err := cmd.Process.Signal(syscall.SIGCONT); err != nil { + t.Error(err) + syscall.Kill(cmd.Process.Pid, syscall.SIGCONT) + } + + cmd.Process.Kill() + + <-ch +} diff --git a/libgo/go/os/exec/exec_test.go b/libgo/go/os/exec/exec_test.go index f13635138a1..a877d8ae232 100644 --- a/libgo/go/os/exec/exec_test.go +++ b/libgo/go/os/exec/exec_test.go @@ -12,6 +12,7 @@ import ( "bytes" "context" "fmt" + "internal/poll" "internal/testenv" "io" "io/ioutil" @@ -292,8 +293,51 @@ func TestStdinCloseRace(t *testing.T) { // Issue 5071 func TestPipeLookPathLeak(t *testing.T) { - fd0, lsof0 := numOpenFDS(t) - for i := 0; i < 4; i++ { + // If we are reading from /proc/self/fd we (should) get an exact result. + tolerance := 0 + + // Reading /proc/self/fd is more reliable than calling lsof, so try that + // first. + numOpenFDs := func() (int, []byte, error) { + fds, err := ioutil.ReadDir("/proc/self/fd") + if err != nil { + return 0, nil, err + } + return len(fds), nil, nil + } + want, before, err := numOpenFDs() + if err != nil { + // We encountered a problem reading /proc/self/fd (we might be on + // a platform that doesn't have it). Fall back onto lsof. + t.Logf("using lsof because: %v", err) + numOpenFDs = func() (int, []byte, error) { + // Android's stock lsof does not obey the -p option, + // so extra filtering is needed. + // https://golang.org/issue/10206 + if runtime.GOOS == "android" { + // numOpenFDsAndroid handles errors itself and + // might skip or fail the test. + n, lsof := numOpenFDsAndroid(t) + return n, lsof, nil + } + lsof, err := exec.Command("lsof", "-b", "-n", "-p", strconv.Itoa(os.Getpid())).Output() + return bytes.Count(lsof, []byte("\n")), lsof, err + } + + // lsof may see file descriptors associated with the fork itself, + // so we allow some extra margin if we have to use it. + // https://golang.org/issue/19243 + tolerance = 5 + + // Retry reading the number of open file descriptors. + want, before, err = numOpenFDs() + if err != nil { + t.Log(err) + t.Skipf("skipping test; error finding or running lsof") + } + } + + for i := 0; i < 6; i++ { cmd := exec.Command("something-that-does-not-exist-binary") cmd.StdoutPipe() cmd.StderrPipe() @@ -302,36 +346,20 @@ func TestPipeLookPathLeak(t *testing.T) { t.Fatal("unexpected success") } } - for triesLeft := 3; triesLeft >= 0; triesLeft-- { - open, lsof := numOpenFDS(t) - fdGrowth := open - fd0 - if fdGrowth > 2 { - if triesLeft > 0 { - // Work around what appears to be a race with Linux's - // proc filesystem (as used by lsof). It seems to only - // be eventually consistent. Give it awhile to settle. - // See golang.org/issue/7808 - time.Sleep(100 * time.Millisecond) - continue - } - t.Errorf("leaked %d fds; want ~0; have:\n%s\noriginally:\n%s", fdGrowth, lsof, lsof0) - } - break - } -} - -func numOpenFDS(t *testing.T) (n int, lsof []byte) { - if runtime.GOOS == "android" { - // Android's stock lsof does not obey the -p option, - // so extra filtering is needed. (golang.org/issue/10206) - return numOpenFDsAndroid(t) - } - - lsof, err := exec.Command("lsof", "-b", "-n", "-p", strconv.Itoa(os.Getpid())).Output() + got, after, err := numOpenFDs() if err != nil { - t.Skip("skipping test; error finding or running lsof") + // numOpenFDs has already succeeded once, it should work here. + t.Errorf("unexpected failure: %v", err) + } + if got-want > tolerance { + t.Errorf("number of open file descriptors changed: got %v, want %v", got, want) + if before != nil { + t.Errorf("before:\n%v\n", before) + } + if after != nil { + t.Errorf("after:\n%v\n", after) + } } - return bytes.Count(lsof, []byte("\n")), lsof } func numOpenFDsAndroid(t *testing.T) (n int, lsof []byte) { @@ -377,12 +405,16 @@ var testedAlreadyLeaked = false // basefds returns the number of expected file descriptors // to be present in a process at start. +// stdin, stdout, stderr, epoll/kqueue func basefds() uintptr { return os.Stderr.Fd() + 1 } func closeUnexpectedFds(t *testing.T, m string) { for fd := basefds(); fd <= 101; fd++ { + if fd == poll.PollDescriptor() { + continue + } err := os.NewFile(fd, "").Close() if err == nil { t.Logf("%s: Something already leaked - closed fd %d", m, fd) @@ -665,6 +697,11 @@ func TestHelperProcess(*testing.T) { iargs = append(iargs, s) } fmt.Println(iargs...) + case "echoenv": + for _, s := range args { + fmt.Println(os.Getenv(s)) + } + os.Exit(0) case "cat": if len(args) == 0 { io.Copy(os.Stdout, os.Stdin) @@ -740,6 +777,9 @@ func TestHelperProcess(*testing.T) { // Now verify that there are no other open fds. var files []*os.File for wantfd := basefds() + 1; wantfd <= 100; wantfd++ { + if wantfd == poll.PollDescriptor() { + continue + } f, err := os.Open(os.Args[0]) if err != nil { fmt.Printf("error opening file with expected fd %d: %v", wantfd, err) @@ -832,31 +872,50 @@ func TestHelperProcess(*testing.T) { case "stderrfail": fmt.Fprintf(os.Stderr, "some stderr text\n") os.Exit(1) + case "sleep": + time.Sleep(3 * time.Second) + os.Exit(0) default: fmt.Fprintf(os.Stderr, "Unknown command %q\n", cmd) os.Exit(2) } } +type delayedInfiniteReader struct{} + +func (delayedInfiniteReader) Read(b []byte) (int, error) { + time.Sleep(100 * time.Millisecond) + for i := range b { + b[i] = 'x' + } + return len(b), nil +} + // Issue 9173: ignore stdin pipe writes if the program completes successfully. func TestIgnorePipeErrorOnSuccess(t *testing.T) { testenv.MustHaveExec(t) - // We really only care about testing this on Unixy things. - if runtime.GOOS == "windows" || runtime.GOOS == "plan9" { + // We really only care about testing this on Unixy and Windowsy things. + if runtime.GOOS == "plan9" { t.Skipf("skipping test on %q", runtime.GOOS) } - cmd := helperCommand(t, "echo", "foo") - var out bytes.Buffer - cmd.Stdin = strings.NewReader(strings.Repeat("x", 10<<20)) - cmd.Stdout = &out - if err := cmd.Run(); err != nil { - t.Fatal(err) - } - if got, want := out.String(), "foo\n"; got != want { - t.Errorf("output = %q; want %q", got, want) + testWith := func(r io.Reader) func(*testing.T) { + return func(t *testing.T) { + cmd := helperCommand(t, "echo", "foo") + var out bytes.Buffer + cmd.Stdin = r + cmd.Stdout = &out + if err := cmd.Run(); err != nil { + t.Fatal(err) + } + if got, want := out.String(), "foo\n"; got != want { + t.Errorf("output = %q; want %q", got, want) + } + } } + t.Run("10MB", testWith(strings.NewReader(strings.Repeat("x", 10<<20)))) + t.Run("Infinite", testWith(delayedInfiniteReader{})) } type badWriter struct{} @@ -1012,3 +1071,18 @@ func TestContextCancel(t *testing.T) { t.Logf("exit status: %v", err) } } + +// test that environment variables are de-duped. +func TestDedupEnvEcho(t *testing.T) { + testenv.MustHaveExec(t) + + cmd := helperCommand(t, "echoenv", "FOO") + cmd.Env = append(cmd.Env, "FOO=bad", "FOO=good") + out, err := cmd.CombinedOutput() + if err != nil { + t.Fatal(err) + } + if got, want := strings.TrimSpace(string(out)), "good"; got != want { + t.Errorf("output = %q; want %q", got, want) + } +} diff --git a/libgo/go/os/exec/exec_unix.go b/libgo/go/os/exec/exec_unix.go new file mode 100644 index 00000000000..9c3e17d23ab --- /dev/null +++ b/libgo/go/os/exec/exec_unix.go @@ -0,0 +1,24 @@ +// Copyright 2015 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build !plan9,!windows + +package exec + +import ( + "os" + "syscall" +) + +func init() { + skipStdinCopyError = func(err error) bool { + // Ignore EPIPE errors copying to stdin if the program + // completed successfully otherwise. + // See Issue 9173. + pe, ok := err.(*os.PathError) + return ok && + pe.Op == "write" && pe.Path == "|1" && + pe.Err == syscall.EPIPE + } +} diff --git a/libgo/go/os/exec/exec_windows.go b/libgo/go/os/exec/exec_windows.go new file mode 100644 index 00000000000..af8cd97218e --- /dev/null +++ b/libgo/go/os/exec/exec_windows.go @@ -0,0 +1,23 @@ +// Copyright 2017 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package exec + +import ( + "os" + "syscall" +) + +func init() { + skipStdinCopyError = func(err error) bool { + // Ignore ERROR_BROKEN_PIPE and ERROR_NO_DATA errors copying + // to stdin if the program completed successfully otherwise. + // See Issue 20445. + const _ERROR_NO_DATA = syscall.Errno(0xe8) + pe, ok := err.(*os.PathError) + return ok && + pe.Op == "write" && pe.Path == "|1" && + (pe.Err == syscall.ERROR_BROKEN_PIPE || pe.Err == _ERROR_NO_DATA) + } +} diff --git a/libgo/go/os/exec_windows.go b/libgo/go/os/exec_windows.go index d89db2022cc..d5d553a2f6b 100644 --- a/libgo/go/os/exec_windows.go +++ b/libgo/go/os/exec_windows.go @@ -97,17 +97,79 @@ func findProcess(pid int) (p *Process, err error) { } func init() { - var argc int32 - cmd := syscall.GetCommandLine() - argv, e := syscall.CommandLineToArgv(cmd, &argc) - if e != nil { - return + p := syscall.GetCommandLine() + cmd := syscall.UTF16ToString((*[0xffff]uint16)(unsafe.Pointer(p))[:]) + if len(cmd) == 0 { + arg0, _ := Executable() + Args = []string{arg0} + } else { + Args = commandLineToArgv(cmd) } - defer syscall.LocalFree(syscall.Handle(uintptr(unsafe.Pointer(argv)))) - Args = make([]string, argc) - for i, v := range (*argv)[:argc] { - Args[i] = syscall.UTF16ToString((*v)[:]) +} + +// appendBSBytes appends n '\\' bytes to b and returns the resulting slice. +func appendBSBytes(b []byte, n int) []byte { + for ; n > 0; n-- { + b = append(b, '\\') + } + return b +} + +// readNextArg splits command line string cmd into next +// argument and command line remainder. +func readNextArg(cmd string) (arg []byte, rest string) { + var b []byte + var inquote bool + var nslash int + for ; len(cmd) > 0; cmd = cmd[1:] { + c := cmd[0] + switch c { + case ' ', '\t': + if !inquote { + return appendBSBytes(b, nslash), cmd[1:] + } + case '"': + b = appendBSBytes(b, nslash/2) + if nslash%2 == 0 { + // use "Prior to 2008" rule from + // http://daviddeley.com/autohotkey/parameters/parameters.htm + // section 5.2 to deal with double double quotes + if inquote && len(cmd) > 1 && cmd[1] == '"' { + b = append(b, c) + cmd = cmd[1:] + } + inquote = !inquote + } else { + b = append(b, c) + } + nslash = 0 + continue + case '\\': + nslash++ + continue + } + b = appendBSBytes(b, nslash) + nslash = 0 + b = append(b, c) + } + return appendBSBytes(b, nslash), "" +} + +// commandLineToArgv splits a command line into individual argument +// strings, following the Windows conventions documented +// at http://daviddeley.com/autohotkey/parameters/parameters.htm#WINARGV +func commandLineToArgv(cmd string) []string { + var args []string + for len(cmd) > 0 { + if cmd[0] == ' ' || cmd[0] == '\t' { + cmd = cmd[1:] + continue + } + var arg []byte + arg, cmd = readNextArg(cmd) + args = append(args, string(arg)) } + return args } func ftToDuration(ft *syscall.Filetime) time.Duration { diff --git a/libgo/go/os/executable.go b/libgo/go/os/executable.go index 8c21246f5a8..17eed10bc9f 100644 --- a/libgo/go/os/executable.go +++ b/libgo/go/os/executable.go @@ -16,8 +16,7 @@ package os // The main use case is finding resources located relative to an // executable. // -// Executable is not supported on nacl or OpenBSD (unless procfs is -// mounted.) +// Executable is not supported on nacl. func Executable() (string, error) { return executable() } diff --git a/libgo/go/os/executable_path.go b/libgo/go/os/executable_path.go index 117320d7949..7b8b83652c8 100644 --- a/libgo/go/os/executable_path.go +++ b/libgo/go/os/executable_path.go @@ -2,7 +2,7 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -// +build aix +// +build aix openbsd package os @@ -12,18 +12,19 @@ package os var initWd, errWd = Getwd() func executable() (string, error) { - var err error var exePath string if len(Args) == 0 || Args[0] == "" { return "", ErrNotExist } - // Args[0] is an absolute path : this is the executable if IsPathSeparator(Args[0][0]) { + // Args[0] is an absolute path, so it is the executable. + // Note that we only need to worry about Unix paths here. exePath = Args[0] } else { for i := 1; i < len(Args[0]); i++ { - // Args[0] is a relative path : append current directory if IsPathSeparator(Args[0][i]) { + // Args[0] is a relative path: prepend the + // initial working directory. if errWd != nil { return "", errWd } @@ -33,18 +34,15 @@ func executable() (string, error) { } } if exePath != "" { - err = isExecutable(exePath) - if err == nil { - return exePath, nil + if err := isExecutable(exePath); err != nil { + return "", err } - // File does not exist or is not executable, - // this is an unexpected situation ! - return "", err + return exePath, nil } - // Search for executable in $PATH + // Search for executable in $PATH. for _, dir := range splitPathList(Getenv("PATH")) { if len(dir) == 0 { - continue + dir = "." } if !IsPathSeparator(dir[0]) { if errWd != nil { @@ -53,12 +51,11 @@ func executable() (string, error) { dir = initWd + string(PathSeparator) + dir } exePath = dir + string(PathSeparator) + Args[0] - err = isExecutable(exePath) - if err == nil { + switch isExecutable(exePath) { + case nil: return exePath, nil - } - if err == ErrPermission { - return "", err + case ErrPermission: + return "", ErrPermission } } return "", ErrNotExist @@ -74,15 +71,18 @@ func isExecutable(path string) error { if !mode.IsRegular() { return ErrPermission } - if (mode & 0111) != 0 { - return nil + if (mode & 0111) == 0 { + return ErrPermission } - return ErrPermission + return nil } // splitPathList splits a path list. // This is based on genSplit from strings/strings.go func splitPathList(pathList string) []string { + if pathList == "" { + return nil + } n := 1 for i := 0; i < len(pathList); i++ { if pathList[i] == PathListSeparator { diff --git a/libgo/go/os/executable_procfs.go b/libgo/go/os/executable_procfs.go index 69a70e18df0..b5fae590468 100644 --- a/libgo/go/os/executable_procfs.go +++ b/libgo/go/os/executable_procfs.go @@ -2,7 +2,7 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -// +build linux netbsd openbsd dragonfly nacl +// +build linux netbsd dragonfly nacl package os @@ -23,8 +23,6 @@ var executablePath, executablePathErr = func() (string, error) { procfn = "/proc/self/exe" case "netbsd": procfn = "/proc/curproc/exe" - case "openbsd": - procfn = "/proc/curproc/file" case "dragonfly": procfn = "/proc/curproc/file" } diff --git a/libgo/go/os/executable_test.go b/libgo/go/os/executable_test.go index a4d89092ac6..7800844e420 100644 --- a/libgo/go/os/executable_test.go +++ b/libgo/go/os/executable_test.go @@ -20,10 +20,6 @@ func TestExecutable(t *testing.T) { testenv.MustHaveExec(t) // will also execlude nacl, which doesn't support Executable anyway ep, err := os.Executable() if err != nil { - switch goos := runtime.GOOS; goos { - case "openbsd": // procfs is not mounted by default - t.Skipf("Executable failed on %s: %v, expected", goos, err) - } t.Fatalf("Executable failed: %v", err) } // we want fn to be of the form "dir/prog" @@ -32,6 +28,7 @@ func TestExecutable(t *testing.T) { if err != nil { t.Fatalf("filepath.Rel: %v", err) } + cmd := &osexec.Cmd{} // make child start with a relative program path cmd.Dir = dir @@ -39,6 +36,10 @@ func TestExecutable(t *testing.T) { // forge argv[0] for child, so that we can verify we could correctly // get real path of the executable without influenced by argv[0]. cmd.Args = []string{"-", "-test.run=XXXX"} + if runtime.GOOS == "openbsd" { + // OpenBSD relies on argv[0] + cmd.Args[0] = fn + } cmd.Env = append(os.Environ(), fmt.Sprintf("%s=1", executable_EnvVar)) out, err := cmd.CombinedOutput() if err != nil { diff --git a/libgo/go/os/export_windows_test.go b/libgo/go/os/export_windows_test.go index 3bb2d2015f7..f36fadb58bc 100644 --- a/libgo/go/os/export_windows_test.go +++ b/libgo/go/os/export_windows_test.go @@ -7,7 +7,7 @@ package os // Export for testing. var ( - FixLongPath = fixLongPath - NewConsoleFile = newConsoleFile - ReadConsoleFunc = &readConsole + FixLongPath = fixLongPath + NewConsoleFile = newConsoleFile + CommandLineToArgv = commandLineToArgv ) diff --git a/libgo/go/os/file.go b/libgo/go/os/file.go index d45a00b1239..4b4d8fb0367 100644 --- a/libgo/go/os/file.go +++ b/libgo/go/os/file.go @@ -37,6 +37,8 @@ package os import ( + "errors" + "internal/poll" "io" "syscall" ) @@ -99,13 +101,7 @@ func (f *File) Read(b []byte) (n int, err error) { return 0, err } n, e := f.read(b) - if n == 0 && len(b) > 0 && e == nil { - return 0, io.EOF - } - if e != nil { - err = &PathError{"read", f.name, e} - } - return n, err + return n, f.wrapErr("read", e) } // ReadAt reads len(b) bytes from the File starting at byte offset off. @@ -116,13 +112,15 @@ func (f *File) ReadAt(b []byte, off int64) (n int, err error) { if err := f.checkValid("read"); err != nil { return 0, err } + + if off < 0 { + return 0, &PathError{"readat", f.name, errors.New("negative offset")} + } + for len(b) > 0 { m, e := f.pread(b, off) - if m == 0 && e == nil { - return n, io.EOF - } if e != nil { - err = &PathError{"read", f.name, e} + err = f.wrapErr("read", e) break } n += m @@ -150,8 +148,9 @@ func (f *File) Write(b []byte) (n int, err error) { epipecheck(f, e) if e != nil { - err = &PathError{"write", f.name, e} + err = f.wrapErr("write", e) } + return n, err } @@ -162,10 +161,15 @@ func (f *File) WriteAt(b []byte, off int64) (n int, err error) { if err := f.checkValid("write"); err != nil { return 0, err } + + if off < 0 { + return 0, &PathError{"writeat", f.name, errors.New("negative offset")} + } + for len(b) > 0 { m, e := f.pwrite(b, off) if e != nil { - err = &PathError{"write", f.name, e} + err = f.wrapErr("write", e) break } n += m @@ -189,7 +193,7 @@ func (f *File) Seek(offset int64, whence int) (ret int64, err error) { e = syscall.EISDIR } if e != nil { - return 0, &PathError{"seek", f.name, e} + return 0, f.wrapErr("seek", e) } return r, nil } @@ -226,19 +230,6 @@ func Chdir(dir string) error { return nil } -// Chdir changes the current working directory to the file, -// which must be a directory. -// If there is an error, it will be of type *PathError. -func (f *File) Chdir() error { - if err := f.checkValid("chdir"); err != nil { - return err - } - if e := syscall.Fchdir(f.fd); e != nil { - return &PathError{"chdir", f.name, e} - } - return nil -} - // Open opens the named file for reading. If successful, methods on // the returned file can be used for reading; the associated file // descriptor has mode O_RDONLY. @@ -276,14 +267,52 @@ func fixCount(n int, err error) (int, error) { return n, err } -// checkValid checks whether f is valid for use. -// If not, it returns an appropriate error, perhaps incorporating the operation name op. -func (f *File) checkValid(op string) error { - if f == nil { - return ErrInvalid +// wrapErr wraps an error that occurred during an operation on an open file. +// It passes io.EOF through unchanged, otherwise converts +// poll.ErrFileClosing to ErrClosed and wraps the error in a PathError. +func (f *File) wrapErr(op string, err error) error { + if err == nil || err == io.EOF { + return err } - if f.fd == badFd { - return &PathError{op, f.name, ErrClosed} + if err == poll.ErrFileClosing { + err = ErrClosed } - return nil + return &PathError{op, f.name, err} +} + +// TempDir returns the default directory to use for temporary files. +// +// On Unix systems, it returns $TMPDIR if non-empty, else /tmp. +// On Windows, it uses GetTempPath, returning the first non-empty +// value from %TMP%, %TEMP%, %USERPROFILE%, or the Windows directory. +// On Plan 9, it returns /tmp. +// +// The directory is neither guaranteed to exist nor have accessible +// permissions. +func TempDir() string { + return tempDir() } + +// Chmod changes the mode of the named file to mode. +// If the file is a symbolic link, it changes the mode of the link's target. +// If there is an error, it will be of type *PathError. +// +// A different subset of the mode bits are used, depending on the +// operating system. +// +// On Unix, the mode's permission bits, ModeSetuid, ModeSetgid, and +// ModeSticky are used. +// +// On Windows, the mode must be non-zero but otherwise only the 0200 +// bit (owner writable) of mode is used; it controls whether the +// file's read-only attribute is set or cleared. attribute. The other +// bits are currently unused. Use mode 0400 for a read-only file and +// 0600 for a readable+writable file. +// +// On Plan 9, the mode's permission bits, ModeAppend, ModeExclusive, +// and ModeTemporary are used. +func Chmod(name string, mode FileMode) error { return chmod(name, mode) } + +// Chmod changes the mode of the file to mode. +// If there is an error, it will be of type *PathError. +func (f *File) Chmod(mode FileMode) error { return f.chmod(mode) } diff --git a/libgo/go/os/file_plan9.go b/libgo/go/os/file_plan9.go index 5276a7ec541..0f4a736c269 100644 --- a/libgo/go/os/file_plan9.go +++ b/libgo/go/os/file_plan9.go @@ -35,7 +35,9 @@ func (f *File) Fd() uintptr { return uintptr(f.fd) } -// NewFile returns a new File with the given file descriptor and name. +// NewFile returns a new File with the given file descriptor and +// name. The returned value will be nil if fd is not a valid file +// descriptor. func NewFile(fd uintptr, name string) *File { fdi := int(fd) if fdi < 0 { @@ -194,9 +196,7 @@ func (f *File) Truncate(size int64) error { const chmodMask = uint32(syscall.DMAPPEND | syscall.DMEXCL | syscall.DMTMP | ModePerm) -// Chmod changes the mode of the file to mode. -// If there is an error, it will be of type *PathError. -func (f *File) Chmod(mode FileMode) error { +func (f *File) chmod(mode FileMode) error { if f == nil { return ErrInvalid } @@ -244,14 +244,22 @@ func (f *File) Sync() error { // read reads up to len(b) bytes from the File. // It returns the number of bytes read and an error, if any. func (f *File) read(b []byte) (n int, err error) { - return fixCount(syscall.Read(f.fd, b)) + n, e := fixCount(syscall.Read(f.fd, b)) + if n == 0 && len(b) > 0 && e == nil { + return 0, io.EOF + } + return n, e } // pread reads len(b) bytes from the File starting at byte offset off. // It returns the number of bytes read and the error, if any. // EOF is signaled by a zero count with err set to nil. func (f *File) pread(b []byte, off int64) (n int, err error) { - return fixCount(syscall.Pread(f.fd, b, off)) + n, e := fixCount(syscall.Pread(f.fd, b, off)) + if n == 0 && len(b) > 0 && e == nil { + return 0, io.EOF + } + return n, e } // write writes len(b) bytes to the File. @@ -365,10 +373,8 @@ func rename(oldname, newname string) error { return nil } -// Chmod changes the mode of the named file to mode. -// If the file is a symbolic link, it changes the mode of the link's target. -// If there is an error, it will be of type *PathError. -func Chmod(name string, mode FileMode) error { +// See docs in file.go:Chmod. +func chmod(name string, mode FileMode) error { var d syscall.Dir odir, e := dirstat(name) @@ -468,7 +474,31 @@ func (f *File) Chown(uid, gid int) error { return &PathError{"chown", f.name, syscall.EPLAN9} } -// TempDir returns the default directory to use for temporary files. -func TempDir() string { +func tempDir() string { return "/tmp" } + +// Chdir changes the current working directory to the file, +// which must be a directory. +// If there is an error, it will be of type *PathError. +func (f *File) Chdir() error { + if err := f.checkValid("chdir"); err != nil { + return err + } + if e := syscall.Fchdir(f.fd); e != nil { + return &PathError{"chdir", f.name, e} + } + return nil +} + +// checkValid checks whether f is valid for use. +// If not, it returns an appropriate error, perhaps incorporating the operation name op. +func (f *File) checkValid(op string) error { + if f == nil { + return ErrInvalid + } + if f.fd == badFd { + return &PathError{op, f.name, ErrClosed} + } + return nil +} diff --git a/libgo/go/os/file_posix.go b/libgo/go/os/file_posix.go index 6634112efce..51cae9de6de 100644 --- a/libgo/go/os/file_posix.go +++ b/libgo/go/os/file_posix.go @@ -48,24 +48,21 @@ func syscallMode(i FileMode) (o uint32) { return } -// Chmod changes the mode of the named file to mode. -// If the file is a symbolic link, it changes the mode of the link's target. -// If there is an error, it will be of type *PathError. -func Chmod(name string, mode FileMode) error { - if e := syscall.Chmod(name, syscallMode(mode)); e != nil { +// See docs in file.go:Chmod. +func chmod(name string, mode FileMode) error { + if e := syscall.Chmod(fixLongPath(name), syscallMode(mode)); e != nil { return &PathError{"chmod", name, e} } return nil } -// Chmod changes the mode of the file to mode. -// If there is an error, it will be of type *PathError. -func (f *File) Chmod(mode FileMode) error { +// See docs in file.go:(*File).Chmod. +func (f *File) chmod(mode FileMode) error { if err := f.checkValid("chmod"); err != nil { return err } - if e := syscall.Fchmod(f.fd, syscallMode(mode)); e != nil { - return &PathError{"chmod", f.name, e} + if e := f.pfd.Fchmod(syscallMode(mode)); e != nil { + return f.wrapErr("chmod", e) } return nil } @@ -73,6 +70,9 @@ func (f *File) Chmod(mode FileMode) error { // Chown changes the numeric uid and gid of the named file. // If the file is a symbolic link, it changes the uid and gid of the link's target. // If there is an error, it will be of type *PathError. +// +// On Windows, it always returns the syscall.EWINDOWS error, wrapped +// in *PathError. func Chown(name string, uid, gid int) error { if e := syscall.Chown(name, uid, gid); e != nil { return &PathError{"chown", name, e} @@ -83,6 +83,9 @@ func Chown(name string, uid, gid int) error { // Lchown changes the numeric uid and gid of the named file. // If the file is a symbolic link, it changes the uid and gid of the link itself. // If there is an error, it will be of type *PathError. +// +// On Windows, it always returns the syscall.EWINDOWS error, wrapped +// in *PathError. func Lchown(name string, uid, gid int) error { if e := syscall.Lchown(name, uid, gid); e != nil { return &PathError{"lchown", name, e} @@ -92,12 +95,15 @@ func Lchown(name string, uid, gid int) error { // Chown changes the numeric uid and gid of the named file. // If there is an error, it will be of type *PathError. +// +// On Windows, it always returns the syscall.EWINDOWS error, wrapped +// in *PathError. func (f *File) Chown(uid, gid int) error { if err := f.checkValid("chown"); err != nil { return err } - if e := syscall.Fchown(f.fd, uid, gid); e != nil { - return &PathError{"chown", f.name, e} + if e := f.pfd.Fchown(uid, gid); e != nil { + return f.wrapErr("chown", e) } return nil } @@ -109,8 +115,8 @@ func (f *File) Truncate(size int64) error { if err := f.checkValid("truncate"); err != nil { return err } - if e := syscall.Ftruncate(f.fd, size); e != nil { - return &PathError{"truncate", f.name, e} + if e := f.pfd.Ftruncate(size); e != nil { + return f.wrapErr("truncate", e) } return nil } @@ -122,8 +128,8 @@ func (f *File) Sync() error { if err := f.checkValid("sync"); err != nil { return err } - if e := syscall.Fsync(f.fd); e != nil { - return &PathError{"sync", f.name, e} + if e := f.pfd.Fsync(); e != nil { + return f.wrapErr("sync", e) } return nil } @@ -143,3 +149,25 @@ func Chtimes(name string, atime time.Time, mtime time.Time) error { } return nil } + +// Chdir changes the current working directory to the file, +// which must be a directory. +// If there is an error, it will be of type *PathError. +func (f *File) Chdir() error { + if err := f.checkValid("chdir"); err != nil { + return err + } + if e := f.pfd.Fchdir(); e != nil { + return f.wrapErr("chdir", e) + } + return nil +} + +// checkValid checks whether f is valid for use. +// If not, it returns an appropriate error, perhaps incorporating the operation name op. +func (f *File) checkValid(op string) error { + if f == nil { + return ErrInvalid + } + return nil +} diff --git a/libgo/go/os/file_unix.go b/libgo/go/os/file_unix.go index 1bba4ed9d63..819999409a9 100644 --- a/libgo/go/os/file_unix.go +++ b/libgo/go/os/file_unix.go @@ -7,6 +7,7 @@ package os import ( + "internal/poll" "runtime" "syscall" ) @@ -19,11 +20,22 @@ func fixLongPath(path string) string { func rename(oldname, newname string) error { fi, err := Lstat(newname) if err == nil && fi.IsDir() { + // There are two independent errors this function can return: + // one for a bad oldname, and one for a bad newname. + // At this point we've determined the newname is bad. + // But just in case oldname is also bad, prioritize returning + // the oldname error because that's what we did historically. + if _, err := Lstat(oldname); err != nil { + if pe, ok := err.(*PathError); ok { + err = pe.Err + } + return &LinkError{"rename", oldname, newname, err} + } return &LinkError{"rename", oldname, newname, syscall.EEXIST} } - e := syscall.Rename(oldname, newname) - if e != nil { - return &LinkError{"rename", oldname, newname, e} + err = syscall.Rename(oldname, newname) + if err != nil { + return &LinkError{"rename", oldname, newname, err} } return nil } @@ -33,9 +45,10 @@ func rename(oldname, newname string) error { // can overwrite this data, which could cause the finalizer // to close the wrong file descriptor. type file struct { - fd int - name string - dirinfo *dirInfo // nil unless directory being read + pfd poll.FD + name string + dirinfo *dirInfo // nil unless directory being read + nonblock bool // whether we set nonblocking mode } // Fd returns the integer Unix file descriptor referencing the open file. @@ -44,16 +57,64 @@ func (f *File) Fd() uintptr { if f == nil { return ^(uintptr(0)) } - return uintptr(f.fd) + + // If we put the file descriptor into nonblocking mode, + // then set it to blocking mode before we return it, + // because historically we have always returned a descriptor + // opened in blocking mode. The File will continue to work, + // but any blocking operation will tie up a thread. + if f.nonblock { + syscall.SetNonblock(f.pfd.Sysfd, false) + } + + return uintptr(f.pfd.Sysfd) } -// NewFile returns a new File with the given file descriptor and name. +// NewFile returns a new File with the given file descriptor and +// name. The returned value will be nil if fd is not a valid file +// descriptor. func NewFile(fd uintptr, name string) *File { + return newFile(fd, name, false) +} + +// newFile is like NewFile, but if pollable is true it tries to add the +// file to the runtime poller. +func newFile(fd uintptr, name string, pollable bool) *File { fdi := int(fd) if fdi < 0 { return nil } - f := &File{&file{fd: fdi, name: name}} + f := &File{&file{ + pfd: poll.FD{ + Sysfd: fdi, + IsStream: true, + ZeroReadIsEOF: true, + }, + name: name, + }} + + // Don't try to use kqueue with regular files on FreeBSD. + // It crashes the system unpredictably while running all.bash. + // Issue 19093. + if runtime.GOOS == "freebsd" { + pollable = false + } + + if err := f.pfd.Init("file", pollable); err != nil { + // An error here indicates a failure to register + // with the netpoll system. That can happen for + // a file descriptor that is not supported by + // epoll/kqueue; for example, disk files on + // GNU/Linux systems. We assume that any real error + // will show up in later I/O. + } else if pollable { + // We successfully registered with netpoll, so put + // the file into nonblocking mode. + if err := syscall.SetNonblock(fdi, true); err == nil { + f.nonblock = true + } + } + runtime.SetFinalizer(f.file, (*file).close) return f } @@ -68,7 +129,7 @@ type dirInfo struct { // output or standard error. See the SIGPIPE docs in os/signal, and // issue 11845. func epipecheck(file *File, e error) { - if e == syscall.EPIPE && (file.fd == 1 || file.fd == 2) { + if e == syscall.EPIPE && (file.pfd.Sysfd == 1 || file.pfd.Sysfd == 2) { sigpipe() } } @@ -119,7 +180,7 @@ func OpenFile(name string, flag int, perm FileMode) (*File, error) { syscall.CloseOnExec(r) } - return NewFile(uintptr(r), name), nil + return newFile(uintptr(r), name, true), nil } // Close closes the File, rendering it unusable for I/O. @@ -132,11 +193,14 @@ func (f *File) Close() error { } func (file *file) close() error { - if file == nil || file.fd == badFd { + if file == nil { return syscall.EINVAL } var err error - if e := syscall.Close(file.fd); e != nil { + if e := file.pfd.Close(); e != nil { + if e == poll.ErrFileClosing { + e = ErrClosed + } err = &PathError{"close", file.name, e} } @@ -151,76 +215,42 @@ func (file *file) close() error { } } - file.fd = -1 // so it can't be closed again - // no need for a finalizer anymore runtime.SetFinalizer(file, nil) return err } -// Darwin and FreeBSD can't read or write 2GB+ at a time, -// even on 64-bit systems. See golang.org/issue/7812. -// Use 1GB instead of, say, 2GB-1, to keep subsequent -// reads aligned. -const ( - needsMaxRW = runtime.GOOS == "darwin" || runtime.GOOS == "freebsd" - maxRW = 1 << 30 -) - // read reads up to len(b) bytes from the File. // It returns the number of bytes read and an error, if any. func (f *File) read(b []byte) (n int, err error) { - if needsMaxRW && len(b) > maxRW { - b = b[:maxRW] - } - return fixCount(syscall.Read(f.fd, b)) + n, err = f.pfd.Read(b) + runtime.KeepAlive(f) + return n, err } // pread reads len(b) bytes from the File starting at byte offset off. // It returns the number of bytes read and the error, if any. // EOF is signaled by a zero count with err set to nil. func (f *File) pread(b []byte, off int64) (n int, err error) { - if needsMaxRW && len(b) > maxRW { - b = b[:maxRW] - } - return fixCount(syscall.Pread(f.fd, b, off)) + n, err = f.pfd.Pread(b, off) + runtime.KeepAlive(f) + return n, err } // write writes len(b) bytes to the File. // It returns the number of bytes written and an error, if any. func (f *File) write(b []byte) (n int, err error) { - for { - bcap := b - if needsMaxRW && len(bcap) > maxRW { - bcap = bcap[:maxRW] - } - m, err := fixCount(syscall.Write(f.fd, bcap)) - n += m - - // If the syscall wrote some data but not all (short write) - // or it returned EINTR, then assume it stopped early for - // reasons that are uninteresting to the caller, and try again. - if 0 < m && m < len(bcap) || err == syscall.EINTR { - b = b[m:] - continue - } - - if needsMaxRW && len(bcap) != len(b) && err == nil { - b = b[m:] - continue - } - - return n, err - } + n, err = f.pfd.Write(b) + runtime.KeepAlive(f) + return n, err } // pwrite writes len(b) bytes to the File starting at byte offset off. // It returns the number of bytes written and an error, if any. func (f *File) pwrite(b []byte, off int64) (n int, err error) { - if needsMaxRW && len(b) > maxRW { - b = b[:maxRW] - } - return fixCount(syscall.Pwrite(f.fd, b, off)) + n, err = f.pfd.Pwrite(b, off) + runtime.KeepAlive(f) + return n, err } // seek sets the offset for the next Read or Write on file to offset, interpreted @@ -228,7 +258,9 @@ func (f *File) pwrite(b []byte, off int64) (n int, err error) { // relative to the current offset, and 2 means relative to the end. // It returns the new offset and an error, if any. func (f *File) seek(offset int64, whence int) (ret int64, err error) { - return syscall.Seek(f.fd, offset, whence) + ret, err = f.pfd.Seek(offset, whence) + runtime.KeepAlive(f) + return ret, err } // Truncate changes the size of the named file. @@ -272,8 +304,7 @@ func Remove(name string) error { return &PathError{"remove", name, e} } -// TempDir returns the default directory to use for temporary files. -func TempDir() string { +func tempDir() string { dir := Getenv("TMPDIR") if dir == "" { if runtime.GOOS == "android" { diff --git a/libgo/go/os/os_test.go b/libgo/go/os/os_test.go index dcc8d762bf8..0f1617ad5d0 100644 --- a/libgo/go/os/os_test.go +++ b/libgo/go/os/os_test.go @@ -17,6 +17,7 @@ import ( "path/filepath" "reflect" "runtime" + "runtime/debug" "sort" "strings" "sync" @@ -52,15 +53,12 @@ var sysdir = func() *sysDir { case "darwin": switch runtime.GOARCH { case "arm", "arm64": - /// At this point the test harness has not had a chance - // to move us into the ./src/os directory, so the - // current working directory is the root of the app. wd, err := syscall.Getwd() if err != nil { wd = err.Error() } return &sysDir{ - wd, + filepath.Join(wd, "..", ".."), []string{ "ResourceRules.plist", "Info.plist", @@ -110,7 +108,7 @@ func size(name string, t *testing.T) int64 { break } if e != nil { - t.Fatal("read failed:", err) + t.Fatal("read failed:", e) } } return int64(len) @@ -174,6 +172,45 @@ func TestStat(t *testing.T) { } } +func TestStatError(t *testing.T) { + defer chtmpdir(t)() + + path := "no-such-file" + Remove(path) // Just in case + + fi, err := Stat(path) + if err == nil { + t.Fatal("got nil, want error") + } + if fi != nil { + t.Errorf("got %v, want nil", fi) + } + if perr, ok := err.(*PathError); !ok { + t.Errorf("got %T, want %T", err, perr) + } + + testenv.MustHaveSymlink(t) + + link := "symlink" + Remove(link) // Just in case + err = Symlink(path, link) + if err != nil { + t.Fatal(err) + } + defer Remove(link) + + fi, err = Stat(link) + if err == nil { + t.Fatal("got nil, want error") + } + if fi != nil { + t.Errorf("got %v, want nil", fi) + } + if perr, ok := err.(*PathError); !ok { + t.Errorf("got %T, want %T", err, perr) + } +} + func TestFstat(t *testing.T) { path := sfdir + "/" + sfname file, err1 := Open(path) @@ -359,6 +396,50 @@ func BenchmarkReaddir(b *testing.B) { benchmarkReaddir(".", b) } +func benchmarkStat(b *testing.B, path string) { + b.ResetTimer() + for i := 0; i < b.N; i++ { + _, err := Stat(path) + if err != nil { + b.Fatalf("Stat(%q) failed: %v", path, err) + } + } +} + +func benchmarkLstat(b *testing.B, path string) { + b.ResetTimer() + for i := 0; i < b.N; i++ { + _, err := Lstat(path) + if err != nil { + b.Fatalf("Lstat(%q) failed: %v", path, err) + } + } +} + +func BenchmarkStatDot(b *testing.B) { + benchmarkStat(b, ".") +} + +func BenchmarkStatFile(b *testing.B) { + benchmarkStat(b, filepath.Join(runtime.GOROOT(), "src/os/os_test.go")) +} + +func BenchmarkStatDir(b *testing.B) { + benchmarkStat(b, filepath.Join(runtime.GOROOT(), "src/os")) +} + +func BenchmarkLstatDot(b *testing.B) { + benchmarkLstat(b, ".") +} + +func BenchmarkLstatFile(b *testing.B) { + benchmarkLstat(b, filepath.Join(runtime.GOROOT(), "src/os/os_test.go")) +} + +func BenchmarkLstatDir(b *testing.B) { + benchmarkLstat(b, filepath.Join(runtime.GOROOT(), "src/os")) +} + // Read the directory one entry at a time. func smallReaddirnames(file *File, length int, t *testing.T) []string { names := make([]string, length) @@ -673,55 +754,58 @@ func TestSymlink(t *testing.T) { Remove(from) // Just in case. file, err := Create(to) if err != nil { - t.Fatalf("open %q failed: %v", to, err) + t.Fatalf("Create(%q) failed: %v", to, err) } defer Remove(to) if err = file.Close(); err != nil { - t.Errorf("close %q failed: %v", to, err) + t.Errorf("Close(%q) failed: %v", to, err) } err = Symlink(to, from) if err != nil { - t.Fatalf("symlink %q, %q failed: %v", to, from, err) + t.Fatalf("Symlink(%q, %q) failed: %v", to, from, err) } defer Remove(from) tostat, err := Lstat(to) if err != nil { - t.Fatalf("stat %q failed: %v", to, err) + t.Fatalf("Lstat(%q) failed: %v", to, err) } if tostat.Mode()&ModeSymlink != 0 { - t.Fatalf("stat %q claims to have found a symlink", to) + t.Fatalf("Lstat(%q).Mode()&ModeSymlink = %v, want 0", to, tostat.Mode()&ModeSymlink) } fromstat, err := Stat(from) if err != nil { - t.Fatalf("stat %q failed: %v", from, err) + t.Fatalf("Stat(%q) failed: %v", from, err) } if !SameFile(tostat, fromstat) { - t.Errorf("symlink %q, %q did not create symlink", to, from) + t.Errorf("Symlink(%q, %q) did not create symlink", to, from) } fromstat, err = Lstat(from) if err != nil { - t.Fatalf("lstat %q failed: %v", from, err) + t.Fatalf("Lstat(%q) failed: %v", from, err) } if fromstat.Mode()&ModeSymlink == 0 { - t.Fatalf("symlink %q, %q did not create symlink", to, from) + t.Fatalf("Lstat(%q).Mode()&ModeSymlink = 0, want %v", from, ModeSymlink) } fromstat, err = Stat(from) if err != nil { - t.Fatalf("stat %q failed: %v", from, err) + t.Fatalf("Stat(%q) failed: %v", from, err) + } + if fromstat.Name() != from { + t.Errorf("Stat(%q).Name() = %q, want %q", from, fromstat.Name(), from) } if fromstat.Mode()&ModeSymlink != 0 { - t.Fatalf("stat %q did not follow symlink", from) + t.Fatalf("Stat(%q).Mode()&ModeSymlink = %v, want 0", from, fromstat.Mode()&ModeSymlink) } s, err := Readlink(from) if err != nil { - t.Fatalf("readlink %q failed: %v", from, err) + t.Fatalf("Readlink(%q) failed: %v", from, err) } if s != to { - t.Fatalf("after symlink %q != %q", s, to) + t.Fatalf("Readlink(%q) = %q, want %q", from, s, to) } file, err = Open(from) if err != nil { - t.Fatalf("open %q failed: %v", from, err) + t.Fatalf("Open(%q) failed: %v", from, err) } file.Close() } @@ -844,6 +928,18 @@ func TestRenameFailed(t *testing.T) { } } +func TestRenameNotExisting(t *testing.T) { + defer chtmpdir(t)() + from, to := "doesnt-exist", "dest" + + Mkdir(to, 0777) + defer Remove(to) + + if err := Rename(from, to); !IsNotExist(err) { + t.Errorf("Rename(%q, %q) = %v; want an IsNotExist error", from, to, err) + } +} + func TestRenameToDirFailed(t *testing.T) { defer chtmpdir(t)() from, to := "renamefrom", "renameto" @@ -1054,14 +1150,22 @@ func testChtimes(t *testing.T, name string) { } postStat := st - /* Plan 9, NaCl: - Mtime is the time of the last change of content. Similarly, atime is set whenever the - contents are accessed; also, it is set whenever mtime is set. - */ pat := Atime(postStat) pmt := postStat.ModTime() - if !pat.Before(at) && runtime.GOOS != "plan9" && runtime.GOOS != "nacl" { - t.Errorf("AccessTime didn't go backwards; was=%d, after=%d", at, pat) + if !pat.Before(at) { + switch runtime.GOOS { + case "plan9", "nacl": + // Ignore. + // Plan 9, NaCl: + // Mtime is the time of the last change of + // content. Similarly, atime is set whenever + // the contents are accessed; also, it is set + // whenever mtime is set. + case "netbsd": + t.Logf("AccessTime didn't go backwards; was=%d, after=%d (Ignoring. See NetBSD issue golang.org/issue/19293)", at, pat) + default: + t.Errorf("AccessTime didn't go backwards; was=%d, after=%d", at, pat) + } } if !pmt.Before(mt) { @@ -1239,6 +1343,32 @@ func TestSeek(t *testing.T) { } } +func TestSeekError(t *testing.T) { + switch runtime.GOOS { + case "plan9", "nacl": + t.Skipf("skipping test on %v", runtime.GOOS) + } + + r, w, err := Pipe() + if err != nil { + t.Fatal(err) + } + _, err = r.Seek(0, 0) + if err == nil { + t.Fatal("Seek on pipe should fail") + } + if perr, ok := err.(*PathError); !ok || perr.Err != syscall.ESPIPE { + t.Errorf("Seek returned error %v, want &PathError{Err: syscall.ESPIPE}", err) + } + _, err = w.Seek(0, 0) + if err == nil { + t.Fatal("Seek on pipe should fail") + } + if perr, ok := err.(*PathError); !ok || perr.Err != syscall.ESPIPE { + t.Errorf("Seek returned error %v, want &PathError{Err: syscall.ESPIPE}", err) + } +} + type openErrorTest struct { path string mode int @@ -1443,6 +1573,26 @@ func TestReadAtOffset(t *testing.T) { } } +// Verify that ReadAt doesn't allow negative offset. +func TestReadAtNegativeOffset(t *testing.T) { + f := newFile("TestReadAtNegativeOffset", t) + defer Remove(f.Name()) + defer f.Close() + + const data = "hello, world\n" + io.WriteString(f, data) + + f.Seek(0, 0) + b := make([]byte, 5) + + n, err := f.ReadAt(b, -10) + + const wantsub = "negative offset" + if !strings.Contains(fmt.Sprint(err), wantsub) || n != 0 { + t.Errorf("ReadAt(-10) = %v, %v; want 0, ...%q...", n, err, wantsub) + } +} + func TestWriteAt(t *testing.T) { f := newFile("TestWriteAt", t) defer Remove(f.Name()) @@ -1465,6 +1615,20 @@ func TestWriteAt(t *testing.T) { } } +// Verify that WriteAt doesn't allow negative offset. +func TestWriteAtNegativeOffset(t *testing.T) { + f := newFile("TestWriteAtNegativeOffset", t) + defer Remove(f.Name()) + defer f.Close() + + n, err := f.WriteAt([]byte("WORLD"), -10) + + const wantsub = "negative offset" + if !strings.Contains(fmt.Sprint(err), wantsub) || n != 0 { + t.Errorf("WriteAt(-10) = %v, %v; want 0, ...%q...", n, err, wantsub) + } +} + func writeFile(t *testing.T, fname string, flag int, text string) string { f, err := OpenFile(fname, flag, 0666) if err != nil { @@ -1667,6 +1831,17 @@ func TestStatStdin(t *testing.T) { Exit(0) } + fi, err := Stdin.Stat() + if err != nil { + t.Fatal(err) + } + switch mode := fi.Mode(); { + case mode&ModeCharDevice != 0: + case mode&ModeNamedPipe != 0: + default: + t.Fatalf("unexpected Stdin mode (%v), want ModeCharDevice or ModeNamedPipe", mode) + } + var cmd *osexec.Cmd if runtime.GOOS == "windows" { cmd = osexec.Command("cmd", "/c", "echo output | "+Args[0]+" -test.run=TestStatStdin") @@ -1686,6 +1861,60 @@ func TestStatStdin(t *testing.T) { } } +func TestStatRelativeSymlink(t *testing.T) { + testenv.MustHaveSymlink(t) + + tmpdir, err := ioutil.TempDir("", "TestStatRelativeSymlink") + if err != nil { + t.Fatal(err) + } + defer RemoveAll(tmpdir) + + target := filepath.Join(tmpdir, "target") + f, err := Create(target) + if err != nil { + t.Fatal(err) + } + defer f.Close() + + st, err := f.Stat() + if err != nil { + t.Fatal(err) + } + + link := filepath.Join(tmpdir, "link") + err = Symlink(filepath.Base(target), link) + if err != nil { + t.Fatal(err) + } + + st1, err := Stat(link) + if err != nil { + t.Fatal(err) + } + + if !SameFile(st, st1) { + t.Error("Stat doesn't follow relative symlink") + } + + if runtime.GOOS == "windows" { + Remove(link) + err = Symlink(target[len(filepath.VolumeName(target)):], link) + if err != nil { + t.Fatal(err) + } + + st1, err := Stat(link) + if err != nil { + t.Fatal(err) + } + + if !SameFile(st, st1) { + t.Error("Stat doesn't follow relative symlink") + } + } +} + func TestReadAtEOF(t *testing.T) { f := newFile("TestReadAtEOF", t) defer Remove(f.Name()) @@ -1759,6 +1988,10 @@ func TestLongPath(t *testing.T) { if dir.Size() != filesize || filesize != wantSize { t.Errorf("Size(%q) is %d, len(ReadFile()) is %d, want %d", path, dir.Size(), filesize, wantSize) } + err = Chmod(path, dir.Mode()) + if err != nil { + t.Fatalf("Chmod(%q) failed: %v", path, err) + } } if err := Truncate(sizedTempDir+"/bar.txt", 0); err != nil { t.Fatalf("Truncate failed: %v", err) @@ -1927,3 +2160,99 @@ func TestRemoveAllRace(t *testing.T) { close(hold) // let workers race to remove root wg.Wait() } + +// Test that reading from a pipe doesn't use up a thread. +func TestPipeThreads(t *testing.T) { + switch runtime.GOOS { + case "freebsd": + t.Skip("skipping on FreeBSD; issue 19093") + case "solaris": + t.Skip("skipping on Solaris; issue 19111") + case "windows": + t.Skip("skipping on Windows; issue 19098") + case "plan9": + t.Skip("skipping on Plan 9; does not support runtime poller") + } + + threads := 100 + + // OpenBSD has a low default for max number of files. + if runtime.GOOS == "openbsd" { + threads = 50 + } + + r := make([]*File, threads) + w := make([]*File, threads) + for i := 0; i < threads; i++ { + rp, wp, err := Pipe() + if err != nil { + for j := 0; j < i; j++ { + r[j].Close() + w[j].Close() + } + t.Fatal(err) + } + r[i] = rp + w[i] = wp + } + + defer debug.SetMaxThreads(debug.SetMaxThreads(threads / 2)) + + var wg sync.WaitGroup + wg.Add(threads) + c := make(chan bool, threads) + for i := 0; i < threads; i++ { + go func(i int) { + defer wg.Done() + var b [1]byte + c <- true + if _, err := r[i].Read(b[:]); err != nil { + t.Error(err) + } + }(i) + } + + for i := 0; i < threads; i++ { + <-c + } + + // If we are still alive, it means that the 100 goroutines did + // not require 100 threads. + + for i := 0; i < threads; i++ { + if _, err := w[i].Write([]byte{0}); err != nil { + t.Error(err) + } + if err := w[i].Close(); err != nil { + t.Error(err) + } + } + + wg.Wait() + + for i := 0; i < threads; i++ { + if err := r[i].Close(); err != nil { + t.Error(err) + } + } +} + +func TestDoubleCloseError(t *testing.T) { + path := sfdir + "/" + sfname + file, err := Open(path) + if err != nil { + t.Fatal(err) + } + if err := file.Close(); err != nil { + t.Fatalf("unexpected error from Close: %v", err) + } + if err := file.Close(); err == nil { + t.Error("second Close did not fail") + } else if pe, ok := err.(*PathError); !ok { + t.Errorf("second Close returned unexpected error type %T; expected os.PathError", pe) + } else if pe.Err != ErrClosed { + t.Errorf("second Close returned %q, wanted %q", err, ErrClosed) + } else { + t.Logf("second close returned expected error %q", err) + } +} diff --git a/libgo/go/os/pipe_bsd.go b/libgo/go/os/pipe_bsd.go index ebe198bb6a8..ae153fa00d6 100644 --- a/libgo/go/os/pipe_bsd.go +++ b/libgo/go/os/pipe_bsd.go @@ -2,7 +2,7 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -// +build aix darwin dragonfly freebsd nacl netbsd openbsd solaris +// +build aix darwin dragonfly nacl netbsd openbsd solaris package os @@ -24,5 +24,5 @@ func Pipe() (r *File, w *File, err error) { syscall.CloseOnExec(p[1]) syscall.ForkLock.RUnlock() - return NewFile(uintptr(p[0]), "|0"), NewFile(uintptr(p[1]), "|1"), nil + return newFile(uintptr(p[0]), "|0", true), newFile(uintptr(p[1]), "|1", true), nil } diff --git a/libgo/go/os/pipe_freebsd.go b/libgo/go/os/pipe_freebsd.go new file mode 100644 index 00000000000..ea6622cd260 --- /dev/null +++ b/libgo/go/os/pipe_freebsd.go @@ -0,0 +1,34 @@ +// Copyright 2017 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package os + +import "syscall" + +// Pipe returns a connected pair of Files; reads from r return bytes written to w. +// It returns the files and an error, if any. +func Pipe() (r *File, w *File, err error) { + var p [2]int + + e := syscall.Pipe2(p[0:], syscall.O_CLOEXEC) + if e != nil { + // Fallback support for FreeBSD 9, which lacks Pipe2. + // + // TODO: remove this for Go 1.10 when FreeBSD 9 + // is removed (Issue 19072). + + // See ../syscall/exec.go for description of lock. + syscall.ForkLock.RLock() + e := syscall.Pipe(p[0:]) + if e != nil { + syscall.ForkLock.RUnlock() + return nil, nil, NewSyscallError("pipe", e) + } + syscall.CloseOnExec(p[0]) + syscall.CloseOnExec(p[1]) + syscall.ForkLock.RUnlock() + } + + return newFile(uintptr(p[0]), "|0", true), newFile(uintptr(p[1]), "|1", true), nil +} diff --git a/libgo/go/os/pipe_linux.go b/libgo/go/os/pipe_linux.go index 9bafad84f9f..96f2ce900cb 100644 --- a/libgo/go/os/pipe_linux.go +++ b/libgo/go/os/pipe_linux.go @@ -29,5 +29,5 @@ func Pipe() (r *File, w *File, err error) { return nil, nil, NewSyscallError("pipe2", e) } - return NewFile(uintptr(p[0]), "|0"), NewFile(uintptr(p[1]), "|1"), nil + return newFile(uintptr(p[0]), "|0", true), newFile(uintptr(p[1]), "|1", true), nil } diff --git a/libgo/go/os/pipe_test.go b/libgo/go/os/pipe_test.go index 74cce80ee4b..9d79d84575d 100644 --- a/libgo/go/os/pipe_test.go +++ b/libgo/go/os/pipe_test.go @@ -10,11 +10,16 @@ package os_test import ( "fmt" "internal/testenv" + "io/ioutil" "os" osexec "os/exec" "os/signal" + "runtime" + "strconv" + "strings" "syscall" "testing" + "time" ) func TestEPIPE(t *testing.T) { @@ -82,7 +87,7 @@ func TestStdPipe(t *testing.T) { t.Errorf("unexpected SIGPIPE signal for descriptor %d sig %t", dest, sig) } } else { - t.Errorf("unexpected exit status %v for descriptor %ds sig %t", err, dest, sig) + t.Errorf("unexpected exit status %v for descriptor %d sig %t", err, dest, sig) } } } @@ -111,3 +116,107 @@ func TestStdPipeHelper(t *testing.T) { // For descriptor 3, a normal exit is expected. os.Exit(0) } + +func testClosedPipeRace(t *testing.T, read bool) { + switch runtime.GOOS { + case "freebsd": + t.Skip("FreeBSD does not use the poller; issue 19093") + } + + limit := 1 + if !read { + // Get the amount we have to write to overload a pipe + // with no reader. + limit = 65537 + if b, err := ioutil.ReadFile("/proc/sys/fs/pipe-max-size"); err == nil { + if i, err := strconv.Atoi(strings.TrimSpace(string(b))); err == nil { + limit = i + 1 + } + } + t.Logf("using pipe write limit of %d", limit) + } + + r, w, err := os.Pipe() + if err != nil { + t.Fatal(err) + } + defer r.Close() + defer w.Close() + + // Close the read end of the pipe in a goroutine while we are + // writing to the write end, or vice-versa. + go func() { + // Give the main goroutine a chance to enter the Read or + // Write call. This is sloppy but the test will pass even + // if we close before the read/write. + time.Sleep(20 * time.Millisecond) + + var err error + if read { + err = r.Close() + } else { + err = w.Close() + } + if err != nil { + t.Error(err) + } + }() + + b := make([]byte, limit) + if read { + _, err = r.Read(b[:]) + } else { + _, err = w.Write(b[:]) + } + if err == nil { + t.Error("I/O on closed pipe unexpectedly succeeded") + } else if pe, ok := err.(*os.PathError); !ok { + t.Errorf("I/O on closed pipe returned unexpected error type %T; expected os.PathError", pe) + } else if pe.Err != os.ErrClosed { + t.Errorf("got error %q but expected %q", pe.Err, os.ErrClosed) + } else { + t.Logf("I/O returned expected error %q", err) + } +} + +func TestClosedPipeRaceRead(t *testing.T) { + testClosedPipeRace(t, true) +} + +func TestClosedPipeRaceWrite(t *testing.T) { + testClosedPipeRace(t, false) +} + +// Issue 20915: Reading on nonblocking fd should not return "waiting +// for unsupported file type." Currently it returns EAGAIN; it is +// possible that in the future it will simply wait for data. +func TestReadNonblockingFd(t *testing.T) { + if os.Getenv("GO_WANT_READ_NONBLOCKING_FD") == "1" { + fd := int(os.Stdin.Fd()) + syscall.SetNonblock(fd, true) + defer syscall.SetNonblock(fd, false) + _, err := os.Stdin.Read(make([]byte, 1)) + if err != nil { + if perr, ok := err.(*os.PathError); !ok || perr.Err != syscall.EAGAIN { + t.Fatalf("read on nonblocking stdin got %q, should have gotten EAGAIN", err) + } + } + os.Exit(0) + } + + testenv.MustHaveExec(t) + r, w, err := os.Pipe() + if err != nil { + t.Fatal(err) + } + defer r.Close() + defer w.Close() + cmd := osexec.Command(os.Args[0], "-test.run="+t.Name()) + cmd.Env = append(os.Environ(), "GO_WANT_READ_NONBLOCKING_FD=1") + cmd.Stdin = r + output, err := cmd.CombinedOutput() + t.Logf("%s", output) + if err != nil { + t.Errorf("child process failed: %v", err) + } +} diff --git a/libgo/go/os/proc.go b/libgo/go/os/proc.go index 33a8b26f78d..804128a1da4 100644 --- a/libgo/go/os/proc.go +++ b/libgo/go/os/proc.go @@ -25,18 +25,29 @@ func init() { func runtime_args() []string // in package runtime // Getuid returns the numeric user id of the caller. +// +// On Windows, it returns -1. func Getuid() int { return syscall.Getuid() } // Geteuid returns the numeric effective user id of the caller. +// +// On Windows, it returns -1. func Geteuid() int { return syscall.Geteuid() } // Getgid returns the numeric group id of the caller. +// +// On Windows, it returns -1. func Getgid() int { return syscall.Getgid() } // Getegid returns the numeric effective group id of the caller. +// +// On Windows, it returns -1. func Getegid() int { return syscall.Getegid() } // Getgroups returns a list of the numeric ids of groups that the caller belongs to. +// +// On Windows, it returns syscall.EWINDOWS. See the os/user package +// for a possible alternative. func Getgroups() ([]int, error) { gids, e := syscall.Getgroups() return gids, NewSyscallError("getgroups", e) diff --git a/libgo/go/os/signal/doc.go b/libgo/go/os/signal/doc.go index 73b01a2764d..16f49c7ab8b 100644 --- a/libgo/go/os/signal/doc.go +++ b/libgo/go/os/signal/doc.go @@ -181,10 +181,11 @@ If the Go runtime sees an existing signal handler for the SIGCANCEL or SIGSETXID signals (which are used only on GNU/Linux), it will turn on the SA_ONSTACK flag and otherwise keep the signal handler. -For the synchronous signals, the Go runtime will install a signal -handler. It will save any existing signal handler. If a synchronous -signal arrives while executing non-Go code, the Go runtime will invoke -the existing signal handler instead of the Go signal handler. +For the synchronous signals and SIGPIPE, the Go runtime will install a +signal handler. It will save any existing signal handler. If a +synchronous signal arrives while executing non-Go code, the Go runtime +will invoke the existing signal handler instead of the Go signal +handler. Go code built with -buildmode=c-archive or -buildmode=c-shared will not install any other signal handlers by default. If there is an diff --git a/libgo/go/os/signal/signal.go b/libgo/go/os/signal/signal.go index c1376daaea6..e5a21e85327 100644 --- a/libgo/go/os/signal/signal.go +++ b/libgo/go/os/signal/signal.go @@ -11,8 +11,21 @@ import ( var handlers struct { sync.Mutex - m map[chan<- os.Signal]*handler + // Map a channel to the signals that should be sent to it. + m map[chan<- os.Signal]*handler + // Map a signal to the number of channels receiving it. ref [numSig]int64 + // Map channels to signals while the channel is being stopped. + // Not a map because entries live here only very briefly. + // We need a separate container because we need m to correspond to ref + // at all times, and we also need to keep track of the *handler + // value for a channel being stopped. See the Stop function. + stopping []stopping +} + +type stopping struct { + c chan<- os.Signal + h *handler } type handler struct { @@ -142,10 +155,10 @@ func Reset(sig ...os.Signal) { // When Stop returns, it is guaranteed that c will receive no more signals. func Stop(c chan<- os.Signal) { handlers.Lock() - defer handlers.Unlock() h := handlers.m[c] if h == nil { + handlers.Unlock() return } delete(handlers.m, c) @@ -158,8 +171,40 @@ func Stop(c chan<- os.Signal) { } } } + + // Signals will no longer be delivered to the channel. + // We want to avoid a race for a signal such as SIGINT: + // it should be either delivered to the channel, + // or the program should take the default action (that is, exit). + // To avoid the possibility that the signal is delivered, + // and the signal handler invoked, and then Stop deregisters + // the channel before the process function below has a chance + // to send it on the channel, put the channel on a list of + // channels being stopped and wait for signal delivery to + // quiesce before fully removing it. + + handlers.stopping = append(handlers.stopping, stopping{c, h}) + + handlers.Unlock() + + signalWaitUntilIdle() + + handlers.Lock() + + for i, s := range handlers.stopping { + if s.c == c { + handlers.stopping = append(handlers.stopping[:i], handlers.stopping[i+1:]...) + break + } + } + + handlers.Unlock() } +// Wait until there are no more signals waiting to be delivered. +// Defined by the runtime package. +func signalWaitUntilIdle() + func process(sig os.Signal) { n := signum(sig) if n < 0 { @@ -178,4 +223,14 @@ func process(sig os.Signal) { } } } + + // Avoid the race mentioned in Stop. + for _, d := range handlers.stopping { + if d.h.want(n) { + select { + case d.c <- sig: + default: + } + } + } } diff --git a/libgo/go/os/signal/signal_test.go b/libgo/go/os/signal/signal_test.go index c8409e73697..10a4146f5e9 100644 --- a/libgo/go/os/signal/signal_test.go +++ b/libgo/go/os/signal/signal_test.go @@ -7,12 +7,16 @@ package signal import ( + "bytes" "flag" + "fmt" + "internal/testenv" "io/ioutil" "os" "os/exec" "runtime" "strconv" + "sync" "syscall" "testing" "time" @@ -301,3 +305,90 @@ func TestSIGCONT(t *testing.T) { syscall.Kill(syscall.Getpid(), syscall.SIGCONT) waitSig(t, c, syscall.SIGCONT) } + +// Test race between stopping and receiving a signal (issue 14571). +func TestAtomicStop(t *testing.T) { + if os.Getenv("GO_TEST_ATOMIC_STOP") != "" { + atomicStopTestProgram() + t.Fatal("atomicStopTestProgram returned") + } + + testenv.MustHaveExec(t) + + const execs = 10 + for i := 0; i < execs; i++ { + cmd := exec.Command(os.Args[0], "-test.run=TestAtomicStop") + cmd.Env = append(os.Environ(), "GO_TEST_ATOMIC_STOP=1") + out, err := cmd.CombinedOutput() + if err == nil { + t.Logf("iteration %d: output %s", i, out) + } else { + t.Logf("iteration %d: exit status %q: output: %s", i, err, out) + } + + lost := bytes.Contains(out, []byte("lost signal")) + if lost { + t.Errorf("iteration %d: lost signal", i) + } + + // The program should either die due to SIGINT, + // or exit with success without printing "lost signal". + if err == nil { + if len(out) > 0 && !lost { + t.Errorf("iteration %d: unexpected output", i) + } + } else { + if ee, ok := err.(*exec.ExitError); !ok { + t.Errorf("iteration %d: error (%v) has type %T; expected exec.ExitError", i, err, err) + } else if ws, ok := ee.Sys().(syscall.WaitStatus); !ok { + t.Errorf("iteration %d: error.Sys (%v) has type %T; expected syscall.WaitStatus", i, ee.Sys(), ee.Sys()) + } else if !ws.Signaled() || ws.Signal() != syscall.SIGINT { + t.Errorf("iteration %d: got exit status %v; expected SIGINT", i, ee) + } + } + } +} + +// atomicStopTestProgram is run in a subprocess by TestAtomicStop. +// It tries to trigger a signal delivery race. This function should +// either catch a signal or die from it. +func atomicStopTestProgram() { + const tries = 10 + pid := syscall.Getpid() + printed := false + for i := 0; i < tries; i++ { + cs := make(chan os.Signal, 1) + Notify(cs, syscall.SIGINT) + + var wg sync.WaitGroup + wg.Add(1) + go func() { + defer wg.Done() + Stop(cs) + }() + + syscall.Kill(pid, syscall.SIGINT) + + // At this point we should either die from SIGINT or + // get a notification on cs. If neither happens, we + // dropped the signal. Give it a second to deliver, + // which is far far longer than it should require. + + select { + case <-cs: + case <-time.After(1 * time.Second): + if !printed { + fmt.Print("lost signal on iterations:") + printed = true + } + fmt.Printf(" %d", i) + } + + wg.Wait() + } + if printed { + fmt.Print("\n") + } + + os.Exit(0) +} diff --git a/libgo/go/os/stat_unix.go b/libgo/go/os/stat_unix.go index 043aefe8fad..7855fbab135 100644 --- a/libgo/go/os/stat_unix.go +++ b/libgo/go/os/stat_unix.go @@ -17,7 +17,7 @@ func (f *File) Stat() (FileInfo, error) { return nil, ErrInvalid } var fs fileStat - err := syscall.Fstat(f.fd, &fs.sys) + err := f.pfd.Fstat(&fs.sys) if err != nil { return nil, &PathError{"stat", f.name, err} } diff --git a/libgo/go/os/sys_darwin.go b/libgo/go/os/sys_darwin.go index 7a8330abb55..11d678ef18d 100644 --- a/libgo/go/os/sys_darwin.go +++ b/libgo/go/os/sys_darwin.go @@ -4,28 +4,8 @@ package os -import "syscall" - // supportsCloseOnExec reports whether the platform supports the // O_CLOEXEC flag. -var supportsCloseOnExec bool - -func init() { - // Seems like kern.osreldate is veiled on latest OS X. We use - // kern.osrelease instead. - osver, err := syscall.Sysctl("kern.osrelease") - if err != nil { - return - } - var i int - for i = range osver { - if osver[i] != '.' { - continue - } - } - // The O_CLOEXEC flag was introduced in OS X 10.7 (Darwin - // 11.0.0). See http://support.apple.com/kb/HT1633. - if i > 2 || i == 2 && osver[0] >= '1' && osver[1] >= '1' { - supportsCloseOnExec = true - } -} +// The O_CLOEXEC flag was introduced in OS X 10.7 (Darwin 11.0.0). +// See http://support.apple.com/kb/HT1633. +const supportsCloseOnExec = true diff --git a/libgo/go/os/types.go b/libgo/go/os/types.go index c56548353f1..db7848759cb 100644 --- a/libgo/go/os/types.go +++ b/libgo/go/os/types.go @@ -45,7 +45,7 @@ const ( ModeDir FileMode = 1 << (32 - 1 - iota) // d: is a directory ModeAppend // a: append-only ModeExclusive // l: exclusive use - ModeTemporary // T: temporary file (not backed up) + ModeTemporary // T: temporary file; Plan 9 only ModeSymlink // L: symbolic link ModeDevice // D: device file ModeNamedPipe // p: named pipe (FIFO) diff --git a/libgo/go/os/types_unix.go b/libgo/go/os/types_unix.go index 1f614812fdd..c0259ae0e84 100644 --- a/libgo/go/os/types_unix.go +++ b/libgo/go/os/types_unix.go @@ -29,5 +29,3 @@ func (fs *fileStat) Sys() interface{} { return &fs.sys } func sameFile(fs1, fs2 *fileStat) bool { return fs1.sys.Dev == fs2.sys.Dev && fs1.sys.Ino == fs2.sys.Ino } - -const badFd = -1 diff --git a/libgo/go/os/types_windows.go b/libgo/go/os/types_windows.go index ad4e863fcbb..01d6b62a16e 100644 --- a/libgo/go/os/types_windows.go +++ b/libgo/go/os/types_windows.go @@ -12,16 +12,17 @@ import ( // A fileStat is the implementation of FileInfo returned by Stat and Lstat. type fileStat struct { - name string - sys syscall.Win32FileAttributeData - pipe bool + name string + sys syscall.Win32FileAttributeData + filetype uint32 // what syscall.GetFileType returns // used to implement SameFile sync.Mutex - path string - vol uint32 - idxhi uint32 - idxlo uint32 + path string + vol uint32 + idxhi uint32 + idxlo uint32 + appendNameToPath bool } func (fs *fileStat) Size() int64 { @@ -32,19 +33,22 @@ func (fs *fileStat) Mode() (m FileMode) { if fs == &devNullStat { return ModeDevice | ModeCharDevice | 0666 } - if fs.sys.FileAttributes&syscall.FILE_ATTRIBUTE_DIRECTORY != 0 { - m |= ModeDir | 0111 - } if fs.sys.FileAttributes&syscall.FILE_ATTRIBUTE_READONLY != 0 { m |= 0444 } else { m |= 0666 } if fs.sys.FileAttributes&syscall.FILE_ATTRIBUTE_REPARSE_POINT != 0 { - m |= ModeSymlink + return m | ModeSymlink + } + if fs.sys.FileAttributes&syscall.FILE_ATTRIBUTE_DIRECTORY != 0 { + m |= ModeDir | 0111 } - if fs.pipe { + switch fs.filetype { + case syscall.FILE_TYPE_PIPE: m |= ModeNamedPipe + case syscall.FILE_TYPE_CHAR: + m |= ModeCharDevice } return m } @@ -63,7 +67,13 @@ func (fs *fileStat) loadFileId() error { // already done return nil } - pathp, err := syscall.UTF16PtrFromString(fs.path) + var path string + if fs.appendNameToPath { + path = fs.path + `\` + fs.name + } else { + path = fs.path + } + pathp, err := syscall.UTF16PtrFromString(path) if err != nil { return err } diff --git a/libgo/go/os/user/cgo_lookup_unix.go b/libgo/go/os/user/cgo_lookup_unix.go new file mode 100644 index 00000000000..8881366cf9f --- /dev/null +++ b/libgo/go/os/user/cgo_lookup_unix.go @@ -0,0 +1,266 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build darwin dragonfly freebsd !android,linux netbsd openbsd solaris +// +build cgo + +package user + +import ( + "fmt" + "strconv" + "strings" + "syscall" + "unsafe" +) + +// bytePtrToString takes a NUL-terminated array of bytes and convert +// it to a Go string. +func bytePtrToString(p *byte) string { + a := (*[10000]byte)(unsafe.Pointer(p)) + i := 0 + for a[i] != 0 { + i++ + } + return string(a[:i]) +} + +func current() (*User, error) { + return lookupUnixUid(syscall.Getuid()) +} + +func lookupUser(username string) (*User, error) { + var pwd syscall.Passwd + var result *syscall.Passwd + p := syscall.StringBytePtr(username) + + buf := alloc(userBuffer) + defer buf.free() + + err := retryWithBuffer(buf, func() syscall.Errno { + syscall.Entersyscall() + rv := libc_getpwnam_r(p, + &pwd, + buf.ptr, + buf.size, + &result) + syscall.Exitsyscall() + if rv != 0 { + return syscall.GetErrno() + } + return 0 + }) + if err != nil { + return nil, fmt.Errorf("user: lookup username %s: %v", username, err) + } + if result == nil { + return nil, UnknownUserError(username) + } + return buildUser(&pwd), err +} + +func lookupUserId(uid string) (*User, error) { + i, e := strconv.Atoi(uid) + if e != nil { + return nil, e + } + return lookupUnixUid(i) +} + +func lookupUnixUid(uid int) (*User, error) { + var pwd syscall.Passwd + var result *syscall.Passwd + + buf := alloc(userBuffer) + defer buf.free() + + err := retryWithBuffer(buf, func() syscall.Errno { + syscall.Entersyscall() + rv := libc_getpwuid_r(syscall.Uid_t(uid), + &pwd, + buf.ptr, + buf.size, + &result) + syscall.Exitsyscall() + if rv != 0 { + return syscall.GetErrno() + } + return 0 + }) + if err != nil { + return nil, fmt.Errorf("user: lookup userid %d: %v", uid, err) + } + if result == nil { + return nil, UnknownUserIdError(uid) + } + return buildUser(&pwd), nil +} + +func buildUser(pwd *syscall.Passwd) *User { + u := &User{ + Uid: strconv.Itoa(int(pwd.Pw_uid)), + Gid: strconv.Itoa(int(pwd.Pw_gid)), + Username: bytePtrToString((*byte)(unsafe.Pointer(pwd.Pw_name))), + Name: bytePtrToString((*byte)(unsafe.Pointer(pwd.Pw_gecos))), + HomeDir: bytePtrToString((*byte)(unsafe.Pointer(pwd.Pw_dir))), + } + // The pw_gecos field isn't quite standardized. Some docs + // say: "It is expected to be a comma separated list of + // personal data where the first item is the full name of the + // user." + if i := strings.Index(u.Name, ","); i >= 0 { + u.Name = u.Name[:i] + } + return u +} + +func currentGroup() (*Group, error) { + return lookupUnixGid(syscall.Getgid()) +} + +func lookupGroup(groupname string) (*Group, error) { + var grp syscall.Group + var result *syscall.Group + + buf := alloc(groupBuffer) + defer buf.free() + p := syscall.StringBytePtr(groupname) + + err := retryWithBuffer(buf, func() syscall.Errno { + syscall.Entersyscall() + rv := libc_getgrnam_r(p, + &grp, + buf.ptr, + buf.size, + &result) + syscall.Exitsyscall() + if rv != 0 { + return syscall.GetErrno() + } + return 0 + }) + if err != nil { + return nil, fmt.Errorf("user: lookup groupname %s: %v", groupname, err) + } + if result == nil { + return nil, UnknownGroupError(groupname) + } + return buildGroup(&grp), nil +} + +func lookupGroupId(gid string) (*Group, error) { + i, e := strconv.Atoi(gid) + if e != nil { + return nil, e + } + return lookupUnixGid(i) +} + +func lookupUnixGid(gid int) (*Group, error) { + var grp syscall.Group + var result *syscall.Group + + buf := alloc(groupBuffer) + defer buf.free() + + err := retryWithBuffer(buf, func() syscall.Errno { + syscall.Entersyscall() + rv := libc_getgrgid_r(syscall.Gid_t(gid), + &grp, + buf.ptr, + buf.size, + &result) + syscall.Exitsyscall() + if rv != 0 { + return syscall.GetErrno() + } + return 0 + }) + if err != nil { + return nil, fmt.Errorf("user: lookup groupid %d: %v", gid, err) + } + if result == nil { + return nil, UnknownGroupIdError(strconv.Itoa(gid)) + } + return buildGroup(&grp), nil +} + +func buildGroup(grp *syscall.Group) *Group { + g := &Group{ + Gid: strconv.Itoa(int(grp.Gr_gid)), + Name: bytePtrToString((*byte)(unsafe.Pointer(grp.Gr_name))), + } + return g +} + +type bufferKind int + +const ( + userBuffer = bufferKind(syscall.SC_GETPW_R_SIZE_MAX) + groupBuffer = bufferKind(syscall.SC_GETGR_R_SIZE_MAX) +) + +func (k bufferKind) initialSize() syscall.Size_t { + sz, _ := syscall.Sysconf(int(k)) + if sz == -1 { + // DragonFly and FreeBSD do not have _SC_GETPW_R_SIZE_MAX. + // Additionally, not all Linux systems have it, either. For + // example, the musl libc returns -1. + return 1024 + } + if !isSizeReasonable(int64(sz)) { + // Truncate. If this truly isn't enough, retryWithBuffer will error on the first run. + return maxBufferSize + } + return syscall.Size_t(sz) +} + +type memBuffer struct { + ptr *byte + size syscall.Size_t +} + +func alloc(kind bufferKind) *memBuffer { + sz := kind.initialSize() + b := make([]byte, sz) + return &memBuffer{ + ptr: &b[0], + size: sz, + } +} + +func (mb *memBuffer) resize(newSize syscall.Size_t) { + b := make([]byte, newSize) + mb.ptr = &b[0] + mb.size = newSize +} + +func (mb *memBuffer) free() { + mb.ptr = nil +} + +// retryWithBuffer repeatedly calls f(), increasing the size of the +// buffer each time, until f succeeds, fails with a non-ERANGE error, +// or the buffer exceeds a reasonable limit. +func retryWithBuffer(buf *memBuffer, f func() syscall.Errno) error { + for { + errno := f() + if errno == 0 { + return nil + } else if errno != syscall.ERANGE { + return errno + } + newSize := buf.size * 2 + if !isSizeReasonable(int64(newSize)) { + return fmt.Errorf("internal buffer exceeds %d bytes", maxBufferSize) + } + buf.resize(newSize) + } +} + +const maxBufferSize = 1 << 20 + +func isSizeReasonable(sz int64) bool { + return sz > 0 && sz <= maxBufferSize +} diff --git a/libgo/go/os/user/lookup.go b/libgo/go/os/user/lookup.go index 3b4421badd5..2243a25788a 100644 --- a/libgo/go/os/user/lookup.go +++ b/libgo/go/os/user/lookup.go @@ -4,20 +4,40 @@ package user +import "sync" + // Current returns the current user. func Current() (*User, error) { - return current() + cache.Do(func() { cache.u, cache.err = current() }) + if cache.err != nil { + return nil, cache.err + } + u := *cache.u // copy + return &u, nil +} + +// cache of the current user +var cache struct { + sync.Once + u *User + err error } // Lookup looks up a user by username. If the user cannot be found, the // returned error is of type UnknownUserError. func Lookup(username string) (*User, error) { + if u, err := Current(); err == nil && u.Username == username { + return u, err + } return lookupUser(username) } // LookupId looks up a user by userid. If the user cannot be found, the // returned error is of type UnknownUserIdError. func LookupId(uid string) (*User, error) { + if u, err := Current(); err == nil && u.Uid == uid { + return u, err + } return lookupUserId(uid) } diff --git a/libgo/go/os/user/lookup_android.go b/libgo/go/os/user/lookup_android.go index b1be3dc1931..8ca30b8c274 100644 --- a/libgo/go/os/user/lookup_android.go +++ b/libgo/go/os/user/lookup_android.go @@ -8,15 +8,6 @@ package user import "errors" -func init() { - userImplemented = false - groupImplemented = false -} - -func current() (*User, error) { - return nil, errors.New("user: Current not implemented on android") -} - func lookupUser(string) (*User, error) { return nil, errors.New("user: Lookup not implemented on android") } @@ -32,7 +23,3 @@ func lookupGroup(string) (*Group, error) { func lookupGroupId(string) (*Group, error) { return nil, errors.New("user: LookupGroupId not implemented on android") } - -func listGroups(*User) ([]string, error) { - return nil, errors.New("user: GroupIds not implemented on android") -} diff --git a/libgo/go/os/user/lookup_stubs.go b/libgo/go/os/user/lookup_stubs.go index ebf24f79dee..d23870fda88 100644 --- a/libgo/go/os/user/lookup_stubs.go +++ b/libgo/go/os/user/lookup_stubs.go @@ -2,7 +2,7 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -// +build !cgo,!windows,!plan9,!android +// +build !cgo,!windows,!plan9 android package user @@ -15,7 +15,6 @@ import ( ) func init() { - userImplemented = false groupImplemented = false } @@ -27,7 +26,9 @@ func current() (*User, error) { Name: "", // ignored HomeDir: os.Getenv("HOME"), } - if runtime.GOOS == "nacl" { + // On NaCL and Android, return a dummy user instead of failing. + switch runtime.GOOS { + case "nacl": if u.Uid == "" { u.Uid = "1" } @@ -35,7 +36,17 @@ func current() (*User, error) { u.Username = "nacl" } if u.HomeDir == "" { - u.HomeDir = "/home/nacl" + u.HomeDir = "/" + } + case "android": + if u.Uid == "" { + u.Uid = "1" + } + if u.Username == "" { + u.Username = "android" + } + if u.HomeDir == "" { + u.HomeDir = "/sdcard" } } // cgo isn't available, but if we found the minimum information @@ -46,23 +57,10 @@ func current() (*User, error) { return u, fmt.Errorf("user: Current not implemented on %s/%s", runtime.GOOS, runtime.GOARCH) } -func lookupUser(username string) (*User, error) { - return nil, errors.New("user: Lookup requires cgo") -} - -func lookupUserId(uid string) (*User, error) { - return nil, errors.New("user: LookupId requires cgo") -} - -func lookupGroup(groupname string) (*Group, error) { - return nil, errors.New("user: LookupGroup requires cgo") -} - -func lookupGroupId(string) (*Group, error) { - return nil, errors.New("user: LookupGroupId requires cgo") -} - func listGroups(*User) ([]string, error) { + if runtime.GOOS == "android" { + return nil, errors.New("user: GroupIds not implemented on Android") + } return nil, errors.New("user: GroupIds requires cgo") } diff --git a/libgo/go/os/user/lookup_unix.go b/libgo/go/os/user/lookup_unix.go index 9670ada4942..5f34ba8611c 100644 --- a/libgo/go/os/user/lookup_unix.go +++ b/libgo/go/os/user/lookup_unix.go @@ -1,266 +1,197 @@ -// Copyright 2011 The Go Authors. All rights reserved. +// Copyright 2016 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -// +build aix darwin dragonfly freebsd !android,linux netbsd openbsd solaris -// +build cgo +// +build darwin dragonfly freebsd !android,linux nacl netbsd openbsd solaris +// +build !cgo package user import ( - "fmt" + "bufio" + "bytes" + "errors" + "io" + "os" "strconv" "strings" - "syscall" - "unsafe" ) -// bytePtrToString takes a NUL-terminated array of bytes and convert -// it to a Go string. -func bytePtrToString(p *byte) string { - a := (*[10000]byte)(unsafe.Pointer(p)) - i := 0 - for a[i] != 0 { - i++ - } - return string(a[:i]) -} +const groupFile = "/etc/group" +const userFile = "/etc/passwd" -func current() (*User, error) { - return lookupUnixUid(syscall.Getuid()) -} +var colon = []byte{':'} -func lookupUser(username string) (*User, error) { - var pwd syscall.Passwd - var result *syscall.Passwd - p := syscall.StringBytePtr(username) +func init() { + groupImplemented = false +} - buf := alloc(userBuffer) - defer buf.free() +// lineFunc returns a value, an error, or (nil, nil) to skip the row. +type lineFunc func(line []byte) (v interface{}, err error) - err := retryWithBuffer(buf, func() syscall.Errno { - syscall.Entersyscall() - rv := libc_getpwnam_r(p, - &pwd, - buf.ptr, - buf.size, - &result) - syscall.Exitsyscall() - if rv != 0 { - return syscall.GetErrno() +// readColonFile parses r as an /etc/group or /etc/passwd style file, running +// fn for each row. readColonFile returns a value, an error, or (nil, nil) if +// the end of the file is reached without a match. +func readColonFile(r io.Reader, fn lineFunc) (v interface{}, err error) { + bs := bufio.NewScanner(r) + for bs.Scan() { + line := bs.Bytes() + // There's no spec for /etc/passwd or /etc/group, but we try to follow + // the same rules as the glibc parser, which allows comments and blank + // space at the beginning of a line. + line = bytes.TrimSpace(line) + if len(line) == 0 || line[0] == '#' { + continue + } + v, err = fn(line) + if v != nil || err != nil { + return } - return 0 - }) - if err != nil { - return nil, fmt.Errorf("user: lookup username %s: %v", username, err) - } - if result == nil { - return nil, UnknownUserError(username) } - return buildUser(&pwd), err + return nil, bs.Err() } -func lookupUserId(uid string) (*User, error) { - i, e := strconv.Atoi(uid) - if e != nil { - return nil, e +func matchGroupIndexValue(value string, idx int) lineFunc { + var leadColon string + if idx > 0 { + leadColon = ":" } - return lookupUnixUid(i) -} - -func lookupUnixUid(uid int) (*User, error) { - var pwd syscall.Passwd - var result *syscall.Passwd - - buf := alloc(userBuffer) - defer buf.free() - - err := retryWithBuffer(buf, func() syscall.Errno { - syscall.Entersyscall() - rv := libc_getpwuid_r(syscall.Uid_t(uid), - &pwd, - buf.ptr, - buf.size, - &result) - syscall.Exitsyscall() - if rv != 0 { - return syscall.GetErrno() + substr := []byte(leadColon + value + ":") + return func(line []byte) (v interface{}, err error) { + if !bytes.Contains(line, substr) || bytes.Count(line, colon) < 3 { + return } - return 0 - }) - if err != nil { - return nil, fmt.Errorf("user: lookup userid %d: %v", uid, err) - } - if result == nil { - return nil, UnknownUserIdError(uid) + // wheel:*:0:root + parts := strings.SplitN(string(line), ":", 4) + if len(parts) < 4 || parts[0] == "" || parts[idx] != value || + // If the file contains +foo and you search for "foo", glibc + // returns an "invalid argument" error. Similarly, if you search + // for a gid for a row where the group name starts with "+" or "-", + // glibc fails to find the record. + parts[0][0] == '+' || parts[0][0] == '-' { + return + } + if _, err := strconv.Atoi(parts[2]); err != nil { + return nil, nil + } + return &Group{Name: parts[0], Gid: parts[2]}, nil } - return buildUser(&pwd), nil } -func buildUser(pwd *syscall.Passwd) *User { - u := &User{ - Uid: strconv.Itoa(int(pwd.Pw_uid)), - Gid: strconv.Itoa(int(pwd.Pw_gid)), - Username: bytePtrToString((*byte)(unsafe.Pointer(pwd.Pw_name))), - Name: bytePtrToString((*byte)(unsafe.Pointer(pwd.Pw_gecos))), - HomeDir: bytePtrToString((*byte)(unsafe.Pointer(pwd.Pw_dir))), - } - // The pw_gecos field isn't quite standardized. Some docs - // say: "It is expected to be a comma separated list of - // personal data where the first item is the full name of the - // user." - if i := strings.Index(u.Name, ","); i >= 0 { - u.Name = u.Name[:i] +func findGroupId(id string, r io.Reader) (*Group, error) { + if v, err := readColonFile(r, matchGroupIndexValue(id, 2)); err != nil { + return nil, err + } else if v != nil { + return v.(*Group), nil } - return u + return nil, UnknownGroupIdError(id) } -func currentGroup() (*Group, error) { - return lookupUnixGid(syscall.Getgid()) +func findGroupName(name string, r io.Reader) (*Group, error) { + if v, err := readColonFile(r, matchGroupIndexValue(name, 0)); err != nil { + return nil, err + } else if v != nil { + return v.(*Group), nil + } + return nil, UnknownGroupError(name) } -func lookupGroup(groupname string) (*Group, error) { - var grp syscall.Group - var result *syscall.Group - - buf := alloc(groupBuffer) - defer buf.free() - p := syscall.StringBytePtr(groupname) - - err := retryWithBuffer(buf, func() syscall.Errno { - syscall.Entersyscall() - rv := libc_getgrnam_r(p, - &grp, - buf.ptr, - buf.size, - &result) - syscall.Exitsyscall() - if rv != 0 { - return syscall.GetErrno() - } - return 0 - }) - if err != nil { - return nil, fmt.Errorf("user: lookup groupname %s: %v", groupname, err) +// returns a *User for a row if that row's has the given value at the +// given index. +func matchUserIndexValue(value string, idx int) lineFunc { + var leadColon string + if idx > 0 { + leadColon = ":" } - if result == nil { - return nil, UnknownGroupError(groupname) + substr := []byte(leadColon + value + ":") + return func(line []byte) (v interface{}, err error) { + if !bytes.Contains(line, substr) || bytes.Count(line, colon) < 6 { + return + } + // kevin:x:1005:1006::/home/kevin:/usr/bin/zsh + parts := strings.SplitN(string(line), ":", 7) + if len(parts) < 6 || parts[idx] != value || parts[0] == "" || + parts[0][0] == '+' || parts[0][0] == '-' { + return + } + if _, err := strconv.Atoi(parts[2]); err != nil { + return nil, nil + } + if _, err := strconv.Atoi(parts[3]); err != nil { + return nil, nil + } + u := &User{ + Username: parts[0], + Uid: parts[2], + Gid: parts[3], + Name: parts[4], + HomeDir: parts[5], + } + // The pw_gecos field isn't quite standardized. Some docs + // say: "It is expected to be a comma separated list of + // personal data where the first item is the full name of the + // user." + if i := strings.Index(u.Name, ","); i >= 0 { + u.Name = u.Name[:i] + } + return u, nil } - return buildGroup(&grp), nil } -func lookupGroupId(gid string) (*Group, error) { - i, e := strconv.Atoi(gid) +func findUserId(uid string, r io.Reader) (*User, error) { + i, e := strconv.Atoi(uid) if e != nil { - return nil, e + return nil, errors.New("user: invalid userid " + uid) } - return lookupUnixGid(i) -} - -func lookupUnixGid(gid int) (*Group, error) { - var grp syscall.Group - var result *syscall.Group - - buf := alloc(groupBuffer) - defer buf.free() - - err := retryWithBuffer(buf, func() syscall.Errno { - syscall.Entersyscall() - rv := libc_getgrgid_r(syscall.Gid_t(gid), - &grp, - buf.ptr, - buf.size, - &result) - syscall.Exitsyscall() - if rv != 0 { - return syscall.GetErrno() - } - return 0 - }) - if err != nil { - return nil, fmt.Errorf("user: lookup groupid %d: %v", gid, err) + if v, err := readColonFile(r, matchUserIndexValue(uid, 2)); err != nil { + return nil, err + } else if v != nil { + return v.(*User), nil } - if result == nil { - return nil, UnknownGroupIdError(strconv.Itoa(gid)) - } - return buildGroup(&grp), nil + return nil, UnknownUserIdError(i) } -func buildGroup(grp *syscall.Group) *Group { - g := &Group{ - Gid: strconv.Itoa(int(grp.Gr_gid)), - Name: bytePtrToString((*byte)(unsafe.Pointer(grp.Gr_name))), +func findUsername(name string, r io.Reader) (*User, error) { + if v, err := readColonFile(r, matchUserIndexValue(name, 0)); err != nil { + return nil, err + } else if v != nil { + return v.(*User), nil } - return g + return nil, UnknownUserError(name) } -type bufferKind int - -const ( - userBuffer = bufferKind(syscall.SC_GETPW_R_SIZE_MAX) - groupBuffer = bufferKind(syscall.SC_GETGR_R_SIZE_MAX) -) - -func (k bufferKind) initialSize() syscall.Size_t { - sz, _ := syscall.Sysconf(int(k)) - if sz == -1 { - // DragonFly and FreeBSD do not have _SC_GETPW_R_SIZE_MAX. - // Additionally, not all Linux systems have it, either. For - // example, the musl libc returns -1. - return 1024 - } - if !isSizeReasonable(int64(sz)) { - // Truncate. If this truly isn't enough, retryWithBuffer will error on the first run. - return maxBufferSize +func lookupGroup(groupname string) (*Group, error) { + f, err := os.Open(groupFile) + if err != nil { + return nil, err } - return syscall.Size_t(sz) + defer f.Close() + return findGroupName(groupname, f) } -type memBuffer struct { - ptr *byte - size syscall.Size_t -} - -func alloc(kind bufferKind) *memBuffer { - sz := kind.initialSize() - b := make([]byte, sz) - return &memBuffer{ - ptr: &b[0], - size: sz, +func lookupGroupId(id string) (*Group, error) { + f, err := os.Open(groupFile) + if err != nil { + return nil, err } + defer f.Close() + return findGroupId(id, f) } -func (mb *memBuffer) resize(newSize syscall.Size_t) { - b := make([]byte, newSize) - mb.ptr = &b[0] - mb.size = newSize -} - -func (mb *memBuffer) free() { - mb.ptr = nil -} - -// retryWithBuffer repeatedly calls f(), increasing the size of the -// buffer each time, until f succeeds, fails with a non-ERANGE error, -// or the buffer exceeds a reasonable limit. -func retryWithBuffer(buf *memBuffer, f func() syscall.Errno) error { - for { - errno := f() - if errno == 0 { - return nil - } else if errno != syscall.ERANGE { - return errno - } - newSize := buf.size * 2 - if !isSizeReasonable(int64(newSize)) { - return fmt.Errorf("internal buffer exceeds %d bytes", maxBufferSize) - } - buf.resize(newSize) +func lookupUser(username string) (*User, error) { + f, err := os.Open(userFile) + if err != nil { + return nil, err } + defer f.Close() + return findUsername(username, f) } -const maxBufferSize = 1 << 20 - -func isSizeReasonable(sz int64) bool { - return sz > 0 && sz <= maxBufferSize +func lookupUserId(uid string) (*User, error) { + f, err := os.Open(userFile) + if err != nil { + return nil, err + } + defer f.Close() + return findUserId(uid, f) } diff --git a/libgo/go/os/user/lookup_unix_test.go b/libgo/go/os/user/lookup_unix_test.go new file mode 100644 index 00000000000..02c88ab8757 --- /dev/null +++ b/libgo/go/os/user/lookup_unix_test.go @@ -0,0 +1,276 @@ +// Copyright 2016 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build darwin dragonfly freebsd !android,linux nacl netbsd openbsd solaris +// +build !cgo + +package user + +import ( + "reflect" + "strings" + "testing" +) + +const testGroupFile = `# See the opendirectoryd(8) man page for additional +# information about Open Directory. +## +nobody:*:-2: +nogroup:*:-1: +wheel:*:0:root +emptyid:*::root +invalidgid:*:notanumber:root ++plussign:*:20:root +-minussign:*:21:root + +daemon:*:1:root + indented:*:7: +# comment:*:4:found + # comment:*:4:found +kmem:*:2:root +` + +var groupTests = []struct { + in string + name string + gid string +}{ + {testGroupFile, "nobody", "-2"}, + {testGroupFile, "kmem", "2"}, + {testGroupFile, "notinthefile", ""}, + {testGroupFile, "comment", ""}, + {testGroupFile, "plussign", ""}, + {testGroupFile, "+plussign", ""}, + {testGroupFile, "-minussign", ""}, + {testGroupFile, "minussign", ""}, + {testGroupFile, "emptyid", ""}, + {testGroupFile, "invalidgid", ""}, + {testGroupFile, "indented", "7"}, + {testGroupFile, "# comment", ""}, + {"", "emptyfile", ""}, +} + +func TestFindGroupName(t *testing.T) { + for _, tt := range groupTests { + got, err := findGroupName(tt.name, strings.NewReader(tt.in)) + if tt.gid == "" { + if err == nil { + t.Errorf("findGroupName(%s): got nil error, expected err", tt.name) + continue + } + switch terr := err.(type) { + case UnknownGroupError: + if terr.Error() != "group: unknown group "+tt.name { + t.Errorf("findGroupName(%s): got %v, want %v", tt.name, terr, tt.name) + } + default: + t.Errorf("findGroupName(%s): got unexpected error %v", tt.name, terr) + } + } else { + if err != nil { + t.Fatalf("findGroupName(%s): got unexpected error %v", tt.name, err) + } + if got.Gid != tt.gid { + t.Errorf("findGroupName(%s): got gid %v, want %s", tt.name, got.Gid, tt.gid) + } + if got.Name != tt.name { + t.Errorf("findGroupName(%s): got name %s, want %s", tt.name, got.Name, tt.name) + } + } + } +} + +var groupIdTests = []struct { + in string + gid string + name string +}{ + {testGroupFile, "-2", "nobody"}, + {testGroupFile, "2", "kmem"}, + {testGroupFile, "notinthefile", ""}, + {testGroupFile, "comment", ""}, + {testGroupFile, "7", "indented"}, + {testGroupFile, "4", ""}, + {testGroupFile, "20", ""}, // row starts with a plus + {testGroupFile, "21", ""}, // row starts with a minus + {"", "emptyfile", ""}, +} + +func TestFindGroupId(t *testing.T) { + for _, tt := range groupIdTests { + got, err := findGroupId(tt.gid, strings.NewReader(tt.in)) + if tt.name == "" { + if err == nil { + t.Errorf("findGroupId(%s): got nil error, expected err", tt.gid) + continue + } + switch terr := err.(type) { + case UnknownGroupIdError: + if terr.Error() != "group: unknown groupid "+tt.gid { + t.Errorf("findGroupId(%s): got %v, want %v", tt.name, terr, tt.name) + } + default: + t.Errorf("findGroupId(%s): got unexpected error %v", tt.name, terr) + } + } else { + if err != nil { + t.Fatalf("findGroupId(%s): got unexpected error %v", tt.name, err) + } + if got.Gid != tt.gid { + t.Errorf("findGroupId(%s): got gid %v, want %s", tt.name, got.Gid, tt.gid) + } + if got.Name != tt.name { + t.Errorf("findGroupId(%s): got name %s, want %s", tt.name, got.Name, tt.name) + } + } + } +} + +const testUserFile = ` # Example user file +root:x:0:0:root:/root:/bin/bash +daemon:x:1:1:daemon:/usr/sbin:/usr/sbin/nologin +bin:x:2:3:bin:/bin:/usr/sbin/nologin + indented:x:3:3:indented:/dev:/usr/sbin/nologin +sync:x:4:65534:sync:/bin:/bin/sync +negative:x:-5:60:games:/usr/games:/usr/sbin/nologin +man:x:6:12:man:/var/cache/man:/usr/sbin/nologin +allfields:x:6:12:mansplit,man2,man3,man4:/home/allfields:/usr/sbin/nologin ++plussign:x:8:10:man:/var/cache/man:/usr/sbin/nologin +-minussign:x:9:10:man:/var/cache/man:/usr/sbin/nologin + +malformed:x:27:12 # more:colons:after:comment + +struid:x:notanumber:12 # more:colons:after:comment + +# commented:x:28:12:commented:/var/cache/man:/usr/sbin/nologin + # commentindented:x:29:12:commentindented:/var/cache/man:/usr/sbin/nologin + +struid2:x:30:badgid:struid2name:/home/struid:/usr/sbin/nologin +` + +var userIdTests = []struct { + in string + uid string + name string +}{ + {testUserFile, "-5", "negative"}, + {testUserFile, "2", "bin"}, + {testUserFile, "100", ""}, // not in the file + {testUserFile, "8", ""}, // plus sign, glibc doesn't find it + {testUserFile, "9", ""}, // minus sign, glibc doesn't find it + {testUserFile, "27", ""}, // malformed + {testUserFile, "28", ""}, // commented out + {testUserFile, "29", ""}, // commented out, indented + {testUserFile, "3", "indented"}, + {testUserFile, "30", ""}, // the Gid is not valid, shouldn't match + {"", "1", ""}, +} + +func TestInvalidUserId(t *testing.T) { + _, err := findUserId("notanumber", strings.NewReader("")) + if err == nil { + t.Fatalf("findUserId('notanumber'): got nil error") + } + if want := "user: invalid userid notanumber"; err.Error() != want { + t.Errorf("findUserId('notanumber'): got %v, want %s", err, want) + } +} + +func TestLookupUserId(t *testing.T) { + for _, tt := range userIdTests { + got, err := findUserId(tt.uid, strings.NewReader(tt.in)) + if tt.name == "" { + if err == nil { + t.Errorf("findUserId(%s): got nil error, expected err", tt.uid) + continue + } + switch terr := err.(type) { + case UnknownUserIdError: + if want := "user: unknown userid " + tt.uid; terr.Error() != want { + t.Errorf("findUserId(%s): got %v, want %v", tt.name, terr, want) + } + default: + t.Errorf("findUserId(%s): got unexpected error %v", tt.name, terr) + } + } else { + if err != nil { + t.Fatalf("findUserId(%s): got unexpected error %v", tt.name, err) + } + if got.Uid != tt.uid { + t.Errorf("findUserId(%s): got uid %v, want %s", tt.name, got.Uid, tt.uid) + } + if got.Username != tt.name { + t.Errorf("findUserId(%s): got name %s, want %s", tt.name, got.Username, tt.name) + } + } + } +} + +func TestLookupUserPopulatesAllFields(t *testing.T) { + u, err := findUsername("allfields", strings.NewReader(testUserFile)) + if err != nil { + t.Fatal(err) + } + want := &User{ + Username: "allfields", + Uid: "6", + Gid: "12", + Name: "mansplit", + HomeDir: "/home/allfields", + } + if !reflect.DeepEqual(u, want) { + t.Errorf("findUsername: got %#v, want %#v", u, want) + } +} + +var userTests = []struct { + in string + name string + uid string +}{ + {testUserFile, "negative", "-5"}, + {testUserFile, "bin", "2"}, + {testUserFile, "notinthefile", ""}, + {testUserFile, "indented", "3"}, + {testUserFile, "plussign", ""}, + {testUserFile, "+plussign", ""}, + {testUserFile, "minussign", ""}, + {testUserFile, "-minussign", ""}, + {testUserFile, " indented", ""}, + {testUserFile, "commented", ""}, + {testUserFile, "commentindented", ""}, + {testUserFile, "malformed", ""}, + {testUserFile, "# commented", ""}, + {"", "emptyfile", ""}, +} + +func TestLookupUser(t *testing.T) { + for _, tt := range userTests { + got, err := findUsername(tt.name, strings.NewReader(tt.in)) + if tt.uid == "" { + if err == nil { + t.Errorf("lookupUser(%s): got nil error, expected err", tt.uid) + continue + } + switch terr := err.(type) { + case UnknownUserError: + if want := "user: unknown user " + tt.name; terr.Error() != want { + t.Errorf("lookupUser(%s): got %v, want %v", tt.name, terr, want) + } + default: + t.Errorf("lookupUser(%s): got unexpected error %v", tt.name, terr) + } + } else { + if err != nil { + t.Fatalf("lookupUser(%s): got unexpected error %v", tt.name, err) + } + if got.Uid != tt.uid { + t.Errorf("lookupUser(%s): got uid %v, want %s", tt.name, got.Uid, tt.uid) + } + if got.Username != tt.name { + t.Errorf("lookupUser(%s): got name %s, want %s", tt.name, got.Username, tt.name) + } + } + } +} diff --git a/libgo/go/os/user/user_test.go b/libgo/go/os/user/user_test.go index 9d8d94d8dae..b3aeed883cd 100644 --- a/libgo/go/os/user/user_test.go +++ b/libgo/go/os/user/user_test.go @@ -16,9 +16,6 @@ func checkUser(t *testing.T) { } func TestCurrent(t *testing.T) { - if runtime.GOOS == "android" { - t.Skipf("skipping on %s", runtime.GOOS) - } u, err := Current() if err != nil { t.Fatalf("Current: %v (got %#v)", err, u) @@ -31,6 +28,12 @@ func TestCurrent(t *testing.T) { } } +func BenchmarkCurrent(b *testing.B) { + for i := 0; i < b.N; i++ { + Current() + } +} + func compare(t *testing.T, want, got *User) { if want.Uid != got.Uid { t.Errorf("got Uid=%q; want %q", got.Uid, want.Uid) @@ -64,6 +67,9 @@ func TestLookup(t *testing.T) { if err != nil { t.Fatalf("Current: %v", err) } + // TODO: Lookup() has a fast path that calls Current() and returns if the + // usernames match, so this test does not exercise very much. It would be + // good to try and test finding a different user than the current user. got, err := Lookup(want.Username) if err != nil { t.Fatalf("Lookup: %v", err) diff --git a/libgo/go/os/wait_unimp.go b/libgo/go/os/wait_unimp.go index 0378b830b76..98243b567f4 100644 --- a/libgo/go/os/wait_unimp.go +++ b/libgo/go/os/wait_unimp.go @@ -2,7 +2,7 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -// +build aix dragonfly nacl netbsd openbsd solaris +// +build aix darwin dragonfly nacl netbsd openbsd solaris package os diff --git a/libgo/go/os/wait_waitid.go b/libgo/go/os/wait_waitid.go index 3337395510e..5a62b27f191 100644 --- a/libgo/go/os/wait_waitid.go +++ b/libgo/go/os/wait_waitid.go @@ -2,7 +2,10 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -// +build darwin linux +// We used to used this code for Darwin, but according to issue #19314 +// waitid returns if the process is stopped, even when using WEXITED. + +// +build linux package os diff --git a/libgo/go/path/example_test.go b/libgo/go/path/example_test.go index 88b76557f2b..21ed1fb2fcd 100644 --- a/libgo/go/path/example_test.go +++ b/libgo/go/path/example_test.go @@ -13,7 +13,12 @@ import ( func ExampleBase() { fmt.Println(path.Base("/a/b")) - // Output: b + fmt.Println(path.Base("/")) + fmt.Println(path.Base("")) + // Output: + // b + // / + // . } func ExampleClean() { @@ -24,6 +29,7 @@ func ExampleClean() { "a/c/b/..", "/../a/c", "/../a/b/../././/c", + "", } for _, p := range paths { @@ -37,16 +43,29 @@ func ExampleClean() { // Clean("a/c/b/..") = "a/c" // Clean("/../a/c") = "/a/c" // Clean("/../a/b/../././/c") = "/a/c" + // Clean("") = "." } func ExampleDir() { fmt.Println(path.Dir("/a/b/c")) - // Output: /a/b + fmt.Println(path.Dir("a/b/c")) + fmt.Println(path.Dir("/")) + fmt.Println(path.Dir("")) + // Output: + // /a/b + // a/b + // / + // . } func ExampleExt() { fmt.Println(path.Ext("/a/b/c/bar.css")) - // Output: .css + fmt.Println(path.Ext("/")) + fmt.Println(path.Ext("")) + // Output: + // .css + // + // } func ExampleIsAbs() { @@ -58,17 +77,26 @@ func ExampleJoin() { fmt.Println(path.Join("a", "b", "c")) fmt.Println(path.Join("a", "b/c")) fmt.Println(path.Join("a/b", "c")) - fmt.Println(path.Join("a/b", "/c")) + fmt.Println(path.Join("", "")) + fmt.Println(path.Join("a", "")) + fmt.Println(path.Join("", "a")) // Output: // a/b/c // a/b/c // a/b/c - // a/b/c + // + // a + // a } func ExampleSplit() { fmt.Println(path.Split("static/myfile.css")) - // Output: static/ myfile.css + fmt.Println(path.Split("myfile.css")) + fmt.Println(path.Split("")) + // Output: + // static/ myfile.css + // myfile.css + // } */ diff --git a/libgo/go/path/filepath/match_test.go b/libgo/go/path/filepath/match_test.go index ae7ca1c228f..12d922f83bc 100644 --- a/libgo/go/path/filepath/match_test.go +++ b/libgo/go/path/filepath/match_test.go @@ -155,8 +155,8 @@ func TestGlob(t *testing.T) { } func TestGlobError(t *testing.T) { - _, err := Glob("[7]") - if err != nil { + _, err := Glob("[]") + if err == nil { t.Error("expected error for bad pattern; got none") } } diff --git a/libgo/go/path/filepath/path.go b/libgo/go/path/filepath/path.go index 1d8e35c969e..c242143c7a3 100644 --- a/libgo/go/path/filepath/path.go +++ b/libgo/go/path/filepath/path.go @@ -4,6 +4,11 @@ // Package filepath implements utility routines for manipulating filename paths // in a way compatible with the target operating system-defined file paths. +// +// The filepath package uses either forward slashes or backslashes, +// depending on the operating system. To process paths such as URLs +// that always use forward slashes regardless of the operating +// system, see the path package. package filepath import ( @@ -461,6 +466,10 @@ func Dir(path string) string { i-- } dir := Clean(path[len(vol) : i+1]) + if dir == "." && len(vol) > 2 { + // must be UNC + return vol + } return vol + dir } diff --git a/libgo/go/path/filepath/path_test.go b/libgo/go/path/filepath/path_test.go index 7389ea29a48..f2e92528a96 100644 --- a/libgo/go/path/filepath/path_test.go +++ b/libgo/go/path/filepath/path_test.go @@ -6,12 +6,14 @@ package filepath_test import ( "errors" + "fmt" "internal/testenv" "io/ioutil" "os" "path/filepath" "reflect" "runtime" + "sort" "strings" "testing" ) @@ -389,7 +391,7 @@ func checkMarks(t *testing.T, report bool) { // Assumes that each node name is unique. Good enough for a test. // If clear is true, any incoming error is cleared before return. The errors // are always accumulated, though. -func mark(path string, info os.FileInfo, err error, errors *[]error, clear bool) error { +func mark(info os.FileInfo, err error, errors *[]error, clear bool) error { if err != nil { *errors = append(*errors, err) if clear { @@ -438,7 +440,7 @@ func TestWalk(t *testing.T) { errors := make([]error, 0, 10) clear := true markFn := func(path string, info os.FileInfo, err error) error { - return mark(path, info, err, &errors, clear) + return mark(info, err, &errors, clear) } // Expect no errors. err := filepath.Walk(tree.name, markFn) @@ -668,6 +670,7 @@ var windirtests = []PathTest{ {`c:\a\b`, `c:\a`}, {`c:a\b`, `c:a`}, {`c:a\b\c`, `c:a\b`}, + {`\\host\share`, `\\host\share`}, {`\\host\share\`, `\\host\share\`}, {`\\host\share\a`, `\\host\share\`}, {`\\host\share\a\b`, `\\host\share\a`}, @@ -1330,3 +1333,53 @@ func TestBug3486(t *testing.T) { // https://golang.org/issue/3486 t.Fatalf("%q not seen", ken) } } + +func testWalkSymlink(t *testing.T, mklink func(target, link string) error) { + tmpdir, err := ioutil.TempDir("", "testWalkSymlink") + if err != nil { + t.Fatal(err) + } + defer os.RemoveAll(tmpdir) + + wd, err := os.Getwd() + if err != nil { + t.Fatal(err) + } + defer os.Chdir(wd) + + err = os.Chdir(tmpdir) + if err != nil { + t.Fatal(err) + } + + err = mklink(tmpdir, "link") + if err != nil { + t.Fatal(err) + } + + var visited []string + err = filepath.Walk(tmpdir, func(path string, info os.FileInfo, err error) error { + if err != nil { + t.Fatal(err) + } + rel, err := filepath.Rel(tmpdir, path) + if err != nil { + t.Fatal(err) + } + visited = append(visited, rel) + return nil + }) + if err != nil { + t.Fatal(err) + } + sort.Strings(visited) + want := []string{".", "link"} + if fmt.Sprintf("%q", visited) != fmt.Sprintf("%q", want) { + t.Errorf("unexpected paths visited %q, want %q", visited, want) + } +} + +func TestWalkSymlink(t *testing.T) { + testenv.MustHaveSymlink(t) + testWalkSymlink(t, os.Symlink) +} diff --git a/libgo/go/path/path.go b/libgo/go/path/path.go index 76c7814c59d..5c905110a1b 100644 --- a/libgo/go/path/path.go +++ b/libgo/go/path/path.go @@ -5,7 +5,10 @@ // Package path implements utility routines for manipulating slash-separated // paths. // -// To manipulate operating system paths, use the path/filepath package. +// The path package should only be used for paths separated by forward +// slashes, such as the paths in URLs. This package does not deal with +// Windows paths with drive letters or backslashes; to manipulate +// operating system paths, use the path/filepath package. package path import ( diff --git a/libgo/go/plugin/plugin.go b/libgo/go/plugin/plugin.go index b86099a4f6f..c7744658122 100644 --- a/libgo/go/plugin/plugin.go +++ b/libgo/go/plugin/plugin.go @@ -4,8 +4,6 @@ // Package plugin implements loading and symbol resolution of Go plugins. // -// Currently plugins only work on Linux. -// // A plugin is a Go main package with exported functions and variables that // has been built with: // @@ -14,6 +12,9 @@ // When a plugin is first opened, the init functions of all packages not // already part of the program are called. The main function is not run. // A plugin is only initialized once, and cannot be closed. +// +// The plugin support is currently incomplete, only supports Linux, +// and has known bugs. Please report any issues. package plugin // Plugin is a loaded Go plugin. @@ -44,9 +45,6 @@ func (p *Plugin) Lookup(symName string) (Symbol, error) { // // package main // -// // // No C code needed. -// import "C" -// // import "fmt" // // var V int diff --git a/libgo/go/plugin/plugin_dlopen.go b/libgo/go/plugin/plugin_dlopen.go index c5b0a4721c5..3237598f06b 100644 --- a/libgo/go/plugin/plugin_dlopen.go +++ b/libgo/go/plugin/plugin_dlopen.go @@ -39,6 +39,47 @@ import ( "unsafe" ) +// avoid a dependency on strings +func lastIndexByte(s string, c byte) int { + for i := len(s) - 1; i >= 0; i-- { + if s[i] == c { + return i + } + } + return -1 +} + +// pathToPrefix converts raw string to the prefix that will be used in the symbol +// table. If modifying, modify the version in internal/obj/sym.go as well. +func pathToPrefix(s string) string { + slash := lastIndexByte(s, '/') + // check for chars that need escaping + n := 0 + for r := 0; r < len(s); r++ { + if c := s[r]; c <= ' ' || (c == '.' && r > slash) || c == '%' || c == '"' || c >= 0x7F { + n++ + } + } + + // quick exit + if n == 0 { + return s + } + + // escape + const hex = "0123456789abcdef" + p := make([]byte, 0, len(s)+2*n) + for r := 0; r < len(s); r++ { + if c := s[r]; c <= ' ' || (c == '.' && r > slash) || c == '%' || c == '"' || c >= 0x7F { + p = append(p, '%', hex[c>>4], hex[c&0xF]) + } else { + p = append(p, c) + } + } + + return string(p) +} + func open(name string) (*Plugin, error) { cPath := (*C.char)(C.malloc(C.PATH_MAX + 1)) defer C.free(unsafe.Pointer(cPath)) @@ -82,7 +123,6 @@ func open(name string) (*Plugin, error) { p := &Plugin{ pluginpath: pluginpath, loaded: make(chan struct{}), - syms: syms, } plugins[filepath] = p pluginsMu.Unlock() @@ -97,14 +137,14 @@ func open(name string) (*Plugin, error) { } // Fill out the value of each plugin symbol. + updatedSyms := map[string]interface{}{} for symName, sym := range syms { isFunc := symName[0] == '.' if isFunc { delete(syms, symName) symName = symName[1:] } - - cname := C.CString(pluginpath + "." + symName) + cname := C.CString(pathToPrefix(pluginpath) + "." + symName) p := C.pluginLookup(h, cname, &cErr) C.free(unsafe.Pointer(cname)) if p == nil { @@ -116,8 +156,12 @@ func open(name string) (*Plugin, error) { } else { (*valp)[1] = p } - syms[symName] = sym + // we can't add to syms during iteration as we'll end up processing + // some symbols twice with the inability to tell if the symbol is a function + updatedSyms[symName] = sym } + p.syms = updatedSyms + close(p.loaded) return p, nil } diff --git a/libgo/go/reflect/all_test.go b/libgo/go/reflect/all_test.go index 6ac33526774..736467319ee 100644 --- a/libgo/go/reflect/all_test.go +++ b/libgo/go/reflect/all_test.go @@ -1576,9 +1576,11 @@ func BenchmarkCallArgCopy(b *testing.B) { args := []Value{size.arg} b.SetBytes(int64(size.arg.Len())) b.ResetTimer() - for i := 0; i < b.N; i++ { - size.fv.Call(args) - } + b.RunParallel(func(pb *testing.PB) { + for pb.Next() { + size.fv.Call(args) + } + }) } name := fmt.Sprintf("size=%v", size.arg.Len()) b.Run(name, bench) @@ -2559,6 +2561,28 @@ func TestPtrToGC(t *testing.T) { } } +func BenchmarkPtrTo(b *testing.B) { + // Construct a type with a zero ptrToThis. + type T struct{ int } + t := SliceOf(TypeOf(T{})) + ptrToThis := ValueOf(t).Elem().FieldByName("ptrToThis") + if !ptrToThis.IsValid() { + b.Fatalf("%v has no ptrToThis field; was it removed from rtype?", t) + } + if ptrToThis.Int() != 0 { + b.Fatalf("%v.ptrToThis unexpectedly nonzero", t) + } + b.ResetTimer() + + // Now benchmark calling PtrTo on it: we'll have to hit the ptrMap cache on + // every call. + b.RunParallel(func(pb *testing.PB) { + for pb.Next() { + PtrTo(t) + } + }) +} + func TestAddr(t *testing.T) { var p struct { X, Y int @@ -3738,7 +3762,7 @@ func checkSameType(t *testing.T, x, y interface{}) { func TestArrayOf(t *testing.T) { // check construction and use of type not in binary - for _, table := range []struct { + tests := []struct { n int value func(i int) interface{} comparable bool @@ -3816,7 +3840,9 @@ func TestArrayOf(t *testing.T) { comparable: true, want: "[{0 0} {1 1} {2 2} {3 3} {4 4} {5 5} {6 6} {7 7} {8 8} {9 9}]", }, - } { + } + + for _, table := range tests { at := ArrayOf(table.n, TypeOf(table.value(0))) v := New(at).Elem() vok := New(at).Elem() @@ -4045,6 +4071,54 @@ func TestSliceOfGC(t *testing.T) { } } +func TestStructOfFieldName(t *testing.T) { + // invalid field name "1nvalid" + shouldPanic(func() { + StructOf([]StructField{ + StructField{Name: "valid", Type: TypeOf("")}, + StructField{Name: "1nvalid", Type: TypeOf("")}, + }) + }) + + // invalid field name "+" + shouldPanic(func() { + StructOf([]StructField{ + StructField{Name: "val1d", Type: TypeOf("")}, + StructField{Name: "+", Type: TypeOf("")}, + }) + }) + + // no field name + shouldPanic(func() { + StructOf([]StructField{ + StructField{Name: "", Type: TypeOf("")}, + }) + }) + + // verify creation of a struct with valid struct fields + validFields := []StructField{ + StructField{ + Name: "φ", + Type: TypeOf(""), + }, + StructField{ + Name: "ValidName", + Type: TypeOf(""), + }, + StructField{ + Name: "Val1dNam5", + Type: TypeOf(""), + }, + } + + validStruct := StructOf(validFields) + + const structStr = `struct { φ string; ValidName string; Val1dNam5 string }` + if got, want := validStruct.String(), structStr; got != want { + t.Errorf("StructOf(validFields).String()=%q, want %q", got, want) + } +} + func TestStructOf(t *testing.T) { // check construction and use of type not in binary fields := []StructField{ @@ -4392,7 +4466,7 @@ func TestStructOfGenericAlg(t *testing.T) { {Name: "S1", Type: st1}, }) - for _, table := range []struct { + tests := []struct { rt Type idx []int }{ @@ -4473,7 +4547,9 @@ func TestStructOfGenericAlg(t *testing.T) { ), idx: []int{2}, }, - } { + } + + for _, table := range tests { v1 := New(table.rt).Elem() v2 := New(table.rt).Elem() @@ -4582,18 +4658,21 @@ func TestStructOfWithInterface(t *testing.T) { type Iface interface { Get() int } - for i, table := range []struct { + tests := []struct { + name string typ Type val Value impl bool }{ { + name: "StructI", typ: TypeOf(StructI(want)), val: ValueOf(StructI(want)), impl: true, }, { - typ: PtrTo(TypeOf(StructI(want))), + name: "StructI", + typ: PtrTo(TypeOf(StructI(want))), val: ValueOf(func() interface{} { v := StructI(want) return &v @@ -4601,7 +4680,8 @@ func TestStructOfWithInterface(t *testing.T) { impl: true, }, { - typ: PtrTo(TypeOf(StructIPtr(want))), + name: "StructIPtr", + typ: PtrTo(TypeOf(StructIPtr(want))), val: ValueOf(func() interface{} { v := StructIPtr(want) return &v @@ -4609,6 +4689,7 @@ func TestStructOfWithInterface(t *testing.T) { impl: true, }, { + name: "StructIPtr", typ: TypeOf(StructIPtr(want)), val: ValueOf(StructIPtr(want)), impl: false, @@ -4618,41 +4699,70 @@ func TestStructOfWithInterface(t *testing.T) { // val: ValueOf(StructI(want)), // impl: true, // }, - } { - rt := StructOf( - []StructField{ - { - Name: "", + } + + for i, table := range tests { + for j := 0; j < 2; j++ { + var fields []StructField + if j == 1 { + fields = append(fields, StructField{ + Name: "Dummy", PkgPath: "", - Type: table.typ, - }, - }, - ) - rv := New(rt).Elem() - rv.Field(0).Set(table.val) + Type: TypeOf(int(0)), + }) + } + fields = append(fields, StructField{ + Name: table.name, + Anonymous: true, + PkgPath: "", + Type: table.typ, + }) - if _, ok := rv.Interface().(Iface); ok != table.impl { - if table.impl { - t.Errorf("test-%d: type=%v fails to implement Iface.\n", i, table.typ) - } else { - t.Errorf("test-%d: type=%v should NOT implement Iface\n", i, table.typ) + // We currently do not correctly implement methods + // for anonymous fields other than the first. + // Therefore, for now, we expect those methods + // to not exist. See issues 15924 and 20824. + // When those issues are fixed, this test of panic + // should be removed. + if j == 1 && table.impl { + func() { + defer func() { + if err := recover(); err == nil { + t.Errorf("test-%d-%d did not panic", i, j) + } + }() + _ = StructOf(fields) + }() + continue } - continue - } - if !table.impl { - continue - } + rt := StructOf(fields) + rv := New(rt).Elem() + rv.Field(j).Set(table.val) - v := rv.Interface().(Iface).Get() - if v != want { - t.Errorf("test-%d: x.Get()=%v. want=%v\n", i, v, want) - } + if _, ok := rv.Interface().(Iface); ok != table.impl { + if table.impl { + t.Errorf("test-%d-%d: type=%v fails to implement Iface.\n", i, j, table.typ) + } else { + t.Errorf("test-%d-%d: type=%v should NOT implement Iface\n", i, j, table.typ) + } + continue + } - fct := rv.MethodByName("Get") - out := fct.Call(nil) - if !DeepEqual(out[0].Interface(), want) { - t.Errorf("test-%d: x.Get()=%v. want=%v\n", i, out[0].Interface(), want) + if !table.impl { + continue + } + + v := rv.Interface().(Iface).Get() + if v != want { + t.Errorf("test-%d-%d: x.Get()=%v. want=%v\n", i, j, v, want) + } + + fct := rv.MethodByName("Get") + out := fct.Call(nil) + if !DeepEqual(out[0].Interface(), want) { + t.Errorf("test-%d-%d: x.Get()=%v. want=%v\n", i, j, out[0].Interface(), want) + } } } } @@ -4914,16 +5024,20 @@ type B1 struct { func BenchmarkFieldByName1(b *testing.B) { t := TypeOf(B1{}) - for i := 0; i < b.N; i++ { - t.FieldByName("Z") - } + b.RunParallel(func(pb *testing.PB) { + for pb.Next() { + t.FieldByName("Z") + } + }) } func BenchmarkFieldByName2(b *testing.B) { t := TypeOf(S3{}) - for i := 0; i < b.N; i++ { - t.FieldByName("B") - } + b.RunParallel(func(pb *testing.PB) { + for pb.Next() { + t.FieldByName("B") + } + }) } type R0 struct { @@ -5006,9 +5120,11 @@ func TestEmbed(t *testing.T) { func BenchmarkFieldByName3(b *testing.B) { t := TypeOf(R0{}) - for i := 0; i < b.N; i++ { - t.FieldByName("X") - } + b.RunParallel(func(pb *testing.PB) { + for pb.Next() { + t.FieldByName("X") + } + }) } type S struct { @@ -5018,9 +5134,11 @@ type S struct { func BenchmarkInterfaceBig(b *testing.B) { v := ValueOf(S{}) - for i := 0; i < b.N; i++ { - v.Interface() - } + b.RunParallel(func(pb *testing.PB) { + for pb.Next() { + v.Interface() + } + }) b.StopTimer() } @@ -5036,9 +5154,11 @@ func TestAllocsInterfaceBig(t *testing.T) { func BenchmarkInterfaceSmall(b *testing.B) { v := ValueOf(int64(0)) - for i := 0; i < b.N; i++ { - v.Interface() - } + b.RunParallel(func(pb *testing.PB) { + for pb.Next() { + v.Interface() + } + }) } func TestAllocsInterfaceSmall(t *testing.T) { @@ -5835,7 +5955,7 @@ func TestTypeOfTypeOf(t *testing.T) { check("SliceOf", SliceOf(TypeOf(T{}))) } -type XM struct{} +type XM struct{ _ bool } func (*XM) String() string { return "" } @@ -5861,6 +5981,24 @@ func TestMapAlloc(t *testing.T) { if allocs > 0.5 { t.Errorf("allocs per map assignment: want 0 got %f", allocs) } + + const size = 1000 + tmp := 0 + val := ValueOf(&tmp).Elem() + allocs = testing.AllocsPerRun(100, func() { + mv := MakeMapWithSize(TypeOf(map[int]int{}), size) + // Only adding half of the capacity to not trigger re-allocations due too many overloaded buckets. + for i := 0; i < size/2; i++ { + val.SetInt(int64(i)) + mv.SetMapIndex(val, val) + } + }) + if allocs > 10 { + t.Errorf("allocs per map assignment: want at most 10 got %f", allocs) + } + // Empirical testing shows that with capacity hint single run will trigger 3 allocations and without 91. I set + // the threshold to 10, to not make it overly brittle if something changes in the initial allocation of the + // map, but to still catch a regression where we keep re-allocating in the hashmap as new entries are added. } func TestChanAlloc(t *testing.T) { @@ -5984,6 +6122,8 @@ func TestTypeStrings(t *testing.T) { {TypeOf(new(XM)).Method(0).Type, "func(*reflect_test.XM) string"}, {ChanOf(3, TypeOf(XM{})), "chan reflect_test.XM"}, {MapOf(TypeOf(int(0)), TypeOf(XM{})), "map[int]reflect_test.XM"}, + {ArrayOf(3, TypeOf(XM{})), "[3]reflect_test.XM"}, + {ArrayOf(3, TypeOf(struct{}{})), "[3]struct {}"}, } for i, test := range stringTests { @@ -6014,9 +6154,11 @@ func TestOffsetLock(t *testing.T) { func BenchmarkNew(b *testing.B) { v := TypeOf(XM{}) - for i := 0; i < b.N; i++ { - New(v) - } + b.RunParallel(func(pb *testing.PB) { + for pb.Next() { + New(v) + } + }) } func TestSwapper(t *testing.T) { @@ -6091,6 +6233,7 @@ func TestSwapper(t *testing.T) { want: []pairPtr{{5, 6, &c}, {3, 4, &b}, {1, 2, &a}}, }, } + for i, tt := range tests { inStr := fmt.Sprint(tt.in) Swapper(tt.in)(tt.i, tt.j) @@ -6116,3 +6259,36 @@ func TestUnaddressableField(t *testing.T) { lv.Set(rv) }) } + +type Tint int + +type Tint2 = Tint + +type Talias1 struct { + byte + uint8 + int + int32 + rune +} + +type Talias2 struct { + Tint + Tint2 +} + +func TestAliasNames(t *testing.T) { + t1 := Talias1{byte: 1, uint8: 2, int: 3, int32: 4, rune: 5} + out := fmt.Sprintf("%#v", t1) + want := "reflect_test.Talias1{byte:0x1, uint8:0x2, int:3, int32:4, rune:5}" + if out != want { + t.Errorf("Talias1 print:\nhave: %s\nwant: %s", out, want) + } + + t2 := Talias2{Tint: 1, Tint2: 2} + out = fmt.Sprintf("%#v", t2) + want = "reflect_test.Talias2{Tint:1, Tint2:2}" + if out != want { + t.Errorf("Talias2 print:\nhave: %s\nwant: %s", out, want) + } +} diff --git a/libgo/go/reflect/deepequal.go b/libgo/go/reflect/deepequal.go index f3fd7043e5a..2fdd6a3d82b 100644 --- a/libgo/go/reflect/deepequal.go +++ b/libgo/go/reflect/deepequal.go @@ -178,6 +178,12 @@ func deepValueEqual(v1, v2 Value, visited map[visit]bool, depth int) bool { // DeepEqual has been defined so that the same short-cut applies // to slices and maps: if x and y are the same slice or the same map, // they are deeply equal regardless of content. +// +// As DeepEqual traverses the data values it may find a cycle. The +// second and subsequent times that DeepEqual compares two pointer +// values that have been compared before, it treats the values as +// equal rather than examining the values to which they point. +// This ensures that DeepEqual terminates. func DeepEqual(x, y interface{}) bool { if x == nil || y == nil { return x == y diff --git a/libgo/go/reflect/set_test.go b/libgo/go/reflect/set_test.go index bc35c78e1bb..7c39623a9db 100644 --- a/libgo/go/reflect/set_test.go +++ b/libgo/go/reflect/set_test.go @@ -7,6 +7,7 @@ package reflect_test import ( "bytes" "go/ast" + "go/token" "io" . "reflect" "testing" @@ -172,6 +173,23 @@ var implementsTests = []struct { {new(bytes.Buffer), new(io.Reader), false}, {new(*bytes.Buffer), new(io.ReaderAt), false}, {new(*ast.Ident), new(ast.Expr), true}, + {new(*notAnExpr), new(ast.Expr), false}, + {new(*ast.Ident), new(notASTExpr), false}, + {new(notASTExpr), new(ast.Expr), false}, + {new(ast.Expr), new(notASTExpr), false}, + {new(*notAnExpr), new(notASTExpr), true}, +} + +type notAnExpr struct{} + +func (notAnExpr) Pos() token.Pos { return token.NoPos } +func (notAnExpr) End() token.Pos { return token.NoPos } +func (notAnExpr) exprNode() {} + +type notASTExpr interface { + Pos() token.Pos + End() token.Pos + exprNode() } func TestImplements(t *testing.T) { diff --git a/libgo/go/reflect/type.go b/libgo/go/reflect/type.go index 97b986a7bba..664d9717a06 100644 --- a/libgo/go/reflect/type.go +++ b/libgo/go/reflect/type.go @@ -18,6 +18,8 @@ package reflect import ( "strconv" "sync" + "unicode" + "unicode/utf8" "unsafe" ) @@ -258,6 +260,8 @@ const ( // It is embedded in other, public struct types, but always // with a unique tag like `reflect:"array"` or `reflect:"ptr"` // so that code cannot convert from, say, *arrayType to *ptrType. +// +// rtype must be kept in sync with ../runtime/type.go:/^type._type. type rtype struct { size uintptr ptrdata uintptr // size of memory prefix holding all pointers @@ -516,79 +520,52 @@ func (t *rtype) pointers() bool { return t.kind&kindNoPointers == 0 } func (t *rtype) common() *rtype { return t } -func (t *uncommonType) Method(i int) (m Method) { - if t == nil || i < 0 || i >= len(t.methods) { - panic("reflect: Method index out of range") - } - found := false - for mi := range t.methods { - if t.methods[mi].pkgPath == nil { - if i == 0 { - i = mi - found = true - break - } - i-- - } - } - if !found { - panic("reflect: Method index out of range") - } +var methodCache sync.Map // map[*rtype][]method - p := &t.methods[i] - if p.name != nil { - m.Name = *p.name - } - fl := flag(Func) - if p.pkgPath != nil { - m.PkgPath = *p.pkgPath - fl |= flagStickyRO +func (t *rtype) exportedMethods() []method { + methodsi, found := methodCache.Load(t) + if found { + return methodsi.([]method) } - mt := p.typ - m.Type = toType(mt) - x := new(unsafe.Pointer) - *x = unsafe.Pointer(&p.tfn) - m.Func = Value{mt, unsafe.Pointer(x), fl | flagIndir | flagMethodFn} - m.Index = i - return -} -func (t *uncommonType) NumMethod() int { - if t == nil { - return 0 + ut := t.uncommon() + if ut == nil { + return nil } - c := 0 - for i := range t.methods { - if t.methods[i].pkgPath == nil { - c++ + allm := ut.methods + allExported := true + for _, m := range allm { + if m.pkgPath != nil { + allExported = false + break } } - return c -} - -func (t *uncommonType) MethodByName(name string) (m Method, ok bool) { - if t == nil { - return - } - var p *method - for i := range t.methods { - p = &t.methods[i] - if p.pkgPath == nil && p.name != nil && *p.name == name { - return t.Method(i), true + var methods []method + if allExported { + methods = allm + } else { + methods = make([]method, 0, len(allm)) + for _, m := range allm { + if m.pkgPath == nil { + methods = append(methods, m) + } } + methods = methods[:len(methods):len(methods)] } - return + + methodsi, _ = methodCache.LoadOrStore(t, methods) + return methodsi.([]method) } -// TODO(rsc): gc supplies these, but they are not -// as efficient as they could be: they have commonType -// as the receiver instead of *rtype. func (t *rtype) NumMethod() int { if t.Kind() == Interface { tt := (*interfaceType)(unsafe.Pointer(t)) return tt.NumMethod() } - return t.uncommonType.NumMethod() + if t.uncommonType == nil { + return 0 // avoid methodCache synchronization + } + return len(t.exportedMethods()) } func (t *rtype) Method(i int) (m Method) { @@ -596,7 +573,22 @@ func (t *rtype) Method(i int) (m Method) { tt := (*interfaceType)(unsafe.Pointer(t)) return tt.Method(i) } - return t.uncommonType.Method(i) + methods := t.exportedMethods() + if i < 0 || i >= len(methods) { + panic("reflect: Method index out of range") + } + p := methods[i] + if p.name != nil { + m.Name = *p.name + } + fl := flag(Func) + mt := p.typ + m.Type = toType(mt) + x := new(unsafe.Pointer) + *x = unsafe.Pointer(&p.tfn) + m.Func = Value{mt, unsafe.Pointer(x), fl | flagIndir | flagMethodFn} + m.Index = i + return m } func (t *rtype) MethodByName(name string) (m Method, ok bool) { @@ -604,7 +596,17 @@ func (t *rtype) MethodByName(name string) (m Method, ok bool) { tt := (*interfaceType)(unsafe.Pointer(t)) return tt.MethodByName(name) } - return t.uncommonType.MethodByName(name) + ut := t.uncommon() + if ut == nil { + return Method{}, false + } + for i := range ut.methods { + p := &ut.methods[i] + if p.pkgPath == nil && p.name != nil && *p.name == name { + return t.Method(i), true + } + } + return Method{}, false } func (t *rtype) PkgPath() string { @@ -983,12 +985,11 @@ func (t *structType) FieldByNameFunc(match func(string) bool) (result StructFiel visited[t] = true for i := range t.fields { f := &t.fields[i] - // Find name and type for field f. + // Find name and (for anonymous field) type for field f. fname := *f.name var ntyp *rtype if f.anon() { // Anonymous field of type T or *T. - // Name taken from type. ntyp = f.typ if ntyp.Kind() == Ptr { ntyp = ntyp.Elem().common() @@ -1072,10 +1073,7 @@ func TypeOf(i interface{}) Type { } // ptrMap is the cache for PtrTo. -var ptrMap struct { - sync.RWMutex - m map[*rtype]*ptrType -} +var ptrMap sync.Map // map[*rtype]*ptrType // PtrTo returns the pointer type with element t. // For example, if t represents type Foo, PtrTo(t) represents *Foo. @@ -1089,24 +1087,8 @@ func (t *rtype) ptrTo() *rtype { } // Check the cache. - ptrMap.RLock() - if m := ptrMap.m; m != nil { - if p := m[t]; p != nil { - ptrMap.RUnlock() - return &p.rtype - } - } - ptrMap.RUnlock() - - ptrMap.Lock() - if ptrMap.m == nil { - ptrMap.m = make(map[*rtype]*ptrType) - } - p := ptrMap.m[t] - if p != nil { - // some other goroutine won the race and created it - ptrMap.Unlock() - return &p.rtype + if pi, ok := ptrMap.Load(t); ok { + return &pi.(*ptrType).rtype } s := "*" + *t.string @@ -1115,9 +1097,9 @@ func (t *rtype) ptrTo() *rtype { r, ok := canonicalType[s] canonicalTypeLock.RUnlock() if ok { - ptrMap.m[t] = (*ptrType)(unsafe.Pointer(r.(*rtype))) - ptrMap.Unlock() - return r.(*rtype) + p := (*ptrType)(unsafe.Pointer(r.(*rtype))) + pi, _ := ptrMap.LoadOrStore(t, p) + return &pi.(*ptrType).rtype } // Create a new ptrType starting with the description @@ -1143,11 +1125,10 @@ func (t *rtype) ptrTo() *rtype { pp.elem = t q := canonicalize(&pp.rtype) - p = (*ptrType)(unsafe.Pointer(q.(*rtype))) + p := (*ptrType)(unsafe.Pointer(q.(*rtype))) - ptrMap.m[t] = p - ptrMap.Unlock() - return &p.rtype + pi, _ := ptrMap.LoadOrStore(t, p) + return &pi.(*ptrType).rtype } // fnv1 incorporates the list of bytes into the hash x using the FNV-1 hash function. @@ -1396,11 +1377,8 @@ func haveIdenticalUnderlyingType(T, V *rtype, cmpTags bool) bool { return false } -// The lookupCache caches ChanOf, MapOf, and SliceOf lookups. -var lookupCache struct { - sync.RWMutex - m map[cacheKey]*rtype -} +// The lookupCache caches ArrayOf, ChanOf, MapOf and SliceOf lookups. +var lookupCache sync.Map // map[cacheKey]*rtype // A cacheKey is the key for use in the lookupCache. // Four values describe any of the types we are looking for: @@ -1412,48 +1390,15 @@ type cacheKey struct { extra uintptr } -// cacheGet looks for a type under the key k in the lookupCache. -// If it finds one, it returns that type. -// If not, it returns nil with the cache locked. -// The caller is expected to use cachePut to unlock the cache. -func cacheGet(k cacheKey) Type { - lookupCache.RLock() - t := lookupCache.m[k] - lookupCache.RUnlock() - if t != nil { - return t - } - - lookupCache.Lock() - t = lookupCache.m[k] - if t != nil { - lookupCache.Unlock() - return t - } - - if lookupCache.m == nil { - lookupCache.m = make(map[cacheKey]*rtype) - } - - return nil -} - -// cachePut stores the given type in the cache, unlocks the cache, -// and returns the type. It is expected that the cache is locked -// because cacheGet returned nil. -func cachePut(k cacheKey, t *rtype) Type { - t = toType(t).common() - lookupCache.m[k] = t - lookupCache.Unlock() - return t -} - // The funcLookupCache caches FuncOf lookups. // FuncOf does not share the common lookupCache since cacheKey is not // sufficient to represent functions unambiguously. var funcLookupCache struct { - sync.RWMutex - m map[uint32][]*rtype // keyed by hash calculated in FuncOf + sync.Mutex // Guards stores (but not loads) on m. + + // m is a map[uint32][]*rtype keyed by the hash calculated in FuncOf. + // Elements of m are append-only and thus safe for concurrent reading. + m sync.Map } // ChanOf returns the channel type with the given direction and element type. @@ -1466,13 +1411,12 @@ func ChanOf(dir ChanDir, t Type) Type { // Look in cache. ckey := cacheKey{Chan, typ, nil, uintptr(dir)} - if ch := cacheGet(ckey); ch != nil { - return ch + if ch, ok := lookupCache.Load(ckey); ok { + return ch.(*rtype) } // This restriction is imposed by the gc compiler and the runtime. if typ.size >= 1<<16 { - lookupCache.Unlock() panic("reflect.ChanOf: element size too large") } @@ -1481,7 +1425,6 @@ func ChanOf(dir ChanDir, t Type) Type { var s string switch dir { default: - lookupCache.Unlock() panic("reflect.ChanOf: invalid dir") case SendDir: s = "chan<- " + *typ.string @@ -1515,7 +1458,8 @@ func ChanOf(dir ChanDir, t Type) Type { ch.uncommonType = nil ch.ptrToThis = nil - return cachePut(ckey, &ch.rtype) + ti, _ := lookupCache.LoadOrStore(ckey, &ch.rtype) + return ti.(Type) } func ismapkey(*rtype) bool // implemented in runtime @@ -1536,8 +1480,8 @@ func MapOf(key, elem Type) Type { // Look in cache. ckey := cacheKey{Map, ktyp, etyp, 0} - if mt := cacheGet(ckey); mt != nil { - return mt + if mt, ok := lookupCache.Load(ckey); ok { + return mt.(Type) } // Look in known types. @@ -1576,7 +1520,8 @@ func MapOf(key, elem Type) Type { mt.reflexivekey = isReflexive(ktyp) mt.needkeyupdate = needKeyUpdate(ktyp) - return cachePut(ckey, &mt.rtype) + ti, _ := lookupCache.LoadOrStore(ckey, &mt.rtype) + return ti.(Type) } // FuncOf returns the function type with the given argument and result types. @@ -1625,25 +1570,32 @@ func FuncOf(in, out []Type, variadic bool) Type { ft.dotdotdot = variadic // Look in cache. - funcLookupCache.RLock() - for _, t := range funcLookupCache.m[hash] { - if haveIdenticalUnderlyingType(&ft.rtype, t, true) { - funcLookupCache.RUnlock() - return t + if ts, ok := funcLookupCache.m.Load(hash); ok { + for _, t := range ts.([]*rtype) { + if haveIdenticalUnderlyingType(&ft.rtype, t, true) { + return t + } } } - funcLookupCache.RUnlock() // Not in cache, lock and retry. funcLookupCache.Lock() defer funcLookupCache.Unlock() - if funcLookupCache.m == nil { - funcLookupCache.m = make(map[uint32][]*rtype) + if ts, ok := funcLookupCache.m.Load(hash); ok { + for _, t := range ts.([]*rtype) { + if haveIdenticalUnderlyingType(&ft.rtype, t, true) { + return t + } + } } - for _, t := range funcLookupCache.m[hash] { - if haveIdenticalUnderlyingType(&ft.rtype, t, true) { - return t + + addToCache := func(tt *rtype) Type { + var rts []*rtype + if rti, ok := funcLookupCache.m.Load(hash); ok { + rts = rti.([]*rtype) } + funcLookupCache.m.Store(hash, append(rts, tt)) + return tt } str := funcStr(ft) @@ -1652,10 +1604,7 @@ func FuncOf(in, out []Type, variadic bool) Type { ft.string = &str ft.uncommonType = nil ft.ptrToThis = nil - - funcLookupCache.m[hash] = append(funcLookupCache.m[hash], &ft.rtype) - - return toType(&ft.rtype) + return addToCache(&ft.rtype) } // funcStr builds a string representation of a funcType. @@ -1771,9 +1720,6 @@ func bucketOf(ktyp, etyp *rtype) *rtype { // Prepare GC data if any. // A bucket is at most bucketSize*(1+maxKeySize+maxValSize)+2*ptrSize bytes, // or 2072 bytes, or 259 pointer-size words, or 33 bytes of pointer bitmap. - // Normally the enforced limit on pointer maps is 16 bytes, - // but larger ones are acceptable, 33 bytes isn't too too big, - // and it's easier to generate a pointer bitmap than a GC program. // Note that since the key and value are known to be <= 128 bytes, // they're guaranteed to have bitmaps instead of GC programs. var gcdata *byte @@ -1812,7 +1758,7 @@ func bucketOf(ktyp, etyp *rtype) *rtype { panic("reflect: unexpected GC program in MapOf") } kmask := (*[16]byte)(unsafe.Pointer(ktyp.gcdata)) - for i := uintptr(0); i < ktyp.size/ptrSize; i++ { + for i := uintptr(0); i < ktyp.ptrdata/ptrSize; i++ { if (kmask[i/8]>>(i%8))&1 != 0 { for j := uintptr(0); j < bucketSize; j++ { word := base + j*ktyp.size/ptrSize + i @@ -1830,7 +1776,7 @@ func bucketOf(ktyp, etyp *rtype) *rtype { panic("reflect: unexpected GC program in MapOf") } emask := (*[16]byte)(unsafe.Pointer(etyp.gcdata)) - for i := uintptr(0); i < etyp.size/ptrSize; i++ { + for i := uintptr(0); i < etyp.ptrdata/ptrSize; i++ { if (emask[i/8]>>(i%8))&1 != 0 { for j := uintptr(0); j < bucketSize; j++ { word := base + j*etyp.size/ptrSize + i @@ -1871,8 +1817,8 @@ func SliceOf(t Type) Type { // Look in cache. ckey := cacheKey{Slice, typ, nil, 0} - if slice := cacheGet(ckey); slice != nil { - return slice + if slice, ok := lookupCache.Load(ckey); ok { + return slice.(Type) } // Look in known types. @@ -1892,17 +1838,44 @@ func SliceOf(t Type) Type { slice.uncommonType = nil slice.ptrToThis = nil - return cachePut(ckey, &slice.rtype) + ti, _ := lookupCache.LoadOrStore(ckey, &slice.rtype) + return ti.(Type) } // The structLookupCache caches StructOf lookups. // StructOf does not share the common lookupCache since we need to pin // the memory associated with *structTypeFixedN. var structLookupCache struct { - sync.RWMutex - m map[uint32][]interface { - common() *rtype - } // keyed by hash calculated in StructOf + sync.Mutex // Guards stores (but not loads) on m. + + // m is a map[uint32][]Type keyed by the hash calculated in StructOf. + // Elements in m are append-only and thus safe for concurrent reading. + m sync.Map +} + +// isLetter returns true if a given 'rune' is classified as a Letter. +func isLetter(ch rune) bool { + return 'a' <= ch && ch <= 'z' || 'A' <= ch && ch <= 'Z' || ch == '_' || ch >= utf8.RuneSelf && unicode.IsLetter(ch) +} + +// isValidFieldName checks if a string is a valid (struct) field name or not. +// +// According to the language spec, a field name should be an identifier. +// +// identifier = letter { letter | unicode_digit } . +// letter = unicode_letter | "_" . +func isValidFieldName(fieldName string) bool { + for i, c := range fieldName { + if i == 0 && !isLetter(c) { + return false + } + + if !(isLetter(c) || unicode.IsDigit(c)) { + return false + } + } + + return len(fieldName) > 0 } // StructOf returns the struct type containing fields. @@ -1930,6 +1903,12 @@ func StructOf(fields []StructField) Type { lastzero := uintptr(0) repr = append(repr, "struct {"...) for i, field := range fields { + if field.Name == "" { + panic("reflect.StructOf: field " + strconv.Itoa(i) + " has no name") + } + if !isValidFieldName(field.Name) { + panic("reflect.StructOf: field " + strconv.Itoa(i) + " has invalid name") + } if field.Type == nil { panic("reflect.StructOf: field " + strconv.Itoa(i) + " has no type") } @@ -1960,30 +1939,29 @@ func StructOf(fields []StructField) Type { } else { name = ft.String() } - // TODO(sbinet) check for syntactically impossible type names? switch f.typ.Kind() { case Interface: ift := (*interfaceType)(unsafe.Pointer(ft)) if len(ift.methods) > 0 { - panic("reflect.StructOf: embedded field with methods not supported") + panic("reflect.StructOf: embedded field with methods not implemented") } case Ptr: ptr := (*ptrType)(unsafe.Pointer(ft)) if unt := ptr.uncommon(); unt != nil { if len(unt.methods) > 0 { - panic("reflect.StructOf: embedded field with methods not supported") + panic("reflect.StructOf: embedded field with methods not implemented") } } if unt := ptr.elem.uncommon(); unt != nil { if len(unt.methods) > 0 { - panic("reflect.StructOf: embedded field with methods not supported") + panic("reflect.StructOf: embedded field with methods not implemented") } } default: if unt := ft.uncommon(); unt != nil { if len(unt.methods) > 0 { - panic("reflect.StructOf: embedded field with methods not supported") + panic("reflect.StructOf: embedded field with methods not implemented") } } } @@ -2044,30 +2022,35 @@ func StructOf(fields []StructField) Type { *typ = *prototype typ.fields = fs - // Look in cache - structLookupCache.RLock() - for _, st := range structLookupCache.m[hash] { - t := st.common() - if haveIdenticalUnderlyingType(&typ.rtype, t, true) { - structLookupCache.RUnlock() - return t + // Look in cache. + if ts, ok := structLookupCache.m.Load(hash); ok { + for _, st := range ts.([]Type) { + t := st.common() + if haveIdenticalUnderlyingType(&typ.rtype, t, true) { + return t + } } } - structLookupCache.RUnlock() - // not in cache, lock and retry + // Not in cache, lock and retry. structLookupCache.Lock() defer structLookupCache.Unlock() - if structLookupCache.m == nil { - structLookupCache.m = make(map[uint32][]interface { - common() *rtype - }) + if ts, ok := structLookupCache.m.Load(hash); ok { + for _, st := range ts.([]Type) { + t := st.common() + if haveIdenticalUnderlyingType(&typ.rtype, t, true) { + return t + } + } } - for _, st := range structLookupCache.m[hash] { - t := st.common() - if haveIdenticalUnderlyingType(&typ.rtype, t, true) { - return t + + addToCache := func(t Type) Type { + var ts []Type + if ti, ok := structLookupCache.m.Load(hash); ok { + ts = ti.([]Type) } + structLookupCache.m.Store(hash, append(ts, t)) + return t } typ.string = &str @@ -2172,24 +2155,19 @@ func StructOf(fields []StructField) Type { typ.uncommonType = nil typ.ptrToThis = nil - structLookupCache.m[hash] = append(structLookupCache.m[hash], typ) - return &typ.rtype + return addToCache(&typ.rtype) } func runtimeStructField(field StructField) structField { - var name *string - if field.Name == "" { - t := field.Type.(*rtype) - if t.Kind() == Ptr { - t = t.Elem().(*rtype) - } - } else if field.PkgPath == "" { - s := field.Name - name = &s - b0 := s[0] - if ('a' <= b0 && b0 <= 'z') || b0 == '_' { - panic("reflect.StructOf: field \"" + field.Name + "\" is unexported but has no PkgPath") - } + if field.PkgPath != "" { + panic("reflect.StructOf: StructOf does not allow unexported fields") + } + + // Best-effort check for misuse. + // Since PkgPath is empty, not much harm done if Unicode lowercase slips through. + c := field.Name[0] + if 'a' <= c && c <= 'z' || c == '_' { + panic("reflect.StructOf: field \"" + field.Name + "\" is unexported but missing PkgPath") } offsetAnon := uintptr(0) @@ -2197,24 +2175,18 @@ func runtimeStructField(field StructField) structField { offsetAnon |= 1 } - var pkgPath *string - if field.PkgPath != "" { - s := field.PkgPath - pkgPath = &s - // This could work with gccgo but we panic to be - // compatible with gc. - panic("reflect: creating a name with a package path is not supported") - } + s := field.Name + name := &s var tag *string if field.Tag != "" { - s := string(field.Tag) - tag = &s + st := string(field.Tag) + tag = &st } return structField{ name: name, - pkgPath: pkgPath, + pkgPath: nil, typ: field.Type.common(), tag: tag, offsetAnon: offsetAnon, @@ -2257,15 +2229,11 @@ const maxPtrmaskBytes = 2048 // ArrayOf panics. func ArrayOf(count int, elem Type) Type { typ := elem.(*rtype) - // call SliceOf here as it calls cacheGet/cachePut. - // ArrayOf also calls cacheGet/cachePut and thus may modify the state of - // the lookupCache mutex. - slice := SliceOf(elem) // Look in cache. ckey := cacheKey{Array, typ, nil, uintptr(count)} - if array := cacheGet(ckey); array != nil { - return array + if array, ok := lookupCache.Load(ckey); ok { + return array.(Type) } // Look in known types. @@ -2287,9 +2255,11 @@ func ArrayOf(count int, elem Type) Type { array.elem = typ array.ptrToThis = nil - max := ^uintptr(0) / typ.size - if uintptr(count) > max { - panic("reflect.ArrayOf: array size would exceed virtual address space") + if typ.size > 0 { + max := ^uintptr(0) / typ.size + if uintptr(count) > max { + panic("reflect.ArrayOf: array size would exceed virtual address space") + } } array.size = typ.size * uintptr(count) if count > 0 && typ.ptrdata != 0 { @@ -2299,7 +2269,7 @@ func ArrayOf(count int, elem Type) Type { array.fieldAlign = typ.fieldAlign array.uncommonType = nil array.len = uintptr(count) - array.slice = slice.(*rtype) + array.slice = SliceOf(elem).(*rtype) array.kind &^= kindNoPointers switch { @@ -2413,7 +2383,8 @@ func ArrayOf(count int, elem Type) Type { } } - return cachePut(ckey, &array.rtype) + ti, _ := lookupCache.LoadOrStore(ckey, &array.rtype) + return ti.(Type) } func appendVarint(x []byte, v uintptr) []byte { @@ -2466,7 +2437,7 @@ func ifaceIndir(t *rtype) bool { return t.kind&kindDirectIface == 0 } -// Layout matches runtime.BitVector (well enough). +// Layout matches runtime.gobitvector (well enough). type bitVector struct { n uint32 // number of bits data []byte diff --git a/libgo/go/reflect/value.go b/libgo/go/reflect/value.go index 8f6a93b3848..792699a6f65 100644 --- a/libgo/go/reflect/value.go +++ b/libgo/go/reflect/value.go @@ -30,9 +30,9 @@ const ptrSize = 4 << (^uintptr(0) >> 63) // unsafe.Sizeof(uintptr(0)) but an ide // the underlying Go value can be used concurrently for the equivalent // direct operations. // -// Using == on two Values does not compare the underlying values -// they represent, but rather the contents of the Value structs. // To compare two Values, compare the results of the Interface method. +// Using == on two Values does not compare the underlying values +// they represent. type Value struct { // typ holds the type of the value represented by a Value. typ *rtype @@ -1000,7 +1000,7 @@ func (v Value) Method(i int) Value { return Value{v.typ, v.ptr, fl} } -// NumMethod returns the number of methods in the value's method set. +// NumMethod returns the number of exported methods in the value's method set. func (v Value) NumMethod() int { if v.typ == nil { panic(&ValueError{"reflect.Value.NumMethod", Invalid}) @@ -1933,12 +1933,18 @@ func MakeChan(typ Type, buffer int) Value { return Value{typ.common(), unsafe.Pointer(&ch), flag(Chan) | flagIndir} } -// MakeMap creates a new map of the specified type. +// MakeMap creates a new map with the specified type. func MakeMap(typ Type) Value { + return MakeMapWithSize(typ, 0) +} + +// MakeMapWithSize creates a new map with the specified type +// and initial space for approximately n elements. +func MakeMapWithSize(typ Type, n int) Value { if typ.Kind() != Map { - panic("reflect.MakeMap of non-map type") + panic("reflect.MakeMapWithSize of non-map type") } - m := makemap(typ.(*rtype)) + m := makemap(typ.(*rtype), n) return Value{typ.common(), unsafe.Pointer(&m), flag(Map) | flagIndir} } @@ -2015,7 +2021,6 @@ func (v Value) assignTo(context string, dst *rtype, target unsafe.Pointer) Value case directlyAssignable(dst, v.typ): // Overwrite type so that they match. // Same memory layout, so no harm done. - v.typ = dst fl := v.flag & (flagRO | flagAddr | flagIndir) fl |= flag(dst.Kind()) return Value{dst, v.ptr, fl} @@ -2333,7 +2338,7 @@ func chanrecv(ch unsafe.Pointer, nb bool, val unsafe.Pointer) (selected, receive func chansend(ch unsafe.Pointer, val unsafe.Pointer, nb bool) bool func makechan(typ *rtype, size uint64) (ch unsafe.Pointer) -func makemap(t *rtype) (m unsafe.Pointer) +func makemap(t *rtype, cap int) (m unsafe.Pointer) //go:noescape func mapaccess(t *rtype, m unsafe.Pointer, key unsafe.Pointer) (val unsafe.Pointer) diff --git a/libgo/go/regexp/all_test.go b/libgo/go/regexp/all_test.go index beb46e70995..28fe20c15d8 100644 --- a/libgo/go/regexp/all_test.go +++ b/libgo/go/regexp/all_test.go @@ -9,6 +9,7 @@ import ( "regexp/syntax" "strings" "testing" + "unicode/utf8" ) var goodRe = []string{ @@ -354,6 +355,7 @@ type MetaTest struct { var metaTests = []MetaTest{ {``, ``, ``, true}, {`foo`, `foo`, `foo`, true}, + {`日本語+`, `日本語\+`, `日本語`, false}, {`foo\.\$`, `foo\\\.\\\$`, `foo.$`, true}, // has meta but no operator {`foo.\$`, `foo\.\\\$`, `foo`, false}, // has escaped operators and real operators {`!@#$%^&*()_+-=[{]}\|,<.>/?~`, `!@#\$%\^&\*\(\)_\+-=\[\{\]\}\\\|,<\.>/\?~`, `!@#`, false}, @@ -822,7 +824,13 @@ func BenchmarkMatchParallelCopied(b *testing.B) { var sink string func BenchmarkQuoteMetaAll(b *testing.B) { - s := string(specialBytes) + specials := make([]byte, 0) + for i := byte(0); i < utf8.RuneSelf; i++ { + if special(i) { + specials = append(specials, i) + } + } + s := string(specials) b.SetBytes(int64(len(s))) b.ResetTimer() for i := 0; i < b.N; i++ { diff --git a/libgo/go/regexp/exec.go b/libgo/go/regexp/exec.go index 977619cb28a..f8fe7b5deff 100644 --- a/libgo/go/regexp/exec.go +++ b/libgo/go/regexp/exec.go @@ -309,12 +309,14 @@ func (m *machine) add(q *queue, pc uint32, pos int, cap []int, cond syntax.Empty // onepass runs the machine over the input starting at pos. // It reports whether a match was found. // If so, m.matchcap holds the submatch information. -func (m *machine) onepass(i input, pos int) bool { +// ncap is the number of captures. +func (m *machine) onepass(i input, pos, ncap int) bool { startCond := m.re.cond if startCond == ^syntax.EmptyOp(0) { // impossible return false } m.matched = false + m.matchcap = m.matchcap[:ncap] for i := range m.matchcap { m.matchcap[i] = -1 } @@ -428,7 +430,7 @@ func (re *Regexp) doExecute(r io.RuneReader, b []byte, s string, pos int, ncap i size = len(s) } if m.op != notOnePass { - if !m.onepass(i, pos) { + if !m.onepass(i, pos, ncap) { re.put(m) return nil } diff --git a/libgo/go/regexp/exec_test.go b/libgo/go/regexp/exec_test.go index 766394de6ee..5f8e747b17b 100644 --- a/libgo/go/regexp/exec_test.go +++ b/libgo/go/regexp/exec_test.go @@ -681,6 +681,35 @@ func BenchmarkMatch(b *testing.B) { } } +func BenchmarkMatch_onepass_regex(b *testing.B) { + isRaceBuilder := strings.HasSuffix(testenv.Builder(), "-race") + r := MustCompile(`(?s)\A.*\z`) + if r.get().op == notOnePass { + b.Fatalf("want onepass regex, but %q is not onepass", r) + } + for _, size := range benchSizes { + if isRaceBuilder && size.n > 1<<10 { + continue + } + t := makeText(size.n) + bs := make([][]byte, len(t)) + for i, s := range t { + bs[i] = []byte{s} + } + b.Run(size.name, func(b *testing.B) { + b.SetBytes(int64(size.n)) + b.ReportAllocs() + for i := 0; i < b.N; i++ { + for _, byts := range bs { + if !r.Match(byts) { + b.Fatal("not match!") + } + } + } + }) + } +} + var benchData = []struct{ name, re string }{ {"Easy0", "ABCDEFGHIJKLMNOPQRSTUVWXYZ$"}, {"Easy0i", "(?i)ABCDEFGHIJklmnopqrstuvwxyz$"}, diff --git a/libgo/go/regexp/onepass.go b/libgo/go/regexp/onepass.go index 1b0564c3fd0..3ceb4619058 100644 --- a/libgo/go/regexp/onepass.go +++ b/libgo/go/regexp/onepass.go @@ -222,9 +222,10 @@ func onePassCopy(prog *syntax.Prog) *onePassProg { p := &onePassProg{ Start: prog.Start, NumCap: prog.NumCap, + Inst: make([]onePassInst, len(prog.Inst)), } - for _, inst := range prog.Inst { - p.Inst = append(p.Inst, onePassInst{Inst: inst}) + for i, inst := range prog.Inst { + p.Inst[i] = onePassInst{Inst: inst} } // rewrites one or more common Prog constructs that enable some otherwise @@ -304,13 +305,13 @@ func makeOnePass(p *onePassProg) *onePassProg { var ( instQueue = newQueue(len(p.Inst)) visitQueue = newQueue(len(p.Inst)) - check func(uint32, map[uint32]bool) bool + check func(uint32, []bool) bool onePassRunes = make([][]rune, len(p.Inst)) ) // check that paths from Alt instructions are unambiguous, and rebuild the new // program as a onepass program - check = func(pc uint32, m map[uint32]bool) (ok bool) { + check = func(pc uint32, m []bool) (ok bool) { ok = true inst := &p.Inst[pc] if visitQueue.contains(pc) { @@ -349,21 +350,20 @@ func makeOnePass(p *onePassProg) *onePassProg { m[pc] = m[inst.Out] // pass matching runes back through these no-ops. onePassRunes[pc] = append([]rune{}, onePassRunes[inst.Out]...) - inst.Next = []uint32{} - for i := len(onePassRunes[pc]) / 2; i >= 0; i-- { - inst.Next = append(inst.Next, inst.Out) + inst.Next = make([]uint32, len(onePassRunes[pc])/2+1) + for i := range inst.Next { + inst.Next[i] = inst.Out } case syntax.InstEmptyWidth: ok = check(inst.Out, m) m[pc] = m[inst.Out] onePassRunes[pc] = append([]rune{}, onePassRunes[inst.Out]...) - inst.Next = []uint32{} - for i := len(onePassRunes[pc]) / 2; i >= 0; i-- { - inst.Next = append(inst.Next, inst.Out) + inst.Next = make([]uint32, len(onePassRunes[pc])/2+1) + for i := range inst.Next { + inst.Next[i] = inst.Out } case syntax.InstMatch, syntax.InstFail: m[pc] = inst.Op == syntax.InstMatch - break case syntax.InstRune: m[pc] = false if len(inst.Next) > 0 { @@ -387,9 +387,9 @@ func makeOnePass(p *onePassProg) *onePassProg { runes = append(runes, inst.Rune...) } onePassRunes[pc] = runes - inst.Next = []uint32{} - for i := len(onePassRunes[pc]) / 2; i >= 0; i-- { - inst.Next = append(inst.Next, inst.Out) + inst.Next = make([]uint32, len(onePassRunes[pc])/2+1) + for i := range inst.Next { + inst.Next[i] = inst.Out } inst.Op = syntax.InstRune case syntax.InstRune1: @@ -411,9 +411,9 @@ func makeOnePass(p *onePassProg) *onePassProg { runes = append(runes, inst.Rune[0], inst.Rune[0]) } onePassRunes[pc] = runes - inst.Next = []uint32{} - for i := len(onePassRunes[pc]) / 2; i >= 0; i-- { - inst.Next = append(inst.Next, inst.Out) + inst.Next = make([]uint32, len(onePassRunes[pc])/2+1) + for i := range inst.Next { + inst.Next[i] = inst.Out } inst.Op = syntax.InstRune case syntax.InstRuneAny: @@ -431,9 +431,9 @@ func makeOnePass(p *onePassProg) *onePassProg { } instQueue.insert(inst.Out) onePassRunes[pc] = append([]rune{}, anyRuneNotNL...) - inst.Next = []uint32{} - for i := len(onePassRunes[pc]) / 2; i >= 0; i-- { - inst.Next = append(inst.Next, inst.Out) + inst.Next = make([]uint32, len(onePassRunes[pc])/2+1) + for i := range inst.Next { + inst.Next[i] = inst.Out } } return @@ -441,7 +441,7 @@ func makeOnePass(p *onePassProg) *onePassProg { instQueue.clear() instQueue.insert(uint32(p.Start)) - m := make(map[uint32]bool, len(p.Inst)) + m := make([]bool, len(p.Inst)) for !instQueue.empty() { visitQueue.clear() pc := instQueue.next() diff --git a/libgo/go/regexp/onepass_test.go b/libgo/go/regexp/onepass_test.go index f4e336c43ba..b1caa445150 100644 --- a/libgo/go/regexp/onepass_test.go +++ b/libgo/go/regexp/onepass_test.go @@ -7,6 +7,7 @@ package regexp import ( "reflect" "regexp/syntax" + "strings" "testing" ) @@ -173,6 +174,7 @@ var onePassTests = []struct { {`^.bc(d|e)*$`, onePass}, {`^(?:(?:aa)|.)$`, notOnePass}, {`^(?:(?:a{1,2}){1,2})$`, notOnePass}, + {`^l` + strings.Repeat("o", 2<<8) + `ng$`, onePass}, } func TestCompileOnePass(t *testing.T) { @@ -223,3 +225,23 @@ func TestRunOnePass(t *testing.T) { } } } + +func BenchmarkCompileOnepass(b *testing.B) { + for _, test := range onePassTests { + if test.onePass == notOnePass { + continue + } + name := test.re + if len(name) > 20 { + name = name[:20] + "..." + } + b.Run(name, func(b *testing.B) { + b.ReportAllocs() + for i := 0; i < b.N; i++ { + if _, err := Compile(test.re); err != nil { + b.Fatal(err) + } + } + }) + } +} diff --git a/libgo/go/regexp/regexp.go b/libgo/go/regexp/regexp.go index 01093d4bd0d..b1af23e8504 100644 --- a/libgo/go/regexp/regexp.go +++ b/libgo/go/regexp/regexp.go @@ -76,7 +76,8 @@ import ( ) // Regexp is the representation of a compiled regular expression. -// A Regexp is safe for concurrent use by multiple goroutines. +// A Regexp is safe for concurrent use by multiple goroutines, +// except for configuration methods, such as Longest. type Regexp struct { // read-only after Compile regexpRO @@ -159,6 +160,8 @@ func CompilePOSIX(expr string) (*Regexp, error) { // That is, when matching against text, the regexp returns a match that // begins as early as possible in the input (leftmost), and among those // it chooses a match that is as long as possible. +// This method modifies the Regexp and may not be called concurrently +// with any other methods. func (re *Regexp) Longest() { re.longest = true } @@ -313,11 +316,19 @@ func (i *inputString) index(re *Regexp, pos int) int { func (i *inputString) context(pos int) syntax.EmptyOp { r1, r2 := endOfText, endOfText - if pos > 0 && pos <= len(i.str) { - r1, _ = utf8.DecodeLastRuneInString(i.str[:pos]) + // 0 < pos && pos <= len(i.str) + if uint(pos-1) < uint(len(i.str)) { + r1 = rune(i.str[pos-1]) + if r1 >= utf8.RuneSelf { + r1, _ = utf8.DecodeLastRuneInString(i.str[:pos]) + } } - if pos < len(i.str) { - r2, _ = utf8.DecodeRuneInString(i.str[pos:]) + // 0 <= pos && pos < len(i.str) + if uint(pos) < uint(len(i.str)) { + r2 = rune(i.str[pos]) + if r2 >= utf8.RuneSelf { + r2, _ = utf8.DecodeRuneInString(i.str[pos:]) + } } return syntax.EmptyOpContext(r1, r2) } @@ -352,11 +363,19 @@ func (i *inputBytes) index(re *Regexp, pos int) int { func (i *inputBytes) context(pos int) syntax.EmptyOp { r1, r2 := endOfText, endOfText - if pos > 0 && pos <= len(i.str) { - r1, _ = utf8.DecodeLastRune(i.str[:pos]) + // 0 < pos && pos <= len(i.str) + if uint(pos-1) < uint(len(i.str)) { + r1 = rune(i.str[pos-1]) + if r1 >= utf8.RuneSelf { + r1, _ = utf8.DecodeLastRune(i.str[:pos]) + } } - if pos < len(i.str) { - r2, _ = utf8.DecodeRune(i.str[pos:]) + // 0 <= pos && pos < len(i.str) + if uint(pos) < uint(len(i.str)) { + r2 = rune(i.str[pos]) + if r2 >= utf8.RuneSelf { + r2, _ = utf8.DecodeRune(i.str[pos:]) + } } return syntax.EmptyOpContext(r1, r2) } @@ -590,10 +609,18 @@ func (re *Regexp) ReplaceAllFunc(src []byte, repl func([]byte) []byte) []byte { }) } -var specialBytes = []byte(`\.+*?()|[]{}^$`) +// Bitmap used by func special to check whether a character needs to be escaped. +var specialBytes [16]byte +// special reports whether byte b needs to be escaped by QuoteMeta. func special(b byte) bool { - return bytes.IndexByte(specialBytes, b) >= 0 + return b < utf8.RuneSelf && specialBytes[b%16]&(1<<(b/16)) != 0 +} + +func init() { + for _, b := range []byte(`\.+*?()|[]{}^$`) { + specialBytes[b%16] |= 1 << (b / 16) + } } // QuoteMeta returns a string that quotes all regular expression metacharacters diff --git a/libgo/go/regexp/syntax/parse.go b/libgo/go/regexp/syntax/parse.go index 7b8be55ddb1..8c6d43a7063 100644 --- a/libgo/go/regexp/syntax/parse.go +++ b/libgo/go/regexp/syntax/parse.go @@ -381,7 +381,7 @@ func (p *parser) collapse(subs []*Regexp, op Op) *Regexp { } } if op == OpAlternate { - re.Sub = p.factor(re.Sub, re.Flags) + re.Sub = p.factor(re.Sub) if len(re.Sub) == 1 { old := re re = re.Sub[0] @@ -402,7 +402,7 @@ func (p *parser) collapse(subs []*Regexp, op Op) *Regexp { // which simplifies by character class introduction to // A(B[CD]|EF)|BC[XY] // -func (p *parser) factor(sub []*Regexp, flags Flags) []*Regexp { +func (p *parser) factor(sub []*Regexp) []*Regexp { if len(sub) < 2 { return sub } diff --git a/libgo/go/runtime/cgo_gccgo.go b/libgo/go/runtime/cgo_gccgo.go index 8236eeabf46..c3bf9552ea8 100644 --- a/libgo/go/runtime/cgo_gccgo.go +++ b/libgo/go/runtime/cgo_gccgo.go @@ -6,7 +6,7 @@ package runtime import ( "runtime/internal/atomic" - _ "unsafe" + "unsafe" ) // For historical reasons these functions are called as though they @@ -41,6 +41,7 @@ func Cgocall() { mp := getg().m mp.ncgocall++ mp.ncgo++ + mp.incgo = true entersyscall(0) } @@ -50,6 +51,7 @@ func CgocallDone() { if gp == nil { throw("no g in CgocallDone") } + gp.m.incgo = false gp.m.ncgo-- // If we are invoked because the C function called _cgo_panic, @@ -68,15 +70,18 @@ func CgocallDone() { // gofunction() //go:nosplit func CgocallBack() { - if getg() == nil || getg().m == nil { + gp := getg() + if gp == nil || gp.m == nil { needm(0) - mp := getg().m + gp = getg() + mp := gp.m mp.dropextram = true } exitsyscall(0) + gp.m.incgo = false - if getg().m.ncgo == 0 { + if gp.m.ncgo == 0 { // The C call to Go came from a thread created by C. // The C call to Go came from a thread not currently running // any Go. In the case of -buildmode=c-archive or c-shared, @@ -85,7 +90,7 @@ func CgocallBack() { <-main_init_done } - mp := getg().m + mp := gp.m if mp.needextram || atomic.Load(&extraMWaiters) > 0 { mp.needextram = false newextram() @@ -120,6 +125,7 @@ func CgocallBackDone() { drop = true } + gp.m.incgo = true entersyscall(0) if drop { @@ -133,3 +139,8 @@ func _cgo_panic(p *byte) { exitsyscall(0) panic(gostringnocopy(p)) } + +// cgo_yield exists in the gc toolchain to let TSAN deliver a signal. +// gccgo does not need this. +var cgo_yield = &_cgo_yield +var _cgo_yield unsafe.Pointer diff --git a/libgo/go/runtime/cgocheck.go b/libgo/go/runtime/cgocheck.go index 09d444dbd1a..30f054b3633 100644 --- a/libgo/go/runtime/cgocheck.go +++ b/libgo/go/runtime/cgocheck.go @@ -125,7 +125,7 @@ func cgoCheckTypedBlock(typ *_type, src unsafe.Pointer, off, size uintptr) { aoff := uintptr(src) - mheap_.arena_start idx := aoff >> _PageShift s := mheap_.spans[idx] - if s.state == _MSpanStack { + if s.state == _MSpanManual { // There are no heap bits for value stored on the stack. // For a channel receive src might be on the stack of some // other goroutine, so we can't unwind the stack even if diff --git a/libgo/go/runtime/chan.go b/libgo/go/runtime/chan.go index d2470bd0442..7bb919c41db 100644 --- a/libgo/go/runtime/chan.go +++ b/libgo/go/runtime/chan.go @@ -185,7 +185,7 @@ func chansend(c *hchan, ep unsafe.Pointer, block bool, callerpc uintptr) bool { if sg := c.recvq.dequeue(); sg != nil { // Found a waiting receiver. We pass the value we want to send // directly to the receiver, bypassing the channel buffer (if any). - send(c, sg, ep, func() { unlock(&c.lock) }) + send(c, sg, ep, func() { unlock(&c.lock) }, 3) return true } @@ -256,7 +256,7 @@ func chansend(c *hchan, ep unsafe.Pointer, block bool, callerpc uintptr) bool { // Channel c must be empty and locked. send unlocks c with unlockf. // sg must already be dequeued from c. // ep must be non-nil and point to the heap or the caller's stack. -func send(c *hchan, sg *sudog, ep unsafe.Pointer, unlockf func()) { +func send(c *hchan, sg *sudog, ep unsafe.Pointer, unlockf func(), skip int) { if raceenabled { if c.dataqsiz == 0 { racesync(c, sg) @@ -286,7 +286,7 @@ func send(c *hchan, sg *sudog, ep unsafe.Pointer, unlockf func()) { if sg.releasetime != 0 { sg.releasetime = cputicks() } - goready(gp, 4) + goready(gp, skip+1) } // Sends and receives on unbuffered or empty-buffered channels are the @@ -466,7 +466,7 @@ func chanrecv(c *hchan, ep unsafe.Pointer, block bool) (selected, received bool) // directly from sender. Otherwise, receive from head of queue // and add sender's value to the tail of the queue (both map to // the same buffer slot because the queue is full). - recv(c, sg, ep, func() { unlock(&c.lock) }) + recv(c, sg, ep, func() { unlock(&c.lock) }, 3) return true, true } @@ -542,7 +542,7 @@ func chanrecv(c *hchan, ep unsafe.Pointer, block bool) (selected, received bool) // Channel c must be full and locked. recv unlocks c with unlockf. // sg must already be dequeued from c. // A non-nil ep must point to the heap or the caller's stack. -func recv(c *hchan, sg *sudog, ep unsafe.Pointer, unlockf func()) { +func recv(c *hchan, sg *sudog, ep unsafe.Pointer, unlockf func(), skip int) { if c.dataqsiz == 0 { if raceenabled { racesync(c, sg) @@ -582,7 +582,7 @@ func recv(c *hchan, sg *sudog, ep unsafe.Pointer, unlockf func()) { if sg.releasetime != 0 { sg.releasetime = cputicks() } - goready(gp, 4) + goready(gp, skip+1) } // compiler implements diff --git a/libgo/go/runtime/cpuprof.go b/libgo/go/runtime/cpuprof.go index e1206f99f10..b031b1a5e75 100644 --- a/libgo/go/runtime/cpuprof.go +++ b/libgo/go/runtime/cpuprof.go @@ -3,118 +3,45 @@ // license that can be found in the LICENSE file. // CPU profiling. -// Based on algorithms and data structures used in -// https://github.com/google/pprof. -// -// The main difference between this code and the google-perftools -// code is that this code is written to allow copying the profile data -// to an arbitrary io.Writer, while the google-perftools code always -// writes to an operating system file. // // The signal handler for the profiling clock tick adds a new stack trace -// to a hash table tracking counts for recent traces. Most clock ticks -// hit in the cache. In the event of a cache miss, an entry must be -// evicted from the hash table, copied to a log that will eventually be -// written as profile data. The google-perftools code flushed the -// log itself during the signal handler. This code cannot do that, because -// the io.Writer might block or need system calls or locks that are not -// safe to use from within the signal handler. Instead, we split the log -// into two halves and let the signal handler fill one half while a goroutine -// is writing out the other half. When the signal handler fills its half, it -// offers to swap with the goroutine. If the writer is not done with its half, -// we lose the stack trace for this clock tick (and record that loss). -// The goroutine interacts with the signal handler by calling getprofile() to -// get the next log piece to write, implicitly handing back the last log -// piece it obtained. -// -// The state of this dance between the signal handler and the goroutine -// is encoded in the Profile.handoff field. If handoff == 0, then the goroutine -// is not using either log half and is waiting (or will soon be waiting) for -// a new piece by calling notesleep(&p.wait). If the signal handler -// changes handoff from 0 to non-zero, it must call notewakeup(&p.wait) -// to wake the goroutine. The value indicates the number of entries in the -// log half being handed off. The goroutine leaves the non-zero value in -// place until it has finished processing the log half and then flips the number -// back to zero. Setting the high bit in handoff means that the profiling is over, -// and the goroutine is now in charge of flushing the data left in the hash table -// to the log and returning that data. -// -// The handoff field is manipulated using atomic operations. -// For the most part, the manipulation of handoff is orderly: if handoff == 0 -// then the signal handler owns it and can change it to non-zero. -// If handoff != 0 then the goroutine owns it and can change it to zero. -// If that were the end of the story then we would not need to manipulate -// handoff using atomic operations. The operations are needed, however, -// in order to let the log closer set the high bit to indicate "EOF" safely -// in the situation when normally the goroutine "owns" handoff. +// to a log of recent traces. The log is read by a user goroutine that +// turns it into formatted profile data. If the reader does not keep up +// with the log, those writes will be recorded as a count of lost records. +// The actual profile buffer is in profbuf.go. package runtime import ( "runtime/internal/atomic" + "runtime/internal/sys" "unsafe" ) -const ( - numBuckets = 1 << 10 - logSize = 1 << 17 - assoc = 4 - maxCPUProfStack = 64 -) +const maxCPUProfStack = 64 -type cpuprofEntry struct { - count uintptr - depth int - stack [maxCPUProfStack]uintptr -} - -//go:notinheap type cpuProfile struct { - on bool // profiling is on - wait note // goroutine waits here - count uintptr // tick count - evicts uintptr // eviction count - lost uintptr // lost ticks that need to be logged - - // Active recent stack traces. - hash [numBuckets]struct { - entry [assoc]cpuprofEntry - } - - // Log of traces evicted from hash. - // Signal handler has filled log[toggle][:nlog]. - // Goroutine is writing log[1-toggle][:handoff]. - log [2][logSize / 2]uintptr - nlog int - toggle int32 - handoff uint32 - - // Writer state. - // Writer maintains its own toggle to avoid races - // looking at signal handler's toggle. - wtoggle uint32 - wholding bool // holding & need to release a log half - flushing bool // flushing hash table - profile is over - eodSent bool // special end-of-data record sent; => flushing + lock mutex + on bool // profiling is on + log *profBuf // profile events written here + + // extra holds extra stacks accumulated in addNonGo + // corresponding to profiling signals arriving on + // non-Go-created threads. Those stacks are written + // to log the next time a normal Go thread gets the + // signal handler. + // Assuming the stacks are 2 words each (we don't get + // a full traceback from those threads), plus one word + // size for framing, 100 Hz profiling would generate + // 300 words per second. + // Hopefully a normal Go thread will get the profiling + // signal at least once every few seconds. + extra [1000]uintptr + numExtra int + lostExtra uint64 // count of frames lost because extra is full } -var ( - cpuprofLock mutex - cpuprof *cpuProfile - - eod = [3]uintptr{0, 1, 0} -) - -func setcpuprofilerate(hz int32) { - systemstack(func() { - setcpuprofilerate_m(hz) - }) -} - -// lostProfileData is a no-op function used in profiles -// to mark the number of profiling stack traces that were -// discarded due to slow data writers. -func lostProfileData() {} +var cpuprof cpuProfile // SetCPUProfileRate sets the CPU profiling rate to hz samples per second. // If hz <= 0, SetCPUProfileRate turns off profiling. @@ -132,323 +59,153 @@ func SetCPUProfileRate(hz int) { hz = 1000000 } - lock(&cpuprofLock) + lock(&cpuprof.lock) if hz > 0 { - if cpuprof == nil { - cpuprof = (*cpuProfile)(sysAlloc(unsafe.Sizeof(cpuProfile{}), &memstats.other_sys)) - if cpuprof == nil { - print("runtime: cpu profiling cannot allocate memory\n") - unlock(&cpuprofLock) - return - } - } - if cpuprof.on || cpuprof.handoff != 0 { + if cpuprof.on || cpuprof.log != nil { print("runtime: cannot set cpu profile rate until previous profile has finished.\n") - unlock(&cpuprofLock) + unlock(&cpuprof.lock) return } cpuprof.on = true - // pprof binary header format. - // https://github.com/gperftools/gperftools/blob/master/src/profiledata.cc#L119 - p := &cpuprof.log[0] - p[0] = 0 // count for header - p[1] = 3 // depth for header - p[2] = 0 // version number - p[3] = uintptr(1e6 / hz) // period (microseconds) - p[4] = 0 - cpuprof.nlog = 5 - cpuprof.toggle = 0 - cpuprof.wholding = false - cpuprof.wtoggle = 0 - cpuprof.flushing = false - cpuprof.eodSent = false - noteclear(&cpuprof.wait) - + cpuprof.log = newProfBuf(1, 1<<17, 1<<14) + hdr := [1]uint64{uint64(hz)} + cpuprof.log.write(nil, nanotime(), hdr[:], nil) setcpuprofilerate(int32(hz)) - } else if cpuprof != nil && cpuprof.on { + } else if cpuprof.on { setcpuprofilerate(0) cpuprof.on = false - - // Now add is not running anymore, and getprofile owns the entire log. - // Set the high bit in cpuprof.handoff to tell getprofile. - for { - n := cpuprof.handoff - if n&0x80000000 != 0 { - print("runtime: setcpuprofile(off) twice\n") - } - if atomic.Cas(&cpuprof.handoff, n, n|0x80000000) { - if n == 0 { - // we did the transition from 0 -> nonzero so we wake getprofile - notewakeup(&cpuprof.wait) - } - break - } - } + cpuprof.addExtra() + cpuprof.log.close() } - unlock(&cpuprofLock) + unlock(&cpuprof.lock) } // add adds the stack trace to the profile. // It is called from signal handlers and other limited environments // and cannot allocate memory or acquire locks that might be // held at the time of the signal, nor can it use substantial amounts -// of stack. It is allowed to call evict. +// of stack. //go:nowritebarrierrec -func (p *cpuProfile) add(pc []uintptr) { - p.addWithFlushlog(pc, p.flushlog) -} - -// addWithFlushlog implements add and addNonGo. -// It is called from signal handlers and other limited environments -// and cannot allocate memory or acquire locks that might be -// held at the time of the signal, nor can it use substantial amounts -// of stack. It may be called by a signal handler with no g or m. -// It is allowed to call evict, passing the flushlog parameter. -//go:nosplit -//go:nowritebarrierrec -func (p *cpuProfile) addWithFlushlog(pc []uintptr, flushlog func() bool) { - if len(pc) > maxCPUProfStack { - pc = pc[:maxCPUProfStack] - } - - // Compute hash. - h := uintptr(0) - for _, x := range pc { - h = h<<8 | (h >> (8 * (unsafe.Sizeof(h) - 1))) - h += x * 41 +func (p *cpuProfile) add(gp *g, stk []uintptr) { + // Simple cas-lock to coordinate with setcpuprofilerate. + for !atomic.Cas(&prof.signalLock, 0, 1) { + osyield() } - p.count++ - // Add to entry count if already present in table. - b := &p.hash[h%numBuckets] -Assoc: - for i := range b.entry { - e := &b.entry[i] - if e.depth != len(pc) { - continue - } - for j := range pc { - if e.stack[j] != pc[j] { - continue Assoc - } + if prof.hz != 0 { // implies cpuprof.log != nil + if p.numExtra > 0 || p.lostExtra > 0 { + p.addExtra() } - e.count++ - return + hdr := [1]uint64{1} + // Note: write "knows" that the argument is &gp.labels, + // because otherwise its write barrier behavior may not + // be correct. See the long comment there before + // changing the argument here. + cpuprof.log.write(&gp.labels, nanotime(), hdr[:], stk) } - // Evict entry with smallest count. - var e *cpuprofEntry - for i := range b.entry { - if e == nil || b.entry[i].count < e.count { - e = &b.entry[i] - } - } - if e.count > 0 { - if !p.evict(e, flushlog) { - // Could not evict entry. Record lost stack. - p.lost++ - return - } - p.evicts++ - } - - // Reuse the newly evicted entry. - e.depth = len(pc) - e.count = 1 - copy(e.stack[:], pc) + atomic.Store(&prof.signalLock, 0) } -// evict copies the given entry's data into the log, so that -// the entry can be reused. evict is called from add, which -// is called from the profiling signal handler, so it must not -// allocate memory or block, and it may be called with no g or m. -// It is safe to call flushlog. evict returns true if the entry was -// copied to the log, false if there was no room available. +// addNonGo adds the non-Go stack trace to the profile. +// It is called from a non-Go thread, so we cannot use much stack at all, +// nor do anything that needs a g or an m. +// In particular, we can't call cpuprof.log.write. +// Instead, we copy the stack into cpuprof.extra, +// which will be drained the next time a Go thread +// gets the signal handling event. //go:nosplit //go:nowritebarrierrec -func (p *cpuProfile) evict(e *cpuprofEntry, flushlog func() bool) bool { - d := e.depth - nslot := d + 2 - log := &p.log[p.toggle] - if p.nlog+nslot > len(log) { - if !flushlog() { - return false - } - log = &p.log[p.toggle] - } - - q := p.nlog - log[q] = e.count - q++ - log[q] = uintptr(d) - q++ - copy(log[q:], e.stack[:d]) - q += d - p.nlog = q - e.count = 0 - return true -} - -// flushlog tries to flush the current log and switch to the other one. -// flushlog is called from evict, called from add, called from the signal handler, -// so it cannot allocate memory or block. It can try to swap logs with -// the writing goroutine, as explained in the comment at the top of this file. -//go:nowritebarrierrec -func (p *cpuProfile) flushlog() bool { - if !atomic.Cas(&p.handoff, 0, uint32(p.nlog)) { - return false +func (p *cpuProfile) addNonGo(stk []uintptr) { + // Simple cas-lock to coordinate with SetCPUProfileRate. + // (Other calls to add or addNonGo should be blocked out + // by the fact that only one SIGPROF can be handled by the + // process at a time. If not, this lock will serialize those too.) + for !atomic.Cas(&prof.signalLock, 0, 1) { + osyield() } - notewakeup(&p.wait) - p.toggle = 1 - p.toggle - log := &p.log[p.toggle] - q := 0 - if p.lost > 0 { - lostPC := funcPC(lostProfileData) - log[0] = p.lost - log[1] = 1 - log[2] = lostPC - q = 3 - p.lost = 0 + if cpuprof.numExtra+1+len(stk) < len(cpuprof.extra) { + i := cpuprof.numExtra + cpuprof.extra[i] = uintptr(1 + len(stk)) + copy(cpuprof.extra[i+1:], stk) + cpuprof.numExtra += 1 + len(stk) + } else { + cpuprof.lostExtra++ } - p.nlog = q - return true -} -// addNonGo is like add, but runs on a non-Go thread. -// It can't do anything that might need a g or an m. -// With this entry point, we don't try to flush the log when evicting an -// old entry. Instead, we just drop the stack trace if we're out of space. -//go:nosplit -//go:nowritebarrierrec -func (p *cpuProfile) addNonGo(pc []uintptr) { - p.addWithFlushlog(pc, func() bool { return false }) + atomic.Store(&prof.signalLock, 0) } -// getprofile blocks until the next block of profiling data is available -// and returns it as a []byte. It is called from the writing goroutine. -func (p *cpuProfile) getprofile() []byte { - if p == nil { - return nil - } - - if p.wholding { - // Release previous log to signal handling side. - // Loop because we are racing against SetCPUProfileRate(0). - for { - n := p.handoff - if n == 0 { - print("runtime: phase error during cpu profile handoff\n") - return nil - } - if n&0x80000000 != 0 { - p.wtoggle = 1 - p.wtoggle - p.wholding = false - p.flushing = true - goto Flush - } - if atomic.Cas(&p.handoff, n, 0) { - break - } - } - p.wtoggle = 1 - p.wtoggle - p.wholding = false - } - - if p.flushing { - goto Flush - } - - if !p.on && p.handoff == 0 { - return nil - } - - // Wait for new log. - notetsleepg(&p.wait, -1) - noteclear(&p.wait) - - switch n := p.handoff; { - case n == 0: - print("runtime: phase error during cpu profile wait\n") - return nil - case n == 0x80000000: - p.flushing = true - goto Flush - default: - n &^= 0x80000000 - - // Return new log to caller. - p.wholding = true - - return uintptrBytes(p.log[p.wtoggle][:n]) - } - - // In flush mode. - // Add is no longer being called. We own the log. - // Also, p.handoff is non-zero, so flushlog will return false. - // Evict the hash table into the log and return it. -Flush: - for i := range p.hash { - b := &p.hash[i] - for j := range b.entry { - e := &b.entry[j] - if e.count > 0 && !p.evict(e, p.flushlog) { - // Filled the log. Stop the loop and return what we've got. - break Flush - } +// addExtra adds the "extra" profiling events, +// queued by addNonGo, to the profile log. +// addExtra is called either from a signal handler on a Go thread +// or from an ordinary goroutine; either way it can use stack +// and has a g. The world may be stopped, though. +func (p *cpuProfile) addExtra() { + // Copy accumulated non-Go profile events. + hdr := [1]uint64{1} + for i := 0; i < p.numExtra; { + p.log.write(nil, 0, hdr[:], p.extra[i+1:i+int(p.extra[i])]) + i += int(p.extra[i]) + } + p.numExtra = 0 + + // Report any lost events. + if p.lostExtra > 0 { + hdr := [1]uint64{p.lostExtra} + lostStk := [2]uintptr{ + funcPC(_LostExternalCode) + sys.PCQuantum, + funcPC(_ExternalCode) + sys.PCQuantum, } + cpuprof.log.write(nil, 0, hdr[:], lostStk[:]) } - - // Return pending log data. - if p.nlog > 0 { - // Note that we're using toggle now, not wtoggle, - // because we're working on the log directly. - n := p.nlog - p.nlog = 0 - return uintptrBytes(p.log[p.toggle][:n]) - } - - // Made it through the table without finding anything to log. - if !p.eodSent { - // We may not have space to append this to the partial log buf, - // so we always return a new slice for the end-of-data marker. - p.eodSent = true - return uintptrBytes(eod[:]) - } - - // Finally done. Clean up and return nil. - p.flushing = false - if !atomic.Cas(&p.handoff, p.handoff, 0) { - print("runtime: profile flush racing with something\n") - } - return nil } -func uintptrBytes(p []uintptr) (ret []byte) { - pp := (*slice)(unsafe.Pointer(&p)) - rp := (*slice)(unsafe.Pointer(&ret)) - - rp.array = pp.array - rp.len = pp.len * int(unsafe.Sizeof(p[0])) - rp.cap = rp.len - - return +func (p *cpuProfile) addLostAtomic64(count uint64) { + hdr := [1]uint64{count} + lostStk := [2]uintptr{ + funcPC(_LostSIGPROFDuringAtomic64) + sys.PCQuantum, + funcPC(_System) + sys.PCQuantum, + } + cpuprof.log.write(nil, 0, hdr[:], lostStk[:]) } -// CPUProfile returns the next chunk of binary CPU profiling stack trace data, -// blocking until data is available. If profiling is turned off and all the profile -// data accumulated while it was on has been returned, CPUProfile returns nil. -// The caller must save the returned data before calling CPUProfile again. +// CPUProfile panics. +// It formerly provided raw access to chunks of +// a pprof-format profile generated by the runtime. +// The details of generating that format have changed, +// so this functionality has been removed. // -// Most clients should use the runtime/pprof package or -// the testing package's -test.cpuprofile flag instead of calling -// CPUProfile directly. +// Deprecated: use the runtime/pprof package, +// or the handlers in the net/http/pprof package, +// or the testing package's -test.cpuprofile flag instead. func CPUProfile() []byte { - return cpuprof.getprofile() + panic("CPUProfile no longer available") } //go:linkname runtime_pprof_runtime_cyclesPerSecond runtime_pprof.runtime_cyclesPerSecond func runtime_pprof_runtime_cyclesPerSecond() int64 { return tickspersecond() } + +// readProfile, provided to runtime/pprof, returns the next chunk of +// binary CPU profiling stack trace data, blocking until data is available. +// If profiling is turned off and all the profile data accumulated while it was +// on has been returned, readProfile returns eof=true. +// The caller must save the returned data and tags before calling readProfile again. +// +//go:linkname runtime_pprof_readProfile runtime_pprof.readProfile +func runtime_pprof_readProfile() ([]uint64, []unsafe.Pointer, bool) { + lock(&cpuprof.lock) + log := cpuprof.log + unlock(&cpuprof.lock) + data, tags, eof := log.read(profBufBlocking) + if len(data) == 0 && eof { + lock(&cpuprof.lock) + cpuprof.log = nil + unlock(&cpuprof.lock) + } + return data, tags, eof +} diff --git a/libgo/go/runtime/crash_cgo_test.go b/libgo/go/runtime/crash_cgo_test.go index b338df985d3..b79873185cc 100644 --- a/libgo/go/runtime/crash_cgo_test.go +++ b/libgo/go/runtime/crash_cgo_test.go @@ -24,7 +24,10 @@ func TestCgoCrashHandler(t *testing.T) { } func TestCgoSignalDeadlock(t *testing.T) { - t.Parallel() + // Don't call t.Parallel, since too much work going on at the + // same time can cause the testprogcgo code to overrun its + // timeouts (issue #18598). + if testing.Short() && runtime.GOOS == "windows" { t.Skip("Skipping in short mode") // takes up to 64 seconds } @@ -291,33 +294,43 @@ func testCgoPprof(t *testing.T, buildArg, runArg string) { got, err := testEnv(exec.Command(exe, runArg)).CombinedOutput() if err != nil { + if testenv.Builder() == "linux-amd64-alpine" { + // See Issue 18243 and Issue 19938. + t.Skipf("Skipping failing test on Alpine (golang.org/issue/18243). Ignoring error: %v", err) + } t.Fatal(err) } fn := strings.TrimSpace(string(got)) defer os.Remove(fn) - cmd := testEnv(exec.Command(testenv.GoToolPath(t), "tool", "pprof", "-top", "-nodecount=1", exe, fn)) - - found := false - for i, e := range cmd.Env { - if strings.HasPrefix(e, "PPROF_TMPDIR=") { - cmd.Env[i] = "PPROF_TMPDIR=" + os.TempDir() - found = true - break + for try := 0; try < 2; try++ { + cmd := testEnv(exec.Command(testenv.GoToolPath(t), "tool", "pprof", "-top", "-nodecount=1")) + // Check that pprof works both with and without explicit executable on command line. + if try == 0 { + cmd.Args = append(cmd.Args, exe, fn) + } else { + cmd.Args = append(cmd.Args, fn) } - } - if !found { - cmd.Env = append(cmd.Env, "PPROF_TMPDIR="+os.TempDir()) - } - top, err := cmd.CombinedOutput() - t.Logf("%s", top) - if err != nil { - t.Fatal(err) - } + found := false + for i, e := range cmd.Env { + if strings.HasPrefix(e, "PPROF_TMPDIR=") { + cmd.Env[i] = "PPROF_TMPDIR=" + os.TempDir() + found = true + break + } + } + if !found { + cmd.Env = append(cmd.Env, "PPROF_TMPDIR="+os.TempDir()) + } - if !bytes.Contains(top, []byte("cpuHog")) { - t.Error("missing cpuHog in pprof output") + top, err := cmd.CombinedOutput() + t.Logf("%s:\n%s", cmd.Args, top) + if err != nil { + t.Error(err) + } else if !bytes.Contains(top, []byte("cpuHog")) { + t.Error("missing cpuHog in pprof output") + } } } @@ -397,3 +410,16 @@ func TestRaceSignal(t *testing.T) { t.Errorf("expected %q got %s", want, got) } } + +func TestCgoNumGoroutine(t *testing.T) { + switch runtime.GOOS { + case "windows", "plan9": + t.Skipf("skipping numgoroutine test on %s", runtime.GOOS) + } + t.Parallel() + got := runTestProg(t, "testprogcgo", "NumGoroutine") + want := "OK\n" + if got != want { + t.Errorf("expected %q got %v", want, got) + } +} diff --git a/libgo/go/runtime/crash_test.go b/libgo/go/runtime/crash_test.go index 4ba9d444b9c..1cde6bf7997 100644 --- a/libgo/go/runtime/crash_test.go +++ b/libgo/go/runtime/crash_test.go @@ -164,6 +164,12 @@ func checkStaleRuntime(t *testing.T) { return } if string(out) != "false\n" { + t.Logf("go list -f {{.Stale}} runtime:\n%s", out) + out, err := testEnv(exec.Command(testenv.GoToolPath(t), "list", "-f", "{{.StaleReason}}", "runtime")).CombinedOutput() + if err != nil { + t.Logf("go list -f {{.StaleReason}} failed: %v", err) + } + t.Logf("go list -f {{.StaleReason}} runtime:\n%s", out) staleRuntimeErr = fmt.Errorf("Stale runtime.a. Run 'go install runtime'.") } }) @@ -305,6 +311,9 @@ func TestNoHelperGoroutines(t *testing.T) { func TestBreakpoint(t *testing.T) { output := runTestProg(t, "testprog", "Breakpoint") + // If runtime.Breakpoint() is inlined, then the stack trace prints + // "runtime.Breakpoint(...)" instead of "runtime.Breakpoint()". + // For gccgo, no parens. want := "runtime.Breakpoint" if !strings.Contains(output, want) { t.Fatalf("output:\n%s\n\nwant output containing: %s", output, want) @@ -481,28 +490,33 @@ func TestMemPprof(t *testing.T) { fn := strings.TrimSpace(string(got)) defer os.Remove(fn) - cmd := testEnv(exec.Command(testenv.GoToolPath(t), "tool", "pprof", "-alloc_space", "-top", exe, fn)) - - found := false - for i, e := range cmd.Env { - if strings.HasPrefix(e, "PPROF_TMPDIR=") { - cmd.Env[i] = "PPROF_TMPDIR=" + os.TempDir() - found = true - break + for try := 0; try < 2; try++ { + cmd := testEnv(exec.Command(testenv.GoToolPath(t), "tool", "pprof", "-alloc_space", "-top")) + // Check that pprof works both with and without explicit executable on command line. + if try == 0 { + cmd.Args = append(cmd.Args, exe, fn) + } else { + cmd.Args = append(cmd.Args, fn) + } + found := false + for i, e := range cmd.Env { + if strings.HasPrefix(e, "PPROF_TMPDIR=") { + cmd.Env[i] = "PPROF_TMPDIR=" + os.TempDir() + found = true + break + } + } + if !found { + cmd.Env = append(cmd.Env, "PPROF_TMPDIR="+os.TempDir()) } - } - if !found { - cmd.Env = append(cmd.Env, "PPROF_TMPDIR="+os.TempDir()) - } - - top, err := cmd.CombinedOutput() - t.Logf("%s", top) - if err != nil { - t.Fatal(err) - } - if !bytes.Contains(top, []byte("MemProf")) { - t.Error("missing MemProf in pprof output") + top, err := cmd.CombinedOutput() + t.Logf("%s:\n%s", cmd.Args, top) + if err != nil { + t.Error(err) + } else if !bytes.Contains(top, []byte("MemProf")) { + t.Error("missing MemProf in pprof output") + } } } @@ -541,3 +555,87 @@ func TestConcurrentMapIterateWrite(t *testing.T) { t.Fatalf("output does not start with %q:\n%s", want, output) } } + +type point struct { + x, y *int +} + +func (p *point) negate() { + *p.x = *p.x * -1 + *p.y = *p.y * -1 +} + +// Test for issue #10152. +func TestPanicInlined(t *testing.T) { + defer func() { + r := recover() + if r == nil { + t.Fatalf("recover failed") + } + buf := make([]byte, 2048) + n := runtime.Stack(buf, false) + buf = buf[:n] + want := []byte("(*point).negate(") + if runtime.Compiler == "gccgo" { + want = []byte("negate.pN18_runtime_test.point") + } + if !bytes.Contains(buf, want) { + t.Logf("%s", buf) + t.Fatalf("expecting stack trace to contain call to %s", want) + } + }() + + pt := new(point) + pt.negate() +} + +// Test for issues #3934 and #20018. +// We want to delay exiting until a panic print is complete. +func TestPanicRace(t *testing.T) { + testenv.MustHaveGoRun(t) + + exe, err := buildTestProg(t, "testprog") + if err != nil { + t.Fatal(err) + } + + // The test is intentionally racy, and in my testing does not + // produce the expected output about 0.05% of the time. + // So run the program in a loop and only fail the test if we + // get the wrong output ten times in a row. + const tries = 10 +retry: + for i := 0; i < tries; i++ { + got, err := testEnv(exec.Command(exe, "PanicRace")).CombinedOutput() + if err == nil { + t.Logf("try %d: program exited successfully, should have failed", i+1) + continue + } + + if i > 0 { + t.Logf("try %d:\n", i+1) + } + t.Logf("%s\n", got) + + wants := []string{ + "panic: crash", + "PanicRace", + "created by ", + } + if runtime.Compiler == "gccgo" { + // gccgo will dump a function name like main.$nested30. + // Match on the file name instead. + wants[1] = "panicrace" + } + for _, want := range wants { + if !bytes.Contains(got, []byte(want)) { + t.Logf("did not find expected string %q", want) + continue retry + } + } + + // Test generated expected output. + return + } + t.Errorf("test ran %d times without producing expected output", tries) +} diff --git a/libgo/go/runtime/crash_unix_test.go b/libgo/go/runtime/crash_unix_test.go index 7a29c1eba79..09c25471d10 100644 --- a/libgo/go/runtime/crash_unix_test.go +++ b/libgo/go/runtime/crash_unix_test.go @@ -24,6 +24,15 @@ import ( // Send SIGQUIT to get a stack trace. var sigquit = syscall.SIGQUIT +func init() { + if runtime.Sigisblocked(int(syscall.SIGQUIT)) { + // We can't use SIGQUIT to kill subprocesses because + // it's blocked. Use SIGKILL instead. See issue + // #19196 for an example of when this happens. + sigquit = syscall.SIGKILL + } +} + func TestCrashDumpsAllThreads(t *testing.T) { switch runtime.GOOS { case "darwin", "dragonfly", "freebsd", "linux", "netbsd", "openbsd", "solaris": @@ -31,6 +40,10 @@ func TestCrashDumpsAllThreads(t *testing.T) { t.Skipf("skipping; not supported on %v", runtime.GOOS) } + if runtime.Sigisblocked(int(syscall.SIGQUIT)) { + t.Skip("skipping; SIGQUIT is blocked, see golang.org/issue/19196") + } + // We don't use executeTest because we need to kill the // program while it is running. @@ -165,6 +178,10 @@ func TestPanicSystemstack(t *testing.T) { t.Skip("Skipping in short mode (GOTRACEBACK=crash is slow)") } + if runtime.Sigisblocked(int(syscall.SIGQUIT)) { + t.Skip("skipping; SIGQUIT is blocked, see golang.org/issue/19196") + } + t.Parallel() cmd := exec.Command(os.Args[0], "testPanicSystemstackInternal") cmd = testEnv(cmd) @@ -251,3 +268,16 @@ func TestSignalIgnoreSIGTRAP(t *testing.T) { t.Fatalf("want %s, got %s\n", want, output) } } + +func TestSignalDuringExec(t *testing.T) { + switch runtime.GOOS { + case "darwin", "dragonfly", "freebsd", "linux", "netbsd", "openbsd": + default: + t.Skipf("skipping test on %s", runtime.GOOS) + } + output := runTestProg(t, "testprognet", "SignalDuringExec") + want := "OK\n" + if output != want { + t.Fatalf("want %s, got %s\n", want, output) + } +} diff --git a/libgo/go/runtime/debug/garbage.go b/libgo/go/runtime/debug/garbage.go index c82c024235b..785e9d4598e 100644 --- a/libgo/go/runtime/debug/garbage.go +++ b/libgo/go/runtime/debug/garbage.go @@ -89,9 +89,7 @@ func ReadGCStats(stats *GCStats) { // at startup, or 100 if the variable is not set. // A negative percentage disables garbage collection. func SetGCPercent(percent int) int { - old := setGCPercent(int32(percent)) - runtime.GC() - return int(old) + return int(setGCPercent(int32(percent))) } // FreeOSMemory forces a garbage collection followed by an diff --git a/libgo/go/runtime/debug/garbage_test.go b/libgo/go/runtime/debug/garbage_test.go index 04e954b1b1e..62eeb2c8078 100644 --- a/libgo/go/runtime/debug/garbage_test.go +++ b/libgo/go/runtime/debug/garbage_test.go @@ -5,6 +5,7 @@ package debug_test import ( + "internal/testenv" "runtime" . "runtime/debug" "testing" @@ -104,15 +105,78 @@ func TestFreeOSMemory(t *testing.T) { } } +var ( + setGCPercentBallast interface{} + setGCPercentSink interface{} +) + func TestSetGCPercent(t *testing.T) { + testenv.SkipFlaky(t, 20076) + // Test that the variable is being set and returned correctly. - // Assume the percentage itself is implemented fine during GC, - // which is harder to test. old := SetGCPercent(123) new := SetGCPercent(old) if new != 123 { t.Errorf("SetGCPercent(123); SetGCPercent(x) = %d, want 123", new) } + + // Test that the percentage is implemented correctly. + defer func() { + SetGCPercent(old) + setGCPercentBallast, setGCPercentSink = nil, nil + }() + SetGCPercent(100) + runtime.GC() + // Create 100 MB of live heap as a baseline. + const baseline = 100 << 20 + var ms runtime.MemStats + runtime.ReadMemStats(&ms) + setGCPercentBallast = make([]byte, baseline-ms.Alloc) + runtime.GC() + runtime.ReadMemStats(&ms) + if abs64(baseline-int64(ms.Alloc)) > 10<<20 { + t.Fatalf("failed to set up baseline live heap; got %d MB, want %d MB", ms.Alloc>>20, baseline>>20) + } + // NextGC should be ~200 MB. + const thresh = 20 << 20 // TODO: Figure out why this is so noisy on some builders + if want := int64(2 * baseline); abs64(want-int64(ms.NextGC)) > thresh { + t.Errorf("NextGC = %d MB, want %d±%d MB", ms.NextGC>>20, want>>20, thresh>>20) + } + // Create some garbage, but not enough to trigger another GC. + for i := 0; float64(i) < 1.2*baseline; i += 1 << 10 { + setGCPercentSink = make([]byte, 1<<10) + } + setGCPercentSink = nil + // Adjust GOGC to 50. NextGC should be ~150 MB. + SetGCPercent(50) + runtime.ReadMemStats(&ms) + if want := int64(1.5 * baseline); abs64(want-int64(ms.NextGC)) > thresh { + t.Errorf("NextGC = %d MB, want %d±%d MB", ms.NextGC>>20, want>>20, thresh>>20) + } + + // Trigger a GC and get back to 100 MB live with GOGC=100. + SetGCPercent(100) + runtime.GC() + // Raise live to 120 MB. + setGCPercentSink = make([]byte, int(0.2*baseline)) + // Lower GOGC to 10. This must force a GC. + runtime.ReadMemStats(&ms) + ngc1 := ms.NumGC + SetGCPercent(10) + // It may require an allocation to actually force the GC. + setGCPercentSink = make([]byte, 1<<20) + runtime.ReadMemStats(&ms) + ngc2 := ms.NumGC + if ngc1 == ngc2 { + t.Errorf("expected GC to run but it did not") + } +} + +func abs64(a int64) int64 { + if a < 0 { + return -a + } + return a } func TestSetMaxThreadsOvf(t *testing.T) { diff --git a/libgo/go/runtime/env_posix.go b/libgo/go/runtime/env_posix.go index 9bf7ddcc535..ddf3c02c025 100644 --- a/libgo/go/runtime/env_posix.go +++ b/libgo/go/runtime/env_posix.go @@ -11,7 +11,7 @@ func gogetenv(key string) string { if env == nil { throw("getenv before env init") } - for _, s := range environ() { + for _, s := range env { if len(s) > len(key) && s[len(key)] == '=' && s[:len(key)] == key { return s[len(key)+1:] } diff --git a/libgo/go/runtime/error.go b/libgo/go/runtime/error.go index 9cf2230ab3f..44e63d8744b 100644 --- a/libgo/go/runtime/error.go +++ b/libgo/go/runtime/error.go @@ -140,8 +140,6 @@ func typestring(x interface{}) string { // For calling from C. // Prints an argument passed to panic. -// There's room for arbitrary complexity here, but we keep it -// simple and handle just a few important cases: int, string, and Stringer. func printany(i interface{}) { switch v := i.(type) { case nil: @@ -150,16 +148,41 @@ func printany(i interface{}) { print(v.String()) case error: print(v.Error()) + case bool: + print(v) case int: print(v) + case int8: + print(v) + case int16: + print(v) + case int32: + print(v) + case int64: + print(v) + case uint: + print(v) + case uint8: + print(v) + case uint16: + print(v) + case uint32: + print(v) + case uint64: + print(v) + case uintptr: + print(v) + case float32: + print(v) + case float64: + print(v) + case complex64: + print(v) + case complex128: + print(v) case string: print(v) default: print("(", typestring(i), ") ", i) } } - -// called from generated code -func panicwrap(pkg, typ, meth string) { - panic(plainError("value method " + pkg + "." + typ + "." + meth + " called using nil *" + typ + " pointer")) -} diff --git a/libgo/go/runtime/example_test.go b/libgo/go/runtime/example_test.go new file mode 100644 index 00000000000..e4912a51588 --- /dev/null +++ b/libgo/go/runtime/example_test.go @@ -0,0 +1,54 @@ +// Copyright 2017 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package runtime_test + +import ( + "fmt" + "runtime" + "strings" +) + +func ExampleFrames() { + c := func() { + // Ask runtime.Callers for up to 10 pcs, including runtime.Callers itself. + pc := make([]uintptr, 10) + n := runtime.Callers(0, pc) + if n == 0 { + // No pcs available. Stop now. + // This can happen if the first argument to runtime.Callers is large. + return + } + + pc = pc[:n] // pass only valid pcs to runtime.CallersFrames + frames := runtime.CallersFrames(pc) + + // Loop to get frames. + // A fixed number of pcs can expand to an indefinite number of Frames. + for { + frame, more := frames.Next() + // To keep this example's output stable + // even if there are changes in the testing package, + // stop unwinding when we leave package runtime. + if !strings.Contains(frame.File, "runtime/") { + break + } + fmt.Printf("- more:%v | %s\n", more, frame.Function) + if !more { + break + } + } + } + + b := func() { c() } + a := func() { b() } + + a() + // Output: + // - more:true | runtime.Callers + // - more:true | runtime_test.ExampleFrames.func1 + // - more:true | runtime_test.ExampleFrames.func2 + // - more:true | runtime_test.ExampleFrames.func3 + // - more:true | runtime_test.ExampleFrames +} diff --git a/libgo/go/runtime/export_test.go b/libgo/go/runtime/export_test.go index bf435f447e0..6325dcb3948 100644 --- a/libgo/go/runtime/export_test.go +++ b/libgo/go/runtime/export_test.go @@ -41,11 +41,11 @@ type LFNode struct { } func LFStackPush(head *uint64, node *LFNode) { - lfstackpush(head, (*lfnode)(unsafe.Pointer(node))) + (*lfstack)(head).push((*lfnode)(unsafe.Pointer(node))) } func LFStackPop(head *uint64) *LFNode { - return (*LFNode)(unsafe.Pointer(lfstackpop(head))) + return (*LFNode)(unsafe.Pointer((*lfstack)(head).pop())) } func GCMask(x interface{}) (ret []byte) { @@ -241,6 +241,97 @@ func CountPagesInUse() (pagesInUse, counted uintptr) { return } +func Fastrand() uint32 { return fastrand() } +func Fastrandn(n uint32) uint32 { return fastrandn(n) } + +type ProfBuf profBuf + +func NewProfBuf(hdrsize, bufwords, tags int) *ProfBuf { + return (*ProfBuf)(newProfBuf(hdrsize, bufwords, tags)) +} + +func (p *ProfBuf) Write(tag *unsafe.Pointer, now int64, hdr []uint64, stk []uintptr) { + (*profBuf)(p).write(tag, now, hdr, stk) +} + +const ( + ProfBufBlocking = profBufBlocking + ProfBufNonBlocking = profBufNonBlocking +) + +func (p *ProfBuf) Read(mode profBufReadMode) ([]uint64, []unsafe.Pointer, bool) { + return (*profBuf)(p).read(profBufReadMode(mode)) +} + +func (p *ProfBuf) Close() { + (*profBuf)(p).close() +} + +// ReadMemStatsSlow returns both the runtime-computed MemStats and +// MemStats accumulated by scanning the heap. +func ReadMemStatsSlow() (base, slow MemStats) { + stopTheWorld("ReadMemStatsSlow") + + // Run on the system stack to avoid stack growth allocation. + systemstack(func() { + // Make sure stats don't change. + getg().m.mallocing++ + + readmemstats_m(&base) + + // Initialize slow from base and zero the fields we're + // recomputing. + slow = base + slow.Alloc = 0 + slow.TotalAlloc = 0 + slow.Mallocs = 0 + slow.Frees = 0 + var bySize [_NumSizeClasses]struct { + Mallocs, Frees uint64 + } + + // Add up current allocations in spans. + for _, s := range mheap_.allspans { + if s.state != mSpanInUse { + continue + } + if sizeclass := s.spanclass.sizeclass(); sizeclass == 0 { + slow.Mallocs++ + slow.Alloc += uint64(s.elemsize) + } else { + slow.Mallocs += uint64(s.allocCount) + slow.Alloc += uint64(s.allocCount) * uint64(s.elemsize) + bySize[sizeclass].Mallocs += uint64(s.allocCount) + } + } + + // Add in frees. readmemstats_m flushed the cached stats, so + // these are up-to-date. + var smallFree uint64 + slow.Frees = mheap_.nlargefree + for i := range mheap_.nsmallfree { + slow.Frees += mheap_.nsmallfree[i] + bySize[i].Frees = mheap_.nsmallfree[i] + bySize[i].Mallocs += mheap_.nsmallfree[i] + smallFree += mheap_.nsmallfree[i] * uint64(class_to_size[i]) + } + slow.Frees += memstats.tinyallocs + slow.Mallocs += slow.Frees + + slow.TotalAlloc = slow.Alloc + mheap_.largefree + smallFree + + for i := range slow.BySize { + slow.BySize[i].Mallocs = bySize[i].Mallocs + slow.BySize[i].Frees = bySize[i].Frees + } + + getg().m.mallocing-- + }) + + startTheWorld() + return +} + // BlockOnSystemStack switches to the system stack, prints "x\n" to // stderr, and blocks in a stack containing // "runtime.blockOnSystemStackInternal". @@ -253,3 +344,23 @@ func blockOnSystemStackInternal() { lock(&deadlock) lock(&deadlock) } + +type RWMutex struct { + rw rwmutex +} + +func (rw *RWMutex) RLock() { + rw.rw.rlock() +} + +func (rw *RWMutex) RUnlock() { + rw.rw.runlock() +} + +func (rw *RWMutex) Lock() { + rw.rw.lock() +} + +func (rw *RWMutex) Unlock() { + rw.rw.unlock() +} diff --git a/libgo/go/runtime/export_unix_test.go b/libgo/go/runtime/export_unix_test.go new file mode 100644 index 00000000000..54d577072ec --- /dev/null +++ b/libgo/go/runtime/export_unix_test.go @@ -0,0 +1,19 @@ +// Copyright 2017 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build darwin dragonfly freebsd linux netbsd openbsd solaris + +package runtime + +func sigismember(mask *sigset, i int) bool { + clear := *mask + sigdelset(&clear, i) + return clear != *mask +} + +func Sigisblocked(i int) bool { + var sigmask sigset + sigprocmask(_SIG_SETMASK, nil, &sigmask) + return sigismember(&sigmask, i) +} diff --git a/libgo/go/runtime/extern.go b/libgo/go/runtime/extern.go index 5c50760b8b8..6ca978980f2 100644 --- a/libgo/go/runtime/extern.go +++ b/libgo/go/runtime/extern.go @@ -50,13 +50,6 @@ It is a comma-separated list of name=val pairs setting these named variables: gcshrinkstackoff: setting gcshrinkstackoff=1 disables moving goroutines onto smaller stacks. In this mode, a goroutine's stack can only grow. - gcstackbarrieroff: setting gcstackbarrieroff=1 disables the use of stack barriers - that allow the garbage collector to avoid repeating a stack scan during the - mark termination phase. - - gcstackbarrierall: setting gcstackbarrierall=1 installs stack barriers - in every stack frame, rather than in exponentially-spaced frames. - gcrescanstacks: setting gcrescanstacks=1 enables stack re-scanning during the STW mark termination phase. This is helpful for debugging if objects are being prematurely @@ -85,7 +78,7 @@ It is a comma-separated list of name=val pairs setting these named variables: for mark/scan are broken down in to assist time (GC performed in line with allocation), background GC time, and idle GC time. If the line ends with "(forced)", this GC was forced by a - runtime.GC() call and all phases are STW. + runtime.GC() call. Setting gctrace to any value > 0 also causes the garbage collector to emit a summary when memory is released back to the system. @@ -173,7 +166,7 @@ func Gosched() // to ascend, with 0 identifying the caller of Caller. (For historical reasons the // meaning of skip differs between Caller and Callers.) The return values report the // program counter, file name, and line number within the file of the corresponding -// call. The boolean ok is false if it was not possible to recover the information. +// call. The boolean ok is false if it was not possible to recover the information. func Caller(skip int) (pc uintptr, file string, line int, ok bool) // Callers fills the slice pc with the return program counters of function invocations @@ -181,6 +174,14 @@ func Caller(skip int) (pc uintptr, file string, line int, ok bool) // to skip before recording in pc, with 0 identifying the frame for Callers itself and // 1 identifying the caller of Callers. // It returns the number of entries written to pc. +// +// To translate these PCs into symbolic information such as function +// names and line numbers, use CallersFrames. CallersFrames accounts +// for inlined functions and adjusts the return program counters into +// call program counters. Iterating over the returned slice of PCs +// directly is discouraged, as is using FuncForPC on any of the +// returned PCs, since these cannot account for inlining or return +// program counter adjustment. func Callers(skip int, pc []uintptr) int // GOROOT returns the root of the Go tree. @@ -206,7 +207,7 @@ func Version() string { const GOOS string = sys.GOOS // GOARCH is the running program's architecture target: -// 386, amd64, arm, or s390x. +// one of 386, amd64, arm, s390x, and so on. const GOARCH string = sys.GOARCH // GCCGOTOOLDIR is the Tool Dir for the gccgo build diff --git a/libgo/go/runtime/fastlog2.go b/libgo/go/runtime/fastlog2.go index 5f3fb534232..1f251bfaab2 100644 --- a/libgo/go/runtime/fastlog2.go +++ b/libgo/go/runtime/fastlog2.go @@ -4,8 +4,6 @@ package runtime -import "unsafe" - // fastlog2 implements a fast approximation to the base 2 log of a // float64. This is used to compute a geometric distribution for heap // sampling, without introducing dependencies into package math. This @@ -27,7 +25,3 @@ func fastlog2(x float64) float64 { low, high := fastlog2Table[xManIndex], fastlog2Table[xManIndex+1] return float64(xExp) + low + (high-low)*float64(xManScale)*fastlogScaleRatio } - -// float64bits returns the IEEE 754 binary representation of f. -// Taken from math.Float64bits to avoid dependencies into package math. -func float64bits(f float64) uint64 { return *(*uint64)(unsafe.Pointer(&f)) } diff --git a/libgo/go/runtime/float.go b/libgo/go/runtime/float.go new file mode 100644 index 00000000000..459e58dd7ef --- /dev/null +++ b/libgo/go/runtime/float.go @@ -0,0 +1,53 @@ +// Copyright 2017 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package runtime + +import "unsafe" + +var inf = float64frombits(0x7FF0000000000000) + +// isNaN reports whether f is an IEEE 754 ``not-a-number'' value. +func isNaN(f float64) (is bool) { + // IEEE 754 says that only NaNs satisfy f != f. + return f != f +} + +// isFinite reports whether f is neither NaN nor an infinity. +func isFinite(f float64) bool { + return !isNaN(f - f) +} + +// isInf reports whether f is an infinity. +func isInf(f float64) bool { + return !isNaN(f) && !isFinite(f) +} + +// Abs returns the absolute value of x. +// +// Special cases are: +// Abs(±Inf) = +Inf +// Abs(NaN) = NaN +func abs(x float64) float64 { + const sign = 1 << 63 + return float64frombits(float64bits(x) &^ sign) +} + +// copysign returns a value with the magnitude +// of x and the sign of y. +func copysign(x, y float64) float64 { + const sign = 1 << 63 + return float64frombits(float64bits(x)&^sign | float64bits(y)&sign) +} + +// Float64bits returns the IEEE 754 binary representation of f. +func float64bits(f float64) uint64 { + return *(*uint64)(unsafe.Pointer(&f)) +} + +// Float64frombits returns the floating point number corresponding +// the IEEE 754 binary representation b. +func float64frombits(b uint64) float64 { + return *(*float64)(unsafe.Pointer(&b)) +} diff --git a/libgo/go/runtime/gc_test.go b/libgo/go/runtime/gc_test.go index ec043ed45be..f14e0d5050e 100644 --- a/libgo/go/runtime/gc_test.go +++ b/libgo/go/runtime/gc_test.go @@ -5,6 +5,7 @@ package runtime_test import ( + "fmt" "os" "reflect" "runtime" @@ -450,3 +451,53 @@ func TestPageAccounting(t *testing.T) { t.Fatalf("mheap_.pagesInUse is %d, but direct count is %d", pagesInUse, counted) } } + +func TestReadMemStats(t *testing.T) { + base, slow := runtime.ReadMemStatsSlow() + if base != slow { + logDiff(t, "MemStats", reflect.ValueOf(base), reflect.ValueOf(slow)) + t.Fatal("memstats mismatch") + } +} + +func logDiff(t *testing.T, prefix string, got, want reflect.Value) { + typ := got.Type() + switch typ.Kind() { + case reflect.Array, reflect.Slice: + if got.Len() != want.Len() { + t.Logf("len(%s): got %v, want %v", prefix, got, want) + return + } + for i := 0; i < got.Len(); i++ { + logDiff(t, fmt.Sprintf("%s[%d]", prefix, i), got.Index(i), want.Index(i)) + } + case reflect.Struct: + for i := 0; i < typ.NumField(); i++ { + gf, wf := got.Field(i), want.Field(i) + logDiff(t, prefix+"."+typ.Field(i).Name, gf, wf) + } + case reflect.Map: + t.Fatal("not implemented: logDiff for map") + default: + if got.Interface() != want.Interface() { + t.Logf("%s: got %v, want %v", prefix, got, want) + } + } +} + +func BenchmarkReadMemStats(b *testing.B) { + var ms runtime.MemStats + const heapSize = 100 << 20 + x := make([]*[1024]byte, heapSize/1024) + for i := range x { + x[i] = new([1024]byte) + } + hugeSink = x + + b.ResetTimer() + for i := 0; i < b.N; i++ { + runtime.ReadMemStats(&ms) + } + + hugeSink = nil +} diff --git a/libgo/go/runtime/hashmap.go b/libgo/go/runtime/hashmap.go index 5b191d45752..a3e50cd9221 100644 --- a/libgo/go/runtime/hashmap.go +++ b/libgo/go/runtime/hashmap.go @@ -129,6 +129,11 @@ type hmap struct { oldbuckets unsafe.Pointer // previous bucket array of half the size, non-nil only when growing nevacuate uintptr // progress counter for evacuation (buckets less than this have been evacuated) + extra *mapextra // optional fields +} + +// mapextra holds fields that are not present on all maps. +type mapextra struct { // If both key and value do not contain pointers and are inline, then we mark bucket // type as containing no pointers. This avoids scanning such maps. // However, bmap.overflow is a pointer. In order to keep overflow buckets @@ -136,9 +141,11 @@ type hmap struct { // Overflow is used only if key and value do not contain pointers. // overflow[0] contains overflow buckets for hmap.buckets. // overflow[1] contains overflow buckets for hmap.oldbuckets. - // The first indirection allows us to reduce static size of hmap. - // The second indirection allows to store a pointer to the slice in hiter. - overflow *[2]*[]*bmap + // The indirection allows to store a pointer to the slice in hiter. + overflow [2]*[]*bmap + + // nextOverflow holds a pointer to a free overflow bucket. + nextOverflow *bmap } // A bucket for a Go map. @@ -183,6 +190,10 @@ func (b *bmap) overflow(t *maptype) *bmap { return *(**bmap)(add(unsafe.Pointer(b), uintptr(t.bucketsize)-sys.PtrSize)) } +func (b *bmap) setoverflow(t *maptype, ovf *bmap) { + *(**bmap)(add(unsafe.Pointer(b), uintptr(t.bucketsize)-sys.PtrSize)) = ovf +} + // incrnoverflow increments h.noverflow. // noverflow counts the number of overflow buckets. // This is used to trigger same-size map growth. @@ -209,21 +220,40 @@ func (h *hmap) incrnoverflow() { } } -func (h *hmap) setoverflow(t *maptype, b, ovf *bmap) { +func (h *hmap) newoverflow(t *maptype, b *bmap) *bmap { + var ovf *bmap + if h.extra != nil && h.extra.nextOverflow != nil { + // We have preallocated overflow buckets available. + // See makeBucketArray for more details. + ovf = h.extra.nextOverflow + if ovf.overflow(t) == nil { + // We're not at the end of the preallocated overflow buckets. Bump the pointer. + h.extra.nextOverflow = (*bmap)(add(unsafe.Pointer(ovf), uintptr(t.bucketsize))) + } else { + // This is the last preallocated overflow bucket. + // Reset the overflow pointer on this bucket, + // which was set to a non-nil sentinel value. + ovf.setoverflow(t, nil) + h.extra.nextOverflow = nil + } + } else { + ovf = (*bmap)(newobject(t.bucket)) + } h.incrnoverflow() if t.bucket.kind&kindNoPointers != 0 { h.createOverflow() - *h.overflow[0] = append(*h.overflow[0], ovf) + *h.extra.overflow[0] = append(*h.extra.overflow[0], ovf) } - *(**bmap)(add(unsafe.Pointer(b), uintptr(t.bucketsize)-sys.PtrSize)) = ovf + b.setoverflow(t, ovf) + return ovf } func (h *hmap) createOverflow() { - if h.overflow == nil { - h.overflow = new([2]*[]*bmap) + if h.extra == nil { + h.extra = new(mapextra) } - if h.overflow[0] == nil { - h.overflow[0] = new([]*bmap) + if h.extra.overflow[0] == nil { + h.extra.overflow[0] = new([]*bmap) } } @@ -238,9 +268,8 @@ func makemap(t *maptype, hint int64, h *hmap, bucket unsafe.Pointer) *hmap { throw("bad hmap size") } - if hint < 0 || int64(int32(hint)) != hint { - panic(plainError("makemap: size out of range")) - // TODO: make hint an int, then none of this nonsense + if hint < 0 || hint > int64(maxSliceCap(t.bucket.size)) { + hint = 0 } if !ismapkey(t.key) { @@ -290,8 +319,14 @@ func makemap(t *maptype, hint int64, h *hmap, bucket unsafe.Pointer) *hmap { // if B == 0, the buckets field is allocated lazily later (in mapassign) // If hint is large zeroing this memory could take a while. buckets := bucket + var extra *mapextra if B != 0 { - buckets = newarray(t.bucket, 1<= 4 { + // Add on the estimated number of overflow buckets + // required to insert the median number of elements + // used with this value of b. + nbuckets += 1 << (b - 4) + sz := t.bucket.size * nbuckets + up := roundupsize(sz) + if up != sz { + nbuckets = up / t.bucket.size + } + } + buckets = newarray(t.bucket, int(nbuckets)) + if base != nbuckets { + // We preallocated some overflow buckets. + // To keep the overhead of tracking these overflow buckets to a minimum, + // we use the convention that if a preallocated overflow bucket's overflow + // pointer is nil, then there are more available by bumping the pointer. + // We need a safe non-nil pointer for the last overflow bucket; just use buckets. + nextOverflow = (*bmap)(add(buckets, base*uintptr(t.bucketsize))) + last := (*bmap)(add(buckets, (nbuckets-1)*uintptr(t.bucketsize))) + last.setoverflow(t, (*bmap)(buckets)) + } + return buckets, nextOverflow +} + func hashGrow(t *maptype, h *hmap) { // If we've hit the load factor, get bigger. // Otherwise, there are too many overflow buckets, @@ -896,7 +967,8 @@ func hashGrow(t *maptype, h *hmap) { h.flags |= sameSizeGrow } oldbuckets := h.buckets - newbuckets := newarray(t.bucket, 1<<(h.B+bigger)) + newbuckets, nextOverflow := makeBucketArray(t, h.B+bigger) + flags := h.flags &^ (iterator | oldIterator) if h.flags&iterator != 0 { flags |= oldIterator @@ -909,13 +981,19 @@ func hashGrow(t *maptype, h *hmap) { h.nevacuate = 0 h.noverflow = 0 - if h.overflow != nil { + if h.extra != nil && h.extra.overflow[0] != nil { // Promote current overflow buckets to the old generation. - if h.overflow[1] != nil { + if h.extra.overflow[1] != nil { throw("overflow is not nil") } - h.overflow[1] = h.overflow[0] - h.overflow[0] = nil + h.extra.overflow[1] = h.extra.overflow[0] + h.extra.overflow[0] = nil + } + if nextOverflow != nil { + if h.extra == nil { + h.extra = new(mapextra) + } + h.extra.nextOverflow = nextOverflow } // the actual copying of the hash table data is done incrementally @@ -925,7 +1003,7 @@ func hashGrow(t *maptype, h *hmap) { // overLoadFactor reports whether count items placed in 1<= bucketCnt && float32(count) >= loadFactor*float32((uintptr(1)<= bucketCnt && float32(count) >= loadFactor*float32((uint64(1)< newbit { + stop = newbit + } + for h.nevacuate != stop && bucketEvacuated(t, h, h.nevacuate) { + h.nevacuate++ + } + if h.nevacuate == newbit { // newbit == # of oldbuckets // Growing is all done. Free old main bucket array. h.oldbuckets = nil // Can discard old overflow buckets as well. // If they are still referenced by an iterator, // then the iterator holds a pointers to the slice. - if h.overflow != nil { - h.overflow[1] = nil + if h.extra != nil { + h.extra.overflow[1] = nil } h.flags &^= sameSizeGrow } @@ -1139,8 +1229,8 @@ func ismapkey(t *_type) bool { // Reflect stubs. Called from ../reflect/asm_*.s //go:linkname reflect_makemap reflect.makemap -func reflect_makemap(t *maptype) *hmap { - return makemap(t, 0, nil, nil) +func reflect_makemap(t *maptype, cap int) *hmap { + return makemap(t, int64(cap), nil, nil) } //go:linkname reflect_mapaccess reflect.mapaccess diff --git a/libgo/go/runtime/hashmap_fast.go b/libgo/go/runtime/hashmap_fast.go index 853da70e966..bec8fdac14e 100644 --- a/libgo/go/runtime/hashmap_fast.go +++ b/libgo/go/runtime/hashmap_fast.go @@ -45,7 +45,7 @@ func mapaccess1_fast32(t *maptype, h *hmap, key uint32) unsafe.Pointer { if k != key { continue } - x := *((*uint8)(add(unsafe.Pointer(b), i))) // b.topbits[i] without the bounds check + x := *((*uint8)(add(unsafe.Pointer(b), i))) // b.tophash[i] without the bounds check if x == empty { continue } @@ -94,7 +94,7 @@ func mapaccess2_fast32(t *maptype, h *hmap, key uint32) (unsafe.Pointer, bool) { if k != key { continue } - x := *((*uint8)(add(unsafe.Pointer(b), i))) // b.topbits[i] without the bounds check + x := *((*uint8)(add(unsafe.Pointer(b), i))) // b.tophash[i] without the bounds check if x == empty { continue } @@ -143,7 +143,7 @@ func mapaccess1_fast64(t *maptype, h *hmap, key uint64) unsafe.Pointer { if k != key { continue } - x := *((*uint8)(add(unsafe.Pointer(b), i))) // b.topbits[i] without the bounds check + x := *((*uint8)(add(unsafe.Pointer(b), i))) // b.tophash[i] without the bounds check if x == empty { continue } @@ -192,7 +192,7 @@ func mapaccess2_fast64(t *maptype, h *hmap, key uint64) (unsafe.Pointer, bool) { if k != key { continue } - x := *((*uint8)(add(unsafe.Pointer(b), i))) // b.topbits[i] without the bounds check + x := *((*uint8)(add(unsafe.Pointer(b), i))) // b.tophash[i] without the bounds check if x == empty { continue } @@ -223,7 +223,7 @@ func mapaccess1_faststr(t *maptype, h *hmap, ky string) unsafe.Pointer { if key.len < 32 { // short key, doing lots of comparisons is ok for i := uintptr(0); i < bucketCnt; i++ { - x := *((*uint8)(add(unsafe.Pointer(b), i))) // b.topbits[i] without the bounds check + x := *((*uint8)(add(unsafe.Pointer(b), i))) // b.tophash[i] without the bounds check if x == empty { continue } @@ -240,7 +240,7 @@ func mapaccess1_faststr(t *maptype, h *hmap, ky string) unsafe.Pointer { // long key, try not to do more comparisons than necessary keymaybe := uintptr(bucketCnt) for i := uintptr(0); i < bucketCnt; i++ { - x := *((*uint8)(add(unsafe.Pointer(b), i))) // b.topbits[i] without the bounds check + x := *((*uint8)(add(unsafe.Pointer(b), i))) // b.tophash[i] without the bounds check if x == empty { continue } @@ -252,8 +252,6 @@ func mapaccess1_faststr(t *maptype, h *hmap, ky string) unsafe.Pointer { return add(unsafe.Pointer(b), dataOffset+bucketCnt*2*sys.PtrSize+i*uintptr(t.valuesize)) } // check first 4 bytes - // TODO: on amd64/386 at least, make this compile to one 4-byte comparison instead of - // four 1-byte comparisons. if *((*[4]byte)(key.str)) != *((*[4]byte)(k.str)) { continue } @@ -295,7 +293,7 @@ dohash: } for { for i := uintptr(0); i < bucketCnt; i++ { - x := *((*uint8)(add(unsafe.Pointer(b), i))) // b.topbits[i] without the bounds check + x := *((*uint8)(add(unsafe.Pointer(b), i))) // b.tophash[i] without the bounds check if x != top { continue } @@ -332,7 +330,7 @@ func mapaccess2_faststr(t *maptype, h *hmap, ky string) (unsafe.Pointer, bool) { if key.len < 32 { // short key, doing lots of comparisons is ok for i := uintptr(0); i < bucketCnt; i++ { - x := *((*uint8)(add(unsafe.Pointer(b), i))) // b.topbits[i] without the bounds check + x := *((*uint8)(add(unsafe.Pointer(b), i))) // b.tophash[i] without the bounds check if x == empty { continue } @@ -349,7 +347,7 @@ func mapaccess2_faststr(t *maptype, h *hmap, ky string) (unsafe.Pointer, bool) { // long key, try not to do more comparisons than necessary keymaybe := uintptr(bucketCnt) for i := uintptr(0); i < bucketCnt; i++ { - x := *((*uint8)(add(unsafe.Pointer(b), i))) // b.topbits[i] without the bounds check + x := *((*uint8)(add(unsafe.Pointer(b), i))) // b.tophash[i] without the bounds check if x == empty { continue } @@ -402,7 +400,7 @@ dohash: } for { for i := uintptr(0); i < bucketCnt; i++ { - x := *((*uint8)(add(unsafe.Pointer(b), i))) // b.topbits[i] without the bounds check + x := *((*uint8)(add(unsafe.Pointer(b), i))) // b.tophash[i] without the bounds check if x != top { continue } @@ -420,3 +418,441 @@ dohash: } } } + +func mapassign_fast32(t *maptype, h *hmap, key uint32) unsafe.Pointer { + if h == nil { + panic(plainError("assignment to entry in nil map")) + } + if raceenabled { + callerpc := getcallerpc(unsafe.Pointer(&t)) + racewritepc(unsafe.Pointer(h), callerpc, funcPC(mapassign_fast32)) + } + if h.flags&hashWriting != 0 { + throw("concurrent map writes") + } + hash := t.key.hashfn(noescape(unsafe.Pointer(&key)), uintptr(h.hash0)) + + // Set hashWriting after calling alg.hash for consistency with mapassign. + h.flags |= hashWriting + + if h.buckets == nil { + h.buckets = newarray(t.bucket, 1) + } + +again: + bucket := hash & (uintptr(1)<> (sys.PtrSize*8 - 8)) + if top < minTopHash { + top += minTopHash + } + + var inserti *uint8 + var insertk unsafe.Pointer + var val unsafe.Pointer + for { + for i := uintptr(0); i < bucketCnt; i++ { + if b.tophash[i] != top { + if b.tophash[i] == empty && inserti == nil { + inserti = &b.tophash[i] + insertk = add(unsafe.Pointer(b), dataOffset+i*4) + val = add(unsafe.Pointer(b), dataOffset+bucketCnt*4+i*uintptr(t.valuesize)) + } + continue + } + k := *((*uint32)(add(unsafe.Pointer(b), dataOffset+i*4))) + if k != key { + continue + } + val = add(unsafe.Pointer(b), dataOffset+bucketCnt*4+i*uintptr(t.valuesize)) + goto done + } + ovf := b.overflow(t) + if ovf == nil { + break + } + b = ovf + } + + // Did not find mapping for key. Allocate new cell & add entry. + + // If we hit the max load factor or we have too many overflow buckets, + // and we're not already in the middle of growing, start growing. + if !h.growing() && (overLoadFactor(int64(h.count), h.B) || tooManyOverflowBuckets(h.noverflow, h.B)) { + hashGrow(t, h) + goto again // Growing the table invalidates everything, so try again + } + + if inserti == nil { + // all current buckets are full, allocate a new one. + newb := h.newoverflow(t, b) + inserti = &newb.tophash[0] + insertk = add(unsafe.Pointer(newb), dataOffset) + val = add(insertk, bucketCnt*4) + } + + // store new key/value at insert position + typedmemmove(t.key, insertk, unsafe.Pointer(&key)) + *inserti = top + h.count++ + +done: + if h.flags&hashWriting == 0 { + throw("concurrent map writes") + } + h.flags &^= hashWriting + return val +} + +func mapassign_fast64(t *maptype, h *hmap, key uint64) unsafe.Pointer { + if h == nil { + panic(plainError("assignment to entry in nil map")) + } + if raceenabled { + callerpc := getcallerpc(unsafe.Pointer(&t)) + racewritepc(unsafe.Pointer(h), callerpc, funcPC(mapassign_fast64)) + } + if h.flags&hashWriting != 0 { + throw("concurrent map writes") + } + hash := t.key.hashfn(noescape(unsafe.Pointer(&key)), uintptr(h.hash0)) + + // Set hashWriting after calling alg.hash for consistency with mapassign. + h.flags |= hashWriting + + if h.buckets == nil { + h.buckets = newarray(t.bucket, 1) + } + +again: + bucket := hash & (uintptr(1)<> (sys.PtrSize*8 - 8)) + if top < minTopHash { + top += minTopHash + } + + var inserti *uint8 + var insertk unsafe.Pointer + var val unsafe.Pointer + for { + for i := uintptr(0); i < bucketCnt; i++ { + if b.tophash[i] != top { + if b.tophash[i] == empty && inserti == nil { + inserti = &b.tophash[i] + insertk = add(unsafe.Pointer(b), dataOffset+i*8) + val = add(unsafe.Pointer(b), dataOffset+bucketCnt*8+i*uintptr(t.valuesize)) + } + continue + } + k := *((*uint64)(add(unsafe.Pointer(b), dataOffset+i*8))) + if k != key { + continue + } + val = add(unsafe.Pointer(b), dataOffset+bucketCnt*8+i*uintptr(t.valuesize)) + goto done + } + ovf := b.overflow(t) + if ovf == nil { + break + } + b = ovf + } + + // Did not find mapping for key. Allocate new cell & add entry. + + // If we hit the max load factor or we have too many overflow buckets, + // and we're not already in the middle of growing, start growing. + if !h.growing() && (overLoadFactor(int64(h.count), h.B) || tooManyOverflowBuckets(h.noverflow, h.B)) { + hashGrow(t, h) + goto again // Growing the table invalidates everything, so try again + } + + if inserti == nil { + // all current buckets are full, allocate a new one. + newb := h.newoverflow(t, b) + inserti = &newb.tophash[0] + insertk = add(unsafe.Pointer(newb), dataOffset) + val = add(insertk, bucketCnt*8) + } + + // store new key/value at insert position + typedmemmove(t.key, insertk, unsafe.Pointer(&key)) + *inserti = top + h.count++ + +done: + if h.flags&hashWriting == 0 { + throw("concurrent map writes") + } + h.flags &^= hashWriting + return val +} + +func mapassign_faststr(t *maptype, h *hmap, ky string) unsafe.Pointer { + if h == nil { + panic(plainError("assignment to entry in nil map")) + } + if raceenabled { + callerpc := getcallerpc(unsafe.Pointer(&t)) + racewritepc(unsafe.Pointer(h), callerpc, funcPC(mapassign_faststr)) + } + if h.flags&hashWriting != 0 { + throw("concurrent map writes") + } + key := stringStructOf(&ky) + hash := t.key.hashfn(noescape(unsafe.Pointer(&ky)), uintptr(h.hash0)) + + // Set hashWriting after calling alg.hash for consistency with mapassign. + h.flags |= hashWriting + + if h.buckets == nil { + h.buckets = newarray(t.bucket, 1) + } + +again: + bucket := hash & (uintptr(1)<> (sys.PtrSize*8 - 8)) + if top < minTopHash { + top += minTopHash + } + + var inserti *uint8 + var insertk unsafe.Pointer + var val unsafe.Pointer + for { + for i := uintptr(0); i < bucketCnt; i++ { + if b.tophash[i] != top { + if b.tophash[i] == empty && inserti == nil { + inserti = &b.tophash[i] + insertk = add(unsafe.Pointer(b), dataOffset+i*uintptr(t.keysize)) + val = add(unsafe.Pointer(b), dataOffset+bucketCnt*uintptr(t.keysize)+i*uintptr(t.valuesize)) + } + continue + } + k := (*stringStruct)(add(unsafe.Pointer(b), dataOffset+i*2*sys.PtrSize)) + if k.len != key.len { + continue + } + if k.str != key.str && !memequal(k.str, key.str, uintptr(key.len)) { + continue + } + // already have a mapping for key. Update it. + val = add(unsafe.Pointer(b), dataOffset+bucketCnt*2*sys.PtrSize+i*uintptr(t.valuesize)) + goto done + } + ovf := b.overflow(t) + if ovf == nil { + break + } + b = ovf + } + + // Did not find mapping for key. Allocate new cell & add entry. + + // If we hit the max load factor or we have too many overflow buckets, + // and we're not already in the middle of growing, start growing. + if !h.growing() && (overLoadFactor(int64(h.count), h.B) || tooManyOverflowBuckets(h.noverflow, h.B)) { + hashGrow(t, h) + goto again // Growing the table invalidates everything, so try again + } + + if inserti == nil { + // all current buckets are full, allocate a new one. + newb := h.newoverflow(t, b) + inserti = &newb.tophash[0] + insertk = add(unsafe.Pointer(newb), dataOffset) + val = add(insertk, bucketCnt*2*sys.PtrSize) + } + + // store new key/value at insert position + *((*stringStruct)(insertk)) = *key + *inserti = top + h.count++ + +done: + if h.flags&hashWriting == 0 { + throw("concurrent map writes") + } + h.flags &^= hashWriting + return val +} + +func mapdelete_fast32(t *maptype, h *hmap, key uint32) { + if raceenabled && h != nil { + callerpc := getcallerpc(unsafe.Pointer(&t)) + racewritepc(unsafe.Pointer(h), callerpc, funcPC(mapdelete_fast32)) + } + if h == nil || h.count == 0 { + return + } + if h.flags&hashWriting != 0 { + throw("concurrent map writes") + } + + hash := t.key.hashfn(noescape(unsafe.Pointer(&key)), uintptr(h.hash0)) + + // Set hashWriting after calling alg.hash for consistency with mapdelete + h.flags |= hashWriting + + bucket := hash & (uintptr(1)<> (sys.PtrSize*8 - 8)) + if top < minTopHash { + top += minTopHash + } + for { + for i := uintptr(0); i < bucketCnt; i++ { + if b.tophash[i] != top { + continue + } + k := (*uint32)(add(unsafe.Pointer(b), dataOffset+i*4)) + if key != *k { + continue + } + typedmemclr(t.key, unsafe.Pointer(k)) + v := unsafe.Pointer(uintptr(unsafe.Pointer(b)) + dataOffset + bucketCnt*4 + i*uintptr(t.valuesize)) + typedmemclr(t.elem, v) + b.tophash[i] = empty + h.count-- + goto done + } + b = b.overflow(t) + if b == nil { + goto done + } + } + +done: + if h.flags&hashWriting == 0 { + throw("concurrent map writes") + } + h.flags &^= hashWriting +} + +func mapdelete_fast64(t *maptype, h *hmap, key uint64) { + if raceenabled && h != nil { + callerpc := getcallerpc(unsafe.Pointer(&t)) + racewritepc(unsafe.Pointer(h), callerpc, funcPC(mapdelete_fast64)) + } + if h == nil || h.count == 0 { + return + } + if h.flags&hashWriting != 0 { + throw("concurrent map writes") + } + + hash := t.key.hashfn(noescape(unsafe.Pointer(&key)), uintptr(h.hash0)) + + // Set hashWriting after calling alg.hash for consistency with mapdelete + h.flags |= hashWriting + + bucket := hash & (uintptr(1)<> (sys.PtrSize*8 - 8)) + if top < minTopHash { + top += minTopHash + } + for { + for i := uintptr(0); i < bucketCnt; i++ { + if b.tophash[i] != top { + continue + } + k := (*uint64)(add(unsafe.Pointer(b), dataOffset+i*8)) + if key != *k { + continue + } + typedmemclr(t.key, unsafe.Pointer(k)) + v := unsafe.Pointer(uintptr(unsafe.Pointer(b)) + dataOffset + bucketCnt*8 + i*uintptr(t.valuesize)) + typedmemclr(t.elem, v) + b.tophash[i] = empty + h.count-- + goto done + } + b = b.overflow(t) + if b == nil { + goto done + } + } + +done: + if h.flags&hashWriting == 0 { + throw("concurrent map writes") + } + h.flags &^= hashWriting +} + +func mapdelete_faststr(t *maptype, h *hmap, ky string) { + if raceenabled && h != nil { + callerpc := getcallerpc(unsafe.Pointer(&t)) + racewritepc(unsafe.Pointer(h), callerpc, funcPC(mapdelete_faststr)) + } + if h == nil || h.count == 0 { + return + } + if h.flags&hashWriting != 0 { + throw("concurrent map writes") + } + + key := stringStructOf(&ky) + hash := t.key.hashfn(noescape(unsafe.Pointer(&ky)), uintptr(h.hash0)) + + // Set hashWriting after calling alg.hash for consistency with mapdelete + h.flags |= hashWriting + + bucket := hash & (uintptr(1)<> (sys.PtrSize*8 - 8)) + if top < minTopHash { + top += minTopHash + } + for { + for i := uintptr(0); i < bucketCnt; i++ { + if b.tophash[i] != top { + continue + } + k := (*stringStruct)(add(unsafe.Pointer(b), dataOffset+i*2*sys.PtrSize)) + if k.len != key.len { + continue + } + if k.str != key.str && !memequal(k.str, key.str, uintptr(key.len)) { + continue + } + typedmemclr(t.key, unsafe.Pointer(k)) + v := unsafe.Pointer(uintptr(unsafe.Pointer(b)) + dataOffset + bucketCnt*2*sys.PtrSize + i*uintptr(t.valuesize)) + typedmemclr(t.elem, v) + b.tophash[i] = empty + h.count-- + goto done + } + b = b.overflow(t) + if b == nil { + goto done + } + } + +done: + if h.flags&hashWriting == 0 { + throw("concurrent map writes") + } + h.flags &^= hashWriting +} diff --git a/libgo/go/runtime/heapdump.go b/libgo/go/runtime/heapdump.go index 0db53f544a5..166199b5ca3 100644 --- a/libgo/go/runtime/heapdump.go +++ b/libgo/go/runtime/heapdump.go @@ -413,7 +413,7 @@ func dumpmemstats() { dumpint(memstats.gc_sys) dumpint(memstats.other_sys) dumpint(memstats.next_gc) - dumpint(memstats.last_gc) + dumpint(memstats.last_gc_unix) dumpint(memstats.pause_total_ns) for i := 0; i < 256; i++ { dumpint(memstats.pause_ns[i]) @@ -515,7 +515,7 @@ func writeheapdump_m(fd uintptr) { // Update stats so we can dump them. // As a side effect, flushes all the MCaches so the MSpan.freelist // lists contain all the free objects. - updatememstats(nil) + updatememstats() // Set dump file. dumpfd = fd diff --git a/libgo/go/runtime/iface_test.go b/libgo/go/runtime/iface_test.go index 3744a4f9700..d63ea796138 100644 --- a/libgo/go/runtime/iface_test.go +++ b/libgo/go/runtime/iface_test.go @@ -29,6 +29,20 @@ func (TM) Method2() {} func (TL) Method1() {} func (TL) Method2() {} +type T8 uint8 +type T16 uint16 +type T32 uint32 +type T64 uint64 +type Tstr string +type Tslice []byte + +func (T8) Method1() {} +func (T16) Method1() {} +func (T32) Method1() {} +func (T64) Method1() {} +func (Tstr) Method1() {} +func (Tslice) Method1() {} + var ( e interface{} e_ interface{} @@ -269,3 +283,133 @@ func TestNonEscapingConvT2I(t *testing.T) { t.Fatalf("want 0 allocs, got %v", n) } } + +func TestZeroConvT2x(t *testing.T) { + if runtime.Compiler == "gccgo" { + t.Skip("does not work on gccgo without better escape analysis") + } + + tests := []struct { + name string + fn func() + }{ + {name: "E8", fn: func() { e = eight8 }}, // any byte-sized value does not allocate + {name: "E16", fn: func() { e = zero16 }}, // zero values do not allocate + {name: "E32", fn: func() { e = zero32 }}, + {name: "E64", fn: func() { e = zero64 }}, + {name: "Estr", fn: func() { e = zerostr }}, + {name: "Eslice", fn: func() { e = zeroslice }}, + {name: "Econstflt", fn: func() { e = 99.0 }}, // constants do not allocate + {name: "Econststr", fn: func() { e = "change" }}, + {name: "I8", fn: func() { i1 = eight8I }}, + {name: "I16", fn: func() { i1 = zero16I }}, + {name: "I32", fn: func() { i1 = zero32I }}, + {name: "I64", fn: func() { i1 = zero64I }}, + {name: "Istr", fn: func() { i1 = zerostrI }}, + {name: "Islice", fn: func() { i1 = zerosliceI }}, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + n := testing.AllocsPerRun(1000, test.fn) + if n != 0 { + t.Errorf("want zero allocs, got %v", n) + } + }) + } +} + +var ( + eight8 uint8 = 8 + eight8I T8 = 8 + + zero16 uint16 = 0 + zero16I T16 = 0 + one16 uint16 = 1 + + zero32 uint32 = 0 + zero32I T32 = 0 + one32 uint32 = 1 + + zero64 uint64 = 0 + zero64I T64 = 0 + one64 uint64 = 1 + + zerostr string = "" + zerostrI Tstr = "" + nzstr string = "abc" + + zeroslice []byte = nil + zerosliceI Tslice = nil + nzslice []byte = []byte("abc") + + zerobig [512]byte + nzbig [512]byte = [512]byte{511: 1} +) + +func BenchmarkConvT2Ezero(b *testing.B) { + b.Run("zero", func(b *testing.B) { + b.Run("16", func(b *testing.B) { + for i := 0; i < b.N; i++ { + e = zero16 + } + }) + b.Run("32", func(b *testing.B) { + for i := 0; i < b.N; i++ { + e = zero32 + } + }) + b.Run("64", func(b *testing.B) { + for i := 0; i < b.N; i++ { + e = zero64 + } + }) + b.Run("str", func(b *testing.B) { + for i := 0; i < b.N; i++ { + e = zerostr + } + }) + b.Run("slice", func(b *testing.B) { + for i := 0; i < b.N; i++ { + e = zeroslice + } + }) + b.Run("big", func(b *testing.B) { + for i := 0; i < b.N; i++ { + e = zerobig + } + }) + }) + b.Run("nonzero", func(b *testing.B) { + b.Run("16", func(b *testing.B) { + for i := 0; i < b.N; i++ { + e = one16 + } + }) + b.Run("32", func(b *testing.B) { + for i := 0; i < b.N; i++ { + e = one32 + } + }) + b.Run("64", func(b *testing.B) { + for i := 0; i < b.N; i++ { + e = one64 + } + }) + b.Run("str", func(b *testing.B) { + for i := 0; i < b.N; i++ { + e = nzstr + } + }) + b.Run("slice", func(b *testing.B) { + for i := 0; i < b.N; i++ { + e = nzslice + } + }) + b.Run("big", func(b *testing.B) { + for i := 0; i < b.N; i++ { + e = nzbig + } + }) + }) +} diff --git a/libgo/go/runtime/internal/sys/intrinsics.go b/libgo/go/runtime/internal/sys/intrinsics.go index 43acf34b801..29282800fdc 100644 --- a/libgo/go/runtime/internal/sys/intrinsics.go +++ b/libgo/go/runtime/internal/sys/intrinsics.go @@ -14,22 +14,22 @@ func builtinCtz64(uint64) int32 // Ctz64 counts trailing (low-order) zeroes, // and if all are zero, then 64. -func Ctz64(x uint64) uint64 { +func Ctz64(x uint64) int { if x == 0 { return 64 } - return uint64(builtinCtz64(x)) + return int(builtinCtz64(x)) } //go:nosplit // Ctz32 counts trailing (low-order) zeroes, // and if all are zero, then 32. -func Ctz32(x uint32) uint32 { +func Ctz32(x uint32) int { if x == 0 { return 32 } - return uint32(builtinCtz32(x)) + return int(builtinCtz32(x)) } //extern __builtin_bswap64 diff --git a/libgo/go/runtime/internal/sys/intrinsics_test.go b/libgo/go/runtime/internal/sys/intrinsics_test.go index 1f2c8daa962..0444183e9dc 100644 --- a/libgo/go/runtime/internal/sys/intrinsics_test.go +++ b/libgo/go/runtime/internal/sys/intrinsics_test.go @@ -6,17 +6,17 @@ import ( ) func TestCtz64(t *testing.T) { - for i := uint(0); i <= 64; i++ { - x := uint64(5) << i - if got := sys.Ctz64(x); got != uint64(i) { + for i := 0; i <= 64; i++ { + x := uint64(5) << uint(i) + if got := sys.Ctz64(x); got != i { t.Errorf("Ctz64(%d)=%d, want %d", x, got, i) } } } func TestCtz32(t *testing.T) { - for i := uint(0); i <= 32; i++ { - x := uint32(5) << i - if got := sys.Ctz32(x); got != uint32(i) { + for i := 0; i <= 32; i++ { + x := uint32(5) << uint(i) + if got := sys.Ctz32(x); got != i { t.Errorf("Ctz32(%d)=%d, want %d", x, got, i) } } diff --git a/libgo/go/runtime/lfstack.go b/libgo/go/runtime/lfstack.go index 2f2958c8869..4787c5be3fa 100644 --- a/libgo/go/runtime/lfstack.go +++ b/libgo/go/runtime/lfstack.go @@ -3,10 +3,6 @@ // license that can be found in the LICENSE file. // Lock-free stack. -// Initialize head to 0, compare with 0 to test for emptiness. -// The stack does not keep pointers to nodes, -// so they can be garbage collected if there are no other pointers to nodes. -// The following code runs only in non-preemptible contexts. package runtime @@ -15,36 +11,47 @@ import ( "unsafe" ) -// Temporary for C code to call: -//go:linkname lfstackpush runtime.lfstackpush -//go:linkname lfstackpop runtime.lfstackpop +// lfstack is the head of a lock-free stack. +// +// The zero value of lfstack is an empty list. +// +// This stack is intrusive. Nodes must embed lfnode as the first field. +// +// The stack does not keep GC-visible pointers to nodes, so the caller +// is responsible for ensuring the nodes are not garbage collected +// (typically by allocating them from manually-managed memory). +type lfstack uint64 -func lfstackpush(head *uint64, node *lfnode) { +func (head *lfstack) push(node *lfnode) { node.pushcnt++ new := lfstackPack(node, node.pushcnt) if node1 := lfstackUnpack(new); node1 != node { - print("runtime: lfstackpush invalid packing: node=", node, " cnt=", hex(node.pushcnt), " packed=", hex(new), " -> node=", node1, "\n") - throw("lfstackpush") + print("runtime: lfstack.push invalid packing: node=", node, " cnt=", hex(node.pushcnt), " packed=", hex(new), " -> node=", node1, "\n") + throw("lfstack.push") } for { - old := atomic.Load64(head) + old := atomic.Load64((*uint64)(head)) node.next = old - if atomic.Cas64(head, old, new) { + if atomic.Cas64((*uint64)(head), old, new) { break } } } -func lfstackpop(head *uint64) unsafe.Pointer { +func (head *lfstack) pop() unsafe.Pointer { for { - old := atomic.Load64(head) + old := atomic.Load64((*uint64)(head)) if old == 0 { return nil } node := lfstackUnpack(old) next := atomic.Load64(&node.next) - if atomic.Cas64(head, old, next) { + if atomic.Cas64((*uint64)(head), old, next) { return unsafe.Pointer(node) } } } + +func (head *lfstack) empty() bool { + return atomic.Load64((*uint64)(head)) == 0 +} diff --git a/libgo/go/runtime/lock_futex.go b/libgo/go/runtime/lock_futex.go index 9877bc35606..7ddd3786624 100644 --- a/libgo/go/runtime/lock_futex.go +++ b/libgo/go/runtime/lock_futex.go @@ -50,6 +50,7 @@ const ( // affect mutex's state. // We use the uintptr mutex.key and note.key as a uint32. +//go:nosplit func key32(p *uintptr) *uint32 { return (*uint32)(unsafe.Pointer(p)) } @@ -152,9 +153,17 @@ func notesleep(n *note) { if gp != gp.m.g0 { throw("notesleep not on g0") } + ns := int64(-1) + if *cgo_yield != nil { + // Sleep for an arbitrary-but-moderate interval to poll libc interceptors. + ns = 10e6 + } for atomic.Load(key32(&n.key)) == 0 { gp.m.blocked = true - futexsleep(key32(&n.key), 0, -1) + futexsleep(key32(&n.key), 0, ns) + if *cgo_yield != nil { + asmcgocall(*cgo_yield, nil) + } gp.m.blocked = false } } @@ -168,9 +177,16 @@ func notetsleep_internal(n *note, ns int64) bool { gp := getg() if ns < 0 { + if *cgo_yield != nil { + // Sleep for an arbitrary-but-moderate interval to poll libc interceptors. + ns = 10e6 + } for atomic.Load(key32(&n.key)) == 0 { gp.m.blocked = true - futexsleep(key32(&n.key), 0, -1) + futexsleep(key32(&n.key), 0, ns) + if *cgo_yield != nil { + asmcgocall(*cgo_yield, nil) + } gp.m.blocked = false } return true @@ -182,8 +198,14 @@ func notetsleep_internal(n *note, ns int64) bool { deadline := nanotime() + ns for { + if *cgo_yield != nil && ns > 10e6 { + ns = 10e6 + } gp.m.blocked = true futexsleep(key32(&n.key), 0, ns) + if *cgo_yield != nil { + asmcgocall(*cgo_yield, nil) + } gp.m.blocked = false if atomic.Load(key32(&n.key)) != 0 { break diff --git a/libgo/go/runtime/lock_sema.go b/libgo/go/runtime/lock_sema.go index 57fee1985e3..52a2376dc5e 100644 --- a/libgo/go/runtime/lock_sema.go +++ b/libgo/go/runtime/lock_sema.go @@ -175,7 +175,16 @@ func notesleep(n *note) { } // Queued. Sleep. gp.m.blocked = true - semasleep(-1) + if *cgo_yield == nil { + semasleep(-1) + } else { + // Sleep for an arbitrary-but-moderate interval to poll libc interceptors. + const ns = 10e6 + for atomic.Loaduintptr(&n.key) == 0 { + semasleep(ns) + asmcgocall(*cgo_yield, nil) + } + } gp.m.blocked = false } @@ -198,7 +207,15 @@ func notetsleep_internal(n *note, ns int64, gp *g, deadline int64) bool { if ns < 0 { // Queued. Sleep. gp.m.blocked = true - semasleep(-1) + if *cgo_yield == nil { + semasleep(-1) + } else { + // Sleep in arbitrary-but-moderate intervals to poll libc interceptors. + const ns = 10e6 + for semasleep(ns) < 0 { + asmcgocall(*cgo_yield, nil) + } + } gp.m.blocked = false return true } @@ -207,12 +224,18 @@ func notetsleep_internal(n *note, ns int64, gp *g, deadline int64) bool { for { // Registered. Sleep. gp.m.blocked = true + if *cgo_yield != nil && ns > 10e6 { + ns = 10e6 + } if semasleep(ns) >= 0 { gp.m.blocked = false // Acquired semaphore, semawakeup unregistered us. // Done. return true } + if *cgo_yield != nil { + asmcgocall(*cgo_yield, nil) + } gp.m.blocked = false // Interrupted or timed out. Still registered. Semaphore not acquired. ns = deadline - nanotime() diff --git a/libgo/go/runtime/malloc.go b/libgo/go/runtime/malloc.go index 3912fc2da58..796cd8a7c64 100644 --- a/libgo/go/runtime/malloc.go +++ b/libgo/go/runtime/malloc.go @@ -122,7 +122,7 @@ const ( // Tiny allocator parameters, see "Tiny allocator" comment in malloc.go. _TinySize = 16 - _TinySizeClass = 2 + _TinySizeClass = int8(2) _FixAllocChunk = 16 << 10 // Chunk size for FixAlloc _MaxMHeapList = 1 << (20 - _PageShift) // Maximum page length for fixed-size list in MHeap. @@ -159,7 +159,11 @@ const ( _MHeapMap_TotalBits = (_64bit*sys.GoosWindows)*35 + (_64bit*(1-sys.GoosWindows)*(1-sys.GoosDarwin*sys.GoarchArm64))*39 + sys.GoosDarwin*sys.GoarchArm64*31 + (1-_64bit)*(32-(sys.GoarchMips+sys.GoarchMipsle)) _MHeapMap_Bits = _MHeapMap_TotalBits - _PageShift - _MaxMem = uintptr(1<<_MHeapMap_TotalBits - 1) + // _MaxMem is the maximum heap arena size minus 1. + // + // On 32-bit, this is also the maximum heap pointer value, + // since the arena starts at address 0. + _MaxMem = 1<<_MHeapMap_TotalBits - 1 // Max number of threads to run garbage collection. // 2, 3, and 4 are all plausible maximums depending @@ -167,8 +171,6 @@ const ( // collector scales well to 32 cpus. _MaxGcproc = 32 - _MaxArena32 = 1<<32 - 1 - // minLegalPointer is the smallest possible legal pointer. // This is the smallest possible architectural page size, // since we assume that the first page is never mapped. @@ -250,18 +252,21 @@ func mallocinit() { throw("bad system page size") } - var p, bitmapSize, spansSize, pSize, limit uintptr + // The auxiliary regions start at p and are laid out in the + // following order: spans, bitmap, arena. + var p, pSize uintptr var reserved bool - // limit = runtime.memlimit(); - // See https://golang.org/issue/5049 - // TODO(rsc): Fix after 1.1. - limit = 0 + // The spans array holds one *mspan per _PageSize of arena. + var spansSize uintptr = (_MaxMem + 1) / _PageSize * sys.PtrSize + spansSize = round(spansSize, _PageSize) + // The bitmap holds 2 bits per word of arena. + var bitmapSize uintptr = (_MaxMem + 1) / (sys.PtrSize * 8 / 2) + bitmapSize = round(bitmapSize, _PageSize) // Set up the allocation arena, a contiguous area of memory where - // allocated data will be found. The arena begins with a bitmap large - // enough to hold 2 bits per allocated word. - if sys.PtrSize == 8 && (limit == 0 || limit > 1<<30) { + // allocated data will be found. + if sys.PtrSize == 8 { // On a 64-bit machine, allocate from a single contiguous reservation. // 512 GB (MaxMem) should be big enough for now. // @@ -294,9 +299,7 @@ func mallocinit() { // On AIX, mmap adresses range start at 0x07000000_00000000 for 64 bits // processes. arenaSize := round(_MaxMem, _PageSize) - bitmapSize = arenaSize / (sys.PtrSize * 8 / 2) - spansSize = arenaSize / _PageSize * sys.PtrSize - spansSize = round(spansSize, _PageSize) + pSize = bitmapSize + spansSize + arenaSize + _PageSize for i := 0; i <= 0x7f; i++ { switch { case GOARCH == "arm64" && GOOS == "darwin": @@ -309,7 +312,6 @@ func mallocinit() { default: p = uintptr(i)<<40 | uintptrMask&(0x00c0<<32) } - pSize = bitmapSize + spansSize + arenaSize + _PageSize p = uintptr(sysReserve(unsafe.Pointer(p), pSize, &reserved)) if p != 0 || GOOS == "aix" { // Useless to loop on AIX, as i is forced to 1 break @@ -327,6 +329,15 @@ func mallocinit() { // When that gets used up, we'll start asking the kernel // for any memory anywhere. + // We want to start the arena low, but if we're linked + // against C code, it's possible global constructors + // have called malloc and adjusted the process' brk. + // Query the brk so we can avoid trying to map the + // arena over it (which will cause the kernel to put + // the arena somewhere else, likely at a high + // address). + procBrk := sbrk0() + // If we fail to allocate, try again with a smaller arena. // This is necessary on Android L where we share a process // with ART, which reserves virtual memory aggressively. @@ -340,15 +351,6 @@ func mallocinit() { } for _, arenaSize := range &arenaSizes { - bitmapSize = (_MaxArena32 + 1) / (sys.PtrSize * 8 / 2) - spansSize = (_MaxArena32 + 1) / _PageSize * sys.PtrSize - if limit > 0 && arenaSize+bitmapSize+spansSize > limit { - bitmapSize = (limit / 9) &^ ((1 << _PageShift) - 1) - arenaSize = bitmapSize * 8 - spansSize = arenaSize / _PageSize * sys.PtrSize - } - spansSize = round(spansSize, _PageSize) - // SysReserve treats the address we ask for, end, as a hint, // not as an absolute requirement. If we ask for the end // of the data segment but the operating system requires @@ -360,6 +362,12 @@ func mallocinit() { // to a MB boundary. p = round(getEnd()+(1<<18), 1<<20) pSize = bitmapSize + spansSize + arenaSize + _PageSize + if p <= procBrk && procBrk < p+pSize { + // Move the start above the brk, + // leaving some room for future brk + // expansion. + p = round(procBrk+(1<<20), 1<<20) + } p = uintptr(sysReserve(unsafe.Pointer(p), pSize, &reserved)) if p != 0 { break @@ -374,18 +382,22 @@ func mallocinit() { // so SysReserve can give us a PageSize-unaligned pointer. // To overcome this we ask for PageSize more and round up the pointer. p1 := round(p, _PageSize) + pSize -= p1 - p spansStart := p1 - mheap_.bitmap = p1 + spansSize + bitmapSize + p1 += spansSize + mheap_.bitmap = p1 + bitmapSize + p1 += bitmapSize if sys.PtrSize == 4 { // Set arena_start such that we can accept memory // reservations located anywhere in the 4GB virtual space. mheap_.arena_start = 0 } else { - mheap_.arena_start = p1 + (spansSize + bitmapSize) + mheap_.arena_start = p1 } mheap_.arena_end = p + pSize - mheap_.arena_used = p1 + (spansSize + bitmapSize) + mheap_.arena_used = p1 + mheap_.arena_alloc = p1 mheap_.arena_reserved = reserved if mheap_.arena_start&(_PageSize-1) != 0 { @@ -404,62 +416,78 @@ func mallocinit() { // h.arena_start and h.arena_end. sysAlloc returns nil on failure. // There is no corresponding free function. func (h *mheap) sysAlloc(n uintptr) unsafe.Pointer { - if n > h.arena_end-h.arena_used { - // We are in 32-bit mode, maybe we didn't use all possible address space yet. - // Reserve some more space. + // strandLimit is the maximum number of bytes to strand from + // the current arena block. If we would need to strand more + // than this, we fall back to sysAlloc'ing just enough for + // this allocation. + const strandLimit = 16 << 20 + + if n > h.arena_end-h.arena_alloc { + // If we haven't grown the arena to _MaxMem yet, try + // to reserve some more address space. p_size := round(n+_PageSize, 256<<20) new_end := h.arena_end + p_size // Careful: can overflow - if h.arena_end <= new_end && new_end-h.arena_start-1 <= _MaxArena32 { + if h.arena_end <= new_end && new_end-h.arena_start-1 <= _MaxMem { // TODO: It would be bad if part of the arena // is reserved and part is not. var reserved bool p := uintptr(sysReserve(unsafe.Pointer(h.arena_end), p_size, &reserved)) if p == 0 { - return nil + // TODO: Try smaller reservation + // growths in case we're in a crowded + // 32-bit address space. + goto reservationFailed } // p can be just about anywhere in the address // space, including before arena_end. if p == h.arena_end { + // The new block is contiguous with + // the current block. Extend the + // current arena block. h.arena_end = new_end h.arena_reserved = reserved - } else if h.arena_end < p && p+p_size-h.arena_start-1 <= _MaxArena32 { + } else if h.arena_start <= p && p+p_size-h.arena_start-1 <= _MaxMem && h.arena_end-h.arena_alloc < strandLimit { + // We were able to reserve more memory + // within the arena space, but it's + // not contiguous with our previous + // reservation. It could be before or + // after our current arena_used. + // // Keep everything page-aligned. // Our pages are bigger than hardware pages. h.arena_end = p + p_size - used := p + (-p & (_PageSize - 1)) - h.mapBits(used) - h.mapSpans(used) - h.arena_used = used + p = round(p, _PageSize) + h.arena_alloc = p h.arena_reserved = reserved } else { - // We got a mapping, but it's not - // linear with our current arena, so - // we can't use it. + // We got a mapping, but either + // + // 1) It's not in the arena, so we + // can't use it. (This should never + // happen on 32-bit.) + // + // 2) We would need to discard too + // much of our current arena block to + // use it. // - // TODO: Make it possible to allocate - // from this. We can't decrease - // arena_used, but we could introduce - // a new variable for the current - // allocation position. - // We haven't added this allocation to // the stats, so subtract it from a // fake stat (but avoid underflow). + // + // We'll fall back to a small sysAlloc. stat := uint64(p_size) sysFree(unsafe.Pointer(p), p_size, &stat) } } } - if n <= h.arena_end-h.arena_used { + if n <= h.arena_end-h.arena_alloc { // Keep taking from our reservation. - p := h.arena_used + p := h.arena_alloc sysMap(unsafe.Pointer(p), n, h.arena_reserved, &memstats.heap_sys) - h.mapBits(p + n) - h.mapSpans(p + n) - h.arena_used = p + n - if raceenabled { - racemapshadow(unsafe.Pointer(p), n) + h.arena_alloc += n + if h.arena_alloc > h.arena_used { + h.setArenaUsed(h.arena_alloc, true) } if p&(_PageSize-1) != 0 { @@ -468,8 +496,9 @@ func (h *mheap) sysAlloc(n uintptr) unsafe.Pointer { return unsafe.Pointer(p) } +reservationFailed: // If using 64-bit, our reservation is all we have. - if h.arena_end-h.arena_start > _MaxArena32 { + if sys.PtrSize != 4 { return nil } @@ -481,28 +510,18 @@ func (h *mheap) sysAlloc(n uintptr) unsafe.Pointer { return nil } - if p < h.arena_start || p+p_size-h.arena_start > _MaxArena32 { - top := ^uintptr(0) - if top-h.arena_start-1 > _MaxArena32 { - top = h.arena_start + _MaxArena32 + 1 - } + if p < h.arena_start || p+p_size-h.arena_start > _MaxMem { + // This shouldn't be possible because _MaxMem is the + // whole address space on 32-bit. + top := uint64(h.arena_start) + _MaxMem print("runtime: memory allocated by OS (", hex(p), ") not in usable range [", hex(h.arena_start), ",", hex(top), ")\n") sysFree(unsafe.Pointer(p), p_size, &memstats.heap_sys) return nil } - p_end := p + p_size p += -p & (_PageSize - 1) if p+n > h.arena_used { - h.mapBits(p + n) - h.mapSpans(p + n) - h.arena_used = p + n - if p_end > h.arena_end { - h.arena_end = p_end - } - if raceenabled { - racemapshadow(unsafe.Pointer(p), n) - } + h.setArenaUsed(p+n, true) } if p&(_PageSize-1) != 0 { @@ -525,7 +544,7 @@ func nextFreeFast(s *mspan) gclinkptr { if freeidx%64 == 0 && freeidx != s.nelems { return 0 } - s.allocCache >>= (theBit + 1) + s.allocCache >>= uint(theBit + 1) s.freeindex = freeidx v := gclinkptr(result*s.elemsize + s.base()) s.allocCount++ @@ -541,8 +560,8 @@ func nextFreeFast(s *mspan) gclinkptr { // weight allocation. If it is a heavy weight allocation the caller must // determine whether a new GC cycle needs to be started or if the GC is active // whether this goroutine needs to assist the GC. -func (c *mcache) nextFree(sizeclass uint8) (v gclinkptr, s *mspan, shouldhelpgc bool) { - s = c.alloc[sizeclass] +func (c *mcache) nextFree(spc spanClass) (v gclinkptr, s *mspan, shouldhelpgc bool) { + s = c.alloc[spc] shouldhelpgc = false freeIndex := s.nextFreeIndex() if freeIndex == s.nelems { @@ -552,10 +571,10 @@ func (c *mcache) nextFree(sizeclass uint8) (v gclinkptr, s *mspan, shouldhelpgc throw("s.allocCount != s.nelems && freeIndex == s.nelems") } systemstack(func() { - c.refill(int32(sizeclass)) + c.refill(spc) }) shouldhelpgc = true - s = c.alloc[sizeclass] + s = c.alloc[spc] freeIndex = s.nextFreeIndex() } @@ -693,10 +712,10 @@ func mallocgc(size uintptr, typ *_type, needzero bool) unsafe.Pointer { return x } // Allocate a new maxTinySize block. - span := c.alloc[tinySizeClass] + span := c.alloc[tinySpanClass] v := nextFreeFast(span) if v == 0 { - v, _, shouldhelpgc = c.nextFree(tinySizeClass) + v, _, shouldhelpgc = c.nextFree(tinySpanClass) } x = unsafe.Pointer(v) (*[2]uint64)(x)[0] = 0 @@ -716,10 +735,11 @@ func mallocgc(size uintptr, typ *_type, needzero bool) unsafe.Pointer { sizeclass = size_to_class128[(size-smallSizeMax+largeSizeDiv-1)/largeSizeDiv] } size = uintptr(class_to_size[sizeclass]) - span := c.alloc[sizeclass] + spc := makeSpanClass(sizeclass, noscan) + span := c.alloc[spc] v := nextFreeFast(span) if v == 0 { - v, span, shouldhelpgc = c.nextFree(sizeclass) + v, span, shouldhelpgc = c.nextFree(spc) } x = unsafe.Pointer(v) if needzero && span.needzero != 0 { @@ -730,7 +750,7 @@ func mallocgc(size uintptr, typ *_type, needzero bool) unsafe.Pointer { var s *mspan shouldhelpgc = true systemstack(func() { - s = largeAlloc(size, needzero) + s = largeAlloc(size, needzero, noscan) }) s.freeindex = 1 s.allocCount = 1 @@ -739,9 +759,7 @@ func mallocgc(size uintptr, typ *_type, needzero bool) unsafe.Pointer { } var scanSize uintptr - if noscan { - heapBitsSetTypeNoScan(uintptr(x)) - } else { + if !noscan { heapBitsSetType(uintptr(x), size, dataSize, typ) if dataSize > typ.size { // Array allocation. If there are any @@ -803,8 +821,10 @@ func mallocgc(size uintptr, typ *_type, needzero bool) unsafe.Pointer { assistG.gcAssistBytes -= int64(size - dataSize) } - if shouldhelpgc && gcShouldStart(false) { - gcStart(gcBackgroundMode, false) + if shouldhelpgc { + if t := (gcTrigger{kind: gcTriggerHeap}); t.test() { + gcStart(gcBackgroundMode, t) + } } if getg().preempt { @@ -818,7 +838,7 @@ func mallocgc(size uintptr, typ *_type, needzero bool) unsafe.Pointer { return x } -func largeAlloc(size uintptr, needzero bool) *mspan { +func largeAlloc(size uintptr, needzero bool, noscan bool) *mspan { // print("largeAlloc size=", size, "\n") if size+_PageSize < size { @@ -834,7 +854,7 @@ func largeAlloc(size uintptr, needzero bool) *mspan { // pays the debt down to npage pages. deductSweepCredit(npages*_PageSize, npages) - s := mheap_.alloc(npages, 0, true, needzero) + s := mheap_.alloc(npages, makeSpanClass(0, noscan), true, needzero) if s == nil { throw("out of memory") } @@ -924,7 +944,7 @@ func nextSampleNoFP() int32 { rate = 0x3fffffff } if rate != 0 { - return int32(int(fastrand()) % (2 * rate)) + return int32(fastrand() % uint32(2*rate)) } return 0 } diff --git a/libgo/go/runtime/malloc_test.go b/libgo/go/runtime/malloc_test.go index bc5530c2f23..0d43cf65976 100644 --- a/libgo/go/runtime/malloc_test.go +++ b/libgo/go/runtime/malloc_test.go @@ -6,6 +6,8 @@ package runtime_test import ( "flag" + "fmt" + "reflect" . "runtime" "testing" "time" @@ -22,24 +24,62 @@ func TestMemStats(t *testing.T) { st := new(MemStats) ReadMemStats(st) - // Everything except HeapReleased, HeapIdle, and NumGC, - // because they indeed can be 0. - if st.Alloc == 0 || st.TotalAlloc == 0 || st.Sys == 0 || st.Lookups == 0 || - st.Mallocs == 0 || st.Frees == 0 || st.HeapAlloc == 0 || st.HeapSys == 0 || - st.HeapInuse == 0 || st.HeapObjects == 0 || st.StackInuse == 0 || - st.StackSys == 0 || st.MSpanInuse == 0 || st.MSpanSys == 0 || st.MCacheInuse == 0 || - st.MCacheSys == 0 || st.BuckHashSys == 0 || st.GCSys == 0 || st.OtherSys == 0 || - st.NextGC == 0 || st.NumForcedGC == 0 { - t.Fatalf("Zero value: %+v", *st) - } - - if st.Alloc > 1e10 || st.TotalAlloc > 1e11 || st.Sys > 1e10 || st.Lookups > 1e10 || - st.Mallocs > 1e10 || st.Frees > 1e10 || st.HeapAlloc > 1e10 || st.HeapSys > 1e10 || - st.HeapIdle > 1e10 || st.HeapInuse > 1e10 || st.HeapObjects > 1e10 || st.StackInuse > 1e10 || - st.StackSys > 1e10 || st.MSpanInuse > 1e10 || st.MSpanSys > 1e10 || st.MCacheInuse > 1e10 || - st.MCacheSys > 1e10 || st.BuckHashSys > 1e10 || st.GCSys > 1e10 || st.OtherSys > 1e10 || - st.NextGC > 1e10 || st.NumGC > 1e9 || st.NumForcedGC > 1e9 || st.PauseTotalNs > 1e11 { - t.Fatalf("Insanely high value (overflow?): %+v", *st) + nz := func(x interface{}) error { + if x != reflect.Zero(reflect.TypeOf(x)).Interface() { + return nil + } + return fmt.Errorf("zero value") + } + le := func(thresh float64) func(interface{}) error { + return func(x interface{}) error { + if reflect.ValueOf(x).Convert(reflect.TypeOf(thresh)).Float() < thresh { + return nil + } + return fmt.Errorf("insanely high value (overflow?); want <= %v", thresh) + } + } + eq := func(x interface{}) func(interface{}) error { + return func(y interface{}) error { + if x == y { + return nil + } + return fmt.Errorf("want %v", x) + } + } + // Of the uint fields, HeapReleased, HeapIdle can be 0. + // PauseTotalNs can be 0 if timer resolution is poor. + // + // TODO: Test that GCCPUFraction is <= 0.99. This currently + // fails on windows/386. (Issue #19319) + fields := map[string][]func(interface{}) error{ + "Alloc": {nz, le(1e10)}, "TotalAlloc": {nz, le(1e11)}, "Sys": {nz, le(1e10)}, + "Lookups": {nz, le(1e10)}, "Mallocs": {nz, le(1e10)}, "Frees": {nz, le(1e10)}, + "HeapAlloc": {nz, le(1e10)}, "HeapSys": {nz, le(1e10)}, "HeapIdle": {le(1e10)}, + "HeapInuse": {nz, le(1e10)}, "HeapReleased": {le(1e10)}, "HeapObjects": {nz, le(1e10)}, + "StackInuse": {nz, le(1e10)}, "StackSys": {nz, le(1e10)}, + "MSpanInuse": {nz, le(1e10)}, "MSpanSys": {nz, le(1e10)}, + "MCacheInuse": {nz, le(1e10)}, "MCacheSys": {nz, le(1e10)}, + "BuckHashSys": {nz, le(1e10)}, "GCSys": {nz, le(1e10)}, "OtherSys": {nz, le(1e10)}, + "NextGC": {nz, le(1e10)}, "LastGC": {nz}, + "PauseTotalNs": {le(1e11)}, "PauseNs": nil, "PauseEnd": nil, + "NumGC": {nz, le(1e9)}, "NumForcedGC": {nz, le(1e9)}, + "GCCPUFraction": nil, "EnableGC": {eq(true)}, "DebugGC": {eq(false)}, + "BySize": nil, + } + + rst := reflect.ValueOf(st).Elem() + for i := 0; i < rst.Type().NumField(); i++ { + name, val := rst.Type().Field(i).Name, rst.Field(i).Interface() + checks, ok := fields[name] + if !ok { + t.Errorf("unknown MemStats field %s", name) + continue + } + for _, check := range checks { + if err := check(val); err != nil { + t.Errorf("%s = %v: %s", name, val, err) + } + } } if st.Sys != st.HeapSys+st.StackSys+st.MSpanSys+st.MCacheSys+ st.BuckHashSys+st.GCSys+st.OtherSys { diff --git a/libgo/go/runtime/map_test.go b/libgo/go/runtime/map_test.go index 9b5b051250e..37c959f8327 100644 --- a/libgo/go/runtime/map_test.go +++ b/libgo/go/runtime/map_test.go @@ -10,6 +10,7 @@ import ( "reflect" "runtime" "sort" + "strconv" "strings" "sync" "testing" @@ -594,6 +595,14 @@ func TestMapLargeValNoPointer(t *testing.T) { } } +// Test that making a map with a large or invalid hint +// doesn't panic. (Issue 19926). +func TestIgnoreBogusMapHint(t *testing.T) { + for _, hint := range []int64{-1, 1 << 62} { + _ = make(map[int]int, hint) + } +} + func benchmarkMapPop(b *testing.B, n int) { m := map[int]int{} for i := 0; i < b.N; i++ { @@ -625,3 +634,86 @@ func TestNonEscapingMap(t *testing.T) { t.Fatalf("want 0 allocs, got %v", n) } } + +func benchmarkMapAssignInt32(b *testing.B, n int) { + a := make(map[int32]int) + for i := 0; i < b.N; i++ { + a[int32(i&(n-1))] = i + } +} + +func benchmarkMapDeleteInt32(b *testing.B, n int) { + a := make(map[int32]int) + for i := 0; i < n*b.N; i++ { + a[int32(i)] = i + } + b.ResetTimer() + for i := 0; i < n*b.N; i = i + n { + delete(a, int32(i)) + } +} + +func benchmarkMapAssignInt64(b *testing.B, n int) { + a := make(map[int64]int) + for i := 0; i < b.N; i++ { + a[int64(i&(n-1))] = i + } +} + +func benchmarkMapDeleteInt64(b *testing.B, n int) { + a := make(map[int64]int) + for i := 0; i < n*b.N; i++ { + a[int64(i)] = i + } + b.ResetTimer() + for i := 0; i < n*b.N; i = i + n { + delete(a, int64(i)) + } +} + +func benchmarkMapAssignStr(b *testing.B, n int) { + k := make([]string, n) + for i := 0; i < len(k); i++ { + k[i] = strconv.Itoa(i) + } + b.ResetTimer() + a := make(map[string]int) + for i := 0; i < b.N; i++ { + a[k[i&(n-1)]] = i + } +} + +func benchmarkMapDeleteStr(b *testing.B, n int) { + k := make([]string, n*b.N) + for i := 0; i < n*b.N; i++ { + k[i] = strconv.Itoa(i) + } + a := make(map[string]int) + for i := 0; i < n*b.N; i++ { + a[k[i]] = i + } + b.ResetTimer() + for i := 0; i < n*b.N; i = i + n { + delete(a, k[i]) + } +} + +func runWith(f func(*testing.B, int), v ...int) func(*testing.B) { + return func(b *testing.B) { + for _, n := range v { + b.Run(strconv.Itoa(n), func(b *testing.B) { f(b, n) }) + } + } +} + +func BenchmarkMapAssign(b *testing.B) { + b.Run("Int32", runWith(benchmarkMapAssignInt32, 1<<8, 1<<16)) + b.Run("Int64", runWith(benchmarkMapAssignInt64, 1<<8, 1<<16)) + b.Run("Str", runWith(benchmarkMapAssignStr, 1<<8, 1<<16)) +} + +func BenchmarkMapDelete(b *testing.B) { + b.Run("Int32", runWith(benchmarkMapDeleteInt32, 1, 2, 4)) + b.Run("Int64", runWith(benchmarkMapDeleteInt64, 1, 2, 4)) + b.Run("Str", runWith(benchmarkMapDeleteStr, 1, 2, 4)) +} diff --git a/libgo/go/runtime/mapspeed_test.go b/libgo/go/runtime/mapspeed_test.go index ac93119d77d..aec0c51f3fd 100644 --- a/libgo/go/runtime/mapspeed_test.go +++ b/libgo/go/runtime/mapspeed_test.go @@ -5,6 +5,7 @@ package runtime_test import ( "fmt" + "strconv" "strings" "testing" ) @@ -308,6 +309,20 @@ func BenchmarkSmallKeyMap(b *testing.B) { } } +func BenchmarkMapPopulate(b *testing.B) { + for size := 1; size < 1000000; size *= 10 { + b.Run(strconv.Itoa(size), func(b *testing.B) { + b.ReportAllocs() + for i := 0; i < b.N; i++ { + m := make(map[int]bool) + for j := 0; j < size; j++ { + m[j] = true + } + } + }) + } +} + type ComplexAlgKey struct { a, b, c int64 _ int diff --git a/libgo/go/runtime/mbarrier.go b/libgo/go/runtime/mbarrier.go index 3a463c8b1ba..d54016f0ba9 100644 --- a/libgo/go/runtime/mbarrier.go +++ b/libgo/go/runtime/mbarrier.go @@ -156,6 +156,11 @@ func gcmarkwb_m(slot *uintptr, ptr uintptr) { // combine the read and the write. Checking inheap is // insufficient since we need to track changes to // roots outside the heap. + // + // Note: profbuf.go omits a barrier during signal handler + // profile logging; that's safe only because this deletion barrier exists. + // If we remove the deletion barrier, we'll have to work out + // a new way to handle the profile logging. if slot1 := uintptr(unsafe.Pointer(slot)); slot1 >= minPhysPageSize { if optr := *slot; optr != 0 { shade(optr) @@ -238,6 +243,7 @@ func writebarrierptr_prewrite(dst *uintptr, src uintptr) { } // typedmemmove copies a value of type t to dst from src. +// Must be nosplit, see #16026. //go:nosplit func typedmemmove(typ *_type, dst, src unsafe.Pointer) { if typ.kind&kindNoPointers == 0 { diff --git a/libgo/go/runtime/mbitmap.go b/libgo/go/runtime/mbitmap.go index a7ccc650ada..d1a58202352 100644 --- a/libgo/go/runtime/mbitmap.go +++ b/libgo/go/runtime/mbitmap.go @@ -45,6 +45,11 @@ // not checkmarked, and is the dead encoding. // These properties must be preserved when modifying the encoding. // +// The bitmap for noscan spans is not maintained. Code must ensure +// that an object is scannable before consulting its bitmap by +// checking either the noscan bit in the span or by consulting its +// type's information. +// // Checkmarks // // In a concurrent garbage collector, one worries about failing to mark @@ -134,13 +139,9 @@ func subtract1(p *byte) *byte { return (*byte)(unsafe.Pointer(uintptr(unsafe.Pointer(p)) - 1)) } -// mHeap_MapBits is called each time arena_used is extended. -// It maps any additional bitmap memory needed for the new arena memory. -// It must be called with the expected new value of arena_used, -// *before* h.arena_used has been updated. -// Waiting to update arena_used until after the memory has been mapped -// avoids faults when other threads try access the bitmap immediately -// after observing the change to arena_used. +// mapBits maps any additional bitmap memory needed for the new arena memory. +// +// Don't call this directly. Call mheap.setArenaUsed. // //go:nowritebarrier func (h *mheap) mapBits(arena_used uintptr) { @@ -186,10 +187,8 @@ type markBits struct { //go:nosplit func (s *mspan) allocBitsForIndex(allocBitIndex uintptr) markBits { - whichByte := allocBitIndex / 8 - whichBit := allocBitIndex % 8 - bytePtr := addb(s.allocBits, whichByte) - return markBits{bytePtr, uint8(1 << whichBit), allocBitIndex} + bytep, mask := s.allocBits.bitp(allocBitIndex) + return markBits{bytep, mask, allocBitIndex} } // refillaCache takes 8 bytes s.allocBits starting at whichByte @@ -197,7 +196,7 @@ func (s *mspan) allocBitsForIndex(allocBitIndex uintptr) markBits { // can be used. It then places these 8 bytes into the cached 64 bit // s.allocCache. func (s *mspan) refillAllocCache(whichByte uintptr) { - bytes := (*[8]uint8)(unsafe.Pointer(addb(s.allocBits, whichByte))) + bytes := (*[8]uint8)(unsafe.Pointer(s.allocBits.bytep(whichByte))) aCache := uint64(0) aCache |= uint64(bytes[0]) aCache |= uint64(bytes[1]) << (1 * 8) @@ -248,7 +247,7 @@ func (s *mspan) nextFreeIndex() uintptr { return snelems } - s.allocCache >>= (bitIndex + 1) + s.allocCache >>= uint(bitIndex + 1) sfreeindex = result + 1 if sfreeindex%64 == 0 && sfreeindex != snelems { @@ -269,10 +268,8 @@ func (s *mspan) isFree(index uintptr) bool { if index < s.freeindex { return false } - whichByte := index / 8 - whichBit := index % 8 - byteVal := *addb(s.allocBits, whichByte) - return byteVal&uint8(1<= mheap_.arena_used { @@ -404,7 +394,7 @@ func heapBitsForObject(p, refBase, refOff uintptr, forStack bool) (base uintptr, // Consult the span table to find the block beginning. s = mheap_.spans[idx] if s == nil || p < s.base() || p >= s.limit || s.state != mSpanInUse { - if s == nil || s.state == _MSpanStack || forStack { + if s == nil || s.state == _MSpanManual || forStack { // If s is nil, the virtual address has never been part of the heap. // This pointer may be to some mmap'd region, so we allow it. // Pointers into stacks are also ok, the runtime manages these explicitly. @@ -434,6 +424,7 @@ func heapBitsForObject(p, refBase, refOff uintptr, forStack bool) (base uintptr, print("runtime: found in object at *(", hex(refBase), "+", hex(refOff), ")\n") gcDumpObject("object", refBase, refOff) } + getg().m.traceback = 2 throw("found bad pointer in Go heap (incorrect use of unsafe or cgo?)") } return @@ -524,16 +515,6 @@ func (h heapBits) isPointer() bool { return h.bits()&bitPointer != 0 } -// hasPointers reports whether the given object has any pointers. -// It must be told how large the object at h is for efficiency. -// h must describe the initial word of the object. -func (h heapBits) hasPointers(size uintptr) bool { - if size == sys.PtrSize { // 1-word objects are always pointers - return true - } - return (*h.bitp>>h.shift)&bitScan != 0 -} - // isCheckmarked reports whether the heap bits have the checkmarked bit set. // It must be told how large the object at h is, because the encoding of the // checkmark bit varies by size. @@ -839,23 +820,23 @@ var oneBitCount = [256]uint8{ 4, 5, 5, 6, 5, 6, 6, 7, 5, 6, 6, 7, 6, 7, 7, 8} -// countFree runs through the mark bits in a span and counts the number of free objects -// in the span. +// countAlloc returns the number of objects allocated in span s by +// scanning the allocation bitmap. // TODO:(rlh) Use popcount intrinsic. -func (s *mspan) countFree() int { +func (s *mspan) countAlloc() int { count := 0 maxIndex := s.nelems / 8 for i := uintptr(0); i < maxIndex; i++ { - mrkBits := *addb(s.gcmarkBits, i) + mrkBits := *s.gcmarkBits.bytep(i) count += int(oneBitCount[mrkBits]) } if bitsInLastByte := s.nelems % 8; bitsInLastByte != 0 { - mrkBits := *addb(s.gcmarkBits, maxIndex) + mrkBits := *s.gcmarkBits.bytep(maxIndex) mask := uint8((1 << bitsInLastByte) - 1) bits := mrkBits & mask count += int(oneBitCount[bits]) } - return int(s.nelems) - count + return count } // heapBitsSetType records that the new allocation [x, x+size) @@ -1076,7 +1057,9 @@ func heapBitsSetType(x, size, dataSize uintptr, typ *_type) { endnb += endnb } // Truncate to a multiple of original ptrmask. - endnb = maxBits / nb * nb + // Because nb+nb <= maxBits, nb fits in a byte. + // Byte division is cheaper than uintptr division. + endnb = uintptr(maxBits/byte(nb)) * nb pbits &= 1< 0 { - reimburseSweepCredit(usedBytes) - } atomic.Xadd64(&memstats.heap_live, int64(spanBytes)-int64(usedBytes)) if trace.enabled { // heap_live changed. @@ -150,6 +166,10 @@ func (c *mcentral) uncacheSpan(s *mspan) { // mCentral_CacheSpan conservatively counted // unallocated slots in heap_live. Undo this. atomic.Xadd64(&memstats.heap_live, -int64(n)*int64(s.elemsize)) + // cacheSpan updated alloc assuming all objects on s + // were going to be allocated. Adjust for any that + // weren't. + atomic.Xadd64(&c.nmalloc, -int64(n)) } unlock(&c.lock) } @@ -205,11 +225,11 @@ func (c *mcentral) freeSpan(s *mspan, preserve bool, wasempty bool) bool { // grow allocates a new empty span from the heap and initializes it for c's size class. func (c *mcentral) grow() *mspan { - npages := uintptr(class_to_allocnpages[c.sizeclass]) - size := uintptr(class_to_size[c.sizeclass]) + npages := uintptr(class_to_allocnpages[c.spanclass.sizeclass()]) + size := uintptr(class_to_size[c.spanclass.sizeclass()]) n := (npages << _PageShift) / size - s := mheap_.alloc(npages, c.sizeclass, false, true) + s := mheap_.alloc(npages, c.spanclass, false, true) if s == nil { return nil } diff --git a/libgo/go/runtime/mfinal.go b/libgo/go/runtime/mfinal.go index 229ccb55387..4353ee57569 100644 --- a/libgo/go/runtime/mfinal.go +++ b/libgo/go/runtime/mfinal.go @@ -12,8 +12,12 @@ import ( "unsafe" ) +// finblock is an array of finalizers to be executed. finblocks are +// arranged in a linked list for the finalizer queue. +// // finblock is allocated from non-GC'd memory, so any heap pointers -// must be specially handled. +// must be specially handled. GC currently assumes that the finalizer +// queue does not grow during marking (but it can shrink). // //go:notinheap type finblock struct { @@ -42,6 +46,16 @@ type finalizer struct { } func queuefinalizer(p unsafe.Pointer, fn *funcval, ft *functype, ot *ptrtype) { + if gcphase != _GCoff { + // Currently we assume that the finalizer queue won't + // grow during marking so we don't have to rescan it + // during mark termination. If we ever need to lift + // this assumption, we can do it by adding the + // necessary barriers to queuefinalizer (which it may + // have automatically). + throw("queuefinalizer during GC") + } + lock(&finlock) if finq == nil || finq.cnt == uint32(len(finq.fin)) { if finc == nil { @@ -399,7 +413,7 @@ func findObject(v unsafe.Pointer) (s *mspan, x unsafe.Pointer, n uintptr) { } n = s.elemsize - if s.sizeclass != 0 { + if s.spanclass.sizeclass() != 0 { x = add(x, (uintptr(v)-uintptr(x))/n*n) } return diff --git a/libgo/go/runtime/mfixalloc.go b/libgo/go/runtime/mfixalloc.go index fe4b0fcf2a9..7496671fbec 100644 --- a/libgo/go/runtime/mfixalloc.go +++ b/libgo/go/runtime/mfixalloc.go @@ -29,7 +29,7 @@ type fixalloc struct { first func(arg, p unsafe.Pointer) // called first time p is returned arg unsafe.Pointer list *mlink - chunk unsafe.Pointer + chunk uintptr // use uintptr instead of unsafe.Pointer to avoid write barriers nchunk uint32 inuse uintptr // in-use bytes now stat *uint64 @@ -54,7 +54,7 @@ func (f *fixalloc) init(size uintptr, first func(arg, p unsafe.Pointer), arg uns f.first = first f.arg = arg f.list = nil - f.chunk = nil + f.chunk = 0 f.nchunk = 0 f.inuse = 0 f.stat = stat @@ -77,15 +77,15 @@ func (f *fixalloc) alloc() unsafe.Pointer { return v } if uintptr(f.nchunk) < f.size { - f.chunk = persistentalloc(_FixAllocChunk, 0, f.stat) + f.chunk = uintptr(persistentalloc(_FixAllocChunk, 0, f.stat)) f.nchunk = _FixAllocChunk } - v := f.chunk + v := unsafe.Pointer(f.chunk) if f.first != nil { f.first(f.arg, v) } - f.chunk = add(f.chunk, f.size) + f.chunk = f.chunk + f.size f.nchunk -= uint32(f.size) f.inuse += f.size return v diff --git a/libgo/go/runtime/mgc.go b/libgo/go/runtime/mgc.go index a4fc2be803c..31c4be86fe4 100644 --- a/libgo/go/runtime/mgc.go +++ b/libgo/go/runtime/mgc.go @@ -178,17 +178,21 @@ func gcinit() { throw("size of Workbuf is suboptimal") } + // No sweep on the first cycle. + mheap_.sweepdone = 1 + + // Set a reasonable initial GC trigger. + memstats.triggerRatio = 7 / 8.0 + + // Fake a heap_marked value so it looks like a trigger at + // heapminimum is the appropriate growth from heap_marked. + // This will go into computing the initial GC goal. + memstats.heap_marked = uint64(float64(heapminimum) / (1 + memstats.triggerRatio)) + + // Set gcpercent from the environment. This will also compute + // and set the GC trigger and goal. _ = setGCPercent(readgogc()) - memstats.gc_trigger = heapminimum - // Compute the goal heap size based on the trigger: - // trigger = marked * (1 + triggerRatio) - // marked = trigger / (1 + triggerRatio) - // goal = marked * (1 + GOGC/100) - // = trigger / (1 + triggerRatio) * (1 + GOGC/100) - memstats.next_gc = uint64(float64(memstats.gc_trigger) / (1 + gcController.triggerRatio) * (1 + float64(gcpercent)/100)) - if gcpercent < 0 { - memstats.next_gc = ^uint64(0) - } + work.startSema = 1 work.markDoneSema = 1 } @@ -224,12 +228,8 @@ func setGCPercent(in int32) (out int32) { } gcpercent = in heapminimum = defaultHeapMinimum * uint64(gcpercent) / 100 - if gcController.triggerRatio > float64(gcpercent)/100 { - gcController.triggerRatio = float64(gcpercent) / 100 - } - // This is either in gcinit or followed by a STW GC, both of - // which will reset other stats like memstats.gc_trigger and - // memstats.next_gc to appropriate values. + // Update pacing in response to gcpercent change. + gcSetTriggerRatio(memstats.triggerRatio) unlock(&mheap_.lock) return out } @@ -239,7 +239,9 @@ func setGCPercent(in int32) (out int32) { var gcphase uint32 // The compiler knows about this variable. -// If you change it, you must change the compiler too. +// If you change it, you must change builtin/runtime.go, too. +// If you change the first four bytes, you must also change the write +// barrier insertion code. var writeBarrier struct { enabled bool // compiler emits a check of this before calling write barrier pad [3]byte // compiler uses 32-bit load for "enabled" field @@ -329,10 +331,10 @@ var gcMarkWorkerModeStrings = [...]string{ // utilization between assist and background marking to be 25% of // GOMAXPROCS. The high-level design of this algorithm is documented // at https://golang.org/s/go15gcpacing. -var gcController = gcControllerState{ - // Initial trigger ratio guess. - triggerRatio: 7 / 8.0, -} +// +// All fields of gcController are used only during a single mark +// cycle. +var gcController gcControllerState type gcControllerState struct { // scanWork is the total scan work performed this cycle. This @@ -403,14 +405,6 @@ type gcControllerState struct { // beginning of each cycle. fractionalUtilizationGoal float64 - // triggerRatio is the heap growth ratio at which the garbage - // collection cycle should start. E.g., if this is 0.6, then - // GC should start when the live heap has reached 1.6 times - // the heap size marked by the previous cycle. This should be - // ≤ GOGC/100 so the trigger heap size is less than the goal - // heap size. This is updated at the end of of each cycle. - triggerRatio float64 - _ [sys.CacheLineSize]byte // fractionalMarkWorkersNeeded is the number of fractional @@ -439,7 +433,7 @@ func (c *gcControllerState) startCycle() { // first cycle) or may be much smaller (resulting in a large // error response). if memstats.gc_trigger <= heapminimum { - memstats.heap_marked = uint64(float64(memstats.gc_trigger) / (1 + c.triggerRatio)) + memstats.heap_marked = uint64(float64(memstats.gc_trigger) / (1 + memstats.triggerRatio)) } // Re-compute the heap goal for this cycle in case something @@ -495,17 +489,12 @@ func (c *gcControllerState) startCycle() { // revise updates the assist ratio during the GC cycle to account for // improved estimates. This should be called either under STW or -// whenever memstats.heap_scan or memstats.heap_live is updated (with -// mheap_.lock held). +// whenever memstats.heap_scan, memstats.heap_live, or +// memstats.next_gc is updated (with mheap_.lock held). // // It should only be called when gcBlackenEnabled != 0 (because this // is when assists are enabled and the necessary statistics are // available). -// -// TODO: Consider removing the periodic controller update altogether. -// Since we switched to allocating black, in theory we shouldn't have -// to change the assist ratio. However, this is still a useful hook -// that we've found many uses for when experimenting. func (c *gcControllerState) revise() { // Compute the expected scan work remaining. // @@ -536,7 +525,7 @@ func (c *gcControllerState) revise() { } // Compute the heap distance remaining. - heapDistance := int64(memstats.next_gc) - int64(memstats.heap_live) + heapDistance := int64(memstats.next_gc) - int64(atomic.Load64(&memstats.heap_live)) if heapDistance <= 0 { // This shouldn't happen, but if it does, avoid // dividing by zero or setting the assist negative. @@ -550,10 +539,15 @@ func (c *gcControllerState) revise() { c.assistBytesPerWork = float64(heapDistance) / float64(scanWorkExpected) } -// endCycle updates the GC controller state at the end of the -// concurrent part of the GC cycle. -func (c *gcControllerState) endCycle() { - h_t := c.triggerRatio // For debugging +// endCycle computes the trigger ratio for the next cycle. +func (c *gcControllerState) endCycle() float64 { + if work.userForced { + // Forced GC means this cycle didn't start at the + // trigger, so where it finished isn't good + // information about how to adjust the trigger. + // Just leave it where it is. + return memstats.triggerRatio + } // Proportional response gain for the trigger controller. Must // be in [0, 1]. Lower values smooth out transient effects but @@ -582,25 +576,17 @@ func (c *gcControllerState) endCycle() { utilization += float64(c.assistTime) / float64(assistDuration*int64(gomaxprocs)) } - triggerError := goalGrowthRatio - c.triggerRatio - utilization/gcGoalUtilization*(actualGrowthRatio-c.triggerRatio) + triggerError := goalGrowthRatio - memstats.triggerRatio - utilization/gcGoalUtilization*(actualGrowthRatio-memstats.triggerRatio) // Finally, we adjust the trigger for next time by this error, // damped by the proportional gain. - c.triggerRatio += triggerGain * triggerError - if c.triggerRatio < 0 { - // This can happen if the mutator is allocating very - // quickly or the GC is scanning very slowly. - c.triggerRatio = 0 - } else if c.triggerRatio > goalGrowthRatio*0.95 { - // Ensure there's always a little margin so that the - // mutator assist ratio isn't infinity. - c.triggerRatio = goalGrowthRatio * 0.95 - } + triggerRatio := memstats.triggerRatio + triggerGain*triggerError if debug.gcpacertrace > 0 { // Print controller state in terms of the design // document. H_m_prev := memstats.heap_marked + h_t := memstats.triggerRatio H_T := memstats.gc_trigger h_a := actualGrowthRatio H_a := memstats.heap_live @@ -620,6 +606,8 @@ func (c *gcControllerState) endCycle() { " u_a/u_g=", u_a/u_g, "\n") } + + return triggerRatio } // enlistWorker encourages another dedicated mark worker to start on @@ -651,7 +639,7 @@ func (c *gcControllerState) enlistWorker() { } myID := gp.m.p.ptr().id for tries := 0; tries < 5; tries++ { - id := int32(fastrand() % uint32(gomaxprocs-1)) + id := int32(fastrandn(uint32(gomaxprocs - 1))) if id >= myID { id++ } @@ -701,9 +689,6 @@ func (c *gcControllerState) findRunnableGCWorker(_p_ *p) *g { // This P is now dedicated to marking until the end of // the concurrent mark phase. _p_.gcMarkWorkerMode = gcMarkWorkerDedicatedMode - // TODO(austin): This P isn't going to run anything - // else for a while, so kick everything out of its run - // queue. } else { if !decIfPositive(&c.fractionalMarkWorkersNeeded) { // No more workers are need right now. @@ -761,6 +746,120 @@ func (c *gcControllerState) findRunnableGCWorker(_p_ *p) *g { return gp } +// gcSetTriggerRatio sets the trigger ratio and updates everything +// derived from it: the absolute trigger, the heap goal, mark pacing, +// and sweep pacing. +// +// This can be called any time. If GC is the in the middle of a +// concurrent phase, it will adjust the pacing of that phase. +// +// This depends on gcpercent, memstats.heap_marked, and +// memstats.heap_live. These must be up to date. +// +// mheap_.lock must be held or the world must be stopped. +func gcSetTriggerRatio(triggerRatio float64) { + // Set the trigger ratio, capped to reasonable bounds. + if triggerRatio < 0 { + // This can happen if the mutator is allocating very + // quickly or the GC is scanning very slowly. + triggerRatio = 0 + } else if gcpercent >= 0 { + // Ensure there's always a little margin so that the + // mutator assist ratio isn't infinity. + maxTriggerRatio := 0.95 * float64(gcpercent) / 100 + if triggerRatio > maxTriggerRatio { + triggerRatio = maxTriggerRatio + } + } + memstats.triggerRatio = triggerRatio + + // Compute the absolute GC trigger from the trigger ratio. + // + // We trigger the next GC cycle when the allocated heap has + // grown by the trigger ratio over the marked heap size. + trigger := ^uint64(0) + if gcpercent >= 0 { + trigger = uint64(float64(memstats.heap_marked) * (1 + triggerRatio)) + // Don't trigger below the minimum heap size. + minTrigger := heapminimum + if !gosweepdone() { + // Concurrent sweep happens in the heap growth + // from heap_live to gc_trigger, so ensure + // that concurrent sweep has some heap growth + // in which to perform sweeping before we + // start the next GC cycle. + sweepMin := atomic.Load64(&memstats.heap_live) + sweepMinHeapDistance*uint64(gcpercent)/100 + if sweepMin > minTrigger { + minTrigger = sweepMin + } + } + if trigger < minTrigger { + trigger = minTrigger + } + if int64(trigger) < 0 { + print("runtime: next_gc=", memstats.next_gc, " heap_marked=", memstats.heap_marked, " heap_live=", memstats.heap_live, " initialHeapLive=", work.initialHeapLive, "triggerRatio=", triggerRatio, " minTrigger=", minTrigger, "\n") + throw("gc_trigger underflow") + } + } + memstats.gc_trigger = trigger + + // Compute the next GC goal, which is when the allocated heap + // has grown by GOGC/100 over the heap marked by the last + // cycle. + goal := ^uint64(0) + if gcpercent >= 0 { + goal = memstats.heap_marked + memstats.heap_marked*uint64(gcpercent)/100 + if goal < trigger { + // The trigger ratio is always less than GOGC/100, but + // other bounds on the trigger may have raised it. + // Push up the goal, too. + goal = trigger + } + } + memstats.next_gc = goal + if trace.enabled { + traceNextGC() + } + + // Update mark pacing. + if gcphase != _GCoff { + gcController.revise() + } + + // Update sweep pacing. + if gosweepdone() { + mheap_.sweepPagesPerByte = 0 + } else { + // Concurrent sweep needs to sweep all of the in-use + // pages by the time the allocated heap reaches the GC + // trigger. Compute the ratio of in-use pages to sweep + // per byte allocated, accounting for the fact that + // some might already be swept. + heapLiveBasis := atomic.Load64(&memstats.heap_live) + heapDistance := int64(trigger) - int64(heapLiveBasis) + // Add a little margin so rounding errors and + // concurrent sweep are less likely to leave pages + // unswept when GC starts. + heapDistance -= 1024 * 1024 + if heapDistance < _PageSize { + // Avoid setting the sweep ratio extremely high + heapDistance = _PageSize + } + pagesSwept := atomic.Load64(&mheap_.pagesSwept) + sweepDistancePages := int64(mheap_.pagesInUse) - int64(pagesSwept) + if sweepDistancePages <= 0 { + mheap_.sweepPagesPerByte = 0 + } else { + mheap_.sweepPagesPerByte = float64(sweepDistancePages) / float64(heapDistance) + mheap_.sweepHeapLiveBasis = heapLiveBasis + // Write pagesSweptBasis last, since this + // signals concurrent sweeps to recompute + // their debt. + atomic.Store64(&mheap_.pagesSweptBasis, pagesSwept) + } + } +} + // gcGoalUtilization is the goal CPU utilization for background // marking as a fraction of GOMAXPROCS. const gcGoalUtilization = 0.25 @@ -783,10 +882,23 @@ const gcAssistTimeSlack = 5000 const gcOverAssistWork = 64 << 10 var work struct { - full uint64 // lock-free list of full blocks workbuf - empty uint64 // lock-free list of empty blocks workbuf + full lfstack // lock-free list of full blocks workbuf + empty lfstack // lock-free list of empty blocks workbuf pad0 [sys.CacheLineSize]uint8 // prevents false-sharing between full/empty and nproc/nwait + wbufSpans struct { + lock mutex + // free is a list of spans dedicated to workbufs, but + // that don't currently contain any workbufs. + free mSpanList + // busy is a list of all spans containing workbufs on + // one of the workbuf lists. + busy mSpanList + } + + // Restore 64-bit alignment on 32-bit. + _ uint32 + // bytesMarked is the number of bytes marked this cycle. This // includes bytes blackened in scanned objects, noscan objects // that go straight to black, and permagrey objects scanned by @@ -816,15 +928,13 @@ var work struct { // should pass gcDrainBlock to gcDrain to block in the // getfull() barrier. Otherwise, they should pass gcDrainNoBlock. // - // TODO: This is a temporary fallback to support - // debug.gcrescanstacks > 0 and to work around some known - // races. Remove this when we remove the debug option and fix - // the races. + // TODO: This is a temporary fallback to work around races + // that cause early mark termination. helperDrainBlock bool // Number of roots of various root types. Set by gcMarkRootPrepare. - nFlushCacheRoots int - nDataRoots, nSpanRoots, nStackRoots, nRescanRoots int + nFlushCacheRoots int + nDataRoots, nSpanRoots, nStackRoots int // markrootDone indicates that roots have been marked at least // once during the current GC cycle. This is checked by root @@ -860,6 +970,10 @@ var work struct { // mode is the concurrency mode of the current GC cycle. mode gcMode + // userForced indicates the current GC cycle was forced by an + // explicit user call. + userForced bool + // totaltime is the CPU nanoseconds spent in GC since the // program started if debug.gctrace > 0. totaltime int64 @@ -876,14 +990,19 @@ var work struct { head, tail guintptr } - // rescan is a list of G's that need to be rescanned during - // mark termination. A G adds itself to this list when it - // first invalidates its stack scan. - rescan struct { + // sweepWaiters is a list of blocked goroutines to wake when + // we transition from mark termination to sweep. + sweepWaiters struct { lock mutex - list []guintptr + head guintptr } + // cycles is the number of completed GC cycles, where a GC + // cycle is sweep termination, mark, mark termination, and + // sweep. This differs from memstats.numgc, which is + // incremented at mark termination. + cycles uint32 + // Timing/utilization stats for this cycle. stwprocs, maxprocs int32 tSweepTerm, tMark, tMarkTerm, tEnd int64 // nanotime() of phase start @@ -899,7 +1018,94 @@ var work struct { // garbage collection is complete. It may also block the entire // program. func GC() { - gcStart(gcForceBlockMode, false) + // We consider a cycle to be: sweep termination, mark, mark + // termination, and sweep. This function shouldn't return + // until a full cycle has been completed, from beginning to + // end. Hence, we always want to finish up the current cycle + // and start a new one. That means: + // + // 1. In sweep termination, mark, or mark termination of cycle + // N, wait until mark termination N completes and transitions + // to sweep N. + // + // 2. In sweep N, help with sweep N. + // + // At this point we can begin a full cycle N+1. + // + // 3. Trigger cycle N+1 by starting sweep termination N+1. + // + // 4. Wait for mark termination N+1 to complete. + // + // 5. Help with sweep N+1 until it's done. + // + // This all has to be written to deal with the fact that the + // GC may move ahead on its own. For example, when we block + // until mark termination N, we may wake up in cycle N+2. + + gp := getg() + + // Prevent the GC phase or cycle count from changing. + lock(&work.sweepWaiters.lock) + n := atomic.Load(&work.cycles) + if gcphase == _GCmark { + // Wait until sweep termination, mark, and mark + // termination of cycle N complete. + gp.schedlink = work.sweepWaiters.head + work.sweepWaiters.head.set(gp) + goparkunlock(&work.sweepWaiters.lock, "wait for GC cycle", traceEvGoBlock, 1) + } else { + // We're in sweep N already. + unlock(&work.sweepWaiters.lock) + } + + // We're now in sweep N or later. Trigger GC cycle N+1, which + // will first finish sweep N if necessary and then enter sweep + // termination N+1. + gcStart(gcBackgroundMode, gcTrigger{kind: gcTriggerCycle, n: n + 1}) + + // Wait for mark termination N+1 to complete. + lock(&work.sweepWaiters.lock) + if gcphase == _GCmark && atomic.Load(&work.cycles) == n+1 { + gp.schedlink = work.sweepWaiters.head + work.sweepWaiters.head.set(gp) + goparkunlock(&work.sweepWaiters.lock, "wait for GC cycle", traceEvGoBlock, 1) + } else { + unlock(&work.sweepWaiters.lock) + } + + // Finish sweep N+1 before returning. We do this both to + // complete the cycle and because runtime.GC() is often used + // as part of tests and benchmarks to get the system into a + // relatively stable and isolated state. + for atomic.Load(&work.cycles) == n+1 && gosweepone() != ^uintptr(0) { + sweep.nbgsweep++ + Gosched() + } + + // Callers may assume that the heap profile reflects the + // just-completed cycle when this returns (historically this + // happened because this was a STW GC), but right now the + // profile still reflects mark termination N, not N+1. + // + // As soon as all of the sweep frees from cycle N+1 are done, + // we can go ahead and publish the heap profile. + // + // First, wait for sweeping to finish. (We know there are no + // more spans on the sweep queue, but we may be concurrently + // sweeping spans, so we have to wait.) + for atomic.Load(&work.cycles) == n+1 && atomic.Load(&mheap_.sweepers) != 0 { + Gosched() + } + + // Now we're really done with sweeping, so we can publish the + // stable heap profile. Only do this if we haven't already hit + // another mark termination. + mp := acquirem() + cycle := atomic.Load(&work.cycles) + if cycle == n+1 || (gcphase == _GCmark && cycle == n+2) { + mProf_PostSweep() + } + releasem(mp) } // gcMode indicates how concurrent a GC cycle should be. @@ -911,24 +1117,75 @@ const ( gcForceBlockMode // stop-the-world GC now and STW sweep (forced by user) ) -// gcShouldStart returns true if the exit condition for the _GCoff -// phase has been met. The exit condition should be tested when -// allocating. -// -// If forceTrigger is true, it ignores the current heap size, but -// checks all other conditions. In general this should be false. -func gcShouldStart(forceTrigger bool) bool { - return gcphase == _GCoff && (forceTrigger || memstats.heap_live >= memstats.gc_trigger) && memstats.enablegc && panicking == 0 && gcpercent >= 0 +// A gcTrigger is a predicate for starting a GC cycle. Specifically, +// it is an exit condition for the _GCoff phase. +type gcTrigger struct { + kind gcTriggerKind + now int64 // gcTriggerTime: current time + n uint32 // gcTriggerCycle: cycle number to start +} + +type gcTriggerKind int + +const ( + // gcTriggerAlways indicates that a cycle should be started + // unconditionally, even if GOGC is off or we're in a cycle + // right now. This cannot be consolidated with other cycles. + gcTriggerAlways gcTriggerKind = iota + + // gcTriggerHeap indicates that a cycle should be started when + // the heap size reaches the trigger heap size computed by the + // controller. + gcTriggerHeap + + // gcTriggerTime indicates that a cycle should be started when + // it's been more than forcegcperiod nanoseconds since the + // previous GC cycle. + gcTriggerTime + + // gcTriggerCycle indicates that a cycle should be started if + // we have not yet started cycle number gcTrigger.n (relative + // to work.cycles). + gcTriggerCycle +) + +// test returns true if the trigger condition is satisfied, meaning +// that the exit condition for the _GCoff phase has been met. The exit +// condition should be tested when allocating. +func (t gcTrigger) test() bool { + if !memstats.enablegc || panicking != 0 { + return false + } + if t.kind == gcTriggerAlways { + return true + } + if gcphase != _GCoff || gcpercent < 0 { + return false + } + switch t.kind { + case gcTriggerHeap: + // Non-atomic access to heap_live for performance. If + // we are going to trigger on this, this thread just + // atomically wrote heap_live anyway and we'll see our + // own write. + return memstats.heap_live >= memstats.gc_trigger + case gcTriggerTime: + lastgc := int64(atomic.Load64(&memstats.last_gc_nanotime)) + return lastgc != 0 && t.now-lastgc > forcegcperiod + case gcTriggerCycle: + // t.n > work.cycles, but accounting for wraparound. + return int32(t.n-work.cycles) > 0 + } + return true } -// gcStart transitions the GC from _GCoff to _GCmark (if mode == -// gcBackgroundMode) or _GCmarktermination (if mode != -// gcBackgroundMode) by performing sweep termination and GC -// initialization. +// gcStart transitions the GC from _GCoff to _GCmark (if +// !mode.stwMark) or _GCmarktermination (if mode.stwMark) by +// performing sweep termination and GC initialization. // // This may return without performing this transition in some cases, // such as when called on a system stack or with locks held. -func gcStart(mode gcMode, forceTrigger bool) { +func gcStart(mode gcMode, trigger gcTrigger) { // Since this is called from malloc and malloc is called in // the guts of a number of libraries that might be holding // locks, don't attempt to start GC in non-preemptible or @@ -951,29 +1208,21 @@ func gcStart(mode gcMode, forceTrigger bool) { // // We check the transition condition continuously here in case // this G gets delayed in to the next GC cycle. - for (mode != gcBackgroundMode || gcShouldStart(forceTrigger)) && gosweepone() != ^uintptr(0) { + for trigger.test() && gosweepone() != ^uintptr(0) { sweep.nbgsweep++ } // Perform GC initialization and the sweep termination // transition. - // - // If this is a forced GC, don't acquire the transition lock - // or re-check the transition condition because we - // specifically *don't* want to share the transition with - // another thread. - useStartSema := mode == gcBackgroundMode - if useStartSema { - semacquire(&work.startSema, 0) - // Re-check transition condition under transition lock. - if !gcShouldStart(forceTrigger) { - semrelease(&work.startSema) - return - } + semacquire(&work.startSema) + // Re-check transition condition under transition lock. + if !trigger.test() { + semrelease(&work.startSema) + return } // For stats, check if this GC was forced by the user. - forced := mode != gcBackgroundMode + work.userForced = trigger.kind == gcTriggerAlways || trigger.kind == gcTriggerCycle // In gcstoptheworld debug mode, upgrade the mode accordingly. // We do this after re-checking the transition condition so @@ -988,7 +1237,7 @@ func gcStart(mode gcMode, forceTrigger bool) { } // Ok, we're doing it! Stop everybody else - semacquire(&worldsema, 0) + semacquire(&worldsema) if trace.enabled { traceGCStart() @@ -1000,13 +1249,13 @@ func gcStart(mode gcMode, forceTrigger bool) { gcResetMarkState() - now := nanotime() work.stwprocs, work.maxprocs = gcprocs(), gomaxprocs - work.tSweepTerm = now - work.heap0 = memstats.heap_live + work.heap0 = atomic.Load64(&memstats.heap_live) work.pauseNS = 0 work.mode = mode + now := nanotime() + work.tSweepTerm = now work.pauseStart = now systemstack(stopTheWorldWithSema) // Finish sweep before we start concurrent scan. @@ -1017,6 +1266,7 @@ func gcStart(mode gcMode, forceTrigger bool) { // reclaimed until the next GC cycle. clearpools() + work.cycles++ if mode == gcBackgroundMode { // Do as much work concurrently as possible gcController.startCycle() work.heapGoal = memstats.next_gc @@ -1029,18 +1279,7 @@ func gcStart(mode gcMode, forceTrigger bool) { // the time we start the world and begin // scanning. // - // It's necessary to enable write barriers - // during the scan phase for several reasons: - // - // They must be enabled for writes to higher - // stack frames before we scan stacks and - // install stack barriers because this is how - // we track writes to inactive stack frames. - // (Alternatively, we could not install stack - // barriers over frame boundaries with - // up-pointers). - // - // They must be enabled before assists are + // Write barriers must be enabled before assists are // enabled because they must be enabled before // any non-leaf heap objects are marked. Since // allocations are blocked until assists can @@ -1079,17 +1318,11 @@ func gcStart(mode gcMode, forceTrigger bool) { work.tMark, work.tMarkTerm = t, t work.heapGoal = work.heap0 - if forced { - memstats.numforcedgc++ - } - // Perform mark termination. This will restart the world. - gcMarkTermination() + gcMarkTermination(memstats.triggerRatio) } - if useStartSema { - semrelease(&work.startSema) - } + semrelease(&work.startSema) } // gcMarkDone transitions the GC from mark 1 to mark 2 and from mark 2 @@ -1109,7 +1342,7 @@ func gcStart(mode gcMode, forceTrigger bool) { // by mark termination. func gcMarkDone() { top: - semacquire(&work.markDoneSema, 0) + semacquire(&work.markDoneSema) // Re-check transition condition under transition lock. if !(gcphase == _GCmark && work.nwait == work.nproc && !gcMarkWorkAvailable(nil)) { @@ -1204,14 +1437,14 @@ top: // endCycle depends on all gcWork cache stats being // flushed. This is ensured by mark 2. - gcController.endCycle() + nextTriggerRatio := gcController.endCycle() // Perform mark termination. This will restart the world. - gcMarkTermination() + gcMarkTermination(nextTriggerRatio) } } -func gcMarkTermination() { +func gcMarkTermination(nextTriggerRatio float64) { // World is stopped. // Start marktermination which includes enabling the write barrier. atomic.Store(&gcBlackenEnabled, 0) @@ -1293,11 +1526,17 @@ func gcMarkTermination() { throw("gc done but gcphase != _GCoff") } + // Update GC trigger and pacing for the next cycle. + gcSetTriggerRatio(nextTriggerRatio) + // Update timing memstats - now, unixNow := nanotime(), unixnanotime() + now := nanotime() + sec, nsec, _ := time_now() + unixNow := sec*1e9 + int64(nsec) work.pauseNS += now - work.pauseStart work.tEnd = now - atomic.Store64(&memstats.last_gc, uint64(unixNow)) // must be Unix time to make sense to user + atomic.Store64(&memstats.last_gc_unix, uint64(unixNow)) // must be Unix time to make sense to user + atomic.Store64(&memstats.last_gc_nanotime, uint64(now)) // monotonic time for us memstats.pause_ns[memstats.numgc%uint32(len(memstats.pause_ns))] = uint64(work.pauseNS) memstats.pause_end[memstats.numgc%uint32(len(memstats.pause_end))] = uint64(unixNow) memstats.pause_total_ns += uint64(work.pauseNS) @@ -1315,25 +1554,36 @@ func gcMarkTermination() { totalCpu := sched.totaltime + (now-sched.procresizetime)*int64(gomaxprocs) memstats.gc_cpu_fraction = float64(work.totaltime) / float64(totalCpu) - memstats.numgc++ - // Reset sweep state. sweep.nbgsweep = 0 sweep.npausesweep = 0 + if work.userForced { + memstats.numforcedgc++ + } + + // Bump GC cycle count and wake goroutines waiting on sweep. + lock(&work.sweepWaiters.lock) + memstats.numgc++ + injectglist(work.sweepWaiters.head.ptr()) + work.sweepWaiters.head = 0 + unlock(&work.sweepWaiters.lock) + + // Finish the current heap profiling cycle and start a new + // heap profiling cycle. We do this before starting the world + // so events don't leak into the wrong cycle. + mProf_NextCycle() + systemstack(startTheWorldWithSema) - // Update heap profile stats if gcSweep didn't do it. This is - // relatively expensive, so we don't want to do it while the - // world is stopped, but it needs to happen ASAP after - // starting the world to prevent too many allocations from the - // next cycle leaking in. It must happen before releasing - // worldsema since there are applications that do a - // runtime.GC() to update the heap profile and then - // immediately collect the profile. - if _ConcurrentSweep && work.mode != gcForceBlockMode { - mProf_GC() - } + // Flush the heap profile so we can start a new cycle next GC. + // This is relatively expensive, so we don't do it with the + // world stopped. + mProf_Flush() + + // Prepare workbufs for freeing by the sweeper. We do this + // asynchronously because it can take non-trivial time. + prepareFreeWorkbufs() // Print gctrace before dropping worldsema. As soon as we drop // worldsema another cycle could start and smash the stats @@ -1368,7 +1618,7 @@ func gcMarkTermination() { work.heap0>>20, "->", work.heap1>>20, "->", work.heap2>>20, " MB, ", work.heapGoal>>20, " MB goal, ", work.maxprocs, " P") - if work.mode != gcBackgroundMode { + if work.userForced { print(" (forced)") } print("\n") @@ -1521,6 +1771,25 @@ func gcBgMarkWorker(_p_ *p) { default: throw("gcBgMarkWorker: unexpected gcMarkWorkerMode") case gcMarkWorkerDedicatedMode: + gcDrain(&_p_.gcw, gcDrainUntilPreempt|gcDrainFlushBgCredit) + if gp.preempt { + // We were preempted. This is + // a useful signal to kick + // everything out of the run + // queue so it can run + // somewhere else. + lock(&sched.lock) + for { + gp, _ := runqget(_p_) + if gp == nil { + break + } + globrunqput(gp) + } + unlock(&sched.lock) + } + // Go back to draining, this time + // without preemption. gcDrain(&_p_.gcw, gcDrainNoBlock|gcDrainFlushBgCredit) case gcMarkWorkerFractionalMode: gcDrain(&_p_.gcw, gcDrainUntilPreempt|gcDrainFlushBgCredit) @@ -1593,7 +1862,7 @@ func gcMarkWorkAvailable(p *p) bool { if p != nil && !p.gcw.empty() { return true } - if atomic.Load64(&work.full) != 0 { + if !work.full.empty() { return true // global work available } if work.markrootNext < work.markrootJobs { @@ -1623,24 +1892,22 @@ func gcMark(start_time int64) { work.ndone = 0 work.nproc = uint32(gcprocs()) - if debug.gcrescanstacks == 0 && work.full == 0 && work.nDataRoots+work.nSpanRoots+work.nStackRoots+work.nRescanRoots == 0 { + if work.full == 0 && work.nDataRoots+work.nSpanRoots+work.nStackRoots == 0 { // There's no work on the work queue and no root jobs // that can produce work, so don't bother entering the // getfull() barrier. // - // With the hybrid barrier enabled, this will be the - // situation the vast majority of the time after - // concurrent mark. However, we still need a fallback - // for STW GC and because there are some known races - // that occasionally leave work around for mark - // termination. + // This will be the situation the vast majority of the + // time after concurrent mark. However, we still need + // a fallback for STW GC and because there are some + // known races that occasionally leave work around for + // mark termination. // // We're still hedging our bets here: if we do // accidentally produce some work, we'll still process // it, just not necessarily in parallel. // - // TODO(austin): When we eliminate - // debug.gcrescanstacks: fix the races, and remove + // TODO(austin): Fix the races and and remove // work draining from mark termination so we don't // need the fallback path. work.helperDrainBlock = false @@ -1704,52 +1971,14 @@ func gcMark(start_time int64) { // Update the marked heap stat. memstats.heap_marked = work.bytesMarked - // Trigger the next GC cycle when the allocated heap has grown - // by triggerRatio over the marked heap size. Assume that - // we're in steady state, so the marked heap size is the - // same now as it was at the beginning of the GC cycle. - memstats.gc_trigger = uint64(float64(memstats.heap_marked) * (1 + gcController.triggerRatio)) - if memstats.gc_trigger < heapminimum { - memstats.gc_trigger = heapminimum - } - if int64(memstats.gc_trigger) < 0 { - print("next_gc=", memstats.next_gc, " bytesMarked=", work.bytesMarked, " heap_live=", memstats.heap_live, " initialHeapLive=", work.initialHeapLive, "\n") - throw("gc_trigger underflow") - } - // Update other GC heap size stats. This must happen after // cachestats (which flushes local statistics to these) and // flushallmcaches (which modifies heap_live). memstats.heap_live = work.bytesMarked memstats.heap_scan = uint64(gcController.scanWork) - minTrigger := memstats.heap_live + sweepMinHeapDistance*uint64(gcpercent)/100 - if memstats.gc_trigger < minTrigger { - // The allocated heap is already past the trigger. - // This can happen if the triggerRatio is very low and - // the marked heap is less than the live heap size. - // - // Concurrent sweep happens in the heap growth from - // heap_live to gc_trigger, so bump gc_trigger up to ensure - // that concurrent sweep has some heap growth in which - // to perform sweeping before we start the next GC - // cycle. - memstats.gc_trigger = minTrigger - } - - // The next GC cycle should finish before the allocated heap - // has grown by GOGC/100. - memstats.next_gc = memstats.heap_marked + memstats.heap_marked*uint64(gcpercent)/100 - if gcpercent < 0 { - memstats.next_gc = ^uint64(0) - } - if memstats.next_gc < memstats.gc_trigger { - memstats.next_gc = memstats.gc_trigger - } - if trace.enabled { traceHeapAlloc() - traceNextGC() } } @@ -1767,6 +1996,7 @@ func gcSweep(mode gcMode) { // with an empty swept list. throw("non-empty swept list") } + mheap_.pagesSwept = 0 unlock(&mheap_.lock) if !_ConcurrentSweep || mode == gcForceBlockMode { @@ -1774,35 +2004,23 @@ func gcSweep(mode gcMode) { // Record that no proportional sweeping has to happen. lock(&mheap_.lock) mheap_.sweepPagesPerByte = 0 - mheap_.pagesSwept = 0 unlock(&mheap_.lock) // Sweep all spans eagerly. for sweepone() != ^uintptr(0) { sweep.npausesweep++ } - // Do an additional mProf_GC, because all 'free' events are now real as well. - mProf_GC() - mProf_GC() + // Free workbufs eagerly. + prepareFreeWorkbufs() + for freeSomeWbufs(false) { + } + // All "free" events for this mark/sweep cycle have + // now happened, so we can make this profile cycle + // available immediately. + mProf_NextCycle() + mProf_Flush() return } - // Concurrent sweep needs to sweep all of the in-use pages by - // the time the allocated heap reaches the GC trigger. Compute - // the ratio of in-use pages to sweep per byte allocated. - heapDistance := int64(memstats.gc_trigger) - int64(memstats.heap_live) - // Add a little margin so rounding errors and concurrent - // sweep are less likely to leave pages unswept when GC starts. - heapDistance -= 1024 * 1024 - if heapDistance < _PageSize { - // Avoid setting the sweep ratio extremely high - heapDistance = _PageSize - } - lock(&mheap_.lock) - mheap_.sweepPagesPerByte = float64(mheap_.pagesInUse) / float64(heapDistance) - mheap_.pagesSwept = 0 - mheap_.spanBytesAlloc = 0 - unlock(&mheap_.lock) - // Background sweep. lock(&sweep.lock) if sweep.parked { @@ -1820,24 +2038,16 @@ func gcSweep(mode gcMode) { func gcResetMarkState() { // This may be called during a concurrent phase, so make sure // allgs doesn't change. - if !(gcphase == _GCoff || gcphase == _GCmarktermination) { - // Accessing gcRescan is unsafe. - throw("bad GC phase") - } lock(&allglock) for _, gp := range allgs { gp.gcscandone = false // set to true in gcphasework gp.gcscanvalid = false // stack has not been scanned - gp.gcRescan = -1 gp.gcAssistBytes = 0 } unlock(&allglock) - // Clear rescan list. - work.rescan.list = work.rescan.list[:0] - work.bytesMarked = 0 - work.initialHeapLive = memstats.heap_live + work.initialHeapLive = atomic.Load64(&memstats.heap_live) work.markrootDone = false } diff --git a/libgo/go/runtime/mgclarge.go b/libgo/go/runtime/mgclarge.go new file mode 100644 index 00000000000..757e88d1d9d --- /dev/null +++ b/libgo/go/runtime/mgclarge.go @@ -0,0 +1,326 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Page heap. +// +// See malloc.go for the general overview. +// +// Large spans are the subject of this file. Spans consisting of less than +// _MaxMHeapLists are held in lists of like sized spans. Larger spans +// are held in a treap. See https://en.wikipedia.org/wiki/Treap or +// http://faculty.washington.edu/aragon/pubs/rst89.pdf for an overview. +// sema.go also holds an implementation of a treap. +// +// Each treapNode holds a single span. The treap is sorted by page size +// and for spans of the same size a secondary sort based on start address +// is done. +// Spans are returned based on a best fit algorithm and for spans of the same +// size the one at the lowest address is selected. +// +// The primary routines are +// insert: adds a span to the treap +// remove: removes the span from that treap that best fits the required size +// removeSpan: which removes a specific span from the treap +// +// _mheap.lock must be held when manipulating this data structure. + +package runtime + +import ( + "unsafe" +) + +//go:notinheap +type mTreap struct { + treap *treapNode +} + +//go:notinheap +type treapNode struct { + right *treapNode // all treapNodes > this treap node + left *treapNode // all treapNodes < this treap node + parent *treapNode // direct parent of this node, nil if root + npagesKey uintptr // number of pages in spanKey, used as primary sort key + spanKey *mspan // span of size npagesKey, used as secondary sort key + priority uint32 // random number used by treap algorithm keep tree probablistically balanced +} + +func (t *treapNode) init() { + t.right = nil + t.left = nil + t.parent = nil + t.spanKey = nil + t.npagesKey = 0 + t.priority = 0 +} + +// isSpanInTreap is handy for debugging. One should hold the heap lock, usually +// mheap_.lock(). +func (t *treapNode) isSpanInTreap(s *mspan) bool { + if t == nil { + return false + } + return t.spanKey == s || t.left.isSpanInTreap(s) || t.right.isSpanInTreap(s) +} + +// walkTreap is handy for debugging. +// Starting at some treapnode t, for example the root, do a depth first preorder walk of +// the tree executing fn at each treap node. One should hold the heap lock, usually +// mheap_.lock(). +func (t *treapNode) walkTreap(fn func(tn *treapNode)) { + if t == nil { + return + } + fn(t) + t.left.walkTreap(fn) + t.right.walkTreap(fn) +} + +// checkTreapNode when used in conjunction with walkTreap can usually detect a +// poorly formed treap. +func checkTreapNode(t *treapNode) { + // lessThan is used to order the treap. + // npagesKey and npages are the primary keys. + // spanKey and span are the secondary keys. + // span == nil (0) will always be lessThan all + // spans of the same size. + lessThan := func(npages uintptr, s *mspan) bool { + if t.npagesKey != npages { + return t.npagesKey < npages + } + // t.npagesKey == npages + return uintptr(unsafe.Pointer(t.spanKey)) < uintptr(unsafe.Pointer(s)) + } + + if t == nil { + return + } + if t.spanKey.npages != t.npagesKey || t.spanKey.next != nil { + println("runtime: checkTreapNode treapNode t=", t, " t.npagesKey=", t.npagesKey, + "t.spanKey.npages=", t.spanKey.npages) + throw("why does span.npages and treap.ngagesKey do not match?") + } + if t.left != nil && lessThan(t.left.npagesKey, t.left.spanKey) { + throw("t.lessThan(t.left.npagesKey, t.left.spanKey) is not false") + } + if t.right != nil && !lessThan(t.right.npagesKey, t.right.spanKey) { + throw("!t.lessThan(t.left.npagesKey, t.left.spanKey) is not false") + } +} + +// insert adds span to the large span treap. +func (root *mTreap) insert(span *mspan) { + npages := span.npages + var last *treapNode + pt := &root.treap + for t := *pt; t != nil; t = *pt { + last = t + if t.npagesKey < npages { + pt = &t.right + } else if t.npagesKey > npages { + pt = &t.left + } else if uintptr(unsafe.Pointer(t.spanKey)) < uintptr(unsafe.Pointer(span)) { + // t.npagesKey == npages, so sort on span addresses. + pt = &t.right + } else if uintptr(unsafe.Pointer(t.spanKey)) > uintptr(unsafe.Pointer(span)) { + pt = &t.left + } else { + throw("inserting span already in treap") + } + } + + // Add t as new leaf in tree of span size and unique addrs. + // The balanced tree is a treap using priority as the random heap priority. + // That is, it is a binary tree ordered according to the npagesKey, + // but then among the space of possible binary trees respecting those + // npagesKeys, it is kept balanced on average by maintaining a heap ordering + // on the priority: s.priority <= both s.right.priority and s.right.priority. + // https://en.wikipedia.org/wiki/Treap + // http://faculty.washington.edu/aragon/pubs/rst89.pdf + + t := (*treapNode)(mheap_.treapalloc.alloc()) + t.init() + t.npagesKey = span.npages + t.priority = fastrand() + t.spanKey = span + t.parent = last + *pt = t // t now at a leaf. + // Rotate up into tree according to priority. + for t.parent != nil && t.parent.priority > t.priority { + if t != nil && t.spanKey.npages != t.npagesKey { + println("runtime: insert t=", t, "t.npagesKey=", t.npagesKey) + println("runtime: t.spanKey=", t.spanKey, "t.spanKey.npages=", t.spanKey.npages) + throw("span and treap sizes do not match?") + } + if t.parent.left == t { + root.rotateRight(t.parent) + } else { + if t.parent.right != t { + throw("treap insert finds a broken treap") + } + root.rotateLeft(t.parent) + } + } +} + +func (root *mTreap) removeNode(t *treapNode) *mspan { + if t.spanKey.npages != t.npagesKey { + throw("span and treap node npages do not match") + } + result := t.spanKey + + // Rotate t down to be leaf of tree for removal, respecting priorities. + for t.right != nil || t.left != nil { + if t.right == nil || t.left != nil && t.left.priority < t.right.priority { + root.rotateRight(t) + } else { + root.rotateLeft(t) + } + } + // Remove t, now a leaf. + if t.parent != nil { + if t.parent.left == t { + t.parent.left = nil + } else { + t.parent.right = nil + } + } else { + root.treap = nil + } + // Return the found treapNode's span after freeing the treapNode. + t.spanKey = nil + t.npagesKey = 0 + mheap_.treapalloc.free(unsafe.Pointer(t)) + return result +} + +// remove searches for, finds, removes from the treap, and returns the smallest +// span that can hold npages. If no span has at least npages return nil. +// This is slightly more complicated than a simple binary tree search +// since if an exact match is not found the next larger node is +// returned. +// If the last node inspected > npagesKey not holding +// a left node (a smaller npages) is the "best fit" node. +func (root *mTreap) remove(npages uintptr) *mspan { + t := root.treap + for t != nil { + if t.spanKey == nil { + throw("treap node with nil spanKey found") + } + if t.npagesKey < npages { + t = t.right + } else if t.left != nil && t.left.npagesKey >= npages { + t = t.left + } else { + result := t.spanKey + root.removeNode(t) + return result + } + } + return nil +} + +// removeSpan searches for, finds, deletes span along with +// the associated treap node. If the span is not in the treap +// then t will eventually be set to nil and the t.spanKey +// will throw. +func (root *mTreap) removeSpan(span *mspan) { + npages := span.npages + t := root.treap + for t.spanKey != span { + if t.npagesKey < npages { + t = t.right + } else if t.npagesKey > npages { + t = t.left + } else if uintptr(unsafe.Pointer(t.spanKey)) < uintptr(unsafe.Pointer(span)) { + t = t.right + } else if uintptr(unsafe.Pointer(t.spanKey)) > uintptr(unsafe.Pointer(span)) { + t = t.left + } + } + root.removeNode(t) +} + +// scavengetreap visits each node in the treap and scavenges the +// treapNode's span. +func scavengetreap(treap *treapNode, now, limit uint64) uintptr { + if treap == nil { + return 0 + } + return scavengeTreapNode(treap, now, limit) + + scavengetreap(treap.left, now, limit) + + scavengetreap(treap.right, now, limit) +} + +// rotateLeft rotates the tree rooted at node x. +// turning (x a (y b c)) into (y (x a b) c). +func (root *mTreap) rotateLeft(x *treapNode) { + // p -> (x a (y b c)) + p := x.parent + a, y := x.left, x.right + b, c := y.left, y.right + + y.left = x + x.parent = y + y.right = c + if c != nil { + c.parent = y + } + x.left = a + if a != nil { + a.parent = x + } + x.right = b + if b != nil { + b.parent = x + } + + y.parent = p + if p == nil { + root.treap = y + } else if p.left == x { + p.left = y + } else { + if p.right != x { + throw("large span treap rotateLeft") + } + p.right = y + } +} + +// rotateRight rotates the tree rooted at node y. +// turning (y (x a b) c) into (x a (y b c)). +func (root *mTreap) rotateRight(y *treapNode) { + // p -> (y (x a b) c) + p := y.parent + x, c := y.left, y.right + a, b := x.left, x.right + + x.left = a + if a != nil { + a.parent = x + } + x.right = y + y.parent = x + y.left = b + if b != nil { + b.parent = y + } + y.right = c + if c != nil { + c.parent = y + } + + x.parent = p + if p == nil { + root.treap = x + } else if p.left == y { + p.left = x + } else { + if p.right != y { + throw("large span treap rotateRight") + } + p.right = x + } +} diff --git a/libgo/go/runtime/mgcmark.go b/libgo/go/runtime/mgcmark.go index 93252ba8a9e..998a830caa8 100644 --- a/libgo/go/runtime/mgcmark.go +++ b/libgo/go/runtime/mgcmark.go @@ -93,21 +93,24 @@ func gcMarkRootPrepare() { // termination, allglen isn't changing, so we'll scan // all Gs. work.nStackRoots = int(atomic.Loaduintptr(&allglen)) - work.nRescanRoots = 0 } else { // We've already scanned span roots and kept the scan // up-to-date during concurrent mark. work.nSpanRoots = 0 - // On the second pass of markroot, we're just scanning - // dirty stacks. It's safe to access rescan since the - // world is stopped. + // The hybrid barrier ensures that stacks can't + // contain pointers to unmarked objects, so on the + // second markroot, there's no need to scan stacks. work.nStackRoots = 0 - work.nRescanRoots = len(work.rescan.list) + + if debug.gcrescanstacks > 0 { + // Scan stacks anyway for debugging. + work.nStackRoots = int(atomic.Loaduintptr(&allglen)) + } } work.markrootNext = 0 - work.markrootJobs = uint32(fixedRootCount + work.nFlushCacheRoots + work.nDataRoots + work.nSpanRoots + work.nStackRoots + work.nRescanRoots) + work.markrootJobs = uint32(fixedRootCount + work.nFlushCacheRoots + work.nDataRoots + work.nSpanRoots + work.nStackRoots) } // gcMarkRootCheck checks that all roots have been scanned. It is @@ -165,8 +168,7 @@ func markroot(gcw *gcWork, i uint32) { baseData := baseFlushCache + uint32(work.nFlushCacheRoots) baseSpans := baseData + uint32(work.nDataRoots) baseStacks := baseSpans + uint32(work.nSpanRoots) - baseRescan := baseStacks + uint32(work.nStackRoots) - end := baseRescan + uint32(work.nRescanRoots) + end := baseStacks + uint32(work.nStackRoots) // Note: if you add a case here, please also update heapdump.go:dumproots. switch { @@ -186,6 +188,11 @@ func markroot(gcw *gcWork, i uint32) { } case i == fixedRootFinalizers: + // Only do this once per GC cycle since we don't call + // queuefinalizer during marking. + if work.markrootDone { + break + } for fb := allfin; fb != nil; fb = fb.alllink { cnt := uintptr(atomic.Load(&fb.cnt)) scanblock(uintptr(unsafe.Pointer(&fb.fin[0])), cnt*unsafe.Sizeof(fb.fin[0]), &finptrmask[0], gcw) @@ -201,15 +208,8 @@ func markroot(gcw *gcWork, i uint32) { default: // the rest is scanning goroutine stacks var gp *g - if baseStacks <= i && i < baseRescan { + if baseStacks <= i && i < end { gp = allgs[i-baseStacks] - } else if baseRescan <= i && i < end { - gp = work.rescan.list[i-baseRescan].ptr() - if gp.gcRescan != int32(i-baseRescan) { - // Looking for issue #17099. - println("runtime: gp", gp, "found at rescan index", i-baseRescan, "but should be at", gp.gcRescan) - throw("bad g rescan index") - } } else { throw("markroot: bad index") } @@ -352,6 +352,7 @@ func gcAssistAlloc(gp *g) { return } + traced := false retry: // Compute the amount of scan work we need to do to make the // balance positive. When the required amount of work is low, @@ -387,10 +388,18 @@ retry: if scanWork == 0 { // We were able to steal all of the credit we // needed. + if traced { + traceGCMarkAssistDone() + } return } } + if trace.enabled && !traced { + traced = true + traceGCMarkAssistStart() + } + // Perform assist work systemstack(func() { gcAssistAlloc1(gp, scanWork) @@ -433,6 +442,9 @@ retry: // At this point either background GC has satisfied // this G's assist debt, or the GC cycle is over. } + if traced { + traceGCMarkAssistDone() + } } // gcAssistAlloc1 is the part of gcAssistAlloc that runs on the system @@ -656,10 +668,6 @@ func doscanstack(*g, *gcWork) // scanstack scans gp's stack, greying all pointers found on the stack. // -// During mark phase, it also installs stack barriers while traversing -// gp's stack. During mark termination, it stops scanning when it -// reaches an unhit stack barrier. -// // scanstack is marked go:systemstack because it must not be preempted // while using a workbuf. // @@ -699,84 +707,9 @@ func scanstack(gp *g, gcw *gcWork) { scanstackblock(uintptr(unsafe.Pointer(&gp.gcregs)), unsafe.Sizeof(gp.gcregs), gcw) scanstackblock(uintptr(unsafe.Pointer(&gp.context)), unsafe.Sizeof(gp.context), gcw) - if gcphase == _GCmark { - // gp may have added itself to the rescan list between - // when GC started and now. It's clean now, so remove - // it. This isn't safe during mark termination because - // mark termination is consuming this list, but it's - // also not necessary. - dequeueRescan(gp) - } gp.gcscanvalid = true } -// queueRescan adds gp to the stack rescan list and clears -// gp.gcscanvalid. The caller must own gp and ensure that gp isn't -// already on the rescan list. -func queueRescan(gp *g) { - if debug.gcrescanstacks == 0 { - // Clear gcscanvalid to keep assertions happy. - // - // TODO: Remove gcscanvalid entirely when we remove - // stack rescanning. - gp.gcscanvalid = false - return - } - - if gcphase == _GCoff { - gp.gcscanvalid = false - return - } - if gp.gcRescan != -1 { - throw("g already on rescan list") - } - - lock(&work.rescan.lock) - gp.gcscanvalid = false - - // Recheck gcphase under the lock in case there was a phase change. - if gcphase == _GCoff { - unlock(&work.rescan.lock) - return - } - if len(work.rescan.list) == cap(work.rescan.list) { - throw("rescan list overflow") - } - n := len(work.rescan.list) - gp.gcRescan = int32(n) - work.rescan.list = work.rescan.list[:n+1] - work.rescan.list[n].set(gp) - unlock(&work.rescan.lock) -} - -// dequeueRescan removes gp from the stack rescan list, if gp is on -// the rescan list. The caller must own gp. -func dequeueRescan(gp *g) { - if debug.gcrescanstacks == 0 { - return - } - - if gp.gcRescan == -1 { - return - } - if gcphase == _GCoff { - gp.gcRescan = -1 - return - } - - lock(&work.rescan.lock) - if work.rescan.list[gp.gcRescan].ptr() != gp { - throw("bad dequeueRescan") - } - // Careful: gp may itself be the last G on the list. - last := work.rescan.list[len(work.rescan.list)-1] - work.rescan.list[gp.gcRescan] = last - last.ptr().gcRescan = gp.gcRescan - gp.gcRescan = -1 - work.rescan.list = work.rescan.list[:len(work.rescan.list)-1] - unlock(&work.rescan.lock) -} - type gcDrainFlags int const ( @@ -1052,7 +985,7 @@ func scanobject(b uintptr, gcw *gcWork) { // paths), in which case we must *not* enqueue // oblets since their bitmaps will be // uninitialized. - if !hbits.hasPointers(n) { + if s.spanclass.noscan() { // Bypass the whole scan. gcw.bytesMarked += uint64(n) return @@ -1185,6 +1118,7 @@ func greyobject(obj, base, off uintptr, hbits heapBits, span *mspan, gcw *gcWork // Dump the object gcDumpObject("obj", obj, ^uintptr(0)) + getg().m.traceback = 2 throw("checkmark found unmarked object") } if hbits.isCheckmarked(span.elemsize) { @@ -1206,6 +1140,7 @@ func greyobject(obj, base, off uintptr, hbits heapBits, span *mspan, gcw *gcWork print("runtime: marking free object ", hex(obj), " found at *(", hex(base), "+", hex(off), ")\n") gcDumpObject("base", base, off) gcDumpObject("obj", obj, ^uintptr(0)) + getg().m.traceback = 2 throw("marking free object") } @@ -1217,7 +1152,7 @@ func greyobject(obj, base, off uintptr, hbits heapBits, span *mspan, gcw *gcWork atomic.Or8(mbits.bytep, mbits.mask) // If this is a noscan object, fast-track it to black // instead of greying it. - if !hbits.hasPointers(span.elemsize) { + if span.spanclass.noscan() { gcw.bytesMarked += uint64(span.elemsize) return } @@ -1250,7 +1185,7 @@ func gcDumpObject(label string, obj, off uintptr) { print(" s=nil\n") return } - print(" s.base()=", hex(s.base()), " s.limit=", hex(s.limit), " s.sizeclass=", s.sizeclass, " s.elemsize=", s.elemsize, " s.state=") + print(" s.base()=", hex(s.base()), " s.limit=", hex(s.limit), " s.spanclass=", s.spanclass, " s.elemsize=", s.elemsize, " s.state=") if 0 <= s.state && int(s.state) < len(mSpanStateNames) { print(mSpanStateNames[s.state], "\n") } else { @@ -1259,7 +1194,7 @@ func gcDumpObject(label string, obj, off uintptr) { skipped := false size := s.elemsize - if s.state == _MSpanStack && size == 0 { + if s.state == _MSpanManual && size == 0 { // We're printing something from a stack frame. We // don't know how big it is, so just show up to an // including off. diff --git a/libgo/go/runtime/mgcsweep.go b/libgo/go/runtime/mgcsweep.go index 2b698bf74a5..c60214cdaf3 100644 --- a/libgo/go/runtime/mgcsweep.go +++ b/libgo/go/runtime/mgcsweep.go @@ -22,10 +22,6 @@ type sweepdata struct { nbgsweep uint32 npausesweep uint32 - - // pacertracegen is the sweepgen at which the last pacer trace - // "sweep finished" message was printed. - pacertracegen uint32 } // finishsweep_m ensures that all spans are swept. @@ -62,6 +58,9 @@ func bgsweep(c chan int) { sweep.nbgsweep++ Gosched() } + for freeSomeWbufs(true) { + Gosched() + } lock(&sweep.lock) if !gosweepdone() { // This can happen if a GC runs between @@ -80,20 +79,24 @@ func bgsweep(c chan int) { //go:nowritebarrier func sweepone() uintptr { _g_ := getg() + sweepRatio := mheap_.sweepPagesPerByte // For debugging // increment locks to ensure that the goroutine is not preempted // in the middle of sweep thus leaving the span in an inconsistent state for next GC _g_.m.locks++ + if atomic.Load(&mheap_.sweepdone) != 0 { + _g_.m.locks-- + return ^uintptr(0) + } + atomic.Xadd(&mheap_.sweepers, +1) + + npages := ^uintptr(0) sg := mheap_.sweepgen for { s := mheap_.sweepSpans[1-sg/2%2].pop() if s == nil { - mheap_.sweepdone = 1 - _g_.m.locks-- - if debug.gcpacertrace > 0 && atomic.Cas(&sweep.pacertracegen, sg-2, sg) { - print("pacer: sweep done at heap size ", memstats.heap_live>>20, "MB; allocated ", mheap_.spanBytesAlloc>>20, "MB of spans; swept ", mheap_.pagesSwept, " pages at ", mheap_.sweepPagesPerByte, " pages/byte\n") - } - return ^uintptr(0) + atomic.Store(&mheap_.sweepdone, 1) + break } if s.state != mSpanInUse { // This can happen if direct sweeping already @@ -108,16 +111,25 @@ func sweepone() uintptr { if s.sweepgen != sg-2 || !atomic.Cas(&s.sweepgen, sg-2, sg-1) { continue } - npages := s.npages + npages = s.npages if !s.sweep(false) { // Span is still in-use, so this returned no // pages to the heap and the span needs to // move to the swept in-use list. npages = 0 } - _g_.m.locks-- - return npages + break + } + + // Decrement the number of active sweepers and if this is the + // last one print trace information. + if atomic.Xadd(&mheap_.sweepers, -1) == 0 && atomic.Load(&mheap_.sweepdone) != 0 { + if debug.gcpacertrace > 0 { + print("pacer: sweep done at heap size ", memstats.heap_live>>20, "MB; allocated ", (memstats.heap_live-mheap_.sweepHeapLiveBasis)>>20, "MB during sweep; swept ", mheap_.pagesSwept, " pages at ", sweepRatio, " pages/byte\n") + } } + _g_.m.locks-- + return npages } //go:nowritebarrier @@ -180,15 +192,14 @@ func (s *mspan) sweep(preserve bool) bool { } if trace.enabled { - traceGCSweepStart() + traceGCSweepSpan(s.npages * _PageSize) } atomic.Xadd64(&mheap_.pagesSwept, int64(s.npages)) - cl := s.sizeclass + spc := s.spanclass size := s.elemsize res := false - nfree := 0 c := _g_.m.mcache freeToHeap := false @@ -278,12 +289,11 @@ func (s *mspan) sweep(preserve bool) bool { } // Count the number of free objects in this span. - nfree = s.countFree() - if cl == 0 && nfree != 0 { + nalloc := uint16(s.countAlloc()) + if spc.sizeclass() == 0 && nalloc == 0 { s.needzero = 1 freeToHeap = true } - nalloc := uint16(s.nelems) - uint16(nfree) nfreed := s.allocCount - nalloc // This test is not reliable with gccgo, because of @@ -300,13 +310,16 @@ func (s *mspan) sweep(preserve bool) bool { // unnecessarily, but provided the pointer is not really live // it is not otherwise a problem. So we disable the test for gccgo. if false && nalloc > s.allocCount { - print("runtime: nelems=", s.nelems, " nfree=", nfree, " nalloc=", nalloc, " previous allocCount=", s.allocCount, " nfreed=", nfreed, "\n") + print("runtime: nelems=", s.nelems, " nalloc=", nalloc, " previous allocCount=", s.allocCount, " nfreed=", nfreed, "\n") throw("sweep increased allocation count") } s.allocCount = nalloc wasempty := s.nextFreeIndex() == s.nelems s.freeindex = 0 // reset allocation index to start of span. + if trace.enabled { + getg().m.p.ptr().traceReclaimed += uintptr(nfreed) * s.elemsize + } // gcmarkBits becomes the allocBits. // get a fresh cleared gcmarkBits in preparation for next GC @@ -334,9 +347,9 @@ func (s *mspan) sweep(preserve bool) bool { atomic.Store(&s.sweepgen, sweepgen) } - if nfreed > 0 && cl != 0 { - c.local_nsmallfree[cl] += uintptr(nfreed) - res = mheap_.central[cl].mcentral.freeSpan(s, preserve, wasempty) + if nfreed > 0 && spc.sizeclass() != 0 { + c.local_nsmallfree[spc.sizeclass()] += uintptr(nfreed) + res = mheap_.central[spc].mcentral.freeSpan(s, preserve, wasempty) // MCentral_FreeSpan updates sweepgen } else if freeToHeap { // Free large span to heap @@ -370,9 +383,6 @@ func (s *mspan) sweep(preserve bool) bool { // it on the swept in-use list. mheap_.sweepSpans[sweepgen/2%2].push(s) } - if trace.enabled { - traceGCSweepDone() - } return res } @@ -385,8 +395,7 @@ func (s *mspan) sweep(preserve bool) bool { // // deductSweepCredit makes a worst-case assumption that all spanBytes // bytes of the ultimately allocated span will be available for object -// allocation. The caller should call reimburseSweepCredit if that -// turns out not to be the case once the span is allocated. +// allocation. // // deductSweepCredit is the core of the "proportional sweep" system. // It uses statistics gathered by the garbage collector to perform @@ -400,31 +409,28 @@ func deductSweepCredit(spanBytes uintptr, callerSweepPages uintptr) { return } - // Account for this span allocation. - spanBytesAlloc := atomic.Xadd64(&mheap_.spanBytesAlloc, int64(spanBytes)) + if trace.enabled { + traceGCSweepStart() + } + +retry: + sweptBasis := atomic.Load64(&mheap_.pagesSweptBasis) // Fix debt if necessary. - pagesOwed := int64(mheap_.sweepPagesPerByte * float64(spanBytesAlloc)) - for pagesOwed-int64(atomic.Load64(&mheap_.pagesSwept)) > int64(callerSweepPages) { + newHeapLive := uintptr(atomic.Load64(&memstats.heap_live)-mheap_.sweepHeapLiveBasis) + spanBytes + pagesTarget := int64(mheap_.sweepPagesPerByte*float64(newHeapLive)) - int64(callerSweepPages) + for pagesTarget > int64(atomic.Load64(&mheap_.pagesSwept)-sweptBasis) { if gosweepone() == ^uintptr(0) { mheap_.sweepPagesPerByte = 0 break } + if atomic.Load64(&mheap_.pagesSweptBasis) != sweptBasis { + // Sweep pacing changed. Recompute debt. + goto retry + } } -} -// reimburseSweepCredit records that unusableBytes bytes of a -// just-allocated span are not available for object allocation. This -// offsets the worst-case charge performed by deductSweepCredit. -func reimburseSweepCredit(unusableBytes uintptr) { - if mheap_.sweepPagesPerByte == 0 { - // Nobody cares about the credit. Avoid the atomic. - return - } - nval := atomic.Xadd64(&mheap_.spanBytesAlloc, -int64(unusableBytes)) - if int64(nval) < 0 { - // Debugging for #18043. - print("runtime: bad spanBytesAlloc=", nval, " (was ", nval+uint64(unusableBytes), ") unusableBytes=", unusableBytes, " sweepPagesPerByte=", mheap_.sweepPagesPerByte, "\n") - throw("spanBytesAlloc underflow") + if trace.enabled { + traceGCSweepDone() } } diff --git a/libgo/go/runtime/mgcwork.go b/libgo/go/runtime/mgcwork.go index 5eb05a767c0..461679b9343 100644 --- a/libgo/go/runtime/mgcwork.go +++ b/libgo/go/runtime/mgcwork.go @@ -12,8 +12,22 @@ import ( const ( _WorkbufSize = 2048 // in bytes; larger values result in less contention + + // workbufAlloc is the number of bytes to allocate at a time + // for new workbufs. This must be a multiple of pageSize and + // should be a multiple of _WorkbufSize. + // + // Larger values reduce workbuf allocation overhead. Smaller + // values reduce heap fragmentation. + workbufAlloc = 32 << 10 ) +func init() { + if workbufAlloc%pageSize != 0 || workbufAlloc%_WorkbufSize != 0 { + throw("bad workbufAlloc") + } +} + // Garbage collector work pool abstraction. // // This implements a producer/consumer model for pointers to grey @@ -25,21 +39,6 @@ const ( // grey objects, thus blackening them, and then scans them, // potentially producing new pointers to grey objects. -// A wbufptr holds a workbuf*, but protects it from write barriers. -// workbufs never live on the heap, so write barriers are unnecessary. -// Write barriers on workbuf pointers may also be dangerous in the GC. -// -// TODO: Since workbuf is now go:notinheap, this isn't necessary. -type wbufptr uintptr - -func wbufptrOf(w *workbuf) wbufptr { - return wbufptr(unsafe.Pointer(w)) -} - -func (wp wbufptr) ptr() *workbuf { - return (*workbuf)(unsafe.Pointer(wp)) -} - // A gcWork provides the interface to produce and consume work for the // garbage collector. // @@ -75,7 +74,7 @@ type gcWork struct { // next. // // Invariant: Both wbuf1 and wbuf2 are nil or neither are. - wbuf1, wbuf2 wbufptr + wbuf1, wbuf2 *workbuf // Bytes marked (blackened) on this gcWork. This is aggregated // into work.bytesMarked by dispose. @@ -87,12 +86,12 @@ type gcWork struct { } func (w *gcWork) init() { - w.wbuf1 = wbufptrOf(getempty()) + w.wbuf1 = getempty() wbuf2 := trygetfull() if wbuf2 == nil { wbuf2 = getempty() } - w.wbuf2 = wbufptrOf(wbuf2) + w.wbuf2 = wbuf2 } // put enqueues a pointer for the garbage collector to trace. @@ -100,18 +99,18 @@ func (w *gcWork) init() { //go:nowritebarrier func (w *gcWork) put(obj uintptr) { flushed := false - wbuf := w.wbuf1.ptr() + wbuf := w.wbuf1 if wbuf == nil { w.init() - wbuf = w.wbuf1.ptr() + wbuf = w.wbuf1 // wbuf is empty at this point. } else if wbuf.nobj == len(wbuf.obj) { w.wbuf1, w.wbuf2 = w.wbuf2, w.wbuf1 - wbuf = w.wbuf1.ptr() + wbuf = w.wbuf1 if wbuf.nobj == len(wbuf.obj) { putfull(wbuf) wbuf = getempty() - w.wbuf1 = wbufptrOf(wbuf) + w.wbuf1 = wbuf flushed = true } } @@ -132,7 +131,7 @@ func (w *gcWork) put(obj uintptr) { // otherwise it returns false and the caller needs to call put. //go:nowritebarrier func (w *gcWork) putFast(obj uintptr) bool { - wbuf := w.wbuf1.ptr() + wbuf := w.wbuf1 if wbuf == nil { return false } else if wbuf.nobj == len(wbuf.obj) { @@ -151,15 +150,15 @@ func (w *gcWork) putFast(obj uintptr) bool { // other gcWork instances or other caches. //go:nowritebarrier func (w *gcWork) tryGet() uintptr { - wbuf := w.wbuf1.ptr() + wbuf := w.wbuf1 if wbuf == nil { w.init() - wbuf = w.wbuf1.ptr() + wbuf = w.wbuf1 // wbuf is empty at this point. } if wbuf.nobj == 0 { w.wbuf1, w.wbuf2 = w.wbuf2, w.wbuf1 - wbuf = w.wbuf1.ptr() + wbuf = w.wbuf1 if wbuf.nobj == 0 { owbuf := wbuf wbuf = trygetfull() @@ -167,7 +166,7 @@ func (w *gcWork) tryGet() uintptr { return 0 } putempty(owbuf) - w.wbuf1 = wbufptrOf(wbuf) + w.wbuf1 = wbuf } } @@ -180,7 +179,7 @@ func (w *gcWork) tryGet() uintptr { // the caller is expected to call tryGet(). //go:nowritebarrier func (w *gcWork) tryGetFast() uintptr { - wbuf := w.wbuf1.ptr() + wbuf := w.wbuf1 if wbuf == nil { return 0 } @@ -197,15 +196,15 @@ func (w *gcWork) tryGetFast() uintptr { // been retrieved. get returns 0 if there are no pointers remaining. //go:nowritebarrier func (w *gcWork) get() uintptr { - wbuf := w.wbuf1.ptr() + wbuf := w.wbuf1 if wbuf == nil { w.init() - wbuf = w.wbuf1.ptr() + wbuf = w.wbuf1 // wbuf is empty at this point. } if wbuf.nobj == 0 { w.wbuf1, w.wbuf2 = w.wbuf2, w.wbuf1 - wbuf = w.wbuf1.ptr() + wbuf = w.wbuf1 if wbuf.nobj == 0 { owbuf := wbuf wbuf = getfull() @@ -213,7 +212,7 @@ func (w *gcWork) get() uintptr { return 0 } putempty(owbuf) - w.wbuf1 = wbufptrOf(wbuf) + w.wbuf1 = wbuf } } @@ -231,21 +230,21 @@ func (w *gcWork) get() uintptr { // //go:nowritebarrier func (w *gcWork) dispose() { - if wbuf := w.wbuf1.ptr(); wbuf != nil { + if wbuf := w.wbuf1; wbuf != nil { if wbuf.nobj == 0 { putempty(wbuf) } else { putfull(wbuf) } - w.wbuf1 = 0 + w.wbuf1 = nil - wbuf = w.wbuf2.ptr() + wbuf = w.wbuf2 if wbuf.nobj == 0 { putempty(wbuf) } else { putfull(wbuf) } - w.wbuf2 = 0 + w.wbuf2 = nil } if w.bytesMarked != 0 { // dispose happens relatively infrequently. If this @@ -265,14 +264,14 @@ func (w *gcWork) dispose() { // global queue. //go:nowritebarrier func (w *gcWork) balance() { - if w.wbuf1 == 0 { + if w.wbuf1 == nil { return } - if wbuf := w.wbuf2.ptr(); wbuf.nobj != 0 { + if wbuf := w.wbuf2; wbuf.nobj != 0 { putfull(wbuf) - w.wbuf2 = wbufptrOf(getempty()) - } else if wbuf := w.wbuf1.ptr(); wbuf.nobj > 4 { - w.wbuf1 = wbufptrOf(handoff(wbuf)) + w.wbuf2 = getempty() + } else if wbuf := w.wbuf1; wbuf.nobj > 4 { + w.wbuf1 = handoff(wbuf) } else { return } @@ -285,7 +284,7 @@ func (w *gcWork) balance() { // empty returns true if w has no mark work available. //go:nowritebarrier func (w *gcWork) empty() bool { - return w.wbuf1 == 0 || (w.wbuf1.ptr().nobj == 0 && w.wbuf2.ptr().nobj == 0) + return w.wbuf1 == nil || (w.wbuf1.nobj == 0 && w.wbuf2.nobj == 0) } // Internally, the GC work pool is kept in arrays in work buffers. @@ -327,23 +326,56 @@ func (b *workbuf) checkempty() { func getempty() *workbuf { var b *workbuf if work.empty != 0 { - b = (*workbuf)(lfstackpop(&work.empty)) + b = (*workbuf)(work.empty.pop()) if b != nil { b.checkempty() } } if b == nil { - b = (*workbuf)(persistentalloc(unsafe.Sizeof(*b), sys.CacheLineSize, &memstats.gc_sys)) + // Allocate more workbufs. + var s *mspan + if work.wbufSpans.free.first != nil { + lock(&work.wbufSpans.lock) + s = work.wbufSpans.free.first + if s != nil { + work.wbufSpans.free.remove(s) + work.wbufSpans.busy.insert(s) + } + unlock(&work.wbufSpans.lock) + } + if s == nil { + systemstack(func() { + s = mheap_.allocManual(workbufAlloc/pageSize, &memstats.gc_sys) + }) + if s == nil { + throw("out of memory") + } + // Record the new span in the busy list. + lock(&work.wbufSpans.lock) + work.wbufSpans.busy.insert(s) + unlock(&work.wbufSpans.lock) + } + // Slice up the span into new workbufs. Return one and + // put the rest on the empty list. + for i := uintptr(0); i+_WorkbufSize <= workbufAlloc; i += _WorkbufSize { + newb := (*workbuf)(unsafe.Pointer(s.base() + i)) + newb.nobj = 0 + if i == 0 { + b = newb + } else { + putempty(newb) + } + } } return b } // putempty puts a workbuf onto the work.empty list. -// Upon entry this go routine owns b. The lfstackpush relinquishes ownership. +// Upon entry this go routine owns b. The lfstack.push relinquishes ownership. //go:nowritebarrier func putempty(b *workbuf) { b.checkempty() - lfstackpush(&work.empty, &b.node) + work.empty.push(&b.node) } // putfull puts the workbuf on the work.full list for the GC. @@ -352,14 +384,14 @@ func putempty(b *workbuf) { //go:nowritebarrier func putfull(b *workbuf) { b.checknonempty() - lfstackpush(&work.full, &b.node) + work.full.push(&b.node) } // trygetfull tries to get a full or partially empty workbuffer. // If one is not immediately available return nil //go:nowritebarrier func trygetfull() *workbuf { - b := (*workbuf)(lfstackpop(&work.full)) + b := (*workbuf)(work.full.pop()) if b != nil { b.checknonempty() return b @@ -380,7 +412,7 @@ func trygetfull() *workbuf { // phase. //go:nowritebarrier func getfull() *workbuf { - b := (*workbuf)(lfstackpop(&work.full)) + b := (*workbuf)(work.full.pop()) if b != nil { b.checknonempty() return b @@ -398,7 +430,7 @@ func getfull() *workbuf { println("runtime: work.nwait=", decnwait, "work.nproc=", work.nproc) throw("work.nwait > work.nproc") } - b = (*workbuf)(lfstackpop(&work.full)) + b = (*workbuf)(work.full.pop()) if b != nil { b.checknonempty() return b @@ -412,15 +444,11 @@ func getfull() *workbuf { if work.nwait == work.nproc && work.markrootNext >= work.markrootJobs { return nil } - _g_ := getg() if i < 10 { - _g_.m.gcstats.nprocyield++ procyield(20) } else if i < 20 { - _g_.m.gcstats.nosyield++ osyield() } else { - _g_.m.gcstats.nsleep++ usleep(100) } } @@ -434,11 +462,49 @@ func handoff(b *workbuf) *workbuf { b.nobj -= n b1.nobj = n memmove(unsafe.Pointer(&b1.obj[0]), unsafe.Pointer(&b.obj[b.nobj]), uintptr(n)*unsafe.Sizeof(b1.obj[0])) - _g_ := getg() - _g_.m.gcstats.nhandoff++ - _g_.m.gcstats.nhandoffcnt += uint64(n) // Put b on full list - let first half of b get stolen. putfull(b) return b1 } + +// prepareFreeWorkbufs moves busy workbuf spans to free list so they +// can be freed to the heap. This must only be called when all +// workbufs are on the empty list. +func prepareFreeWorkbufs() { + lock(&work.wbufSpans.lock) + if work.full != 0 { + throw("cannot free workbufs when work.full != 0") + } + // Since all workbufs are on the empty list, we don't care + // which ones are in which spans. We can wipe the entire empty + // list and move all workbuf spans to the free list. + work.empty = 0 + work.wbufSpans.free.takeAll(&work.wbufSpans.busy) + unlock(&work.wbufSpans.lock) +} + +// freeSomeWbufs frees some workbufs back to the heap and returns +// true if it should be called again to free more. +func freeSomeWbufs(preemptible bool) bool { + const batchSize = 64 // ~1–2 µs per span. + lock(&work.wbufSpans.lock) + if gcphase != _GCoff || work.wbufSpans.free.isEmpty() { + unlock(&work.wbufSpans.lock) + return false + } + systemstack(func() { + gp := getg().m.curg + for i := 0; i < batchSize && !(preemptible && gp.preempt); i++ { + span := work.wbufSpans.free.first + if span == nil { + break + } + work.wbufSpans.free.remove(span) + mheap_.freeManual(span, &memstats.gc_sys) + } + }) + more := !work.wbufSpans.free.isEmpty() + unlock(&work.wbufSpans.lock) + return more +} diff --git a/libgo/go/runtime/mheap.go b/libgo/go/runtime/mheap.go index 72627485b87..8749f971065 100644 --- a/libgo/go/runtime/mheap.go +++ b/libgo/go/runtime/mheap.go @@ -29,12 +29,13 @@ const minPhysPageSize = 4096 //go:notinheap type mheap struct { lock mutex - free [_MaxMHeapList]mSpanList // free lists of given length - freelarge mSpanList // free lists length >= _MaxMHeapList - busy [_MaxMHeapList]mSpanList // busy lists of large objects of given length - busylarge mSpanList // busy lists of large objects length >= _MaxMHeapList + free [_MaxMHeapList]mSpanList // free lists of given length up to _MaxMHeapList + freelarge mTreap // free treap of length >= _MaxMHeapList + busy [_MaxMHeapList]mSpanList // busy lists of large spans of given length + busylarge mSpanList // busy lists of large spans length >= _MaxMHeapList sweepgen uint32 // sweep generation, see comment in mspan sweepdone uint32 // all spans are swept + sweepers uint32 // number of active sweepone calls // allspans is a slice of all mspans ever created. Each mspan // appears exactly once. @@ -74,37 +75,82 @@ type mheap struct { _ uint32 // align uint64 fields on 32-bit for atomics // Proportional sweep - pagesInUse uint64 // pages of spans in stats _MSpanInUse; R/W with mheap.lock - spanBytesAlloc uint64 // bytes of spans allocated this cycle; updated atomically - pagesSwept uint64 // pages swept this cycle; updated atomically - sweepPagesPerByte float64 // proportional sweep ratio; written with lock, read without + // + // These parameters represent a linear function from heap_live + // to page sweep count. The proportional sweep system works to + // stay in the black by keeping the current page sweep count + // above this line at the current heap_live. + // + // The line has slope sweepPagesPerByte and passes through a + // basis point at (sweepHeapLiveBasis, pagesSweptBasis). At + // any given time, the system is at (memstats.heap_live, + // pagesSwept) in this space. + // + // It's important that the line pass through a point we + // control rather than simply starting at a (0,0) origin + // because that lets us adjust sweep pacing at any time while + // accounting for current progress. If we could only adjust + // the slope, it would create a discontinuity in debt if any + // progress has already been made. + pagesInUse uint64 // pages of spans in stats _MSpanInUse; R/W with mheap.lock + pagesSwept uint64 // pages swept this cycle; updated atomically + pagesSweptBasis uint64 // pagesSwept to use as the origin of the sweep ratio; updated atomically + sweepHeapLiveBasis uint64 // value of heap_live to use as the origin of sweep ratio; written with lock, read without + sweepPagesPerByte float64 // proportional sweep ratio; written with lock, read without // TODO(austin): pagesInUse should be a uintptr, but the 386 // compiler can't 8-byte align fields. // Malloc stats. - largefree uint64 // bytes freed for large objects (>maxsmallsize) - nlargefree uint64 // number of frees for large objects (>maxsmallsize) - nsmallfree [_NumSizeClasses]uint64 // number of frees for small objects (<=maxsmallsize) + largealloc uint64 // bytes allocated for large objects + nlargealloc uint64 // number of large object allocations + largefree uint64 // bytes freed for large objects (>maxsmallsize) + nlargefree uint64 // number of frees for large objects (>maxsmallsize) + nsmallfree [_NumSizeClasses]uint64 // number of frees for small objects (<=maxsmallsize) // range of addresses we might see in the heap - bitmap uintptr // Points to one byte past the end of the bitmap - bitmap_mapped uintptr - arena_start uintptr - arena_used uintptr // always mHeap_Map{Bits,Spans} before updating - arena_end uintptr + bitmap uintptr // Points to one byte past the end of the bitmap + bitmap_mapped uintptr + + // The arena_* fields indicate the addresses of the Go heap. + // + // The maximum range of the Go heap is + // [arena_start, arena_start+_MaxMem+1). + // + // The range of the current Go heap is + // [arena_start, arena_used). Parts of this range may not be + // mapped, but the metadata structures are always mapped for + // the full range. + arena_start uintptr + arena_used uintptr // Set with setArenaUsed. + + // The heap is grown using a linear allocator that allocates + // from the block [arena_alloc, arena_end). arena_alloc is + // often, but *not always* equal to arena_used. + arena_alloc uintptr + arena_end uintptr + + // arena_reserved indicates that the memory [arena_alloc, + // arena_end) is reserved (e.g., mapped PROT_NONE). If this is + // false, we have to be careful not to clobber existing + // mappings here. If this is true, then we own the mapping + // here and *must* clobber it to use it. arena_reserved bool + _ uint32 // ensure 64-bit alignment + // central free lists for small size classes. // the padding makes sure that the MCentrals are // spaced CacheLineSize bytes apart, so that each MCentral.lock // gets its own cache line. - central [_NumSizeClasses]struct { + // central is indexed by spanClass. + central [numSpanClasses]struct { mcentral mcentral - pad [sys.CacheLineSize]byte + pad [sys.CacheLineSize - unsafe.Sizeof(mcentral{})%sys.CacheLineSize]byte } spanalloc fixalloc // allocator for span* cachealloc fixalloc // allocator for mcache* + treapalloc fixalloc // allocator for treapNodes* used by large objects specialfinalizeralloc fixalloc // allocator for specialfinalizer* specialprofilealloc fixalloc // allocator for specialprofile* speciallock mutex // lock for special record allocators. @@ -117,7 +163,7 @@ var mheap_ mheap // When a MSpan is in the heap free list, state == MSpanFree // and heapmap(s->start) == span, heapmap(s->start+s->npages-1) == span. // -// When a MSpan is allocated, state == MSpanInUse or MSpanStack +// When a MSpan is allocated, state == MSpanInUse or MSpanManual // and heapmap(i) == span for all s->start <= i < s->start+s->npages. // Every MSpan is in one doubly-linked list, @@ -125,25 +171,25 @@ var mheap_ mheap // MCentral's span lists. // An MSpan representing actual memory has state _MSpanInUse, -// _MSpanStack, or _MSpanFree. Transitions between these states are +// _MSpanManual, or _MSpanFree. Transitions between these states are // constrained as follows: // -// * A span may transition from free to in-use or stack during any GC +// * A span may transition from free to in-use or manual during any GC // phase. // // * During sweeping (gcphase == _GCoff), a span may transition from -// in-use to free (as a result of sweeping) or stack to free (as a +// in-use to free (as a result of sweeping) or manual to free (as a // result of stacks being freed). // // * During GC (gcphase != _GCoff), a span *must not* transition from -// stack or in-use to free. Because concurrent GC may read a pointer +// manual or in-use to free. Because concurrent GC may read a pointer // and then look up its span, the span state must be monotonic. type mSpanState uint8 const ( - _MSpanDead mSpanState = iota - _MSpanInUse // allocated for garbage collected heap - _MSpanStack // allocated for use by stack allocator + _MSpanDead mSpanState = iota + _MSpanInUse // allocated for garbage collected heap + _MSpanManual // allocated for manual management (e.g., stack allocator) _MSpanFree ) @@ -152,7 +198,7 @@ const ( var mSpanStateNames = []string{ "_MSpanDead", "_MSpanInUse", - "_MSpanStack", + "_MSpanManual", "_MSpanFree", } @@ -170,15 +216,16 @@ type mspan struct { prev *mspan // previous span in list, or nil if none list *mSpanList // For debugging. TODO: Remove. - startAddr uintptr // address of first byte of span aka s.base() - npages uintptr // number of pages in span - stackfreelist gclinkptr // list of free stacks, avoids overloading freelist + startAddr uintptr // address of first byte of span aka s.base() + npages uintptr // number of pages in span + + manualFreeList gclinkptr // list of free objects in _MSpanManual spans // freeindex is the slot index between 0 and nelems at which to begin scanning // for the next free object in this span. // Each allocation scans allocBits starting at freeindex until it encounters a 0 // indicating a free object. freeindex is then adjusted so that subsequent scans begin - // just past the the newly discovered free object. + // just past the newly discovered free object. // // If freeindex == nelem, this span has no free objects. // @@ -224,8 +271,8 @@ type mspan struct { // The sweep will free the old allocBits and set allocBits to the // gcmarkBits. The gcmarkBits are replaced with a fresh zeroed // out memory. - allocBits *uint8 - gcmarkBits *uint8 + allocBits *gcBits + gcmarkBits *gcBits // sweep generation: // if sweepgen == h->sweepgen - 2, the span needs sweeping @@ -236,8 +283,8 @@ type mspan struct { sweepgen uint32 divMul uint16 // for divide by elemsize - divMagic.mul baseMask uint16 // if non-0, elemsize is a power of 2, & this will get object allocation base - allocCount uint16 // capacity - number of objects in freelist - sizeclass uint8 // size class + allocCount uint16 // number of allocated objects + spanclass spanClass // size class and noscan (uint8) incache bool // being used by an mcache state mSpanState // mspaninuse etc needzero uint8 // needs to be zeroed before allocation @@ -292,8 +339,33 @@ func recordspan(vh unsafe.Pointer, p unsafe.Pointer) { h.allspans = append(h.allspans, s) } +// A spanClass represents the size class and noscan-ness of a span. +// +// Each size class has a noscan spanClass and a scan spanClass. The +// noscan spanClass contains only noscan objects, which do not contain +// pointers and thus do not need to be scanned by the garbage +// collector. +type spanClass uint8 + +const ( + numSpanClasses = _NumSizeClasses << 1 + tinySpanClass = spanClass(tinySizeClass<<1 | 1) +) + +func makeSpanClass(sizeclass uint8, noscan bool) spanClass { + return spanClass(sizeclass<<1) | spanClass(bool2int(noscan)) +} + +func (sc spanClass) sizeclass() int8 { + return int8(sc >> 1) +} + +func (sc spanClass) noscan() bool { + return sc&1 != 0 +} + // inheap reports whether b is a pointer into a (potentially dead) heap object. -// It returns false for pointers into stack spans. +// It returns false for pointers into _MSpanManual spans. // Non-preemptible because it is used by write barriers. //go:nowritebarrier //go:nosplit @@ -309,7 +381,9 @@ func inheap(b uintptr) bool { return true } -// inHeapOrStack is a variant of inheap that returns true for pointers into stack spans. +// inHeapOrStack is a variant of inheap that returns true for pointers +// into any allocated heap span. +// //go:nowritebarrier //go:nosplit func inHeapOrStack(b uintptr) bool { @@ -322,10 +396,8 @@ func inHeapOrStack(b uintptr) bool { return false } switch s.state { - case mSpanInUse: + case mSpanInUse, _MSpanManual: return b < s.limit - case _MSpanStack: - return b < s.base()+s.npages<<_PageShift default: return false } @@ -376,7 +448,7 @@ func mlookup(v uintptr, base *uintptr, size *uintptr, sp **mspan) int32 { } p := s.base() - if s.sizeclass == 0 { + if s.spanclass.sizeclass() == 0 { // Large object. if base != nil { *base = p @@ -401,6 +473,7 @@ func mlookup(v uintptr, base *uintptr, size *uintptr, sp **mspan) int32 { // Initialize the heap. func (h *mheap) init(spansStart, spansBytes uintptr) { + h.treapalloc.init(unsafe.Sizeof(treapNode{}), nil, nil, &memstats.other_sys) h.spanalloc.init(unsafe.Sizeof(mspan{}), recordspan, unsafe.Pointer(h), &memstats.mspan_sys) h.cachealloc.init(unsafe.Sizeof(mcache{}), nil, nil, &memstats.mcache_sys) h.specialfinalizeralloc.init(unsafe.Sizeof(specialfinalizer{}), nil, nil, &memstats.other_sys) @@ -421,26 +494,51 @@ func (h *mheap) init(spansStart, spansBytes uintptr) { h.busy[i].init() } - h.freelarge.init() h.busylarge.init() for i := range h.central { - h.central[i].mcentral.init(int32(i)) + h.central[i].mcentral.init(spanClass(i)) } sp := (*slice)(unsafe.Pointer(&h.spans)) sp.array = unsafe.Pointer(spansStart) sp.len = 0 sp.cap = int(spansBytes / sys.PtrSize) + + // Map metadata structures. But don't map race detector memory + // since we're not actually growing the arena here (and TSAN + // gets mad if you map 0 bytes). + h.setArenaUsed(h.arena_used, false) } -// mHeap_MapSpans makes sure that the spans are mapped +// setArenaUsed extends the usable arena to address arena_used and +// maps auxiliary VM regions for any newly usable arena space. +// +// racemap indicates that this memory should be managed by the race +// detector. racemap should be true unless this is covering a VM hole. +func (h *mheap) setArenaUsed(arena_used uintptr, racemap bool) { + // Map auxiliary structures *before* h.arena_used is updated. + // Waiting to update arena_used until after the memory has been mapped + // avoids faults when other threads try access these regions immediately + // after observing the change to arena_used. + + // Map the bitmap. + h.mapBits(arena_used) + + // Map spans array. + h.mapSpans(arena_used) + + // Tell the race detector about the new heap memory. + if racemap && raceenabled { + racemapshadow(unsafe.Pointer(h.arena_used), arena_used-h.arena_used) + } + + h.arena_used = arena_used +} + +// mapSpans makes sure that the spans are mapped // up to the new value of arena_used. // -// It must be called with the expected new value of arena_used, -// *before* h.arena_used has been updated. -// Waiting to update arena_used until after the memory has been mapped -// avoids faults when other threads try access the bitmap immediately -// after observing the change to arena_used. +// Don't call this directly. Call mheap.setArenaUsed. func (h *mheap) mapSpans(arena_used uintptr) { // Map spans array, PageSize at a time. n := arena_used @@ -466,7 +564,7 @@ retry: if s.sweepgen == sg-2 && atomic.Cas(&s.sweepgen, sg-2, sg-1) { list.remove(s) // swept spans are at the end of the list - list.insertBack(s) + list.insertBack(s) // Puts it back on a busy list. s is not in the treap at this point. unlock(&h.lock) snpages := s.npages if s.sweep(false) { @@ -533,7 +631,7 @@ func (h *mheap) reclaim(npage uintptr) { // Allocate a new span of npage pages from the heap for GC'd memory // and record its size class in the HeapMap and HeapMapCache. -func (h *mheap) alloc_m(npage uintptr, sizeclass int32, large bool) *mspan { +func (h *mheap) alloc_m(npage uintptr, spanclass spanClass, large bool) *mspan { _g_ := getg() lock(&h.lock) @@ -547,7 +645,13 @@ func (h *mheap) alloc_m(npage uintptr, sizeclass int32, large bool) *mspan { // If GC kept a bit for whether there were any marks // in a span, we could release these free spans // at the end of GC and eliminate this entirely. + if trace.enabled { + traceGCSweepStart() + } h.reclaim(npage) + if trace.enabled { + traceGCSweepDone() + } } // transfer stats from cache to global @@ -556,7 +660,7 @@ func (h *mheap) alloc_m(npage uintptr, sizeclass int32, large bool) *mspan { memstats.tinyallocs += uint64(_g_.m.mcache.local_tinyallocs) _g_.m.mcache.local_tinyallocs = 0 - s := h.allocSpanLocked(npage) + s := h.allocSpanLocked(npage, &memstats.heap_inuse) if s != nil { // Record span info, because gc needs to be // able to map interior pointer to containing span. @@ -564,8 +668,8 @@ func (h *mheap) alloc_m(npage uintptr, sizeclass int32, large bool) *mspan { h.sweepSpans[h.sweepgen/2%2].push(s) // Add to swept in-use list. s.state = _MSpanInUse s.allocCount = 0 - s.sizeclass = uint8(sizeclass) - if sizeclass == 0 { + s.spanclass = spanclass + if sizeclass := spanclass.sizeclass(); sizeclass == 0 { s.elemsize = s.npages << _PageShift s.divShift = 0 s.divMul = 0 @@ -584,9 +688,11 @@ func (h *mheap) alloc_m(npage uintptr, sizeclass int32, large bool) *mspan { h.pagesInUse += uint64(npage) if large { memstats.heap_objects++ + mheap_.largealloc += uint64(s.elemsize) + mheap_.nlargealloc++ atomic.Xadd64(&memstats.heap_live, int64(npage<<_PageShift)) // Swept spans are at the end of lists. - if s.npages < uintptr(len(h.free)) { + if s.npages < uintptr(len(h.busy)) { h.busy[s.npages].insertBack(s) } else { h.busylarge.insertBack(s) @@ -615,13 +721,13 @@ func (h *mheap) alloc_m(npage uintptr, sizeclass int32, large bool) *mspan { return s } -func (h *mheap) alloc(npage uintptr, sizeclass int32, large bool, needzero bool) *mspan { +func (h *mheap) alloc(npage uintptr, spanclass spanClass, large bool, needzero bool) *mspan { // Don't do any operations that lock the heap on the G stack. // It might trigger stack growth, and the stack growth code needs // to be able to allocate heap. var s *mspan systemstack(func() { - s = h.alloc_m(npage, sizeclass, large) + s = h.alloc_m(npage, spanclass, large) }) if s != nil { @@ -633,29 +739,46 @@ func (h *mheap) alloc(npage uintptr, sizeclass int32, large bool, needzero bool) return s } -func (h *mheap) allocStack(npage uintptr) *mspan { - _g_ := getg() - if _g_ != _g_.m.g0 { - throw("mheap_allocstack not on g0 stack") - } +// allocManual allocates a manually-managed span of npage pages. +// allocManual returns nil if allocation fails. +// +// allocManual adds the bytes used to *stat, which should be a +// memstats in-use field. Unlike allocations in the GC'd heap, the +// allocation does *not* count toward heap_inuse or heap_sys. +// +// The memory backing the returned span may not be zeroed if +// span.needzero is set. +// +// allocManual must be called on the system stack to prevent stack +// growth. Since this is used by the stack allocator, stack growth +// during allocManual would self-deadlock. +// +//go:systemstack +func (h *mheap) allocManual(npage uintptr, stat *uint64) *mspan { lock(&h.lock) - s := h.allocSpanLocked(npage) + s := h.allocSpanLocked(npage, stat) if s != nil { - s.state = _MSpanStack - s.stackfreelist = 0 + s.state = _MSpanManual + s.manualFreeList = 0 s.allocCount = 0 - memstats.stacks_inuse += uint64(s.npages << _PageShift) + s.spanclass = 0 + s.nelems = 0 + s.elemsize = 0 + s.limit = s.base() + s.npages<<_PageShift + // Manually manged memory doesn't count toward heap_sys. + memstats.heap_sys -= uint64(s.npages << _PageShift) } - // This unlock acts as a release barrier. See mHeap_Alloc_m. + // This unlock acts as a release barrier. See mheap.alloc_m. unlock(&h.lock) + return s } // Allocates a span of the given size. h must be locked. // The returned span has been removed from the // free list, but its state is still MSpanFree. -func (h *mheap) allocSpanLocked(npage uintptr) *mspan { +func (h *mheap) allocSpanLocked(npage uintptr, stat *uint64) *mspan { var list *mSpanList var s *mspan @@ -664,13 +787,12 @@ func (h *mheap) allocSpanLocked(npage uintptr) *mspan { list = &h.free[i] if !list.isEmpty() { s = list.first + list.remove(s) goto HaveSpan } } - // Best fit in list of large spans. - list = &h.freelarge - s = h.allocLarge(npage) + s = h.allocLarge(npage) // allocLarge removed s from h.freelarge for us if s == nil { if !h.grow(npage) { return nil @@ -689,10 +811,6 @@ HaveSpan: if s.npages < npage { throw("MHeap_AllocLocked - bad npages") } - list.remove(s) - if s.inList() { - throw("still in list") - } if s.npreleased > 0 { sysUsed(unsafe.Pointer(s.base()), s.npages<<_PageShift) memstats.heap_released -= uint64(s.npreleased << _PageShift) @@ -711,8 +829,8 @@ HaveSpan: h.spans[p] = t h.spans[p+t.npages-1] = t t.needzero = s.needzero - s.state = _MSpanStack // prevent coalescing with s - t.state = _MSpanStack + s.state = _MSpanManual // prevent coalescing with s + t.state = _MSpanManual h.freeSpanLocked(t, false, false, s.unusedsince) s.state = _MSpanFree } @@ -723,7 +841,7 @@ HaveSpan: h.spans[p+n] = s } - memstats.heap_inuse += uint64(npage << _PageShift) + *stat += uint64(npage << _PageShift) memstats.heap_idle -= uint64(npage << _PageShift) //println("spanalloc", hex(s.start<<_PageShift)) @@ -733,24 +851,19 @@ HaveSpan: return s } -// Allocate a span of exactly npage pages from the list of large spans. -func (h *mheap) allocLarge(npage uintptr) *mspan { - return bestFit(&h.freelarge, npage, nil) +// Large spans have a minimum size of 1MByte. The maximum number of large spans to support +// 1TBytes is 1 million, experimentation using random sizes indicates that the depth of +// the tree is less that 2x that of a perfectly balanced tree. For 1TByte can be referenced +// by a perfectly balanced tree with a a depth of 20. Twice that is an acceptable 40. +func (h *mheap) isLargeSpan(npages uintptr) bool { + return npages >= uintptr(len(h.free)) } -// Search list for smallest span with >= npage pages. -// If there are multiple smallest spans, take the one -// with the earliest starting address. -func bestFit(list *mSpanList, npage uintptr, best *mspan) *mspan { - for s := list.first; s != nil; s = s.next { - if s.npages < npage { - continue - } - if best == nil || s.npages < best.npages || (s.npages == best.npages && s.base() < best.base()) { - best = s - } - } - return best +// allocLarge allocates a span of at least npage pages from the treap of large spans. +// Returns nil if no such span currently exists. +func (h *mheap) allocLarge(npage uintptr) *mspan { + // Search treap for smallest span with >= npage pages. + return h.freelarge.remove(npage) } // Try to add at least npage pages of memory to the heap, @@ -849,22 +962,30 @@ func (h *mheap) freeSpan(s *mspan, acct int32) { }) } -func (h *mheap) freeStack(s *mspan) { - _g_ := getg() - if _g_ != _g_.m.g0 { - throw("mheap_freestack not on g0 stack") - } +// freeManual frees a manually-managed span returned by allocManual. +// stat must be the same as the stat passed to the allocManual that +// allocated s. +// +// This must only be called when gcphase == _GCoff. See mSpanState for +// an explanation. +// +// freeManual must be called on the system stack to prevent stack +// growth, just like allocManual. +// +//go:systemstack +func (h *mheap) freeManual(s *mspan, stat *uint64) { s.needzero = 1 lock(&h.lock) - memstats.stacks_inuse -= uint64(s.npages << _PageShift) - h.freeSpanLocked(s, true, true, 0) + *stat -= uint64(s.npages << _PageShift) + memstats.heap_sys += uint64(s.npages << _PageShift) + h.freeSpanLocked(s, false, true, 0) unlock(&h.lock) } // s must be on a busy list (h.busy or h.busylarge) or unlinked. func (h *mheap) freeSpanLocked(s *mspan, acctinuse, acctidle bool, unusedsince int64) { switch s.state { - case _MSpanStack: + case _MSpanManual: if s.allocCount != 0 { throw("MHeap_FreeSpanLocked - invalid stack free") } @@ -900,50 +1021,98 @@ func (h *mheap) freeSpanLocked(s *mspan, acctinuse, acctidle bool, unusedsince i // Coalesce with earlier, later spans. p := (s.base() - h.arena_start) >> _PageShift if p > 0 { - t := h.spans[p-1] - if t != nil && t.state == _MSpanFree { - s.startAddr = t.startAddr - s.npages += t.npages - s.npreleased = t.npreleased // absorb released pages - s.needzero |= t.needzero - p -= t.npages + before := h.spans[p-1] + if before != nil && before.state == _MSpanFree { + // Now adjust s. + s.startAddr = before.startAddr + s.npages += before.npages + s.npreleased = before.npreleased // absorb released pages + s.needzero |= before.needzero + p -= before.npages h.spans[p] = s - h.freeList(t.npages).remove(t) - t.state = _MSpanDead - h.spanalloc.free(unsafe.Pointer(t)) + // The size is potentially changing so the treap needs to delete adjacent nodes and + // insert back as a combined node. + if h.isLargeSpan(before.npages) { + // We have a t, it is large so it has to be in the treap so we can remove it. + h.freelarge.removeSpan(before) + } else { + h.freeList(before.npages).remove(before) + } + before.state = _MSpanDead + h.spanalloc.free(unsafe.Pointer(before)) } } + + // Now check to see if next (greater addresses) span is free and can be coalesced. if (p + s.npages) < uintptr(len(h.spans)) { - t := h.spans[p+s.npages] - if t != nil && t.state == _MSpanFree { - s.npages += t.npages - s.npreleased += t.npreleased - s.needzero |= t.needzero + after := h.spans[p+s.npages] + if after != nil && after.state == _MSpanFree { + s.npages += after.npages + s.npreleased += after.npreleased + s.needzero |= after.needzero h.spans[p+s.npages-1] = s - h.freeList(t.npages).remove(t) - t.state = _MSpanDead - h.spanalloc.free(unsafe.Pointer(t)) + if h.isLargeSpan(after.npages) { + h.freelarge.removeSpan(after) + } else { + h.freeList(after.npages).remove(after) + } + after.state = _MSpanDead + h.spanalloc.free(unsafe.Pointer(after)) } } - // Insert s into appropriate list. - h.freeList(s.npages).insert(s) + // Insert s into appropriate list or treap. + if h.isLargeSpan(s.npages) { + h.freelarge.insert(s) + } else { + h.freeList(s.npages).insert(s) + } } func (h *mheap) freeList(npages uintptr) *mSpanList { - if npages < uintptr(len(h.free)) { - return &h.free[npages] - } - return &h.freelarge + return &h.free[npages] } func (h *mheap) busyList(npages uintptr) *mSpanList { - if npages < uintptr(len(h.free)) { + if npages < uintptr(len(h.busy)) { return &h.busy[npages] } return &h.busylarge } +func scavengeTreapNode(t *treapNode, now, limit uint64) uintptr { + s := t.spanKey + var sumreleased uintptr + if (now-uint64(s.unusedsince)) > limit && s.npreleased != s.npages { + start := s.base() + end := start + s.npages<<_PageShift + if physPageSize > _PageSize { + // We can only release pages in + // physPageSize blocks, so round start + // and end in. (Otherwise, madvise + // will round them *out* and release + // more memory than we want.) + start = (start + physPageSize - 1) &^ (physPageSize - 1) + end &^= physPageSize - 1 + if end <= start { + // start and end don't span a + // whole physical page. + return sumreleased + } + } + len := end - start + released := len - (s.npreleased << _PageShift) + if physPageSize > _PageSize && released == 0 { + return sumreleased + } + memstats.heap_released += uint64(released) + sumreleased += released + s.npreleased = len >> _PageShift + sysUnused(unsafe.Pointer(start), len) + } + return sumreleased +} + func scavengelist(list *mSpanList, now, limit uint64) uintptr { if list.isEmpty() { return 0 @@ -984,27 +1153,31 @@ func scavengelist(list *mSpanList, now, limit uint64) uintptr { } func (h *mheap) scavenge(k int32, now, limit uint64) { + // Disallow malloc or panic while holding the heap lock. We do + // this here because this is an non-mallocgc entry-point to + // the mheap API. + gp := getg() + gp.m.mallocing++ lock(&h.lock) var sumreleased uintptr for i := 0; i < len(h.free); i++ { sumreleased += scavengelist(&h.free[i], now, limit) } - sumreleased += scavengelist(&h.freelarge, now, limit) + sumreleased += scavengetreap(h.freelarge.treap, now, limit) unlock(&h.lock) + gp.m.mallocing-- if debug.gctrace > 0 { if sumreleased > 0 { print("scvg", k, ": ", sumreleased>>20, " MB released\n") } - // TODO(dvyukov): these stats are incorrect as we don't subtract stack usage from heap. - // But we can't call ReadMemStats on g0 holding locks. print("scvg", k, ": inuse: ", memstats.heap_inuse>>20, ", idle: ", memstats.heap_idle>>20, ", sys: ", memstats.heap_sys>>20, ", released: ", memstats.heap_released>>20, ", consumed: ", (memstats.heap_sys-memstats.heap_released)>>20, " (MB)\n") } } //go:linkname runtime_debug_freeOSMemory runtime_debug.freeOSMemory func runtime_debug_freeOSMemory() { - gcStart(gcForceBlockMode, false) + GC() systemstack(func() { mheap_.scavenge(-1, ^uint64(0), 0) }) } @@ -1017,7 +1190,7 @@ func (span *mspan) init(base uintptr, npages uintptr) { span.startAddr = base span.npages = npages span.allocCount = 0 - span.sizeclass = 0 + span.spanclass = 0 span.incache = false span.elemsize = 0 span.state = _MSpanDead @@ -1043,7 +1216,8 @@ func (list *mSpanList) init() { func (list *mSpanList) remove(span *mspan) { if span.list != list { - println("runtime: failed MSpanList_Remove", span, span.prev, span.list, list) + print("runtime: failed MSpanList_Remove span.npages=", span.npages, + " span=", span, " prev=", span.prev, " span.list=", span.list, " list=", list, "\n") throw("MSpanList_Remove") } if list.first == span { @@ -1085,7 +1259,7 @@ func (list *mSpanList) insert(span *mspan) { func (list *mSpanList) insertBack(span *mspan) { if span.next != nil || span.prev != nil || span.list != nil { - println("failed MSpanList_InsertBack", span, span.next, span.prev, span.list) + println("runtime: failed MSpanList_InsertBack", span, span.next, span.prev, span.list) throw("MSpanList_InsertBack") } span.prev = list.last @@ -1100,6 +1274,31 @@ func (list *mSpanList) insertBack(span *mspan) { span.list = list } +// takeAll removes all spans from other and inserts them at the front +// of list. +func (list *mSpanList) takeAll(other *mSpanList) { + if other.isEmpty() { + return + } + + // Reparent everything in other to list. + for s := other.first; s != nil; s = s.next { + s.list = list + } + + // Concatenate the lists. + if list.isEmpty() { + *list = *other + } else { + // Neither list is empty. Put other before list. + other.last.next = list.first + list.first.prev = other.last + list.first = other.first + } + + other.first, other.last = nil, nil +} + const ( _KindSpecialFinalizer = 1 _KindSpecialProfile = 2 @@ -1311,6 +1510,22 @@ func freespecial(s *special, p unsafe.Pointer, size uintptr) { } } +// gcBits is an alloc/mark bitmap. This is always used as *gcBits. +// +//go:notinheap +type gcBits uint8 + +// bytep returns a pointer to the n'th byte of b. +func (b *gcBits) bytep(n uintptr) *uint8 { + return addb((*uint8)(b), n) +} + +// bitp returns a pointer to the byte containing bit n and a mask for +// selecting that bit from *bytep. +func (b *gcBits) bitp(n uintptr) (bytep *uint8, mask uint8) { + return b.bytep(n / 8), 1 << (n % 8) +} + const gcBitsChunkBytes = uintptr(64 << 10) const gcBitsHeaderBytes = unsafe.Sizeof(gcBitsHeader{}) @@ -1320,42 +1535,87 @@ type gcBitsHeader struct { } //go:notinheap -type gcBits struct { +type gcBitsArena struct { // gcBitsHeader // side step recursive type bug (issue 14620) by including fields by hand. - free uintptr // free is the index into bits of the next free byte. - next *gcBits - bits [gcBitsChunkBytes - gcBitsHeaderBytes]uint8 + free uintptr // free is the index into bits of the next free byte; read/write atomically + next *gcBitsArena + bits [gcBitsChunkBytes - gcBitsHeaderBytes]gcBits } var gcBitsArenas struct { lock mutex - free *gcBits - next *gcBits - current *gcBits - previous *gcBits + free *gcBitsArena + next *gcBitsArena // Read atomically. Write atomically under lock. + current *gcBitsArena + previous *gcBitsArena +} + +// tryAlloc allocates from b or returns nil if b does not have enough room. +// This is safe to call concurrently. +func (b *gcBitsArena) tryAlloc(bytes uintptr) *gcBits { + if b == nil || atomic.Loaduintptr(&b.free)+bytes > uintptr(len(b.bits)) { + return nil + } + // Try to allocate from this block. + end := atomic.Xadduintptr(&b.free, bytes) + if end > uintptr(len(b.bits)) { + return nil + } + // There was enough room. + start := end - bytes + return &b.bits[start] } // newMarkBits returns a pointer to 8 byte aligned bytes // to be used for a span's mark bits. -func newMarkBits(nelems uintptr) *uint8 { - lock(&gcBitsArenas.lock) +func newMarkBits(nelems uintptr) *gcBits { blocksNeeded := uintptr((nelems + 63) / 64) bytesNeeded := blocksNeeded * 8 - if gcBitsArenas.next == nil || - gcBitsArenas.next.free+bytesNeeded > uintptr(len(gcBits{}.bits)) { - // Allocate a new arena. - fresh := newArena() - fresh.next = gcBitsArenas.next - gcBitsArenas.next = fresh - } - if gcBitsArenas.next.free >= gcBitsChunkBytes { - println("runtime: gcBitsArenas.next.free=", gcBitsArenas.next.free, gcBitsChunkBytes) + + // Try directly allocating from the current head arena. + head := (*gcBitsArena)(atomic.Loadp(unsafe.Pointer(&gcBitsArenas.next))) + if p := head.tryAlloc(bytesNeeded); p != nil { + return p + } + + // There's not enough room in the head arena. We may need to + // allocate a new arena. + lock(&gcBitsArenas.lock) + // Try the head arena again, since it may have changed. Now + // that we hold the lock, the list head can't change, but its + // free position still can. + if p := gcBitsArenas.next.tryAlloc(bytesNeeded); p != nil { + unlock(&gcBitsArenas.lock) + return p + } + + // Allocate a new arena. This may temporarily drop the lock. + fresh := newArenaMayUnlock() + // If newArenaMayUnlock dropped the lock, another thread may + // have put a fresh arena on the "next" list. Try allocating + // from next again. + if p := gcBitsArenas.next.tryAlloc(bytesNeeded); p != nil { + // Put fresh back on the free list. + // TODO: Mark it "already zeroed" + fresh.next = gcBitsArenas.free + gcBitsArenas.free = fresh + unlock(&gcBitsArenas.lock) + return p + } + + // Allocate from the fresh arena. We haven't linked it in yet, so + // this cannot race and is guaranteed to succeed. + p := fresh.tryAlloc(bytesNeeded) + if p == nil { throw("markBits overflow") } - result := &gcBitsArenas.next.bits[gcBitsArenas.next.free] - gcBitsArenas.next.free += bytesNeeded + + // Add the fresh arena to the "next" list. + fresh.next = gcBitsArenas.next + atomic.StorepNoWB(unsafe.Pointer(&gcBitsArenas.next), unsafe.Pointer(fresh)) + unlock(&gcBitsArenas.lock) - return result + return p } // newAllocBits returns a pointer to 8 byte aligned bytes @@ -1364,7 +1624,7 @@ func newMarkBits(nelems uintptr) *uint8 { // allocation bits. For spans not being initialized the // the mark bits are repurposed as allocation bits when // the span is swept. -func newAllocBits(nelems uintptr) *uint8 { +func newAllocBits(nelems uintptr) *gcBits { return newMarkBits(nelems) } @@ -1398,18 +1658,21 @@ func nextMarkBitArenaEpoch() { } gcBitsArenas.previous = gcBitsArenas.current gcBitsArenas.current = gcBitsArenas.next - gcBitsArenas.next = nil // newMarkBits calls newArena when needed + atomic.StorepNoWB(unsafe.Pointer(&gcBitsArenas.next), nil) // newMarkBits calls newArena when needed unlock(&gcBitsArenas.lock) } -// newArena allocates and zeroes a gcBits arena. -func newArena() *gcBits { - var result *gcBits +// newArenaMayUnlock allocates and zeroes a gcBits arena. +// The caller must hold gcBitsArena.lock. This may temporarily release it. +func newArenaMayUnlock() *gcBitsArena { + var result *gcBitsArena if gcBitsArenas.free == nil { - result = (*gcBits)(sysAlloc(gcBitsChunkBytes, &memstats.gc_sys)) + unlock(&gcBitsArenas.lock) + result = (*gcBitsArena)(sysAlloc(gcBitsChunkBytes, &memstats.gc_sys)) if result == nil { throw("runtime: cannot allocate memory") } + lock(&gcBitsArenas.lock) } else { result = gcBitsArenas.free gcBitsArenas.free = gcBitsArenas.free.next @@ -1418,7 +1681,7 @@ func newArena() *gcBits { result.next = nil // If result.bits is not 8 byte aligned adjust index so // that &result.bits[result.free] is 8 byte aligned. - if uintptr(unsafe.Offsetof(gcBits{}.bits))&7 == 0 { + if uintptr(unsafe.Offsetof(gcBitsArena{}.bits))&7 == 0 { result.free = 0 } else { result.free = 8 - (uintptr(unsafe.Pointer(&result.bits[0])) & 7) diff --git a/libgo/go/runtime/mksizeclasses.go b/libgo/go/runtime/mksizeclasses.go index 0f897ba8e69..0cb2b33a8cd 100644 --- a/libgo/go/runtime/mksizeclasses.go +++ b/libgo/go/runtime/mksizeclasses.go @@ -48,7 +48,7 @@ func main() { flag.Parse() var b bytes.Buffer - fmt.Fprintln(&b, "// AUTO-GENERATED by mksizeclasses.go; DO NOT EDIT") + fmt.Fprintln(&b, "// Code generated by mksizeclasses.go; DO NOT EDIT.") fmt.Fprintln(&b, "//go:generate go run mksizeclasses.go") fmt.Fprintln(&b) fmt.Fprintln(&b, "package runtime") diff --git a/libgo/go/runtime/mprof.go b/libgo/go/runtime/mprof.go index 87f84a72acd..f31c88c5cae 100644 --- a/libgo/go/runtime/mprof.go +++ b/libgo/go/runtime/mprof.go @@ -64,27 +64,70 @@ type memRecord struct { // come only after a GC during concurrent sweeping. So if we would // naively count them, we would get a skew toward mallocs. // - // Mallocs are accounted in recent stats. - // Explicit frees are accounted in recent stats. - // GC frees are accounted in prev stats. - // After GC prev stats are added to final stats and - // recent stats are moved into prev stats. - allocs uintptr - frees uintptr - alloc_bytes uintptr - free_bytes uintptr - - // changes between next-to-last GC and last GC - prev_allocs uintptr - prev_frees uintptr - prev_alloc_bytes uintptr - prev_free_bytes uintptr - - // changes since last GC - recent_allocs uintptr - recent_frees uintptr - recent_alloc_bytes uintptr - recent_free_bytes uintptr + // Hence, we delay information to get consistent snapshots as + // of mark termination. Allocations count toward the next mark + // termination's snapshot, while sweep frees count toward the + // previous mark termination's snapshot: + // + // MT MT MT MT + // .·| .·| .·| .·| + // .·˙ | .·˙ | .·˙ | .·˙ | + // .·˙ | .·˙ | .·˙ | .·˙ | + // .·˙ |.·˙ |.·˙ |.·˙ | + // + // alloc → ▲ ← free + // ┠┅┅┅┅┅┅┅┅┅┅┅P + // C+2 → C+1 → C + // + // alloc → ▲ ← free + // ┠┅┅┅┅┅┅┅┅┅┅┅P + // C+2 → C+1 → C + // + // Since we can't publish a consistent snapshot until all of + // the sweep frees are accounted for, we wait until the next + // mark termination ("MT" above) to publish the previous mark + // termination's snapshot ("P" above). To do this, allocation + // and free events are accounted to *future* heap profile + // cycles ("C+n" above) and we only publish a cycle once all + // of the events from that cycle must be done. Specifically: + // + // Mallocs are accounted to cycle C+2. + // Explicit frees are accounted to cycle C+2. + // GC frees (done during sweeping) are accounted to cycle C+1. + // + // After mark termination, we increment the global heap + // profile cycle counter and accumulate the stats from cycle C + // into the active profile. + + // active is the currently published profile. A profiling + // cycle can be accumulated into active once its complete. + active memRecordCycle + + // future records the profile events we're counting for cycles + // that have not yet been published. This is ring buffer + // indexed by the global heap profile cycle C and stores + // cycles C, C+1, and C+2. Unlike active, these counts are + // only for a single cycle; they are not cumulative across + // cycles. + // + // We store cycle C here because there's a window between when + // C becomes the active cycle and when we've flushed it to + // active. + future [3]memRecordCycle +} + +// memRecordCycle +type memRecordCycle struct { + allocs, frees uintptr + alloc_bytes, free_bytes uintptr +} + +// add accumulates b into a. It does not zero b. +func (a *memRecordCycle) add(b *memRecordCycle) { + a.allocs += b.allocs + a.frees += b.frees + a.alloc_bytes += b.alloc_bytes + a.free_bytes += b.free_bytes } // A blockRecord is the bucket data for a bucket of type blockProfile, @@ -100,8 +143,21 @@ var ( xbuckets *bucket // mutex profile buckets buckhash *[179999]*bucket bucketmem uintptr + + mProf struct { + // All fields in mProf are protected by proflock. + + // cycle is the global heap profile cycle. This wraps + // at mProfCycleWrap. + cycle uint32 + // flushed indicates that future[cycle] in all buckets + // has been flushed to the active profile. + flushed bool + } ) +const mProfCycleWrap = uint32(len(memRecord{}.future)) * (2 << 24) + // newBucket allocates a bucket with the given type and number of stack entries. func newBucket(typ bucketType, nstk int) *bucket { size := unsafe.Sizeof(bucket{}) + uintptr(nstk)*unsafe.Sizeof(location{}) @@ -212,30 +268,71 @@ func eqslice(x, y []location) bool { return true } -func mprof_GC() { +// mProf_NextCycle publishes the next heap profile cycle and creates a +// fresh heap profile cycle. This operation is fast and can be done +// during STW. The caller must call mProf_Flush before calling +// mProf_NextCycle again. +// +// This is called by mark termination during STW so allocations and +// frees after the world is started again count towards a new heap +// profiling cycle. +func mProf_NextCycle() { + lock(&proflock) + // We explicitly wrap mProf.cycle rather than depending on + // uint wraparound because the memRecord.future ring does not + // itself wrap at a power of two. + mProf.cycle = (mProf.cycle + 1) % mProfCycleWrap + mProf.flushed = false + unlock(&proflock) +} + +// mProf_Flush flushes the events from the current heap profiling +// cycle into the active profile. After this it is safe to start a new +// heap profiling cycle with mProf_NextCycle. +// +// This is called by GC after mark termination starts the world. In +// contrast with mProf_NextCycle, this is somewhat expensive, but safe +// to do concurrently. +func mProf_Flush() { + lock(&proflock) + if !mProf.flushed { + mProf_FlushLocked() + mProf.flushed = true + } + unlock(&proflock) +} + +func mProf_FlushLocked() { + c := mProf.cycle for b := mbuckets; b != nil; b = b.allnext { mp := b.mp() - mp.allocs += mp.prev_allocs - mp.frees += mp.prev_frees - mp.alloc_bytes += mp.prev_alloc_bytes - mp.free_bytes += mp.prev_free_bytes - mp.prev_allocs = mp.recent_allocs - mp.prev_frees = mp.recent_frees - mp.prev_alloc_bytes = mp.recent_alloc_bytes - mp.prev_free_bytes = mp.recent_free_bytes - - mp.recent_allocs = 0 - mp.recent_frees = 0 - mp.recent_alloc_bytes = 0 - mp.recent_free_bytes = 0 + // Flush cycle C into the published profile and clear + // it for reuse. + mpc := &mp.future[c%uint32(len(mp.future))] + mp.active.add(mpc) + *mpc = memRecordCycle{} } } -// Record that a gc just happened: all the 'recent' statistics are now real. -func mProf_GC() { +// mProf_PostSweep records that all sweep frees for this GC cycle have +// completed. This has the effect of publishing the heap profile +// snapshot as of the last mark termination without advancing the heap +// profile cycle. +func mProf_PostSweep() { lock(&proflock) - mprof_GC() + // Flush cycle C+1 to the active profile so everything as of + // the last mark termination becomes visible. *Don't* advance + // the cycle, since we're still accumulating allocs in cycle + // C+2, which have to become C+1 in the next mark termination + // and so on. + c := mProf.cycle + for b := mbuckets; b != nil; b = b.allnext { + mp := b.mp() + mpc := &mp.future[(c+1)%uint32(len(mp.future))] + mp.active.add(mpc) + *mpc = memRecordCycle{} + } unlock(&proflock) } @@ -245,9 +342,11 @@ func mProf_Malloc(p unsafe.Pointer, size uintptr) { nstk := callers(4, stk[:]) lock(&proflock) b := stkbucket(memProfile, size, stk[:nstk], true) + c := mProf.cycle mp := b.mp() - mp.recent_allocs++ - mp.recent_alloc_bytes += size + mpc := &mp.future[(c+2)%uint32(len(mp.future))] + mpc.allocs++ + mpc.alloc_bytes += size unlock(&proflock) // Setprofilebucket locks a bunch of other mutexes, so we call it outside of proflock. @@ -262,9 +361,11 @@ func mProf_Malloc(p unsafe.Pointer, size uintptr) { // Called when freeing a profiled block. func mProf_Free(b *bucket, size uintptr) { lock(&proflock) + c := mProf.cycle mp := b.mp() - mp.prev_frees++ - mp.prev_free_bytes += size + mpc := &mp.future[(c+1)%uint32(len(mp.future))] + mpc.frees++ + mpc.free_bytes += size unlock(&proflock) } @@ -298,7 +399,7 @@ func blockevent(cycles int64, skip int) { cycles = 1 } if blocksampled(cycles) { - saveblockevent(cycles, skip+1, blockProfile, &blockprofilerate) + saveblockevent(cycles, skip+1, blockProfile) } } @@ -310,7 +411,7 @@ func blocksampled(cycles int64) bool { return true } -func saveblockevent(cycles int64, skip int, which bucketType, ratep *uint64) { +func saveblockevent(cycles int64, skip int, which bucketType) { gp := getg() var nstk int var stk [maxStack]location @@ -355,7 +456,7 @@ func mutexevent(cycles int64, skip int) { // TODO(pjw): measure impact of always calling fastrand vs using something // like malloc.go:nextSample() if rate > 0 && int64(fastrand())%rate == 0 { - saveblockevent(cycles, skip+1, mutexProfile, &mutexprofilerate) + saveblockevent(cycles, skip+1, mutexProfile) } } @@ -443,13 +544,17 @@ func (r *MemProfileRecord) Stack() []uintptr { // of calling MemProfile directly. func MemProfile(p []MemProfileRecord, inuseZero bool) (n int, ok bool) { lock(&proflock) + // If we're between mProf_NextCycle and mProf_Flush, take care + // of flushing to the active profile so we only have to look + // at the active profile below. + mProf_FlushLocked() clear := true for b := mbuckets; b != nil; b = b.allnext { mp := b.mp() - if inuseZero || mp.alloc_bytes != mp.free_bytes { + if inuseZero || mp.active.alloc_bytes != mp.active.free_bytes { n++ } - if mp.allocs != 0 || mp.frees != 0 { + if mp.active.allocs != 0 || mp.active.frees != 0 { clear = false } } @@ -457,13 +562,15 @@ func MemProfile(p []MemProfileRecord, inuseZero bool) (n int, ok bool) { // Absolutely no data, suggesting that a garbage collection // has not yet happened. In order to allow profiling when // garbage collection is disabled from the beginning of execution, - // accumulate stats as if a GC just happened, and recount buckets. - mprof_GC() - mprof_GC() + // accumulate all of the cycles, and recount buckets. n = 0 for b := mbuckets; b != nil; b = b.allnext { mp := b.mp() - if inuseZero || mp.alloc_bytes != mp.free_bytes { + for c := range mp.future { + mp.active.add(&mp.future[c]) + mp.future[c] = memRecordCycle{} + } + if inuseZero || mp.active.alloc_bytes != mp.active.free_bytes { n++ } } @@ -473,7 +580,7 @@ func MemProfile(p []MemProfileRecord, inuseZero bool) (n int, ok bool) { idx := 0 for b := mbuckets; b != nil; b = b.allnext { mp := b.mp() - if inuseZero || mp.alloc_bytes != mp.free_bytes { + if inuseZero || mp.active.alloc_bytes != mp.active.free_bytes { record(&p[idx], b) idx++ } @@ -486,10 +593,10 @@ func MemProfile(p []MemProfileRecord, inuseZero bool) (n int, ok bool) { // Write b's data to r. func record(r *MemProfileRecord, b *bucket) { mp := b.mp() - r.AllocBytes = int64(mp.alloc_bytes) - r.FreeBytes = int64(mp.free_bytes) - r.AllocObjects = int64(mp.allocs) - r.FreeObjects = int64(mp.frees) + r.AllocBytes = int64(mp.active.alloc_bytes) + r.FreeBytes = int64(mp.active.free_bytes) + r.AllocObjects = int64(mp.active.allocs) + r.FreeObjects = int64(mp.active.frees) for i, loc := range b.stk() { if i >= len(r.Stack0) { break @@ -505,7 +612,7 @@ func iterate_memprof(fn func(*bucket, uintptr, *location, uintptr, uintptr, uint lock(&proflock) for b := mbuckets; b != nil; b = b.allnext { mp := b.mp() - fn(b, b.nstk, &b.stk()[0], b.size, mp.allocs, mp.frees) + fn(b, b.nstk, &b.stk()[0], b.size, mp.active.allocs, mp.active.frees) } unlock(&proflock) } diff --git a/libgo/go/runtime/msize.go b/libgo/go/runtime/msize.go index 438c9875135..0accb83eb89 100644 --- a/libgo/go/runtime/msize.go +++ b/libgo/go/runtime/msize.go @@ -9,28 +9,6 @@ package runtime -// sizeToClass(0 <= n <= MaxSmallSize) returns the size class, -// 1 <= sizeclass < NumSizeClasses, for n. -// Size class 0 is reserved to mean "not small". -// -// The sizeToClass lookup is implemented using two arrays, -// one mapping sizes <= 1024 to their class and one mapping -// sizes >= 1024 and <= MaxSmallSize to their class. -// All objects are 8-aligned, so the first array is indexed by -// the size divided by 8 (rounded up). Objects >= 1024 bytes -// are 128-aligned, so the second array is indexed by the -// size divided by 128 (rounded up). The arrays are constants -// in sizeclass.go generated by mksizeclass.go. -func sizeToClass(size uint32) uint32 { - if size > _MaxSmallSize { - throw("invalid size") - } - if size > smallSizeMax-8 { - return uint32(size_to_class128[(size-smallSizeMax+largeSizeDiv-1)/largeSizeDiv]) - } - return uint32(size_to_class8[(size+smallSizeDiv-1)/smallSizeDiv]) -} - // Returns size of the memory block that mallocgc will allocate if you ask for the size. func roundupsize(size uintptr) uintptr { if size < _MaxSmallSize { diff --git a/libgo/go/runtime/mstats.go b/libgo/go/runtime/mstats.go index aa3cfef0e10..71dc2239854 100644 --- a/libgo/go/runtime/mstats.go +++ b/libgo/go/runtime/mstats.go @@ -33,13 +33,12 @@ type mstats struct { // Statistics about malloc heap. // Protected by mheap.lock // - // In mstats, heap_sys and heap_inuse includes stack memory, - // while in MemStats stack memory is separated out from the - // heap stats. + // Like MemStats, heap_sys and heap_inuse do not count memory + // in manually-managed spans. heap_alloc uint64 // bytes allocated and not yet freed (same as alloc above) - heap_sys uint64 // virtual address space obtained from system + heap_sys uint64 // virtual address space obtained from system for GC'd heap heap_idle uint64 // bytes in idle spans - heap_inuse uint64 // bytes in non-idle spans + heap_inuse uint64 // bytes in _MSpanInUse spans heap_released uint64 // bytes released to the os heap_objects uint64 // total number of allocated objects @@ -59,7 +58,7 @@ type mstats struct { // Statistics about allocation of low-level fixed-size structures. // Protected by FixAlloc locks. - stacks_inuse uint64 // this number is included in heap_inuse above; differs from MemStats.StackInuse + stacks_inuse uint64 // bytes in manually-managed stack spans stacks_sys uint64 // only counts newosproc0 stack in mstats; differs from MemStats.StackSys mspan_inuse uint64 // mspan structures mspan_sys uint64 @@ -72,7 +71,7 @@ type mstats struct { // Statistics about garbage collector. // Protected by mheap or stopping the world during GC. next_gc uint64 // goal heap_live for when next GC ends; ^0 if disabled - last_gc uint64 // last gc (in absolute time) + last_gc_unix uint64 // last gc (in unix time) pause_total_ns uint64 pause_ns [256]uint64 // circular buffer of recent gc pause lengths pause_end [256]uint64 // circular buffer of recent gc end times (nanoseconds since 1970) @@ -92,13 +91,26 @@ type mstats struct { // Statistics below here are not exported to MemStats directly. - tinyallocs uint64 // number of tiny allocations that didn't cause actual allocation; not exported to go directly + last_gc_nanotime uint64 // last gc (monotonic time) + tinyallocs uint64 // number of tiny allocations that didn't cause actual allocation; not exported to go directly + + // triggerRatio is the heap growth ratio that triggers marking. + // + // E.g., if this is 0.6, then GC should start when the live + // heap has reached 1.6 times the heap size marked by the + // previous cycle. This should be ≤ GOGC/100 so the trigger + // heap size is less than the goal heap size. This is set + // during mark termination for the next cycle's trigger. + triggerRatio float64 // gc_trigger is the heap size that triggers marking. // // When heap_live ≥ gc_trigger, the mark phase will start. // This is also the heap size by which proportional sweeping // must be complete. + // + // This is computed from triggerRatio during mark termination + // for the next cycle's trigger. gc_trigger uint64 // heap_live is the number of bytes considered live by the GC. @@ -121,6 +133,8 @@ type mstats struct { // leads to a conservative GC rate rather than a GC rate that // is potentially too low. // + // Reads should likewise be atomic (or during STW). + // // Whenever this is updated, call traceHeapAlloc() and // gcController.revise(). heap_live uint64 @@ -451,22 +465,18 @@ func ReadMemStats(m *MemStats) { } func readmemstats_m(stats *MemStats) { - updatememstats(nil) + updatememstats() // The size of the trailing by_size array differs between // mstats and MemStats. NumSizeClasses was changed, but we // cannot change MemStats because of backward compatibility. memmove(unsafe.Pointer(stats), unsafe.Pointer(&memstats), sizeof_C_MStats) - // Stack numbers are part of the heap numbers, separate those out for user consumption + // memstats.stacks_sys is only memory mapped directly for OS stacks. + // Add in heap-allocated stack memory for user consumption. stats.StackSys += stats.StackInuse - stats.HeapInuse -= stats.StackInuse - stats.HeapSys -= stats.StackInuse } -// For gccgo this is in runtime/mgc0.c. -func updatememstats(stats *gcstats) - //go:linkname readGCStats runtime_debug.readGCStats func readGCStats(pauses *[]uint64) { systemstack(func() { @@ -500,7 +510,7 @@ func readGCStats_m(pauses *[]uint64) { p[n+i] = memstats.pause_end[j] } - p[n+n] = memstats.last_gc + p[n+n] = memstats.last_gc_unix p[n+n+1] = uint64(memstats.numgc) p[n+n+2] = memstats.pause_total_ns unlock(&mheap_.lock) @@ -508,26 +518,15 @@ func readGCStats_m(pauses *[]uint64) { } //go:nowritebarrier -func updatememstats(stats *gcstats) { - if stats != nil { - *stats = gcstats{} - } - for mp := allm; mp != nil; mp = mp.alllink { - if stats != nil { - src := (*[unsafe.Sizeof(gcstats{}) / 8]uint64)(unsafe.Pointer(&mp.gcstats)) - dst := (*[unsafe.Sizeof(gcstats{}) / 8]uint64)(unsafe.Pointer(stats)) - for i, v := range src { - dst[i] += v - } - mp.gcstats = gcstats{} - } - } - +func updatememstats() { memstats.mcache_inuse = uint64(mheap_.cachealloc.inuse) memstats.mspan_inuse = uint64(mheap_.spanalloc.inuse) memstats.sys = memstats.heap_sys + memstats.stacks_sys + memstats.mspan_sys + memstats.mcache_sys + memstats.buckhash_sys + memstats.gc_sys + memstats.other_sys + // We also count stacks_inuse as sys memory. + memstats.sys += memstats.stacks_inuse + // Calculate memory allocator stats. // During program execution we only count number of frees and amount of freed memory. // Current number of alive object in the heap and amount of alive heap memory @@ -550,45 +549,49 @@ func updatememstats(stats *gcstats) { // Aggregate local stats. cachestats() - // Scan all spans and count number of alive objects. - lock(&mheap_.lock) - for _, s := range mheap_.allspans { - if s.state != mSpanInUse { + // Collect allocation stats. This is safe and consistent + // because the world is stopped. + var smallFree, totalAlloc, totalFree uint64 + // Collect per-spanclass stats. + for spc := range mheap_.central { + // The mcaches are now empty, so mcentral stats are + // up-to-date. + c := &mheap_.central[spc].mcentral + memstats.nmalloc += c.nmalloc + i := spanClass(spc).sizeclass() + memstats.by_size[i].nmalloc += c.nmalloc + totalAlloc += c.nmalloc * uint64(class_to_size[i]) + } + // Collect per-sizeclass stats. + for i := 0; i < _NumSizeClasses; i++ { + if i == 0 { + memstats.nmalloc += mheap_.nlargealloc + totalAlloc += mheap_.largealloc + totalFree += mheap_.largefree + memstats.nfree += mheap_.nlargefree continue } - if s.sizeclass == 0 { - memstats.nmalloc++ - memstats.alloc += uint64(s.elemsize) - } else { - memstats.nmalloc += uint64(s.allocCount) - memstats.by_size[s.sizeclass].nmalloc += uint64(s.allocCount) - memstats.alloc += uint64(s.allocCount) * uint64(s.elemsize) - } - } - unlock(&mheap_.lock) - // Aggregate by size class. - smallfree := uint64(0) - memstats.nfree = mheap_.nlargefree - for i := 0; i < len(memstats.by_size); i++ { + // The mcache stats have been flushed to mheap_. memstats.nfree += mheap_.nsmallfree[i] memstats.by_size[i].nfree = mheap_.nsmallfree[i] - memstats.by_size[i].nmalloc += mheap_.nsmallfree[i] - smallfree += mheap_.nsmallfree[i] * uint64(class_to_size[i]) + smallFree += mheap_.nsmallfree[i] * uint64(class_to_size[i]) } + totalFree += smallFree + memstats.nfree += memstats.tinyallocs - memstats.nmalloc += memstats.nfree + memstats.nmalloc += memstats.tinyallocs // Calculate derived stats. - memstats.total_alloc = memstats.alloc + mheap_.largefree + smallfree + memstats.total_alloc = totalAlloc + memstats.alloc = totalAlloc - totalFree memstats.heap_alloc = memstats.alloc memstats.heap_objects = memstats.nmalloc - memstats.nfree } //go:nowritebarrier func cachestats() { - for i := 0; ; i++ { - p := allp[i] + for _, p := range &allp { if p == nil { break } diff --git a/libgo/go/runtime/mstkbar.go b/libgo/go/runtime/mstkbar.go deleted file mode 100644 index 616c220132f..00000000000 --- a/libgo/go/runtime/mstkbar.go +++ /dev/null @@ -1,395 +0,0 @@ -// Copyright 2015 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// +build ignore - -// Garbage collector: stack barriers -// -// Stack barriers enable the garbage collector to determine how much -// of a gorountine stack has changed between when a stack is scanned -// during the concurrent scan phase and when it is re-scanned during -// the stop-the-world mark termination phase. Mark termination only -// needs to re-scan the changed part, so for deep stacks this can -// significantly reduce GC pause time compared to the alternative of -// re-scanning whole stacks. The deeper the stacks, the more stack -// barriers help. -// -// When stacks are scanned during the concurrent scan phase, the stack -// scan installs stack barriers by selecting stack frames and -// overwriting the saved return PCs (or link registers) of these -// frames with the PC of a "stack barrier trampoline". Later, when a -// selected frame returns, it "returns" to this trampoline instead of -// returning to its actual caller. The trampoline records that the -// stack has unwound past this frame and jumps to the original return -// PC recorded when the stack barrier was installed. Mark termination -// re-scans only as far as the first frame that hasn't hit a stack -// barrier and then removes and un-hit stack barriers. -// -// This scheme is very lightweight. No special code is required in the -// mutator to record stack unwinding and the trampoline is only a few -// assembly instructions. -// -// Book-keeping -// ------------ -// -// The primary cost of stack barriers is book-keeping: the runtime has -// to record the locations of all stack barriers and the original -// return PCs in order to return to the correct caller when a stack -// barrier is hit and so it can remove un-hit stack barriers. In order -// to minimize this cost, the Go runtime places stack barriers in -// exponentially-spaced frames, starting 1K past the current frame. -// The book-keeping structure hence grows logarithmically with the -// size of the stack and mark termination re-scans at most twice as -// much stack as necessary. -// -// The runtime reserves space for this book-keeping structure at the -// top of the stack allocation itself (just above the outermost -// frame). This is necessary because the regular memory allocator can -// itself grow the stack, and hence can't be used when allocating -// stack-related structures. -// -// For debugging, the runtime also supports installing stack barriers -// at every frame. However, this requires significantly more -// book-keeping space. -// -// Correctness -// ----------- -// -// The runtime and the compiler cooperate to ensure that all objects -// reachable from the stack as of mark termination are marked. -// Anything unchanged since the concurrent scan phase will be marked -// because it is marked by the concurrent scan. After the concurrent -// scan, there are three possible classes of stack modifications that -// must be tracked: -// -// 1) Mutator writes below the lowest un-hit stack barrier. This -// includes all writes performed by an executing function to its own -// stack frame. This part of the stack will be re-scanned by mark -// termination, which will mark any objects made reachable from -// modifications to this part of the stack. -// -// 2) Mutator writes above the lowest un-hit stack barrier. It's -// possible for a mutator to modify the stack above the lowest un-hit -// stack barrier if a higher frame has passed down a pointer to a -// stack variable in its frame. This is called an "up-pointer". The -// compiler ensures that writes through up-pointers have an -// accompanying write barrier (it simply doesn't distinguish between -// writes through up-pointers and writes through heap pointers). This -// write barrier marks any object made reachable from modifications to -// this part of the stack. -// -// 3) Runtime writes to the stack. Various runtime operations such as -// sends to unbuffered channels can write to arbitrary parts of the -// stack, including above the lowest un-hit stack barrier. We solve -// this in two ways. In many cases, the runtime can perform an -// explicit write barrier operation like in case 2. However, in the -// case of bulk memory move (typedmemmove), the runtime doesn't -// necessary have ready access to a pointer bitmap for the memory -// being copied, so it simply unwinds any stack barriers below the -// destination. -// -// Gotchas -// ------- -// -// Anything that inspects or manipulates the stack potentially needs -// to understand stack barriers. The most obvious case is that -// gentraceback needs to use the original return PC when it encounters -// the stack barrier trampoline. Anything that unwinds the stack such -// as panic/recover must unwind stack barriers in tandem with -// unwinding the stack. -// -// Stack barriers require that any goroutine whose stack has been -// scanned must execute write barriers. Go solves this by simply -// enabling write barriers globally during the concurrent scan phase. -// However, traditionally, write barriers are not enabled during this -// phase. -// -// Synchronization -// --------------- -// -// For the most part, accessing and modifying stack barriers is -// synchronized around GC safe points. Installing stack barriers -// forces the G to a safe point, while all other operations that -// modify stack barriers run on the G and prevent it from reaching a -// safe point. -// -// Subtlety arises when a G may be tracebacked when *not* at a safe -// point. This happens during sigprof. For this, each G has a "stack -// barrier lock" (see gcLockStackBarriers, gcUnlockStackBarriers). -// Operations that manipulate stack barriers acquire this lock, while -// sigprof tries to acquire it and simply skips the traceback if it -// can't acquire it. There is one exception for performance and -// complexity reasons: hitting a stack barrier manipulates the stack -// barrier list without acquiring the stack barrier lock. For this, -// gentraceback performs a special fix up if the traceback starts in -// the stack barrier function. - -package runtime - -import ( - "runtime/internal/atomic" - "runtime/internal/sys" - "unsafe" -) - -const debugStackBarrier = false - -// firstStackBarrierOffset is the approximate byte offset at -// which to place the first stack barrier from the current SP. -// This is a lower bound on how much stack will have to be -// re-scanned during mark termination. Subsequent barriers are -// placed at firstStackBarrierOffset * 2^n offsets. -// -// For debugging, this can be set to 0, which will install a -// stack barrier at every frame. If you do this, you may also -// have to raise _StackMin, since the stack barrier -// bookkeeping will use a large amount of each stack. -var firstStackBarrierOffset = 1024 - -// gcMaxStackBarriers returns the maximum number of stack barriers -// that can be installed in a stack of stackSize bytes. -func gcMaxStackBarriers(stackSize int) (n int) { - if debug.gcstackbarrieroff > 0 { - return 0 - } - - if firstStackBarrierOffset == 0 { - // Special debugging case for inserting stack barriers - // at every frame. Steal half of the stack for the - // []stkbar. Technically, if the stack were to consist - // solely of return PCs we would need two thirds of - // the stack, but stealing that much breaks things and - // this doesn't happen in practice. - return stackSize / 2 / int(unsafe.Sizeof(stkbar{})) - } - - offset := firstStackBarrierOffset - for offset < stackSize { - n++ - offset *= 2 - } - return n + 1 -} - -// gcInstallStackBarrier installs a stack barrier over the return PC of frame. -//go:nowritebarrier -func gcInstallStackBarrier(gp *g, frame *stkframe) bool { - if frame.lr == 0 { - if debugStackBarrier { - print("not installing stack barrier with no LR, goid=", gp.goid, "\n") - } - return false - } - - if frame.fn.entry == cgocallback_gofuncPC { - // cgocallback_gofunc doesn't return to its LR; - // instead, its return path puts LR in g.sched.pc and - // switches back to the system stack on which - // cgocallback_gofunc was originally called. We can't - // have a stack barrier in g.sched.pc, so don't - // install one in this frame. - if debugStackBarrier { - print("not installing stack barrier over LR of cgocallback_gofunc, goid=", gp.goid, "\n") - } - return false - } - - // Save the return PC and overwrite it with stackBarrier. - var lrUintptr uintptr - if usesLR { - lrUintptr = frame.sp - } else { - lrUintptr = frame.fp - sys.RegSize - } - lrPtr := (*sys.Uintreg)(unsafe.Pointer(lrUintptr)) - if debugStackBarrier { - print("install stack barrier at ", hex(lrUintptr), " over ", hex(*lrPtr), ", goid=", gp.goid, "\n") - if uintptr(*lrPtr) != frame.lr { - print("frame.lr=", hex(frame.lr)) - throw("frame.lr differs from stack LR") - } - } - - gp.stkbar = gp.stkbar[:len(gp.stkbar)+1] - stkbar := &gp.stkbar[len(gp.stkbar)-1] - stkbar.savedLRPtr = lrUintptr - stkbar.savedLRVal = uintptr(*lrPtr) - *lrPtr = sys.Uintreg(stackBarrierPC) - return true -} - -// gcRemoveStackBarriers removes all stack barriers installed in gp's stack. -// -// gp's stack barriers must be locked. -// -//go:nowritebarrier -func gcRemoveStackBarriers(gp *g) { - if debugStackBarrier && gp.stkbarPos != 0 { - print("hit ", gp.stkbarPos, " stack barriers, goid=", gp.goid, "\n") - } - - // Remove stack barriers that we didn't hit. - for _, stkbar := range gp.stkbar[gp.stkbarPos:] { - gcRemoveStackBarrier(gp, stkbar) - } - - // Clear recorded stack barriers so copystack doesn't try to - // adjust them. - gp.stkbarPos = 0 - gp.stkbar = gp.stkbar[:0] -} - -// gcRemoveStackBarrier removes a single stack barrier. It is the -// inverse operation of gcInstallStackBarrier. -// -// This is nosplit to ensure gp's stack does not move. -// -//go:nowritebarrier -//go:nosplit -func gcRemoveStackBarrier(gp *g, stkbar stkbar) { - if debugStackBarrier { - print("remove stack barrier at ", hex(stkbar.savedLRPtr), " with ", hex(stkbar.savedLRVal), ", goid=", gp.goid, "\n") - } - lrPtr := (*sys.Uintreg)(unsafe.Pointer(stkbar.savedLRPtr)) - if val := *lrPtr; val != sys.Uintreg(stackBarrierPC) { - printlock() - print("at *", hex(stkbar.savedLRPtr), " expected stack barrier PC ", hex(stackBarrierPC), ", found ", hex(val), ", goid=", gp.goid, "\n") - print("gp.stkbar=") - gcPrintStkbars(gp, -1) - print(", gp.stack=[", hex(gp.stack.lo), ",", hex(gp.stack.hi), ")\n") - throw("stack barrier lost") - } - *lrPtr = sys.Uintreg(stkbar.savedLRVal) -} - -// gcTryRemoveAllStackBarriers tries to remove stack barriers from all -// Gs in gps. It is best-effort and efficient. If it can't remove -// barriers from a G immediately, it will simply skip it. -func gcTryRemoveAllStackBarriers(gps []*g) { - for _, gp := range gps { - retry: - for { - switch s := readgstatus(gp); s { - default: - break retry - - case _Grunnable, _Gsyscall, _Gwaiting: - if !castogscanstatus(gp, s, s|_Gscan) { - continue - } - gcLockStackBarriers(gp) - gcRemoveStackBarriers(gp) - gcUnlockStackBarriers(gp) - restartg(gp) - break retry - } - } - } -} - -// gcPrintStkbars prints the stack barriers of gp for debugging. It -// places a "@@@" marker at gp.stkbarPos. If marker >= 0, it will also -// place a "==>" marker before the marker'th entry. -func gcPrintStkbars(gp *g, marker int) { - print("[") - for i, s := range gp.stkbar { - if i > 0 { - print(" ") - } - if i == int(gp.stkbarPos) { - print("@@@ ") - } - if i == marker { - print("==> ") - } - print("*", hex(s.savedLRPtr), "=", hex(s.savedLRVal)) - } - if int(gp.stkbarPos) == len(gp.stkbar) { - print(" @@@") - } - if marker == len(gp.stkbar) { - print(" ==>") - } - print("]") -} - -// gcUnwindBarriers marks all stack barriers up the frame containing -// sp as hit and removes them. This is used during stack unwinding for -// panic/recover and by heapBitsBulkBarrier to force stack re-scanning -// when its destination is on the stack. -// -// This is nosplit to ensure gp's stack does not move. -// -//go:nosplit -func gcUnwindBarriers(gp *g, sp uintptr) { - gcLockStackBarriers(gp) - // On LR machines, if there is a stack barrier on the return - // from the frame containing sp, this will mark it as hit even - // though it isn't, but it's okay to be conservative. - before := gp.stkbarPos - for int(gp.stkbarPos) < len(gp.stkbar) && gp.stkbar[gp.stkbarPos].savedLRPtr < sp { - gcRemoveStackBarrier(gp, gp.stkbar[gp.stkbarPos]) - gp.stkbarPos++ - } - gcUnlockStackBarriers(gp) - if debugStackBarrier && gp.stkbarPos != before { - print("skip barriers below ", hex(sp), " in goid=", gp.goid, ": ") - // We skipped barriers between the "==>" marker - // (before) and the "@@@" marker (gp.stkbarPos). - gcPrintStkbars(gp, int(before)) - print("\n") - } -} - -// nextBarrierPC returns the original return PC of the next stack barrier. -// Used by getcallerpc, so it must be nosplit. -//go:nosplit -func nextBarrierPC() uintptr { - gp := getg() - return gp.stkbar[gp.stkbarPos].savedLRVal -} - -// setNextBarrierPC sets the return PC of the next stack barrier. -// Used by setcallerpc, so it must be nosplit. -//go:nosplit -func setNextBarrierPC(pc uintptr) { - gp := getg() - gcLockStackBarriers(gp) - gp.stkbar[gp.stkbarPos].savedLRVal = pc - gcUnlockStackBarriers(gp) -} - -// gcLockStackBarriers synchronizes with tracebacks of gp's stack -// during sigprof for installation or removal of stack barriers. It -// blocks until any current sigprof is done tracebacking gp's stack -// and then disallows profiling tracebacks of gp's stack. -// -// This is necessary because a sigprof during barrier installation or -// removal could observe inconsistencies between the stkbar array and -// the stack itself and crash. -// -//go:nosplit -func gcLockStackBarriers(gp *g) { - // Disable preemption so scanstack cannot run while the caller - // is manipulating the stack barriers. - acquirem() - for !atomic.Cas(&gp.stackLock, 0, 1) { - osyield() - } -} - -//go:nosplit -func gcTryLockStackBarriers(gp *g) bool { - mp := acquirem() - result := atomic.Cas(&gp.stackLock, 0, 1) - if !result { - releasem(mp) - } - return result -} - -func gcUnlockStackBarriers(gp *g) { - atomic.Store(&gp.stackLock, 0) - releasem(getg().m) -} diff --git a/libgo/go/runtime/net_plan9.go b/libgo/go/runtime/net_plan9.go index 10fd089aea3..77ae8c612d7 100644 --- a/libgo/go/runtime/net_plan9.go +++ b/libgo/go/runtime/net_plan9.go @@ -8,12 +8,12 @@ import ( _ "unsafe" ) -//go:linkname runtime_ignoreHangup net.runtime_ignoreHangup +//go:linkname runtime_ignoreHangup internal_poll.runtime_ignoreHangup func runtime_ignoreHangup() { getg().m.ignoreHangup = true } -//go:linkname runtime_unignoreHangup net.runtime_unignoreHangup +//go:linkname runtime_unignoreHangup internal_poll.runtime_unignoreHangup func runtime_unignoreHangup(sig string) { getg().m.ignoreHangup = false } diff --git a/libgo/go/runtime/netpoll.go b/libgo/go/runtime/netpoll.go index 8932455a19a..e9bbfecb5b3 100644 --- a/libgo/go/runtime/netpoll.go +++ b/libgo/go/runtime/netpoll.go @@ -80,12 +80,13 @@ type pollCache struct { } var ( - netpollInited uint32 - pollcache pollCache + netpollInited uint32 + pollcache pollCache + netpollWaiters uint32 ) -//go:linkname net_runtime_pollServerInit net.runtime_pollServerInit -func net_runtime_pollServerInit() { +//go:linkname poll_runtime_pollServerInit internal_poll.runtime_pollServerInit +func poll_runtime_pollServerInit() { netpollinit() atomic.Store(&netpollInited, 1) } @@ -94,15 +95,23 @@ func netpollinited() bool { return atomic.Load(&netpollInited) != 0 } -//go:linkname net_runtime_pollOpen net.runtime_pollOpen -func net_runtime_pollOpen(fd uintptr) (*pollDesc, int) { +//go:linkname poll_runtime_pollServerDescriptor internal_poll.runtime_pollServerDescriptor + +// poll_runtime_pollServerDescriptor returns the descriptor being used, +// or ^uintptr(0) if the system does not use a poll descriptor. +func poll_runtime_pollServerDescriptor() uintptr { + return netpolldescriptor() +} + +//go:linkname poll_runtime_pollOpen internal_poll.runtime_pollOpen +func poll_runtime_pollOpen(fd uintptr) (*pollDesc, int) { pd := pollcache.alloc() lock(&pd.lock) if pd.wg != 0 && pd.wg != pdReady { - throw("netpollOpen: blocked write on free descriptor") + throw("runtime: blocked write on free polldesc") } if pd.rg != 0 && pd.rg != pdReady { - throw("netpollOpen: blocked read on free descriptor") + throw("runtime: blocked read on free polldesc") } pd.fd = fd pd.closing = false @@ -118,16 +127,16 @@ func net_runtime_pollOpen(fd uintptr) (*pollDesc, int) { return pd, int(errno) } -//go:linkname net_runtime_pollClose net.runtime_pollClose -func net_runtime_pollClose(pd *pollDesc) { +//go:linkname poll_runtime_pollClose internal_poll.runtime_pollClose +func poll_runtime_pollClose(pd *pollDesc) { if !pd.closing { - throw("netpollClose: close w/o unblock") + throw("runtime: close polldesc w/o unblock") } if pd.wg != 0 && pd.wg != pdReady { - throw("netpollClose: blocked write on closing descriptor") + throw("runtime: blocked write on closing polldesc") } if pd.rg != 0 && pd.rg != pdReady { - throw("netpollClose: blocked read on closing descriptor") + throw("runtime: blocked read on closing polldesc") } netpollclose(pd.fd) pollcache.free(pd) @@ -140,8 +149,8 @@ func (c *pollCache) free(pd *pollDesc) { unlock(&c.lock) } -//go:linkname net_runtime_pollReset net.runtime_pollReset -func net_runtime_pollReset(pd *pollDesc, mode int) int { +//go:linkname poll_runtime_pollReset internal_poll.runtime_pollReset +func poll_runtime_pollReset(pd *pollDesc, mode int) int { err := netpollcheckerr(pd, int32(mode)) if err != 0 { return err @@ -154,8 +163,8 @@ func net_runtime_pollReset(pd *pollDesc, mode int) int { return 0 } -//go:linkname net_runtime_pollWait net.runtime_pollWait -func net_runtime_pollWait(pd *pollDesc, mode int) int { +//go:linkname poll_runtime_pollWait internal_poll.runtime_pollWait +func poll_runtime_pollWait(pd *pollDesc, mode int) int { err := netpollcheckerr(pd, int32(mode)) if err != 0 { return err @@ -176,16 +185,16 @@ func net_runtime_pollWait(pd *pollDesc, mode int) int { return 0 } -//go:linkname net_runtime_pollWaitCanceled net.runtime_pollWaitCanceled -func net_runtime_pollWaitCanceled(pd *pollDesc, mode int) { +//go:linkname poll_runtime_pollWaitCanceled internal_poll.runtime_pollWaitCanceled +func poll_runtime_pollWaitCanceled(pd *pollDesc, mode int) { // This function is used only on windows after a failed attempt to cancel // a pending async IO operation. Wait for ioready, ignore closing or timeouts. for !netpollblock(pd, int32(mode), true) { } } -//go:linkname net_runtime_pollSetDeadline net.runtime_pollSetDeadline -func net_runtime_pollSetDeadline(pd *pollDesc, d int64, mode int) { +//go:linkname poll_runtime_pollSetDeadline internal_poll.runtime_pollSetDeadline +func poll_runtime_pollSetDeadline(pd *pollDesc, d int64, mode int) { lock(&pd.lock) if pd.closing { unlock(&pd.lock) @@ -247,18 +256,18 @@ func net_runtime_pollSetDeadline(pd *pollDesc, d int64, mode int) { } unlock(&pd.lock) if rg != nil { - goready(rg, 3) + netpollgoready(rg, 3) } if wg != nil { - goready(wg, 3) + netpollgoready(wg, 3) } } -//go:linkname net_runtime_pollUnblock net.runtime_pollUnblock -func net_runtime_pollUnblock(pd *pollDesc) { +//go:linkname poll_runtime_pollUnblock internal_poll.runtime_pollUnblock +func poll_runtime_pollUnblock(pd *pollDesc) { lock(&pd.lock) if pd.closing { - throw("netpollUnblock: already closing") + throw("runtime: unblock on closing polldesc") } pd.closing = true pd.seq++ @@ -276,10 +285,10 @@ func net_runtime_pollUnblock(pd *pollDesc) { } unlock(&pd.lock) if rg != nil { - goready(rg, 3) + netpollgoready(rg, 3) } if wg != nil { - goready(wg, 3) + netpollgoready(wg, 3) } } @@ -315,7 +324,19 @@ func netpollcheckerr(pd *pollDesc, mode int32) int { } func netpollblockcommit(gp *g, gpp unsafe.Pointer) bool { - return atomic.Casuintptr((*uintptr)(gpp), pdWait, uintptr(unsafe.Pointer(gp))) + r := atomic.Casuintptr((*uintptr)(gpp), pdWait, uintptr(unsafe.Pointer(gp))) + if r { + // Bump the count of goroutines waiting for the poller. + // The scheduler uses this to decide whether to block + // waiting for the poller if there is nothing else to do. + atomic.Xadd(&netpollWaiters, 1) + } + return r +} + +func netpollgoready(gp *g, traceskip int) { + atomic.Xadd(&netpollWaiters, -1) + goready(gp, traceskip+1) } // returns true if IO is ready, or false if timedout or closed @@ -334,7 +355,7 @@ func netpollblock(pd *pollDesc, mode int32, waitio bool) bool { return true } if old != 0 { - throw("netpollblock: double wait") + throw("runtime: double wait") } if atomic.Casuintptr(gpp, 0, pdWait) { break @@ -350,7 +371,7 @@ func netpollblock(pd *pollDesc, mode int32, waitio bool) bool { // be careful to not lose concurrent READY notification old := atomic.Xchguintptr(gpp, 0) if old > pdWait { - throw("netpollblock: corrupted state") + throw("runtime: corrupted polldesc") } return old == pdReady } @@ -396,7 +417,7 @@ func netpolldeadlineimpl(pd *pollDesc, seq uintptr, read, write bool) { var rg *g if read { if pd.rd <= 0 || pd.rt.f == nil { - throw("netpolldeadlineimpl: inconsistent read deadline") + throw("runtime: inconsistent read deadline") } pd.rd = -1 atomicstorep(unsafe.Pointer(&pd.rt.f), nil) // full memory barrier between store to rd and load of rg in netpollunblock @@ -405,7 +426,7 @@ func netpolldeadlineimpl(pd *pollDesc, seq uintptr, read, write bool) { var wg *g if write { if pd.wd <= 0 || pd.wt.f == nil && !read { - throw("netpolldeadlineimpl: inconsistent write deadline") + throw("runtime: inconsistent write deadline") } pd.wd = -1 atomicstorep(unsafe.Pointer(&pd.wt.f), nil) // full memory barrier between store to wd and load of wg in netpollunblock @@ -413,10 +434,10 @@ func netpolldeadlineimpl(pd *pollDesc, seq uintptr, read, write bool) { } unlock(&pd.lock) if rg != nil { - goready(rg, 0) + netpollgoready(rg, 0) } if wg != nil { - goready(wg, 0) + netpollgoready(wg, 0) } } diff --git a/libgo/go/runtime/netpoll_epoll.go b/libgo/go/runtime/netpoll_epoll.go index 247692ef042..ced399d781e 100644 --- a/libgo/go/runtime/netpoll_epoll.go +++ b/libgo/go/runtime/netpoll_epoll.go @@ -47,6 +47,10 @@ func netpollinit() { throw("netpollinit: failed to create descriptor") } +func netpolldescriptor() uintptr { + return uintptr(epfd) +} + func netpollopen(fd uintptr, pd *pollDesc) int32 { var ev epollevent ev.events = _EPOLLIN | _EPOLLOUT | _EPOLLRDHUP | _EPOLLETpos @@ -66,7 +70,7 @@ func netpollclose(fd uintptr) int32 { } func netpollarm(pd *pollDesc, mode int) { - throw("unused") + throw("runtime: unused") } // polls for ready network connections @@ -86,7 +90,7 @@ retry: e := errno() if e != _EINTR { println("runtime: epollwait on fd", epfd, "failed with", e) - throw("epollwait failed") + throw("runtime: netpoll failed") } goto retry } diff --git a/libgo/go/runtime/netpoll_kqueue.go b/libgo/go/runtime/netpoll_kqueue.go index eae4f21d1df..47927fe7c37 100644 --- a/libgo/go/runtime/netpoll_kqueue.go +++ b/libgo/go/runtime/netpoll_kqueue.go @@ -32,11 +32,15 @@ func netpollinit() { kq = kqueue() if kq < 0 { println("netpollinit: kqueue failed with", errno()) - throw("netpollinit: kqueue failed") + throw("runtime: netpollinit failed") } closeonexec(kq) } +func netpolldescriptor() uintptr { + return uintptr(kq) +} + func netpollopen(fd uintptr, pd *pollDesc) int32 { // Arm both EVFILT_READ and EVFILT_WRITE in edge-triggered mode (EV_CLEAR) // for the whole fd lifetime. The notifications are automatically unregistered @@ -64,7 +68,7 @@ func netpollclose(fd uintptr) int32 { } func netpollarm(pd *pollDesc, mode int) { - throw("unused") + throw("runtime: unused") } // Polls for ready network connections. @@ -85,7 +89,7 @@ retry: e := errno() if e != _EINTR { println("runtime: kevent on fd", kq, "failed with", e) - throw("kevent failed") + throw("runtime: netpoll failed") } goto retry } diff --git a/libgo/go/runtime/netpoll_nacl.go b/libgo/go/runtime/netpoll_nacl.go index 5cbc3003214..dc5a55ec846 100644 --- a/libgo/go/runtime/netpoll_nacl.go +++ b/libgo/go/runtime/netpoll_nacl.go @@ -10,6 +10,10 @@ package runtime func netpollinit() { } +func netpolldescriptor() uintptr { + return ^uintptr(0) +} + func netpollopen(fd uintptr, pd *pollDesc) int32 { return 0 } diff --git a/libgo/go/runtime/netpoll_solaris.go b/libgo/go/runtime/netpoll_solaris.go index cc6754cd2eb..e1e73857f40 100644 --- a/libgo/go/runtime/netpoll_solaris.go +++ b/libgo/go/runtime/netpoll_solaris.go @@ -96,8 +96,12 @@ func netpollinit() { return } - print("netpollinit: failed to create port (", errno(), ")\n") - throw("netpollinit: failed to create port") + print("runtime: port_create failed (errno=", errno(), ")\n") + throw("runtime: netpollinit failed") +} + +func netpolldescriptor() uintptr { + return uintptr(portfd) } func netpollopen(fd uintptr, pd *pollDesc) int32 { @@ -139,8 +143,8 @@ func netpollupdate(pd *pollDesc, set, clear uint32) { } if events != 0 && port_associate(portfd, _PORT_SOURCE_FD, pd.fd, events, uintptr(unsafe.Pointer(pd))) != 0 { - print("netpollupdate: failed to associate (", errno(), ")\n") - throw("netpollupdate: failed to associate") + print("runtime: port_associate failed (errno=", errno(), ")\n") + throw("runtime: netpollupdate failed") } pd.user = events } @@ -154,7 +158,7 @@ func netpollarm(pd *pollDesc, mode int) { case 'w': netpollupdate(pd, _POLLOUT, 0) default: - throw("netpollarm: bad mode") + throw("runtime: bad mode") } unlock(&pd.lock) } @@ -177,8 +181,8 @@ retry: var n uint32 = 1 if port_getn(portfd, &events[0], uint32(len(events)), &n, wait) < 0 { if e := errno(); e != _EINTR { - print("runtime: port_getn on fd ", portfd, " failed with ", e, "\n") - throw("port_getn failed") + print("runtime: port_getn on fd ", portfd, " failed (errno=", e, ")\n") + throw("runtime: netpoll failed") } goto retry } diff --git a/libgo/go/runtime/netpoll_stub.go b/libgo/go/runtime/netpoll_stub.go index 09f64ad9b5b..a4d6b4608ac 100644 --- a/libgo/go/runtime/netpoll_stub.go +++ b/libgo/go/runtime/netpoll_stub.go @@ -6,6 +6,8 @@ package runtime +var netpollWaiters uint32 + // Polls for ready network connections. // Returns list of goroutines that become runnable. func netpoll(block bool) (gp *g) { diff --git a/libgo/go/runtime/netpoll_windows.go b/libgo/go/runtime/netpoll_windows.go index 7ad115850d5..79dafb02796 100644 --- a/libgo/go/runtime/netpoll_windows.go +++ b/libgo/go/runtime/netpoll_windows.go @@ -12,7 +12,8 @@ const _DWORD_MAX = 0xffffffff const _INVALID_HANDLE_VALUE = ^uintptr(0) -// net_op must be the same as beginning of net.operation. Keep these in sync. +// net_op must be the same as beginning of internal/poll.operation. +// Keep these in sync. type net_op struct { // used by windows o overlapped @@ -35,11 +36,15 @@ var iocphandle uintptr = _INVALID_HANDLE_VALUE // completion port io handle func netpollinit() { iocphandle = stdcall4(_CreateIoCompletionPort, _INVALID_HANDLE_VALUE, 0, 0, _DWORD_MAX) if iocphandle == 0 { - println("netpoll: failed to create iocp handle (errno=", getlasterror(), ")") - throw("netpoll: failed to create iocp handle") + println("runtime: CreateIoCompletionPort failed (errno=", getlasterror(), ")") + throw("runtime: netpollinit failed") } } +func netpolldescriptor() uintptr { + return iocphandle +} + func netpollopen(fd uintptr, pd *pollDesc) int32 { if stdcall4(_CreateIoCompletionPort, fd, iocphandle, 0, 0) == 0 { return -int32(getlasterror()) @@ -53,7 +58,7 @@ func netpollclose(fd uintptr) int32 { } func netpollarm(pd *pollDesc, mode int) { - throw("unused") + throw("runtime: unused") } // Polls for completed network IO. @@ -89,8 +94,8 @@ retry: if !block && errno == _WAIT_TIMEOUT { return nil } - println("netpoll: GetQueuedCompletionStatusEx failed (errno=", errno, ")") - throw("netpoll: GetQueuedCompletionStatusEx failed") + println("runtime: GetQueuedCompletionStatusEx failed (errno=", errno, ")") + throw("runtime: netpoll failed") } mp.blocked = false for i = 0; i < n; i++ { @@ -116,8 +121,8 @@ retry: return nil } if op == nil { - println("netpoll: GetQueuedCompletionStatus failed (errno=", errno, ")") - throw("netpoll: GetQueuedCompletionStatus failed") + println("runtime: GetQueuedCompletionStatus failed (errno=", errno, ")") + throw("runtime: netpoll failed") } // dequeued failed IO packet, so report that } @@ -132,12 +137,13 @@ retry: func handlecompletion(gpp *guintptr, op *net_op, errno int32, qty uint32) { if op == nil { - throw("netpoll: GetQueuedCompletionStatus returned op == nil") + println("runtime: GetQueuedCompletionStatus returned op == nil") + throw("runtime: netpoll failed") } mode := op.mode if mode != 'r' && mode != 'w' { - println("netpoll: GetQueuedCompletionStatus returned invalid mode=", mode) - throw("netpoll: GetQueuedCompletionStatus returned invalid mode") + println("runtime: GetQueuedCompletionStatus returned invalid mode=", mode) + throw("runtime: netpoll failed") } op.errno = errno op.qty = qty diff --git a/libgo/go/runtime/numcpu_freebsd_test.go b/libgo/go/runtime/numcpu_freebsd_test.go new file mode 100644 index 00000000000..e78890a6a4b --- /dev/null +++ b/libgo/go/runtime/numcpu_freebsd_test.go @@ -0,0 +1,15 @@ +// Copyright 2017 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package runtime_test + +import "testing" + +func TestFreeBSDNumCPU(t *testing.T) { + got := runTestProg(t, "testprog", "FreeBSDNumCPU") + want := "OK\n" + if got != want { + t.Fatalf("expected %q, but got:\n%s", want, got) + } +} diff --git a/libgo/go/runtime/os_gccgo.go b/libgo/go/runtime/os_gccgo.go index db3ea48ef48..5709555acdb 100644 --- a/libgo/go/runtime/os_gccgo.go +++ b/libgo/go/runtime/os_gccgo.go @@ -8,7 +8,7 @@ import ( "unsafe" ) -// Temporary for C code to call: +// For C code to call: //go:linkname minit runtime.minit func goenvs() { diff --git a/libgo/go/runtime/panic.go b/libgo/go/runtime/panic.go index 43d595f667e..2f656038a9e 100644 --- a/libgo/go/runtime/panic.go +++ b/libgo/go/runtime/panic.go @@ -450,6 +450,8 @@ func gopanic(e interface{}) { } gp._panic = p + atomic.Xadd(&runningPanicDefers, 1) + for { d := gp._defer if d == nil { @@ -486,8 +488,8 @@ func gopanic(e interface{}) { d._panic = nil if p.recovered { - // Some deferred function called recover. - // Stop running this panic. + atomic.Xadd(&runningPanicDefers, -1) + gp._panic = p.link // Aborted panics are marked but remain on the g.panic list. @@ -531,6 +533,11 @@ func gopanic(e interface{}) { // and String methods to prepare the panic strings before startpanic. preprintpanics(gp._panic) startpanic() + + // startpanic set panicking, which will block main from exiting, + // so now OK to decrement runningPanicDefers. + atomic.Xadd(&runningPanicDefers, -1) + printpanics(gp._panic) dopanic(0) // should not return *(*int)(nil) = 0 // not reached @@ -801,7 +808,17 @@ func throw(s string) { *(*int)(nil) = 0 // not reached } -//uint32 runtime·panicking; +// runningPanicDefers is non-zero while running deferred functions for panic. +// runningPanicDefers is incremented and decremented atomically. +// This is used to try hard to get a panic stack trace out when exiting. +var runningPanicDefers uint32 + +// panicking is non-zero when crashing the program for an unrecovered panic. +// panicking is incremented and decremented atomically. +var panicking uint32 + +// paniclk is held while printing the panic information and stack trace, +// so that two concurrent panics don't overlap their output. var paniclk mutex func startpanic() { diff --git a/libgo/go/runtime/pprof/elf.go b/libgo/go/runtime/pprof/elf.go new file mode 100644 index 00000000000..a8b5ea68175 --- /dev/null +++ b/libgo/go/runtime/pprof/elf.go @@ -0,0 +1,109 @@ +// Copyright 2017 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package pprof + +import ( + "encoding/binary" + "errors" + "fmt" + "os" +) + +var ( + errBadELF = errors.New("malformed ELF binary") + errNoBuildID = errors.New("no NT_GNU_BUILD_ID found in ELF binary") +) + +// elfBuildID returns the GNU build ID of the named ELF binary, +// without introducing a dependency on debug/elf and its dependencies. +func elfBuildID(file string) (string, error) { + buf := make([]byte, 256) + f, err := os.Open(file) + if err != nil { + return "", err + } + defer f.Close() + + if _, err := f.ReadAt(buf[:64], 0); err != nil { + return "", err + } + + // ELF file begins with \x7F E L F. + if buf[0] != 0x7F || buf[1] != 'E' || buf[2] != 'L' || buf[3] != 'F' { + return "", errBadELF + } + + var byteOrder binary.ByteOrder + switch buf[5] { + default: + return "", errBadELF + case 1: // little-endian + byteOrder = binary.LittleEndian + case 2: // big-endian + byteOrder = binary.BigEndian + } + + var shnum int + var shoff, shentsize int64 + switch buf[4] { + default: + return "", errBadELF + case 1: // 32-bit file header + shoff = int64(byteOrder.Uint32(buf[32:])) + shentsize = int64(byteOrder.Uint16(buf[46:])) + if shentsize != 40 { + return "", errBadELF + } + shnum = int(byteOrder.Uint16(buf[48:])) + case 2: // 64-bit file header + shoff = int64(byteOrder.Uint64(buf[40:])) + shentsize = int64(byteOrder.Uint16(buf[58:])) + if shentsize != 64 { + return "", errBadELF + } + shnum = int(byteOrder.Uint16(buf[60:])) + } + + for i := 0; i < shnum; i++ { + if _, err := f.ReadAt(buf[:shentsize], shoff+int64(i)*shentsize); err != nil { + return "", err + } + if typ := byteOrder.Uint32(buf[4:]); typ != 7 { // SHT_NOTE + continue + } + var off, size int64 + if shentsize == 40 { + // 32-bit section header + off = int64(byteOrder.Uint32(buf[16:])) + size = int64(byteOrder.Uint32(buf[20:])) + } else { + // 64-bit section header + off = int64(byteOrder.Uint64(buf[24:])) + size = int64(byteOrder.Uint64(buf[32:])) + } + size += off + for off < size { + if _, err := f.ReadAt(buf[:16], off); err != nil { // room for header + name GNU\x00 + return "", err + } + nameSize := int(byteOrder.Uint32(buf[0:])) + descSize := int(byteOrder.Uint32(buf[4:])) + noteType := int(byteOrder.Uint32(buf[8:])) + descOff := off + int64(12+(nameSize+3)&^3) + off = descOff + int64((descSize+3)&^3) + if nameSize != 4 || noteType != 3 || buf[12] != 'G' || buf[13] != 'N' || buf[14] != 'U' || buf[15] != '\x00' { // want name GNU\x00 type 3 (NT_GNU_BUILD_ID) + continue + } + if descSize > len(buf) { + return "", errBadELF + } + if _, err := f.ReadAt(buf[:descSize], descOff); err != nil { + return "", err + } + return fmt.Sprintf("%x", buf[:descSize]), nil + } + } + return "", errNoBuildID +} diff --git a/libgo/go/runtime/pprof/internal/profile/encode.go b/libgo/go/runtime/pprof/internal/profile/encode.go new file mode 100644 index 00000000000..6b879a84acd --- /dev/null +++ b/libgo/go/runtime/pprof/internal/profile/encode.go @@ -0,0 +1,470 @@ +// Copyright 2014 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package profile + +import ( + "errors" + "fmt" + "sort" +) + +func (p *Profile) decoder() []decoder { + return profileDecoder +} + +// preEncode populates the unexported fields to be used by encode +// (with suffix X) from the corresponding exported fields. The +// exported fields are cleared up to facilitate testing. +func (p *Profile) preEncode() { + strings := make(map[string]int) + addString(strings, "") + + for _, st := range p.SampleType { + st.typeX = addString(strings, st.Type) + st.unitX = addString(strings, st.Unit) + } + + for _, s := range p.Sample { + s.labelX = nil + var keys []string + for k := range s.Label { + keys = append(keys, k) + } + sort.Strings(keys) + for _, k := range keys { + vs := s.Label[k] + for _, v := range vs { + s.labelX = append(s.labelX, + Label{ + keyX: addString(strings, k), + strX: addString(strings, v), + }, + ) + } + } + var numKeys []string + for k := range s.NumLabel { + numKeys = append(numKeys, k) + } + sort.Strings(numKeys) + for _, k := range numKeys { + vs := s.NumLabel[k] + for _, v := range vs { + s.labelX = append(s.labelX, + Label{ + keyX: addString(strings, k), + numX: v, + }, + ) + } + } + s.locationIDX = nil + for _, l := range s.Location { + s.locationIDX = append(s.locationIDX, l.ID) + } + } + + for _, m := range p.Mapping { + m.fileX = addString(strings, m.File) + m.buildIDX = addString(strings, m.BuildID) + } + + for _, l := range p.Location { + for i, ln := range l.Line { + if ln.Function != nil { + l.Line[i].functionIDX = ln.Function.ID + } else { + l.Line[i].functionIDX = 0 + } + } + if l.Mapping != nil { + l.mappingIDX = l.Mapping.ID + } else { + l.mappingIDX = 0 + } + } + for _, f := range p.Function { + f.nameX = addString(strings, f.Name) + f.systemNameX = addString(strings, f.SystemName) + f.filenameX = addString(strings, f.Filename) + } + + p.dropFramesX = addString(strings, p.DropFrames) + p.keepFramesX = addString(strings, p.KeepFrames) + + if pt := p.PeriodType; pt != nil { + pt.typeX = addString(strings, pt.Type) + pt.unitX = addString(strings, pt.Unit) + } + + p.stringTable = make([]string, len(strings)) + for s, i := range strings { + p.stringTable[i] = s + } +} + +func (p *Profile) encode(b *buffer) { + for _, x := range p.SampleType { + encodeMessage(b, 1, x) + } + for _, x := range p.Sample { + encodeMessage(b, 2, x) + } + for _, x := range p.Mapping { + encodeMessage(b, 3, x) + } + for _, x := range p.Location { + encodeMessage(b, 4, x) + } + for _, x := range p.Function { + encodeMessage(b, 5, x) + } + encodeStrings(b, 6, p.stringTable) + encodeInt64Opt(b, 7, p.dropFramesX) + encodeInt64Opt(b, 8, p.keepFramesX) + encodeInt64Opt(b, 9, p.TimeNanos) + encodeInt64Opt(b, 10, p.DurationNanos) + if pt := p.PeriodType; pt != nil && (pt.typeX != 0 || pt.unitX != 0) { + encodeMessage(b, 11, p.PeriodType) + } + encodeInt64Opt(b, 12, p.Period) +} + +var profileDecoder = []decoder{ + nil, // 0 + // repeated ValueType sample_type = 1 + func(b *buffer, m message) error { + x := new(ValueType) + pp := m.(*Profile) + pp.SampleType = append(pp.SampleType, x) + return decodeMessage(b, x) + }, + // repeated Sample sample = 2 + func(b *buffer, m message) error { + x := new(Sample) + pp := m.(*Profile) + pp.Sample = append(pp.Sample, x) + return decodeMessage(b, x) + }, + // repeated Mapping mapping = 3 + func(b *buffer, m message) error { + x := new(Mapping) + pp := m.(*Profile) + pp.Mapping = append(pp.Mapping, x) + return decodeMessage(b, x) + }, + // repeated Location location = 4 + func(b *buffer, m message) error { + x := new(Location) + pp := m.(*Profile) + pp.Location = append(pp.Location, x) + return decodeMessage(b, x) + }, + // repeated Function function = 5 + func(b *buffer, m message) error { + x := new(Function) + pp := m.(*Profile) + pp.Function = append(pp.Function, x) + return decodeMessage(b, x) + }, + // repeated string string_table = 6 + func(b *buffer, m message) error { + err := decodeStrings(b, &m.(*Profile).stringTable) + if err != nil { + return err + } + if *&m.(*Profile).stringTable[0] != "" { + return errors.New("string_table[0] must be ''") + } + return nil + }, + // repeated int64 drop_frames = 7 + func(b *buffer, m message) error { return decodeInt64(b, &m.(*Profile).dropFramesX) }, + // repeated int64 keep_frames = 8 + func(b *buffer, m message) error { return decodeInt64(b, &m.(*Profile).keepFramesX) }, + // repeated int64 time_nanos = 9 + func(b *buffer, m message) error { return decodeInt64(b, &m.(*Profile).TimeNanos) }, + // repeated int64 duration_nanos = 10 + func(b *buffer, m message) error { return decodeInt64(b, &m.(*Profile).DurationNanos) }, + // optional string period_type = 11 + func(b *buffer, m message) error { + x := new(ValueType) + pp := m.(*Profile) + pp.PeriodType = x + return decodeMessage(b, x) + }, + // repeated int64 period = 12 + func(b *buffer, m message) error { return decodeInt64(b, &m.(*Profile).Period) }, +} + +// postDecode takes the unexported fields populated by decode (with +// suffix X) and populates the corresponding exported fields. +// The unexported fields are cleared up to facilitate testing. +func (p *Profile) postDecode() error { + var err error + + mappings := make(map[uint64]*Mapping) + for _, m := range p.Mapping { + m.File, err = getString(p.stringTable, &m.fileX, err) + m.BuildID, err = getString(p.stringTable, &m.buildIDX, err) + mappings[m.ID] = m + } + + functions := make(map[uint64]*Function) + for _, f := range p.Function { + f.Name, err = getString(p.stringTable, &f.nameX, err) + f.SystemName, err = getString(p.stringTable, &f.systemNameX, err) + f.Filename, err = getString(p.stringTable, &f.filenameX, err) + functions[f.ID] = f + } + + locations := make(map[uint64]*Location) + for _, l := range p.Location { + l.Mapping = mappings[l.mappingIDX] + l.mappingIDX = 0 + for i, ln := range l.Line { + if id := ln.functionIDX; id != 0 { + l.Line[i].Function = functions[id] + if l.Line[i].Function == nil { + return fmt.Errorf("Function ID %d not found", id) + } + l.Line[i].functionIDX = 0 + } + } + locations[l.ID] = l + } + + for _, st := range p.SampleType { + st.Type, err = getString(p.stringTable, &st.typeX, err) + st.Unit, err = getString(p.stringTable, &st.unitX, err) + } + + for _, s := range p.Sample { + labels := make(map[string][]string) + numLabels := make(map[string][]int64) + for _, l := range s.labelX { + var key, value string + key, err = getString(p.stringTable, &l.keyX, err) + if l.strX != 0 { + value, err = getString(p.stringTable, &l.strX, err) + labels[key] = append(labels[key], value) + } else { + numLabels[key] = append(numLabels[key], l.numX) + } + } + if len(labels) > 0 { + s.Label = labels + } + if len(numLabels) > 0 { + s.NumLabel = numLabels + } + s.Location = nil + for _, lid := range s.locationIDX { + s.Location = append(s.Location, locations[lid]) + } + s.locationIDX = nil + } + + p.DropFrames, err = getString(p.stringTable, &p.dropFramesX, err) + p.KeepFrames, err = getString(p.stringTable, &p.keepFramesX, err) + + if pt := p.PeriodType; pt == nil { + p.PeriodType = &ValueType{} + } + + if pt := p.PeriodType; pt != nil { + pt.Type, err = getString(p.stringTable, &pt.typeX, err) + pt.Unit, err = getString(p.stringTable, &pt.unitX, err) + } + p.stringTable = nil + return nil +} + +func (p *ValueType) decoder() []decoder { + return valueTypeDecoder +} + +func (p *ValueType) encode(b *buffer) { + encodeInt64Opt(b, 1, p.typeX) + encodeInt64Opt(b, 2, p.unitX) +} + +var valueTypeDecoder = []decoder{ + nil, // 0 + // optional int64 type = 1 + func(b *buffer, m message) error { return decodeInt64(b, &m.(*ValueType).typeX) }, + // optional int64 unit = 2 + func(b *buffer, m message) error { return decodeInt64(b, &m.(*ValueType).unitX) }, +} + +func (p *Sample) decoder() []decoder { + return sampleDecoder +} + +func (p *Sample) encode(b *buffer) { + encodeUint64s(b, 1, p.locationIDX) + for _, x := range p.Value { + encodeInt64(b, 2, x) + } + for _, x := range p.labelX { + encodeMessage(b, 3, x) + } +} + +var sampleDecoder = []decoder{ + nil, // 0 + // repeated uint64 location = 1 + func(b *buffer, m message) error { return decodeUint64s(b, &m.(*Sample).locationIDX) }, + // repeated int64 value = 2 + func(b *buffer, m message) error { return decodeInt64s(b, &m.(*Sample).Value) }, + // repeated Label label = 3 + func(b *buffer, m message) error { + s := m.(*Sample) + n := len(s.labelX) + s.labelX = append(s.labelX, Label{}) + return decodeMessage(b, &s.labelX[n]) + }, +} + +func (p Label) decoder() []decoder { + return labelDecoder +} + +func (p Label) encode(b *buffer) { + encodeInt64Opt(b, 1, p.keyX) + encodeInt64Opt(b, 2, p.strX) + encodeInt64Opt(b, 3, p.numX) +} + +var labelDecoder = []decoder{ + nil, // 0 + // optional int64 key = 1 + func(b *buffer, m message) error { return decodeInt64(b, &m.(*Label).keyX) }, + // optional int64 str = 2 + func(b *buffer, m message) error { return decodeInt64(b, &m.(*Label).strX) }, + // optional int64 num = 3 + func(b *buffer, m message) error { return decodeInt64(b, &m.(*Label).numX) }, +} + +func (p *Mapping) decoder() []decoder { + return mappingDecoder +} + +func (p *Mapping) encode(b *buffer) { + encodeUint64Opt(b, 1, p.ID) + encodeUint64Opt(b, 2, p.Start) + encodeUint64Opt(b, 3, p.Limit) + encodeUint64Opt(b, 4, p.Offset) + encodeInt64Opt(b, 5, p.fileX) + encodeInt64Opt(b, 6, p.buildIDX) + encodeBoolOpt(b, 7, p.HasFunctions) + encodeBoolOpt(b, 8, p.HasFilenames) + encodeBoolOpt(b, 9, p.HasLineNumbers) + encodeBoolOpt(b, 10, p.HasInlineFrames) +} + +var mappingDecoder = []decoder{ + nil, // 0 + func(b *buffer, m message) error { return decodeUint64(b, &m.(*Mapping).ID) }, // optional uint64 id = 1 + func(b *buffer, m message) error { return decodeUint64(b, &m.(*Mapping).Start) }, // optional uint64 memory_offset = 2 + func(b *buffer, m message) error { return decodeUint64(b, &m.(*Mapping).Limit) }, // optional uint64 memory_limit = 3 + func(b *buffer, m message) error { return decodeUint64(b, &m.(*Mapping).Offset) }, // optional uint64 file_offset = 4 + func(b *buffer, m message) error { return decodeInt64(b, &m.(*Mapping).fileX) }, // optional int64 filename = 5 + func(b *buffer, m message) error { return decodeInt64(b, &m.(*Mapping).buildIDX) }, // optional int64 build_id = 6 + func(b *buffer, m message) error { return decodeBool(b, &m.(*Mapping).HasFunctions) }, // optional bool has_functions = 7 + func(b *buffer, m message) error { return decodeBool(b, &m.(*Mapping).HasFilenames) }, // optional bool has_filenames = 8 + func(b *buffer, m message) error { return decodeBool(b, &m.(*Mapping).HasLineNumbers) }, // optional bool has_line_numbers = 9 + func(b *buffer, m message) error { return decodeBool(b, &m.(*Mapping).HasInlineFrames) }, // optional bool has_inline_frames = 10 +} + +func (p *Location) decoder() []decoder { + return locationDecoder +} + +func (p *Location) encode(b *buffer) { + encodeUint64Opt(b, 1, p.ID) + encodeUint64Opt(b, 2, p.mappingIDX) + encodeUint64Opt(b, 3, p.Address) + for i := range p.Line { + encodeMessage(b, 4, &p.Line[i]) + } +} + +var locationDecoder = []decoder{ + nil, // 0 + func(b *buffer, m message) error { return decodeUint64(b, &m.(*Location).ID) }, // optional uint64 id = 1; + func(b *buffer, m message) error { return decodeUint64(b, &m.(*Location).mappingIDX) }, // optional uint64 mapping_id = 2; + func(b *buffer, m message) error { return decodeUint64(b, &m.(*Location).Address) }, // optional uint64 address = 3; + func(b *buffer, m message) error { // repeated Line line = 4 + pp := m.(*Location) + n := len(pp.Line) + pp.Line = append(pp.Line, Line{}) + return decodeMessage(b, &pp.Line[n]) + }, +} + +func (p *Line) decoder() []decoder { + return lineDecoder +} + +func (p *Line) encode(b *buffer) { + encodeUint64Opt(b, 1, p.functionIDX) + encodeInt64Opt(b, 2, p.Line) +} + +var lineDecoder = []decoder{ + nil, // 0 + // optional uint64 function_id = 1 + func(b *buffer, m message) error { return decodeUint64(b, &m.(*Line).functionIDX) }, + // optional int64 line = 2 + func(b *buffer, m message) error { return decodeInt64(b, &m.(*Line).Line) }, +} + +func (p *Function) decoder() []decoder { + return functionDecoder +} + +func (p *Function) encode(b *buffer) { + encodeUint64Opt(b, 1, p.ID) + encodeInt64Opt(b, 2, p.nameX) + encodeInt64Opt(b, 3, p.systemNameX) + encodeInt64Opt(b, 4, p.filenameX) + encodeInt64Opt(b, 5, p.StartLine) +} + +var functionDecoder = []decoder{ + nil, // 0 + // optional uint64 id = 1 + func(b *buffer, m message) error { return decodeUint64(b, &m.(*Function).ID) }, + // optional int64 function_name = 2 + func(b *buffer, m message) error { return decodeInt64(b, &m.(*Function).nameX) }, + // optional int64 function_system_name = 3 + func(b *buffer, m message) error { return decodeInt64(b, &m.(*Function).systemNameX) }, + // repeated int64 filename = 4 + func(b *buffer, m message) error { return decodeInt64(b, &m.(*Function).filenameX) }, + // optional int64 start_line = 5 + func(b *buffer, m message) error { return decodeInt64(b, &m.(*Function).StartLine) }, +} + +func addString(strings map[string]int, s string) int64 { + i, ok := strings[s] + if !ok { + i = len(strings) + strings[s] = i + } + return int64(i) +} + +func getString(strings []string, strng *int64, err error) (string, error) { + if err != nil { + return "", err + } + s := int(*strng) + if s < 0 || s >= len(strings) { + return "", errMalformed + } + *strng = 0 + return strings[s], nil +} diff --git a/libgo/go/runtime/pprof/internal/profile/filter.go b/libgo/go/runtime/pprof/internal/profile/filter.go new file mode 100644 index 00000000000..1baa096a49c --- /dev/null +++ b/libgo/go/runtime/pprof/internal/profile/filter.go @@ -0,0 +1,158 @@ +// Copyright 2014 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Implements methods to filter samples from profiles. + +package profile + +import "regexp" + +// FilterSamplesByName filters the samples in a profile and only keeps +// samples where at least one frame matches focus but none match ignore. +// Returns true is the corresponding regexp matched at least one sample. +func (p *Profile) FilterSamplesByName(focus, ignore, hide *regexp.Regexp) (fm, im, hm bool) { + focusOrIgnore := make(map[uint64]bool) + hidden := make(map[uint64]bool) + for _, l := range p.Location { + if ignore != nil && l.matchesName(ignore) { + im = true + focusOrIgnore[l.ID] = false + } else if focus == nil || l.matchesName(focus) { + fm = true + focusOrIgnore[l.ID] = true + } + if hide != nil && l.matchesName(hide) { + hm = true + l.Line = l.unmatchedLines(hide) + if len(l.Line) == 0 { + hidden[l.ID] = true + } + } + } + + s := make([]*Sample, 0, len(p.Sample)) + for _, sample := range p.Sample { + if focusedAndNotIgnored(sample.Location, focusOrIgnore) { + if len(hidden) > 0 { + var locs []*Location + for _, loc := range sample.Location { + if !hidden[loc.ID] { + locs = append(locs, loc) + } + } + if len(locs) == 0 { + // Remove sample with no locations (by not adding it to s). + continue + } + sample.Location = locs + } + s = append(s, sample) + } + } + p.Sample = s + + return +} + +// matchesName returns whether the function name or file in the +// location matches the regular expression. +func (loc *Location) matchesName(re *regexp.Regexp) bool { + for _, ln := range loc.Line { + if fn := ln.Function; fn != nil { + if re.MatchString(fn.Name) { + return true + } + if re.MatchString(fn.Filename) { + return true + } + } + } + return false +} + +// unmatchedLines returns the lines in the location that do not match +// the regular expression. +func (loc *Location) unmatchedLines(re *regexp.Regexp) []Line { + var lines []Line + for _, ln := range loc.Line { + if fn := ln.Function; fn != nil { + if re.MatchString(fn.Name) { + continue + } + if re.MatchString(fn.Filename) { + continue + } + } + lines = append(lines, ln) + } + return lines +} + +// focusedAndNotIgnored looks up a slice of ids against a map of +// focused/ignored locations. The map only contains locations that are +// explicitly focused or ignored. Returns whether there is at least +// one focused location but no ignored locations. +func focusedAndNotIgnored(locs []*Location, m map[uint64]bool) bool { + var f bool + for _, loc := range locs { + if focus, focusOrIgnore := m[loc.ID]; focusOrIgnore { + if focus { + // Found focused location. Must keep searching in case there + // is an ignored one as well. + f = true + } else { + // Found ignored location. Can return false right away. + return false + } + } + } + return f +} + +// TagMatch selects tags for filtering +type TagMatch func(key, val string, nval int64) bool + +// FilterSamplesByTag removes all samples from the profile, except +// those that match focus and do not match the ignore regular +// expression. +func (p *Profile) FilterSamplesByTag(focus, ignore TagMatch) (fm, im bool) { + samples := make([]*Sample, 0, len(p.Sample)) + for _, s := range p.Sample { + focused, ignored := focusedSample(s, focus, ignore) + fm = fm || focused + im = im || ignored + if focused && !ignored { + samples = append(samples, s) + } + } + p.Sample = samples + return +} + +// focusedTag checks a sample against focus and ignore regexps. +// Returns whether the focus/ignore regexps match any tags +func focusedSample(s *Sample, focus, ignore TagMatch) (fm, im bool) { + fm = focus == nil + for key, vals := range s.Label { + for _, val := range vals { + if ignore != nil && ignore(key, val, 0) { + im = true + } + if !fm && focus(key, val, 0) { + fm = true + } + } + } + for key, vals := range s.NumLabel { + for _, val := range vals { + if ignore != nil && ignore(key, "", val) { + im = true + } + if !fm && focus(key, "", val) { + fm = true + } + } + } + return fm, im +} diff --git a/libgo/go/runtime/pprof/internal/profile/legacy_profile.go b/libgo/go/runtime/pprof/internal/profile/legacy_profile.go new file mode 100644 index 00000000000..d69f8deee7c --- /dev/null +++ b/libgo/go/runtime/pprof/internal/profile/legacy_profile.go @@ -0,0 +1,1266 @@ +// Copyright 2014 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// This file implements parsers to convert legacy profiles into the +// profile.proto format. + +package profile + +import ( + "bufio" + "bytes" + "fmt" + "io" + "math" + "regexp" + "strconv" + "strings" +) + +var ( + countStartRE = regexp.MustCompile(`\A(\w+) profile: total \d+\n\z`) + countRE = regexp.MustCompile(`\A(\d+) @(( 0x[0-9a-f]+)+)\n\z`) + + heapHeaderRE = regexp.MustCompile(`heap profile: *(\d+): *(\d+) *\[ *(\d+): *(\d+) *\] *@ *(heap[_a-z0-9]*)/?(\d*)`) + heapSampleRE = regexp.MustCompile(`(-?\d+): *(-?\d+) *\[ *(\d+): *(\d+) *] @([ x0-9a-f]*)`) + + contentionSampleRE = regexp.MustCompile(`(\d+) *(\d+) @([ x0-9a-f]*)`) + + hexNumberRE = regexp.MustCompile(`0x[0-9a-f]+`) + + growthHeaderRE = regexp.MustCompile(`heap profile: *(\d+): *(\d+) *\[ *(\d+): *(\d+) *\] @ growthz`) + + fragmentationHeaderRE = regexp.MustCompile(`heap profile: *(\d+): *(\d+) *\[ *(\d+): *(\d+) *\] @ fragmentationz`) + + threadzStartRE = regexp.MustCompile(`--- threadz \d+ ---`) + threadStartRE = regexp.MustCompile(`--- Thread ([[:xdigit:]]+) \(name: (.*)/(\d+)\) stack: ---`) + + procMapsRE = regexp.MustCompile(`([[:xdigit:]]+)-([[:xdigit:]]+)\s+([-rwxp]+)\s+([[:xdigit:]]+)\s+([[:xdigit:]]+):([[:xdigit:]]+)\s+([[:digit:]]+)\s*(\S+)?`) + + briefMapsRE = regexp.MustCompile(`\s*([[:xdigit:]]+)-([[:xdigit:]]+):\s*(\S+)(\s.*@)?([[:xdigit:]]+)?`) + + // LegacyHeapAllocated instructs the heapz parsers to use the + // allocated memory stats instead of the default in-use memory. Note + // that tcmalloc doesn't provide all allocated memory, only in-use + // stats. + LegacyHeapAllocated bool +) + +func isSpaceOrComment(line string) bool { + trimmed := strings.TrimSpace(line) + return len(trimmed) == 0 || trimmed[0] == '#' +} + +// parseGoCount parses a Go count profile (e.g., threadcreate or +// goroutine) and returns a new Profile. +func parseGoCount(b []byte) (*Profile, error) { + r := bytes.NewBuffer(b) + + var line string + var err error + for { + // Skip past comments and empty lines seeking a real header. + line, err = r.ReadString('\n') + if err != nil { + return nil, err + } + if !isSpaceOrComment(line) { + break + } + } + + m := countStartRE.FindStringSubmatch(line) + if m == nil { + return nil, errUnrecognized + } + profileType := m[1] + p := &Profile{ + PeriodType: &ValueType{Type: profileType, Unit: "count"}, + Period: 1, + SampleType: []*ValueType{{Type: profileType, Unit: "count"}}, + } + locations := make(map[uint64]*Location) + for { + line, err = r.ReadString('\n') + if err != nil { + if err == io.EOF { + break + } + return nil, err + } + if isSpaceOrComment(line) { + continue + } + if strings.HasPrefix(line, "---") { + break + } + m := countRE.FindStringSubmatch(line) + if m == nil { + return nil, errMalformed + } + n, err := strconv.ParseInt(m[1], 0, 64) + if err != nil { + return nil, errMalformed + } + fields := strings.Fields(m[2]) + locs := make([]*Location, 0, len(fields)) + for _, stk := range fields { + addr, err := strconv.ParseUint(stk, 0, 64) + if err != nil { + return nil, errMalformed + } + // Adjust all frames by -1 to land on the call instruction. + addr-- + loc := locations[addr] + if loc == nil { + loc = &Location{ + Address: addr, + } + locations[addr] = loc + p.Location = append(p.Location, loc) + } + locs = append(locs, loc) + } + p.Sample = append(p.Sample, &Sample{ + Location: locs, + Value: []int64{n}, + }) + } + + if err = parseAdditionalSections(strings.TrimSpace(line), r, p); err != nil { + return nil, err + } + return p, nil +} + +// remapLocationIDs ensures there is a location for each address +// referenced by a sample, and remaps the samples to point to the new +// location ids. +func (p *Profile) remapLocationIDs() { + seen := make(map[*Location]bool, len(p.Location)) + var locs []*Location + + for _, s := range p.Sample { + for _, l := range s.Location { + if seen[l] { + continue + } + l.ID = uint64(len(locs) + 1) + locs = append(locs, l) + seen[l] = true + } + } + p.Location = locs +} + +func (p *Profile) remapFunctionIDs() { + seen := make(map[*Function]bool, len(p.Function)) + var fns []*Function + + for _, l := range p.Location { + for _, ln := range l.Line { + fn := ln.Function + if fn == nil || seen[fn] { + continue + } + fn.ID = uint64(len(fns) + 1) + fns = append(fns, fn) + seen[fn] = true + } + } + p.Function = fns +} + +// remapMappingIDs matches location addresses with existing mappings +// and updates them appropriately. This is O(N*M), if this ever shows +// up as a bottleneck, evaluate sorting the mappings and doing a +// binary search, which would make it O(N*log(M)). +func (p *Profile) remapMappingIDs() { + if len(p.Mapping) == 0 { + return + } + + // Some profile handlers will incorrectly set regions for the main + // executable if its section is remapped. Fix them through heuristics. + + // Remove the initial mapping if named '/anon_hugepage' and has a + // consecutive adjacent mapping. + if m := p.Mapping[0]; strings.HasPrefix(m.File, "/anon_hugepage") { + if len(p.Mapping) > 1 && m.Limit == p.Mapping[1].Start { + p.Mapping = p.Mapping[1:] + } + } + + // Subtract the offset from the start of the main mapping if it + // ends up at a recognizable start address. + const expectedStart = 0x400000 + if m := p.Mapping[0]; m.Start-m.Offset == expectedStart { + m.Start = expectedStart + m.Offset = 0 + } + + for _, l := range p.Location { + if a := l.Address; a != 0 { + for _, m := range p.Mapping { + if m.Start <= a && a < m.Limit { + l.Mapping = m + break + } + } + } + } + + // Reset all mapping IDs. + for i, m := range p.Mapping { + m.ID = uint64(i + 1) + } +} + +var cpuInts = []func([]byte) (uint64, []byte){ + get32l, + get32b, + get64l, + get64b, +} + +func get32l(b []byte) (uint64, []byte) { + if len(b) < 4 { + return 0, nil + } + return uint64(b[0]) | uint64(b[1])<<8 | uint64(b[2])<<16 | uint64(b[3])<<24, b[4:] +} + +func get32b(b []byte) (uint64, []byte) { + if len(b) < 4 { + return 0, nil + } + return uint64(b[3]) | uint64(b[2])<<8 | uint64(b[1])<<16 | uint64(b[0])<<24, b[4:] +} + +func get64l(b []byte) (uint64, []byte) { + if len(b) < 8 { + return 0, nil + } + return uint64(b[0]) | uint64(b[1])<<8 | uint64(b[2])<<16 | uint64(b[3])<<24 | uint64(b[4])<<32 | uint64(b[5])<<40 | uint64(b[6])<<48 | uint64(b[7])<<56, b[8:] +} + +func get64b(b []byte) (uint64, []byte) { + if len(b) < 8 { + return 0, nil + } + return uint64(b[7]) | uint64(b[6])<<8 | uint64(b[5])<<16 | uint64(b[4])<<24 | uint64(b[3])<<32 | uint64(b[2])<<40 | uint64(b[1])<<48 | uint64(b[0])<<56, b[8:] +} + +// ParseTracebacks parses a set of tracebacks and returns a newly +// populated profile. It will accept any text file and generate a +// Profile out of it with any hex addresses it can identify, including +// a process map if it can recognize one. Each sample will include a +// tag "source" with the addresses recognized in string format. +func ParseTracebacks(b []byte) (*Profile, error) { + r := bytes.NewBuffer(b) + + p := &Profile{ + PeriodType: &ValueType{Type: "trace", Unit: "count"}, + Period: 1, + SampleType: []*ValueType{ + {Type: "trace", Unit: "count"}, + }, + } + + var sources []string + var sloc []*Location + + locs := make(map[uint64]*Location) + for { + l, err := r.ReadString('\n') + if err != nil { + if err != io.EOF { + return nil, err + } + if l == "" { + break + } + } + if sectionTrigger(l) == memoryMapSection { + break + } + if s, addrs := extractHexAddresses(l); len(s) > 0 { + for _, addr := range addrs { + // Addresses from stack traces point to the next instruction after + // each call. Adjust by -1 to land somewhere on the actual call. + addr-- + loc := locs[addr] + if locs[addr] == nil { + loc = &Location{ + Address: addr, + } + p.Location = append(p.Location, loc) + locs[addr] = loc + } + sloc = append(sloc, loc) + } + + sources = append(sources, s...) + } else { + if len(sources) > 0 || len(sloc) > 0 { + addTracebackSample(sloc, sources, p) + sloc, sources = nil, nil + } + } + } + + // Add final sample to save any leftover data. + if len(sources) > 0 || len(sloc) > 0 { + addTracebackSample(sloc, sources, p) + } + + if err := p.ParseMemoryMap(r); err != nil { + return nil, err + } + return p, nil +} + +func addTracebackSample(l []*Location, s []string, p *Profile) { + p.Sample = append(p.Sample, + &Sample{ + Value: []int64{1}, + Location: l, + Label: map[string][]string{"source": s}, + }) +} + +// parseCPU parses a profilez legacy profile and returns a newly +// populated Profile. +// +// The general format for profilez samples is a sequence of words in +// binary format. The first words are a header with the following data: +// 1st word -- 0 +// 2nd word -- 3 +// 3rd word -- 0 if a c++ application, 1 if a java application. +// 4th word -- Sampling period (in microseconds). +// 5th word -- Padding. +func parseCPU(b []byte) (*Profile, error) { + var parse func([]byte) (uint64, []byte) + var n1, n2, n3, n4, n5 uint64 + for _, parse = range cpuInts { + var tmp []byte + n1, tmp = parse(b) + n2, tmp = parse(tmp) + n3, tmp = parse(tmp) + n4, tmp = parse(tmp) + n5, tmp = parse(tmp) + + if tmp != nil && n1 == 0 && n2 == 3 && n3 == 0 && n4 > 0 && n5 == 0 { + b = tmp + return cpuProfile(b, int64(n4), parse) + } + } + return nil, errUnrecognized +} + +// cpuProfile returns a new Profile from C++ profilez data. +// b is the profile bytes after the header, period is the profiling +// period, and parse is a function to parse 8-byte chunks from the +// profile in its native endianness. +func cpuProfile(b []byte, period int64, parse func(b []byte) (uint64, []byte)) (*Profile, error) { + p := &Profile{ + Period: period * 1000, + PeriodType: &ValueType{Type: "cpu", Unit: "nanoseconds"}, + SampleType: []*ValueType{ + {Type: "samples", Unit: "count"}, + {Type: "cpu", Unit: "nanoseconds"}, + }, + } + var err error + if b, _, err = parseCPUSamples(b, parse, true, p); err != nil { + return nil, err + } + + // If all samples have the same second-to-the-bottom frame, it + // strongly suggests that it is an uninteresting artifact of + // measurement -- a stack frame pushed by the signal handler. The + // bottom frame is always correct as it is picked up from the signal + // structure, not the stack. Check if this is the case and if so, + // remove. + if len(p.Sample) > 1 && len(p.Sample[0].Location) > 1 { + allSame := true + id1 := p.Sample[0].Location[1].Address + for _, s := range p.Sample { + if len(s.Location) < 2 || id1 != s.Location[1].Address { + allSame = false + break + } + } + if allSame { + for _, s := range p.Sample { + s.Location = append(s.Location[:1], s.Location[2:]...) + } + } + } + + if err := p.ParseMemoryMap(bytes.NewBuffer(b)); err != nil { + return nil, err + } + return p, nil +} + +// parseCPUSamples parses a collection of profilez samples from a +// profile. +// +// profilez samples are a repeated sequence of stack frames of the +// form: +// 1st word -- The number of times this stack was encountered. +// 2nd word -- The size of the stack (StackSize). +// 3rd word -- The first address on the stack. +// ... +// StackSize + 2 -- The last address on the stack +// The last stack trace is of the form: +// 1st word -- 0 +// 2nd word -- 1 +// 3rd word -- 0 +// +// Addresses from stack traces may point to the next instruction after +// each call. Optionally adjust by -1 to land somewhere on the actual +// call (except for the leaf, which is not a call). +func parseCPUSamples(b []byte, parse func(b []byte) (uint64, []byte), adjust bool, p *Profile) ([]byte, map[uint64]*Location, error) { + locs := make(map[uint64]*Location) + for len(b) > 0 { + var count, nstk uint64 + count, b = parse(b) + nstk, b = parse(b) + if b == nil || nstk > uint64(len(b)/4) { + return nil, nil, errUnrecognized + } + var sloc []*Location + addrs := make([]uint64, nstk) + for i := 0; i < int(nstk); i++ { + addrs[i], b = parse(b) + } + + if count == 0 && nstk == 1 && addrs[0] == 0 { + // End of data marker + break + } + for i, addr := range addrs { + if adjust && i > 0 { + addr-- + } + loc := locs[addr] + if loc == nil { + loc = &Location{ + Address: addr, + } + locs[addr] = loc + p.Location = append(p.Location, loc) + } + sloc = append(sloc, loc) + } + p.Sample = append(p.Sample, + &Sample{ + Value: []int64{int64(count), int64(count) * p.Period}, + Location: sloc, + }) + } + // Reached the end without finding the EOD marker. + return b, locs, nil +} + +// parseHeap parses a heapz legacy or a growthz profile and +// returns a newly populated Profile. +func parseHeap(b []byte) (p *Profile, err error) { + r := bytes.NewBuffer(b) + l, err := r.ReadString('\n') + if err != nil { + return nil, errUnrecognized + } + + sampling := "" + + if header := heapHeaderRE.FindStringSubmatch(l); header != nil { + p = &Profile{ + SampleType: []*ValueType{ + {Type: "objects", Unit: "count"}, + {Type: "space", Unit: "bytes"}, + }, + PeriodType: &ValueType{Type: "objects", Unit: "bytes"}, + } + + var period int64 + if len(header[6]) > 0 { + if period, err = strconv.ParseInt(header[6], 10, 64); err != nil { + return nil, errUnrecognized + } + } + + switch header[5] { + case "heapz_v2", "heap_v2": + sampling, p.Period = "v2", period + case "heapprofile": + sampling, p.Period = "", 1 + case "heap": + sampling, p.Period = "v2", period/2 + default: + return nil, errUnrecognized + } + } else if header = growthHeaderRE.FindStringSubmatch(l); header != nil { + p = &Profile{ + SampleType: []*ValueType{ + {Type: "objects", Unit: "count"}, + {Type: "space", Unit: "bytes"}, + }, + PeriodType: &ValueType{Type: "heapgrowth", Unit: "count"}, + Period: 1, + } + } else if header = fragmentationHeaderRE.FindStringSubmatch(l); header != nil { + p = &Profile{ + SampleType: []*ValueType{ + {Type: "objects", Unit: "count"}, + {Type: "space", Unit: "bytes"}, + }, + PeriodType: &ValueType{Type: "allocations", Unit: "count"}, + Period: 1, + } + } else { + return nil, errUnrecognized + } + + if LegacyHeapAllocated { + for _, st := range p.SampleType { + st.Type = "alloc_" + st.Type + } + } else { + for _, st := range p.SampleType { + st.Type = "inuse_" + st.Type + } + } + + locs := make(map[uint64]*Location) + for { + l, err = r.ReadString('\n') + if err != nil { + if err != io.EOF { + return nil, err + } + + if l == "" { + break + } + } + + if isSpaceOrComment(l) { + continue + } + l = strings.TrimSpace(l) + + if sectionTrigger(l) != unrecognizedSection { + break + } + + value, blocksize, addrs, err := parseHeapSample(l, p.Period, sampling) + if err != nil { + return nil, err + } + var sloc []*Location + for _, addr := range addrs { + // Addresses from stack traces point to the next instruction after + // each call. Adjust by -1 to land somewhere on the actual call. + addr-- + loc := locs[addr] + if locs[addr] == nil { + loc = &Location{ + Address: addr, + } + p.Location = append(p.Location, loc) + locs[addr] = loc + } + sloc = append(sloc, loc) + } + + p.Sample = append(p.Sample, &Sample{ + Value: value, + Location: sloc, + NumLabel: map[string][]int64{"bytes": {blocksize}}, + }) + } + + if err = parseAdditionalSections(l, r, p); err != nil { + return nil, err + } + return p, nil +} + +// parseHeapSample parses a single row from a heap profile into a new Sample. +func parseHeapSample(line string, rate int64, sampling string) (value []int64, blocksize int64, addrs []uint64, err error) { + sampleData := heapSampleRE.FindStringSubmatch(line) + if len(sampleData) != 6 { + return value, blocksize, addrs, fmt.Errorf("unexpected number of sample values: got %d, want 6", len(sampleData)) + } + + // Use first two values by default; tcmalloc sampling generates the + // same value for both, only the older heap-profile collect separate + // stats for in-use and allocated objects. + valueIndex := 1 + if LegacyHeapAllocated { + valueIndex = 3 + } + + var v1, v2 int64 + if v1, err = strconv.ParseInt(sampleData[valueIndex], 10, 64); err != nil { + return value, blocksize, addrs, fmt.Errorf("malformed sample: %s: %v", line, err) + } + if v2, err = strconv.ParseInt(sampleData[valueIndex+1], 10, 64); err != nil { + return value, blocksize, addrs, fmt.Errorf("malformed sample: %s: %v", line, err) + } + + if v1 == 0 { + if v2 != 0 { + return value, blocksize, addrs, fmt.Errorf("allocation count was 0 but allocation bytes was %d", v2) + } + } else { + blocksize = v2 / v1 + if sampling == "v2" { + v1, v2 = scaleHeapSample(v1, v2, rate) + } + } + + value = []int64{v1, v2} + addrs = parseHexAddresses(sampleData[5]) + + return value, blocksize, addrs, nil +} + +// extractHexAddresses extracts hex numbers from a string and returns +// them, together with their numeric value, in a slice. +func extractHexAddresses(s string) ([]string, []uint64) { + hexStrings := hexNumberRE.FindAllString(s, -1) + var ids []uint64 + for _, s := range hexStrings { + if id, err := strconv.ParseUint(s, 0, 64); err == nil { + ids = append(ids, id) + } else { + // Do not expect any parsing failures due to the regexp matching. + panic("failed to parse hex value:" + s) + } + } + return hexStrings, ids +} + +// parseHexAddresses parses hex numbers from a string and returns them +// in a slice. +func parseHexAddresses(s string) []uint64 { + _, ids := extractHexAddresses(s) + return ids +} + +// scaleHeapSample adjusts the data from a heapz Sample to +// account for its probability of appearing in the collected +// data. heapz profiles are a sampling of the memory allocations +// requests in a program. We estimate the unsampled value by dividing +// each collected sample by its probability of appearing in the +// profile. heapz v2 profiles rely on a poisson process to determine +// which samples to collect, based on the desired average collection +// rate R. The probability of a sample of size S to appear in that +// profile is 1-exp(-S/R). +func scaleHeapSample(count, size, rate int64) (int64, int64) { + if count == 0 || size == 0 { + return 0, 0 + } + + if rate <= 1 { + // if rate==1 all samples were collected so no adjustment is needed. + // if rate<1 treat as unknown and skip scaling. + return count, size + } + + avgSize := float64(size) / float64(count) + scale := 1 / (1 - math.Exp(-avgSize/float64(rate))) + + return int64(float64(count) * scale), int64(float64(size) * scale) +} + +// parseContention parses a mutex or contention profile. There are 2 cases: +// "--- contentionz " for legacy C++ profiles (and backwards compatibility) +// "--- mutex:" or "--- contention:" for profiles generated by the Go runtime. +// This code converts the text output from runtime into a *Profile. (In the future +// the runtime might write a serialized Profile directly making this unnecessary.) +func parseContention(b []byte) (*Profile, error) { + r := bytes.NewBuffer(b) + var l string + var err error + for { + // Skip past comments and empty lines seeking a real header. + l, err = r.ReadString('\n') + if err != nil { + return nil, err + } + if !isSpaceOrComment(l) { + break + } + } + + if strings.HasPrefix(l, "--- contentionz ") { + return parseCppContention(r) + } else if strings.HasPrefix(l, "--- mutex:") { + return parseCppContention(r) + } else if strings.HasPrefix(l, "--- contention:") { + return parseCppContention(r) + } + return nil, errUnrecognized +} + +// parseCppContention parses the output from synchronization_profiling.cc +// for backward compatibility, and the compatible (non-debug) block profile +// output from the Go runtime. +func parseCppContention(r *bytes.Buffer) (*Profile, error) { + p := &Profile{ + PeriodType: &ValueType{Type: "contentions", Unit: "count"}, + Period: 1, + SampleType: []*ValueType{ + {Type: "contentions", Unit: "count"}, + {Type: "delay", Unit: "nanoseconds"}, + }, + } + + var cpuHz int64 + var l string + var err error + // Parse text of the form "attribute = value" before the samples. + const delimiter = "=" + for { + l, err = r.ReadString('\n') + if err != nil { + if err != io.EOF { + return nil, err + } + + if l == "" { + break + } + } + if isSpaceOrComment(l) { + continue + } + + if l = strings.TrimSpace(l); l == "" { + continue + } + + if strings.HasPrefix(l, "---") { + break + } + + attr := strings.SplitN(l, delimiter, 2) + if len(attr) != 2 { + break + } + key, val := strings.TrimSpace(attr[0]), strings.TrimSpace(attr[1]) + var err error + switch key { + case "cycles/second": + if cpuHz, err = strconv.ParseInt(val, 0, 64); err != nil { + return nil, errUnrecognized + } + case "sampling period": + if p.Period, err = strconv.ParseInt(val, 0, 64); err != nil { + return nil, errUnrecognized + } + case "ms since reset": + ms, err := strconv.ParseInt(val, 0, 64) + if err != nil { + return nil, errUnrecognized + } + p.DurationNanos = ms * 1000 * 1000 + case "format": + // CPP contentionz profiles don't have format. + return nil, errUnrecognized + case "resolution": + // CPP contentionz profiles don't have resolution. + return nil, errUnrecognized + case "discarded samples": + default: + return nil, errUnrecognized + } + } + + locs := make(map[uint64]*Location) + for { + if !isSpaceOrComment(l) { + if l = strings.TrimSpace(l); strings.HasPrefix(l, "---") { + break + } + value, addrs, err := parseContentionSample(l, p.Period, cpuHz) + if err != nil { + return nil, err + } + var sloc []*Location + for _, addr := range addrs { + // Addresses from stack traces point to the next instruction after + // each call. Adjust by -1 to land somewhere on the actual call. + addr-- + loc := locs[addr] + if locs[addr] == nil { + loc = &Location{ + Address: addr, + } + p.Location = append(p.Location, loc) + locs[addr] = loc + } + sloc = append(sloc, loc) + } + p.Sample = append(p.Sample, &Sample{ + Value: value, + Location: sloc, + }) + } + + if l, err = r.ReadString('\n'); err != nil { + if err != io.EOF { + return nil, err + } + if l == "" { + break + } + } + } + + if err = parseAdditionalSections(l, r, p); err != nil { + return nil, err + } + + return p, nil +} + +// parseContentionSample parses a single row from a contention profile +// into a new Sample. +func parseContentionSample(line string, period, cpuHz int64) (value []int64, addrs []uint64, err error) { + sampleData := contentionSampleRE.FindStringSubmatch(line) + if sampleData == nil { + return value, addrs, errUnrecognized + } + + v1, err := strconv.ParseInt(sampleData[1], 10, 64) + if err != nil { + return value, addrs, fmt.Errorf("malformed sample: %s: %v", line, err) + } + v2, err := strconv.ParseInt(sampleData[2], 10, 64) + if err != nil { + return value, addrs, fmt.Errorf("malformed sample: %s: %v", line, err) + } + + // Unsample values if period and cpuHz are available. + // - Delays are scaled to cycles and then to nanoseconds. + // - Contentions are scaled to cycles. + if period > 0 { + if cpuHz > 0 { + cpuGHz := float64(cpuHz) / 1e9 + v1 = int64(float64(v1) * float64(period) / cpuGHz) + } + v2 = v2 * period + } + + value = []int64{v2, v1} + addrs = parseHexAddresses(sampleData[3]) + + return value, addrs, nil +} + +// parseThread parses a Threadz profile and returns a new Profile. +func parseThread(b []byte) (*Profile, error) { + r := bytes.NewBuffer(b) + + var line string + var err error + for { + // Skip past comments and empty lines seeking a real header. + line, err = r.ReadString('\n') + if err != nil { + return nil, err + } + if !isSpaceOrComment(line) { + break + } + } + + if m := threadzStartRE.FindStringSubmatch(line); m != nil { + // Advance over initial comments until first stack trace. + for { + line, err = r.ReadString('\n') + if err != nil { + if err != io.EOF { + return nil, err + } + + if line == "" { + break + } + } + if sectionTrigger(line) != unrecognizedSection || line[0] == '-' { + break + } + } + } else if t := threadStartRE.FindStringSubmatch(line); len(t) != 4 { + return nil, errUnrecognized + } + + p := &Profile{ + SampleType: []*ValueType{{Type: "thread", Unit: "count"}}, + PeriodType: &ValueType{Type: "thread", Unit: "count"}, + Period: 1, + } + + locs := make(map[uint64]*Location) + // Recognize each thread and populate profile samples. + for sectionTrigger(line) == unrecognizedSection { + if strings.HasPrefix(line, "---- no stack trace for") { + line = "" + break + } + if t := threadStartRE.FindStringSubmatch(line); len(t) != 4 { + return nil, errUnrecognized + } + + var addrs []uint64 + line, addrs, err = parseThreadSample(r) + if err != nil { + return nil, errUnrecognized + } + if len(addrs) == 0 { + // We got a --same as previous threads--. Bump counters. + if len(p.Sample) > 0 { + s := p.Sample[len(p.Sample)-1] + s.Value[0]++ + } + continue + } + + var sloc []*Location + for _, addr := range addrs { + // Addresses from stack traces point to the next instruction after + // each call. Adjust by -1 to land somewhere on the actual call. + addr-- + loc := locs[addr] + if locs[addr] == nil { + loc = &Location{ + Address: addr, + } + p.Location = append(p.Location, loc) + locs[addr] = loc + } + sloc = append(sloc, loc) + } + + p.Sample = append(p.Sample, &Sample{ + Value: []int64{1}, + Location: sloc, + }) + } + + if err = parseAdditionalSections(line, r, p); err != nil { + return nil, err + } + + return p, nil +} + +// parseThreadSample parses a symbolized or unsymbolized stack trace. +// Returns the first line after the traceback, the sample (or nil if +// it hits a 'same-as-previous' marker) and an error. +func parseThreadSample(b *bytes.Buffer) (nextl string, addrs []uint64, err error) { + var l string + sameAsPrevious := false + for { + if l, err = b.ReadString('\n'); err != nil { + if err != io.EOF { + return "", nil, err + } + if l == "" { + break + } + } + if l = strings.TrimSpace(l); l == "" { + continue + } + + if strings.HasPrefix(l, "---") { + break + } + if strings.Contains(l, "same as previous thread") { + sameAsPrevious = true + continue + } + + addrs = append(addrs, parseHexAddresses(l)...) + } + + if sameAsPrevious { + return l, nil, nil + } + return l, addrs, nil +} + +// parseAdditionalSections parses any additional sections in the +// profile, ignoring any unrecognized sections. +func parseAdditionalSections(l string, b *bytes.Buffer, p *Profile) (err error) { + for { + if sectionTrigger(l) == memoryMapSection { + break + } + // Ignore any unrecognized sections. + if l, err := b.ReadString('\n'); err != nil { + if err != io.EOF { + return err + } + if l == "" { + break + } + } + } + return p.ParseMemoryMap(b) +} + +// ParseMemoryMap parses a memory map in the format of +// /proc/self/maps, and overrides the mappings in the current profile. +// It renumbers the samples and locations in the profile correspondingly. +func (p *Profile) ParseMemoryMap(rd io.Reader) error { + b := bufio.NewReader(rd) + + var attrs []string + var r *strings.Replacer + const delimiter = "=" + for { + l, err := b.ReadString('\n') + if err != nil { + if err != io.EOF { + return err + } + if l == "" { + break + } + } + if l = strings.TrimSpace(l); l == "" { + continue + } + + if r != nil { + l = r.Replace(l) + } + m, err := parseMappingEntry(l) + if err != nil { + if err == errUnrecognized { + // Recognize assignments of the form: attr=value, and replace + // $attr with value on subsequent mappings. + if attr := strings.SplitN(l, delimiter, 2); len(attr) == 2 { + attrs = append(attrs, "$"+strings.TrimSpace(attr[0]), strings.TrimSpace(attr[1])) + r = strings.NewReplacer(attrs...) + } + // Ignore any unrecognized entries + continue + } + return err + } + if m == nil || (m.File == "" && len(p.Mapping) != 0) { + // In some cases the first entry may include the address range + // but not the name of the file. It should be followed by + // another entry with the name. + continue + } + if len(p.Mapping) == 1 && p.Mapping[0].File == "" { + // Update the name if this is the entry following that empty one. + p.Mapping[0].File = m.File + continue + } + p.Mapping = append(p.Mapping, m) + } + p.remapLocationIDs() + p.remapFunctionIDs() + p.remapMappingIDs() + return nil +} + +func parseMappingEntry(l string) (*Mapping, error) { + mapping := &Mapping{} + var err error + if me := procMapsRE.FindStringSubmatch(l); len(me) == 9 { + if !strings.Contains(me[3], "x") { + // Skip non-executable entries. + return nil, nil + } + if mapping.Start, err = strconv.ParseUint(me[1], 16, 64); err != nil { + return nil, errUnrecognized + } + if mapping.Limit, err = strconv.ParseUint(me[2], 16, 64); err != nil { + return nil, errUnrecognized + } + if me[4] != "" { + if mapping.Offset, err = strconv.ParseUint(me[4], 16, 64); err != nil { + return nil, errUnrecognized + } + } + mapping.File = me[8] + return mapping, nil + } + + if me := briefMapsRE.FindStringSubmatch(l); len(me) == 6 { + if mapping.Start, err = strconv.ParseUint(me[1], 16, 64); err != nil { + return nil, errUnrecognized + } + if mapping.Limit, err = strconv.ParseUint(me[2], 16, 64); err != nil { + return nil, errUnrecognized + } + mapping.File = me[3] + if me[5] != "" { + if mapping.Offset, err = strconv.ParseUint(me[5], 16, 64); err != nil { + return nil, errUnrecognized + } + } + return mapping, nil + } + + return nil, errUnrecognized +} + +type sectionType int + +const ( + unrecognizedSection sectionType = iota + memoryMapSection +) + +var memoryMapTriggers = []string{ + "--- Memory map: ---", + "MAPPED_LIBRARIES:", +} + +func sectionTrigger(line string) sectionType { + for _, trigger := range memoryMapTriggers { + if strings.Contains(line, trigger) { + return memoryMapSection + } + } + return unrecognizedSection +} + +func (p *Profile) addLegacyFrameInfo() { + switch { + case isProfileType(p, heapzSampleTypes) || + isProfileType(p, heapzInUseSampleTypes) || + isProfileType(p, heapzAllocSampleTypes): + p.DropFrames, p.KeepFrames = allocRxStr, allocSkipRxStr + case isProfileType(p, contentionzSampleTypes): + p.DropFrames, p.KeepFrames = lockRxStr, "" + default: + p.DropFrames, p.KeepFrames = cpuProfilerRxStr, "" + } +} + +var heapzSampleTypes = []string{"allocations", "size"} // early Go pprof profiles +var heapzInUseSampleTypes = []string{"inuse_objects", "inuse_space"} +var heapzAllocSampleTypes = []string{"alloc_objects", "alloc_space"} +var contentionzSampleTypes = []string{"contentions", "delay"} + +func isProfileType(p *Profile, t []string) bool { + st := p.SampleType + if len(st) != len(t) { + return false + } + + for i := range st { + if st[i].Type != t[i] { + return false + } + } + return true +} + +var allocRxStr = strings.Join([]string{ + // POSIX entry points. + `calloc`, + `cfree`, + `malloc`, + `free`, + `memalign`, + `do_memalign`, + `(__)?posix_memalign`, + `pvalloc`, + `valloc`, + `realloc`, + + // TC malloc. + `tcmalloc::.*`, + `tc_calloc`, + `tc_cfree`, + `tc_malloc`, + `tc_free`, + `tc_memalign`, + `tc_posix_memalign`, + `tc_pvalloc`, + `tc_valloc`, + `tc_realloc`, + `tc_new`, + `tc_delete`, + `tc_newarray`, + `tc_deletearray`, + `tc_new_nothrow`, + `tc_newarray_nothrow`, + + // Memory-allocation routines on OS X. + `malloc_zone_malloc`, + `malloc_zone_calloc`, + `malloc_zone_valloc`, + `malloc_zone_realloc`, + `malloc_zone_memalign`, + `malloc_zone_free`, + + // Go runtime + `runtime\..*`, + + // Other misc. memory allocation routines + `BaseArena::.*`, + `(::)?do_malloc_no_errno`, + `(::)?do_malloc_pages`, + `(::)?do_malloc`, + `DoSampledAllocation`, + `MallocedMemBlock::MallocedMemBlock`, + `_M_allocate`, + `__builtin_(vec_)?delete`, + `__builtin_(vec_)?new`, + `__gnu_cxx::new_allocator::allocate`, + `__libc_malloc`, + `__malloc_alloc_template::allocate`, + `allocate`, + `cpp_alloc`, + `operator new(\[\])?`, + `simple_alloc::allocate`, +}, `|`) + +var allocSkipRxStr = strings.Join([]string{ + // Preserve Go runtime frames that appear in the middle/bottom of + // the stack. + `runtime\.panic`, + `runtime\.reflectcall`, + `runtime\.call[0-9]*`, +}, `|`) + +var cpuProfilerRxStr = strings.Join([]string{ + `ProfileData::Add`, + `ProfileData::prof_handler`, + `CpuProfiler::prof_handler`, + `__pthread_sighandler`, + `__restore`, +}, `|`) + +var lockRxStr = strings.Join([]string{ + `RecordLockProfileData`, + `(base::)?RecordLockProfileData.*`, + `(base::)?SubmitMutexProfileData.*`, + `(base::)?SubmitSpinLockProfileData.*`, + `(Mutex::)?AwaitCommon.*`, + `(Mutex::)?Unlock.*`, + `(Mutex::)?UnlockSlow.*`, + `(Mutex::)?ReaderUnlock.*`, + `(MutexLock::)?~MutexLock.*`, + `(SpinLock::)?Unlock.*`, + `(SpinLock::)?SlowUnlock.*`, + `(SpinLockHolder::)?~SpinLockHolder.*`, +}, `|`) diff --git a/libgo/go/runtime/pprof/internal/profile/profile.go b/libgo/go/runtime/pprof/internal/profile/profile.go new file mode 100644 index 00000000000..9b6a6f9aa9b --- /dev/null +++ b/libgo/go/runtime/pprof/internal/profile/profile.go @@ -0,0 +1,575 @@ +// Copyright 2014 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package profile provides a representation of profile.proto and +// methods to encode/decode profiles in this format. +// +// This package is only for testing runtime/pprof. +// It is not used by production Go programs. +package profile + +import ( + "bytes" + "compress/gzip" + "fmt" + "io" + "io/ioutil" + "regexp" + "strings" + "time" +) + +// Profile is an in-memory representation of profile.proto. +type Profile struct { + SampleType []*ValueType + Sample []*Sample + Mapping []*Mapping + Location []*Location + Function []*Function + + DropFrames string + KeepFrames string + + TimeNanos int64 + DurationNanos int64 + PeriodType *ValueType + Period int64 + + dropFramesX int64 + keepFramesX int64 + stringTable []string +} + +// ValueType corresponds to Profile.ValueType +type ValueType struct { + Type string // cpu, wall, inuse_space, etc + Unit string // seconds, nanoseconds, bytes, etc + + typeX int64 + unitX int64 +} + +// Sample corresponds to Profile.Sample +type Sample struct { + Location []*Location + Value []int64 + Label map[string][]string + NumLabel map[string][]int64 + + locationIDX []uint64 + labelX []Label +} + +// Label corresponds to Profile.Label +type Label struct { + keyX int64 + // Exactly one of the two following values must be set + strX int64 + numX int64 // Integer value for this label +} + +// Mapping corresponds to Profile.Mapping +type Mapping struct { + ID uint64 + Start uint64 + Limit uint64 + Offset uint64 + File string + BuildID string + HasFunctions bool + HasFilenames bool + HasLineNumbers bool + HasInlineFrames bool + + fileX int64 + buildIDX int64 +} + +// Location corresponds to Profile.Location +type Location struct { + ID uint64 + Mapping *Mapping + Address uint64 + Line []Line + + mappingIDX uint64 +} + +// Line corresponds to Profile.Line +type Line struct { + Function *Function + Line int64 + + functionIDX uint64 +} + +// Function corresponds to Profile.Function +type Function struct { + ID uint64 + Name string + SystemName string + Filename string + StartLine int64 + + nameX int64 + systemNameX int64 + filenameX int64 +} + +// Parse parses a profile and checks for its validity. The input +// may be a gzip-compressed encoded protobuf or one of many legacy +// profile formats which may be unsupported in the future. +func Parse(r io.Reader) (*Profile, error) { + orig, err := ioutil.ReadAll(r) + if err != nil { + return nil, err + } + + var p *Profile + if len(orig) >= 2 && orig[0] == 0x1f && orig[1] == 0x8b { + gz, err := gzip.NewReader(bytes.NewBuffer(orig)) + if err != nil { + return nil, fmt.Errorf("decompressing profile: %v", err) + } + data, err := ioutil.ReadAll(gz) + if err != nil { + return nil, fmt.Errorf("decompressing profile: %v", err) + } + orig = data + } + if p, err = parseUncompressed(orig); err != nil { + if p, err = parseLegacy(orig); err != nil { + return nil, fmt.Errorf("parsing profile: %v", err) + } + } + + if err := p.CheckValid(); err != nil { + return nil, fmt.Errorf("malformed profile: %v", err) + } + return p, nil +} + +var errUnrecognized = fmt.Errorf("unrecognized profile format") +var errMalformed = fmt.Errorf("malformed profile format") + +func parseLegacy(data []byte) (*Profile, error) { + parsers := []func([]byte) (*Profile, error){ + parseCPU, + parseHeap, + parseGoCount, // goroutine, threadcreate + parseThread, + parseContention, + } + + for _, parser := range parsers { + p, err := parser(data) + if err == nil { + p.setMain() + p.addLegacyFrameInfo() + return p, nil + } + if err != errUnrecognized { + return nil, err + } + } + return nil, errUnrecognized +} + +func parseUncompressed(data []byte) (*Profile, error) { + p := &Profile{} + if err := unmarshal(data, p); err != nil { + return nil, err + } + + if err := p.postDecode(); err != nil { + return nil, err + } + + return p, nil +} + +var libRx = regexp.MustCompile(`([.]so$|[.]so[._][0-9]+)`) + +// setMain scans Mapping entries and guesses which entry is main +// because legacy profiles don't obey the convention of putting main +// first. +func (p *Profile) setMain() { + for i := 0; i < len(p.Mapping); i++ { + file := strings.TrimSpace(strings.Replace(p.Mapping[i].File, "(deleted)", "", -1)) + if len(file) == 0 { + continue + } + if len(libRx.FindStringSubmatch(file)) > 0 { + continue + } + if strings.HasPrefix(file, "[") { + continue + } + // Swap what we guess is main to position 0. + tmp := p.Mapping[i] + p.Mapping[i] = p.Mapping[0] + p.Mapping[0] = tmp + break + } +} + +// Write writes the profile as a gzip-compressed marshaled protobuf. +func (p *Profile) Write(w io.Writer) error { + p.preEncode() + b := marshal(p) + zw := gzip.NewWriter(w) + defer zw.Close() + _, err := zw.Write(b) + return err +} + +// CheckValid tests whether the profile is valid. Checks include, but are +// not limited to: +// - len(Profile.Sample[n].value) == len(Profile.value_unit) +// - Sample.id has a corresponding Profile.Location +func (p *Profile) CheckValid() error { + // Check that sample values are consistent + sampleLen := len(p.SampleType) + if sampleLen == 0 && len(p.Sample) != 0 { + return fmt.Errorf("missing sample type information") + } + for _, s := range p.Sample { + if len(s.Value) != sampleLen { + return fmt.Errorf("mismatch: sample has: %d values vs. %d types", len(s.Value), len(p.SampleType)) + } + } + + // Check that all mappings/locations/functions are in the tables + // Check that there are no duplicate ids + mappings := make(map[uint64]*Mapping, len(p.Mapping)) + for _, m := range p.Mapping { + if m.ID == 0 { + return fmt.Errorf("found mapping with reserved ID=0") + } + if mappings[m.ID] != nil { + return fmt.Errorf("multiple mappings with same id: %d", m.ID) + } + mappings[m.ID] = m + } + functions := make(map[uint64]*Function, len(p.Function)) + for _, f := range p.Function { + if f.ID == 0 { + return fmt.Errorf("found function with reserved ID=0") + } + if functions[f.ID] != nil { + return fmt.Errorf("multiple functions with same id: %d", f.ID) + } + functions[f.ID] = f + } + locations := make(map[uint64]*Location, len(p.Location)) + for _, l := range p.Location { + if l.ID == 0 { + return fmt.Errorf("found location with reserved id=0") + } + if locations[l.ID] != nil { + return fmt.Errorf("multiple locations with same id: %d", l.ID) + } + locations[l.ID] = l + if m := l.Mapping; m != nil { + if m.ID == 0 || mappings[m.ID] != m { + return fmt.Errorf("inconsistent mapping %p: %d", m, m.ID) + } + } + for _, ln := range l.Line { + if f := ln.Function; f != nil { + if f.ID == 0 || functions[f.ID] != f { + return fmt.Errorf("inconsistent function %p: %d", f, f.ID) + } + } + } + } + return nil +} + +// Aggregate merges the locations in the profile into equivalence +// classes preserving the request attributes. It also updates the +// samples to point to the merged locations. +func (p *Profile) Aggregate(inlineFrame, function, filename, linenumber, address bool) error { + for _, m := range p.Mapping { + m.HasInlineFrames = m.HasInlineFrames && inlineFrame + m.HasFunctions = m.HasFunctions && function + m.HasFilenames = m.HasFilenames && filename + m.HasLineNumbers = m.HasLineNumbers && linenumber + } + + // Aggregate functions + if !function || !filename { + for _, f := range p.Function { + if !function { + f.Name = "" + f.SystemName = "" + } + if !filename { + f.Filename = "" + } + } + } + + // Aggregate locations + if !inlineFrame || !address || !linenumber { + for _, l := range p.Location { + if !inlineFrame && len(l.Line) > 1 { + l.Line = l.Line[len(l.Line)-1:] + } + if !linenumber { + for i := range l.Line { + l.Line[i].Line = 0 + } + } + if !address { + l.Address = 0 + } + } + } + + return p.CheckValid() +} + +// Print dumps a text representation of a profile. Intended mainly +// for debugging purposes. +func (p *Profile) String() string { + + ss := make([]string, 0, len(p.Sample)+len(p.Mapping)+len(p.Location)) + if pt := p.PeriodType; pt != nil { + ss = append(ss, fmt.Sprintf("PeriodType: %s %s", pt.Type, pt.Unit)) + } + ss = append(ss, fmt.Sprintf("Period: %d", p.Period)) + if p.TimeNanos != 0 { + ss = append(ss, fmt.Sprintf("Time: %v", time.Unix(0, p.TimeNanos))) + } + if p.DurationNanos != 0 { + ss = append(ss, fmt.Sprintf("Duration: %v", time.Duration(p.DurationNanos))) + } + + ss = append(ss, "Samples:") + var sh1 string + for _, s := range p.SampleType { + sh1 = sh1 + fmt.Sprintf("%s/%s ", s.Type, s.Unit) + } + ss = append(ss, strings.TrimSpace(sh1)) + for _, s := range p.Sample { + var sv string + for _, v := range s.Value { + sv = fmt.Sprintf("%s %10d", sv, v) + } + sv = sv + ": " + for _, l := range s.Location { + sv = sv + fmt.Sprintf("%d ", l.ID) + } + ss = append(ss, sv) + const labelHeader = " " + if len(s.Label) > 0 { + ls := labelHeader + for k, v := range s.Label { + ls = ls + fmt.Sprintf("%s:%v ", k, v) + } + ss = append(ss, ls) + } + if len(s.NumLabel) > 0 { + ls := labelHeader + for k, v := range s.NumLabel { + ls = ls + fmt.Sprintf("%s:%v ", k, v) + } + ss = append(ss, ls) + } + } + + ss = append(ss, "Locations") + for _, l := range p.Location { + locStr := fmt.Sprintf("%6d: %#x ", l.ID, l.Address) + if m := l.Mapping; m != nil { + locStr = locStr + fmt.Sprintf("M=%d ", m.ID) + } + if len(l.Line) == 0 { + ss = append(ss, locStr) + } + for li := range l.Line { + lnStr := "??" + if fn := l.Line[li].Function; fn != nil { + lnStr = fmt.Sprintf("%s %s:%d s=%d", + fn.Name, + fn.Filename, + l.Line[li].Line, + fn.StartLine) + if fn.Name != fn.SystemName { + lnStr = lnStr + "(" + fn.SystemName + ")" + } + } + ss = append(ss, locStr+lnStr) + // Do not print location details past the first line + locStr = " " + } + } + + ss = append(ss, "Mappings") + for _, m := range p.Mapping { + bits := "" + if m.HasFunctions { + bits = bits + "[FN]" + } + if m.HasFilenames { + bits = bits + "[FL]" + } + if m.HasLineNumbers { + bits = bits + "[LN]" + } + if m.HasInlineFrames { + bits = bits + "[IN]" + } + ss = append(ss, fmt.Sprintf("%d: %#x/%#x/%#x %s %s %s", + m.ID, + m.Start, m.Limit, m.Offset, + m.File, + m.BuildID, + bits)) + } + + return strings.Join(ss, "\n") + "\n" +} + +// Merge adds profile p adjusted by ratio r into profile p. Profiles +// must be compatible (same Type and SampleType). +// TODO(rsilvera): consider normalizing the profiles based on the +// total samples collected. +func (p *Profile) Merge(pb *Profile, r float64) error { + if err := p.Compatible(pb); err != nil { + return err + } + + pb = pb.Copy() + + // Keep the largest of the two periods. + if pb.Period > p.Period { + p.Period = pb.Period + } + + p.DurationNanos += pb.DurationNanos + + p.Mapping = append(p.Mapping, pb.Mapping...) + for i, m := range p.Mapping { + m.ID = uint64(i + 1) + } + p.Location = append(p.Location, pb.Location...) + for i, l := range p.Location { + l.ID = uint64(i + 1) + } + p.Function = append(p.Function, pb.Function...) + for i, f := range p.Function { + f.ID = uint64(i + 1) + } + + if r != 1.0 { + for _, s := range pb.Sample { + for i, v := range s.Value { + s.Value[i] = int64((float64(v) * r)) + } + } + } + p.Sample = append(p.Sample, pb.Sample...) + return p.CheckValid() +} + +// Compatible determines if two profiles can be compared/merged. +// returns nil if the profiles are compatible; otherwise an error with +// details on the incompatibility. +func (p *Profile) Compatible(pb *Profile) error { + if !compatibleValueTypes(p.PeriodType, pb.PeriodType) { + return fmt.Errorf("incompatible period types %v and %v", p.PeriodType, pb.PeriodType) + } + + if len(p.SampleType) != len(pb.SampleType) { + return fmt.Errorf("incompatible sample types %v and %v", p.SampleType, pb.SampleType) + } + + for i := range p.SampleType { + if !compatibleValueTypes(p.SampleType[i], pb.SampleType[i]) { + return fmt.Errorf("incompatible sample types %v and %v", p.SampleType, pb.SampleType) + } + } + + return nil +} + +// HasFunctions determines if all locations in this profile have +// symbolized function information. +func (p *Profile) HasFunctions() bool { + for _, l := range p.Location { + if l.Mapping == nil || !l.Mapping.HasFunctions { + return false + } + } + return true +} + +// HasFileLines determines if all locations in this profile have +// symbolized file and line number information. +func (p *Profile) HasFileLines() bool { + for _, l := range p.Location { + if l.Mapping == nil || (!l.Mapping.HasFilenames || !l.Mapping.HasLineNumbers) { + return false + } + } + return true +} + +func compatibleValueTypes(v1, v2 *ValueType) bool { + if v1 == nil || v2 == nil { + return true // No grounds to disqualify. + } + return v1.Type == v2.Type && v1.Unit == v2.Unit +} + +// Copy makes a fully independent copy of a profile. +func (p *Profile) Copy() *Profile { + p.preEncode() + b := marshal(p) + + pp := &Profile{} + if err := unmarshal(b, pp); err != nil { + panic(err) + } + if err := pp.postDecode(); err != nil { + panic(err) + } + + return pp +} + +// Demangler maps symbol names to a human-readable form. This may +// include C++ demangling and additional simplification. Names that +// are not demangled may be missing from the resulting map. +type Demangler func(name []string) (map[string]string, error) + +// Demangle attempts to demangle and optionally simplify any function +// names referenced in the profile. It works on a best-effort basis: +// it will silently preserve the original names in case of any errors. +func (p *Profile) Demangle(d Demangler) error { + // Collect names to demangle. + var names []string + for _, fn := range p.Function { + names = append(names, fn.SystemName) + } + + // Update profile with demangled names. + demangled, err := d(names) + if err != nil { + return err + } + for _, fn := range p.Function { + if dd, ok := demangled[fn.SystemName]; ok { + fn.Name = dd + } + } + return nil +} + +// Empty returns true if the profile contains no samples. +func (p *Profile) Empty() bool { + return len(p.Sample) == 0 +} diff --git a/libgo/go/runtime/pprof/internal/profile/profile_test.go b/libgo/go/runtime/pprof/internal/profile/profile_test.go new file mode 100644 index 00000000000..e1963f33515 --- /dev/null +++ b/libgo/go/runtime/pprof/internal/profile/profile_test.go @@ -0,0 +1,79 @@ +// Copyright 2015 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package profile + +import ( + "bytes" + "testing" +) + +func TestEmptyProfile(t *testing.T) { + var buf bytes.Buffer + p, err := Parse(&buf) + if err != nil { + t.Error("Want no error, got", err) + } + if p == nil { + t.Fatal("Want a valid profile, got ") + } + if !p.Empty() { + t.Errorf("Profile should be empty, got %#v", p) + } +} + +func TestParseContention(t *testing.T) { + tests := []struct { + name string + in string + wantErr bool + }{ + { + name: "valid", + in: `--- mutex: +cycles/second=3491920901 +sampling period=1 +43227965305 1659640 @ 0x45e851 0x45f764 0x4a2be1 0x44ea31 +34035731690 15760 @ 0x45e851 0x45f764 0x4a2b17 0x44ea31 +`, + }, + { + name: "valid with comment", + in: `--- mutex: +cycles/second=3491920901 +sampling period=1 +43227965305 1659640 @ 0x45e851 0x45f764 0x4a2be1 0x44ea31 +# 0x45e850 sync.(*Mutex).Unlock+0x80 /go/src/sync/mutex.go:126 +# 0x45f763 sync.(*RWMutex).Unlock+0x83 /go/src/sync/rwmutex.go:125 +# 0x4a2be0 main.main.func3+0x70 /go/src/internal/pprof/profile/a_binary.go:58 + +34035731690 15760 @ 0x45e851 0x45f764 0x4a2b17 0x44ea31 +# 0x45e850 sync.(*Mutex).Unlock+0x80 /go/src/sync/mutex.go:126 +# 0x45f763 sync.(*RWMutex).Unlock+0x83 /go/src/sync/rwmutex.go:125 +# 0x4a2b16 main.main.func2+0xd6 /go/src/internal/pprof/profile/a_binary.go:48 +`, + }, + { + name: "empty", + in: `--- mutex:`, + wantErr: true, + }, + { + name: "invalid header", + in: `--- channel: +43227965305 1659640 @ 0x45e851 0x45f764 0x4a2be1 0x44ea31`, + wantErr: true, + }, + } + for _, tc := range tests { + _, err := parseContention([]byte(tc.in)) + if tc.wantErr && err == nil { + t.Errorf("parseContention(%q) succeeded unexpectedly", tc.name) + } + if !tc.wantErr && err != nil { + t.Errorf("parseContention(%q) failed unexpectedly: %v", tc.name, err) + } + } + +} diff --git a/libgo/go/runtime/pprof/internal/profile/proto.go b/libgo/go/runtime/pprof/internal/profile/proto.go new file mode 100644 index 00000000000..11d7f9ff9b3 --- /dev/null +++ b/libgo/go/runtime/pprof/internal/profile/proto.go @@ -0,0 +1,360 @@ +// Copyright 2014 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// This file is a simple protocol buffer encoder and decoder. +// +// A protocol message must implement the message interface: +// decoder() []decoder +// encode(*buffer) +// +// The decode method returns a slice indexed by field number that gives the +// function to decode that field. +// The encode method encodes its receiver into the given buffer. +// +// The two methods are simple enough to be implemented by hand rather than +// by using a protocol compiler. +// +// See profile.go for examples of messages implementing this interface. +// +// There is no support for groups, message sets, or "has" bits. + +package profile + +import "errors" + +type buffer struct { + field int + typ int + u64 uint64 + data []byte + tmp [16]byte +} + +type decoder func(*buffer, message) error + +type message interface { + decoder() []decoder + encode(*buffer) +} + +func marshal(m message) []byte { + var b buffer + m.encode(&b) + return b.data +} + +func encodeVarint(b *buffer, x uint64) { + for x >= 128 { + b.data = append(b.data, byte(x)|0x80) + x >>= 7 + } + b.data = append(b.data, byte(x)) +} + +func encodeLength(b *buffer, tag int, len int) { + encodeVarint(b, uint64(tag)<<3|2) + encodeVarint(b, uint64(len)) +} + +func encodeUint64(b *buffer, tag int, x uint64) { + // append varint to b.data + encodeVarint(b, uint64(tag)<<3|0) + encodeVarint(b, x) +} + +func encodeUint64s(b *buffer, tag int, x []uint64) { + if len(x) > 2 { + // Use packed encoding + n1 := len(b.data) + for _, u := range x { + encodeVarint(b, u) + } + n2 := len(b.data) + encodeLength(b, tag, n2-n1) + n3 := len(b.data) + copy(b.tmp[:], b.data[n2:n3]) + copy(b.data[n1+(n3-n2):], b.data[n1:n2]) + copy(b.data[n1:], b.tmp[:n3-n2]) + return + } + for _, u := range x { + encodeUint64(b, tag, u) + } +} + +func encodeUint64Opt(b *buffer, tag int, x uint64) { + if x == 0 { + return + } + encodeUint64(b, tag, x) +} + +func encodeInt64(b *buffer, tag int, x int64) { + u := uint64(x) + encodeUint64(b, tag, u) +} + +func encodeInt64Opt(b *buffer, tag int, x int64) { + if x == 0 { + return + } + encodeInt64(b, tag, x) +} + +func encodeInt64s(b *buffer, tag int, x []int64) { + if len(x) > 2 { + // Use packed encoding + n1 := len(b.data) + for _, u := range x { + encodeVarint(b, uint64(u)) + } + n2 := len(b.data) + encodeLength(b, tag, n2-n1) + n3 := len(b.data) + copy(b.tmp[:], b.data[n2:n3]) + copy(b.data[n1+(n3-n2):], b.data[n1:n2]) + copy(b.data[n1:], b.tmp[:n3-n2]) + return + } + for _, u := range x { + encodeInt64(b, tag, u) + } +} + +func encodeString(b *buffer, tag int, x string) { + encodeLength(b, tag, len(x)) + b.data = append(b.data, x...) +} + +func encodeStrings(b *buffer, tag int, x []string) { + for _, s := range x { + encodeString(b, tag, s) + } +} + +func encodeStringOpt(b *buffer, tag int, x string) { + if x == "" { + return + } + encodeString(b, tag, x) +} + +func encodeBool(b *buffer, tag int, x bool) { + if x { + encodeUint64(b, tag, 1) + } else { + encodeUint64(b, tag, 0) + } +} + +func encodeBoolOpt(b *buffer, tag int, x bool) { + if x == false { + return + } + encodeBool(b, tag, x) +} + +func encodeMessage(b *buffer, tag int, m message) { + n1 := len(b.data) + m.encode(b) + n2 := len(b.data) + encodeLength(b, tag, n2-n1) + n3 := len(b.data) + copy(b.tmp[:], b.data[n2:n3]) + copy(b.data[n1+(n3-n2):], b.data[n1:n2]) + copy(b.data[n1:], b.tmp[:n3-n2]) +} + +func unmarshal(data []byte, m message) (err error) { + b := buffer{data: data, typ: 2} + return decodeMessage(&b, m) +} + +func le64(p []byte) uint64 { + return uint64(p[0]) | uint64(p[1])<<8 | uint64(p[2])<<16 | uint64(p[3])<<24 | uint64(p[4])<<32 | uint64(p[5])<<40 | uint64(p[6])<<48 | uint64(p[7])<<56 +} + +func le32(p []byte) uint32 { + return uint32(p[0]) | uint32(p[1])<<8 | uint32(p[2])<<16 | uint32(p[3])<<24 +} + +func decodeVarint(data []byte) (uint64, []byte, error) { + var i int + var u uint64 + for i = 0; ; i++ { + if i >= 10 || i >= len(data) { + return 0, nil, errors.New("bad varint") + } + u |= uint64(data[i]&0x7F) << uint(7*i) + if data[i]&0x80 == 0 { + return u, data[i+1:], nil + } + } +} + +func decodeField(b *buffer, data []byte) ([]byte, error) { + x, data, err := decodeVarint(data) + if err != nil { + return nil, err + } + b.field = int(x >> 3) + b.typ = int(x & 7) + b.data = nil + b.u64 = 0 + switch b.typ { + case 0: + b.u64, data, err = decodeVarint(data) + if err != nil { + return nil, err + } + case 1: + if len(data) < 8 { + return nil, errors.New("not enough data") + } + b.u64 = le64(data[:8]) + data = data[8:] + case 2: + var n uint64 + n, data, err = decodeVarint(data) + if err != nil { + return nil, err + } + if n > uint64(len(data)) { + return nil, errors.New("too much data") + } + b.data = data[:n] + data = data[n:] + case 5: + if len(data) < 4 { + return nil, errors.New("not enough data") + } + b.u64 = uint64(le32(data[:4])) + data = data[4:] + default: + return nil, errors.New("unknown type: " + string(b.typ)) + } + + return data, nil +} + +func checkType(b *buffer, typ int) error { + if b.typ != typ { + return errors.New("type mismatch") + } + return nil +} + +func decodeMessage(b *buffer, m message) error { + if err := checkType(b, 2); err != nil { + return err + } + dec := m.decoder() + data := b.data + for len(data) > 0 { + // pull varint field# + type + var err error + data, err = decodeField(b, data) + if err != nil { + return err + } + if b.field >= len(dec) || dec[b.field] == nil { + continue + } + if err := dec[b.field](b, m); err != nil { + return err + } + } + return nil +} + +func decodeInt64(b *buffer, x *int64) error { + if err := checkType(b, 0); err != nil { + return err + } + *x = int64(b.u64) + return nil +} + +func decodeInt64s(b *buffer, x *[]int64) error { + if b.typ == 2 { + // Packed encoding + data := b.data + for len(data) > 0 { + var u uint64 + var err error + + if u, data, err = decodeVarint(data); err != nil { + return err + } + *x = append(*x, int64(u)) + } + return nil + } + var i int64 + if err := decodeInt64(b, &i); err != nil { + return err + } + *x = append(*x, i) + return nil +} + +func decodeUint64(b *buffer, x *uint64) error { + if err := checkType(b, 0); err != nil { + return err + } + *x = b.u64 + return nil +} + +func decodeUint64s(b *buffer, x *[]uint64) error { + if b.typ == 2 { + data := b.data + // Packed encoding + for len(data) > 0 { + var u uint64 + var err error + + if u, data, err = decodeVarint(data); err != nil { + return err + } + *x = append(*x, u) + } + return nil + } + var u uint64 + if err := decodeUint64(b, &u); err != nil { + return err + } + *x = append(*x, u) + return nil +} + +func decodeString(b *buffer, x *string) error { + if err := checkType(b, 2); err != nil { + return err + } + *x = string(b.data) + return nil +} + +func decodeStrings(b *buffer, x *[]string) error { + var s string + if err := decodeString(b, &s); err != nil { + return err + } + *x = append(*x, s) + return nil +} + +func decodeBool(b *buffer, x *bool) error { + if err := checkType(b, 0); err != nil { + return err + } + if int64(b.u64) == 0 { + *x = false + } else { + *x = true + } + return nil +} diff --git a/libgo/go/runtime/pprof/internal/profile/proto_test.go b/libgo/go/runtime/pprof/internal/profile/proto_test.go new file mode 100644 index 00000000000..c2613fc375a --- /dev/null +++ b/libgo/go/runtime/pprof/internal/profile/proto_test.go @@ -0,0 +1,67 @@ +package profile + +import ( + "reflect" + "testing" +) + +func TestPackedEncoding(t *testing.T) { + + type testcase struct { + uint64s []uint64 + int64s []int64 + encoded []byte + } + for i, tc := range []testcase{ + { + []uint64{0, 1, 10, 100, 1000, 10000}, + []int64{1000, 0, 1000}, + []byte{10, 8, 0, 1, 10, 100, 232, 7, 144, 78, 18, 5, 232, 7, 0, 232, 7}, + }, + { + []uint64{10000}, + nil, + []byte{8, 144, 78}, + }, + { + nil, + []int64{-10000}, + []byte{16, 240, 177, 255, 255, 255, 255, 255, 255, 255, 1}, + }, + } { + source := &packedInts{tc.uint64s, tc.int64s} + if got, want := marshal(source), tc.encoded; !reflect.DeepEqual(got, want) { + t.Errorf("failed encode %d, got %v, want %v", i, got, want) + } + + dest := new(packedInts) + if err := unmarshal(tc.encoded, dest); err != nil { + t.Errorf("failed decode %d: %v", i, err) + continue + } + if got, want := dest.uint64s, tc.uint64s; !reflect.DeepEqual(got, want) { + t.Errorf("failed decode uint64s %d, got %v, want %v", i, got, want) + } + if got, want := dest.int64s, tc.int64s; !reflect.DeepEqual(got, want) { + t.Errorf("failed decode int64s %d, got %v, want %v", i, got, want) + } + } +} + +type packedInts struct { + uint64s []uint64 + int64s []int64 +} + +func (u *packedInts) decoder() []decoder { + return []decoder{ + nil, + func(b *buffer, m message) error { return decodeUint64s(b, &m.(*packedInts).uint64s) }, + func(b *buffer, m message) error { return decodeInt64s(b, &m.(*packedInts).int64s) }, + } +} + +func (u *packedInts) encode(b *buffer) { + encodeUint64s(b, 1, u.uint64s) + encodeInt64s(b, 2, u.int64s) +} diff --git a/libgo/go/runtime/pprof/internal/profile/prune.go b/libgo/go/runtime/pprof/internal/profile/prune.go new file mode 100644 index 00000000000..1924fada7a5 --- /dev/null +++ b/libgo/go/runtime/pprof/internal/profile/prune.go @@ -0,0 +1,97 @@ +// Copyright 2014 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Implements methods to remove frames from profiles. + +package profile + +import ( + "fmt" + "regexp" +) + +// Prune removes all nodes beneath a node matching dropRx, and not +// matching keepRx. If the root node of a Sample matches, the sample +// will have an empty stack. +func (p *Profile) Prune(dropRx, keepRx *regexp.Regexp) { + prune := make(map[uint64]bool) + pruneBeneath := make(map[uint64]bool) + + for _, loc := range p.Location { + var i int + for i = len(loc.Line) - 1; i >= 0; i-- { + if fn := loc.Line[i].Function; fn != nil && fn.Name != "" { + funcName := fn.Name + // Account for leading '.' on the PPC ELF v1 ABI. + if funcName[0] == '.' { + funcName = funcName[1:] + } + if dropRx.MatchString(funcName) { + if keepRx == nil || !keepRx.MatchString(funcName) { + break + } + } + } + } + + if i >= 0 { + // Found matching entry to prune. + pruneBeneath[loc.ID] = true + + // Remove the matching location. + if i == len(loc.Line)-1 { + // Matched the top entry: prune the whole location. + prune[loc.ID] = true + } else { + loc.Line = loc.Line[i+1:] + } + } + } + + // Prune locs from each Sample + for _, sample := range p.Sample { + // Scan from the root to the leaves to find the prune location. + // Do not prune frames before the first user frame, to avoid + // pruning everything. + foundUser := false + for i := len(sample.Location) - 1; i >= 0; i-- { + id := sample.Location[i].ID + if !prune[id] && !pruneBeneath[id] { + foundUser = true + continue + } + if !foundUser { + continue + } + if prune[id] { + sample.Location = sample.Location[i+1:] + break + } + if pruneBeneath[id] { + sample.Location = sample.Location[i:] + break + } + } + } +} + +// RemoveUninteresting prunes and elides profiles using built-in +// tables of uninteresting function names. +func (p *Profile) RemoveUninteresting() error { + var keep, drop *regexp.Regexp + var err error + + if p.DropFrames != "" { + if drop, err = regexp.Compile("^(" + p.DropFrames + ")$"); err != nil { + return fmt.Errorf("failed to compile regexp %s: %v", p.DropFrames, err) + } + if p.KeepFrames != "" { + if keep, err = regexp.Compile("^(" + p.KeepFrames + ")$"); err != nil { + return fmt.Errorf("failed to compile regexp %s: %v", p.KeepFrames, err) + } + } + p.Prune(drop, keep) + } + return nil +} diff --git a/libgo/go/runtime/pprof/internal/protopprof/protomemprofile.go b/libgo/go/runtime/pprof/internal/protopprof/protomemprofile.go deleted file mode 100644 index c2ab5b57025..00000000000 --- a/libgo/go/runtime/pprof/internal/protopprof/protomemprofile.go +++ /dev/null @@ -1,83 +0,0 @@ -// Copyright 2016 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package protopprof - -import ( - "internal/pprof/profile" - "math" - "runtime" - "time" -) - -// EncodeMemProfile converts MemProfileRecords to a Profile. -func EncodeMemProfile(mr []runtime.MemProfileRecord, rate int64, t time.Time) *profile.Profile { - p := &profile.Profile{ - Period: rate, - PeriodType: &profile.ValueType{Type: "space", Unit: "bytes"}, - SampleType: []*profile.ValueType{ - {Type: "alloc_objects", Unit: "count"}, - {Type: "alloc_space", Unit: "bytes"}, - {Type: "inuse_objects", Unit: "count"}, - {Type: "inuse_space", Unit: "bytes"}, - }, - TimeNanos: int64(t.UnixNano()), - } - - locs := make(map[uintptr]*profile.Location) - for _, r := range mr { - stack := r.Stack() - sloc := make([]*profile.Location, len(stack)) - for i, addr := range stack { - loc := locs[addr] - if loc == nil { - loc = &profile.Location{ - ID: uint64(len(p.Location) + 1), - Address: uint64(addr), - } - locs[addr] = loc - p.Location = append(p.Location, loc) - } - sloc[i] = loc - } - - ao, ab := scaleHeapSample(r.AllocObjects, r.AllocBytes, rate) - uo, ub := scaleHeapSample(r.InUseObjects(), r.InUseBytes(), rate) - - p.Sample = append(p.Sample, &profile.Sample{ - Value: []int64{ao, ab, uo, ub}, - Location: sloc, - }) - } - if runtime.GOOS == "linux" { - addMappings(p) - } - return p -} - -// scaleHeapSample adjusts the data from a heap Sample to -// account for its probability of appearing in the collected -// data. heap profiles are a sampling of the memory allocations -// requests in a program. We estimate the unsampled value by dividing -// each collected sample by its probability of appearing in the -// profile. heap profiles rely on a poisson process to determine -// which samples to collect, based on the desired average collection -// rate R. The probability of a sample of size S to appear in that -// profile is 1-exp(-S/R). -func scaleHeapSample(count, size, rate int64) (int64, int64) { - if count == 0 || size == 0 { - return 0, 0 - } - - if rate <= 1 { - // if rate==1 all samples were collected so no adjustment is needed. - // if rate<1 treat as unknown and skip scaling. - return count, size - } - - avgSize := float64(size) / float64(count) - scale := 1 / (1 - math.Exp(-avgSize/float64(rate))) - - return int64(float64(count) * scale), int64(float64(size) * scale) -} diff --git a/libgo/go/runtime/pprof/internal/protopprof/protomemprofile_test.go b/libgo/go/runtime/pprof/internal/protopprof/protomemprofile_test.go deleted file mode 100644 index a10fe772ccf..00000000000 --- a/libgo/go/runtime/pprof/internal/protopprof/protomemprofile_test.go +++ /dev/null @@ -1,104 +0,0 @@ -// Copyright 2016 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package protopprof - -import ( - "bytes" - "internal/pprof/profile" - "io/ioutil" - "reflect" - "runtime" - "testing" - "time" -) - -// TestSampledHeapAllocProfile tests encoding of a memory profile from -// runtime.MemProfileRecord data. -func TestSampledHeapAllocProfile(t *testing.T) { - if runtime.GOOS != "linux" { - t.Skip("Test requires a system with /proc/self/maps") - } - - // Figure out two addresses from /proc/self/maps. - mmap, err := ioutil.ReadFile("/proc/self/maps") - if err != nil { - t.Fatal("Cannot read /proc/self/maps") - } - rd := bytes.NewReader(mmap) - mprof := &profile.Profile{} - if err = mprof.ParseMemoryMap(rd); err != nil { - t.Fatalf("Cannot parse /proc/self/maps") - } - if len(mprof.Mapping) < 2 { - // It is possible for a binary to only have 1 executable - // region of memory. - t.Skipf("need 2 or more mappings, got %v", len(mprof.Mapping)) - } - address1 := mprof.Mapping[0].Start - address2 := mprof.Mapping[1].Start - - var buf bytes.Buffer - - rec, rate := testMemRecords(address1, address2) - p := EncodeMemProfile(rec, rate, time.Now()) - if err := p.Write(&buf); err != nil { - t.Fatalf("Failed to write profile: %v", err) - } - - p, err = profile.Parse(&buf) - if err != nil { - t.Fatalf("Could not parse Profile profile: %v", err) - } - - // Expected PeriodType, SampleType and Sample. - expectedPeriodType := &profile.ValueType{Type: "space", Unit: "bytes"} - expectedSampleType := []*profile.ValueType{ - {Type: "alloc_objects", Unit: "count"}, - {Type: "alloc_space", Unit: "bytes"}, - {Type: "inuse_objects", Unit: "count"}, - {Type: "inuse_space", Unit: "bytes"}, - } - // Expected samples, with values unsampled according to the profiling rate. - expectedSample := []*profile.Sample{ - {Value: []int64{2050, 2099200, 1537, 1574400}, Location: []*profile.Location{ - {ID: 1, Mapping: mprof.Mapping[0], Address: address1}, - {ID: 2, Mapping: mprof.Mapping[1], Address: address2}, - }}, - {Value: []int64{1, 829411, 1, 829411}, Location: []*profile.Location{ - {ID: 3, Mapping: mprof.Mapping[1], Address: address2 + 1}, - {ID: 4, Mapping: mprof.Mapping[1], Address: address2 + 2}, - }}, - {Value: []int64{1, 829411, 0, 0}, Location: []*profile.Location{ - {ID: 5, Mapping: mprof.Mapping[0], Address: address1 + 1}, - {ID: 6, Mapping: mprof.Mapping[0], Address: address1 + 2}, - {ID: 7, Mapping: mprof.Mapping[1], Address: address2 + 3}, - }}, - } - - if p.Period != 512*1024 { - t.Fatalf("Sampling periods do not match") - } - if !reflect.DeepEqual(p.PeriodType, expectedPeriodType) { - t.Fatalf("Period types do not match") - } - if !reflect.DeepEqual(p.SampleType, expectedSampleType) { - t.Fatalf("Sample types do not match") - } - if !reflect.DeepEqual(p.Sample, expectedSample) { - t.Fatalf("Samples do not match: Expected: %v, Got:%v", getSampleAsString(expectedSample), - getSampleAsString(p.Sample)) - } -} - -func testMemRecords(a1, a2 uint64) ([]runtime.MemProfileRecord, int64) { - addr1, addr2 := uintptr(a1), uintptr(a2) - rate := int64(512 * 1024) - rec := []runtime.MemProfileRecord{ - {4096, 1024, 4, 1, [32]uintptr{addr1, addr2}}, - {512 * 1024, 0, 1, 0, [32]uintptr{addr2 + 1, addr2 + 2}}, - {512 * 1024, 512 * 1024, 1, 1, [32]uintptr{addr1 + 1, addr1 + 2, addr2 + 3}}, - } - return rec, rate -} diff --git a/libgo/go/runtime/pprof/internal/protopprof/protopprof.go b/libgo/go/runtime/pprof/internal/protopprof/protopprof.go deleted file mode 100644 index 5d269c4f652..00000000000 --- a/libgo/go/runtime/pprof/internal/protopprof/protopprof.go +++ /dev/null @@ -1,105 +0,0 @@ -// Copyright 2016 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// Package protopprof converts the runtime's raw profile logs -// to Profile structs containing a representation of the pprof -// protocol buffer profile format. -package protopprof - -import ( - "fmt" - "os" - "runtime" - "time" - "unsafe" - - "internal/pprof/profile" -) - -// TranslateCPUProfile parses binary CPU profiling stack trace data -// generated by runtime.CPUProfile() into a profile struct. -func TranslateCPUProfile(b []byte, startTime time.Time) (*profile.Profile, error) { - const wordSize = unsafe.Sizeof(uintptr(0)) - const minRawProfile = 5 * wordSize // Need a minimum of 5 words. - if uintptr(len(b)) < minRawProfile { - return nil, fmt.Errorf("truncated profile") - } - n := int(uintptr(len(b)) / wordSize) - data := ((*[1 << 28]uintptr)(unsafe.Pointer(&b[0])))[:n:n] - period := data[3] - data = data[5:] // skip header - - // profile initialization taken from pprof tool - p := &profile.Profile{ - Period: int64(period) * 1000, - PeriodType: &profile.ValueType{Type: "cpu", Unit: "nanoseconds"}, - SampleType: []*profile.ValueType{ - {Type: "samples", Unit: "count"}, - {Type: "cpu", Unit: "nanoseconds"}, - }, - TimeNanos: int64(startTime.UnixNano()), - DurationNanos: time.Since(startTime).Nanoseconds(), - } - // Parse CPU samples from the profile. - locs := make(map[uint64]*profile.Location) - for len(b) > 0 { - if len(data) < 2 || uintptr(len(data)) < 2+data[1] { - return nil, fmt.Errorf("truncated profile") - } - count := data[0] - nstk := data[1] - if uintptr(len(data)) < 2+nstk { - return nil, fmt.Errorf("truncated profile") - } - stk := data[2 : 2+nstk] - data = data[2+nstk:] - - if count == 0 && nstk == 1 && stk[0] == 0 { - // end of data marker - break - } - - sloc := make([]*profile.Location, len(stk)) - for i, addr := range stk { - addr := uint64(addr) - // Addresses from stack traces point to the next instruction after - // each call. Adjust by -1 to land somewhere on the actual call - // (except for the leaf, which is not a call). - if i > 0 { - addr-- - } - loc := locs[addr] - if loc == nil { - loc = &profile.Location{ - ID: uint64(len(p.Location) + 1), - Address: addr, - } - locs[addr] = loc - p.Location = append(p.Location, loc) - } - sloc[i] = loc - } - p.Sample = append(p.Sample, &profile.Sample{ - Value: []int64{int64(count), int64(count) * int64(p.Period)}, - Location: sloc, - }) - } - - if runtime.GOOS == "linux" { - if err := addMappings(p); err != nil { - return nil, err - } - } - return p, nil -} - -func addMappings(p *profile.Profile) error { - // Parse memory map from /proc/self/maps - f, err := os.Open("/proc/self/maps") - if err != nil { - return err - } - defer f.Close() - return p.ParseMemoryMap(f) -} diff --git a/libgo/go/runtime/pprof/internal/protopprof/protopprof_test.go b/libgo/go/runtime/pprof/internal/protopprof/protopprof_test.go deleted file mode 100644 index f1937b5bd06..00000000000 --- a/libgo/go/runtime/pprof/internal/protopprof/protopprof_test.go +++ /dev/null @@ -1,171 +0,0 @@ -// Copyright 2016 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package protopprof - -import ( - "bytes" - "fmt" - "internal/pprof/profile" - "io/ioutil" - "reflect" - "runtime" - "testing" - "time" - "unsafe" -) - -// Helper function to initialize empty cpu profile with sampling period provided. -func createEmptyProfileWithPeriod(t *testing.T, periodMs uint64) bytes.Buffer { - // Mock the sample header produced by cpu profiler. Write a sample - // period of 2000 microseconds, followed by no samples. - buf := new(bytes.Buffer) - // Profile header is as follows: - // The first, third and fifth words are 0. The second word is 3. - // The fourth word is the period. - // EOD marker: - // The sixth word -- count is initialized to 0 above. - // The code below sets the seventh word -- nstk to 1 - // The eighth word -- addr is initialized to 0 above. - words := []int{0, 3, 0, int(periodMs), 0, 0, 1, 0} - n := int(unsafe.Sizeof(0)) * len(words) - data := ((*[1 << 29]byte)(unsafe.Pointer(&words[0])))[:n:n] - if _, err := buf.Write(data); err != nil { - t.Fatalf("createEmptyProfileWithPeriod failed: %v", err) - } - return *buf -} - -// Helper function to initialize cpu profile with two sample values. -func createProfileWithTwoSamples(t *testing.T, periodMs uintptr, count1 uintptr, count2 uintptr, - address1 uintptr, address2 uintptr) bytes.Buffer { - // Mock the sample header produced by cpu profiler. Write a sample - // period of 2000 microseconds, followed by no samples. - buf := new(bytes.Buffer) - words := []uintptr{0, 3, 0, uintptr(periodMs), 0, uintptr(count1), 2, - uintptr(address1), uintptr(address1 + 2), - uintptr(count2), 2, uintptr(address2), uintptr(address2 + 2), - 0, 1, 0} - for _, n := range words { - var err error - switch unsafe.Sizeof(int(0)) { - case 8: - _, err = buf.Write((*[8]byte)(unsafe.Pointer(&n))[:8:8]) - case 4: - _, err = buf.Write((*[4]byte)(unsafe.Pointer(&n))[:4:4]) - } - if err != nil { - t.Fatalf("createProfileWithTwoSamples failed: %v", err) - } - } - return *buf -} - -// Tests TranslateCPUProfile parses correct sampling period in an otherwise empty cpu profile. -func TestTranlateCPUProfileSamplingPeriod(t *testing.T) { - // A test server with mock cpu profile data. - var buf bytes.Buffer - - startTime := time.Now() - b := createEmptyProfileWithPeriod(t, 2000) - p, err := TranslateCPUProfile(b.Bytes(), startTime) - if err != nil { - t.Fatalf("translate failed: %v", err) - } - if err := p.Write(&buf); err != nil { - t.Fatalf("write failed: %v", err) - } - - p, err = profile.Parse(&buf) - if err != nil { - t.Fatalf("Could not parse Profile profile: %v", err) - } - - // Expected PeriodType and SampleType. - expectedPeriodType := &profile.ValueType{Type: "cpu", Unit: "nanoseconds"} - expectedSampleType := []*profile.ValueType{ - {Type: "samples", Unit: "count"}, - {Type: "cpu", Unit: "nanoseconds"}, - } - if p.Period != 2000*1000 || !reflect.DeepEqual(p.PeriodType, expectedPeriodType) || - !reflect.DeepEqual(p.SampleType, expectedSampleType) || p.Sample != nil { - t.Fatalf("Unexpected Profile fields") - } -} - -func getSampleAsString(sample []*profile.Sample) string { - var str string - for _, x := range sample { - for _, y := range x.Location { - if y.Mapping != nil { - str += fmt.Sprintf("Mapping:%v\n", *y.Mapping) - } - str += fmt.Sprintf("Location:%v\n", y) - } - str += fmt.Sprintf("Sample:%v\n", *x) - } - return str -} - -// Tests TranslateCPUProfile parses a cpu profile with sample values present. -func TestTranslateCPUProfileWithSamples(t *testing.T) { - if runtime.GOOS != "linux" { - t.Skip("test requires a system with /proc/self/maps") - } - // Figure out two addresses from /proc/self/maps. - mmap, err := ioutil.ReadFile("/proc/self/maps") - if err != nil { - t.Fatal("Cannot read /proc/self/maps") - } - rd := bytes.NewReader(mmap) - mprof := &profile.Profile{} - if err = mprof.ParseMemoryMap(rd); err != nil { - t.Fatalf("Cannot parse /proc/self/maps") - } - if len(mprof.Mapping) < 2 { - // It is possible for a binary to only have 1 executable - // region of memory. - t.Skipf("need 2 or more mappings, got %v", len(mprof.Mapping)) - } - address1 := mprof.Mapping[0].Start - address2 := mprof.Mapping[1].Start - // A test server with mock cpu profile data. - - startTime := time.Now() - b := createProfileWithTwoSamples(t, 2000, 20, 40, uintptr(address1), uintptr(address2)) - p, err := TranslateCPUProfile(b.Bytes(), startTime) - - if err != nil { - t.Fatalf("Could not parse Profile profile: %v", err) - } - // Expected PeriodType, SampleType and Sample. - expectedPeriodType := &profile.ValueType{Type: "cpu", Unit: "nanoseconds"} - expectedSampleType := []*profile.ValueType{ - {Type: "samples", Unit: "count"}, - {Type: "cpu", Unit: "nanoseconds"}, - } - expectedSample := []*profile.Sample{ - {Value: []int64{20, 20 * 2000 * 1000}, Location: []*profile.Location{ - {ID: 1, Mapping: mprof.Mapping[0], Address: address1}, - {ID: 2, Mapping: mprof.Mapping[0], Address: address1 + 1}, - }}, - {Value: []int64{40, 40 * 2000 * 1000}, Location: []*profile.Location{ - {ID: 3, Mapping: mprof.Mapping[1], Address: address2}, - {ID: 4, Mapping: mprof.Mapping[1], Address: address2 + 1}, - }}, - } - if p.Period != 2000*1000 { - t.Fatalf("Sampling periods do not match") - } - if !reflect.DeepEqual(p.PeriodType, expectedPeriodType) { - t.Fatalf("Period types do not match") - } - if !reflect.DeepEqual(p.SampleType, expectedSampleType) { - t.Fatalf("Sample types do not match") - } - if !reflect.DeepEqual(p.Sample, expectedSample) { - t.Fatalf("Samples do not match: Expected: %v, Got:%v", getSampleAsString(expectedSample), - getSampleAsString(p.Sample)) - } -} diff --git a/libgo/go/runtime/pprof/label.go b/libgo/go/runtime/pprof/label.go new file mode 100644 index 00000000000..35647ee3ce1 --- /dev/null +++ b/libgo/go/runtime/pprof/label.go @@ -0,0 +1,85 @@ +// Copyright 2016 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package pprof + +import ( + "context" +) + +type label struct { + key string + value string +} + +// LabelSet is a set of labels. +type LabelSet struct { + list []label +} + +// labelContextKey is the type of contextKeys used for profiler labels. +type labelContextKey struct{} + +func labelValue(ctx context.Context) labelMap { + labels, _ := ctx.Value(labelContextKey{}).(*labelMap) + if labels == nil { + return labelMap(nil) + } + return *labels +} + +// labelMap is the representation of the label set held in the context type. +// This is an initial implementation, but it will be replaced with something +// that admits incremental immutable modification more efficiently. +type labelMap map[string]string + +// WithLabels returns a new context.Context with the given labels added. +// A label overwrites a prior label with the same key. +func WithLabels(ctx context.Context, labels LabelSet) context.Context { + childLabels := make(labelMap) + parentLabels := labelValue(ctx) + // TODO(matloob): replace the map implementation with something + // more efficient so creating a child context WithLabels doesn't need + // to clone the map. + for k, v := range parentLabels { + childLabels[k] = v + } + for _, label := range labels.list { + childLabels[label.key] = label.value + } + return context.WithValue(ctx, labelContextKey{}, &childLabels) +} + +// Labels takes an even number of strings representing key-value pairs +// and makes a LabelSet containing them. +// A label overwrites a prior label with the same key. +func Labels(args ...string) LabelSet { + if len(args)%2 != 0 { + panic("uneven number of arguments to pprof.Labels") + } + labels := LabelSet{} + for i := 0; i+1 < len(args); i += 2 { + labels.list = append(labels.list, label{key: args[i], value: args[i+1]}) + } + return labels +} + +// Label returns the value of the label with the given key on ctx, and a boolean indicating +// whether that label exists. +func Label(ctx context.Context, key string) (string, bool) { + ctxLabels := labelValue(ctx) + v, ok := ctxLabels[key] + return v, ok +} + +// ForLabels invokes f with each label set on the context. +// The function f should return true to continue iteration or false to stop iteration early. +func ForLabels(ctx context.Context, f func(key, value string) bool) { + ctxLabels := labelValue(ctx) + for k, v := range ctxLabels { + if !f(k, v) { + break + } + } +} diff --git a/libgo/go/runtime/pprof/label_test.go b/libgo/go/runtime/pprof/label_test.go new file mode 100644 index 00000000000..240445f098e --- /dev/null +++ b/libgo/go/runtime/pprof/label_test.go @@ -0,0 +1,82 @@ +package pprof + +import ( + "context" + "reflect" + "sort" + "testing" +) + +func labelsSorted(ctx context.Context) []label { + ls := []label{} + ForLabels(ctx, func(key, value string) bool { + ls = append(ls, label{key, value}) + return true + }) + sort.Sort(labelSorter(ls)) + return ls +} + +type labelSorter []label + +func (s labelSorter) Len() int { return len(s) } +func (s labelSorter) Swap(i, j int) { s[i], s[j] = s[j], s[i] } +func (s labelSorter) Less(i, j int) bool { return s[i].key < s[j].key } + +func TestContextLabels(t *testing.T) { + // Background context starts with no lablels. + ctx := context.Background() + labels := labelsSorted(ctx) + if len(labels) != 0 { + t.Errorf("labels on background context: want [], got %v ", labels) + } + + // Add a single label. + ctx = WithLabels(ctx, Labels("key", "value")) + // Retrieve it with Label. + v, ok := Label(ctx, "key") + if !ok || v != "value" { + t.Errorf(`Label(ctx, "key"): got %v, %v; want "value", ok`, v, ok) + } + gotLabels := labelsSorted(ctx) + wantLabels := []label{{"key", "value"}} + if !reflect.DeepEqual(gotLabels, wantLabels) { + t.Errorf("(sorted) labels on context: got %v, want %v", gotLabels, wantLabels) + } + + // Add a label with a different key. + ctx = WithLabels(ctx, Labels("key2", "value2")) + v, ok = Label(ctx, "key2") + if !ok || v != "value2" { + t.Errorf(`Label(ctx, "key2"): got %v, %v; want "value2", ok`, v, ok) + } + gotLabels = labelsSorted(ctx) + wantLabels = []label{{"key", "value"}, {"key2", "value2"}} + if !reflect.DeepEqual(gotLabels, wantLabels) { + t.Errorf("(sorted) labels on context: got %v, want %v", gotLabels, wantLabels) + } + + // Add label with first key to test label replacement. + ctx = WithLabels(ctx, Labels("key", "value3")) + v, ok = Label(ctx, "key") + if !ok || v != "value3" { + t.Errorf(`Label(ctx, "key3"): got %v, %v; want "value3", ok`, v, ok) + } + gotLabels = labelsSorted(ctx) + wantLabels = []label{{"key", "value3"}, {"key2", "value2"}} + if !reflect.DeepEqual(gotLabels, wantLabels) { + t.Errorf("(sorted) labels on context: got %v, want %v", gotLabels, wantLabels) + } + + // Labels called with two labels with the same key should pick the second. + ctx = WithLabels(ctx, Labels("key4", "value4a", "key4", "value4b")) + v, ok = Label(ctx, "key4") + if !ok || v != "value4b" { + t.Errorf(`Label(ctx, "key4"): got %v, %v; want "value4b", ok`, v, ok) + } + gotLabels = labelsSorted(ctx) + wantLabels = []label{{"key", "value3"}, {"key2", "value2"}, {"key4", "value4b"}} + if !reflect.DeepEqual(gotLabels, wantLabels) { + t.Errorf("(sorted) labels on context: got %v, want %v", gotLabels, wantLabels) + } +} diff --git a/libgo/go/runtime/pprof/map.go b/libgo/go/runtime/pprof/map.go new file mode 100644 index 00000000000..a271ad022e7 --- /dev/null +++ b/libgo/go/runtime/pprof/map.go @@ -0,0 +1,89 @@ +// Copyright 2017 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package pprof + +import "unsafe" + +// A profMap is a map from (stack, tag) to mapEntry. +// It grows without bound, but that's assumed to be OK. +type profMap struct { + hash map[uintptr]*profMapEntry + all *profMapEntry + last *profMapEntry + free []profMapEntry + freeStk []uintptr +} + +// A profMapEntry is a single entry in the profMap. +type profMapEntry struct { + nextHash *profMapEntry // next in hash list + nextAll *profMapEntry // next in list of all entries + stk []uintptr + tag unsafe.Pointer + count int64 +} + +func (m *profMap) lookup(stk []uint64, tag unsafe.Pointer) *profMapEntry { + // Compute hash of (stk, tag). + h := uintptr(0) + for _, x := range stk { + h = h<<8 | (h >> (8 * (unsafe.Sizeof(h) - 1))) + h += uintptr(x) * 41 + } + h = h<<8 | (h >> (8 * (unsafe.Sizeof(h) - 1))) + h += uintptr(tag) * 41 + + // Find entry if present. + var last *profMapEntry +Search: + for e := m.hash[h]; e != nil; last, e = e, e.nextHash { + if len(e.stk) != len(stk) || e.tag != tag { + continue + } + for j := range stk { + if e.stk[j] != uintptr(stk[j]) { + continue Search + } + } + // Move to front. + if last != nil { + last.nextHash = e.nextHash + e.nextHash = m.hash[h] + m.hash[h] = e + } + return e + } + + // Add new entry. + if len(m.free) < 1 { + m.free = make([]profMapEntry, 128) + } + e := &m.free[0] + m.free = m.free[1:] + e.nextHash = m.hash[h] + e.tag = tag + + if len(m.freeStk) < len(stk) { + m.freeStk = make([]uintptr, 1024) + } + e.stk = m.freeStk[:len(stk)] + m.freeStk = m.freeStk[len(stk):] + + for j := range stk { + e.stk[j] = uintptr(stk[j]) + } + if m.hash == nil { + m.hash = make(map[uintptr]*profMapEntry) + } + m.hash[h] = e + if m.all == nil { + m.all = e + m.last = e + } else { + m.last.nextAll = e + m.last = e + } + return e +} diff --git a/libgo/go/runtime/pprof/mprof_test.go b/libgo/go/runtime/pprof/mprof_test.go index 5ebd46b198f..5d77a1d8986 100644 --- a/libgo/go/runtime/pprof/mprof_test.go +++ b/libgo/go/runtime/pprof/mprof_test.go @@ -2,7 +2,7 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -package pprof_test +package pprof import ( "bytes" @@ -10,7 +10,6 @@ import ( "reflect" "regexp" "runtime" - . "runtime/pprof" "testing" "unsafe" ) @@ -87,26 +86,26 @@ func TestMemoryProfiler(t *testing.T) { tests := []string{ fmt.Sprintf(`%v: %v \[%v: %v\] @ 0x[0-9,a-f x]+ -# 0x[0-9,a-f]+ pprof_test\.allocatePersistent1K\+0x[0-9,a-f]+ .*/mprof_test\.go:41 -# 0x[0-9,a-f]+ runtime_pprof_test\.TestMemoryProfiler\+0x[0-9,a-f]+ .*/mprof_test\.go:75 +# 0x[0-9,a-f]+ pprof\.allocatePersistent1K\+0x[0-9,a-f]+ .*/mprof_test\.go:40 +# 0x[0-9,a-f]+ runtime_pprof\.TestMemoryProfiler\+0x[0-9,a-f]+ .*/mprof_test\.go:74 `, 32*memoryProfilerRun, 1024*memoryProfilerRun, 32*memoryProfilerRun, 1024*memoryProfilerRun), fmt.Sprintf(`0: 0 \[%v: %v\] @ 0x[0-9,a-f x]+ -# 0x[0-9,a-f]+ pprof_test\.allocateTransient1M\+0x[0-9,a-f]+ .*/mprof_test.go:22 -# 0x[0-9,a-f]+ runtime_pprof_test\.TestMemoryProfiler\+0x[0-9,a-f]+ .*/mprof_test.go:73 +# 0x[0-9,a-f]+ pprof\.allocateTransient1M\+0x[0-9,a-f]+ .*/mprof_test.go:21 +# 0x[0-9,a-f]+ runtime_pprof\.TestMemoryProfiler\+0x[0-9,a-f]+ .*/mprof_test.go:72 `, (1<<10)*memoryProfilerRun, (1<<20)*memoryProfilerRun), // This should start with "0: 0" but gccgo's imprecise // GC means that sometimes the value is not collected. fmt.Sprintf(`(0|%v): (0|%v) \[%v: %v\] @ 0x[0-9,a-f x]+ -# 0x[0-9,a-f]+ pprof_test\.allocateTransient2M\+0x[0-9,a-f]+ .*/mprof_test.go:28 -# 0x[0-9,a-f]+ runtime_pprof_test\.TestMemoryProfiler\+0x[0-9,a-f]+ .*/mprof_test.go:74 +# 0x[0-9,a-f]+ pprof\.allocateTransient2M\+0x[0-9,a-f]+ .*/mprof_test.go:27 +# 0x[0-9,a-f]+ runtime_pprof\.TestMemoryProfiler\+0x[0-9,a-f]+ .*/mprof_test.go:73 `, memoryProfilerRun, (2<<20)*memoryProfilerRun, memoryProfilerRun, (2<<20)*memoryProfilerRun), // This should start with "0: 0" but gccgo's imprecise // GC means that sometimes the value is not collected. fmt.Sprintf(`(0|%v): (0|%v) \[%v: %v\] @( 0x[0-9,a-f]+)+ -# 0x[0-9,a-f]+ pprof_test\.allocateReflectTransient\+0x[0-9,a-f]+ .*/mprof_test.go:49 +# 0x[0-9,a-f]+ pprof\.allocateReflectTransient\+0x[0-9,a-f]+ .*/mprof_test.go:48 `, memoryProfilerRun, (2<<20)*memoryProfilerRun, memoryProfilerRun, (2<<20)*memoryProfilerRun), } diff --git a/libgo/go/runtime/pprof/pprof.go b/libgo/go/runtime/pprof/pprof.go index 0db1dedd7ac..a57b69dca35 100644 --- a/libgo/go/runtime/pprof/pprof.go +++ b/libgo/go/runtime/pprof/pprof.go @@ -33,7 +33,9 @@ // } // defer pprof.StopCPUProfile() // } -// ... +// +// // ... rest of the program ... +// // if *memprofile != "" { // f, err := os.Create(*memprofile) // if err != nil { @@ -73,15 +75,14 @@ import ( "bufio" "bytes" "fmt" - "internal/pprof/profile" "io" "runtime" - "runtime/pprof/internal/protopprof" "sort" "strings" "sync" "text/tabwriter" "time" + "unsafe" ) // BUG(rsc): Profiles are only as good as the kernel support used to generate them. @@ -183,6 +184,8 @@ func unlockProfiles() { // If a profile with that name already exists, NewProfile panics. // The convention is to use a 'import/path.' prefix to create // separate name spaces for each package. +// For compatibility with various tools that read pprof data, +// profile names should not contain spaces. func NewProfile(name string) *Profile { lockProfiles() defer unlockProfiles() @@ -264,13 +267,18 @@ func (p *Profile) Add(value interface{}, skip int) { stk := make([]uintptr, 32) n := runtime.Callers(skip+1, stk[:]) + stk = stk[:n] + if len(stk) == 0 { + // The value for skip is too large, and there's no stack trace to record. + stk = []uintptr{funcPC(lostProfileEvent) + 1} + } p.mu.Lock() defer p.mu.Unlock() if p.m[value] != nil { panic("pprof: Profile.Add of duplicate value") } - p.m[value] = stk[:n] + p.m[value] = stk } // Remove removes the execution stack associated with value from the profile. @@ -303,8 +311,8 @@ func (p *Profile) WriteTo(w io.Writer, debug int) error { } // Obtain consistent snapshot under lock; then process without lock. - all := make([][]uintptr, 0, len(p.m)) p.mu.Lock() + all := make([][]uintptr, 0, len(p.m)) for _, stk := range p.m { all = append(all, stk) } @@ -380,35 +388,29 @@ func printCountProfile(w io.Writer, debug int, name string, p countProfile) erro } // Output profile in protobuf form. - prof := &profile.Profile{ - PeriodType: &profile.ValueType{Type: name, Unit: "count"}, - Period: 1, - Sample: make([]*profile.Sample, 0, len(keys)), - SampleType: []*profile.ValueType{{Type: name, Unit: "count"}}, - } - locMap := make(map[uintptr]*profile.Location) + b := newProfileBuilder(w) + b.pbValueType(tagProfile_PeriodType, name, "count") + b.pb.int64Opt(tagProfile_Period, 1) + b.pbValueType(tagProfile_SampleType, name, "count") + + values := []int64{0} + var locs []uint64 for _, k := range keys { - stk := p.Stack(index[k]) - c := count[k] - locs := make([]*profile.Location, len(stk)) - for i, addr := range stk { - loc := locMap[addr] - if loc == nil { - loc = &profile.Location{ - ID: uint64(len(locMap) + 1), - Address: uint64(addr - 1), - } - prof.Location = append(prof.Location, loc) - locMap[addr] = loc + values[0] = int64(count[k]) + locs = locs[:0] + for _, addr := range p.Stack(index[k]) { + // For count profiles, all stack addresses are + // return PCs, which is what locForPC expects. + l := b.locForPC(addr) + if l == 0 { // runtime.goexit + continue } - locs[i] = loc + locs = append(locs, l) } - prof.Sample = append(prof.Sample, &profile.Sample{ - Location: locs, - Value: []int64{int64(c)}, - }) + b.pbSample(values, locs, nil) } - return prof.Write(w) + b.build() + return nil } // keysByCount sorts keys with higher counts first, breaking ties by key string order. @@ -510,8 +512,7 @@ func writeHeap(w io.Writer, debug int) error { } if debug == 0 { - pp := protopprof.EncodeMemProfile(p, int64(runtime.MemProfileRate), time.Now()) - return pp.Write(w) + return writeHeapProto(w, p, int64(runtime.MemProfileRate)) } sort.Slice(p, func(i, j int) bool { return p[i].InUseBytes() > p[j].InUseBytes() }) @@ -576,8 +577,12 @@ func writeHeap(w io.Writer, debug int) error { fmt.Fprintf(w, "# OtherSys = %d\n", s.OtherSys) fmt.Fprintf(w, "# NextGC = %d\n", s.NextGC) + fmt.Fprintf(w, "# LastGC = %d\n", s.LastGC) fmt.Fprintf(w, "# PauseNs = %d\n", s.PauseNs) + fmt.Fprintf(w, "# PauseEnd = %d\n", s.PauseEnd) fmt.Fprintf(w, "# NumGC = %d\n", s.NumGC) + fmt.Fprintf(w, "# NumForcedGC = %d\n", s.NumForcedGC) + fmt.Fprintf(w, "# GCCPUFraction = %v\n", s.GCCPUFraction) fmt.Fprintf(w, "# DebugGC = %v\n", s.DebugGC) tw.Flush() @@ -703,30 +708,32 @@ func StartCPUProfile(w io.Writer) error { return nil } +// readProfile, provided by the runtime, returns the next chunk of +// binary CPU profiling stack trace data, blocking until data is available. +// If profiling is turned off and all the profile data accumulated while it was +// on has been returned, readProfile returns eof=true. +// The caller must save the returned data and tags before calling readProfile again. +func readProfile() (data []uint64, tags []unsafe.Pointer, eof bool) + func profileWriter(w io.Writer) { - startTime := time.Now() - // This will buffer the entire profile into buf and then - // translate it into a profile.Profile structure. This will - // create two copies of all the data in the profile in memory. - // TODO(matloob): Convert each chunk of the proto output and - // stream it out instead of converting the entire profile. - var buf bytes.Buffer + b := newProfileBuilder(w) + var err error for { - data := runtime.CPUProfile() - if data == nil { + time.Sleep(100 * time.Millisecond) + data, tags, eof := readProfile() + if e := b.addCPUData(data, tags); e != nil && err == nil { + err = e + } + if eof { break } - buf.Write(data) } - - profile, err := protopprof.TranslateCPUProfile(buf.Bytes(), startTime) if err != nil { // The runtime should never produce an invalid or truncated profile. // It drops records that can't fit into its log buffers. - panic(fmt.Errorf("could not translate binary profile to proto format: %v", err)) + panic("runtime/pprof: converting profile: " + err.Error()) } - - profile.Write(w) + b.build() cpu.done <- true } diff --git a/libgo/go/runtime/pprof/pprof_test.go b/libgo/go/runtime/pprof/pprof_test.go index 60340582d5f..9e5e403b741 100644 --- a/libgo/go/runtime/pprof/pprof_test.go +++ b/libgo/go/runtime/pprof/pprof_test.go @@ -4,13 +4,12 @@ // +build !nacl -package pprof_test +package pprof import ( "bytes" - "compress/gzip" + "context" "fmt" - "internal/pprof/profile" "internal/testenv" "io" "io/ioutil" @@ -19,14 +18,15 @@ import ( "os/exec" "regexp" "runtime" - . "runtime/pprof" + "runtime/pprof/internal/profile" "strings" "sync" + "sync/atomic" "testing" "time" ) -func cpuHogger(f func(), dur time.Duration) { +func cpuHogger(f func() int, dur time.Duration) { // We only need to get one 100 Hz clock tick, so we've got // a large safety buffer. // But do at least 500 iterations (which should take about 100ms), @@ -46,7 +46,7 @@ var ( // The actual CPU hogging function. // Must not call other functions nor access heap/globals in the loop, // otherwise under race detector the samples will be in the race runtime. -func cpuHog1() { +func cpuHog1() int { foo := salt1 for i := 0; i < 1e5; i++ { if foo > 0 { @@ -55,10 +55,10 @@ func cpuHog1() { foo *= foo + 1 } } - salt1 = foo + return foo } -func cpuHog2() { +func cpuHog2() int { foo := salt2 for i := 0; i < 1e5; i++ { if foo > 0 { @@ -67,18 +67,18 @@ func cpuHog2() { foo *= foo + 2 } } - salt2 = foo + return foo } func TestCPUProfile(t *testing.T) { - testCPUProfile(t, []string{"pprof_test.cpuHog1"}, func(dur time.Duration) { + testCPUProfile(t, []string{"pprof.cpuHog1"}, func(dur time.Duration) { cpuHogger(cpuHog1, dur) }) } func TestCPUProfileMultithreaded(t *testing.T) { defer runtime.GOMAXPROCS(runtime.GOMAXPROCS(2)) - testCPUProfile(t, []string{"pprof_test.cpuHog1", "pprof_test.cpuHog2"}, func(dur time.Duration) { + testCPUProfile(t, []string{"pprof.cpuHog1", "pprof.cpuHog2"}, func(dur time.Duration) { c := make(chan int) go func() { cpuHogger(cpuHog1, dur) @@ -89,18 +89,42 @@ func TestCPUProfileMultithreaded(t *testing.T) { }) } -func parseProfile(t *testing.T, valBytes []byte, f func(uintptr, []uintptr)) { +func TestCPUProfileInlining(t *testing.T) { + testCPUProfile(t, []string{"pprof.inlinedCallee", "pprof.inlinedCaller"}, func(dur time.Duration) { + cpuHogger(inlinedCaller, dur) + }) +} + +func inlinedCaller() int { + inlinedCallee() + return 0 +} + +func inlinedCallee() { + // We could just use cpuHog1, but for loops prevent inlining + // right now. :( + foo := salt1 + i := 0 +loop: + if foo > 0 { + foo *= foo + } else { + foo *= foo + 1 + } + if i++; i < 1e5 { + goto loop + } + salt1 = foo +} + +func parseProfile(t *testing.T, valBytes []byte, f func(uintptr, []*profile.Location, map[string][]string)) { p, err := profile.Parse(bytes.NewReader(valBytes)) if err != nil { t.Fatal(err) } for _, sample := range p.Sample { count := uintptr(sample.Value[0]) - stk := make([]uintptr, len(sample.Location)) - for i := range sample.Location { - stk[i] = uintptr(sample.Location[i].Address) - } - f(count, stk) + f(count, sample.Location, sample.Label) } } @@ -124,8 +148,7 @@ func testCPUProfile(t *testing.T, need []string, f func(dur time.Duration)) { const maxDuration = 5 * time.Second // If we're running a long test, start with a long duration - // because some of the tests (e.g., TestStackBarrierProfiling) - // are trying to make sure something *doesn't* happen. + // for tests that try to make sure something *doesn't* happen. duration := 5 * time.Second if testing.Short() { duration = 200 * time.Millisecond @@ -169,32 +192,45 @@ func testCPUProfile(t *testing.T, need []string, f func(dur time.Duration)) { t.FailNow() } +func contains(slice []string, s string) bool { + for i := range slice { + if slice[i] == s { + return true + } + } + return false +} + func profileOk(t *testing.T, need []string, prof bytes.Buffer, duration time.Duration) (ok bool) { ok = true // Check that profile is well formed and contains need. have := make([]uintptr, len(need)) var samples uintptr - parseProfile(t, prof.Bytes(), func(count uintptr, stk []uintptr) { + var buf bytes.Buffer + parseProfile(t, prof.Bytes(), func(count uintptr, stk []*profile.Location, labels map[string][]string) { + fmt.Fprintf(&buf, "%d:", count) + fprintStack(&buf, stk) samples += count - for _, pc := range stk { - f := runtime.FuncForPC(pc) - if f == nil { - continue - } - t.Log(f.Name(), count) - for i, name := range need { - if strings.Contains(f.Name(), name) { - have[i] += count + for i, name := range need { + if semi := strings.Index(name, ";"); semi > -1 { + kv := strings.SplitN(name[semi+1:], "=", 2) + if len(kv) != 2 || !contains(labels[kv[0]], kv[1]) { + continue } + name = name[:semi] } - if strings.Contains(f.Name(), "stackBarrier") { - // The runtime should have unwound this. - t.Fatalf("profile includes stackBarrier") + for _, loc := range stk { + for _, line := range loc.Line { + if strings.Contains(line.Function.Name, name) { + have[i] += count + } + } } } + fmt.Fprintf(&buf, "\n") }) - t.Logf("total %d CPU profile samples collected", samples) + t.Logf("total %d CPU profile samples collected:\n%s", samples, buf.String()) if samples < 10 && runtime.GOOS == "windows" { // On some windows machines we end up with @@ -301,36 +337,43 @@ func TestGoroutineSwitch(t *testing.T) { // Read profile to look for entries for runtime.gogo with an attempt at a traceback. // The special entry - parseProfile(t, prof.Bytes(), func(count uintptr, stk []uintptr) { + parseProfile(t, prof.Bytes(), func(count uintptr, stk []*profile.Location, _ map[string][]string) { // An entry with two frames with 'System' in its top frame // exists to record a PC without a traceback. Those are okay. if len(stk) == 2 { - f := runtime.FuncForPC(stk[1]) - if f != nil && (f.Name() == "runtime._System" || f.Name() == "runtime._ExternalCode" || f.Name() == "runtime._GC") { + name := stk[1].Line[0].Function.Name + if name == "runtime._System" || name == "runtime._ExternalCode" || name == "runtime._GC" { return } } // Otherwise, should not see runtime.gogo. // The place we'd see it would be the inner most frame. - f := runtime.FuncForPC(stk[0]) - if f != nil && f.Name() == "runtime.gogo" { + name := stk[0].Line[0].Function.Name + if name == "runtime.gogo" { var buf bytes.Buffer - for _, pc := range stk { - f := runtime.FuncForPC(pc) - if f == nil { - fmt.Fprintf(&buf, "%#x ?:0\n", pc) - } else { - file, line := f.FileLine(pc) - fmt.Fprintf(&buf, "%#x %s:%d\n", pc, file, line) - } - } + fprintStack(&buf, stk) t.Fatalf("found profile entry for runtime.gogo:\n%s", buf.String()) } }) } } +func fprintStack(w io.Writer, stk []*profile.Location) { + for _, loc := range stk { + fmt.Fprintf(w, " %#x", loc.Address) + fmt.Fprintf(w, " (") + for i, line := range loc.Line { + if i > 0 { + fmt.Fprintf(w, " ") + } + fmt.Fprintf(w, "%s:%d", line.Function.Name, line.Line) + } + fmt.Fprintf(w, ")") + } + fmt.Fprintf(w, "\n") +} + // Test that profiling of division operations is okay, especially on ARM. See issue 6681. func TestMathBigDivide(t *testing.T) { testCPUProfile(t, nil, func(duration time.Duration) { @@ -351,111 +394,6 @@ func TestMathBigDivide(t *testing.T) { }) } -func slurpString(r io.Reader) string { - slurp, _ := ioutil.ReadAll(r) - return string(slurp) -} - -func getLinuxKernelConfig() string { - if f, err := os.Open("/proc/config"); err == nil { - defer f.Close() - return slurpString(f) - } - if f, err := os.Open("/proc/config.gz"); err == nil { - defer f.Close() - r, err := gzip.NewReader(f) - if err != nil { - return "" - } - return slurpString(r) - } - if f, err := os.Open("/boot/config"); err == nil { - defer f.Close() - return slurpString(f) - } - uname, _ := exec.Command("uname", "-r").Output() - if len(uname) > 0 { - if f, err := os.Open("/boot/config-" + strings.TrimSpace(string(uname))); err == nil { - defer f.Close() - return slurpString(f) - } - } - return "" -} - -func haveLinuxHiresTimers() bool { - config := getLinuxKernelConfig() - return strings.Contains(config, "CONFIG_HIGH_RES_TIMERS=y") -} - -func TestStackBarrierProfiling(t *testing.T) { - if (runtime.GOOS == "linux" && runtime.GOARCH == "arm") || - runtime.GOOS == "openbsd" || - runtime.GOOS == "solaris" || - runtime.GOOS == "dragonfly" || - runtime.GOOS == "freebsd" { - // This test currently triggers a large number of - // usleep(100)s. These kernels/arches have poor - // resolution timers, so this gives up a whole - // scheduling quantum. On Linux and the BSDs (and - // probably Solaris), profiling signals are only - // generated when a process completes a whole - // scheduling quantum, so this test often gets zero - // profiling signals and fails. - t.Skipf("low resolution timers inhibit profiling signals (golang.org/issue/13405)") - return - } - - if runtime.GOOS == "linux" && strings.HasPrefix(runtime.GOARCH, "mips") { - if !haveLinuxHiresTimers() { - t.Skipf("low resolution timers inhibit profiling signals (golang.org/issue/13405, golang.org/issue/17936)") - } - } - - if !strings.Contains(os.Getenv("GODEBUG"), "gcstackbarrierall=1") { - // Re-execute this test with constant GC and stack - // barriers at every frame. - testenv.MustHaveExec(t) - if runtime.GOARCH == "ppc64" || runtime.GOARCH == "ppc64le" { - t.Skip("gcstackbarrierall doesn't work on ppc64") - } - args := []string{"-test.run=TestStackBarrierProfiling"} - if testing.Short() { - args = append(args, "-test.short") - } - cmd := exec.Command(os.Args[0], args...) - cmd.Env = append([]string{"GODEBUG=gcstackbarrierall=1", "GOGC=1", "GOTRACEBACK=system"}, os.Environ()...) - if out, err := cmd.CombinedOutput(); err != nil { - t.Fatalf("subprocess failed with %v:\n%s", err, out) - } - return - } - - testCPUProfile(t, nil, func(duration time.Duration) { - // In long mode, we're likely to get one or two - // samples in stackBarrier. - t := time.After(duration) - for { - deepStack(1000) - select { - case <-t: - return - default: - } - } - }) -} - -var x []byte - -func deepStack(depth int) int { - if depth == 0 { - return 0 - } - x = make([]byte, 1024) - return deepStack(depth-1) + 1 -} - // Operating systems that are expected to fail the tests. See issue 13841. var badOS = map[string]bool{ "darwin": true, @@ -474,46 +412,46 @@ func TestBlockProfile(t *testing.T) { } tests := [...]TestCase{ {"chan recv", blockChanRecv, ` -[0-9]+ [0-9]+ @ 0x[0-9,a-f]+ 0x[0-9,a-f]+ 0x[0-9,a-f]+ 0x[0-9,a-f]+ 0x[0-9,a-f]+ -# 0x[0-9,a-f]+ runtime\.chanrecv1\+0x[0-9,a-f]+ .*/src/runtime/chan.go:[0-9]+ -# 0x[0-9,a-f]+ runtime/pprof_test\.blockChanRecv\+0x[0-9,a-f]+ .*/src/runtime/pprof/pprof_test.go:[0-9]+ -# 0x[0-9,a-f]+ runtime/pprof_test\.TestBlockProfile\+0x[0-9,a-f]+ .*/src/runtime/pprof/pprof_test.go:[0-9]+ +[0-9]+ [0-9]+ @( 0x[[:xdigit:]]+)+ +# 0x[0-9a-f]+ runtime\.chanrecv1\+0x[0-9a-f]+ .*/src/runtime/chan.go:[0-9]+ +# 0x[0-9a-f]+ runtime/pprof\.blockChanRecv\+0x[0-9a-f]+ .*/src/runtime/pprof/pprof_test.go:[0-9]+ +# 0x[0-9a-f]+ runtime/pprof\.TestBlockProfile\+0x[0-9a-f]+ .*/src/runtime/pprof/pprof_test.go:[0-9]+ `}, {"chan send", blockChanSend, ` -[0-9]+ [0-9]+ @ 0x[0-9,a-f]+ 0x[0-9,a-f]+ 0x[0-9,a-f]+ 0x[0-9,a-f]+ 0x[0-9,a-f]+ -# 0x[0-9,a-f]+ runtime\.chansend1\+0x[0-9,a-f]+ .*/src/runtime/chan.go:[0-9]+ -# 0x[0-9,a-f]+ runtime/pprof_test\.blockChanSend\+0x[0-9,a-f]+ .*/src/runtime/pprof/pprof_test.go:[0-9]+ -# 0x[0-9,a-f]+ runtime/pprof_test\.TestBlockProfile\+0x[0-9,a-f]+ .*/src/runtime/pprof/pprof_test.go:[0-9]+ +[0-9]+ [0-9]+ @( 0x[[:xdigit:]]+)+ +# 0x[0-9a-f]+ runtime\.chansend1\+0x[0-9a-f]+ .*/src/runtime/chan.go:[0-9]+ +# 0x[0-9a-f]+ runtime/pprof\.blockChanSend\+0x[0-9a-f]+ .*/src/runtime/pprof/pprof_test.go:[0-9]+ +# 0x[0-9a-f]+ runtime/pprof\.TestBlockProfile\+0x[0-9a-f]+ .*/src/runtime/pprof/pprof_test.go:[0-9]+ `}, {"chan close", blockChanClose, ` -[0-9]+ [0-9]+ @ 0x[0-9,a-f]+ 0x[0-9,a-f]+ 0x[0-9,a-f]+ 0x[0-9,a-f]+ 0x[0-9,a-f]+ -# 0x[0-9,a-f]+ runtime\.chanrecv1\+0x[0-9,a-f]+ .*/src/runtime/chan.go:[0-9]+ -# 0x[0-9,a-f]+ runtime/pprof_test\.blockChanClose\+0x[0-9,a-f]+ .*/src/runtime/pprof/pprof_test.go:[0-9]+ -# 0x[0-9,a-f]+ runtime/pprof_test\.TestBlockProfile\+0x[0-9,a-f]+ .*/src/runtime/pprof/pprof_test.go:[0-9]+ +[0-9]+ [0-9]+ @( 0x[[:xdigit:]]+)+ +# 0x[0-9a-f]+ runtime\.chanrecv1\+0x[0-9a-f]+ .*/src/runtime/chan.go:[0-9]+ +# 0x[0-9a-f]+ runtime/pprof\.blockChanClose\+0x[0-9a-f]+ .*/src/runtime/pprof/pprof_test.go:[0-9]+ +# 0x[0-9a-f]+ runtime/pprof\.TestBlockProfile\+0x[0-9a-f]+ .*/src/runtime/pprof/pprof_test.go:[0-9]+ `}, {"select recv async", blockSelectRecvAsync, ` -[0-9]+ [0-9]+ @ 0x[0-9,a-f]+ 0x[0-9,a-f]+ 0x[0-9,a-f]+ 0x[0-9,a-f]+ 0x[0-9,a-f]+ -# 0x[0-9,a-f]+ runtime\.selectgo\+0x[0-9,a-f]+ .*/src/runtime/select.go:[0-9]+ -# 0x[0-9,a-f]+ runtime/pprof_test\.blockSelectRecvAsync\+0x[0-9,a-f]+ .*/src/runtime/pprof/pprof_test.go:[0-9]+ -# 0x[0-9,a-f]+ runtime/pprof_test\.TestBlockProfile\+0x[0-9,a-f]+ .*/src/runtime/pprof/pprof_test.go:[0-9]+ +[0-9]+ [0-9]+ @( 0x[[:xdigit:]]+)+ +# 0x[0-9a-f]+ runtime\.selectgo\+0x[0-9a-f]+ .*/src/runtime/select.go:[0-9]+ +# 0x[0-9a-f]+ runtime/pprof\.blockSelectRecvAsync\+0x[0-9a-f]+ .*/src/runtime/pprof/pprof_test.go:[0-9]+ +# 0x[0-9a-f]+ runtime/pprof\.TestBlockProfile\+0x[0-9a-f]+ .*/src/runtime/pprof/pprof_test.go:[0-9]+ `}, {"select send sync", blockSelectSendSync, ` -[0-9]+ [0-9]+ @ 0x[0-9,a-f]+ 0x[0-9,a-f]+ 0x[0-9,a-f]+ 0x[0-9,a-f]+ 0x[0-9,a-f]+ -# 0x[0-9,a-f]+ runtime\.selectgo\+0x[0-9,a-f]+ .*/src/runtime/select.go:[0-9]+ -# 0x[0-9,a-f]+ runtime/pprof_test\.blockSelectSendSync\+0x[0-9,a-f]+ .*/src/runtime/pprof/pprof_test.go:[0-9]+ -# 0x[0-9,a-f]+ runtime/pprof_test\.TestBlockProfile\+0x[0-9,a-f]+ .*/src/runtime/pprof/pprof_test.go:[0-9]+ +[0-9]+ [0-9]+ @( 0x[[:xdigit:]]+)+ +# 0x[0-9a-f]+ runtime\.selectgo\+0x[0-9a-f]+ .*/src/runtime/select.go:[0-9]+ +# 0x[0-9a-f]+ runtime/pprof\.blockSelectSendSync\+0x[0-9a-f]+ .*/src/runtime/pprof/pprof_test.go:[0-9]+ +# 0x[0-9a-f]+ runtime/pprof\.TestBlockProfile\+0x[0-9a-f]+ .*/src/runtime/pprof/pprof_test.go:[0-9]+ `}, {"mutex", blockMutex, ` -[0-9]+ [0-9]+ @ 0x[0-9,a-f]+ 0x[0-9,a-f]+ 0x[0-9,a-f]+ 0x[0-9,a-f]+ 0x[0-9,a-f]+ -# 0x[0-9,a-f]+ sync\.\(\*Mutex\)\.Lock\+0x[0-9,a-f]+ .*/src/sync/mutex\.go:[0-9]+ -# 0x[0-9,a-f]+ runtime/pprof_test\.blockMutex\+0x[0-9,a-f]+ .*/src/runtime/pprof/pprof_test.go:[0-9]+ -# 0x[0-9,a-f]+ runtime/pprof_test\.TestBlockProfile\+0x[0-9,a-f]+ .*/src/runtime/pprof/pprof_test.go:[0-9]+ +[0-9]+ [0-9]+ @( 0x[[:xdigit:]]+)+ +# 0x[0-9a-f]+ sync\.\(\*Mutex\)\.Lock\+0x[0-9a-f]+ .*/src/sync/mutex\.go:[0-9]+ +# 0x[0-9a-f]+ runtime/pprof\.blockMutex\+0x[0-9a-f]+ .*/src/runtime/pprof/pprof_test.go:[0-9]+ +# 0x[0-9a-f]+ runtime/pprof\.TestBlockProfile\+0x[0-9a-f]+ .*/src/runtime/pprof/pprof_test.go:[0-9]+ `}, {"cond", blockCond, ` -[0-9]+ [0-9]+ @ 0x[0-9,a-f]+ 0x[0-9,a-f]+ 0x[0-9,a-f]+ 0x[0-9,a-f]+ 0x[0-9,a-f]+ -# 0x[0-9,a-f]+ sync\.\(\*Cond\)\.Wait\+0x[0-9,a-f]+ .*/src/sync/cond\.go:[0-9]+ -# 0x[0-9,a-f]+ runtime/pprof_test\.blockCond\+0x[0-9,a-f]+ .*/src/runtime/pprof/pprof_test.go:[0-9]+ -# 0x[0-9,a-f]+ runtime/pprof_test\.TestBlockProfile\+0x[0-9,a-f]+ .*/src/runtime/pprof/pprof_test.go:[0-9]+ +[0-9]+ [0-9]+ @( 0x[[:xdigit:]]+)+ +# 0x[0-9a-f]+ sync\.\(\*Cond\)\.Wait\+0x[0-9a-f]+ .*/src/sync/cond\.go:[0-9]+ +# 0x[0-9a-f]+ runtime/pprof\.blockCond\+0x[0-9a-f]+ .*/src/runtime/pprof/pprof_test.go:[0-9]+ +# 0x[0-9a-f]+ runtime/pprof\.TestBlockProfile\+0x[0-9a-f]+ .*/src/runtime/pprof/pprof_test.go:[0-9]+ `}, } @@ -608,6 +546,10 @@ func blockMutex() { time.Sleep(blockDelay) mu.Unlock() }() + // Note: Unlock releases mu before recording the mutex event, + // so it's theoretically possible for this to proceed and + // capture the profile before the event is recorded. As long + // as this is blocked before the unlock happens, it's okay. mu.Lock() } @@ -656,7 +598,7 @@ func TestMutexProfile(t *testing.T) { if ok, err := regexp.MatchString(r2, lines[3]); err != nil || !ok { t.Errorf("%q didn't match %q", lines[3], r2) } - r3 := "^#.*pprof_test.\\$nested.*$" + r3 := "^#.*pprof.\\$nested.*$" match := false for _, i := range []int{5, 6} { if ok, _ := regexp.MatchString(r3, lines[i]); ok { @@ -678,22 +620,26 @@ func TestGoroutineCounts(t *testing.T) { if runtime.Compiler == "gccgo" { t.Skip("goroutine stacks not supported on gccgo") } - if runtime.GOOS == "openbsd" { - testenv.SkipFlaky(t, 15156) - } + + // Setting GOMAXPROCS to 1 ensures we can force all goroutines to the + // desired blocking point. + defer runtime.GOMAXPROCS(runtime.GOMAXPROCS(1)) + c := make(chan int) for i := 0; i < 100; i++ { - if i%10 == 0 { + switch { + case i%10 == 0: go func1(c) - continue - } - if i%2 == 0 { + case i%2 == 0: go func2(c) - continue + default: + go func3(c) + } + // Let goroutines block on channel + for j := 0; j < 5; j++ { + runtime.Gosched() } - go func3(c) } - time.Sleep(10 * time.Millisecond) // let goroutines block on channel var w bytes.Buffer goroutineProf := Lookup("goroutine") @@ -756,3 +702,81 @@ func containsCounts(prof *profile.Profile, counts []int64) bool { } return true } + +// Issue 18836. +func TestEmptyCallStack(t *testing.T) { + t.Parallel() + var buf bytes.Buffer + p := NewProfile("test18836") + p.Add("foo", 47674) + p.WriteTo(&buf, 1) + p.Remove("foo") + got := buf.String() + prefix := "test18836 profile: total 1\n" + if !strings.HasPrefix(got, prefix) { + t.Fatalf("got:\n\t%q\nwant prefix:\n\t%q\n", got, prefix) + } + lostevent := "lostProfileEvent" + if !strings.Contains(got, lostevent) { + t.Fatalf("got:\n\t%q\ndoes not contain:\n\t%q\n", got, lostevent) + } +} + +func TestCPUProfileLabel(t *testing.T) { + testCPUProfile(t, []string{"pprof.cpuHogger;key=value"}, func(dur time.Duration) { + Do(context.Background(), Labels("key", "value"), func(context.Context) { + cpuHogger(cpuHog1, dur) + }) + }) +} + +func TestLabelRace(t *testing.T) { + // Test the race detector annotations for synchronization + // between settings labels and consuming them from the + // profile. + testCPUProfile(t, []string{"pprof.cpuHogger;key=value"}, func(dur time.Duration) { + start := time.Now() + var wg sync.WaitGroup + for time.Since(start) < dur { + for i := 0; i < 10; i++ { + wg.Add(1) + go func() { + Do(context.Background(), Labels("key", "value"), func(context.Context) { + cpuHogger(cpuHog1, time.Millisecond) + }) + wg.Done() + }() + } + wg.Wait() + } + }) +} + +// Check that there is no deadlock when the program receives SIGPROF while in +// 64bit atomics' critical section. Used to happen on mips{,le}. See #20146. +func TestAtomicLoadStore64(t *testing.T) { + f, err := ioutil.TempFile("", "profatomic") + if err != nil { + t.Fatalf("TempFile: %v", err) + } + defer os.Remove(f.Name()) + defer f.Close() + + if err := StartCPUProfile(f); err != nil { + t.Fatal(err) + } + defer StopCPUProfile() + + var flag uint64 + done := make(chan bool, 1) + + go func() { + for atomic.LoadUint64(&flag) == 0 { + runtime.Gosched() + } + done <- true + }() + time.Sleep(50 * time.Millisecond) + atomic.StoreUint64(&flag, 1) + <-done +} diff --git a/libgo/go/runtime/pprof/proto.go b/libgo/go/runtime/pprof/proto.go new file mode 100644 index 00000000000..5e1d71c7e72 --- /dev/null +++ b/libgo/go/runtime/pprof/proto.go @@ -0,0 +1,515 @@ +// Copyright 2016 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package pprof + +import ( + "bytes" + "compress/gzip" + "fmt" + "io" + "io/ioutil" + "runtime" + "sort" + "strconv" + "time" + "unsafe" +) + +// lostProfileEvent is the function to which lost profiling +// events are attributed. +// (The name shows up in the pprof graphs.) +func lostProfileEvent() { lostProfileEvent() } + +// funcPC returns the PC for the func value f. +func funcPC(f interface{}) uintptr { + type iface struct { + tab unsafe.Pointer + data unsafe.Pointer + } + i := (*iface)(unsafe.Pointer(&f)) + return **(**uintptr)(i.data) +} + +// A profileBuilder writes a profile incrementally from a +// stream of profile samples delivered by the runtime. +type profileBuilder struct { + start time.Time + end time.Time + havePeriod bool + period int64 + m profMap + + // encoding state + w io.Writer + zw *gzip.Writer + pb protobuf + strings []string + stringMap map[string]int + locs map[uintptr]int + funcs map[string]int // Package path-qualified function name to Function.ID + mem []memMap +} + +type memMap struct { + start uintptr + end uintptr +} + +const ( + // message Profile + tagProfile_SampleType = 1 // repeated ValueType + tagProfile_Sample = 2 // repeated Sample + tagProfile_Mapping = 3 // repeated Mapping + tagProfile_Location = 4 // repeated Location + tagProfile_Function = 5 // repeated Function + tagProfile_StringTable = 6 // repeated string + tagProfile_DropFrames = 7 // int64 (string table index) + tagProfile_KeepFrames = 8 // int64 (string table index) + tagProfile_TimeNanos = 9 // int64 + tagProfile_DurationNanos = 10 // int64 + tagProfile_PeriodType = 11 // ValueType (really optional string???) + tagProfile_Period = 12 // int64 + + // message ValueType + tagValueType_Type = 1 // int64 (string table index) + tagValueType_Unit = 2 // int64 (string table index) + + // message Sample + tagSample_Location = 1 // repeated uint64 + tagSample_Value = 2 // repeated int64 + tagSample_Label = 3 // repeated Label + + // message Label + tagLabel_Key = 1 // int64 (string table index) + tagLabel_Str = 2 // int64 (string table index) + tagLabel_Num = 3 // int64 + + // message Mapping + tagMapping_ID = 1 // uint64 + tagMapping_Start = 2 // uint64 + tagMapping_Limit = 3 // uint64 + tagMapping_Offset = 4 // uint64 + tagMapping_Filename = 5 // int64 (string table index) + tagMapping_BuildID = 6 // int64 (string table index) + tagMapping_HasFunctions = 7 // bool + tagMapping_HasFilenames = 8 // bool + tagMapping_HasLineNumbers = 9 // bool + tagMapping_HasInlineFrames = 10 // bool + + // message Location + tagLocation_ID = 1 // uint64 + tagLocation_MappingID = 2 // uint64 + tagLocation_Address = 3 // uint64 + tagLocation_Line = 4 // repeated Line + + // message Line + tagLine_FunctionID = 1 // uint64 + tagLine_Line = 2 // int64 + + // message Function + tagFunction_ID = 1 // uint64 + tagFunction_Name = 2 // int64 (string table index) + tagFunction_SystemName = 3 // int64 (string table index) + tagFunction_Filename = 4 // int64 (string table index) + tagFunction_StartLine = 5 // int64 +) + +// stringIndex adds s to the string table if not already present +// and returns the index of s in the string table. +func (b *profileBuilder) stringIndex(s string) int64 { + id, ok := b.stringMap[s] + if !ok { + id = len(b.strings) + b.strings = append(b.strings, s) + b.stringMap[s] = id + } + return int64(id) +} + +func (b *profileBuilder) flush() { + const dataFlush = 4096 + if b.pb.nest == 0 && len(b.pb.data) > dataFlush { + b.zw.Write(b.pb.data) + b.pb.data = b.pb.data[:0] + } +} + +// pbValueType encodes a ValueType message to b.pb. +func (b *profileBuilder) pbValueType(tag int, typ, unit string) { + start := b.pb.startMessage() + b.pb.int64(tagValueType_Type, b.stringIndex(typ)) + b.pb.int64(tagValueType_Unit, b.stringIndex(unit)) + b.pb.endMessage(tag, start) +} + +// pbSample encodes a Sample message to b.pb. +func (b *profileBuilder) pbSample(values []int64, locs []uint64, labels func()) { + start := b.pb.startMessage() + b.pb.int64s(tagSample_Value, values) + b.pb.uint64s(tagSample_Location, locs) + if labels != nil { + labels() + } + b.pb.endMessage(tagProfile_Sample, start) + b.flush() +} + +// pbLabel encodes a Label message to b.pb. +func (b *profileBuilder) pbLabel(tag int, key, str string, num int64) { + start := b.pb.startMessage() + b.pb.int64Opt(tagLabel_Key, b.stringIndex(key)) + b.pb.int64Opt(tagLabel_Str, b.stringIndex(str)) + b.pb.int64Opt(tagLabel_Num, num) + b.pb.endMessage(tag, start) +} + +// pbLine encodes a Line message to b.pb. +func (b *profileBuilder) pbLine(tag int, funcID uint64, line int64) { + start := b.pb.startMessage() + b.pb.uint64Opt(tagLine_FunctionID, funcID) + b.pb.int64Opt(tagLine_Line, line) + b.pb.endMessage(tag, start) +} + +// pbMapping encodes a Mapping message to b.pb. +func (b *profileBuilder) pbMapping(tag int, id, base, limit, offset uint64, file, buildID string) { + start := b.pb.startMessage() + b.pb.uint64Opt(tagMapping_ID, id) + b.pb.uint64Opt(tagMapping_Start, base) + b.pb.uint64Opt(tagMapping_Limit, limit) + b.pb.uint64Opt(tagMapping_Offset, offset) + b.pb.int64Opt(tagMapping_Filename, b.stringIndex(file)) + b.pb.int64Opt(tagMapping_BuildID, b.stringIndex(buildID)) + // TODO: Set any of HasInlineFrames, HasFunctions, HasFilenames, HasLineNumbers? + // It seems like they should all be true, but they've never been set. + b.pb.endMessage(tag, start) +} + +// locForPC returns the location ID for addr. +// addr must be a return PC. This returns the location of the call. +// It may emit to b.pb, so there must be no message encoding in progress. +func (b *profileBuilder) locForPC(addr uintptr) uint64 { + id := uint64(b.locs[addr]) + if id != 0 { + return id + } + + // Expand this one address using CallersFrames so we can cache + // each expansion. In general, CallersFrames takes a whole + // stack, but in this case we know there will be no skips in + // the stack and we have return PCs anyway. + frames := runtime.CallersFrames([]uintptr{addr}) + frame, more := frames.Next() + if frame.Function == "runtime.goexit" { + // Short-circuit if we see runtime.goexit so the loop + // below doesn't allocate a useless empty location. + return 0 + } + + if frame.PC == 0 { + // If we failed to resolve the frame, at least make up + // a reasonable call PC. This mostly happens in tests. + frame.PC = addr - 1 + } + + // We can't write out functions while in the middle of the + // Location message, so record new functions we encounter and + // write them out after the Location. + type newFunc struct { + id uint64 + name, file string + } + newFuncs := make([]newFunc, 0, 8) + + id = uint64(len(b.locs)) + 1 + b.locs[addr] = int(id) + start := b.pb.startMessage() + b.pb.uint64Opt(tagLocation_ID, id) + b.pb.uint64Opt(tagLocation_Address, uint64(frame.PC)) + for frame.Function != "runtime.goexit" { + // Write out each line in frame expansion. + funcID := uint64(b.funcs[frame.Function]) + if funcID == 0 { + funcID = uint64(len(b.funcs)) + 1 + b.funcs[frame.Function] = int(funcID) + newFuncs = append(newFuncs, newFunc{funcID, frame.Function, frame.File}) + } + b.pbLine(tagLocation_Line, funcID, int64(frame.Line)) + if !more { + break + } + frame, more = frames.Next() + } + if len(b.mem) > 0 { + i := sort.Search(len(b.mem), func(i int) bool { + return b.mem[i].end > addr + }) + if i < len(b.mem) && b.mem[i].start <= addr && addr < b.mem[i].end { + b.pb.uint64Opt(tagLocation_MappingID, uint64(i+1)) + } + } + b.pb.endMessage(tagProfile_Location, start) + + // Write out functions we found during frame expansion. + for _, fn := range newFuncs { + start := b.pb.startMessage() + b.pb.uint64Opt(tagFunction_ID, fn.id) + b.pb.int64Opt(tagFunction_Name, b.stringIndex(fn.name)) + b.pb.int64Opt(tagFunction_SystemName, b.stringIndex(fn.name)) + b.pb.int64Opt(tagFunction_Filename, b.stringIndex(fn.file)) + b.pb.endMessage(tagProfile_Function, start) + } + + b.flush() + return id +} + +// newProfileBuilder returns a new profileBuilder. +// CPU profiling data obtained from the runtime can be added +// by calling b.addCPUData, and then the eventual profile +// can be obtained by calling b.finish. +func newProfileBuilder(w io.Writer) *profileBuilder { + zw, _ := gzip.NewWriterLevel(w, gzip.BestSpeed) + b := &profileBuilder{ + w: w, + zw: zw, + start: time.Now(), + strings: []string{""}, + stringMap: map[string]int{"": 0}, + locs: map[uintptr]int{}, + funcs: map[string]int{}, + } + b.readMapping() + return b +} + +// addCPUData adds the CPU profiling data to the profile. +// The data must be a whole number of records, +// as delivered by the runtime. +func (b *profileBuilder) addCPUData(data []uint64, tags []unsafe.Pointer) error { + if !b.havePeriod { + // first record is period + if len(data) < 3 { + return fmt.Errorf("truncated profile") + } + if data[0] != 3 || data[2] == 0 { + return fmt.Errorf("malformed profile") + } + // data[2] is sampling rate in Hz. Convert to sampling + // period in nanoseconds. + b.period = 1e9 / int64(data[2]) + b.havePeriod = true + data = data[3:] + } + + // Parse CPU samples from the profile. + // Each sample is 3+n uint64s: + // data[0] = 3+n + // data[1] = time stamp (ignored) + // data[2] = count + // data[3:3+n] = stack + // If the count is 0 and the stack has length 1, + // that's an overflow record inserted by the runtime + // to indicate that stack[0] samples were lost. + // Otherwise the count is usually 1, + // but in a few special cases like lost non-Go samples + // there can be larger counts. + // Because many samples with the same stack arrive, + // we want to deduplicate immediately, which we do + // using the b.m profMap. + for len(data) > 0 { + if len(data) < 3 || data[0] > uint64(len(data)) { + return fmt.Errorf("truncated profile") + } + if data[0] < 3 || tags != nil && len(tags) < 1 { + return fmt.Errorf("malformed profile") + } + count := data[2] + stk := data[3:data[0]] + data = data[data[0]:] + var tag unsafe.Pointer + if tags != nil { + tag = tags[0] + tags = tags[1:] + } + + if count == 0 && len(stk) == 1 { + // overflow record + count = uint64(stk[0]) + stk = []uint64{ + uint64(funcPC(lostProfileEvent)), + } + } + b.m.lookup(stk, tag).count += int64(count) + } + return nil +} + +// build completes and returns the constructed profile. +func (b *profileBuilder) build() error { + b.end = time.Now() + + b.pb.int64Opt(tagProfile_TimeNanos, b.start.UnixNano()) + if b.havePeriod { // must be CPU profile + b.pbValueType(tagProfile_SampleType, "samples", "count") + b.pbValueType(tagProfile_SampleType, "cpu", "nanoseconds") + b.pb.int64Opt(tagProfile_DurationNanos, b.end.Sub(b.start).Nanoseconds()) + b.pbValueType(tagProfile_PeriodType, "cpu", "nanoseconds") + b.pb.int64Opt(tagProfile_Period, b.period) + } + + values := []int64{0, 0} + var locs []uint64 + for e := b.m.all; e != nil; e = e.nextAll { + values[0] = e.count + values[1] = e.count * b.period + + var labels func() + if e.tag != nil { + labels = func() { + for k, v := range *(*labelMap)(e.tag) { + b.pbLabel(tagSample_Label, k, v, 0) + } + } + } + + locs = locs[:0] + for i, addr := range e.stk { + // Addresses from stack traces point to the + // next instruction after each call, except + // for the leaf, which points to where the + // signal occurred. locForPC expects return + // PCs, so increment the leaf address to look + // like a return PC. + if i == 0 { + addr++ + } + l := b.locForPC(addr) + if l == 0 { // runtime.goexit + continue + } + locs = append(locs, l) + } + b.pbSample(values, locs, labels) + } + + // TODO: Anything for tagProfile_DropFrames? + // TODO: Anything for tagProfile_KeepFrames? + + b.pb.strings(tagProfile_StringTable, b.strings) + b.zw.Write(b.pb.data) + b.zw.Close() + return nil +} + +// readMapping reads /proc/self/maps and writes mappings to b.pb. +// It saves the address ranges of the mappings in b.mem for use +// when emitting locations. +func (b *profileBuilder) readMapping() { + data, _ := ioutil.ReadFile("/proc/self/maps") + parseProcSelfMaps(data, b.addMapping) +} + +func parseProcSelfMaps(data []byte, addMapping func(lo, hi, offset uint64, file, buildID string)) { + // $ cat /proc/self/maps + // 00400000-0040b000 r-xp 00000000 fc:01 787766 /bin/cat + // 0060a000-0060b000 r--p 0000a000 fc:01 787766 /bin/cat + // 0060b000-0060c000 rw-p 0000b000 fc:01 787766 /bin/cat + // 014ab000-014cc000 rw-p 00000000 00:00 0 [heap] + // 7f7d76af8000-7f7d7797c000 r--p 00000000 fc:01 1318064 /usr/lib/locale/locale-archive + // 7f7d7797c000-7f7d77b36000 r-xp 00000000 fc:01 1180226 /lib/x86_64-linux-gnu/libc-2.19.so + // 7f7d77b36000-7f7d77d36000 ---p 001ba000 fc:01 1180226 /lib/x86_64-linux-gnu/libc-2.19.so + // 7f7d77d36000-7f7d77d3a000 r--p 001ba000 fc:01 1180226 /lib/x86_64-linux-gnu/libc-2.19.so + // 7f7d77d3a000-7f7d77d3c000 rw-p 001be000 fc:01 1180226 /lib/x86_64-linux-gnu/libc-2.19.so + // 7f7d77d3c000-7f7d77d41000 rw-p 00000000 00:00 0 + // 7f7d77d41000-7f7d77d64000 r-xp 00000000 fc:01 1180217 /lib/x86_64-linux-gnu/ld-2.19.so + // 7f7d77f3f000-7f7d77f42000 rw-p 00000000 00:00 0 + // 7f7d77f61000-7f7d77f63000 rw-p 00000000 00:00 0 + // 7f7d77f63000-7f7d77f64000 r--p 00022000 fc:01 1180217 /lib/x86_64-linux-gnu/ld-2.19.so + // 7f7d77f64000-7f7d77f65000 rw-p 00023000 fc:01 1180217 /lib/x86_64-linux-gnu/ld-2.19.so + // 7f7d77f65000-7f7d77f66000 rw-p 00000000 00:00 0 + // 7ffc342a2000-7ffc342c3000 rw-p 00000000 00:00 0 [stack] + // 7ffc34343000-7ffc34345000 r-xp 00000000 00:00 0 [vdso] + // ffffffffff600000-ffffffffff601000 r-xp 00000000 00:00 0 [vsyscall] + + var line []byte + // next removes and returns the next field in the line. + // It also removes from line any spaces following the field. + next := func() []byte { + j := bytes.IndexByte(line, ' ') + if j < 0 { + f := line + line = nil + return f + } + f := line[:j] + line = line[j+1:] + for len(line) > 0 && line[0] == ' ' { + line = line[1:] + } + return f + } + + for len(data) > 0 { + i := bytes.IndexByte(data, '\n') + if i < 0 { + line, data = data, nil + } else { + line, data = data[:i], data[i+1:] + } + addr := next() + i = bytes.IndexByte(addr, '-') + if i < 0 { + continue + } + lo, err := strconv.ParseUint(string(addr[:i]), 16, 64) + if err != nil { + continue + } + hi, err := strconv.ParseUint(string(addr[i+1:]), 16, 64) + if err != nil { + continue + } + perm := next() + if len(perm) < 4 || perm[2] != 'x' { + // Only interested in executable mappings. + continue + } + offset, err := strconv.ParseUint(string(next()), 16, 64) + if err != nil { + continue + } + next() // dev + inode := next() // inode + if line == nil { + continue + } + file := string(line) + if len(inode) == 1 && inode[0] == '0' && file == "" { + // Huge-page text mappings list the initial fragment of + // mapped but unpopulated memory as being inode 0. + // Don't report that part. + // But [vdso] and [vsyscall] are inode 0, so let non-empty file names through. + continue + } + + // TODO: pprof's remapMappingIDs makes two adjustments: + // 1. If there is an /anon_hugepage mapping first and it is + // consecutive to a next mapping, drop the /anon_hugepage. + // 2. If start-offset = 0x400000, change start to 0x400000 and offset to 0. + // There's no indication why either of these is needed. + // Let's try not doing these and see what breaks. + // If we do need them, they would go here, before we + // enter the mappings into b.mem in the first place. + + buildID, _ := elfBuildID(file) + addMapping(lo, hi, offset, file, buildID) + } +} + +func (b *profileBuilder) addMapping(lo, hi, offset uint64, file, buildID string) { + b.mem = append(b.mem, memMap{uintptr(lo), uintptr(hi)}) + b.pbMapping(tagProfile_Mapping, uint64(len(b.mem)), lo, hi, offset, file, buildID) +} diff --git a/libgo/go/runtime/pprof/proto_test.go b/libgo/go/runtime/pprof/proto_test.go new file mode 100644 index 00000000000..a268c3a95d9 --- /dev/null +++ b/libgo/go/runtime/pprof/proto_test.go @@ -0,0 +1,224 @@ +// Copyright 2016 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package pprof + +import ( + "bytes" + "encoding/json" + "fmt" + "io/ioutil" + "reflect" + "runtime" + "runtime/pprof/internal/profile" + "strings" + "testing" +) + +// translateCPUProfile parses binary CPU profiling stack trace data +// generated by runtime.CPUProfile() into a profile struct. +// This is only used for testing. Real conversions stream the +// data into the profileBuilder as it becomes available. +func translateCPUProfile(data []uint64) (*profile.Profile, error) { + var buf bytes.Buffer + b := newProfileBuilder(&buf) + if err := b.addCPUData(data, nil); err != nil { + return nil, err + } + b.build() + return profile.Parse(&buf) +} + +// fmtJSON returns a pretty-printed JSON form for x. +// It works reasonbly well for printing protocol-buffer +// data structures like profile.Profile. +func fmtJSON(x interface{}) string { + js, _ := json.MarshalIndent(x, "", "\t") + return string(js) +} + +func TestConvertCPUProfileEmpty(t *testing.T) { + // A test server with mock cpu profile data. + var buf bytes.Buffer + + b := []uint64{3, 0, 500} // empty profile at 500 Hz (2ms sample period) + p, err := translateCPUProfile(b) + if err != nil { + t.Fatalf("translateCPUProfile: %v", err) + } + if err := p.Write(&buf); err != nil { + t.Fatalf("writing profile: %v", err) + } + + p, err = profile.Parse(&buf) + if err != nil { + t.Fatalf("profile.Parse: %v", err) + } + + // Expected PeriodType and SampleType. + periodType := &profile.ValueType{Type: "cpu", Unit: "nanoseconds"} + sampleType := []*profile.ValueType{ + {Type: "samples", Unit: "count"}, + {Type: "cpu", Unit: "nanoseconds"}, + } + + checkProfile(t, p, 2000*1000, periodType, sampleType, nil) +} + +// For gccgo make these functions different so that gccgo doesn't +// merge them with each other and with lostProfileEvent. +func f1(i int) { f1(i + 1) } +func f2(i int) { f2(i + 2) } + +// testPCs returns two PCs and two corresponding memory mappings +// to use in test profiles. +func testPCs(t *testing.T) (addr1, addr2 uint64, map1, map2 *profile.Mapping) { + switch runtime.GOOS { + case "linux", "android", "netbsd": + // Figure out two addresses from /proc/self/maps. + mmap, err := ioutil.ReadFile("/proc/self/maps") + if err != nil { + t.Fatal(err) + } + mprof := &profile.Profile{} + if err = mprof.ParseMemoryMap(bytes.NewReader(mmap)); err != nil { + t.Fatalf("parsing /proc/self/maps: %v", err) + } + if len(mprof.Mapping) < 2 { + // It is possible for a binary to only have 1 executable + // region of memory. + t.Skipf("need 2 or more mappings, got %v", len(mprof.Mapping)) + } + addr1 = mprof.Mapping[0].Start + map1 = mprof.Mapping[0] + map1.BuildID, _ = elfBuildID(map1.File) + addr2 = mprof.Mapping[1].Start + map2 = mprof.Mapping[1] + map2.BuildID, _ = elfBuildID(map2.File) + default: + addr1 = uint64(funcPC(f1)) + addr2 = uint64(funcPC(f2)) + } + return +} + +func TestConvertCPUProfile(t *testing.T) { + addr1, addr2, map1, map2 := testPCs(t) + + b := []uint64{ + 3, 0, 500, // hz = 500 + 5, 0, 10, uint64(addr1), uint64(addr1 + 2), // 10 samples in addr1 + 5, 0, 40, uint64(addr2), uint64(addr2 + 2), // 40 samples in addr2 + 5, 0, 10, uint64(addr1), uint64(addr1 + 2), // 10 samples in addr1 + } + p, err := translateCPUProfile(b) + if err != nil { + t.Fatalf("translating profile: %v", err) + } + period := int64(2000 * 1000) + periodType := &profile.ValueType{Type: "cpu", Unit: "nanoseconds"} + sampleType := []*profile.ValueType{ + {Type: "samples", Unit: "count"}, + {Type: "cpu", Unit: "nanoseconds"}, + } + samples := []*profile.Sample{ + {Value: []int64{20, 20 * 2000 * 1000}, Location: []*profile.Location{ + {ID: 1, Mapping: map1, Address: addr1}, + {ID: 2, Mapping: map1, Address: addr1 + 1}, + }}, + {Value: []int64{40, 40 * 2000 * 1000}, Location: []*profile.Location{ + {ID: 3, Mapping: map2, Address: addr2}, + {ID: 4, Mapping: map2, Address: addr2 + 1}, + }}, + } + checkProfile(t, p, period, periodType, sampleType, samples) +} + +func checkProfile(t *testing.T, p *profile.Profile, period int64, periodType *profile.ValueType, sampleType []*profile.ValueType, samples []*profile.Sample) { + if p.Period != period { + t.Fatalf("p.Period = %d, want %d", p.Period, period) + } + if !reflect.DeepEqual(p.PeriodType, periodType) { + t.Fatalf("p.PeriodType = %v\nwant = %v", fmtJSON(p.PeriodType), fmtJSON(periodType)) + } + if !reflect.DeepEqual(p.SampleType, sampleType) { + t.Fatalf("p.SampleType = %v\nwant = %v", fmtJSON(p.SampleType), fmtJSON(sampleType)) + } + // Clear line info since it is not in the expected samples. + // If we used f1 and f2 above, then the samples will have line info. + for _, s := range p.Sample { + for _, l := range s.Location { + l.Line = nil + } + } + if fmtJSON(p.Sample) != fmtJSON(samples) { // ignore unexported fields + if len(p.Sample) == len(samples) { + for i := range p.Sample { + if !reflect.DeepEqual(p.Sample[i], samples[i]) { + t.Errorf("sample %d = %v\nwant = %v\n", i, fmtJSON(p.Sample[i]), fmtJSON(samples[i])) + } + } + if t.Failed() { + t.FailNow() + } + } + t.Fatalf("p.Sample = %v\nwant = %v", fmtJSON(p.Sample), fmtJSON(samples)) + } +} + +var profSelfMapsTests = ` +00400000-0040b000 r-xp 00000000 fc:01 787766 /bin/cat +0060a000-0060b000 r--p 0000a000 fc:01 787766 /bin/cat +0060b000-0060c000 rw-p 0000b000 fc:01 787766 /bin/cat +014ab000-014cc000 rw-p 00000000 00:00 0 [heap] +7f7d76af8000-7f7d7797c000 r--p 00000000 fc:01 1318064 /usr/lib/locale/locale-archive +7f7d7797c000-7f7d77b36000 r-xp 00000000 fc:01 1180226 /lib/x86_64-linux-gnu/libc-2.19.so +7f7d77b36000-7f7d77d36000 ---p 001ba000 fc:01 1180226 /lib/x86_64-linux-gnu/libc-2.19.so +7f7d77d36000-7f7d77d3a000 r--p 001ba000 fc:01 1180226 /lib/x86_64-linux-gnu/libc-2.19.so +7f7d77d3a000-7f7d77d3c000 rw-p 001be000 fc:01 1180226 /lib/x86_64-linux-gnu/libc-2.19.so +7f7d77d3c000-7f7d77d41000 rw-p 00000000 00:00 0 +7f7d77d41000-7f7d77d64000 r-xp 00000000 fc:01 1180217 /lib/x86_64-linux-gnu/ld-2.19.so +7f7d77f3f000-7f7d77f42000 rw-p 00000000 00:00 0 +7f7d77f61000-7f7d77f63000 rw-p 00000000 00:00 0 +7f7d77f63000-7f7d77f64000 r--p 00022000 fc:01 1180217 /lib/x86_64-linux-gnu/ld-2.19.so +7f7d77f64000-7f7d77f65000 rw-p 00023000 fc:01 1180217 /lib/x86_64-linux-gnu/ld-2.19.so +7f7d77f65000-7f7d77f66000 rw-p 00000000 00:00 0 +7ffc342a2000-7ffc342c3000 rw-p 00000000 00:00 0 [stack] +7ffc34343000-7ffc34345000 r-xp 00000000 00:00 0 [vdso] +ffffffffff600000-ffffffffff601000 r-xp 00000090 00:00 0 [vsyscall] +-> +00400000 0040b000 00000000 /bin/cat +7f7d7797c000 7f7d77b36000 00000000 /lib/x86_64-linux-gnu/libc-2.19.so +7f7d77d41000 7f7d77d64000 00000000 /lib/x86_64-linux-gnu/ld-2.19.so +7ffc34343000 7ffc34345000 00000000 [vdso] +ffffffffff600000 ffffffffff601000 00000090 [vsyscall] + +00400000-07000000 r-xp 00000000 00:00 0 +07000000-07093000 r-xp 06c00000 00:2e 536754 /path/to/gobench_server_main +07093000-0722d000 rw-p 06c92000 00:2e 536754 /path/to/gobench_server_main +0722d000-07b21000 rw-p 00000000 00:00 0 +c000000000-c000036000 rw-p 00000000 00:00 0 +-> +07000000 07093000 06c00000 /path/to/gobench_server_main +` + +func TestProcSelfMaps(t *testing.T) { + for tx, tt := range strings.Split(profSelfMapsTests, "\n\n") { + i := strings.Index(tt, "->\n") + if i < 0 { + t.Fatal("malformed test case") + } + in, out := tt[:i], tt[i+len("->\n"):] + if len(out) > 0 && out[len(out)-1] != '\n' { + out += "\n" + } + var buf bytes.Buffer + parseProcSelfMaps([]byte(in), func(lo, hi, offset uint64, file, buildID string) { + fmt.Fprintf(&buf, "%08x %08x %08x %s\n", lo, hi, offset, file) + }) + if buf.String() != out { + t.Errorf("#%d: have:\n%s\nwant:\n%s\n%q\n%q", tx, buf.String(), out, buf.String(), out) + } + } +} diff --git a/libgo/go/runtime/pprof/protobuf.go b/libgo/go/runtime/pprof/protobuf.go new file mode 100644 index 00000000000..7b99095a13a --- /dev/null +++ b/libgo/go/runtime/pprof/protobuf.go @@ -0,0 +1,141 @@ +// Copyright 2014 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package pprof + +// A protobuf is a simple protocol buffer encoder. +type protobuf struct { + data []byte + tmp [16]byte + nest int +} + +func (b *protobuf) varint(x uint64) { + for x >= 128 { + b.data = append(b.data, byte(x)|0x80) + x >>= 7 + } + b.data = append(b.data, byte(x)) +} + +func (b *protobuf) length(tag int, len int) { + b.varint(uint64(tag)<<3 | 2) + b.varint(uint64(len)) +} + +func (b *protobuf) uint64(tag int, x uint64) { + // append varint to b.data + b.varint(uint64(tag)<<3 | 0) + b.varint(x) +} + +func (b *protobuf) uint64s(tag int, x []uint64) { + if len(x) > 2 { + // Use packed encoding + n1 := len(b.data) + for _, u := range x { + b.varint(u) + } + n2 := len(b.data) + b.length(tag, n2-n1) + n3 := len(b.data) + copy(b.tmp[:], b.data[n2:n3]) + copy(b.data[n1+(n3-n2):], b.data[n1:n2]) + copy(b.data[n1:], b.tmp[:n3-n2]) + return + } + for _, u := range x { + b.uint64(tag, u) + } +} + +func (b *protobuf) uint64Opt(tag int, x uint64) { + if x == 0 { + return + } + b.uint64(tag, x) +} + +func (b *protobuf) int64(tag int, x int64) { + u := uint64(x) + b.uint64(tag, u) +} + +func (b *protobuf) int64Opt(tag int, x int64) { + if x == 0 { + return + } + b.int64(tag, x) +} + +func (b *protobuf) int64s(tag int, x []int64) { + if len(x) > 2 { + // Use packed encoding + n1 := len(b.data) + for _, u := range x { + b.varint(uint64(u)) + } + n2 := len(b.data) + b.length(tag, n2-n1) + n3 := len(b.data) + copy(b.tmp[:], b.data[n2:n3]) + copy(b.data[n1+(n3-n2):], b.data[n1:n2]) + copy(b.data[n1:], b.tmp[:n3-n2]) + return + } + for _, u := range x { + b.int64(tag, u) + } +} + +func (b *protobuf) string(tag int, x string) { + b.length(tag, len(x)) + b.data = append(b.data, x...) +} + +func (b *protobuf) strings(tag int, x []string) { + for _, s := range x { + b.string(tag, s) + } +} + +func (b *protobuf) stringOpt(tag int, x string) { + if x == "" { + return + } + b.string(tag, x) +} + +func (b *protobuf) bool(tag int, x bool) { + if x { + b.uint64(tag, 1) + } else { + b.uint64(tag, 0) + } +} + +func (b *protobuf) boolOpt(tag int, x bool) { + if x == false { + return + } + b.bool(tag, x) +} + +type msgOffset int + +func (b *protobuf) startMessage() msgOffset { + b.nest++ + return msgOffset(len(b.data)) +} + +func (b *protobuf) endMessage(tag int, start msgOffset) { + n1 := int(start) + n2 := len(b.data) + b.length(tag, n2-n1) + n3 := len(b.data) + copy(b.tmp[:], b.data[n2:n3]) + copy(b.data[n1+(n3-n2):], b.data[n1:n2]) + copy(b.data[n1:], b.tmp[:n3-n2]) + b.nest-- +} diff --git a/libgo/go/runtime/pprof/protomem.go b/libgo/go/runtime/pprof/protomem.go new file mode 100644 index 00000000000..2756cfd28d8 --- /dev/null +++ b/libgo/go/runtime/pprof/protomem.go @@ -0,0 +1,93 @@ +// Copyright 2016 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package pprof + +import ( + "io" + "math" + "runtime" + "strings" +) + +// writeHeapProto writes the current heap profile in protobuf format to w. +func writeHeapProto(w io.Writer, p []runtime.MemProfileRecord, rate int64) error { + b := newProfileBuilder(w) + b.pbValueType(tagProfile_PeriodType, "space", "bytes") + b.pb.int64Opt(tagProfile_Period, rate) + b.pbValueType(tagProfile_SampleType, "alloc_objects", "count") + b.pbValueType(tagProfile_SampleType, "alloc_space", "bytes") + b.pbValueType(tagProfile_SampleType, "inuse_objects", "count") + b.pbValueType(tagProfile_SampleType, "inuse_space", "bytes") + + values := []int64{0, 0, 0, 0} + var locs []uint64 + for _, r := range p { + locs = locs[:0] + hideRuntime := true + for tries := 0; tries < 2; tries++ { + for _, addr := range r.Stack() { + // For heap profiles, all stack + // addresses are return PCs, which is + // what locForPC expects. + if hideRuntime { + if f := runtime.FuncForPC(addr); f != nil && strings.HasPrefix(f.Name(), "runtime.") { + continue + } + // Found non-runtime. Show any runtime uses above it. + hideRuntime = false + } + l := b.locForPC(addr) + if l == 0 { // runtime.goexit + continue + } + locs = append(locs, l) + } + if len(locs) > 0 { + break + } + hideRuntime = false // try again, and show all frames + } + + values[0], values[1] = scaleHeapSample(r.AllocObjects, r.AllocBytes, rate) + values[2], values[3] = scaleHeapSample(r.InUseObjects(), r.InUseBytes(), rate) + var blockSize int64 + if values[0] > 0 { + blockSize = values[1] / values[0] + } + b.pbSample(values, locs, func() { + if blockSize != 0 { + b.pbLabel(tagSample_Label, "bytes", "", blockSize) + } + }) + } + b.build() + return nil +} + +// scaleHeapSample adjusts the data from a heap Sample to +// account for its probability of appearing in the collected +// data. heap profiles are a sampling of the memory allocations +// requests in a program. We estimate the unsampled value by dividing +// each collected sample by its probability of appearing in the +// profile. heap profiles rely on a poisson process to determine +// which samples to collect, based on the desired average collection +// rate R. The probability of a sample of size S to appear in that +// profile is 1-exp(-S/R). +func scaleHeapSample(count, size, rate int64) (int64, int64) { + if count == 0 || size == 0 { + return 0, 0 + } + + if rate <= 1 { + // if rate==1 all samples were collected so no adjustment is needed. + // if rate<1 treat as unknown and skip scaling. + return count, size + } + + avgSize := float64(size) / float64(count) + scale := 1 / (1 - math.Exp(-avgSize/float64(rate))) + + return int64(float64(count) * scale), int64(float64(size) * scale) +} diff --git a/libgo/go/runtime/pprof/protomem_test.go b/libgo/go/runtime/pprof/protomem_test.go new file mode 100644 index 00000000000..1e30ed93a36 --- /dev/null +++ b/libgo/go/runtime/pprof/protomem_test.go @@ -0,0 +1,74 @@ +// Copyright 2016 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package pprof + +import ( + "bytes" + "runtime" + "runtime/pprof/internal/profile" + "testing" +) + +func TestConvertMemProfile(t *testing.T) { + addr1, addr2, map1, map2 := testPCs(t) + + var buf bytes.Buffer + // MemProfileRecord stacks are return PCs, so add one to the + // addresses recorded in the "profile". The proto profile + // locations are call PCs, so conversion will subtract one + // from these and get back to addr1 and addr2. + a1, a2 := uintptr(addr1)+1, uintptr(addr2)+1 + rate := int64(512 * 1024) + rec := []runtime.MemProfileRecord{ + {AllocBytes: 4096, FreeBytes: 1024, AllocObjects: 4, FreeObjects: 1, Stack0: [32]uintptr{a1, a2}}, + {AllocBytes: 512 * 1024, FreeBytes: 0, AllocObjects: 1, FreeObjects: 0, Stack0: [32]uintptr{a2 + 1, a2 + 2}}, + {AllocBytes: 512 * 1024, FreeBytes: 512 * 1024, AllocObjects: 1, FreeObjects: 1, Stack0: [32]uintptr{a1 + 1, a1 + 2, a2 + 3}}, + } + + if err := writeHeapProto(&buf, rec, rate); err != nil { + t.Fatalf("writing profile: %v", err) + } + + p, err := profile.Parse(&buf) + if err != nil { + t.Fatalf("profile.Parse: %v", err) + } + + periodType := &profile.ValueType{Type: "space", Unit: "bytes"} + sampleType := []*profile.ValueType{ + {Type: "alloc_objects", Unit: "count"}, + {Type: "alloc_space", Unit: "bytes"}, + {Type: "inuse_objects", Unit: "count"}, + {Type: "inuse_space", Unit: "bytes"}, + } + samples := []*profile.Sample{ + { + Value: []int64{2050, 2099200, 1537, 1574400}, + Location: []*profile.Location{ + {ID: 1, Mapping: map1, Address: addr1}, + {ID: 2, Mapping: map2, Address: addr2}, + }, + NumLabel: map[string][]int64{"bytes": {1024}}, + }, + { + Value: []int64{1, 829411, 1, 829411}, + Location: []*profile.Location{ + {ID: 3, Mapping: map2, Address: addr2 + 1}, + {ID: 4, Mapping: map2, Address: addr2 + 2}, + }, + NumLabel: map[string][]int64{"bytes": {829411}}, + }, + { + Value: []int64{1, 829411, 0, 0}, + Location: []*profile.Location{ + {ID: 5, Mapping: map1, Address: addr1 + 1}, + {ID: 6, Mapping: map1, Address: addr1 + 2}, + {ID: 7, Mapping: map2, Address: addr2 + 3}, + }, + NumLabel: map[string][]int64{"bytes": {829411}}, + }, + } + checkProfile(t, p, rate, periodType, sampleType, samples) +} diff --git a/libgo/go/runtime/pprof/runtime.go b/libgo/go/runtime/pprof/runtime.go new file mode 100644 index 00000000000..e6aace83e26 --- /dev/null +++ b/libgo/go/runtime/pprof/runtime.go @@ -0,0 +1,36 @@ +// Copyright 2017 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package pprof + +import ( + "context" + "unsafe" +) + +// runtime_setProfLabel is defined in runtime/proflabel.go. +func runtime_setProfLabel(labels unsafe.Pointer) + +// runtime_getProfLabel is defined in runtime/proflabel.go. +func runtime_getProfLabel() unsafe.Pointer + +// SetGoroutineLabels sets the current goroutine's labels to match ctx. +// This is a lower-level API than Do, which should be used instead when possible. +func SetGoroutineLabels(ctx context.Context) { + ctxLabels, _ := ctx.Value(labelContextKey{}).(*labelMap) + runtime_setProfLabel(unsafe.Pointer(ctxLabels)) +} + +// Do calls f with a copy of the parent context with the +// given labels added to the parent's label map. +// Each key/value pair in labels is inserted into the label map in the +// order provided, overriding any previous value for the same key. +// The augmented label map will be set for the duration of the call to f +// and restored once f returns. +func Do(ctx context.Context, labels LabelSet, f func(context.Context)) { + defer SetGoroutineLabels(ctx) + ctx = WithLabels(ctx, labels) + SetGoroutineLabels(ctx) + f(ctx) +} diff --git a/libgo/go/runtime/pprof/runtime_test.go b/libgo/go/runtime/pprof/runtime_test.go new file mode 100644 index 00000000000..0dd5324b42e --- /dev/null +++ b/libgo/go/runtime/pprof/runtime_test.go @@ -0,0 +1,96 @@ +// Copyright 2017 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package pprof + +import ( + "context" + "fmt" + "reflect" + "testing" +) + +func TestSetGoroutineLabels(t *testing.T) { + sync := make(chan struct{}) + + wantLabels := map[string]string{} + if gotLabels := getProfLabel(); !reflect.DeepEqual(gotLabels, wantLabels) { + t.Errorf("Expected parent goroutine's profile labels to be empty before test, got %v", gotLabels) + } + go func() { + if gotLabels := getProfLabel(); !reflect.DeepEqual(gotLabels, wantLabels) { + t.Errorf("Expected child goroutine's profile labels to be empty before test, got %v", gotLabels) + } + sync <- struct{}{} + }() + <-sync + + wantLabels = map[string]string{"key": "value"} + ctx := WithLabels(context.Background(), Labels("key", "value")) + SetGoroutineLabels(ctx) + if gotLabels := getProfLabel(); !reflect.DeepEqual(gotLabels, wantLabels) { + t.Errorf("parent goroutine's profile labels: got %v, want %v", gotLabels, wantLabels) + } + go func() { + if gotLabels := getProfLabel(); !reflect.DeepEqual(gotLabels, wantLabels) { + t.Errorf("child goroutine's profile labels: got %v, want %v", gotLabels, wantLabels) + } + sync <- struct{}{} + }() + <-sync + + wantLabels = map[string]string{} + ctx = context.Background() + SetGoroutineLabels(ctx) + if gotLabels := getProfLabel(); !reflect.DeepEqual(gotLabels, wantLabels) { + t.Errorf("Expected parent goroutine's profile labels to be empty, got %v", gotLabels) + } + go func() { + if gotLabels := getProfLabel(); !reflect.DeepEqual(gotLabels, wantLabels) { + t.Errorf("Expected child goroutine's profile labels to be empty, got %v", gotLabels) + } + sync <- struct{}{} + }() + <-sync +} + +func TestDo(t *testing.T) { + wantLabels := map[string]string{} + if gotLabels := getProfLabel(); !reflect.DeepEqual(gotLabels, wantLabels) { + t.Errorf("Expected parent goroutine's profile labels to be empty before Do, got %v", gotLabels) + } + + Do(context.Background(), Labels("key1", "value1", "key2", "value2"), func(ctx context.Context) { + wantLabels := map[string]string{"key1": "value1", "key2": "value2"} + if gotLabels := getProfLabel(); !reflect.DeepEqual(gotLabels, wantLabels) { + t.Errorf("parent goroutine's profile labels: got %v, want %v", gotLabels, wantLabels) + } + + sync := make(chan struct{}) + go func() { + wantLabels := map[string]string{"key1": "value1", "key2": "value2"} + if gotLabels := getProfLabel(); !reflect.DeepEqual(gotLabels, wantLabels) { + t.Errorf("child goroutine's profile labels: got %v, want %v", gotLabels, wantLabels) + } + sync <- struct{}{} + }() + <-sync + + }) + + wantLabels = map[string]string{} + if gotLabels := getProfLabel(); !reflect.DeepEqual(gotLabels, wantLabels) { + fmt.Printf("%#v", gotLabels) + fmt.Printf("%#v", wantLabels) + t.Errorf("Expected parent goroutine's profile labels to be empty after Do, got %v", gotLabels) + } +} + +func getProfLabel() map[string]string { + l := (*labelMap)(runtime_getProfLabel()) + if l == nil { + return map[string]string{} + } + return *l +} diff --git a/libgo/go/runtime/proc.go b/libgo/go/runtime/proc.go index 6b6f9c9c379..345f57b6875 100644 --- a/libgo/go/runtime/proc.go +++ b/libgo/go/runtime/proc.go @@ -214,8 +214,17 @@ func main() { // Make racy client program work: if panicking on // another goroutine at the same time as main returns, // let the other goroutine finish printing the panic trace. - // Once it does, it will exit. See issue 3934. - if panicking != 0 { + // Once it does, it will exit. See issues 3934 and 20018. + if atomic.Load(&runningPanicDefers) != 0 { + // Running deferred functions should not take long. + for c := 0; c < 1000; c++ { + if atomic.Load(&runningPanicDefers) == 0 { + break + } + Gosched() + } + } + if atomic.Load(&panicking) != 0 { gopark(nil, nil, "panicwait", traceEvGoStop, 1) } @@ -255,18 +264,25 @@ func forcegchelper() { if debug.gctrace > 0 { println("GC forced") } - gcStart(gcBackgroundMode, true) + // Time-triggered, fully concurrent. + gcStart(gcBackgroundMode, gcTrigger{kind: gcTriggerTime, now: nanotime()}) } } -//go:nosplit - // Gosched yields the processor, allowing other goroutines to run. It does not // suspend the current goroutine, so execution resumes automatically. +//go:nosplit func Gosched() { mcall(gosched_m) } +// goschedguarded yields the processor like gosched, but also checks +// for forbidden states and opts out of the yield in those cases. +//go:nosplit +func goschedguarded() { + mcall(goschedguarded_m) +} + // Puts the current goroutine into a waiting state and calls unlockf. // If unlockf returns false, the goroutine is resumed. // unlockf must not access this G's stack, as it may be moved between @@ -419,16 +435,6 @@ func allgadd(gp *g) { lock(&allglock) allgs = append(allgs, gp) allglen = uintptr(len(allgs)) - - // Grow GC rescan list if necessary. - if len(allgs) > cap(work.rescan.list) { - lock(&work.rescan.lock) - l := work.rescan.list - // Let append do the heavy lifting, but keep the - // length the same. - work.rescan.list = append(l[:cap(l)], 0)[:len(l)] - unlock(&work.rescan.lock) - } unlock(&allglock) } @@ -765,9 +771,8 @@ func casgstatus(gp *g, oldval, newval uint32) { nextYield = nanotime() + yieldDelay/2 } } - if newval == _Grunning && gp.gcscanvalid { - // Run queueRescan on the system stack so it has more space. - systemstack(func() { queueRescan(gp) }) + if newval == _Grunning { + gp.gcscanvalid = false } } @@ -779,8 +784,6 @@ func scang(gp *g, gcw *gcWork) { // Nothing is racing with us now, but gcscandone might be set to true left over // from an earlier round of stack scanning (we scan twice per GC). // We use gcscandone to record whether the scan has been done during this round. - // It is important that the scan happens exactly once: if called twice, - // the installation of stack barriers will detect the double scan and die. gp.gcscandone = false @@ -902,7 +905,7 @@ func restartg(gp *g) { // in panic or being exited, this may not reliably stop all // goroutines. func stopTheWorld(reason string) { - semacquire(&worldsema, 0) + semacquire(&worldsema) getg().m.preemptoff = reason systemstack(stopTheWorldWithSema) } @@ -1129,10 +1132,10 @@ func mstart1() { } asminit() - minit() // Install signal handlers; after minit so that minit can // prepare the thread to be able to handle the signals. + // For gccgo minit was called by C code. if _g_.m == &m0 { // Create an extra M for callbacks on threads not created by Go. if iscgo && !cgoHasExtraM { @@ -1363,6 +1366,7 @@ func needm(x byte) { // running at all (that is, there's no garbage collection // running right now). mp.needextram = mp.schedlink == 0 + extraMCount-- unlockextra(mp.schedlink.ptr()) // Save and block signals before installing g. @@ -1376,12 +1380,16 @@ func needm(x byte) { // Install g (= m->curg). setg(mp.curg) - atomic.Store(&mp.curg.atomicstatus, _Gsyscall) - setGContext() // Initialize this thread to use the m. asminit() minit() + + setGContext() + + // mp.curg is now a real goroutine. + casgstatus(mp.curg, _Gdead, _Gsyscall) + atomic.Xadd(&sched.ngsys, -1) } var earlycgocallback = []byte("fatal error: cgo callback before cgo call\n") @@ -1414,14 +1422,12 @@ func oneNewExtraM() { // the goroutine stack ends. mp, g0SP, g0SPSize := allocm(nil, nil, true) gp := malg(true, false, nil, nil) - gp.gcscanvalid = true // fresh G, so no dequeueRescan necessary + gp.gcscanvalid = true gp.gcscandone = true - gp.gcRescan = -1 - - // malg returns status as Gidle, change to Gdead before adding to allg - // where GC will see it. - // gccgo uses Gdead here, not Gsyscall, because the split - // stack context is not initialized. + // malg returns status as _Gidle. Change to _Gdead before + // adding to allg where GC can see it. We use _Gdead to hide + // this from tracebacks and stack scans since it isn't a + // "real" goroutine until needm grabs it. casgstatus(gp, _Gidle, _Gdead) gp.m = mp mp.curg = gp @@ -1436,9 +1442,16 @@ func oneNewExtraM() { // Here we need to set the context for g0. makeGContext(mp.g0, g0SP, g0SPSize) + // gp is now on the allg list, but we don't want it to be + // counted by gcount. It would be more "proper" to increment + // sched.ngfree, but that requires locking. Incrementing ngsys + // has the same effect. + atomic.Xadd(&sched.ngsys, +1) + // Add m to the extra list. mnext := lockextra(true) mp.schedlink.set(mnext) + extraMCount++ unlockextra(mp) } @@ -1474,6 +1487,10 @@ func dropm() { // with no pointer manipulation. mp := getg().m + // Return mp.curg to dead state. + casgstatus(mp.curg, _Gsyscall, _Gdead) + atomic.Xadd(&sched.ngsys, +1) + // Block signals before unminit. // Unminit unregisters the signal handling stack (but needs g on some systems). // Setg(nil) clears g, which is the signal handler's cue not to run Go handlers. @@ -1489,6 +1506,7 @@ func dropm() { mp.curg.gcnextsp = 0 mnext := lockextra(true) + extraMCount++ mp.schedlink.set(mnext) setg(nil) @@ -1505,6 +1523,7 @@ func getm() uintptr { } var extram uintptr +var extraMCount uint32 // Protected by lockextra var extraMWaiters uint32 // lockextra locks the extra list and returns the list head. @@ -1551,6 +1570,10 @@ func unlockextra(mp *m) { atomic.Storeuintptr(&extram, uintptr(unsafe.Pointer(mp))) } +// execLock serializes exec and clone to avoid bugs or unspecified behaviour +// around exec'ing while creating/destroying threads. See issue #19546. +var execLock rwmutex + // Create a new m. It will start off with a call to fn, or else the scheduler. // fn needs to be static and not a heap allocated closure. // May run with m.p==nil, so write barriers are not allowed. @@ -1559,7 +1582,9 @@ func newm(fn func(), _p_ *p) { mp, _, _ := allocm(_p_, fn, false) mp.nextp.set(_p_) mp.sigmask = initSigmask + execLock.rlock() // Prevent process clone. newosproc(mp) + execLock.runlock() } // Stops execution of the current m until new work is available. @@ -1812,7 +1837,7 @@ func execute(gp *g, inheritTime bool) { // Check whether the profiler needs to be turned on or off. hz := sched.profilehz if _g_.m.profilehz != hz { - resetcpuprofiler(hz) + setThreadCPUProfiler(hz) } if trace.enabled { @@ -1850,6 +1875,9 @@ top: ready(gp, 0, true) } } + if *cgo_yield != nil { + asmcgocall(*cgo_yield, nil) + } // local runq if gp, inheritTime := runqget(_p_); gp != nil { @@ -2007,7 +2035,7 @@ stop: } // poll network - if netpollinited() && atomic.Xchg64(&sched.lastpoll, 0) != 0 { + if netpollinited() && atomic.Load(&netpollWaiters) > 0 && atomic.Xchg64(&sched.lastpoll, 0) != 0 { if _g_.m.p != 0 { throw("findrunnable: netpoll with p") } @@ -2048,7 +2076,7 @@ func pollWork() bool { if !runqempty(p) { return true } - if netpollinited() && sched.lastpoll != 0 { + if netpollinited() && atomic.Load(&netpollWaiters) > 0 && sched.lastpoll != 0 { if gp := netpoll(false); gp != nil { injectglist(gp) return true @@ -2211,7 +2239,7 @@ func park_m(gp *g) { _g_ := getg() if trace.enabled { - traceGoPark(_g_.m.waittraceev, _g_.m.waittraceskip, gp) + traceGoPark(_g_.m.waittraceev, _g_.m.waittraceskip) } casgstatus(gp, _Grunning, _Gwaiting) @@ -2256,6 +2284,19 @@ func gosched_m(gp *g) { goschedImpl(gp) } +// goschedguarded is a forbidden-states-avoided version of gosched_m +func goschedguarded_m(gp *g) { + + if gp.m.locks != 0 || gp.m.mallocing != 0 || gp.m.preemptoff != "" || gp.m.p.ptr().status != _Prunning { + gogo(gp) // never return + } + + if trace.enabled { + traceGoSched() + } + goschedImpl(gp) +} + func gopreempt_m(gp *g) { if trace.enabled { traceGoPreempt() @@ -2290,10 +2331,11 @@ func goexit0(gp *g) { gp.writebuf = nil gp.waitreason = "" gp.param = nil + gp.labels = nil + gp.timer = nil // Note that gp's stack scan is now "valid" because it has no - // stack. We could dequeueRescan, but that takes a lock and - // isn't really necessary. + // stack. gp.gcscanvalid = true dropg() @@ -2641,12 +2683,12 @@ func syscall_exitsyscall() { func beforefork() { gp := getg().m.curg - // Fork can hang if preempted with signals frequently enough (see issue 5517). - // Ensure that we stay on the same M where we disable profiling. + // Block signals during a fork, so that the child does not run + // a signal handler before exec if a signal is sent to the process + // group. See issue #18600. gp.m.locks++ - if gp.m.profilehz != 0 { - resetcpuprofiler(0) - } + msigsave(gp.m) + sigblock() } // Called from syscall package before fork. @@ -2659,10 +2701,8 @@ func syscall_runtime_BeforeFork() { func afterfork() { gp := getg().m.curg - hz := sched.profilehz - if hz != 0 { - resetcpuprofiler(hz) - } + msigrestore(gp.m.sigmask) + gp.m.locks-- } @@ -2673,6 +2713,50 @@ func syscall_runtime_AfterFork() { systemstack(afterfork) } +// inForkedChild is true while manipulating signals in the child process. +// This is used to avoid calling libc functions in case we are using vfork. +var inForkedChild bool + +// Called from syscall package after fork in child. +// It resets non-sigignored signals to the default handler, and +// restores the signal mask in preparation for the exec. +// +// Because this might be called during a vfork, and therefore may be +// temporarily sharing address space with the parent process, this must +// not change any global variables or calling into C code that may do so. +// +//go:linkname syscall_runtime_AfterForkInChild syscall.runtime_AfterForkInChild +//go:nosplit +//go:nowritebarrierrec +func syscall_runtime_AfterForkInChild() { + // It's OK to change the global variable inForkedChild here + // because we are going to change it back. There is no race here, + // because if we are sharing address space with the parent process, + // then the parent process can not be running concurrently. + inForkedChild = true + + clearSignalHandlers() + + // When we are the child we are the only thread running, + // so we know that nothing else has changed gp.m.sigmask. + msigrestore(getg().m.sigmask) + + inForkedChild = false +} + +// Called from syscall package before Exec. +//go:linkname syscall_runtime_BeforeExec syscall.runtime_BeforeExec +func syscall_runtime_BeforeExec() { + // Prevent thread creation during exec. + execLock.lock() +} + +// Called from syscall package after Exec. +//go:linkname syscall_runtime_AfterExec syscall.runtime_AfterExec +func syscall_runtime_AfterExec() { + execLock.unlock() +} + // Create a new g running fn passing arg as the single argument. // Put it on the queue of g's waiting to run. // The compiler turns a go statement into a call to this. @@ -2695,7 +2779,6 @@ func newproc(fn uintptr, arg unsafe.Pointer) *g { if newg == nil { newg = malg(true, false, &sp, &spsize) casgstatus(newg, _Gidle, _Gdead) - newg.gcRescan = -1 allgadd(newg) // publishes with a g->status of Gdead so GC scanner doesn't look at uninitialized stack. } else { resetNewG(newg, &sp, &spsize) @@ -2717,17 +2800,13 @@ func newproc(fn uintptr, arg unsafe.Pointer) *g { newg.param = arg newg.gopc = getcallerpc(unsafe.Pointer(&fn)) newg.startpc = fn - // The stack is dirty from the argument frame, so queue it for - // scanning. Do this before setting it to runnable so we still - // own the G. If we're recycling a G, it may already be on the - // rescan list. - if newg.gcRescan == -1 { - queueRescan(newg) - } else { - // The recycled G is already on the rescan list. Just - // mark the stack dirty. - newg.gcscanvalid = false + if _g_.m.curg != nil { + newg.labels = _g_.m.curg.labels + } + if isSystemGoroutine(newg) { + atomic.Xadd(&sched.ngsys, +1) } + newg.gcscanvalid = false casgstatus(newg, _Gdead, _Grunnable) if _p_.goidcache == _p_.goidcacheend { @@ -2926,8 +3005,7 @@ func badunlockosthread() { func gcount() int32 { n := int32(allglen) - sched.ngfree - int32(atomic.Load(&sched.ngsys)) - for i := 0; ; i++ { - _p_ := allp[i] + for _, _p_ := range &allp { if _p_ == nil { break } @@ -2947,13 +3025,18 @@ func mcount() int32 { } var prof struct { - lock uint32 - hz int32 + signalLock uint32 + hz int32 } -func _System() { _System() } -func _ExternalCode() { _ExternalCode() } -func _GC() { _GC() } +func _System() { _System() } +func _ExternalCode() { _ExternalCode() } +func _LostExternalCode() { _LostExternalCode() } +func _GC() { _GC() } +func _LostSIGPROFDuringAtomic64() { _LostSIGPROFDuringAtomic64() } + +// Counts SIGPROFs received while in atomic64 critical section, on mips{,le} +var lostAtomic64Count uint64 var _SystemPC = funcPC(_System) var _ExternalCodePC = funcPC(_ExternalCode) @@ -3009,14 +3092,11 @@ func sigprof(pc uintptr, gp *g, mp *m) { } if prof.hz != 0 { - // Simple cas-lock to coordinate with setcpuprofilerate. - for !atomic.Cas(&prof.lock, 0, 1) { - osyield() + if (GOARCH == "mips" || GOARCH == "mipsle") && lostAtomic64Count > 0 { + cpuprof.addLostAtomic64(lostAtomic64Count) + lostAtomic64Count = 0 } - if prof.hz != 0 { - cpuprof.add(stk[:n]) - } - atomic.Store(&prof.lock, 0) + cpuprof.add(gp, stk[:n]) } getg().m.mallocing-- } @@ -3047,19 +3127,28 @@ func sigprofNonGo(pc uintptr) { nonprofGoStk[1] = _ExternalCodePC + sys.PCQuantum } - // Simple cas-lock to coordinate with setcpuprofilerate. - for !atomic.Cas(&prof.lock, 0, 1) { - osyield() - } - if prof.hz != 0 { - cpuprof.addNonGo(nonprofGoStk[:n]) + cpuprof.addNonGo(nonprofGoStk[:n]) + } +} + +// sigprofNonGoPC is called when a profiling signal arrived on a +// non-Go thread and we have a single PC value, not a stack trace. +// g is nil, and what we can do is very limited. +//go:nosplit +//go:nowritebarrierrec +func sigprofNonGoPC(pc uintptr) { + if prof.hz != 0 { + stk := []uintptr{ + pc, + funcPC(_ExternalCode) + sys.PCQuantum, } - atomic.Store(&prof.lock, 0) + cpuprof.addNonGo(stk) } } -// Arrange to call fn with a traceback hz times a second. -func setcpuprofilerate_m(hz int32) { +// setcpuprofilerate sets the CPU profiling rate to hz times per second. +// If hz <= 0, setcpuprofilerate turns off CPU profiling. +func setcpuprofilerate(hz int32) { // Force sane arguments. if hz < 0 { hz = 0 @@ -3073,20 +3162,23 @@ func setcpuprofilerate_m(hz int32) { // Stop profiler on this thread so that it is safe to lock prof. // if a profiling signal came in while we had prof locked, // it would deadlock. - resetcpuprofiler(0) + setThreadCPUProfiler(0) - for !atomic.Cas(&prof.lock, 0, 1) { + for !atomic.Cas(&prof.signalLock, 0, 1) { osyield() } - prof.hz = hz - atomic.Store(&prof.lock, 0) + if prof.hz != hz { + setProcessCPUProfiler(hz) + prof.hz = hz + } + atomic.Store(&prof.signalLock, 0) lock(&sched.lock) sched.profilehz = hz unlock(&sched.lock) if hz != 0 { - resetcpuprofiler(hz) + setThreadCPUProfiler(hz) } _g_.m.locks-- @@ -3424,7 +3516,25 @@ func sysmon() { if scavengelimit < forcegcperiod { maxsleep = scavengelimit / 2 } + shouldRelax := true + if osRelaxMinNS > 0 { + lock(&timers.lock) + if timers.sleeping { + now := nanotime() + next := timers.sleepUntil + if next-now < osRelaxMinNS { + shouldRelax = false + } + } + unlock(&timers.lock) + } + if shouldRelax { + osRelax(true) + } notetsleep(&sched.sysmonnote, maxsleep) + if shouldRelax { + osRelax(false) + } lock(&sched.lock) atomic.Store(&sched.sysmonwait, 0) noteclear(&sched.sysmonnote) @@ -3433,10 +3543,13 @@ func sysmon() { } unlock(&sched.lock) } + // trigger libc interceptors if needed + if *cgo_yield != nil { + asmcgocall(*cgo_yield, nil) + } // poll network if not polled for more than 10ms lastpoll := int64(atomic.Load64(&sched.lastpoll)) now := nanotime() - unixnow := unixnanotime() if lastpoll != 0 && lastpoll+10*1000*1000 < now { atomic.Cas64(&sched.lastpoll, uint64(lastpoll), uint64(now)) gp := netpoll(false) // non-blocking - returns list of goroutines @@ -3461,8 +3574,7 @@ func sysmon() { idle++ } // check if we need to force a GC - lastgc := int64(atomic.Load64(&memstats.last_gc)) - if gcphase == _GCoff && lastgc != 0 && unixnow-lastgc > forcegcperiod && atomic.Load(&forcegc.idle) != 0 { + if t := (gcTrigger{kind: gcTriggerTime, now: now}); t.test() && atomic.Load(&forcegc.idle) != 0 { lock(&forcegc.lock) forcegc.idle = 0 forcegc.g.schedlink = 0 @@ -3482,7 +3594,7 @@ func sysmon() { } } -var pdesc [_MaxGomaxprocs]struct { +type sysmontick struct { schedtick uint32 schedwhen int64 syscalltick uint32 @@ -3500,7 +3612,7 @@ func retake(now int64) uint32 { if _p_ == nil { continue } - pd := &pdesc[i] + pd := &_p_.sysmontick s := _p_.status if s == _Psyscall { // Retake P from syscall if it's there for more than 1 sysmon tick (at least 20us). @@ -3905,7 +4017,7 @@ func runqputslow(_p_ *p, gp *g, h, t uint32) bool { if randomizeScheduler { for i := uint32(1); i <= n; i++ { - j := fastrand() % (i + 1) + j := fastrandn(i + 1) batch[i], batch[j] = batch[j], batch[i] } } diff --git a/libgo/go/runtime/proc_test.go b/libgo/go/runtime/proc_test.go index 813c92912b9..313a9610e0e 100644 --- a/libgo/go/runtime/proc_test.go +++ b/libgo/go/runtime/proc_test.go @@ -53,14 +53,14 @@ func TestStopTheWorldDeadlock(t *testing.T) { } func TestYieldProgress(t *testing.T) { - testYieldProgress(t, false) + testYieldProgress(false) } func TestYieldLockedProgress(t *testing.T) { - testYieldProgress(t, true) + testYieldProgress(true) } -func testYieldProgress(t *testing.T, locked bool) { +func testYieldProgress(locked bool) { c := make(chan bool) cack := make(chan bool) go func() { @@ -430,10 +430,13 @@ func TestPingPongHog(t *testing.T) { <-lightChan // Check that hogCount and lightCount are within a factor of - // 2, which indicates that both pairs of goroutines handed off - // the P within a time-slice to their buddy. - if hogCount > lightCount*2 || lightCount > hogCount*2 { - t.Fatalf("want hogCount/lightCount in [0.5, 2]; got %d/%d = %g", hogCount, lightCount, float64(hogCount)/float64(lightCount)) + // 5, which indicates that both pairs of goroutines handed off + // the P within a time-slice to their buddy. We can use a + // fairly large factor here to make this robust: if the + // scheduler isn't working right, the gap should be ~1000X. + const factor = 5 + if hogCount > lightCount*factor || lightCount > hogCount*factor { + t.Fatalf("want hogCount/lightCount in [%v, %v]; got %d/%d = %g", 1.0/factor, factor, hogCount, lightCount, float64(hogCount)/float64(lightCount)) } } diff --git a/libgo/go/runtime/profbuf.go b/libgo/go/runtime/profbuf.go new file mode 100644 index 00000000000..f40881aed51 --- /dev/null +++ b/libgo/go/runtime/profbuf.go @@ -0,0 +1,561 @@ +// Copyright 2017 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package runtime + +import ( + "runtime/internal/atomic" + "unsafe" +) + +// A profBuf is a lock-free buffer for profiling events, +// safe for concurrent use by one reader and one writer. +// The writer may be a signal handler running without a user g. +// The reader is assumed to be a user g. +// +// Each logged event corresponds to a fixed size header, a list of +// uintptrs (typically a stack), and exactly one unsafe.Pointer tag. +// The header and uintptrs are stored in the circular buffer data and the +// tag is stored in a circular buffer tags, running in parallel. +// In the circular buffer data, each event takes 2+hdrsize+len(stk) +// words: the value 2+hdrsize+len(stk), then the time of the event, then +// hdrsize words giving the fixed-size header, and then len(stk) words +// for the stack. +// +// The current effective offsets into the tags and data circular buffers +// for reading and writing are stored in the high 30 and low 32 bits of r and w. +// The bottom bits of the high 32 are additional flag bits in w, unused in r. +// "Effective" offsets means the total number of reads or writes, mod 2^length. +// The offset in the buffer is the effective offset mod the length of the buffer. +// To make wraparound mod 2^length match wraparound mod length of the buffer, +// the length of the buffer must be a power of two. +// +// If the reader catches up to the writer, a flag passed to read controls +// whether the read blocks until more data is available. A read returns a +// pointer to the buffer data itself; the caller is assumed to be done with +// that data at the next read. The read offset rNext tracks the next offset to +// be returned by read. By definition, r ≤ rNext ≤ w (before wraparound), +// and rNext is only used by the reader, so it can be accessed without atomics. +// +// If the writer gets ahead of the reader, so that the buffer fills, +// future writes are discarded and replaced in the output stream by an +// overflow entry, which has size 2+hdrsize+1, time set to the time of +// the first discarded write, a header of all zeroed words, and a "stack" +// containing one word, the number of discarded writes. +// +// Between the time the buffer fills and the buffer becomes empty enough +// to hold more data, the overflow entry is stored as a pending overflow +// entry in the fields overflow and overflowTime. The pending overflow +// entry can be turned into a real record by either the writer or the +// reader. If the writer is called to write a new record and finds that +// the output buffer has room for both the pending overflow entry and the +// new record, the writer emits the pending overflow entry and the new +// record into the buffer. If the reader is called to read data and finds +// that the output buffer is empty but that there is a pending overflow +// entry, the reader will return a synthesized record for the pending +// overflow entry. +// +// Only the writer can create or add to a pending overflow entry, but +// either the reader or the writer can clear the pending overflow entry. +// A pending overflow entry is indicated by the low 32 bits of 'overflow' +// holding the number of discarded writes, and overflowTime holding the +// time of the first discarded write. The high 32 bits of 'overflow' +// increment each time the low 32 bits transition from zero to non-zero +// or vice versa. This sequence number avoids ABA problems in the use of +// compare-and-swap to coordinate between reader and writer. +// The overflowTime is only written when the low 32 bits of overflow are +// zero, that is, only when there is no pending overflow entry, in +// preparation for creating a new one. The reader can therefore fetch and +// clear the entry atomically using +// +// for { +// overflow = load(&b.overflow) +// if uint32(overflow) == 0 { +// // no pending entry +// break +// } +// time = load(&b.overflowTime) +// if cas(&b.overflow, overflow, ((overflow>>32)+1)<<32) { +// // pending entry cleared +// break +// } +// } +// if uint32(overflow) > 0 { +// emit entry for uint32(overflow), time +// } +// +type profBuf struct { + // accessed atomically + r, w profAtomic + overflow uint64 + overflowTime uint64 + eof uint32 + + // immutable (excluding slice content) + hdrsize uintptr + data []uint64 + tags []unsafe.Pointer + + // owned by reader + rNext profIndex + overflowBuf []uint64 // for use by reader to return overflow record + wait note +} + +// A profAtomic is the atomically-accessed word holding a profIndex. +type profAtomic uint64 + +// A profIndex is the packet tag and data counts and flags bits, described above. +type profIndex uint64 + +const ( + profReaderSleeping profIndex = 1 << 32 // reader is sleeping and must be woken up + profWriteExtra profIndex = 1 << 33 // overflow or eof waiting +) + +func (x *profAtomic) load() profIndex { + return profIndex(atomic.Load64((*uint64)(x))) +} + +func (x *profAtomic) store(new profIndex) { + atomic.Store64((*uint64)(x), uint64(new)) +} + +func (x *profAtomic) cas(old, new profIndex) bool { + return atomic.Cas64((*uint64)(x), uint64(old), uint64(new)) +} + +func (x profIndex) dataCount() uint32 { + return uint32(x) +} + +func (x profIndex) tagCount() uint32 { + return uint32(x >> 34) +} + +// countSub subtracts two counts obtained from profIndex.dataCount or profIndex.tagCount, +// assuming that they are no more than 2^29 apart (guaranteed since they are never more than +// len(data) or len(tags) apart, respectively). +// tagCount wraps at 2^30, while dataCount wraps at 2^32. +// This function works for both. +func countSub(x, y uint32) int { + // x-y is 32-bit signed or 30-bit signed; sign-extend to 32 bits and convert to int. + return int(int32(x-y) << 2 >> 2) +} + +// addCountsAndClearFlags returns the packed form of "x + (data, tag) - all flags". +func (x profIndex) addCountsAndClearFlags(data, tag int) profIndex { + return profIndex((uint64(x)>>34+uint64(uint32(tag)<<2>>2))<<34 | uint64(uint32(x)+uint32(data))) +} + +// hasOverflow reports whether b has any overflow records pending. +func (b *profBuf) hasOverflow() bool { + return uint32(atomic.Load64(&b.overflow)) > 0 +} + +// takeOverflow consumes the pending overflow records, returning the overflow count +// and the time of the first overflow. +// When called by the reader, it is racing against incrementOverflow. +func (b *profBuf) takeOverflow() (count uint32, time uint64) { + overflow := atomic.Load64(&b.overflow) + time = atomic.Load64(&b.overflowTime) + for { + count = uint32(overflow) + if count == 0 { + time = 0 + break + } + // Increment generation, clear overflow count in low bits. + if atomic.Cas64(&b.overflow, overflow, ((overflow>>32)+1)<<32) { + break + } + overflow = atomic.Load64(&b.overflow) + time = atomic.Load64(&b.overflowTime) + } + return uint32(overflow), time +} + +// incrementOverflow records a single overflow at time now. +// It is racing against a possible takeOverflow in the reader. +func (b *profBuf) incrementOverflow(now int64) { + for { + overflow := atomic.Load64(&b.overflow) + + // Once we see b.overflow reach 0, it's stable: no one else is changing it underfoot. + // We need to set overflowTime if we're incrementing b.overflow from 0. + if uint32(overflow) == 0 { + // Store overflowTime first so it's always available when overflow != 0. + atomic.Store64(&b.overflowTime, uint64(now)) + atomic.Store64(&b.overflow, (((overflow>>32)+1)<<32)+1) + break + } + // Otherwise we're racing to increment against reader + // who wants to set b.overflow to 0. + // Out of paranoia, leave 2³²-1 a sticky overflow value, + // to avoid wrapping around. Extremely unlikely. + if int32(overflow) == -1 { + break + } + if atomic.Cas64(&b.overflow, overflow, overflow+1) { + break + } + } +} + +// newProfBuf returns a new profiling buffer with room for +// a header of hdrsize words and a buffer of at least bufwords words. +func newProfBuf(hdrsize, bufwords, tags int) *profBuf { + if min := 2 + hdrsize + 1; bufwords < min { + bufwords = min + } + + // Buffer sizes must be power of two, so that we don't have to + // worry about uint32 wraparound changing the effective position + // within the buffers. We store 30 bits of count; limiting to 28 + // gives us some room for intermediate calculations. + if bufwords >= 1<<28 || tags >= 1<<28 { + throw("newProfBuf: buffer too large") + } + var i int + for i = 1; i < bufwords; i <<= 1 { + } + bufwords = i + for i = 1; i < tags; i <<= 1 { + } + tags = i + + b := new(profBuf) + b.hdrsize = uintptr(hdrsize) + b.data = make([]uint64, bufwords) + b.tags = make([]unsafe.Pointer, tags) + b.overflowBuf = make([]uint64, 2+b.hdrsize+1) + return b +} + +// canWriteRecord reports whether the buffer has room +// for a single contiguous record with a stack of length nstk. +func (b *profBuf) canWriteRecord(nstk int) bool { + br := b.r.load() + bw := b.w.load() + + // room for tag? + if countSub(br.tagCount(), bw.tagCount())+len(b.tags) < 1 { + return false + } + + // room for data? + nd := countSub(br.dataCount(), bw.dataCount()) + len(b.data) + want := 2 + int(b.hdrsize) + nstk + i := int(bw.dataCount() % uint32(len(b.data))) + if i+want > len(b.data) { + // Can't fit in trailing fragment of slice. + // Skip over that and start over at beginning of slice. + nd -= len(b.data) - i + } + return nd >= want +} + +// canWriteTwoRecords reports whether the buffer has room +// for two records with stack lengths nstk1, nstk2, in that order. +// Each record must be contiguous on its own, but the two +// records need not be contiguous (one can be at the end of the buffer +// and the other can wrap around and start at the beginning of the buffer). +func (b *profBuf) canWriteTwoRecords(nstk1, nstk2 int) bool { + br := b.r.load() + bw := b.w.load() + + // room for tag? + if countSub(br.tagCount(), bw.tagCount())+len(b.tags) < 2 { + return false + } + + // room for data? + nd := countSub(br.dataCount(), bw.dataCount()) + len(b.data) + + // first record + want := 2 + int(b.hdrsize) + nstk1 + i := int(bw.dataCount() % uint32(len(b.data))) + if i+want > len(b.data) { + // Can't fit in trailing fragment of slice. + // Skip over that and start over at beginning of slice. + nd -= len(b.data) - i + i = 0 + } + i += want + nd -= want + + // second record + want = 2 + int(b.hdrsize) + nstk2 + if i+want > len(b.data) { + // Can't fit in trailing fragment of slice. + // Skip over that and start over at beginning of slice. + nd -= len(b.data) - i + i = 0 + } + return nd >= want +} + +// write writes an entry to the profiling buffer b. +// The entry begins with a fixed hdr, which must have +// length b.hdrsize, followed by a variable-sized stack +// and a single tag pointer *tagPtr (or nil if tagPtr is nil). +// No write barriers allowed because this might be called from a signal handler. +func (b *profBuf) write(tagPtr *unsafe.Pointer, now int64, hdr []uint64, stk []uintptr) { + if b == nil { + return + } + if len(hdr) > int(b.hdrsize) { + throw("misuse of profBuf.write") + } + + if hasOverflow := b.hasOverflow(); hasOverflow && b.canWriteTwoRecords(1, len(stk)) { + // Room for both an overflow record and the one being written. + // Write the overflow record if the reader hasn't gotten to it yet. + // Only racing against reader, not other writers. + count, time := b.takeOverflow() + if count > 0 { + var stk [1]uintptr + stk[0] = uintptr(count) + b.write(nil, int64(time), nil, stk[:]) + } + } else if hasOverflow || !b.canWriteRecord(len(stk)) { + // Pending overflow without room to write overflow and new records + // or no overflow but also no room for new record. + b.incrementOverflow(now) + b.wakeupExtra() + return + } + + // There's room: write the record. + br := b.r.load() + bw := b.w.load() + + // Profiling tag + // + // The tag is a pointer, but we can't run a write barrier here. + // We have interrupted the OS-level execution of gp, but the + // runtime still sees gp as executing. In effect, we are running + // in place of the real gp. Since gp is the only goroutine that + // can overwrite gp.labels, the value of gp.labels is stable during + // this signal handler: it will still be reachable from gp when + // we finish executing. If a GC is in progress right now, it must + // keep gp.labels alive, because gp.labels is reachable from gp. + // If gp were to overwrite gp.labels, the deletion barrier would + // still shade that pointer, which would preserve it for the + // in-progress GC, so all is well. Any future GC will see the + // value we copied when scanning b.tags (heap-allocated). + // We arrange that the store here is always overwriting a nil, + // so there is no need for a deletion barrier on b.tags[wt]. + wt := int(bw.tagCount() % uint32(len(b.tags))) + if tagPtr != nil { + *(*uintptr)(unsafe.Pointer(&b.tags[wt])) = uintptr(unsafe.Pointer(*tagPtr)) + } + + // Main record. + // It has to fit in a contiguous section of the slice, so if it doesn't fit at the end, + // leave a rewind marker (0) and start over at the beginning of the slice. + wd := int(bw.dataCount() % uint32(len(b.data))) + nd := countSub(br.dataCount(), bw.dataCount()) + len(b.data) + skip := 0 + if wd+2+int(b.hdrsize)+len(stk) > len(b.data) { + b.data[wd] = 0 + skip = len(b.data) - wd + nd -= skip + wd = 0 + } + data := b.data[wd:] + data[0] = uint64(2 + b.hdrsize + uintptr(len(stk))) // length + data[1] = uint64(now) // time stamp + // header, zero-padded + i := uintptr(copy(data[2:2+b.hdrsize], hdr)) + for ; i < b.hdrsize; i++ { + data[2+i] = 0 + } + for i, pc := range stk { + data[2+b.hdrsize+uintptr(i)] = uint64(pc) + } + + for { + // Commit write. + // Racing with reader setting flag bits in b.w, to avoid lost wakeups. + old := b.w.load() + new := old.addCountsAndClearFlags(skip+2+len(stk)+int(b.hdrsize), 1) + if !b.w.cas(old, new) { + continue + } + // If there was a reader, wake it up. + if old&profReaderSleeping != 0 { + notewakeup(&b.wait) + } + break + } +} + +// close signals that there will be no more writes on the buffer. +// Once all the data has been read from the buffer, reads will return eof=true. +func (b *profBuf) close() { + if atomic.Load(&b.eof) > 0 { + throw("runtime: profBuf already closed") + } + atomic.Store(&b.eof, 1) + b.wakeupExtra() +} + +// wakeupExtra must be called after setting one of the "extra" +// atomic fields b.overflow or b.eof. +// It records the change in b.w and wakes up the reader if needed. +func (b *profBuf) wakeupExtra() { + for { + old := b.w.load() + new := old | profWriteExtra + if !b.w.cas(old, new) { + continue + } + if old&profReaderSleeping != 0 { + notewakeup(&b.wait) + } + break + } +} + +// profBufReadMode specifies whether to block when no data is available to read. +type profBufReadMode int + +const ( + profBufBlocking profBufReadMode = iota + profBufNonBlocking +) + +var overflowTag [1]unsafe.Pointer // always nil + +func (b *profBuf) read(mode profBufReadMode) (data []uint64, tags []unsafe.Pointer, eof bool) { + if b == nil { + return nil, nil, true + } + + br := b.rNext + + // Commit previous read, returning that part of the ring to the writer. + // First clear tags that have now been read, both to avoid holding + // up the memory they point at for longer than necessary + // and so that b.write can assume it is always overwriting + // nil tag entries (see comment in b.write). + rPrev := b.r.load() + if rPrev != br { + ntag := countSub(br.tagCount(), rPrev.tagCount()) + ti := int(rPrev.tagCount() % uint32(len(b.tags))) + for i := 0; i < ntag; i++ { + b.tags[ti] = nil + if ti++; ti == len(b.tags) { + ti = 0 + } + } + b.r.store(br) + } + +Read: + bw := b.w.load() + numData := countSub(bw.dataCount(), br.dataCount()) + if numData == 0 { + if b.hasOverflow() { + // No data to read, but there is overflow to report. + // Racing with writer flushing b.overflow into a real record. + count, time := b.takeOverflow() + if count == 0 { + // Lost the race, go around again. + goto Read + } + // Won the race, report overflow. + dst := b.overflowBuf + dst[0] = uint64(2 + b.hdrsize + 1) + dst[1] = uint64(time) + for i := uintptr(0); i < b.hdrsize; i++ { + dst[2+i] = 0 + } + dst[2+b.hdrsize] = uint64(count) + return dst[:2+b.hdrsize+1], overflowTag[:1], false + } + if atomic.Load(&b.eof) > 0 { + // No data, no overflow, EOF set: done. + return nil, nil, true + } + if bw&profWriteExtra != 0 { + // Writer claims to have published extra information (overflow or eof). + // Attempt to clear notification and then check again. + // If we fail to clear the notification it means b.w changed, + // so we still need to check again. + b.w.cas(bw, bw&^profWriteExtra) + goto Read + } + + // Nothing to read right now. + // Return or sleep according to mode. + if mode == profBufNonBlocking { + return nil, nil, false + } + if !b.w.cas(bw, bw|profReaderSleeping) { + goto Read + } + // Committed to sleeping. + notetsleepg(&b.wait, -1) + noteclear(&b.wait) + goto Read + } + data = b.data[br.dataCount()%uint32(len(b.data)):] + if len(data) > numData { + data = data[:numData] + } else { + numData -= len(data) // available in case of wraparound + } + skip := 0 + if data[0] == 0 { + // Wraparound record. Go back to the beginning of the ring. + skip = len(data) + data = b.data + if len(data) > numData { + data = data[:numData] + } + } + + ntag := countSub(bw.tagCount(), br.tagCount()) + if ntag == 0 { + throw("runtime: malformed profBuf buffer - tag and data out of sync") + } + tags = b.tags[br.tagCount()%uint32(len(b.tags)):] + if len(tags) > ntag { + tags = tags[:ntag] + } + + // Count out whole data records until either data or tags is done. + // They are always in sync in the buffer, but due to an end-of-slice + // wraparound we might need to stop early and return the rest + // in the next call. + di := 0 + ti := 0 + for di < len(data) && data[di] != 0 && ti < len(tags) { + if uintptr(di)+uintptr(data[di]) > uintptr(len(data)) { + throw("runtime: malformed profBuf buffer - invalid size") + } + di += int(data[di]) + ti++ + } + + // Remember how much we returned, to commit read on next call. + b.rNext = br.addCountsAndClearFlags(skip+di, ti) + + if raceenabled { + // Match racereleasemerge in runtime_setProfLabel, + // so that the setting of the labels in runtime_setProfLabel + // is treated as happening before any use of the labels + // by our caller. The synchronization on labelSync itself is a fiction + // for the race detector. The actual synchronization is handled + // by the fact that the signal handler only reads from the current + // goroutine and uses atomics to write the updated queue indices, + // and then the read-out from the signal handler buffer uses + // atomics to read those queue indices. + raceacquire(unsafe.Pointer(&labelSync)) + } + + return data[:di], tags[:ti], false +} diff --git a/libgo/go/runtime/profbuf_test.go b/libgo/go/runtime/profbuf_test.go new file mode 100644 index 00000000000..d9c5264b732 --- /dev/null +++ b/libgo/go/runtime/profbuf_test.go @@ -0,0 +1,182 @@ +// Copyright 2017 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package runtime_test + +import ( + "reflect" + . "runtime" + "testing" + "time" + "unsafe" +) + +func TestProfBuf(t *testing.T) { + const hdrSize = 2 + + write := func(t *testing.T, b *ProfBuf, tag unsafe.Pointer, now int64, hdr []uint64, stk []uintptr) { + b.Write(&tag, now, hdr, stk) + } + read := func(t *testing.T, b *ProfBuf, data []uint64, tags []unsafe.Pointer) { + rdata, rtags, eof := b.Read(ProfBufNonBlocking) + if !reflect.DeepEqual(rdata, data) || !reflect.DeepEqual(rtags, tags) { + t.Fatalf("unexpected profile read:\nhave data %#x\nwant data %#x\nhave tags %#x\nwant tags %#x", rdata, data, rtags, tags) + } + if eof { + t.Fatalf("unexpected eof") + } + } + readBlock := func(t *testing.T, b *ProfBuf, data []uint64, tags []unsafe.Pointer) func() { + c := make(chan int) + go func() { + eof := data == nil + rdata, rtags, reof := b.Read(ProfBufBlocking) + if !reflect.DeepEqual(rdata, data) || !reflect.DeepEqual(rtags, tags) || reof != eof { + // Errorf, not Fatalf, because called in goroutine. + t.Errorf("unexpected profile read:\nhave data %#x\nwant data %#x\nhave tags %#x\nwant tags %#x\nhave eof=%v, want %v", rdata, data, rtags, tags, reof, eof) + } + c <- 1 + }() + time.Sleep(10 * time.Millisecond) // let goroutine run and block + return func() { + select { + case <-c: + case <-time.After(1 * time.Second): + t.Fatalf("timeout waiting for blocked read") + } + } + } + readEOF := func(t *testing.T, b *ProfBuf) { + rdata, rtags, eof := b.Read(ProfBufBlocking) + if rdata != nil || rtags != nil || !eof { + t.Errorf("unexpected profile read: %#x, %#x, eof=%v; want nil, nil, eof=true", rdata, rtags, eof) + } + rdata, rtags, eof = b.Read(ProfBufNonBlocking) + if rdata != nil || rtags != nil || !eof { + t.Errorf("unexpected profile read (non-blocking): %#x, %#x, eof=%v; want nil, nil, eof=true", rdata, rtags, eof) + } + } + + myTags := make([]byte, 100) + t.Logf("myTags is %p", &myTags[0]) + + t.Run("BasicWriteRead", func(t *testing.T) { + b := NewProfBuf(2, 11, 1) + write(t, b, unsafe.Pointer(&myTags[0]), 1, []uint64{2, 3}, []uintptr{4, 5, 6, 7, 8, 9}) + read(t, b, []uint64{10, 1, 2, 3, 4, 5, 6, 7, 8, 9}, []unsafe.Pointer{unsafe.Pointer(&myTags[0])}) + read(t, b, nil, nil) // release data returned by previous read + write(t, b, unsafe.Pointer(&myTags[2]), 99, []uint64{101, 102}, []uintptr{201, 202, 203, 204}) + read(t, b, []uint64{8, 99, 101, 102, 201, 202, 203, 204}, []unsafe.Pointer{unsafe.Pointer(&myTags[2])}) + }) + + t.Run("ReadMany", func(t *testing.T) { + b := NewProfBuf(2, 50, 50) + write(t, b, unsafe.Pointer(&myTags[0]), 1, []uint64{2, 3}, []uintptr{4, 5, 6, 7, 8, 9}) + write(t, b, unsafe.Pointer(&myTags[2]), 99, []uint64{101, 102}, []uintptr{201, 202, 203, 204}) + write(t, b, unsafe.Pointer(&myTags[1]), 500, []uint64{502, 504}, []uintptr{506}) + read(t, b, []uint64{10, 1, 2, 3, 4, 5, 6, 7, 8, 9, 8, 99, 101, 102, 201, 202, 203, 204, 5, 500, 502, 504, 506}, []unsafe.Pointer{unsafe.Pointer(&myTags[0]), unsafe.Pointer(&myTags[2]), unsafe.Pointer(&myTags[1])}) + }) + + t.Run("ReadManyShortData", func(t *testing.T) { + b := NewProfBuf(2, 50, 50) + write(t, b, unsafe.Pointer(&myTags[0]), 1, []uint64{2, 3}, []uintptr{4, 5, 6, 7, 8, 9}) + write(t, b, unsafe.Pointer(&myTags[2]), 99, []uint64{101, 102}, []uintptr{201, 202, 203, 204}) + read(t, b, []uint64{10, 1, 2, 3, 4, 5, 6, 7, 8, 9, 8, 99, 101, 102, 201, 202, 203, 204}, []unsafe.Pointer{unsafe.Pointer(&myTags[0]), unsafe.Pointer(&myTags[2])}) + }) + + t.Run("ReadManyShortTags", func(t *testing.T) { + b := NewProfBuf(2, 50, 50) + write(t, b, unsafe.Pointer(&myTags[0]), 1, []uint64{2, 3}, []uintptr{4, 5, 6, 7, 8, 9}) + write(t, b, unsafe.Pointer(&myTags[2]), 99, []uint64{101, 102}, []uintptr{201, 202, 203, 204}) + read(t, b, []uint64{10, 1, 2, 3, 4, 5, 6, 7, 8, 9, 8, 99, 101, 102, 201, 202, 203, 204}, []unsafe.Pointer{unsafe.Pointer(&myTags[0]), unsafe.Pointer(&myTags[2])}) + }) + + t.Run("ReadAfterOverflow1", func(t *testing.T) { + // overflow record synthesized by write + b := NewProfBuf(2, 16, 5) + write(t, b, unsafe.Pointer(&myTags[0]), 1, []uint64{2, 3}, []uintptr{4, 5, 6, 7, 8, 9}) // uses 10 + read(t, b, []uint64{10, 1, 2, 3, 4, 5, 6, 7, 8, 9}, []unsafe.Pointer{unsafe.Pointer(&myTags[0])}) // reads 10 but still in use until next read + write(t, b, unsafe.Pointer(&myTags[0]), 1, []uint64{2, 3}, []uintptr{4, 5}) // uses 6 + read(t, b, []uint64{6, 1, 2, 3, 4, 5}, []unsafe.Pointer{unsafe.Pointer(&myTags[0])}) // reads 6 but still in use until next read + // now 10 available + write(t, b, unsafe.Pointer(&myTags[2]), 99, []uint64{101, 102}, []uintptr{201, 202, 203, 204, 205, 206, 207, 208, 209}) // no room + for i := 0; i < 299; i++ { + write(t, b, unsafe.Pointer(&myTags[3]), int64(100+i), []uint64{101, 102}, []uintptr{201, 202, 203, 204}) // no room for overflow+this record + } + write(t, b, unsafe.Pointer(&myTags[1]), 500, []uint64{502, 504}, []uintptr{506}) // room for overflow+this record + read(t, b, []uint64{5, 99, 0, 0, 300, 5, 500, 502, 504, 506}, []unsafe.Pointer{nil, unsafe.Pointer(&myTags[1])}) + }) + + t.Run("ReadAfterOverflow2", func(t *testing.T) { + // overflow record synthesized by read + b := NewProfBuf(2, 16, 5) + write(t, b, unsafe.Pointer(&myTags[0]), 1, []uint64{2, 3}, []uintptr{4, 5, 6, 7, 8, 9}) + write(t, b, unsafe.Pointer(&myTags[2]), 99, []uint64{101, 102}, []uintptr{201, 202, 203, 204, 205, 206, 207, 208, 209, 210, 211, 212, 213}) + for i := 0; i < 299; i++ { + write(t, b, unsafe.Pointer(&myTags[3]), 100, []uint64{101, 102}, []uintptr{201, 202, 203, 204}) + } + read(t, b, []uint64{10, 1, 2, 3, 4, 5, 6, 7, 8, 9}, []unsafe.Pointer{unsafe.Pointer(&myTags[0])}) // reads 10 but still in use until next read + write(t, b, unsafe.Pointer(&myTags[1]), 500, []uint64{502, 504}, []uintptr{}) // still overflow + read(t, b, []uint64{5, 99, 0, 0, 301}, []unsafe.Pointer{nil}) // overflow synthesized by read + write(t, b, unsafe.Pointer(&myTags[1]), 500, []uint64{502, 505}, []uintptr{506}) // written + read(t, b, []uint64{5, 500, 502, 505, 506}, []unsafe.Pointer{unsafe.Pointer(&myTags[1])}) + }) + + t.Run("ReadAtEndAfterOverflow", func(t *testing.T) { + b := NewProfBuf(2, 12, 5) + write(t, b, unsafe.Pointer(&myTags[0]), 1, []uint64{2, 3}, []uintptr{4, 5, 6, 7, 8, 9}) + write(t, b, unsafe.Pointer(&myTags[2]), 99, []uint64{101, 102}, []uintptr{201, 202, 203, 204}) + for i := 0; i < 299; i++ { + write(t, b, unsafe.Pointer(&myTags[3]), 100, []uint64{101, 102}, []uintptr{201, 202, 203, 204}) + } + read(t, b, []uint64{10, 1, 2, 3, 4, 5, 6, 7, 8, 9}, []unsafe.Pointer{unsafe.Pointer(&myTags[0])}) + read(t, b, []uint64{5, 99, 0, 0, 300}, []unsafe.Pointer{nil}) + write(t, b, unsafe.Pointer(&myTags[1]), 500, []uint64{502, 504}, []uintptr{506}) + read(t, b, []uint64{5, 500, 502, 504, 506}, []unsafe.Pointer{unsafe.Pointer(&myTags[1])}) + }) + + t.Run("BlockingWriteRead", func(t *testing.T) { + b := NewProfBuf(2, 11, 1) + wait := readBlock(t, b, []uint64{10, 1, 2, 3, 4, 5, 6, 7, 8, 9}, []unsafe.Pointer{unsafe.Pointer(&myTags[0])}) + write(t, b, unsafe.Pointer(&myTags[0]), 1, []uint64{2, 3}, []uintptr{4, 5, 6, 7, 8, 9}) + wait() + wait = readBlock(t, b, []uint64{8, 99, 101, 102, 201, 202, 203, 204}, []unsafe.Pointer{unsafe.Pointer(&myTags[2])}) + time.Sleep(10 * time.Millisecond) + write(t, b, unsafe.Pointer(&myTags[2]), 99, []uint64{101, 102}, []uintptr{201, 202, 203, 204}) + wait() + wait = readBlock(t, b, nil, nil) + b.Close() + wait() + wait = readBlock(t, b, nil, nil) + wait() + readEOF(t, b) + }) + + t.Run("DataWraparound", func(t *testing.T) { + b := NewProfBuf(2, 16, 1024) + for i := 0; i < 10; i++ { + write(t, b, unsafe.Pointer(&myTags[0]), 1, []uint64{2, 3}, []uintptr{4, 5, 6, 7, 8, 9}) + read(t, b, []uint64{10, 1, 2, 3, 4, 5, 6, 7, 8, 9}, []unsafe.Pointer{unsafe.Pointer(&myTags[0])}) + read(t, b, nil, nil) // release data returned by previous read + } + }) + + t.Run("TagWraparound", func(t *testing.T) { + b := NewProfBuf(2, 1024, 2) + for i := 0; i < 10; i++ { + write(t, b, unsafe.Pointer(&myTags[0]), 1, []uint64{2, 3}, []uintptr{4, 5, 6, 7, 8, 9}) + read(t, b, []uint64{10, 1, 2, 3, 4, 5, 6, 7, 8, 9}, []unsafe.Pointer{unsafe.Pointer(&myTags[0])}) + read(t, b, nil, nil) // release data returned by previous read + } + }) + + t.Run("BothWraparound", func(t *testing.T) { + b := NewProfBuf(2, 16, 2) + for i := 0; i < 10; i++ { + write(t, b, unsafe.Pointer(&myTags[0]), 1, []uint64{2, 3}, []uintptr{4, 5, 6, 7, 8, 9}) + read(t, b, []uint64{10, 1, 2, 3, 4, 5, 6, 7, 8, 9}, []unsafe.Pointer{unsafe.Pointer(&myTags[0])}) + read(t, b, nil, nil) // release data returned by previous read + } + }) +} diff --git a/libgo/go/runtime/proflabel.go b/libgo/go/runtime/proflabel.go new file mode 100644 index 00000000000..ff73fe4856f --- /dev/null +++ b/libgo/go/runtime/proflabel.go @@ -0,0 +1,40 @@ +// Copyright 2017 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package runtime + +import "unsafe" + +var labelSync uintptr + +//go:linkname runtime_setProfLabel runtime_pprof.runtime_setProfLabel +func runtime_setProfLabel(labels unsafe.Pointer) { + // Introduce race edge for read-back via profile. + // This would more properly use &getg().labels as the sync address, + // but we do the read in a signal handler and can't call the race runtime then. + // + // This uses racereleasemerge rather than just racerelease so + // the acquire in profBuf.read synchronizes with *all* prior + // setProfLabel operations, not just the most recent one. This + // is important because profBuf.read will observe different + // labels set by different setProfLabel operations on + // different goroutines, so it needs to synchronize with all + // of them (this wouldn't be an issue if we could synchronize + // on &getg().labels since we would synchronize with each + // most-recent labels write separately.) + // + // racereleasemerge is like a full read-modify-write on + // labelSync, rather than just a store-release, so it carries + // a dependency on the previous racereleasemerge, which + // ultimately carries forward to the acquire in profBuf.read. + if raceenabled { + racereleasemerge(unsafe.Pointer(&labelSync)) + } + getg().labels = labels +} + +//go:linkname runtime_getProfLabel runtime_pprof.runtime_getProfLabel +func runtime_getProfLabel() unsafe.Pointer { + return getg().labels +} diff --git a/libgo/go/runtime/rand_test.go b/libgo/go/runtime/rand_test.go new file mode 100644 index 00000000000..f8831b05f9f --- /dev/null +++ b/libgo/go/runtime/rand_test.go @@ -0,0 +1,45 @@ +// Copyright 2017 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package runtime_test + +import ( + . "runtime" + "strconv" + "testing" +) + +func BenchmarkFastrand(b *testing.B) { + b.RunParallel(func(pb *testing.PB) { + for pb.Next() { + Fastrand() + } + }) +} + +func BenchmarkFastrandHashiter(b *testing.B) { + var m = make(map[int]int, 10) + for i := 0; i < 10; i++ { + m[i] = i + } + b.RunParallel(func(pb *testing.PB) { + for pb.Next() { + for _ = range m { + break + } + } + }) +} + +var sink32 uint32 + +func BenchmarkFastrandn(b *testing.B) { + for n := uint32(2); n <= 5; n++ { + b.Run(strconv.Itoa(int(n)), func(b *testing.B) { + for i := 0; i < b.N; i++ { + sink32 = Fastrandn(n) + } + }) + } +} diff --git a/libgo/go/runtime/relax_stub.go b/libgo/go/runtime/relax_stub.go new file mode 100644 index 00000000000..81ed1291b8b --- /dev/null +++ b/libgo/go/runtime/relax_stub.go @@ -0,0 +1,17 @@ +// Copyright 2017 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build !windows + +package runtime + +// osRelaxMinNS is the number of nanoseconds of idleness to tolerate +// without performing an osRelax. Since osRelax may reduce the +// precision of timers, this should be enough larger than the relaxed +// timer precision to keep the timer error acceptable. +const osRelaxMinNS = 0 + +// osRelax is called by the scheduler when transitioning to and from +// all Ps being idle. +func osRelax(relax bool) {} diff --git a/libgo/go/runtime/runtime1.go b/libgo/go/runtime/runtime1.go index dd3f7b277a6..627adf74765 100644 --- a/libgo/go/runtime/runtime1.go +++ b/libgo/go/runtime/runtime1.go @@ -47,15 +47,14 @@ var traceback_env uint32 //go:nosplit func gotraceback() (level int32, all, crash bool) { _g_ := getg() - all = _g_.m.throwing > 0 + t := atomic.Load(&traceback_cache) + crash = t&tracebackCrash != 0 + all = _g_.m.throwing > 0 || t&tracebackAll != 0 if _g_.m.traceback != 0 { level = int32(_g_.m.traceback) - return + } else { + level = int32(t >> tracebackShift) } - t := atomic.Load(&traceback_cache) - crash = t&tracebackCrash != 0 - all = all || t&tracebackAll != 0 - level = int32(t >> tracebackShift) return } @@ -330,35 +329,23 @@ type dbgVar struct { // except for "memprofilerate" since there is an // existing int var for that value, which may // already have an initial value. - -// For gccgo we use a named type so that the C code can see the -// definition. -type debugVars struct { - allocfreetrace int32 - cgocheck int32 - efence int32 - gccheckmark int32 - gcpacertrace int32 - gcshrinkstackoff int32 - gcstackbarrieroff int32 - gcstackbarrierall int32 - gcrescanstacks int32 - gcstoptheworld int32 - gctrace int32 - invalidptr int32 - sbrk int32 - scavenge int32 - scheddetail int32 - schedtrace int32 - wbshadow int32 +var debug struct { + allocfreetrace int32 + cgocheck int32 + efence int32 + gccheckmark int32 + gcpacertrace int32 + gcshrinkstackoff int32 + gcrescanstacks int32 + gcstoptheworld int32 + gctrace int32 + invalidptr int32 + sbrk int32 + scavenge int32 + scheddetail int32 + schedtrace int32 } -var debug debugVars - -// For gccgo's C code. -//extern runtime_setdebug -func runtime_setdebug(*debugVars) - var dbgvars = []dbgVar{ {"allocfreetrace", &debug.allocfreetrace}, {"cgocheck", &debug.cgocheck}, @@ -366,8 +353,6 @@ var dbgvars = []dbgVar{ {"gccheckmark", &debug.gccheckmark}, {"gcpacertrace", &debug.gcpacertrace}, {"gcshrinkstackoff", &debug.gcshrinkstackoff}, - {"gcstackbarrieroff", &debug.gcstackbarrieroff}, - {"gcstackbarrierall", &debug.gcstackbarrierall}, {"gcrescanstacks", &debug.gcrescanstacks}, {"gcstoptheworld", &debug.gcstoptheworld}, {"gctrace", &debug.gctrace}, @@ -376,7 +361,6 @@ var dbgvars = []dbgVar{ {"scavenge", &debug.scavenge}, {"scheddetail", &debug.scheddetail}, {"schedtrace", &debug.schedtrace}, - {"wbshadow", &debug.wbshadow}, } func parsedebugvars() { @@ -430,26 +414,12 @@ func parsedebugvars() { setTraceback(gogetenv("GOTRACEBACK")) traceback_env = traceback_cache - if debug.gcrescanstacks == 0 { - // Without rescanning, there's no need for stack - // barriers. - debug.gcstackbarrieroff = 1 - debug.gcstackbarrierall = 0 - } - - // if debug.gcstackbarrierall > 0 { - // firstStackBarrierOffset = 0 - // } - // For cgocheck > 1, we turn on the write barrier at all times // and check all pointer writes. if debug.cgocheck > 1 { writeBarrier.cgo = true writeBarrier.enabled = true } - - // Tell the C code what the value is. - runtime_setdebug(&debug) } //go:linkname setTraceback runtime_debug.SetTraceback diff --git a/libgo/go/runtime/runtime2.go b/libgo/go/runtime/runtime2.go index cdd3fcc7911..045e76ff4df 100644 --- a/libgo/go/runtime/runtime2.go +++ b/libgo/go/runtime/runtime2.go @@ -254,7 +254,7 @@ func setMNoWB(mp **m, new *m) { type sudog struct { // The following fields are protected by the hchan.lock of the // channel this sudog is blocking on. shrinkstack depends on - // this. + // this for sudogs involved in channel ops. g *g selectdone *uint32 // CAS to 1 to win select race (may point to stack) @@ -263,25 +263,19 @@ type sudog struct { elem unsafe.Pointer // data element (may point to stack) // The following fields are never accessed concurrently. - // waitlink is only accessed by g. + // For channels, waitlink is only accessed by g. + // For semaphores, all fields (including the ones above) + // are only accessed when holding a semaRoot lock. acquiretime int64 releasetime int64 ticket uint32 - waitlink *sudog // g.waiting list + parent *sudog // semaRoot binary tree + waitlink *sudog // g.waiting list or semaRoot + waittail *sudog // semaRoot c *hchan // channel } -type gcstats struct { - // the struct must consist of only uint64's, - // because it is casted to uint64[]. - nhandoff uint64 - nhandoffcnt uint64 - nprocyield uint64 - nosyield uint64 - nsleep uint64 -} - /* Not used by gccgo. @@ -318,12 +312,6 @@ type stack struct { lo uintptr hi uintptr } - -// stkbar records the state of a G's stack barrier. -type stkbar struct { - savedLRPtr uintptr // location overwritten by stack barrier PC - savedLRVal uintptr // value overwritten at savedLRPtr -} */ type g struct { @@ -341,12 +329,9 @@ type g struct { _panic *_panic // innermost panic - offset known to liblink _defer *_defer // innermost defer m *m // current m; offset known to arm liblink - // Not for gccgo: stackAlloc uintptr // stack allocation is [stack.lo,stack.lo+stackAlloc) // Not for gccgo: sched gobuf syscallsp uintptr // if status==Gsyscall, syscallsp = sched.sp to use during gc syscallpc uintptr // if status==Gsyscall, syscallpc = sched.pc to use during gc - // Not for gccgo: stkbar []stkbar // stack barriers, from low to high (see top of mstkbar.go) - // Not for gccgo: stkbarPos uintptr // index of lowest stack barrier not hit // Not for gccgo: stktopsp uintptr // expected sp at top of stack, to check in traceback param unsafe.Pointer // passed parameter on wakeup atomicstatus uint32 @@ -359,7 +344,7 @@ type g struct { paniconfault bool // panic (instead of crash) on unexpected fault address preemptscan bool // preempted g does scan for gc gcscandone bool // g has scanned stack; protected by _Gscan bit in status - gcscanvalid bool // false at start of gc cycle, true if G has not run since last scan; transition from true to false by calling queueRescan and false to true by calling dequeueRescan + gcscanvalid bool // false at start of gc cycle, true if G has not run since last scan; TODO: remove? throwsplit bool // must not split stack raceignore int8 // ignore race detection events sysblocktraced bool // StartTrace has emitted EvGoInSyscall about this goroutine @@ -376,17 +361,12 @@ type g struct { startpc uintptr // pc of goroutine function // Not for gccgo: racectx uintptr waiting *sudog // sudog structures this g is waiting on (that have a valid elem ptr); in lock order - // Not for gccgo: cgoCtxt []uintptr // cgo traceback context + // Not for gccgo: cgoCtxt []uintptr // cgo traceback context + labels unsafe.Pointer // profiler labels + timer *timer // cached timer for time.Sleep // Per-G GC state - // gcRescan is this G's index in work.rescan.list. If this is - // -1, this G is not on the rescan list. - // - // If gcphase != _GCoff and this G is visible to the garbage - // collector, writes to this are protected by work.rescan.lock. - gcRescan int32 - // gcAssistBytes is this G's GC assist credit in terms of // bytes allocated. If this is positive, then the G has credit // to allocate gcAssistBytes bytes without assisting. If this @@ -452,6 +432,7 @@ type m struct { inwb bool // m is executing a write barrier newSigstack bool // minit on C thread called sigaltstack printlock int8 + incgo bool // m is executing a cgo call fastrand uint32 ncgocall uint64 // number of cgo calls in total ncgo int32 // number of cgo calls currently in progress @@ -468,7 +449,6 @@ type m struct { // Not for gccgo: fflag uint32 // floating point compare flags locked uint32 // tracking for lockosthread nextwaitm uintptr // next m waiting for lock - gcstats gcstats needextram bool traceback uint8 waitunlockf unsafe.Pointer // todo go func(*g, unsafe.pointer) bool @@ -505,9 +485,10 @@ type p struct { id int32 status uint32 // one of pidle/prunning/... link puintptr - schedtick uint32 // incremented on every scheduler call - syscalltick uint32 // incremented on every system call - m muintptr // back-link to associated m (nil if idle) + schedtick uint32 // incremented on every scheduler call + syscalltick uint32 // incremented on every system call + sysmontick sysmontick // last tick observed by sysmon + m muintptr // back-link to associated m (nil if idle) mcache *mcache // Not for gccgo: racectx uintptr @@ -543,6 +524,14 @@ type p struct { tracebuf traceBufPtr + // traceSweep indicates the sweep events should be traced. + // This is used to defer the sweep start event until a span + // has actually been swept. + traceSweep bool + // traceSwept and traceReclaimed track the number of bytes + // swept and reclaimed by sweeping in the current sweep loop. + traceSwept, traceReclaimed uintptr + palloc persistentAlloc // per-P to avoid mutex // Per-P GC state @@ -563,7 +552,7 @@ type p struct { const ( // The max value of GOMAXPROCS. // There are no fundamental restrictions on the value. - _MaxGomaxprocs = 1 << 8 + _MaxGomaxprocs = 1 << 10 ) type schedt struct { @@ -639,7 +628,6 @@ const ( _SigThrow // if signal.Notify doesn't take it, exit loudly _SigPanic // if the signal is from the kernel, panic _SigDefault // if the signal isn't explicitly requested, don't monitor it - _SigHandling // our signal handler is registered _SigGoExit // cause all runtime procs to exit (only used on Plan 9). _SigSetStack // add SA_ONSTACK to libc handler _SigUnblock // unblocked in minit @@ -753,13 +741,10 @@ const ( const _TracebackMaxFrames = 100 var ( - // emptystring string - allglen uintptr allm *m allp [_MaxGomaxprocs + 1]*p gomaxprocs int32 - panicking uint32 ncpu int32 forcegc forcegcstate sched schedt @@ -767,6 +752,8 @@ var ( // Information about what cpu features are available. // Set on startup in asm_{x86,amd64}.s. + // Packages outside the runtime should not use these + // as they are not an external api. cpuid_ecx uint32 support_aes bool diff --git a/libgo/go/runtime/runtime_test.go b/libgo/go/runtime/runtime_test.go index 1f403a17056..b8f6ac2aed4 100644 --- a/libgo/go/runtime/runtime_test.go +++ b/libgo/go/runtime/runtime_test.go @@ -50,6 +50,23 @@ func BenchmarkIfaceCmpNil100(b *testing.B) { } } +var efaceCmp1 interface{} +var efaceCmp2 interface{} + +func BenchmarkEfaceCmpDiff(b *testing.B) { + x := 5 + efaceCmp1 = &x + y := 6 + efaceCmp2 = &y + for i := 0; i < b.N; i++ { + for j := 0; j < 100; j++ { + if efaceCmp1 == efaceCmp2 { + b.Fatal("bad comparison") + } + } + } +} + func BenchmarkDefer(b *testing.B) { for i := 0; i < b.N; i++ { defer1() @@ -62,7 +79,6 @@ func defer1() { panic("bad recover") } }(1, 2, 3) - return } func BenchmarkDefer10(b *testing.B) { diff --git a/libgo/go/runtime/rwmutex.go b/libgo/go/runtime/rwmutex.go new file mode 100644 index 00000000000..7eeb559adb5 --- /dev/null +++ b/libgo/go/runtime/rwmutex.go @@ -0,0 +1,125 @@ +// Copyright 2017 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package runtime + +import ( + "runtime/internal/atomic" +) + +// This is a copy of sync/rwmutex.go rewritten to work in the runtime. + +// An rwmutex is a reader/writer mutual exclusion lock. +// The lock can be held by an arbitrary number of readers or a single writer. +// This is a variant of sync.RWMutex, for the runtime package. +// Like mutex, rwmutex blocks the calling M. +// It does not interact with the goroutine scheduler. +type rwmutex struct { + rLock mutex // protects readers, readerPass, writer + readers muintptr // list of pending readers + readerPass uint32 // number of pending readers to skip readers list + + wLock mutex // serializes writers + writer muintptr // pending writer waiting for completing readers + + readerCount uint32 // number of pending readers + readerWait uint32 // number of departing readers +} + +const rwmutexMaxReaders = 1 << 30 + +// rlock locks rw for reading. +func (rw *rwmutex) rlock() { + // The reader must not be allowed to lose its P or else other + // things blocking on the lock may consume all of the Ps and + // deadlock (issue #20903). Alternatively, we could drop the P + // while sleeping. + acquirem() + if int32(atomic.Xadd(&rw.readerCount, 1)) < 0 { + // A writer is pending. Park on the reader queue. + systemstack(func() { + lock(&rw.rLock) + if rw.readerPass > 0 { + // Writer finished. + rw.readerPass -= 1 + unlock(&rw.rLock) + } else { + // Queue this reader to be woken by + // the writer. + m := getg().m + m.schedlink = rw.readers + rw.readers.set(m) + unlock(&rw.rLock) + notesleep(&m.park) + noteclear(&m.park) + } + }) + } +} + +// runlock undoes a single rlock call on rw. +func (rw *rwmutex) runlock() { + if r := int32(atomic.Xadd(&rw.readerCount, -1)); r < 0 { + if r+1 == 0 || r+1 == -rwmutexMaxReaders { + throw("runlock of unlocked rwmutex") + } + // A writer is pending. + if atomic.Xadd(&rw.readerWait, -1) == 0 { + // The last reader unblocks the writer. + lock(&rw.rLock) + w := rw.writer.ptr() + if w != nil { + notewakeup(&w.park) + } + unlock(&rw.rLock) + } + } + releasem(getg().m) +} + +// lock locks rw for writing. +func (rw *rwmutex) lock() { + // Resolve competition with other writers and stick to our P. + lock(&rw.wLock) + m := getg().m + // Announce that there is a pending writer. + r := int32(atomic.Xadd(&rw.readerCount, -rwmutexMaxReaders)) + rwmutexMaxReaders + // Wait for any active readers to complete. + lock(&rw.rLock) + if r != 0 && atomic.Xadd(&rw.readerWait, r) != 0 { + // Wait for reader to wake us up. + systemstack(func() { + rw.writer.set(m) + unlock(&rw.rLock) + notesleep(&m.park) + noteclear(&m.park) + }) + } else { + unlock(&rw.rLock) + } +} + +// unlock unlocks rw for writing. +func (rw *rwmutex) unlock() { + // Announce to readers that there is no active writer. + r := int32(atomic.Xadd(&rw.readerCount, rwmutexMaxReaders)) + if r >= rwmutexMaxReaders { + throw("unlock of unlocked rwmutex") + } + // Unblock blocked readers. + lock(&rw.rLock) + for rw.readers.ptr() != nil { + reader := rw.readers.ptr() + rw.readers = reader.schedlink + reader.schedlink.set(nil) + notewakeup(&reader.park) + r -= 1 + } + // If r > 0, there are pending readers that aren't on the + // queue. Tell them to skip waiting. + rw.readerPass += uint32(r) + unlock(&rw.rLock) + // Allow other writers to proceed. + unlock(&rw.wLock) +} diff --git a/libgo/go/runtime/rwmutex_test.go b/libgo/go/runtime/rwmutex_test.go new file mode 100644 index 00000000000..a69eca1511f --- /dev/null +++ b/libgo/go/runtime/rwmutex_test.go @@ -0,0 +1,178 @@ +// Copyright 2017 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// GOMAXPROCS=10 go test + +// This is a copy of sync/rwmutex_test.go rewritten to test the +// runtime rwmutex. + +package runtime_test + +import ( + "fmt" + . "runtime" + "sync/atomic" + "testing" +) + +func parallelReader(m *RWMutex, clocked chan bool, cunlock *uint32, cdone chan bool) { + m.RLock() + clocked <- true + for atomic.LoadUint32(cunlock) == 0 { + } + m.RUnlock() + cdone <- true +} + +func doTestParallelReaders(numReaders int) { + GOMAXPROCS(numReaders + 1) + var m RWMutex + clocked := make(chan bool, numReaders) + var cunlock uint32 + cdone := make(chan bool) + for i := 0; i < numReaders; i++ { + go parallelReader(&m, clocked, &cunlock, cdone) + } + // Wait for all parallel RLock()s to succeed. + for i := 0; i < numReaders; i++ { + <-clocked + } + atomic.StoreUint32(&cunlock, 1) + // Wait for the goroutines to finish. + for i := 0; i < numReaders; i++ { + <-cdone + } +} + +func TestParallelRWMutexReaders(t *testing.T) { + defer GOMAXPROCS(GOMAXPROCS(-1)) + doTestParallelReaders(1) + doTestParallelReaders(3) + doTestParallelReaders(4) +} + +func reader(rwm *RWMutex, num_iterations int, activity *int32, cdone chan bool) { + for i := 0; i < num_iterations; i++ { + rwm.RLock() + n := atomic.AddInt32(activity, 1) + if n < 1 || n >= 10000 { + panic(fmt.Sprintf("wlock(%d)\n", n)) + } + for i := 0; i < 100; i++ { + } + atomic.AddInt32(activity, -1) + rwm.RUnlock() + } + cdone <- true +} + +func writer(rwm *RWMutex, num_iterations int, activity *int32, cdone chan bool) { + for i := 0; i < num_iterations; i++ { + rwm.Lock() + n := atomic.AddInt32(activity, 10000) + if n != 10000 { + panic(fmt.Sprintf("wlock(%d)\n", n)) + } + for i := 0; i < 100; i++ { + } + atomic.AddInt32(activity, -10000) + rwm.Unlock() + } + cdone <- true +} + +func HammerRWMutex(gomaxprocs, numReaders, num_iterations int) { + GOMAXPROCS(gomaxprocs) + // Number of active readers + 10000 * number of active writers. + var activity int32 + var rwm RWMutex + cdone := make(chan bool) + go writer(&rwm, num_iterations, &activity, cdone) + var i int + for i = 0; i < numReaders/2; i++ { + go reader(&rwm, num_iterations, &activity, cdone) + } + go writer(&rwm, num_iterations, &activity, cdone) + for ; i < numReaders; i++ { + go reader(&rwm, num_iterations, &activity, cdone) + } + // Wait for the 2 writers and all readers to finish. + for i := 0; i < 2+numReaders; i++ { + <-cdone + } +} + +func TestRWMutex(t *testing.T) { + defer GOMAXPROCS(GOMAXPROCS(-1)) + n := 1000 + if testing.Short() { + n = 5 + } + HammerRWMutex(1, 1, n) + HammerRWMutex(1, 3, n) + HammerRWMutex(1, 10, n) + HammerRWMutex(4, 1, n) + HammerRWMutex(4, 3, n) + HammerRWMutex(4, 10, n) + HammerRWMutex(10, 1, n) + HammerRWMutex(10, 3, n) + HammerRWMutex(10, 10, n) + HammerRWMutex(10, 5, n) +} + +func BenchmarkRWMutexUncontended(b *testing.B) { + type PaddedRWMutex struct { + RWMutex + pad [32]uint32 + } + b.RunParallel(func(pb *testing.PB) { + var rwm PaddedRWMutex + for pb.Next() { + rwm.RLock() + rwm.RLock() + rwm.RUnlock() + rwm.RUnlock() + rwm.Lock() + rwm.Unlock() + } + }) +} + +func benchmarkRWMutex(b *testing.B, localWork, writeRatio int) { + var rwm RWMutex + b.RunParallel(func(pb *testing.PB) { + foo := 0 + for pb.Next() { + foo++ + if foo%writeRatio == 0 { + rwm.Lock() + rwm.Unlock() + } else { + rwm.RLock() + for i := 0; i != localWork; i += 1 { + foo *= 2 + foo /= 2 + } + rwm.RUnlock() + } + } + _ = foo + }) +} + +func BenchmarkRWMutexWrite100(b *testing.B) { + benchmarkRWMutex(b, 0, 100) +} + +func BenchmarkRWMutexWrite10(b *testing.B) { + benchmarkRWMutex(b, 0, 10) +} + +func BenchmarkRWMutexWorkWrite100(b *testing.B) { + benchmarkRWMutex(b, 100, 100) +} + +func BenchmarkRWMutexWorkWrite10(b *testing.B) { + benchmarkRWMutex(b, 100, 10) +} diff --git a/libgo/go/runtime/select.go b/libgo/go/runtime/select.go index f0cad1bf990..9f8ac49d972 100644 --- a/libgo/go/runtime/select.go +++ b/libgo/go/runtime/select.go @@ -98,7 +98,6 @@ func selectsend(sel *hselect, c *hchan, elem unsafe.Pointer) { return } cas := (*scase)(add(unsafe.Pointer(&sel.scase), uintptr(i)*unsafe.Sizeof(sel.scase[0]))) - cas.pc = pc cas.c = c cas.kind = caseSend @@ -246,7 +245,7 @@ func selectgo(sel *hselect) int { pollslice := slice{unsafe.Pointer(sel.pollorder), int(sel.ncase), int(sel.ncase)} pollorder := *(*[]uint16)(unsafe.Pointer(&pollslice)) for i := 1; i < int(sel.ncase); i++ { - j := int(fastrand()) % (i + 1) + j := fastrandn(uint32(i + 1)) pollorder[i] = pollorder[j] pollorder[j] = uint16(i) } @@ -408,7 +407,7 @@ loop: // wait for someone to wake us up gp.param = nil - gopark(selparkcommit, nil, "select", traceEvGoBlockSelect, 2) + gopark(selparkcommit, nil, "select", traceEvGoBlockSelect, 1) // While we were asleep, some goroutine came along and completed // one of the cases in the select and woke us up (called ready). @@ -602,7 +601,7 @@ bufsend: recv: // can receive from sleeping sender (sg) - recv(c, sg, cas.elem, func() { selunlock(scases, lockorder) }) + recv(c, sg, cas.elem, func() { selunlock(scases, lockorder) }, 2) if debugSelect { print("syncrecv: sel=", sel, " c=", c, "\n") } @@ -633,7 +632,7 @@ send: if msanenabled { msanread(cas.elem, c.elemtype.size) } - send(c, sg, cas.elem, func() { selunlock(scases, lockorder) }) + send(c, sg, cas.elem, func() { selunlock(scases, lockorder) }, 2) if debugSelect { print("syncsend: sel=", sel, " c=", c, "\n") } @@ -641,7 +640,7 @@ send: retc: if cas.releasetime > 0 { - blockevent(cas.releasetime-t0, 2) + blockevent(cas.releasetime-t0, 1) } return casi diff --git a/libgo/go/runtime/sema.go b/libgo/go/runtime/sema.go index 37318ff9d55..d04e6f592fc 100644 --- a/libgo/go/runtime/sema.go +++ b/libgo/go/runtime/sema.go @@ -27,10 +27,19 @@ import ( // Asynchronous semaphore for sync.Mutex. +// A semaRoot holds a balanced tree of sudog with distinct addresses (s.elem). +// Each of those sudog may in turn point (through s.waitlink) to a list +// of other sudogs waiting on the same address. +// The operations on the inner lists of sudogs with the same address +// are all O(1). The scanning of the top-level semaRoot list is O(log n), +// where n is the number of distinct addresses with goroutines blocked +// on them that hash to the given semaRoot. +// See golang.org/issue/17953 for a program that worked badly +// before we introduced the second level of list, and test/locklinear.go +// for a test that exercises this. type semaRoot struct { lock mutex - head *sudog - tail *sudog + treap *sudog // root of balanced tree of unique waiters. nwait uint32 // Number of waiters. Read w/o the lock. } @@ -44,26 +53,26 @@ var semtable [semTabSize]struct { //go:linkname sync_runtime_Semacquire sync.runtime_Semacquire func sync_runtime_Semacquire(addr *uint32) { - semacquire(addr, semaBlockProfile) + semacquire1(addr, false, semaBlockProfile) } -//go:linkname net_runtime_Semacquire net.runtime_Semacquire -func net_runtime_Semacquire(addr *uint32) { - semacquire(addr, semaBlockProfile) +//go:linkname poll_runtime_Semacquire internal_poll.runtime_Semacquire +func poll_runtime_Semacquire(addr *uint32) { + semacquire1(addr, false, semaBlockProfile) } //go:linkname sync_runtime_Semrelease sync.runtime_Semrelease -func sync_runtime_Semrelease(addr *uint32) { - semrelease(addr) +func sync_runtime_Semrelease(addr *uint32, handoff bool) { + semrelease1(addr, handoff) } //go:linkname sync_runtime_SemacquireMutex sync.runtime_SemacquireMutex -func sync_runtime_SemacquireMutex(addr *uint32) { - semacquire(addr, semaBlockProfile|semaMutexProfile) +func sync_runtime_SemacquireMutex(addr *uint32, lifo bool) { + semacquire1(addr, lifo, semaBlockProfile|semaMutexProfile) } -//go:linkname net_runtime_Semrelease net.runtime_Semrelease -func net_runtime_Semrelease(addr *uint32) { +//go:linkname poll_runtime_Semrelease internal_poll.runtime_Semrelease +func poll_runtime_Semrelease(addr *uint32) { semrelease(addr) } @@ -82,7 +91,11 @@ const ( ) // Called from runtime. -func semacquire(addr *uint32, profile semaProfileFlags) { +func semacquire(addr *uint32) { + semacquire1(addr, false, 0) +} + +func semacquire1(addr *uint32, lifo bool, profile semaProfileFlags) { gp := getg() if gp != gp.m.curg { throw("semacquire not on the G stack") @@ -104,6 +117,7 @@ func semacquire(addr *uint32, profile semaProfileFlags) { t0 := int64(0) s.releasetime = 0 s.acquiretime = 0 + s.ticket = 0 if profile&semaBlockProfile != 0 && blockprofilerate > 0 { t0 = cputicks() s.releasetime = -1 @@ -126,9 +140,9 @@ func semacquire(addr *uint32, profile semaProfileFlags) { } // Any semrelease after the cansemacquire knows we're waiting // (we set nwait above), so go to sleep. - root.queue(addr, s) + root.queue(addr, s, lifo) goparkunlock(&root.lock, "semacquire", traceEvGoBlockSync, 4) - if cansemacquire(addr) { + if s.ticket != 0 || cansemacquire(addr) { break } } @@ -139,6 +153,10 @@ func semacquire(addr *uint32, profile semaProfileFlags) { } func semrelease(addr *uint32) { + semrelease1(addr, false) +} + +func semrelease1(addr *uint32, handoff bool) { root := semroot(addr) atomic.Xadd(addr, 1) @@ -157,28 +175,22 @@ func semrelease(addr *uint32) { unlock(&root.lock) return } - s := root.head - for ; s != nil; s = s.next { - if s.elem == unsafe.Pointer(addr) { - atomic.Xadd(&root.nwait, -1) - root.dequeue(s) - break - } - } + s, t0 := root.dequeue(addr) if s != nil { - if s.acquiretime != 0 { - t0 := cputicks() - for x := root.head; x != nil; x = x.next { - if x.elem == unsafe.Pointer(addr) { - x.acquiretime = t0 - break - } - } - mutexevent(t0-s.acquiretime, 3) - } + atomic.Xadd(&root.nwait, -1) } unlock(&root.lock) if s != nil { // May be slow, so unlock first + acquiretime := s.acquiretime + if acquiretime != 0 { + mutexevent(t0-acquiretime, 3) + } + if s.ticket != 0 { + throw("corrupted semaphore ticket") + } + if handoff && cansemacquire(addr) { + s.ticket = 1 + } readyWithTime(s, 5) } } @@ -199,33 +211,230 @@ func cansemacquire(addr *uint32) bool { } } -func (root *semaRoot) queue(addr *uint32, s *sudog) { +// queue adds s to the blocked goroutines in semaRoot. +func (root *semaRoot) queue(addr *uint32, s *sudog, lifo bool) { s.g = getg() s.elem = unsafe.Pointer(addr) s.next = nil - s.prev = root.tail - if root.tail != nil { - root.tail.next = s - } else { - root.head = s + s.prev = nil + + var last *sudog + pt := &root.treap + for t := *pt; t != nil; t = *pt { + if t.elem == unsafe.Pointer(addr) { + // Already have addr in list. + if lifo { + // Substitute s in t's place in treap. + *pt = s + s.ticket = t.ticket + s.acquiretime = t.acquiretime + s.parent = t.parent + s.prev = t.prev + s.next = t.next + if s.prev != nil { + s.prev.parent = s + } + if s.next != nil { + s.next.parent = s + } + // Add t first in s's wait list. + s.waitlink = t + s.waittail = t.waittail + if s.waittail == nil { + s.waittail = t + } + t.parent = nil + t.prev = nil + t.next = nil + t.waittail = nil + } else { + // Add s to end of t's wait list. + if t.waittail == nil { + t.waitlink = s + } else { + t.waittail.waitlink = s + } + t.waittail = s + s.waitlink = nil + } + return + } + last = t + if uintptr(unsafe.Pointer(addr)) < uintptr(t.elem) { + pt = &t.prev + } else { + pt = &t.next + } + } + + // Add s as new leaf in tree of unique addrs. + // The balanced tree is a treap using ticket as the random heap priority. + // That is, it is a binary tree ordered according to the elem addresses, + // but then among the space of possible binary trees respecting those + // addresses, it is kept balanced on average by maintaining a heap ordering + // on the ticket: s.ticket <= both s.prev.ticket and s.next.ticket. + // https://en.wikipedia.org/wiki/Treap + // http://faculty.washington.edu/aragon/pubs/rst89.pdf + s.ticket = fastrand() + s.parent = last + *pt = s + + // Rotate up into tree according to ticket (priority). + for s.parent != nil && s.parent.ticket > s.ticket { + if s.parent.prev == s { + root.rotateRight(s.parent) + } else { + if s.parent.next != s { + panic("semaRoot queue") + } + root.rotateLeft(s.parent) + } } - root.tail = s } -func (root *semaRoot) dequeue(s *sudog) { - if s.next != nil { - s.next.prev = s.prev - } else { - root.tail = s.prev +// dequeue searches for and finds the first goroutine +// in semaRoot blocked on addr. +// If the sudog was being profiled, dequeue returns the time +// at which it was woken up as now. Otherwise now is 0. +func (root *semaRoot) dequeue(addr *uint32) (found *sudog, now int64) { + ps := &root.treap + s := *ps + for ; s != nil; s = *ps { + if s.elem == unsafe.Pointer(addr) { + goto Found + } + if uintptr(unsafe.Pointer(addr)) < uintptr(s.elem) { + ps = &s.prev + } else { + ps = &s.next + } } - if s.prev != nil { - s.prev.next = s.next + return nil, 0 + +Found: + now = int64(0) + if s.acquiretime != 0 { + now = cputicks() + } + if t := s.waitlink; t != nil { + // Substitute t, also waiting on addr, for s in root tree of unique addrs. + *ps = t + t.ticket = s.ticket + t.parent = s.parent + t.prev = s.prev + if t.prev != nil { + t.prev.parent = t + } + t.next = s.next + if t.next != nil { + t.next.parent = t + } + if t.waitlink != nil { + t.waittail = s.waittail + } else { + t.waittail = nil + } + t.acquiretime = now + s.waitlink = nil + s.waittail = nil } else { - root.head = s.next + // Rotate s down to be leaf of tree for removal, respecting priorities. + for s.next != nil || s.prev != nil { + if s.next == nil || s.prev != nil && s.prev.ticket < s.next.ticket { + root.rotateRight(s) + } else { + root.rotateLeft(s) + } + } + // Remove s, now a leaf. + if s.parent != nil { + if s.parent.prev == s { + s.parent.prev = nil + } else { + s.parent.next = nil + } + } else { + root.treap = nil + } } + s.parent = nil s.elem = nil s.next = nil s.prev = nil + s.ticket = 0 + return s, now +} + +// rotateLeft rotates the tree rooted at node x. +// turning (x a (y b c)) into (y (x a b) c). +func (root *semaRoot) rotateLeft(x *sudog) { + // p -> (x a (y b c)) + p := x.parent + a, y := x.prev, x.next + b, c := y.prev, y.next + + y.prev = x + x.parent = y + y.next = c + if c != nil { + c.parent = y + } + x.prev = a + if a != nil { + a.parent = x + } + x.next = b + if b != nil { + b.parent = x + } + + y.parent = p + if p == nil { + root.treap = y + } else if p.prev == x { + p.prev = y + } else { + if p.next != x { + throw("semaRoot rotateLeft") + } + p.next = y + } +} + +// rotateRight rotates the tree rooted at node y. +// turning (y (x a b) c) into (x a (y b c)). +func (root *semaRoot) rotateRight(y *sudog) { + // p -> (y (x a b) c) + p := y.parent + x, c := y.prev, y.next + a, b := x.prev, x.next + + x.prev = a + if a != nil { + a.parent = x + } + x.next = y + y.parent = x + y.prev = b + if b != nil { + b.parent = y + } + y.next = c + if c != nil { + c.parent = y + } + + x.parent = p + if p == nil { + root.treap = x + } else if p.prev == y { + p.prev = x + } else { + if p.next != y { + throw("semaRoot rotateRight") + } + p.next = x + } } // notifyList is a ticket-based notification list used to implement sync.Cond. @@ -352,10 +561,22 @@ func notifyListNotifyOne(l *notifyList) { return } - // Update the next notify ticket number, and try to find the G that - // needs to be notified. If it hasn't made it to the list yet we won't - // find it, but it won't park itself once it sees the new notify number. + // Update the next notify ticket number. atomic.Store(&l.notify, t+1) + + // Try to find the g that needs to be notified. + // If it hasn't made it to the list yet we won't find it, + // but it won't park itself once it sees the new notify number. + // + // This scan looks linear but essentially always stops quickly. + // Because g's queue separately from taking numbers, + // there may be minor reorderings in the list, but we + // expect the g we're looking for to be near the front. + // The g has others in front of it on the list only to the + // extent that it lost the race, so the iteration will not + // be too long. This applies even when the g is missing: + // it hasn't yet gotten to sleep and has lost the race to + // the (few) other g's that we find on the list. for p, s := (*sudog)(nil), l.head; s != nil; p, s = s, s.next { if s.ticket == t { n := s.next @@ -383,3 +604,8 @@ func notifyListCheck(sz uintptr) { throw("bad notifyList size") } } + +//go:linkname sync_nanotime sync.runtime_nanotime +func sync_nanotime() int64 { + return nanotime() +} diff --git a/libgo/go/runtime/signal_sighandler.go b/libgo/go/runtime/signal_sighandler.go index b71b21e1d5c..378c68e1d90 100644 --- a/libgo/go/runtime/signal_sighandler.go +++ b/libgo/go/runtime/signal_sighandler.go @@ -111,7 +111,7 @@ func sighandler(sig uint32, info *_siginfo_t, ctxt unsafe.Pointer, gp *g) { if docrash { crashing++ - if crashing < sched.mcount { + if crashing < sched.mcount-int32(extraMCount) { // There are other m's that need to dump their stacks. // Relay SIGQUIT to the next m by sending it to the current process. // All m's that have already received SIGQUIT have signal masks blocking diff --git a/libgo/go/runtime/signal_unix.go b/libgo/go/runtime/signal_unix.go index e2642ee25fc..3237e18765f 100644 --- a/libgo/go/runtime/signal_unix.go +++ b/libgo/go/runtime/signal_unix.go @@ -7,14 +7,13 @@ package runtime import ( + "runtime/internal/atomic" "runtime/internal/sys" "unsafe" ) // For gccgo's C code to call: //go:linkname initsig runtime.initsig -//go:linkname crash runtime.crash -//go:linkname resetcpuprofiler runtime.resetcpuprofiler //go:linkname sigtrampgo runtime.sigtrampgo //go:linkname os_sigpipe os.sigpipe @@ -37,11 +36,18 @@ const ( // Stores the signal handlers registered before Go installed its own. // These signal handlers will be invoked in cases where Go doesn't want to // handle a particular signal (e.g., signal occurred on a non-Go thread). -// See sigfwdgo() for more information on when the signals are forwarded. +// See sigfwdgo for more information on when the signals are forwarded. // -// Signal forwarding is currently available only on Darwin and Linux. +// This is read by the signal handler; accesses should use +// atomic.Loaduintptr and atomic.Storeuintptr. var fwdSig [_NSIG]uintptr +// handlingSig is indexed by signal number and is non-zero if we are +// currently handling the signal. Or, to put it another way, whether +// the signal handler is currently set to the Go signal handler or not. +// This is uint32 rather than bool so that we can use atomic instructions. +var handlingSig [_NSIG]uint32 + // channels for synchronizing signal mask updates with the signal mask // thread var ( @@ -87,6 +93,9 @@ func initsig(preinit bool) { if t.flags == 0 || t.flags&_SigDefault != 0 { continue } + + // We don't need to use atomic operations here because + // there shouldn't be any other goroutines running yet. fwdSig[i] = getsig(i) if !sigInstallGoHandler(i) { @@ -98,7 +107,7 @@ func initsig(preinit bool) { continue } - t.flags |= _SigHandling + handlingSig[i] = 1 setsig(i, getSigtramp()) } } @@ -111,7 +120,7 @@ func sigInstallGoHandler(sig uint32) bool { // Even these signals can be fetched using the os/signal package. switch sig { case _SIGHUP, _SIGINT: - if fwdSig[sig] == _SIG_IGN { + if atomic.Loaduintptr(&fwdSig[sig]) == _SIG_IGN { return false } } @@ -122,37 +131,52 @@ func sigInstallGoHandler(sig uint32) bool { } // When built using c-archive or c-shared, only install signal - // handlers for synchronous signals. - if (isarchive || islibrary) && t.flags&_SigPanic == 0 { + // handlers for synchronous signals and SIGPIPE. + if (isarchive || islibrary) && t.flags&_SigPanic == 0 && sig != _SIGPIPE { return false } return true } +// sigenable enables the Go signal handler to catch the signal sig. +// It is only called while holding the os/signal.handlers lock, +// via os/signal.enableSignal and signal_enable. func sigenable(sig uint32) { if sig >= uint32(len(sigtable)) { return } + // SIGPROF is handled specially for profiling. + if sig == _SIGPROF { + return + } + t := &sigtable[sig] if t.flags&_SigNotify != 0 { ensureSigM() enableSigChan <- sig <-maskUpdatedChan - if t.flags&_SigHandling == 0 { - t.flags |= _SigHandling - fwdSig[sig] = getsig(sig) + if atomic.Cas(&handlingSig[sig], 0, 1) { + atomic.Storeuintptr(&fwdSig[sig], getsig(sig)) setsig(sig, getSigtramp()) } } } +// sigdisable disables the Go signal handler for the signal sig. +// It is only called while holding the os/signal.handlers lock, +// via os/signal.disableSignal and signal_disable. func sigdisable(sig uint32) { if sig >= uint32(len(sigtable)) { return } + // SIGPROF is handled specially for profiling. + if sig == _SIGPROF { + return + } + t := &sigtable[sig] if t.flags&_SigNotify != 0 { ensureSigM() @@ -163,25 +187,71 @@ func sigdisable(sig uint32) { // signal, then to go back to the state before Notify // we should remove the one we installed. if !sigInstallGoHandler(sig) { - t.flags &^= _SigHandling - setsig(sig, fwdSig[sig]) + atomic.Store(&handlingSig[sig], 0) + setsig(sig, atomic.Loaduintptr(&fwdSig[sig])) } } } +// sigignore ignores the signal sig. +// It is only called while holding the os/signal.handlers lock, +// via os/signal.ignoreSignal and signal_ignore. func sigignore(sig uint32) { if sig >= uint32(len(sigtable)) { return } + // SIGPROF is handled specially for profiling. + if sig == _SIGPROF { + return + } + t := &sigtable[sig] if t.flags&_SigNotify != 0 { - t.flags &^= _SigHandling + atomic.Store(&handlingSig[sig], 0) setsig(sig, _SIG_IGN) } } -func resetcpuprofiler(hz int32) { +// clearSignalHandlers clears all signal handlers that are not ignored +// back to the default. This is called by the child after a fork, so that +// we can enable the signal mask for the exec without worrying about +// running a signal handler in the child. +//go:nosplit +//go:nowritebarrierrec +func clearSignalHandlers() { + for i := uint32(0); i < _NSIG; i++ { + if atomic.Load(&handlingSig[i]) != 0 { + setsig(i, _SIG_DFL) + } + } +} + +// setProcessCPUProfiler is called when the profiling timer changes. +// It is called with prof.lock held. hz is the new timer, and is 0 if +// profiling is being disabled. Enable or disable the signal as +// required for -buildmode=c-archive. +func setProcessCPUProfiler(hz int32) { + if hz != 0 { + // Enable the Go signal handler if not enabled. + if atomic.Cas(&handlingSig[_SIGPROF], 0, 1) { + atomic.Storeuintptr(&fwdSig[_SIGPROF], getsig(_SIGPROF)) + setsig(_SIGPROF, funcPC(sighandler)) + } + } else { + // If the Go signal handler should be disabled by default, + // disable it if it is enabled. + if !sigInstallGoHandler(_SIGPROF) { + if atomic.Cas(&handlingSig[_SIGPROF], 1, 0) { + setsig(_SIGPROF, atomic.Loaduintptr(&fwdSig[_SIGPROF])) + } + } + } +} + +// setThreadCPUProfiler makes any thread-specific changes required to +// implement profiling at a rate of hz. +func setThreadCPUProfiler(hz int32) { var it _itimerval if hz == 0 { setitimer(_ITIMER_PROF, &it, nil) @@ -315,7 +385,7 @@ func raisebadsignal(sig uint32, c *sigctxt) { if sig >= _NSIG { handler = _SIG_DFL } else { - handler = fwdSig[sig] + handler = atomic.Loaduintptr(&fwdSig[sig]) } // Reset the signal handler and raise the signal. @@ -431,6 +501,16 @@ func sigNotOnStack(sig uint32) { throw("non-Go code set up signal handler without SA_ONSTACK flag") } +// signalDuringFork is called if we receive a signal while doing a fork. +// We do not want signals at that time, as a signal sent to the process +// group may be delivered to the child process, causing confusion. +// This should never be called, because we block signals across the fork; +// this function is just a safety check. See issue 18600 for background. +func signalDuringFork(sig uint32) { + println("signal", sig, "received during fork") + throw("signal received during fork") +} + // This runs on a foreign stack, without an m or a g. No stack split. //go:nosplit //go:norace @@ -455,7 +535,7 @@ func sigfwdgo(sig uint32, info *_siginfo_t, ctx unsafe.Pointer) bool { if sig >= uint32(len(sigtable)) { return false } - fwdFn := fwdSig[sig] + fwdFn := atomic.Loaduintptr(&fwdSig[sig]) if !signalsOK { // The only way we can get here is if we are in a @@ -470,35 +550,44 @@ func sigfwdgo(sig uint32, info *_siginfo_t, ctx unsafe.Pointer) bool { return true } - flags := sigtable[sig].flags - // If there is no handler to forward to, no need to forward. if fwdFn == _SIG_DFL { return false } // If we aren't handling the signal, forward it. - if flags&_SigHandling == 0 { + // Really if we aren't handling the signal, we shouldn't get here, + // but on Darwin setsigstack can lead us here because it sets + // the sa_tramp field. The sa_tramp field is not returned by + // sigaction, so the fix for that is non-obvious. + if atomic.Load(&handlingSig[sig]) == 0 { sigfwd(fwdFn, sig, info, ctx) return true } - // Only forward synchronous signals. + flags := sigtable[sig].flags + c := sigctxt{info, ctx} - if c.sigcode() == _SI_USER || flags&_SigPanic == 0 { + // Only forward synchronous signals and SIGPIPE. + // Unfortunately, user generated SIGPIPEs will also be forwarded, because si_code + // is set to _SI_USER even for a SIGPIPE raised from a write to a closed socket + // or pipe. + if (c.sigcode() == _SI_USER || flags&_SigPanic == 0) && sig != _SIGPIPE { return false } // Determine if the signal occurred inside Go code. We test that: // (1) we were in a goroutine (i.e., m.curg != nil), and - // (2) we weren't in CGO (i.e., m.curg.syscallsp == 0). + // (2) we weren't in CGO. g := getg() - if g != nil && g.m != nil && g.m.curg != nil && g.m.curg.syscallsp == 0 { + if g != nil && g.m != nil && g.m.curg != nil && !g.m.incgo { return false } + // Signal not handled by Go, forward it. if fwdFn != _SIG_IGN { sigfwd(fwdFn, sig, info, ctx) } + return true } diff --git a/libgo/go/runtime/sigqueue.go b/libgo/go/runtime/sigqueue.go index a6d498f9b03..cd036ce364c 100644 --- a/libgo/go/runtime/sigqueue.go +++ b/libgo/go/runtime/sigqueue.go @@ -33,6 +33,17 @@ import ( _ "unsafe" // for go:linkname ) +// sig handles communication between the signal handler and os/signal. +// Other than the inuse and recv fields, the fields are accessed atomically. +// +// The wanted and ignored fields are only written by one goroutine at +// a time; access is controlled by the handlers Mutex in os/signal. +// The fields are only read by that one goroutine and by the signal handler. +// We access them atomically to minimize the race between setting them +// in the goroutine calling os/signal and the signal handler, +// which may be running in a different thread. That race is unavoidable, +// as there is no connection between handling a signal and receiving one, +// but atomic instructions should minimize it. var sig struct { note note mask [(_NSIG + 31) / 32]uint32 @@ -53,7 +64,11 @@ const ( // Reports whether the signal was sent. If not, the caller typically crashes the program. func sigsend(s uint32) bool { bit := uint32(1) << uint(s&31) - if !sig.inuse || s >= uint32(32*len(sig.wanted)) || sig.wanted[s/32]&bit == 0 { + if !sig.inuse || s >= uint32(32*len(sig.wanted)) { + return false + } + + if w := atomic.Load(&sig.wanted[s/32]); w&bit == 0 { return false } @@ -131,6 +146,23 @@ func signal_recv() uint32 { } } +// signalWaitUntilIdle waits until the signal delivery mechanism is idle. +// This is used to ensure that we do not drop a signal notification due +// to a race between disabling a signal and receiving a signal. +// This assumes that signal delivery has already been disabled for +// the signal(s) in question, and here we are just waiting to make sure +// that all the signals have been delivered to the user channels +// by the os/signal package. +//go:linkname signalWaitUntilIdle os_signal.signalWaitUntilIdle +func signalWaitUntilIdle() { + // Although WaitUntilIdle seems like the right name for this + // function, the state we are looking for is sigReceiving, not + // sigIdle. The sigIdle state is really more like sigProcessing. + for atomic.Load(&sig.state) != sigReceiving { + Gosched() + } +} + // Must only be called from a single goroutine at a time. //go:linkname signal_enable os_signal.signal_enable func signal_enable(s uint32) { @@ -146,8 +178,15 @@ func signal_enable(s uint32) { if s >= uint32(len(sig.wanted)*32) { return } - sig.wanted[s/32] |= 1 << (s & 31) - sig.ignored[s/32] &^= 1 << (s & 31) + + w := sig.wanted[s/32] + w |= 1 << (s & 31) + atomic.Store(&sig.wanted[s/32], w) + + i := sig.ignored[s/32] + i &^= 1 << (s & 31) + atomic.Store(&sig.ignored[s/32], i) + sigenable(s) } @@ -157,8 +196,11 @@ func signal_disable(s uint32) { if s >= uint32(len(sig.wanted)*32) { return } - sig.wanted[s/32] &^= 1 << (s & 31) sigdisable(s) + + w := sig.wanted[s/32] + w &^= 1 << (s & 31) + atomic.Store(&sig.wanted[s/32], w) } // Must only be called from a single goroutine at a time. @@ -167,12 +209,19 @@ func signal_ignore(s uint32) { if s >= uint32(len(sig.wanted)*32) { return } - sig.wanted[s/32] &^= 1 << (s & 31) - sig.ignored[s/32] |= 1 << (s & 31) sigignore(s) + + w := sig.wanted[s/32] + w &^= 1 << (s & 31) + atomic.Store(&sig.wanted[s/32], w) + + i := sig.ignored[s/32] + i |= 1 << (s & 31) + atomic.Store(&sig.ignored[s/32], i) } // Checked by signal handlers. func signal_ignored(s uint32) bool { - return sig.ignored[s/32]&(1<<(s&31)) != 0 + i := atomic.Load(&sig.ignored[s/32]) + return i&(1<<(s&31)) != 0 } diff --git a/libgo/go/runtime/sizeclasses.go b/libgo/go/runtime/sizeclasses.go index e616e95148c..5366564afda 100644 --- a/libgo/go/runtime/sizeclasses.go +++ b/libgo/go/runtime/sizeclasses.go @@ -1,4 +1,4 @@ -// AUTO-GENERATED by mksizeclasses.go; DO NOT EDIT +// Code generated by mksizeclasses.go; DO NOT EDIT. //go:generate go run mksizeclasses.go package runtime diff --git a/libgo/go/runtime/string.go b/libgo/go/runtime/string.go index 9cc28737434..7436ddfdf4b 100644 --- a/libgo/go/runtime/string.go +++ b/libgo/go/runtime/string.go @@ -88,7 +88,7 @@ func concatstring5(buf *tmpBuf, a [5]string) string { // Buf is a fixed-size buffer for the result, // it is not nil if the result does not escape. -func slicebytetostring(buf *tmpBuf, b []byte) string { +func slicebytetostring(buf *tmpBuf, b []byte) (str string) { l := len(b) if l == 0 { // Turns out to be a relatively common case. @@ -96,18 +96,26 @@ func slicebytetostring(buf *tmpBuf, b []byte) string { // you find the indices and convert the subslice to string. return "" } - if raceenabled && l > 0 { + if raceenabled { racereadrangepc(unsafe.Pointer(&b[0]), uintptr(l), getcallerpc(unsafe.Pointer(&buf)), funcPC(slicebytetostring)) } - if msanenabled && l > 0 { + if msanenabled { msanread(unsafe.Pointer(&b[0]), uintptr(l)) } - s, c := rawstringtmp(buf, l) - copy(c, b) - return s + + var p unsafe.Pointer + if buf != nil && len(b) <= len(buf) { + p = unsafe.Pointer(buf) + } else { + p = mallocgc(uintptr(len(b)), nil, false) + } + stringStructOf(&str).str = p + stringStructOf(&str).len = len(b) + memmove(p, (*(*slice)(unsafe.Pointer(&b))).array, uintptr(len(b))) + return } func rawstringtmp(buf *tmpBuf, l int) (s string, b []byte) { diff --git a/libgo/go/runtime/string_test.go b/libgo/go/runtime/string_test.go index ee306999e2e..555a7fc7ef9 100644 --- a/libgo/go/runtime/string_test.go +++ b/libgo/go/runtime/string_test.go @@ -6,6 +6,7 @@ package runtime_test import ( "runtime" + "strconv" "strings" "testing" ) @@ -89,6 +90,20 @@ func BenchmarkConcatStringAndBytes(b *testing.B) { } } +var escapeString string + +func BenchmarkSliceByteToString(b *testing.B) { + buf := []byte{'!'} + for n := 0; n < 8; n++ { + b.Run(strconv.Itoa(len(buf)), func(b *testing.B) { + for i := 0; i < b.N; i++ { + escapeString = string(buf) + } + }) + buf = append(buf, buf...) + } +} + var stringdata = []struct{ name, data string }{ {"ASCII", "01234567890"}, {"Japanese", "日本語日本語日本語"}, diff --git a/libgo/go/runtime/stubs.go b/libgo/go/runtime/stubs.go index 30d87c4121b..84fa1c79689 100644 --- a/libgo/go/runtime/stubs.go +++ b/libgo/go/runtime/stubs.go @@ -109,8 +109,22 @@ func memcmp(a, b unsafe.Pointer, size uintptr) int32 // exported value for testing var hashLoad = loadFactor -// in asm_*.s -func fastrand() uint32 +//go:nosplit +func fastrand() uint32 { + mp := getg().m + fr := mp.fastrand + mx := uint32(int32(fr)>>31) & 0xa8888eef + fr = fr<<1 ^ mx + mp.fastrand = fr + return fr +} + +//go:nosplit +func fastrandn(n uint32) uint32 { + // This is similar to fastrand() % n, but faster. + // See http://lemire.me/blog/2016/06/27/a-fast-alternative-to-the-modulo-reduction/ + return uint32(uint64(fastrand()) * uint64(n) >> 32) +} //go:linkname sync_fastrand sync.fastrand func sync_fastrand() uint32 { return fastrand() } @@ -156,7 +170,7 @@ type neverCallThisFunction struct{} // This function must never be called directly. Call goexit1 instead. // gentraceback assumes that goexit terminates the stack. A direct // call on the stack will cause gentraceback to stop walking the stack -// prematurely and if there are leftover stack barriers it may panic. +// prematurely and if there is leftover state it may panic. func goexit(neverCallThisFunction) // publicationBarrier performs a store/store barrier (a "publication" @@ -176,9 +190,6 @@ func goexit(neverCallThisFunction) // data dependency ordering. func publicationBarrier() -//go:noescape -func setcallerpc(argp unsafe.Pointer, pc uintptr) - // getcallerpc returns the program counter (PC) of its caller's caller. // getcallersp returns the stack pointer (SP) of its caller's caller. // For both, the argp must be a pointer to the caller's first function argument. @@ -213,12 +224,14 @@ func getcallerpc(argp unsafe.Pointer) uintptr //go:noescape func getcallersp(argp unsafe.Pointer) uintptr +func asmcgocall(fn, arg unsafe.Pointer) int32 { + throw("asmcgocall") + return 0 +} + // argp used in Defer structs when there is no argp. const _NoArgs = ^uintptr(0) -//go:linkname time_now time.now -func time_now() (sec int64, nsec int32) - //extern __builtin_prefetch func prefetch(addr unsafe.Pointer, rw int32, locality int32) @@ -238,13 +251,6 @@ func prefetchnta(addr uintptr) { prefetch(unsafe.Pointer(addr), 0, 0) } -// For gccgo, expose this for C callers. -//go:linkname unixnanotime runtime.unixnanotime -func unixnanotime() int64 { - sec, nsec := time_now() - return sec*1e9 + int64(nsec) -} - // round n up to a multiple of a. a must be a power of 2. func round(n, a uintptr) uintptr { return (n + a - 1) &^ (a - 1) @@ -315,18 +321,6 @@ func entersyscallblock(int32) // Here for gccgo until we port mgc.go. func GC() -// For gccgo to call from C code. -//go:linkname acquireWorldsema runtime.acquireWorldsema -func acquireWorldsema() { - semacquire(&worldsema, 0) -} - -// For gccgo to call from C code. -//go:linkname releaseWorldsema runtime.releaseWorldsema -func releaseWorldsema() { - semrelease(&worldsema) -} - // For gccgo to call from C code, so that the C code and the Go code // can share the memstats variable for now. //go:linkname getMstats runtime.getMstats @@ -436,10 +430,6 @@ func setpagesize(s uintptr) { } } -// Temporary for gccgo until we port more of proc.go. -func sigprofNonGoPC(pc uintptr) { -} - // Temporary for gccgo until we port mgc.go. //go:linkname runtime_m0 runtime.runtime_m0 func runtime_m0() *m { @@ -458,3 +448,11 @@ type bitvector struct { n int32 // # of bits bytedata *uint8 } + +// bool2int returns 0 if x is false or 1 if x is true. +func bool2int(x bool) int { + if x { + return 1 + } + return 0 +} diff --git a/libgo/go/runtime/stubs_linux.go b/libgo/go/runtime/stubs_linux.go new file mode 100644 index 00000000000..d10f657197f --- /dev/null +++ b/libgo/go/runtime/stubs_linux.go @@ -0,0 +1,9 @@ +// Copyright 2017 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build linux + +package runtime + +func sbrk0() uintptr diff --git a/libgo/go/runtime/stubs_nonlinux.go b/libgo/go/runtime/stubs_nonlinux.go new file mode 100644 index 00000000000..e1ea05cf0b1 --- /dev/null +++ b/libgo/go/runtime/stubs_nonlinux.go @@ -0,0 +1,12 @@ +// Copyright 2017 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build !linux + +package runtime + +// sbrk0 returns the current process brk, or 0 if not implemented. +func sbrk0() uintptr { + return 0 +} diff --git a/libgo/go/runtime/symtab.go b/libgo/go/runtime/symtab.go index bad03471c46..3d15fc8039e 100644 --- a/libgo/go/runtime/symtab.go +++ b/libgo/go/runtime/symtab.go @@ -7,6 +7,7 @@ package runtime // Frames may be used to get function/file/line information for a // slice of PC values returned by Callers. type Frames struct { + // callers is a slice of PCs that have not yet been expanded. callers []uintptr // The last PC we saw. @@ -18,23 +19,34 @@ type Frames struct { // Frame is the information returned by Frames for each call frame. type Frame struct { - // Program counter for this frame; multiple frames may have - // the same PC value. + // PC is the program counter for the location in this frame. + // For a frame that calls another frame, this will be the + // program counter of a call instruction. Because of inlining, + // multiple frames may have the same PC value, but different + // symbolic information. PC uintptr - // Func for this frame; may be nil for non-Go code or fully - // inlined functions. + // Func is the Func value of this call frame. This may be nil + // for non-Go code or fully inlined functions. Func *Func - // Function name, file name, and line number for this call frame. - // May be the empty string or zero if not known. + // Function is the package path-qualified function name of + // this call frame. If non-empty, this string uniquely + // identifies a single function in the program. + // This may be the empty string if not known. // If Func is not nil then Function == Func.Name(). Function string - File string - Line int - // Entry point for the function; may be zero if not known. - // If Func is not nil then Entry == Func.Entry(). + // File and Line are the file name and line number of the + // location in this frame. For non-leaf frames, this will be + // the location of a call. These may be the empty string and + // zero, respectively, if not known. + File string + Line int + + // Entry point program counter for the function; may be zero + // if not known. If Func is not nil then Entry == + // Func.Entry(). Entry uintptr } @@ -94,7 +106,8 @@ func (ci *Frames) Next() (frame Frame, more bool) { // NOTE: Func does not expose the actual unexported fields, because we return *Func // values to users, and we want to keep them from being able to overwrite the data // with (say) *f = Func{}. -// All code operating on a *Func must call raw to get the *_func instead. +// All code operating on a *Func must call raw() to get the *_func +// or funcInfo() to get the funcInfo instead. // A Func represents a Go function in the running binary. type Func struct { @@ -104,6 +117,9 @@ type Func struct { // FuncForPC returns a *Func describing the function that contains the // given program counter address, or else nil. +// +// If pc represents multiple functions because of inlining, it returns +// the *Func describing the outermost function. func FuncForPC(pc uintptr) *Func { name, _, _ := funcfileline(pc, -1) if name == "" { diff --git a/libgo/go/runtime/symtab_test.go b/libgo/go/runtime/symtab_test.go index 8c8281f78a1..807b50de990 100644 --- a/libgo/go/runtime/symtab_test.go +++ b/libgo/go/runtime/symtab_test.go @@ -31,10 +31,14 @@ func TestCaller(t *testing.T) { } } +// These are marked noinline so that we can use FuncForPC +// in testCallerBar. +//go:noinline func testCallerFoo(t *testing.T) { testCallerBar(t) } +//go:noinline func testCallerBar(t *testing.T) { for i := 0; i < 2; i++ { pc, file, line, ok := runtime.Caller(i) @@ -94,7 +98,7 @@ var mapLit = map[int]int{ // 28 } // 33 var intLit = lineNumber() + // 34 lineNumber() + // 35 - lineNumber() // 36 + lineNumber() // 36 func trythis() { // 37 recordLines(lineNumber(), // 38 lineNumber(), // 39 @@ -156,3 +160,14 @@ func TestLineNumber(t *testing.T) { } } } + +func TestNilName(t *testing.T) { + defer func() { + if ex := recover(); ex != nil { + t.Fatalf("expected no nil panic, got=%v", ex) + } + }() + if got := (*runtime.Func)(nil).Name(); got != "" { + t.Errorf("Name() = %q, want %q", got, "") + } +} diff --git a/libgo/go/runtime/testdata/testprog/numcpu_freebsd.go b/libgo/go/runtime/testdata/testprog/numcpu_freebsd.go new file mode 100644 index 00000000000..035c53470be --- /dev/null +++ b/libgo/go/runtime/testdata/testprog/numcpu_freebsd.go @@ -0,0 +1,126 @@ +// Copyright 2017 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package main + +import ( + "bytes" + "fmt" + "os" + "os/exec" + "runtime" + "strconv" + "strings" + "syscall" +) + +func init() { + register("FreeBSDNumCPU", FreeBSDNumCPU) + register("FreeBSDNumCPUHelper", FreeBSDNumCPUHelper) +} + +func FreeBSDNumCPUHelper() { + fmt.Printf("%d\n", runtime.NumCPU()) +} + +func FreeBSDNumCPU() { + _, err := exec.LookPath("cpuset") + if err != nil { + // Can not test without cpuset command. + fmt.Println("OK") + return + } + _, err = exec.LookPath("sysctl") + if err != nil { + // Can not test without sysctl command. + fmt.Println("OK") + return + } + cmd := exec.Command("sysctl", "-n", "kern.smp.active") + output, err := cmd.CombinedOutput() + if err != nil { + fmt.Printf("fail to launch '%s', error: %s, output: %s\n", strings.Join(cmd.Args, " "), err, output) + return + } + if bytes.Equal(output, []byte("1\n")) == false { + // SMP mode deactivated in kernel. + fmt.Println("OK") + return + } + + list, err := getList() + if err != nil { + fmt.Printf("%s\n", err) + return + } + err = checkNCPU(list) + if err != nil { + fmt.Printf("%s\n", err) + return + } + if len(list) >= 2 { + err = checkNCPU(list[:len(list)-1]) + if err != nil { + fmt.Printf("%s\n", err) + return + } + } + fmt.Println("OK") + return +} + +func getList() ([]string, error) { + pid := syscall.Getpid() + + // Launch cpuset to print a list of available CPUs: pid mask: 0, 1, 2, 3. + cmd := exec.Command("cpuset", "-g", "-p", strconv.Itoa(pid)) + cmdline := strings.Join(cmd.Args, " ") + output, err := cmd.CombinedOutput() + if err != nil { + return nil, fmt.Errorf("fail to execute '%s': %s", cmdline, err) + } + pos := bytes.IndexRune(output, ':') + if pos == -1 { + return nil, fmt.Errorf("invalid output from '%s', ':' not found: %s", cmdline, output) + } + + var list []string + for _, val := range bytes.Split(output[pos+1:], []byte(",")) { + index := string(bytes.TrimSpace(val)) + if len(index) == 0 { + continue + } + list = append(list, index) + } + if len(list) == 0 { + return nil, fmt.Errorf("empty CPU list from '%s': %s", cmdline, output) + } + return list, nil +} + +func checkNCPU(list []string) error { + listString := strings.Join(list, ",") + if len(listString) == 0 { + return fmt.Errorf("could not check against an empty CPU list") + } + + // Launch FreeBSDNumCPUHelper() with specified CPUs list. + cmd := exec.Command("cpuset", "-l", listString, os.Args[0], "FreeBSDNumCPUHelper") + cmdline := strings.Join(cmd.Args, " ") + output, err := cmd.CombinedOutput() + if err != nil { + return fmt.Errorf("fail to launch child '%s', error: %s, output: %s", cmdline, err, output) + } + + // NumCPU from FreeBSDNumCPUHelper come with '\n'. + output = bytes.TrimSpace(output) + n, err := strconv.Atoi(string(output)) + if err != nil { + return fmt.Errorf("fail to parse output from child '%s', error: %s, output: %s", cmdline, err, output) + } + if n != len(list) { + return fmt.Errorf("runtime.NumCPU() expected to %d, got %d when run with CPU list %s", len(list), n, listString) + } + return nil +} diff --git a/libgo/go/runtime/testdata/testprog/panicrace.go b/libgo/go/runtime/testdata/testprog/panicrace.go new file mode 100644 index 00000000000..f0589940b56 --- /dev/null +++ b/libgo/go/runtime/testdata/testprog/panicrace.go @@ -0,0 +1,27 @@ +// Copyright 2017 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package main + +import ( + "runtime" + "sync" +) + +func init() { + register("PanicRace", PanicRace) +} + +func PanicRace() { + var wg sync.WaitGroup + wg.Add(1) + go func() { + defer func() { + wg.Done() + runtime.Gosched() + }() + panic("crash") + }() + wg.Wait() +} diff --git a/libgo/go/runtime/testdata/testprogcgo/cgo.go b/libgo/go/runtime/testdata/testprogcgo/cgo.go index 870d4efdead..209524a24db 100644 --- a/libgo/go/runtime/testdata/testprogcgo/cgo.go +++ b/libgo/go/runtime/testdata/testprogcgo/cgo.go @@ -45,10 +45,13 @@ func CgoSignalDeadlock() { }() var s *string *s = "" + fmt.Printf("continued after expected panic\n") }() } }() time.Sleep(time.Millisecond) + start := time.Now() + var times []time.Duration for i := 0; i < 64; i++ { go func() { runtime.LockOSThread() @@ -62,8 +65,9 @@ func CgoSignalDeadlock() { ping <- false select { case <-ping: + times = append(times, time.Since(start)) case <-time.After(time.Second): - fmt.Printf("HANG\n") + fmt.Printf("HANG 1 %v\n", times) return } } @@ -71,7 +75,7 @@ func CgoSignalDeadlock() { select { case <-ping: case <-time.After(time.Second): - fmt.Printf("HANG\n") + fmt.Printf("HANG 2 %v\n", times) return } fmt.Printf("OK\n") diff --git a/libgo/go/runtime/testdata/testprogcgo/numgoroutine.go b/libgo/go/runtime/testdata/testprogcgo/numgoroutine.go new file mode 100644 index 00000000000..12fda49a131 --- /dev/null +++ b/libgo/go/runtime/testdata/testprogcgo/numgoroutine.go @@ -0,0 +1,99 @@ +// Copyright 2017 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build !plan9,!windows + +package main + +/* +#include +#include + +extern void CallbackNumGoroutine(); + +static void* thread2(void* arg __attribute__ ((unused))) { + CallbackNumGoroutine(); + return NULL; +} + +static void CheckNumGoroutine() { + pthread_t tid; + pthread_create(&tid, NULL, thread2, NULL); + pthread_join(tid, NULL); +} +*/ +import "C" + +import ( + "fmt" + "runtime" + "strings" +) + +var baseGoroutines int + +func init() { + register("NumGoroutine", NumGoroutine) +} + +func NumGoroutine() { + // Test that there are just the expected number of goroutines + // running. Specifically, test that the spare M's goroutine + // doesn't show up. + // + // On non-Windows platforms there's a signal handling thread + // started by os/signal.init in addition to the main + // goroutine. + if runtime.GOOS != "windows" { + baseGoroutines = 1 + } + if _, ok := checkNumGoroutine("first", 1+baseGoroutines); !ok { + return + } + + // Test that the goroutine for a callback from C appears. + if C.CheckNumGoroutine(); !callbackok { + return + } + + // Make sure we're back to the initial goroutines. + if _, ok := checkNumGoroutine("third", 1+baseGoroutines); !ok { + return + } + + fmt.Println("OK") +} + +func checkNumGoroutine(label string, want int) (string, bool) { + n := runtime.NumGoroutine() + if n != want { + fmt.Printf("%s NumGoroutine: want %d; got %d\n", label, want, n) + return "", false + } + + sbuf := make([]byte, 32<<10) + sbuf = sbuf[:runtime.Stack(sbuf, true)] + n = strings.Count(string(sbuf), "goroutine ") + if n != want { + fmt.Printf("%s Stack: want %d; got %d:\n%s\n", label, want, n, string(sbuf)) + return "", false + } + return string(sbuf), true +} + +var callbackok bool + +//export CallbackNumGoroutine +func CallbackNumGoroutine() { + stk, ok := checkNumGoroutine("second", 2+baseGoroutines) + if !ok { + return + } + if !strings.Contains(stk, "CallbackNumGoroutine") { + fmt.Printf("missing CallbackNumGoroutine from stack:\n%s\n", stk) + return + } + + callbackok = true +} diff --git a/libgo/go/runtime/testdata/testprognet/signalexec.go b/libgo/go/runtime/testdata/testprognet/signalexec.go new file mode 100644 index 00000000000..4a988ef6c1d --- /dev/null +++ b/libgo/go/runtime/testdata/testprognet/signalexec.go @@ -0,0 +1,70 @@ +// Copyright 2017 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build darwin dragonfly freebsd linux netbsd openbsd + +// This is in testprognet instead of testprog because testprog +// must not import anything (like net, but also like os/signal) +// that kicks off background goroutines during init. + +package main + +import ( + "fmt" + "os" + "os/exec" + "os/signal" + "sync" + "syscall" + "time" +) + +func init() { + register("SignalDuringExec", SignalDuringExec) + register("Nop", Nop) +} + +func SignalDuringExec() { + pgrp := syscall.Getpgrp() + + const tries = 10 + + var wg sync.WaitGroup + c := make(chan os.Signal, tries) + signal.Notify(c, syscall.SIGWINCH) + wg.Add(1) + go func() { + defer wg.Done() + for range c { + } + }() + + for i := 0; i < tries; i++ { + time.Sleep(time.Microsecond) + wg.Add(2) + go func() { + defer wg.Done() + cmd := exec.Command(os.Args[0], "Nop") + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + if err := cmd.Run(); err != nil { + fmt.Printf("Start failed: %v", err) + } + }() + go func() { + defer wg.Done() + syscall.Kill(-pgrp, syscall.SIGWINCH) + }() + } + + signal.Stop(c) + close(c) + wg.Wait() + + fmt.Println("OK") +} + +func Nop() { + // This is just for SignalDuringExec. +} diff --git a/libgo/go/runtime/time.go b/libgo/go/runtime/time.go index cc167a8bcfe..f204830a6f7 100644 --- a/libgo/go/runtime/time.go +++ b/libgo/go/runtime/time.go @@ -31,6 +31,7 @@ var timers struct { created bool sleeping bool rescheduling bool + sleepUntil int64 waitnote note t []*timer } @@ -50,7 +51,12 @@ func timeSleep(ns int64) { return } - t := new(timer) + t := getg().timer + if t == nil { + t = new(timer) + getg().timer = t + } + *t = timer{} t.when = nanotime() + ns t.f = goroutineReady t.arg = getg() @@ -207,6 +213,7 @@ func timerproc() { } // At least one timer pending. Sleep until then. timers.sleeping = true + timers.sleepUntil = now + delta noteclear(&timers.waitnote) unlock(&timers.lock) notetsleepg(&timers.waitnote, delta) @@ -295,8 +302,8 @@ func siftdownTimer(i int) { // Entry points for net, time to call nanotime. -//go:linkname net_runtimeNano net.runtimeNano -func net_runtimeNano() int64 { +//go:linkname poll_runtimeNano internal_poll.runtimeNano +func poll_runtimeNano() int64 { return nanotime() } @@ -304,3 +311,5 @@ func net_runtimeNano() int64 { func time_runtimeNano() int64 { return nanotime() } + +var startNano int64 = nanotime() diff --git a/libgo/go/runtime/timeasm.go b/libgo/go/runtime/timeasm.go new file mode 100644 index 00000000000..d5f5ea33a30 --- /dev/null +++ b/libgo/go/runtime/timeasm.go @@ -0,0 +1,17 @@ +// Copyright 2017 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Declarations for operating systems implementing time.now directly in assembly. +// Those systems are also expected to have nanotime subtract startNano, +// so that time.now and nanotime return the same monotonic clock readings. + +// +build ignore +// +build darwin,amd64 darwin,386 windows + +package runtime + +import _ "unsafe" + +//go:linkname time_now time.now +func time_now() (sec int64, nsec int32, mono int64) diff --git a/libgo/go/runtime/timestub.go b/libgo/go/runtime/timestub.go new file mode 100644 index 00000000000..033734ed92f --- /dev/null +++ b/libgo/go/runtime/timestub.go @@ -0,0 +1,21 @@ +// Copyright 2017 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Declarations for operating systems implementing time.now +// indirectly, in terms of walltime and nanotime assembly. + +// -build !darwin !amd64,!386 +// -build !windows + +package runtime + +import _ "unsafe" // for go:linkname + +func walltime() (sec int64, nsec int32) + +//go:linkname time_now time.now +func time_now() (sec int64, nsec int32, mono int64) { + sec, nsec = walltime() + return sec, nsec, nanotime() - startNano +} diff --git a/libgo/go/runtime/trace.go b/libgo/go/runtime/trace.go index 61cfa8e751c..af9313be37a 100644 --- a/libgo/go/runtime/trace.go +++ b/libgo/go/runtime/trace.go @@ -19,50 +19,52 @@ import ( // Event types in the trace, args are given in square brackets. const ( - traceEvNone = 0 // unused - traceEvBatch = 1 // start of per-P batch of events [pid, timestamp] - traceEvFrequency = 2 // contains tracer timer frequency [frequency (ticks per second)] - traceEvStack = 3 // stack [stack id, number of PCs, array of {PC, func string ID, file string ID, line}] - traceEvGomaxprocs = 4 // current value of GOMAXPROCS [timestamp, GOMAXPROCS, stack id] - traceEvProcStart = 5 // start of P [timestamp, thread id] - traceEvProcStop = 6 // stop of P [timestamp] - traceEvGCStart = 7 // GC start [timestamp, seq, stack id] - traceEvGCDone = 8 // GC done [timestamp] - traceEvGCScanStart = 9 // GC mark termination start [timestamp] - traceEvGCScanDone = 10 // GC mark termination done [timestamp] - traceEvGCSweepStart = 11 // GC sweep start [timestamp, stack id] - traceEvGCSweepDone = 12 // GC sweep done [timestamp] - traceEvGoCreate = 13 // goroutine creation [timestamp, new goroutine id, new stack id, stack id] - traceEvGoStart = 14 // goroutine starts running [timestamp, goroutine id, seq] - traceEvGoEnd = 15 // goroutine ends [timestamp] - traceEvGoStop = 16 // goroutine stops (like in select{}) [timestamp, stack] - traceEvGoSched = 17 // goroutine calls Gosched [timestamp, stack] - traceEvGoPreempt = 18 // goroutine is preempted [timestamp, stack] - traceEvGoSleep = 19 // goroutine calls Sleep [timestamp, stack] - traceEvGoBlock = 20 // goroutine blocks [timestamp, stack] - traceEvGoUnblock = 21 // goroutine is unblocked [timestamp, goroutine id, seq, stack] - traceEvGoBlockSend = 22 // goroutine blocks on chan send [timestamp, stack] - traceEvGoBlockRecv = 23 // goroutine blocks on chan recv [timestamp, stack] - traceEvGoBlockSelect = 24 // goroutine blocks on select [timestamp, stack] - traceEvGoBlockSync = 25 // goroutine blocks on Mutex/RWMutex [timestamp, stack] - traceEvGoBlockCond = 26 // goroutine blocks on Cond [timestamp, stack] - traceEvGoBlockNet = 27 // goroutine blocks on network [timestamp, stack] - traceEvGoSysCall = 28 // syscall enter [timestamp, stack] - traceEvGoSysExit = 29 // syscall exit [timestamp, goroutine id, seq, real timestamp] - traceEvGoSysBlock = 30 // syscall blocks [timestamp] - traceEvGoWaiting = 31 // denotes that goroutine is blocked when tracing starts [timestamp, goroutine id] - traceEvGoInSyscall = 32 // denotes that goroutine is in syscall when tracing starts [timestamp, goroutine id] - traceEvHeapAlloc = 33 // memstats.heap_live change [timestamp, heap_alloc] - traceEvNextGC = 34 // memstats.next_gc change [timestamp, next_gc] - traceEvTimerGoroutine = 35 // denotes timer goroutine [timer goroutine id] - traceEvFutileWakeup = 36 // denotes that the previous wakeup of this goroutine was futile [timestamp] - traceEvString = 37 // string dictionary entry [ID, length, string] - traceEvGoStartLocal = 38 // goroutine starts running on the same P as the last event [timestamp, goroutine id] - traceEvGoUnblockLocal = 39 // goroutine is unblocked on the same P as the last event [timestamp, goroutine id, stack] - traceEvGoSysExitLocal = 40 // syscall exit on the same P as the last event [timestamp, goroutine id, real timestamp] - traceEvGoStartLabel = 41 // goroutine starts running with label [timestamp, goroutine id, seq, label string id] - traceEvGoBlockGC = 42 // goroutine blocks on GC assist [timestamp, stack] - traceEvCount = 43 + traceEvNone = 0 // unused + traceEvBatch = 1 // start of per-P batch of events [pid, timestamp] + traceEvFrequency = 2 // contains tracer timer frequency [frequency (ticks per second)] + traceEvStack = 3 // stack [stack id, number of PCs, array of {PC, func string ID, file string ID, line}] + traceEvGomaxprocs = 4 // current value of GOMAXPROCS [timestamp, GOMAXPROCS, stack id] + traceEvProcStart = 5 // start of P [timestamp, thread id] + traceEvProcStop = 6 // stop of P [timestamp] + traceEvGCStart = 7 // GC start [timestamp, seq, stack id] + traceEvGCDone = 8 // GC done [timestamp] + traceEvGCScanStart = 9 // GC mark termination start [timestamp] + traceEvGCScanDone = 10 // GC mark termination done [timestamp] + traceEvGCSweepStart = 11 // GC sweep start [timestamp, stack id] + traceEvGCSweepDone = 12 // GC sweep done [timestamp, swept, reclaimed] + traceEvGoCreate = 13 // goroutine creation [timestamp, new goroutine id, new stack id, stack id] + traceEvGoStart = 14 // goroutine starts running [timestamp, goroutine id, seq] + traceEvGoEnd = 15 // goroutine ends [timestamp] + traceEvGoStop = 16 // goroutine stops (like in select{}) [timestamp, stack] + traceEvGoSched = 17 // goroutine calls Gosched [timestamp, stack] + traceEvGoPreempt = 18 // goroutine is preempted [timestamp, stack] + traceEvGoSleep = 19 // goroutine calls Sleep [timestamp, stack] + traceEvGoBlock = 20 // goroutine blocks [timestamp, stack] + traceEvGoUnblock = 21 // goroutine is unblocked [timestamp, goroutine id, seq, stack] + traceEvGoBlockSend = 22 // goroutine blocks on chan send [timestamp, stack] + traceEvGoBlockRecv = 23 // goroutine blocks on chan recv [timestamp, stack] + traceEvGoBlockSelect = 24 // goroutine blocks on select [timestamp, stack] + traceEvGoBlockSync = 25 // goroutine blocks on Mutex/RWMutex [timestamp, stack] + traceEvGoBlockCond = 26 // goroutine blocks on Cond [timestamp, stack] + traceEvGoBlockNet = 27 // goroutine blocks on network [timestamp, stack] + traceEvGoSysCall = 28 // syscall enter [timestamp, stack] + traceEvGoSysExit = 29 // syscall exit [timestamp, goroutine id, seq, real timestamp] + traceEvGoSysBlock = 30 // syscall blocks [timestamp] + traceEvGoWaiting = 31 // denotes that goroutine is blocked when tracing starts [timestamp, goroutine id] + traceEvGoInSyscall = 32 // denotes that goroutine is in syscall when tracing starts [timestamp, goroutine id] + traceEvHeapAlloc = 33 // memstats.heap_live change [timestamp, heap_alloc] + traceEvNextGC = 34 // memstats.next_gc change [timestamp, next_gc] + traceEvTimerGoroutine = 35 // denotes timer goroutine [timer goroutine id] + traceEvFutileWakeup = 36 // denotes that the previous wakeup of this goroutine was futile [timestamp] + traceEvString = 37 // string dictionary entry [ID, length, string] + traceEvGoStartLocal = 38 // goroutine starts running on the same P as the last event [timestamp, goroutine id] + traceEvGoUnblockLocal = 39 // goroutine is unblocked on the same P as the last event [timestamp, goroutine id, stack] + traceEvGoSysExitLocal = 40 // syscall exit on the same P as the last event [timestamp, goroutine id, real timestamp] + traceEvGoStartLabel = 41 // goroutine starts running with label [timestamp, goroutine id, seq, label string id] + traceEvGoBlockGC = 42 // goroutine blocks on GC assist [timestamp, stack] + traceEvGCMarkAssistStart = 43 // GC mark assist start [timestamp, stack] + traceEvGCMarkAssistDone = 44 // GC mark assist done [timestamp] + traceEvCount = 45 ) const ( @@ -311,7 +313,7 @@ func StopTrace() { // The world is started but we've set trace.shutdown, so new tracing can't start. // Wait for the trace reader to flush pending buffers and stop. - semacquire(&trace.shutdownSema, 0) + semacquire(&trace.shutdownSema) if raceenabled { raceacquire(unsafe.Pointer(&trace.shutdownSema)) } @@ -380,7 +382,7 @@ func ReadTrace() []byte { trace.headerWritten = true trace.lockOwner = nil unlock(&trace.lock) - return []byte("go 1.8 trace\x00\x00\x00\x00") + return []byte("go 1.9 trace\x00\x00\x00\x00") } // Wait for new data. if trace.fullHead == 0 && !trace.shutdown { @@ -776,11 +778,12 @@ func (tab *traceStackTable) dump() { for ; stk != nil; stk = stk.link.ptr() { tmpbuf := tmp[:0] tmpbuf = traceAppend(tmpbuf, uint64(stk.id)) - tmpbuf = traceAppend(tmpbuf, uint64(stk.n)) - for _, pc := range stk.stack() { + frames := stk.stack() + tmpbuf = traceAppend(tmpbuf, uint64(len(frames))) + for _, f := range frames { var frame traceFrame - frame, buf = traceFrameForPC(buf, pc) - tmpbuf = traceAppend(tmpbuf, uint64(pc.pc)) + frame, buf = traceFrameForPC(buf, f) + tmpbuf = traceAppend(tmpbuf, uint64(f.pc)) tmpbuf = traceAppend(tmpbuf, uint64(frame.funcID)) tmpbuf = traceAppend(tmpbuf, uint64(frame.fileID)) tmpbuf = traceAppend(tmpbuf, uint64(frame.line)) @@ -810,16 +813,17 @@ type traceFrame struct { line uint64 } -func traceFrameForPC(buf *traceBuf, loc location) (traceFrame, *traceBuf) { +func traceFrameForPC(buf *traceBuf, f location) (traceFrame, *traceBuf) { var frame traceFrame - fn := loc.function + + fn := f.function const maxLen = 1 << 10 if len(fn) > maxLen { fn = fn[len(fn)-maxLen:] } frame.funcID, buf = traceString(buf, fn) - file, line := loc.filename, loc.lineno - frame.line = uint64(line) + frame.line = uint64(f.lineno) + file := f.filename if len(file) > maxLen { file = file[len(file)-maxLen:] } @@ -921,12 +925,52 @@ func traceGCScanDone() { traceEvent(traceEvGCScanDone, -1) } +// traceGCSweepStart prepares to trace a sweep loop. This does not +// emit any events until traceGCSweepSpan is called. +// +// traceGCSweepStart must be paired with traceGCSweepDone and there +// must be no preemption points between these two calls. func traceGCSweepStart() { - traceEvent(traceEvGCSweepStart, 1) + // Delay the actual GCSweepStart event until the first span + // sweep. If we don't sweep anything, don't emit any events. + _p_ := getg().m.p.ptr() + if _p_.traceSweep { + throw("double traceGCSweepStart") + } + _p_.traceSweep, _p_.traceSwept, _p_.traceReclaimed = true, 0, 0 +} + +// traceGCSweepSpan traces the sweep of a single page. +// +// This may be called outside a traceGCSweepStart/traceGCSweepDone +// pair; however, it will not emit any trace events in this case. +func traceGCSweepSpan(bytesSwept uintptr) { + _p_ := getg().m.p.ptr() + if _p_.traceSweep { + if _p_.traceSwept == 0 { + traceEvent(traceEvGCSweepStart, 1) + } + _p_.traceSwept += bytesSwept + } } func traceGCSweepDone() { - traceEvent(traceEvGCSweepDone, -1) + _p_ := getg().m.p.ptr() + if !_p_.traceSweep { + throw("missing traceGCSweepStart") + } + if _p_.traceSwept != 0 { + traceEvent(traceEvGCSweepDone, -1, uint64(_p_.traceSwept), uint64(_p_.traceReclaimed)) + } + _p_.traceSweep = false +} + +func traceGCMarkAssistStart() { + traceEvent(traceEvGCMarkAssistStart, 1) +} + +func traceGCMarkAssistDone() { + traceEvent(traceEvGCMarkAssistDone, -1) } func traceGoCreate(newg *g, pc uintptr) { @@ -967,7 +1011,7 @@ func traceGoPreempt() { traceEvent(traceEvGoPreempt, 1) } -func traceGoPark(traceEv byte, skip int, gp *g) { +func traceGoPark(traceEv byte, skip int) { if traceEv&traceFutileWakeup != 0 { traceEvent(traceEvFutileWakeup, -1) } diff --git a/libgo/go/runtime/trace/trace_stack_test.go b/libgo/go/runtime/trace/trace_stack_test.go index c37b33de862..274cdf7800a 100644 --- a/libgo/go/runtime/trace/trace_stack_test.go +++ b/libgo/go/runtime/trace/trace_stack_test.go @@ -151,7 +151,7 @@ func TestTraceSymbolize(t *testing.T) { {"testing.tRunner", 0}, }}, {trace.EvGoCreate, []frame{ - {"runtime/trace_test.TestTraceSymbolize", 39}, + {"runtime/trace_test.TestTraceSymbolize", 37}, {"testing.tRunner", 0}, }}, {trace.EvGoStop, []frame{ @@ -231,6 +231,7 @@ func TestTraceSymbolize(t *testing.T) { if runtime.GOOS != "windows" && runtime.GOOS != "plan9" { want = append(want, []eventDesc{ {trace.EvGoBlockNet, []frame{ + {"internal/poll.(*FD).Accept", 0}, {"net.(*netFD).accept", 0}, {"net.(*TCPListener).accept", 0}, {"net.(*TCPListener).Accept", 0}, @@ -239,6 +240,7 @@ func TestTraceSymbolize(t *testing.T) { {trace.EvGoSysCall, []frame{ {"syscall.read", 0}, {"syscall.Read", 0}, + {"internal/poll.(*FD).Read", 0}, {"os.(*File).read", 0}, {"os.(*File).Read", 0}, {"runtime/trace_test.TestTraceSymbolize.func11", 102}, @@ -274,9 +276,10 @@ func TestTraceSymbolize(t *testing.T) { continue } for _, f := range ev.Stk { - t.Logf(" %v:%v", f.Fn, f.Line) + t.Logf(" %v :: %s:%v", f.Fn, f.File, f.Line) } t.Logf("---") } + t.Logf("======") } } diff --git a/libgo/go/runtime/traceback_gccgo.go b/libgo/go/runtime/traceback_gccgo.go index 715772edf92..f29ccd7b564 100644 --- a/libgo/go/runtime/traceback_gccgo.go +++ b/libgo/go/runtime/traceback_gccgo.go @@ -12,14 +12,6 @@ import ( _ "unsafe" // for go:linkname ) -// For gccgo, use go:linkname to rename compiler-called functions to -// themselves, so that the compiler will export them. -// These are temporary for C runtime code to call. -//go:linkname traceback runtime.traceback -//go:linkname printtrace runtime.printtrace -//go:linkname goroutineheader runtime.goroutineheader -//go:linkname printcreatedby runtime.printcreatedby - func printcreatedby(gp *g) { // Show what created goroutine, except main goroutine (goid 1). pc := gp.gopc @@ -71,6 +63,7 @@ func traceback(skip int32) { var locbuf [100]location c := c_callers(skip+1, &locbuf[0], int32(len(locbuf)), false) printtrace(locbuf[:c], getg()) + printcreatedby(getg()) } // printtrace prints a traceback from locbuf. @@ -223,7 +216,7 @@ func tracebackothers(me *g) { print("\tgoroutine running on other thread; stack unavailable\n") printcreatedby(gp) } else if readgstatus(gp)&^_Gscan == _Gsyscall { - print("\tgoroutine in C code; stack unavailable\n") + print("\tin C code; stack unavailable\n") printcreatedby(gp) } else { gp.traceback = &tb diff --git a/libgo/go/runtime/write_err_android.go b/libgo/go/runtime/write_err_android.go index 748dec644c6..bf99b5f6c5b 100644 --- a/libgo/go/runtime/write_err_android.go +++ b/libgo/go/runtime/write_err_android.go @@ -144,7 +144,7 @@ func writeLogdHeader() int { // hdr[3:7] sec unsigned uint32, little endian. // hdr[7:11] nsec unsigned uint32, little endian. hdr[0] = 0 // LOG_ID_MAIN - sec, nsec := time_now() + sec, nsec := walltime() packUint32(hdr[3:7], uint32(sec)) packUint32(hdr[7:11], uint32(nsec)) diff --git a/libgo/go/sort/example_test.go b/libgo/go/sort/example_test.go index 980c0d03680..f8d8491bc4c 100644 --- a/libgo/go/sort/example_test.go +++ b/libgo/go/sort/example_test.go @@ -41,3 +41,38 @@ func ExampleSlice() { // Output: By name: [{Alice 55} {Bob 75} {Gopher 7} {Vera 24}] // By age: [{Gopher 7} {Vera 24} {Alice 55} {Bob 75}] } + +func ExampleSliceStable() { + + people := []struct { + Name string + Age int + }{ + {"Alice", 25}, + {"Elizabeth", 75}, + {"Alice", 75}, + {"Bob", 75}, + {"Alice", 75}, + {"Bob", 25}, + {"Colin", 25}, + {"Elizabeth", 25}, + } + + // Sort by name, preserving original order + sort.SliceStable(people, func(i, j int) bool { return people[i].Name < people[j].Name }) + fmt.Println("By name:", people) + + // Sort by age preserving name order + sort.SliceStable(people, func(i, j int) bool { return people[i].Age < people[j].Age }) + fmt.Println("By age,name:", people) + + // Output: By name: [{Alice 25} {Alice 75} {Alice 75} {Bob 75} {Bob 25} {Colin 25} {Elizabeth 75} {Elizabeth 25}] + // By age,name: [{Alice 25} {Bob 25} {Colin 25} {Elizabeth 25} {Alice 75} {Alice 75} {Bob 75} {Elizabeth 75}] +} + +func ExampleStrings() { + s := []string{"Go", "Bravo", "Gopher", "Alpha", "Grin", "Delta"} + sort.Strings(s) + fmt.Println(s) + // Output: [Alpha Bravo Delta Go Gopher Grin] +} diff --git a/libgo/go/sort/genzfunc.go b/libgo/go/sort/genzfunc.go index 6d2b471b62e..3bb7691f6a8 100644 --- a/libgo/go/sort/genzfunc.go +++ b/libgo/go/sort/genzfunc.go @@ -115,6 +115,10 @@ func rewriteCall(ce *ast.CallExpr) { // e.g. skip SelectorExpr (data.Less(..) calls) return } + // skip casts + if ident.Name == "int" || ident.Name == "uint" { + return + } if len(ce.Args) < 1 { return } diff --git a/libgo/go/sort/search.go b/libgo/go/sort/search.go index b9640a40aff..fcff0f9491b 100644 --- a/libgo/go/sort/search.go +++ b/libgo/go/sort/search.go @@ -24,7 +24,7 @@ package sort // // For instance, given a slice data sorted in ascending order, // the call Search(len(data), func(i int) bool { return data[i] >= 23 }) -// returns the smallest index i such that data[i] >= 23. If the caller +// returns the smallest index i such that data[i] >= 23. If the caller // wants to find whether 23 is in the slice, it must test data[i] == 23 // separately. // @@ -61,7 +61,7 @@ func Search(n int, f func(int) bool) int { // Invariant: f(i-1) == false, f(j) == true. i, j := 0, n for i < j { - h := i + (j-i)/2 // avoid overflow when computing h + h := int(uint(i+j) >> 1) // avoid overflow when computing h // i ≤ h < j if !f(h) { i = h + 1 // preserves f(i-1) == false diff --git a/libgo/go/sort/sort.go b/libgo/go/sort/sort.go index 72d24efceab..081b7007989 100644 --- a/libgo/go/sort/sort.go +++ b/libgo/go/sort/sort.go @@ -96,7 +96,7 @@ func swapRange(data Interface, a, b, n int) { } func doPivot(data Interface, lo, hi int) (midlo, midhi int) { - m := lo + (hi-lo)/2 // Written like this to avoid integer overflow. + m := int(uint(lo+hi) >> 1) // Written like this to avoid integer overflow. if hi-lo > 40 { // Tukey's ``Ninther,'' median of three medians of three. s := (hi - lo) / 8 @@ -314,7 +314,8 @@ func (p IntSlice) Swap(i, j int) { p[i], p[j] = p[j], p[i] } // Sort is a convenience method. func (p IntSlice) Sort() { Sort(p) } -// Float64Slice attaches the methods of Interface to []float64, sorting in increasing order. +// Float64Slice attaches the methods of Interface to []float64, sorting in increasing order +// (not-a-number values are treated as less than other values). type Float64Slice []float64 func (p Float64Slice) Len() int { return len(p) } @@ -344,7 +345,8 @@ func (p StringSlice) Sort() { Sort(p) } // Ints sorts a slice of ints in increasing order. func Ints(a []int) { Sort(IntSlice(a)) } -// Float64s sorts a slice of float64s in increasing order. +// Float64s sorts a slice of float64s in increasing order +// (not-a-number values are treated as less than other values). func Float64s(a []float64) { Sort(Float64Slice(a)) } // Strings sorts a slice of strings in increasing order. @@ -353,7 +355,8 @@ func Strings(a []string) { Sort(StringSlice(a)) } // IntsAreSorted tests whether a slice of ints is sorted in increasing order. func IntsAreSorted(a []int) bool { return IsSorted(IntSlice(a)) } -// Float64sAreSorted tests whether a slice of float64s is sorted in increasing order. +// Float64sAreSorted tests whether a slice of float64s is sorted in increasing order +// (not-a-number values are treated as less than other values). func Float64sAreSorted(a []float64) bool { return IsSorted(Float64Slice(a)) } // StringsAreSorted tests whether a slice of strings is sorted in increasing order. @@ -447,7 +450,7 @@ func symMerge(data Interface, a, m, b int) { i := m j := b for i < j { - h := i + (j-i)/2 + h := int(uint(i+j) >> 1) if data.Less(h, a) { i = h + 1 } else { @@ -471,7 +474,7 @@ func symMerge(data Interface, a, m, b int) { i := a j := m for i < j { - h := i + (j-i)/2 + h := int(uint(i+j) >> 1) if !data.Less(m, h) { i = h + 1 } else { @@ -485,7 +488,7 @@ func symMerge(data Interface, a, m, b int) { return } - mid := a + (b-a)/2 + mid := int(uint(a+b) >> 1) n := mid + m var start, r int if m > mid { @@ -498,7 +501,7 @@ func symMerge(data Interface, a, m, b int) { p := n - 1 for start < r { - c := start + (r-start)/2 + c := int(uint(start+r) >> 1) if !data.Less(p-c, c) { start = c + 1 } else { diff --git a/libgo/go/sort/zfuncversion.go b/libgo/go/sort/zfuncversion.go index 7abb18a24d5..99c95a22c1f 100644 --- a/libgo/go/sort/zfuncversion.go +++ b/libgo/go/sort/zfuncversion.go @@ -70,7 +70,7 @@ func swapRange_func(data lessSwap, a, b, n int) { // Auto-generated variant of sort.go:doPivot func doPivot_func(data lessSwap, lo, hi int) (midlo, midhi int) { - m := lo + (hi-lo)/2 + m := int(uint(lo+hi) >> 1) if hi-lo > 40 { s := (hi - lo) / 8 medianOfThree_func(data, lo, lo+s, lo+2*s) @@ -189,7 +189,7 @@ func symMerge_func(data lessSwap, a, m, b int) { i := m j := b for i < j { - h := i + (j-i)/2 + h := int(uint(i+j) >> 1) if data.Less(h, a) { i = h + 1 } else { @@ -205,7 +205,7 @@ func symMerge_func(data lessSwap, a, m, b int) { i := a j := m for i < j { - h := i + (j-i)/2 + h := int(uint(i+j) >> 1) if !data.Less(m, h) { i = h + 1 } else { @@ -217,7 +217,7 @@ func symMerge_func(data lessSwap, a, m, b int) { } return } - mid := a + (b-a)/2 + mid := int(uint(a+b) >> 1) n := mid + m var start, r int if m > mid { @@ -229,7 +229,7 @@ func symMerge_func(data lessSwap, a, m, b int) { } p := n - 1 for start < r { - c := start + (r-start)/2 + c := int(uint(start+r) >> 1) if !data.Less(p-c, c) { start = c + 1 } else { diff --git a/libgo/go/strconv/atof_test.go b/libgo/go/strconv/atof_test.go index 0a89c3e0bfa..3380b200884 100644 --- a/libgo/go/strconv/atof_test.go +++ b/libgo/go/strconv/atof_test.go @@ -10,6 +10,7 @@ import ( "reflect" . "strconv" "strings" + "sync" "testing" "time" ) @@ -213,12 +214,17 @@ type atofSimpleTest struct { } var ( + atofOnce sync.Once atofRandomTests []atofSimpleTest benchmarksRandomBits [1024]string benchmarksRandomNormal [1024]string ) -func init() { +func initAtof() { + atofOnce.Do(initAtofOnce) +} + +func initAtofOnce() { // The atof routines return NumErrors wrapping // the error and the string. Convert the table above. for i := range atoftests { @@ -261,6 +267,7 @@ func init() { } func testAtof(t *testing.T, opt bool) { + initAtof() oldopt := SetOptimize(opt) for i := 0; i < len(atoftests); i++ { test := &atoftests[i] @@ -306,6 +313,7 @@ func TestAtof(t *testing.T) { testAtof(t, true) } func TestAtofSlow(t *testing.T) { testAtof(t, false) } func TestAtofRandom(t *testing.T) { + initAtof() for _, test := range atofRandomTests { x, _ := ParseFloat(test.s, 64) switch { diff --git a/libgo/go/strconv/decimal.go b/libgo/go/strconv/decimal.go index 957acd98918..b58001888e8 100644 --- a/libgo/go/strconv/decimal.go +++ b/libgo/go/strconv/decimal.go @@ -15,8 +15,8 @@ type decimal struct { d [800]byte // digits, big-endian representation nd int // number of digits used dp int // decimal point - neg bool - trunc bool // discarded nonzero digits beyond d[:nd] + neg bool // negative flag + trunc bool // discarded nonzero digits beyond d[:nd] } func (a *decimal) String() string { diff --git a/libgo/go/strconv/itoa.go b/libgo/go/strconv/itoa.go index f50d8779408..78527c8ae66 100644 --- a/libgo/go/strconv/itoa.go +++ b/libgo/go/strconv/itoa.go @@ -4,10 +4,15 @@ package strconv +const fastSmalls = true // enable fast path for small integers + // FormatUint returns the string representation of i in the given base, // for 2 <= base <= 36. The result uses the lower-case letters 'a' to 'z' // for digit values >= 10. func FormatUint(i uint64, base int) string { + if fastSmalls && i < nSmalls && base == 10 { + return small(int(i)) + } _, s := formatBits(nil, i, base, false, false) return s } @@ -16,6 +21,9 @@ func FormatUint(i uint64, base int) string { // for 2 <= base <= 36. The result uses the lower-case letters 'a' to 'z' // for digit values >= 10. func FormatInt(i int64, base int) string { + if fastSmalls && 0 <= i && i < nSmalls && base == 10 { + return small(int(i)) + } _, s := formatBits(nil, uint64(i), base, i < 0, false) return s } @@ -28,6 +36,9 @@ func Itoa(i int) string { // AppendInt appends the string form of the integer i, // as generated by FormatInt, to dst and returns the extended buffer. func AppendInt(dst []byte, i int64, base int) []byte { + if fastSmalls && 0 <= i && i < nSmalls && base == 10 { + return append(dst, small(int(i))...) + } dst, _ = formatBits(dst, uint64(i), base, i < 0, true) return dst } @@ -35,13 +46,38 @@ func AppendInt(dst []byte, i int64, base int) []byte { // AppendUint appends the string form of the unsigned integer i, // as generated by FormatUint, to dst and returns the extended buffer. func AppendUint(dst []byte, i uint64, base int) []byte { + if fastSmalls && i < nSmalls && base == 10 { + return append(dst, small(int(i))...) + } dst, _ = formatBits(dst, i, base, false, true) return dst } -const ( - digits = "0123456789abcdefghijklmnopqrstuvwxyz" -) +// small returns the string for an i with 0 <= i < nSmalls. +func small(i int) string { + off := 0 + if i < 10 { + off = 1 + } + return smallsString[i*2+off : i*2+2] +} + +const nSmalls = 100 + +const smallsString = "00010203040506070809" + + "10111213141516171819" + + "20212223242526272829" + + "30313233343536373839" + + "40414243444546474849" + + "50515253545556575859" + + "60616263646566676869" + + "70717273747576777879" + + "80818283848586878889" + + "90919293949596979899" + +const host32bit = ^uint(0)>>32 == 0 + +const digits = "0123456789abcdefghijklmnopqrstuvwxyz" var shifts = [len(digits) + 1]uint{ 1 << 1: 1, @@ -71,61 +107,84 @@ func formatBits(dst []byte, u uint64, base int, neg, append_ bool) (d []byte, s } // convert bits + // We use uint values where we can because those will + // fit into a single register even on a 32bit machine. if base == 10 { // common case: use constants for / because // the compiler can optimize it into a multiply+shift - if ^uintptr(0)>>32 == 0 { - for u > uint64(^uintptr(0)) { + if host32bit { + // convert the lower digits using 32bit operations + for u >= 1e9 { + // Avoid using r = a%b in addition to q = a/b + // since 64bit division and modulo operations + // are calculated by runtime functions on 32bit machines. q := u / 1e9 - us := uintptr(u - q*1e9) // us % 1e9 fits into a uintptr - for j := 9; j > 0; j-- { - i-- - qs := us / 10 - a[i] = byte(us - qs*10 + '0') - us = qs + us := uint(u - q*1e9) // u % 1e9 fits into a uint + for j := 4; j > 0; j-- { + is := us % 100 * 2 + us /= 100 + i -= 2 + a[i+1] = smallsString[is+1] + a[i+0] = smallsString[is+0] } + + // us < 10, since it contains the last digit + // from the initial 9-digit us. + i-- + a[i] = smallsString[us*2+1] + u = q } + // u < 1e9 } - // u guaranteed to fit into a uintptr - us := uintptr(u) - for us >= 10 { - i-- - q := us / 10 - a[i] = byte(us - q*10 + '0') - us = q + // u guaranteed to fit into a uint + us := uint(u) + for us >= 100 { + is := us % 100 * 2 + us /= 100 + i -= 2 + a[i+1] = smallsString[is+1] + a[i+0] = smallsString[is+0] } - // u < 10 + + // us < 100 + is := us * 2 i-- - a[i] = byte(us + '0') + a[i] = smallsString[is+1] + if us >= 10 { + i-- + a[i] = smallsString[is] + } } else if s := shifts[base]; s > 0 { // base is power of 2: use shifts and masks instead of / and % b := uint64(base) - m := uintptr(b) - 1 // == 1<= b { i-- - a[i] = digits[uintptr(u)&m] + a[i] = digits[uint(u)&m] u >>= s } // u < base i-- - a[i] = digits[uintptr(u)] - + a[i] = digits[uint(u)] } else { // general case b := uint64(base) for u >= b { i-- + // Avoid using r = a%b in addition to q = a/b + // since 64bit division and modulo operations + // are calculated by runtime functions on 32bit machines. q := u / b - a[i] = digits[uintptr(u-q*b)] + a[i] = digits[uint(u-q*b)] u = q } // u < base i-- - a[i] = digits[uintptr(u)] + a[i] = digits[uint(u)] } // add sign, if any diff --git a/libgo/go/strconv/itoa_test.go b/libgo/go/strconv/itoa_test.go index 48dc03e8390..89c2de6941a 100644 --- a/libgo/go/strconv/itoa_test.go +++ b/libgo/go/strconv/itoa_test.go @@ -126,10 +126,46 @@ func TestUitoa(t *testing.T) { } } +var varlenUints = []struct { + in uint64 + out string +}{ + {1, "1"}, + {12, "12"}, + {123, "123"}, + {1234, "1234"}, + {12345, "12345"}, + {123456, "123456"}, + {1234567, "1234567"}, + {12345678, "12345678"}, + {123456789, "123456789"}, + {1234567890, "1234567890"}, + {12345678901, "12345678901"}, + {123456789012, "123456789012"}, + {1234567890123, "1234567890123"}, + {12345678901234, "12345678901234"}, + {123456789012345, "123456789012345"}, + {1234567890123456, "1234567890123456"}, + {12345678901234567, "12345678901234567"}, + {123456789012345678, "123456789012345678"}, + {1234567890123456789, "1234567890123456789"}, + {12345678901234567890, "12345678901234567890"}, +} + +func TestFormatUintVarlen(t *testing.T) { + for _, test := range varlenUints { + s := FormatUint(test.in, 10) + if s != test.out { + t.Errorf("FormatUint(%v, 10) = %v want %v", test.in, s, test.out) + } + } +} + func BenchmarkFormatInt(b *testing.B) { for i := 0; i < b.N; i++ { for _, test := range itob64tests { - FormatInt(test.in, test.base) + s := FormatInt(test.in, test.base) + BenchSink += len(s) } } } @@ -138,7 +174,8 @@ func BenchmarkAppendInt(b *testing.B) { dst := make([]byte, 0, 30) for i := 0; i < b.N; i++ { for _, test := range itob64tests { - AppendInt(dst, test.in, test.base) + dst = AppendInt(dst[:0], test.in, test.base) + BenchSink += len(dst) } } } @@ -146,7 +183,8 @@ func BenchmarkAppendInt(b *testing.B) { func BenchmarkFormatUint(b *testing.B) { for i := 0; i < b.N; i++ { for _, test := range uitob64tests { - FormatUint(test.in, test.base) + s := FormatUint(test.in, test.base) + BenchSink += len(s) } } } @@ -155,7 +193,39 @@ func BenchmarkAppendUint(b *testing.B) { dst := make([]byte, 0, 30) for i := 0; i < b.N; i++ { for _, test := range uitob64tests { - AppendUint(dst, test.in, test.base) + dst = AppendUint(dst[:0], test.in, test.base) + BenchSink += len(dst) } } } + +func BenchmarkFormatIntSmall(b *testing.B) { + const smallInt = 42 + for i := 0; i < b.N; i++ { + s := FormatInt(smallInt, 10) + BenchSink += len(s) + } +} + +func BenchmarkAppendIntSmall(b *testing.B) { + dst := make([]byte, 0, 30) + const smallInt = 42 + for i := 0; i < b.N; i++ { + dst = AppendInt(dst[:0], smallInt, 10) + BenchSink += len(dst) + } +} + +func BenchmarkAppendUintVarlen(b *testing.B) { + for _, test := range varlenUints { + b.Run(test.out, func(b *testing.B) { + dst := make([]byte, 0, 30) + for j := 0; j < b.N; j++ { + dst = AppendUint(dst[:0], test.in, 10) + BenchSink += len(dst) + } + }) + } +} + +var BenchSink int // make sure compiler cannot optimize away benchmarks diff --git a/libgo/go/strconv/quote.go b/libgo/go/strconv/quote.go index 76c5c2a1cbb..db57065cac1 100644 --- a/libgo/go/strconv/quote.go +++ b/libgo/go/strconv/quote.go @@ -32,7 +32,7 @@ func appendQuotedWith(buf []byte, s string, quote byte, ASCIIonly, graphicOnly b buf = append(buf, lowerhex[s[0]&0xF]) continue } - buf = appendEscapedRune(buf, r, width, quote, ASCIIonly, graphicOnly) + buf = appendEscapedRune(buf, r, quote, ASCIIonly, graphicOnly) } buf = append(buf, quote) return buf @@ -43,12 +43,12 @@ func appendQuotedRuneWith(buf []byte, r rune, quote byte, ASCIIonly, graphicOnly if !utf8.ValidRune(r) { r = utf8.RuneError } - buf = appendEscapedRune(buf, r, utf8.RuneLen(r), quote, ASCIIonly, graphicOnly) + buf = appendEscapedRune(buf, r, quote, ASCIIonly, graphicOnly) buf = append(buf, quote) return buf } -func appendEscapedRune(buf []byte, r rune, width int, quote byte, ASCIIonly, graphicOnly bool) []byte { +func appendEscapedRune(buf []byte, r rune, quote byte, ASCIIonly, graphicOnly bool) []byte { var runeTmp [utf8.UTFMax]byte if r == rune(quote) || r == '\\' { // always backslashed buf = append(buf, '\\') diff --git a/libgo/go/strings/example_test.go b/libgo/go/strings/example_test.go index 3f9d63b5a4a..e9621522ef2 100644 --- a/libgo/go/strings/example_test.go +++ b/libgo/go/strings/example_test.go @@ -23,6 +23,16 @@ func ExampleFieldsFunc() { // Output: Fields are: ["foo1" "bar2" "baz3"] } +func ExampleCompare() { + fmt.Println(strings.Compare("a", "b")) + fmt.Println(strings.Compare("a", "a")) + fmt.Println(strings.Compare("b", "a")) + // Output: + // -1 + // 0 + // 1 +} + func ExampleContains() { fmt.Println(strings.Contains("seafood", "foo")) fmt.Println(strings.Contains("seafood", "bar")) @@ -47,6 +57,16 @@ func ExampleContainsAny() { // false } +func ExampleContainsRune() { + // Finds whether a string contains a particular Unicode code point. + // The code point for the lowercase letter "a", for example, is 97. + fmt.Println(strings.ContainsRune("aardvark", 97)) + fmt.Println(strings.ContainsRune("timeout", 97)) + // Output: + // true + // false +} + func ExampleCount() { fmt.Println(strings.Count("cheese", "e")) fmt.Println(strings.Count("five", "")) // before & after each rune @@ -109,6 +129,15 @@ func ExampleIndexAny() { // -1 } +func ExampleIndexByte() { + fmt.Println(strings.IndexByte("golang", 'g')) + fmt.Println(strings.IndexByte("gophers", 'h')) + fmt.Println(strings.IndexByte("golang", 'x')) + // Output: + // 0 + // 3 + // -1 +} func ExampleIndexRune() { fmt.Println(strings.IndexRune("chicken", 'k')) fmt.Println(strings.IndexRune("chicken", 'd')) @@ -127,6 +156,16 @@ func ExampleLastIndex() { // -1 } +func ExampleLastIndexAny() { + fmt.Println(strings.LastIndexAny("go gopher", "go")) + fmt.Println(strings.LastIndexAny("go gopher", "rodent")) + fmt.Println(strings.LastIndexAny("go gopher", "fail")) + // Output: + // 4 + // 8 + // -1 +} + func ExampleJoin() { s := []string{"foo", "bar", "baz"} fmt.Println(strings.Join(s, ", ")) @@ -195,6 +234,14 @@ func ExampleTrim() { // Output: ["Achtung! Achtung"] } +func ExampleTrimFunc() { + f := func(c rune) bool { + return !unicode.IsLetter(c) && !unicode.IsNumber(c) + } + fmt.Printf("[%q]", strings.TrimFunc(" Achtung1! Achtung2,...", f)) + // Output: ["Achtung1! Achtung2"] +} + func ExampleMap() { rot13 := func(r rune) rune { switch { diff --git a/libgo/go/strings/replace_test.go b/libgo/go/strings/replace_test.go index 77e48b988bc..34b5badfadf 100644 --- a/libgo/go/strings/replace_test.go +++ b/libgo/go/strings/replace_test.go @@ -540,3 +540,44 @@ func BenchmarkByteByteMap(b *testing.B) { Map(fn, str) } } + +var mapdata = []struct{ name, data string }{ + {"ASCII", "a b c d e f g h i j k l m n o p q r s t u v w x y z"}, + {"Greek", "α β γ δ ε ζ η θ ι κ λ μ ν ξ ο π ρ ς σ τ υ φ χ ψ ω"}, +} + +func BenchmarkMap(b *testing.B) { + mapidentity := func(r rune) rune { + return r + } + + b.Run("identity", func(b *testing.B) { + for _, md := range mapdata { + b.Run(md.name, func(b *testing.B) { + for i := 0; i < b.N; i++ { + Map(mapidentity, md.data) + } + }) + } + }) + + mapchange := func(r rune) rune { + if 'a' <= r && r <= 'z' { + return r + 'A' - 'a' + } + if 'α' <= r && r <= 'ω' { + return r + 'Α' - 'α' + } + return r + } + + b.Run("change", func(b *testing.B) { + for _, md := range mapdata { + b.Run(md.name, func(b *testing.B) { + for i := 0; i < b.N; i++ { + Map(mapchange, md.data) + } + }) + } + }) +} diff --git a/libgo/go/strings/strings.go b/libgo/go/strings/strings.go index 60a281a6ac5..0c836c09d46 100644 --- a/libgo/go/strings/strings.go +++ b/libgo/go/strings/strings.go @@ -72,22 +72,20 @@ func hashStrRev(sep string) (uint32, uint32) { return hash, pow } -// Count counts the number of non-overlapping instances of sep in s. -// If sep is an empty string, Count returns 1 + the number of Unicode code points in s. -func Count(s, sep string) int { - n := 0 - // special cases - if len(sep) == 0 { +// countGeneric implements Count. +func countGeneric(s, substr string) int { + // special case + if len(substr) == 0 { return utf8.RuneCountInString(s) + 1 } - offset := 0 + n := 0 for { - i := Index(s[offset:], sep) + i := Index(s, substr) if i == -1 { return n } n++ - offset += i + len(sep) + s = s[i+len(substr):] } } @@ -106,16 +104,16 @@ func ContainsRune(s string, r rune) bool { return IndexRune(s, r) >= 0 } -// LastIndex returns the index of the last instance of sep in s, or -1 if sep is not present in s. -func LastIndex(s, sep string) int { - n := len(sep) +// LastIndex returns the index of the last instance of substr in s, or -1 if substr is not present in s. +func LastIndex(s, substr string) int { + n := len(substr) switch { case n == 0: return len(s) case n == 1: - return LastIndexByte(s, sep[0]) + return LastIndexByte(s, substr[0]) case n == len(s): - if sep == s { + if substr == s { return 0 } return -1 @@ -123,20 +121,20 @@ func LastIndex(s, sep string) int { return -1 } // Rabin-Karp search from the end of the string - hashsep, pow := hashStrRev(sep) + hashss, pow := hashStrRev(substr) last := len(s) - n var h uint32 for i := len(s) - 1; i >= last; i-- { h = h*primeRK + uint32(s[i]) } - if h == hashsep && s[last:] == sep { + if h == hashss && s[last:] == substr { return last } for i := last - 1; i >= 0; i-- { h *= primeRK h += uint32(s[i]) h -= pow * uint32(s[i+n]) - if h == hashsep && s[i:i+n] == sep { + if h == hashss && s[i:i+n] == substr { return i } } @@ -240,61 +238,187 @@ func genSplit(s, sep string, sepSave, n int) []string { if n < 0 { n = Count(s, sep) + 1 } - c := sep[0] - start := 0 + a := make([]string, n) - na := 0 - for i := 0; i+len(sep) <= len(s) && na+1 < n; i++ { - if s[i] == c && (len(sep) == 1 || s[i:i+len(sep)] == sep) { - a[na] = s[start : i+sepSave] - na++ - start = i + len(sep) - i += len(sep) - 1 - } + n-- + i := 0 + for i < n { + m := Index(s, sep) + if m < 0 { + break + } + a[i] = s[:m+sepSave] + s = s[m+len(sep):] + i++ } - a[na] = s[start:] - return a[0 : na+1] + a[i] = s + return a[:i+1] } // SplitN slices s into substrings separated by sep and returns a slice of // the substrings between those separators. -// If sep is empty, SplitN splits after each UTF-8 sequence. +// // The count determines the number of substrings to return: // n > 0: at most n substrings; the last substring will be the unsplit remainder. // n == 0: the result is nil (zero substrings) // n < 0: all substrings +// +// Edge cases for s and sep (for example, empty strings) are handled +// as described in the documentation for Split. func SplitN(s, sep string, n int) []string { return genSplit(s, sep, 0, n) } // SplitAfterN slices s into substrings after each instance of sep and // returns a slice of those substrings. -// If sep is empty, SplitAfterN splits after each UTF-8 sequence. +// // The count determines the number of substrings to return: // n > 0: at most n substrings; the last substring will be the unsplit remainder. // n == 0: the result is nil (zero substrings) // n < 0: all substrings +// +// Edge cases for s and sep (for example, empty strings) are handled +// as described in the documentation for SplitAfter. func SplitAfterN(s, sep string, n int) []string { return genSplit(s, sep, len(sep), n) } // Split slices s into all substrings separated by sep and returns a slice of // the substrings between those separators. -// If sep is empty, Split splits after each UTF-8 sequence. +// +// If s does not contain sep and sep is not empty, Split returns a +// slice of length 1 whose only element is s. +// +// If sep is empty, Split splits after each UTF-8 sequence. If both s +// and sep are empty, Split returns an empty slice. +// // It is equivalent to SplitN with a count of -1. func Split(s, sep string) []string { return genSplit(s, sep, 0, -1) } // SplitAfter slices s into all substrings after each instance of sep and // returns a slice of those substrings. -// If sep is empty, SplitAfter splits after each UTF-8 sequence. +// +// If s does not contain sep and sep is not empty, SplitAfter returns +// a slice of length 1 whose only element is s. +// +// If sep is empty, SplitAfter splits after each UTF-8 sequence. If +// both s and sep are empty, SplitAfter returns an empty slice. +// // It is equivalent to SplitAfterN with a count of -1. func SplitAfter(s, sep string) []string { return genSplit(s, sep, len(sep), -1) } +var asciiSpace = [256]uint8{'\t': 1, '\n': 1, '\v': 1, '\f': 1, '\r': 1, ' ': 1} + // Fields splits the string s around each instance of one or more consecutive white space // characters, as defined by unicode.IsSpace, returning an array of substrings of s or an // empty list if s contains only white space. func Fields(s string) []string { - return FieldsFunc(s, unicode.IsSpace) + // First count the fields. + // This is an exact count if s is ASCII, otherwise it is an approximation. + n := 0 + wasSpace := 1 + // setBits is used to track which bits are set in the bytes of s. + setBits := uint8(0) + for i := 0; i < len(s); i++ { + r := s[i] + setBits |= r + isSpace := int(asciiSpace[r]) + n += wasSpace & ^isSpace + wasSpace = isSpace + } + + if setBits < utf8.RuneSelf { // ASCII fast path + a := make([]string, n) + na := 0 + fieldStart := 0 + i := 0 + // Skip spaces in the front of the input. + for i < len(s) && asciiSpace[s[i]] != 0 { + i++ + } + fieldStart = i + for i < len(s) { + if asciiSpace[s[i]] == 0 { + i++ + continue + } + a[na] = s[fieldStart:i] + na++ + i++ + // Skip spaces in between fields. + for i < len(s) && asciiSpace[s[i]] != 0 { + i++ + } + fieldStart = i + } + if fieldStart < len(s) { // Last field might end at EOF. + a[na] = s[fieldStart:] + } + return a + } + + // Some runes in the input string are not ASCII. + // Same general approach as in the ASCII path but + // uses DecodeRuneInString and unicode.IsSpace if + // a non-ASCII rune needs to be decoded and checked + // if it corresponds to a space. + a := make([]string, 0, n) + fieldStart := 0 + i := 0 + // Skip spaces in the front of the input. + for i < len(s) { + if c := s[i]; c < utf8.RuneSelf { + if asciiSpace[c] == 0 { + break + } + i++ + } else { + r, w := utf8.DecodeRuneInString(s[i:]) + if !unicode.IsSpace(r) { + break + } + i += w + } + } + fieldStart = i + for i < len(s) { + if c := s[i]; c < utf8.RuneSelf { + if asciiSpace[c] == 0 { + i++ + continue + } + a = append(a, s[fieldStart:i]) + i++ + } else { + r, w := utf8.DecodeRuneInString(s[i:]) + if !unicode.IsSpace(r) { + i += w + continue + } + a = append(a, s[fieldStart:i]) + i += w + } + // Skip spaces in between fields. + for i < len(s) { + if c := s[i]; c < utf8.RuneSelf { + if asciiSpace[c] == 0 { + break + } + i++ + } else { + r, w := utf8.DecodeRuneInString(s[i:]) + if !unicode.IsSpace(r) { + break + } + i += w + } + } + fieldStart = i + } + if fieldStart < len(s) { // Last field might end at EOF. + a = append(a, s[fieldStart:]) + } + return a } // FieldsFunc splits the string s at each run of Unicode code points c satisfying f(c) @@ -383,40 +507,71 @@ func Map(mapping func(rune) rune, s string) string { // In the worst case, the string can grow when mapped, making // things unpleasant. But it's so rare we barge in assuming it's // fine. It could also shrink but that falls out naturally. - maxbytes := len(s) // length of b - nbytes := 0 // number of bytes encoded in b + // The output buffer b is initialized on demand, the first // time a character differs. var b []byte + // nbytes is the number of bytes encoded in b. + var nbytes int for i, c := range s { r := mapping(c) - if b == nil { - if r == c { - continue - } - b = make([]byte, maxbytes) - nbytes = copy(b, s[:i]) + if r == c { + continue } + + b = make([]byte, len(s)+utf8.UTFMax) + nbytes = copy(b, s[:i]) if r >= 0 { - wid := 1 - if r >= utf8.RuneSelf { - wid = utf8.RuneLen(r) + if r <= utf8.RuneSelf { + b[nbytes] = byte(r) + nbytes++ + } else { + nbytes += utf8.EncodeRune(b[nbytes:], r) } - if nbytes+wid > maxbytes { - // Grow the buffer. - maxbytes = maxbytes*2 + utf8.UTFMax - nb := make([]byte, maxbytes) - copy(nb, b[0:nbytes]) - b = nb - } - nbytes += utf8.EncodeRune(b[nbytes:maxbytes], r) } + + if c == utf8.RuneError { + // RuneError is the result of either decoding + // an invalid sequence or '\uFFFD'. Determine + // the correct number of bytes we need to advance. + _, w := utf8.DecodeRuneInString(s[i:]) + i += w + } else { + i += utf8.RuneLen(c) + } + + s = s[i:] + break } + if b == nil { return s } - return string(b[0:nbytes]) + + for _, c := range s { + r := mapping(c) + + // common case + if (0 <= r && r <= utf8.RuneSelf) && nbytes < len(b) { + b[nbytes] = byte(r) + nbytes++ + continue + } + + // b is not big enough or r is not a ASCII rune. + if r >= 0 { + if nbytes+utf8.UTFMax >= len(b) { + // Grow the buffer. + nb := make([]byte, 2*len(b)) + copy(nb, b[:nbytes]) + b = nb + } + nbytes += utf8.EncodeRune(b[nbytes:], r) + } + } + + return string(b[:nbytes]) } // Repeat returns a new string consisting of count copies of the string s. @@ -561,17 +716,10 @@ func LastIndexFunc(s string, f func(rune) bool) int { // truth==false, the sense of the predicate function is // inverted. func indexFunc(s string, f func(rune) bool, truth bool) int { - start := 0 - for start < len(s) { - wid := 1 - r := rune(s[start]) - if r >= utf8.RuneSelf { - r, wid = utf8.DecodeRuneInString(s[start:]) - } + for i, r := range s { if f(r) == truth { - return start + return i } - start += wid } return -1 } diff --git a/libgo/go/strings/strings_amd64.go b/libgo/go/strings/strings_amd64.go index e55afd53d01..9648912fd5b 100644 --- a/libgo/go/strings/strings_amd64.go +++ b/libgo/go/strings/strings_amd64.go @@ -6,44 +6,46 @@ package strings +import "internal/cpu" + //go:noescape // indexShortStr returns the index of the first instance of c in s, or -1 if c is not present in s. // indexShortStr requires 2 <= len(c) <= shortStringLen -func indexShortStr(s, c string) int // ../runtime/asm_$GOARCH.s -func supportAVX2() bool // ../runtime/asm_$GOARCH.s +func indexShortStr(s, c string) int // ../runtime/asm_amd64.s +func countByte(s string, c byte) int // ../runtime/asm_amd64.s var shortStringLen int func init() { - if supportAVX2() { + if cpu.X86.HasAVX2 { shortStringLen = 63 } else { shortStringLen = 31 } } -// Index returns the index of the first instance of sep in s, or -1 if sep is not present in s. -func Index(s, sep string) int { - n := len(sep) +// Index returns the index of the first instance of substr in s, or -1 if substr is not present in s. +func Index(s, substr string) int { + n := len(substr) switch { case n == 0: return 0 case n == 1: - return IndexByte(s, sep[0]) + return IndexByte(s, substr[0]) case n == len(s): - if sep == s { + if substr == s { return 0 } return -1 case n > len(s): return -1 case n <= shortStringLen: - // Use brute force when s and sep both are small + // Use brute force when s and substr both are small if len(s) <= 64 { - return indexShortStr(s, sep) + return indexShortStr(s, substr) } - c := sep[0] + c := substr[0] i := 0 t := s[:len(s)-n+1] fails := 0 @@ -57,7 +59,7 @@ func Index(s, sep string) int { } i += o } - if s[i:i+n] == sep { + if s[i:i+n] == substr { return i } fails++ @@ -66,7 +68,7 @@ func Index(s, sep string) int { // Too many means more that 1 error per 8 characters. // Allow some errors in the beginning. if fails > (i+16)/8 { - r := indexShortStr(s[i:], sep) + r := indexShortStr(s[i:], substr) if r >= 0 { return r + i } @@ -76,12 +78,12 @@ func Index(s, sep string) int { return -1 } // Rabin-Karp search - hashsep, pow := hashStr(sep) + hashss, pow := hashStr(substr) var h uint32 for i := 0; i < n; i++ { h = h*primeRK + uint32(s[i]) } - if h == hashsep && s[:n] == sep { + if h == hashss && s[:n] == substr { return 0 } for i := n; i < len(s); { @@ -89,9 +91,18 @@ func Index(s, sep string) int { h += uint32(s[i]) h -= pow * uint32(s[i-n]) i++ - if h == hashsep && s[i-n:i] == sep { + if h == hashss && s[i-n:i] == substr { return i - n } } return -1 } + +// Count counts the number of non-overlapping instances of substr in s. +// If substr is an empty string, Count returns 1 + the number of Unicode code points in s. +func Count(s, substr string) int { + if len(substr) == 1 && cpu.X86.HasPOPCNT { + return countByte(s, byte(substr[0])) + } + return countGeneric(s, substr) +} diff --git a/libgo/go/strings/strings_generic.go b/libgo/go/strings/strings_generic.go index a3ad515444b..9844201db30 100644 --- a/libgo/go/strings/strings_generic.go +++ b/libgo/go/strings/strings_generic.go @@ -9,16 +9,16 @@ package strings // TODO: implements short string optimization on non amd64 platforms // and get rid of strings_amd64.go -// Index returns the index of the first instance of sep in s, or -1 if sep is not present in s. -func Index(s, sep string) int { - n := len(sep) +// Index returns the index of the first instance of substr in s, or -1 if substr is not present in s. +func Index(s, substr string) int { + n := len(substr) switch { case n == 0: return 0 case n == 1: - return IndexByte(s, sep[0]) + return IndexByte(s, substr[0]) case n == len(s): - if sep == s { + if substr == s { return 0 } return -1 @@ -26,12 +26,12 @@ func Index(s, sep string) int { return -1 } // Rabin-Karp search - hashsep, pow := hashStr(sep) + hashss, pow := hashStr(substr) var h uint32 for i := 0; i < n; i++ { h = h*primeRK + uint32(s[i]) } - if h == hashsep && s[:n] == sep { + if h == hashss && s[:n] == substr { return 0 } for i := n; i < len(s); { @@ -39,9 +39,15 @@ func Index(s, sep string) int { h += uint32(s[i]) h -= pow * uint32(s[i-n]) i++ - if h == hashsep && s[i-n:i] == sep { + if h == hashss && s[i-n:i] == substr { return i - n } } return -1 } + +// Count counts the number of non-overlapping instances of substr in s. +// If substr is an empty string, Count returns 1 + the number of Unicode code points in s. +func Count(s, substr string) int { + return countGeneric(s, substr) +} diff --git a/libgo/go/strings/strings_s390x.go b/libgo/go/strings/strings_s390x.go index b47702fd51a..b05fb2b025a 100644 --- a/libgo/go/strings/strings_s390x.go +++ b/libgo/go/strings/strings_s390x.go @@ -26,27 +26,27 @@ func init() { } } -// Index returns the index of the first instance of sep in s, or -1 if sep is not present in s. -func Index(s, sep string) int { - n := len(sep) +// Index returns the index of the first instance of substr in s, or -1 if substr is not present in s. +func Index(s, substr string) int { + n := len(substr) switch { case n == 0: return 0 case n == 1: - return IndexByte(s, sep[0]) + return IndexByte(s, substr[0]) case n == len(s): - if sep == s { + if substr == s { return 0 } return -1 case n > len(s): return -1 case n <= shortStringLen: - // Use brute force when s and sep both are small + // Use brute force when s and substr both are small if len(s) <= 64 { - return indexShortStr(s, sep) + return indexShortStr(s, substr) } - c := sep[0] + c := substr[0] i := 0 t := s[:len(s)-n+1] fails := 0 @@ -60,7 +60,7 @@ func Index(s, sep string) int { } i += o } - if s[i:i+n] == sep { + if s[i:i+n] == substr { return i } fails++ @@ -69,7 +69,7 @@ func Index(s, sep string) int { // Too many means more that 1 error per 8 characters. // Allow some errors in the beginning. if fails > (i+16)/8 { - r := indexShortStr(s[i:], sep) + r := indexShortStr(s[i:], substr) if r >= 0 { return r + i } @@ -79,12 +79,12 @@ func Index(s, sep string) int { return -1 } // Rabin-Karp search - hashsep, pow := hashStr(sep) + hashss, pow := hashStr(substr) var h uint32 for i := 0; i < n; i++ { h = h*primeRK + uint32(s[i]) } - if h == hashsep && s[:n] == sep { + if h == hashss && s[:n] == substr { return 0 } for i := n; i < len(s); { @@ -92,9 +92,15 @@ func Index(s, sep string) int { h += uint32(s[i]) h -= pow * uint32(s[i-n]) i++ - if h == hashsep && s[i-n:i] == sep { + if h == hashss && s[i-n:i] == substr { return i - n } } return -1 } + +// Count counts the number of non-overlapping instances of substr in s. +// If substr is an empty string, Count returns 1 + the number of Unicode code points in s. +func Count(s, substr string) int { + return countGeneric(s, substr) +} diff --git a/libgo/go/strings/strings_test.go b/libgo/go/strings/strings_test.go index 449fb502d64..0fddaf0e4a6 100644 --- a/libgo/go/strings/strings_test.go +++ b/libgo/go/strings/strings_test.go @@ -456,6 +456,7 @@ var fieldstests = []FieldsTest{ {"", []string{}}, {" ", []string{}}, {" \t ", []string{}}, + {"\u2000", []string{}}, {" abc ", []string{"abc"}}, {"1 2 3 4", []string{"1", "2", "3", "4"}}, {"1 2 3 4", []string{"1", "2", "3", "4"}}, @@ -463,6 +464,9 @@ var fieldstests = []FieldsTest{ {"1\u20002\u20013\u20024", []string{"1", "2", "3", "4"}}, {"\u2000\u2001\u2002", []string{}}, {"\n™\t™\n", []string{"™", "™"}}, + {"\n\u20001™2\u2000 \u2001 ™", []string{"1™2", "™"}}, + {"\n1\uFFFD \uFFFD2\u20003\uFFFD4", []string{"1\uFFFD", "\uFFFD2", "3\uFFFD4"}}, + {"1\xFF\u2000\xFF2\xFF \xFF", []string{"1\xFF", "\xFF2\xFF", "\xFF"}}, {faces, []string{faces}}, } @@ -629,6 +633,19 @@ func TestMap(t *testing.T) { (*reflect.StringHeader)(unsafe.Pointer(&m)).Data { t.Error("unexpected copy during identity map") } + + // 7. Handle invalid UTF-8 sequence + replaceNotLatin := func(r rune) rune { + if unicode.Is(unicode.Latin, r) { + return r + } + return '?' + } + m = Map(replaceNotLatin, "Hello\255World") + expect = "Hello?World" + if m != expect { + t.Errorf("replace invalid sequence: expected %q got %q", expect, m) + } } func TestToUpper(t *testing.T) { runStringTests(t, ToUpper, "ToUpper", upperTests) } @@ -1444,6 +1461,24 @@ func BenchmarkCountTortureOverlapping(b *testing.B) { } } +func BenchmarkCountByte(b *testing.B) { + indexSizes := []int{10, 32, 4 << 10, 4 << 20, 64 << 20} + benchStr := Repeat(benchmarkString, + (indexSizes[len(indexSizes)-1]+len(benchmarkString)-1)/len(benchmarkString)) + benchFunc := func(b *testing.B, benchStr string) { + b.SetBytes(int64(len(benchStr))) + for i := 0; i < b.N; i++ { + Count(benchStr, "=") + } + } + for _, size := range indexSizes { + b.Run(fmt.Sprintf("%d", size), func(b *testing.B) { + benchFunc(b, benchStr[:size]) + }) + } + +} + var makeFieldsInput = func() string { x := make([]byte, 1<<20) // Input is ~10% space, ~10% 2-byte UTF-8, rest ASCII non-space. @@ -1464,40 +1499,88 @@ var makeFieldsInput = func() string { return string(x) } -var fieldsInput = makeFieldsInput() +var makeFieldsInputASCII = func() string { + x := make([]byte, 1<<20) + // Input is ~10% space, rest ASCII non-space. + for i := range x { + if rand.Intn(10) == 0 { + x[i] = ' ' + } else { + x[i] = 'x' + } + } + return string(x) +} + +var stringdata = []struct{ name, data string }{ + {"ASCII", makeFieldsInputASCII()}, + {"Mixed", makeFieldsInput()}, +} func BenchmarkFields(b *testing.B) { - b.SetBytes(int64(len(fieldsInput))) - for i := 0; i < b.N; i++ { - Fields(fieldsInput) + for _, sd := range stringdata { + b.Run(sd.name, func(b *testing.B) { + for j := 1 << 4; j <= 1<<20; j <<= 4 { + b.Run(fmt.Sprintf("%d", j), func(b *testing.B) { + b.ReportAllocs() + b.SetBytes(int64(j)) + data := sd.data[:j] + for i := 0; i < b.N; i++ { + Fields(data) + } + }) + } + }) } } func BenchmarkFieldsFunc(b *testing.B) { - b.SetBytes(int64(len(fieldsInput))) - for i := 0; i < b.N; i++ { - FieldsFunc(fieldsInput, unicode.IsSpace) + for _, sd := range stringdata { + b.Run(sd.name, func(b *testing.B) { + for j := 1 << 4; j <= 1<<20; j <<= 4 { + b.Run(fmt.Sprintf("%d", j), func(b *testing.B) { + b.ReportAllocs() + b.SetBytes(int64(j)) + data := sd.data[:j] + for i := 0; i < b.N; i++ { + FieldsFunc(data, unicode.IsSpace) + } + }) + } + }) } } -func BenchmarkSplit1(b *testing.B) { +func BenchmarkSplitEmptySeparator(b *testing.B) { for i := 0; i < b.N; i++ { Split(benchInputHard, "") } } -func BenchmarkSplit2(b *testing.B) { +func BenchmarkSplitSingleByteSeparator(b *testing.B) { for i := 0; i < b.N; i++ { Split(benchInputHard, "/") } } -func BenchmarkSplit3(b *testing.B) { +func BenchmarkSplitMultiByteSeparator(b *testing.B) { for i := 0; i < b.N; i++ { Split(benchInputHard, "hello") } } +func BenchmarkSplitNSingleByteSeparator(b *testing.B) { + for i := 0; i < b.N; i++ { + SplitN(benchInputHard, "/", 10) + } +} + +func BenchmarkSplitNMultiByteSeparator(b *testing.B) { + for i := 0; i < b.N; i++ { + SplitN(benchInputHard, "hello", 10) + } +} + func BenchmarkRepeat(b *testing.B) { for i := 0; i < b.N; i++ { Repeat("-", 80) diff --git a/libgo/go/sync/atomic/atomic_test.go b/libgo/go/sync/atomic/atomic_test.go index 6d0831c3f9d..17baccb4683 100644 --- a/libgo/go/sync/atomic/atomic_test.go +++ b/libgo/go/sync/atomic/atomic_test.go @@ -953,16 +953,20 @@ func hammerSwapUint64(addr *uint64, count int) { } } +const arch32 = unsafe.Sizeof(uintptr(0)) == 4 + func hammerSwapUintptr64(uaddr *uint64, count int) { // only safe when uintptr is 64-bit. // not called on 32-bit systems. - addr := (*uintptr)(unsafe.Pointer(uaddr)) - seed := int(uintptr(unsafe.Pointer(&count))) - for i := 0; i < count; i++ { - new := uintptr(seed+i)<<32 | uintptr(seed+i)<<32>>32 - old := SwapUintptr(addr, new) - if old>>32 != old<<32>>32 { - panic(fmt.Sprintf("SwapUintptr is not atomic: %v", old)) + if !arch32 { + addr := (*uintptr)(unsafe.Pointer(uaddr)) + seed := int(uintptr(unsafe.Pointer(&count))) + for i := 0; i < count; i++ { + new := uintptr(seed+i)<<32 | uintptr(seed+i)<<32>>32 + old := SwapUintptr(addr, new) + if old>>32 != old<<32>>32 { + panic(fmt.Sprintf("SwapUintptr is not atomic: %v", old)) + } } } } @@ -1116,8 +1120,6 @@ func hammerStoreLoadUint64(t *testing.T, paddr unsafe.Pointer) { func hammerStoreLoadUintptr(t *testing.T, paddr unsafe.Pointer) { addr := (*uintptr)(paddr) - var test64 uint64 = 1 << 50 - arch32 := uintptr(test64) == 0 v := LoadUintptr(addr) new := v if arch32 { @@ -1144,8 +1146,6 @@ func hammerStoreLoadUintptr(t *testing.T, paddr unsafe.Pointer) { func hammerStoreLoadPointer(t *testing.T, paddr unsafe.Pointer) { addr := (*unsafe.Pointer)(paddr) - var test64 uint64 = 1 << 50 - arch32 := uintptr(test64) == 0 v := uintptr(LoadPointer(addr)) new := v if arch32 { @@ -1398,7 +1398,7 @@ func TestUnaligned64(t *testing.T) { switch runtime.GOARCH { default: - if unsafe.Sizeof(int(0)) != 4 { + if !arch32 { t.Skip("test only runs on 32-bit systems") } case "amd64p32": diff --git a/libgo/go/sync/atomic/doc.go b/libgo/go/sync/atomic/doc.go index 302ff43070a..7c007d7a150 100644 --- a/libgo/go/sync/atomic/doc.go +++ b/libgo/go/sync/atomic/doc.go @@ -48,8 +48,8 @@ import ( // On non-Linux ARM, the 64-bit functions use instructions unavailable before the ARMv6k core. // // On both ARM and x86-32, it is the caller's responsibility to arrange for 64-bit -// alignment of 64-bit words accessed atomically. The first word in a global -// variable or in an allocated struct or slice can be relied upon to be +// alignment of 64-bit words accessed atomically. The first word in a +// variable or in an allocated struct, array, or slice can be relied upon to be // 64-bit aligned. // SwapInt32 atomically stores new into *addr and returns the previous *addr value. diff --git a/libgo/go/sync/atomic/value.go b/libgo/go/sync/atomic/value.go index 30abf726344..1fc1f681f20 100644 --- a/libgo/go/sync/atomic/value.go +++ b/libgo/go/sync/atomic/value.go @@ -9,7 +9,6 @@ import ( ) // A Value provides an atomic load and store of a consistently typed value. -// Values can be created as part of other data structures. // The zero value for a Value returns nil from Load. // Once Store has been called, a Value must not be copied. // diff --git a/libgo/go/sync/cond.go b/libgo/go/sync/cond.go index c070d9d84ef..14e2f6b24d4 100644 --- a/libgo/go/sync/cond.go +++ b/libgo/go/sync/cond.go @@ -17,7 +17,6 @@ import ( // which must be held when changing the condition and // when calling the Wait method. // -// A Cond can be created as part of other structures. // A Cond must not be copied after first use. type Cond struct { noCopy noCopy diff --git a/libgo/go/sync/export_test.go b/libgo/go/sync/export_test.go index 6ed38dad89d..669076efad3 100644 --- a/libgo/go/sync/export_test.go +++ b/libgo/go/sync/export_test.go @@ -7,3 +7,5 @@ package sync // Export for testing. var Runtime_Semacquire = runtime_Semacquire var Runtime_Semrelease = runtime_Semrelease +var Runtime_procPin = runtime_procPin +var Runtime_procUnpin = runtime_procUnpin diff --git a/libgo/go/sync/map.go b/libgo/go/sync/map.go new file mode 100644 index 00000000000..083f4a563f8 --- /dev/null +++ b/libgo/go/sync/map.go @@ -0,0 +1,375 @@ +// Copyright 2016 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package sync + +import ( + "sync/atomic" + "unsafe" +) + +// Map is a concurrent map with amortized-constant-time loads, stores, and deletes. +// It is safe for multiple goroutines to call a Map's methods concurrently. +// +// It is optimized for use in concurrent loops with keys that are +// stable over time, and either few steady-state stores, or stores +// localized to one goroutine per key. +// +// For use cases that do not share these attributes, it will likely have +// comparable or worse performance and worse type safety than an ordinary +// map paired with a read-write mutex. +// +// The zero Map is valid and empty. +// +// A Map must not be copied after first use. +type Map struct { + mu Mutex + + // read contains the portion of the map's contents that are safe for + // concurrent access (with or without mu held). + // + // The read field itself is always safe to load, but must only be stored with + // mu held. + // + // Entries stored in read may be updated concurrently without mu, but updating + // a previously-expunged entry requires that the entry be copied to the dirty + // map and unexpunged with mu held. + read atomic.Value // readOnly + + // dirty contains the portion of the map's contents that require mu to be + // held. To ensure that the dirty map can be promoted to the read map quickly, + // it also includes all of the non-expunged entries in the read map. + // + // Expunged entries are not stored in the dirty map. An expunged entry in the + // clean map must be unexpunged and added to the dirty map before a new value + // can be stored to it. + // + // If the dirty map is nil, the next write to the map will initialize it by + // making a shallow copy of the clean map, omitting stale entries. + dirty map[interface{}]*entry + + // misses counts the number of loads since the read map was last updated that + // needed to lock mu to determine whether the key was present. + // + // Once enough misses have occurred to cover the cost of copying the dirty + // map, the dirty map will be promoted to the read map (in the unamended + // state) and the next store to the map will make a new dirty copy. + misses int +} + +// readOnly is an immutable struct stored atomically in the Map.read field. +type readOnly struct { + m map[interface{}]*entry + amended bool // true if the dirty map contains some key not in m. +} + +// expunged is an arbitrary pointer that marks entries which have been deleted +// from the dirty map. +var expunged = unsafe.Pointer(new(interface{})) + +// An entry is a slot in the map corresponding to a particular key. +type entry struct { + // p points to the interface{} value stored for the entry. + // + // If p == nil, the entry has been deleted and m.dirty == nil. + // + // If p == expunged, the entry has been deleted, m.dirty != nil, and the entry + // is missing from m.dirty. + // + // Otherwise, the entry is valid and recorded in m.read.m[key] and, if m.dirty + // != nil, in m.dirty[key]. + // + // An entry can be deleted by atomic replacement with nil: when m.dirty is + // next created, it will atomically replace nil with expunged and leave + // m.dirty[key] unset. + // + // An entry's associated value can be updated by atomic replacement, provided + // p != expunged. If p == expunged, an entry's associated value can be updated + // only after first setting m.dirty[key] = e so that lookups using the dirty + // map find the entry. + p unsafe.Pointer // *interface{} +} + +func newEntry(i interface{}) *entry { + return &entry{p: unsafe.Pointer(&i)} +} + +// Load returns the value stored in the map for a key, or nil if no +// value is present. +// The ok result indicates whether value was found in the map. +func (m *Map) Load(key interface{}) (value interface{}, ok bool) { + read, _ := m.read.Load().(readOnly) + e, ok := read.m[key] + if !ok && read.amended { + m.mu.Lock() + // Avoid reporting a spurious miss if m.dirty got promoted while we were + // blocked on m.mu. (If further loads of the same key will not miss, it's + // not worth copying the dirty map for this key.) + read, _ = m.read.Load().(readOnly) + e, ok = read.m[key] + if !ok && read.amended { + e, ok = m.dirty[key] + // Regardless of whether the entry was present, record a miss: this key + // will take the slow path until the dirty map is promoted to the read + // map. + m.missLocked() + } + m.mu.Unlock() + } + if !ok { + return nil, false + } + return e.load() +} + +func (e *entry) load() (value interface{}, ok bool) { + p := atomic.LoadPointer(&e.p) + if p == nil || p == expunged { + return nil, false + } + return *(*interface{})(p), true +} + +// Store sets the value for a key. +func (m *Map) Store(key, value interface{}) { + read, _ := m.read.Load().(readOnly) + if e, ok := read.m[key]; ok && e.tryStore(&value) { + return + } + + m.mu.Lock() + read, _ = m.read.Load().(readOnly) + if e, ok := read.m[key]; ok { + if e.unexpungeLocked() { + // The entry was previously expunged, which implies that there is a + // non-nil dirty map and this entry is not in it. + m.dirty[key] = e + } + e.storeLocked(&value) + } else if e, ok := m.dirty[key]; ok { + e.storeLocked(&value) + } else { + if !read.amended { + // We're adding the first new key to the dirty map. + // Make sure it is allocated and mark the read-only map as incomplete. + m.dirtyLocked() + m.read.Store(readOnly{m: read.m, amended: true}) + } + m.dirty[key] = newEntry(value) + } + m.mu.Unlock() +} + +// tryStore stores a value if the entry has not been expunged. +// +// If the entry is expunged, tryStore returns false and leaves the entry +// unchanged. +func (e *entry) tryStore(i *interface{}) bool { + p := atomic.LoadPointer(&e.p) + if p == expunged { + return false + } + for { + if atomic.CompareAndSwapPointer(&e.p, p, unsafe.Pointer(i)) { + return true + } + p = atomic.LoadPointer(&e.p) + if p == expunged { + return false + } + } +} + +// unexpungeLocked ensures that the entry is not marked as expunged. +// +// If the entry was previously expunged, it must be added to the dirty map +// before m.mu is unlocked. +func (e *entry) unexpungeLocked() (wasExpunged bool) { + return atomic.CompareAndSwapPointer(&e.p, expunged, nil) +} + +// storeLocked unconditionally stores a value to the entry. +// +// The entry must be known not to be expunged. +func (e *entry) storeLocked(i *interface{}) { + atomic.StorePointer(&e.p, unsafe.Pointer(i)) +} + +// LoadOrStore returns the existing value for the key if present. +// Otherwise, it stores and returns the given value. +// The loaded result is true if the value was loaded, false if stored. +func (m *Map) LoadOrStore(key, value interface{}) (actual interface{}, loaded bool) { + // Avoid locking if it's a clean hit. + read, _ := m.read.Load().(readOnly) + if e, ok := read.m[key]; ok { + actual, loaded, ok := e.tryLoadOrStore(value) + if ok { + return actual, loaded + } + } + + m.mu.Lock() + read, _ = m.read.Load().(readOnly) + if e, ok := read.m[key]; ok { + if e.unexpungeLocked() { + m.dirty[key] = e + } + actual, loaded, _ = e.tryLoadOrStore(value) + } else if e, ok := m.dirty[key]; ok { + actual, loaded, _ = e.tryLoadOrStore(value) + m.missLocked() + } else { + if !read.amended { + // We're adding the first new key to the dirty map. + // Make sure it is allocated and mark the read-only map as incomplete. + m.dirtyLocked() + m.read.Store(readOnly{m: read.m, amended: true}) + } + m.dirty[key] = newEntry(value) + actual, loaded = value, false + } + m.mu.Unlock() + + return actual, loaded +} + +// tryLoadOrStore atomically loads or stores a value if the entry is not +// expunged. +// +// If the entry is expunged, tryLoadOrStore leaves the entry unchanged and +// returns with ok==false. +func (e *entry) tryLoadOrStore(i interface{}) (actual interface{}, loaded, ok bool) { + p := atomic.LoadPointer(&e.p) + if p == expunged { + return nil, false, false + } + if p != nil { + return *(*interface{})(p), true, true + } + + // Copy the interface after the first load to make this method more amenable + // to escape analysis: if we hit the "load" path or the entry is expunged, we + // shouldn't bother heap-allocating. + ic := i + for { + if atomic.CompareAndSwapPointer(&e.p, nil, unsafe.Pointer(&ic)) { + return i, false, true + } + p = atomic.LoadPointer(&e.p) + if p == expunged { + return nil, false, false + } + if p != nil { + return *(*interface{})(p), true, true + } + } +} + +// Delete deletes the value for a key. +func (m *Map) Delete(key interface{}) { + read, _ := m.read.Load().(readOnly) + e, ok := read.m[key] + if !ok && read.amended { + m.mu.Lock() + read, _ = m.read.Load().(readOnly) + e, ok = read.m[key] + if !ok && read.amended { + delete(m.dirty, key) + } + m.mu.Unlock() + } + if ok { + e.delete() + } +} + +func (e *entry) delete() (hadValue bool) { + for { + p := atomic.LoadPointer(&e.p) + if p == nil || p == expunged { + return false + } + if atomic.CompareAndSwapPointer(&e.p, p, nil) { + return true + } + } +} + +// Range calls f sequentially for each key and value present in the map. +// If f returns false, range stops the iteration. +// +// Range does not necessarily correspond to any consistent snapshot of the Map's +// contents: no key will be visited more than once, but if the value for any key +// is stored or deleted concurrently, Range may reflect any mapping for that key +// from any point during the Range call. +// +// Range may be O(N) with the number of elements in the map even if f returns +// false after a constant number of calls. +func (m *Map) Range(f func(key, value interface{}) bool) { + // We need to be able to iterate over all of the keys that were already + // present at the start of the call to Range. + // If read.amended is false, then read.m satisfies that property without + // requiring us to hold m.mu for a long time. + read, _ := m.read.Load().(readOnly) + if read.amended { + // m.dirty contains keys not in read.m. Fortunately, Range is already O(N) + // (assuming the caller does not break out early), so a call to Range + // amortizes an entire copy of the map: we can promote the dirty copy + // immediately! + m.mu.Lock() + read, _ = m.read.Load().(readOnly) + if read.amended { + read = readOnly{m: m.dirty} + m.read.Store(read) + m.dirty = nil + m.misses = 0 + } + m.mu.Unlock() + } + + for k, e := range read.m { + v, ok := e.load() + if !ok { + continue + } + if !f(k, v) { + break + } + } +} + +func (m *Map) missLocked() { + m.misses++ + if m.misses < len(m.dirty) { + return + } + m.read.Store(readOnly{m: m.dirty}) + m.dirty = nil + m.misses = 0 +} + +func (m *Map) dirtyLocked() { + if m.dirty != nil { + return + } + + read, _ := m.read.Load().(readOnly) + m.dirty = make(map[interface{}]*entry, len(read.m)) + for k, e := range read.m { + if !e.tryExpungeLocked() { + m.dirty[k] = e + } + } +} + +func (e *entry) tryExpungeLocked() (isExpunged bool) { + p := atomic.LoadPointer(&e.p) + for p == nil { + if atomic.CompareAndSwapPointer(&e.p, nil, expunged) { + return true + } + p = atomic.LoadPointer(&e.p) + } + return p == expunged +} diff --git a/libgo/go/sync/map_bench_test.go b/libgo/go/sync/map_bench_test.go new file mode 100644 index 00000000000..e6a8badddba --- /dev/null +++ b/libgo/go/sync/map_bench_test.go @@ -0,0 +1,215 @@ +// Copyright 2016 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package sync_test + +import ( + "fmt" + "reflect" + "sync" + "sync/atomic" + "testing" +) + +type bench struct { + setup func(*testing.B, mapInterface) + perG func(b *testing.B, pb *testing.PB, i int, m mapInterface) +} + +func benchMap(b *testing.B, bench bench) { + for _, m := range [...]mapInterface{&DeepCopyMap{}, &RWMutexMap{}, &sync.Map{}} { + b.Run(fmt.Sprintf("%T", m), func(b *testing.B) { + m = reflect.New(reflect.TypeOf(m).Elem()).Interface().(mapInterface) + if bench.setup != nil { + bench.setup(b, m) + } + + b.ResetTimer() + + var i int64 + b.RunParallel(func(pb *testing.PB) { + id := int(atomic.AddInt64(&i, 1) - 1) + bench.perG(b, pb, id*b.N, m) + }) + }) + } +} + +func BenchmarkLoadMostlyHits(b *testing.B) { + const hits, misses = 1023, 1 + + benchMap(b, bench{ + setup: func(_ *testing.B, m mapInterface) { + for i := 0; i < hits; i++ { + m.LoadOrStore(i, i) + } + // Prime the map to get it into a steady state. + for i := 0; i < hits*2; i++ { + m.Load(i % hits) + } + }, + + perG: func(b *testing.B, pb *testing.PB, i int, m mapInterface) { + for ; pb.Next(); i++ { + m.Load(i % (hits + misses)) + } + }, + }) +} + +func BenchmarkLoadMostlyMisses(b *testing.B) { + const hits, misses = 1, 1023 + + benchMap(b, bench{ + setup: func(_ *testing.B, m mapInterface) { + for i := 0; i < hits; i++ { + m.LoadOrStore(i, i) + } + // Prime the map to get it into a steady state. + for i := 0; i < hits*2; i++ { + m.Load(i % hits) + } + }, + + perG: func(b *testing.B, pb *testing.PB, i int, m mapInterface) { + for ; pb.Next(); i++ { + m.Load(i % (hits + misses)) + } + }, + }) +} + +func BenchmarkLoadOrStoreBalanced(b *testing.B) { + const hits, misses = 128, 128 + + benchMap(b, bench{ + setup: func(b *testing.B, m mapInterface) { + if _, ok := m.(*DeepCopyMap); ok { + b.Skip("DeepCopyMap has quadratic running time.") + } + for i := 0; i < hits; i++ { + m.LoadOrStore(i, i) + } + // Prime the map to get it into a steady state. + for i := 0; i < hits*2; i++ { + m.Load(i % hits) + } + }, + + perG: func(b *testing.B, pb *testing.PB, i int, m mapInterface) { + for ; pb.Next(); i++ { + j := i % (hits + misses) + if j < hits { + if _, ok := m.LoadOrStore(j, i); !ok { + b.Fatalf("unexpected miss for %v", j) + } + } else { + if v, loaded := m.LoadOrStore(i, i); loaded { + b.Fatalf("failed to store %v: existing value %v", i, v) + } + } + } + }, + }) +} + +func BenchmarkLoadOrStoreUnique(b *testing.B) { + benchMap(b, bench{ + setup: func(b *testing.B, m mapInterface) { + if _, ok := m.(*DeepCopyMap); ok { + b.Skip("DeepCopyMap has quadratic running time.") + } + }, + + perG: func(b *testing.B, pb *testing.PB, i int, m mapInterface) { + for ; pb.Next(); i++ { + m.LoadOrStore(i, i) + } + }, + }) +} + +func BenchmarkLoadOrStoreCollision(b *testing.B) { + benchMap(b, bench{ + setup: func(_ *testing.B, m mapInterface) { + m.LoadOrStore(0, 0) + }, + + perG: func(b *testing.B, pb *testing.PB, i int, m mapInterface) { + for ; pb.Next(); i++ { + m.LoadOrStore(0, 0) + } + }, + }) +} + +func BenchmarkRange(b *testing.B) { + const mapSize = 1 << 10 + + benchMap(b, bench{ + setup: func(_ *testing.B, m mapInterface) { + for i := 0; i < mapSize; i++ { + m.Store(i, i) + } + }, + + perG: func(b *testing.B, pb *testing.PB, i int, m mapInterface) { + for ; pb.Next(); i++ { + m.Range(func(_, _ interface{}) bool { return true }) + } + }, + }) +} + +// BenchmarkAdversarialAlloc tests performance when we store a new value +// immediately whenever the map is promoted to clean and otherwise load a +// unique, missing key. +// +// This forces the Load calls to always acquire the map's mutex. +func BenchmarkAdversarialAlloc(b *testing.B) { + benchMap(b, bench{ + perG: func(b *testing.B, pb *testing.PB, i int, m mapInterface) { + var stores, loadsSinceStore int64 + for ; pb.Next(); i++ { + m.Load(i) + if loadsSinceStore++; loadsSinceStore > stores { + m.LoadOrStore(i, stores) + loadsSinceStore = 0 + stores++ + } + } + }, + }) +} + +// BenchmarkAdversarialDelete tests performance when we periodically delete +// one key and add a different one in a large map. +// +// This forces the Load calls to always acquire the map's mutex and periodically +// makes a full copy of the map despite changing only one entry. +func BenchmarkAdversarialDelete(b *testing.B) { + const mapSize = 1 << 10 + + benchMap(b, bench{ + setup: func(_ *testing.B, m mapInterface) { + for i := 0; i < mapSize; i++ { + m.Store(i, i) + } + }, + + perG: func(b *testing.B, pb *testing.PB, i int, m mapInterface) { + for ; pb.Next(); i++ { + m.Load(i) + + if i%mapSize == 0 { + m.Range(func(k, _ interface{}) bool { + m.Delete(k) + return false + }) + m.Store(i, i) + } + } + }, + }) +} diff --git a/libgo/go/sync/map_reference_test.go b/libgo/go/sync/map_reference_test.go new file mode 100644 index 00000000000..9f27b07c329 --- /dev/null +++ b/libgo/go/sync/map_reference_test.go @@ -0,0 +1,151 @@ +// Copyright 2016 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package sync_test + +import ( + "sync" + "sync/atomic" +) + +// This file contains reference map implementations for unit-tests. + +// mapInterface is the interface Map implements. +type mapInterface interface { + Load(interface{}) (interface{}, bool) + Store(key, value interface{}) + LoadOrStore(key, value interface{}) (actual interface{}, loaded bool) + Delete(interface{}) + Range(func(key, value interface{}) (shouldContinue bool)) +} + +// RWMutexMap is an implementation of mapInterface using a sync.RWMutex. +type RWMutexMap struct { + mu sync.RWMutex + dirty map[interface{}]interface{} +} + +func (m *RWMutexMap) Load(key interface{}) (value interface{}, ok bool) { + m.mu.RLock() + value, ok = m.dirty[key] + m.mu.RUnlock() + return +} + +func (m *RWMutexMap) Store(key, value interface{}) { + m.mu.Lock() + if m.dirty == nil { + m.dirty = make(map[interface{}]interface{}) + } + m.dirty[key] = value + m.mu.Unlock() +} + +func (m *RWMutexMap) LoadOrStore(key, value interface{}) (actual interface{}, loaded bool) { + m.mu.Lock() + actual, loaded = m.dirty[key] + if !loaded { + actual = value + if m.dirty == nil { + m.dirty = make(map[interface{}]interface{}) + } + m.dirty[key] = value + } + m.mu.Unlock() + return actual, loaded +} + +func (m *RWMutexMap) Delete(key interface{}) { + m.mu.Lock() + delete(m.dirty, key) + m.mu.Unlock() +} + +func (m *RWMutexMap) Range(f func(key, value interface{}) (shouldContinue bool)) { + m.mu.RLock() + keys := make([]interface{}, 0, len(m.dirty)) + for k := range m.dirty { + keys = append(keys, k) + } + m.mu.RUnlock() + + for _, k := range keys { + v, ok := m.Load(k) + if !ok { + continue + } + if !f(k, v) { + break + } + } +} + +// DeepCopyMap is an implementation of mapInterface using a Mutex and +// atomic.Value. It makes deep copies of the map on every write to avoid +// acquiring the Mutex in Load. +type DeepCopyMap struct { + mu sync.Mutex + clean atomic.Value +} + +func (m *DeepCopyMap) Load(key interface{}) (value interface{}, ok bool) { + clean, _ := m.clean.Load().(map[interface{}]interface{}) + value, ok = clean[key] + return value, ok +} + +func (m *DeepCopyMap) Store(key, value interface{}) { + m.mu.Lock() + dirty := m.dirty() + dirty[key] = value + m.clean.Store(dirty) + m.mu.Unlock() +} + +func (m *DeepCopyMap) LoadOrStore(key, value interface{}) (actual interface{}, loaded bool) { + clean, _ := m.clean.Load().(map[interface{}]interface{}) + actual, loaded = clean[key] + if loaded { + return actual, loaded + } + + m.mu.Lock() + // Reload clean in case it changed while we were waiting on m.mu. + clean, _ = m.clean.Load().(map[interface{}]interface{}) + actual, loaded = clean[key] + if !loaded { + dirty := m.dirty() + dirty[key] = value + actual = value + m.clean.Store(dirty) + } + m.mu.Unlock() + return actual, loaded +} + +func (m *DeepCopyMap) Delete(key interface{}) { + m.mu.Lock() + dirty := m.dirty() + delete(dirty, key) + m.clean.Store(dirty) + m.mu.Unlock() +} + +func (m *DeepCopyMap) Range(f func(key, value interface{}) (shouldContinue bool)) { + clean, _ := m.clean.Load().(map[interface{}]interface{}) + for k, v := range clean { + if !f(k, v) { + break + } + } +} + +func (m *DeepCopyMap) dirty() map[interface{}]interface{} { + clean, _ := m.clean.Load().(map[interface{}]interface{}) + dirty := make(map[interface{}]interface{}, len(clean)+1) + for k, v := range clean { + dirty[k] = v + } + return dirty +} diff --git a/libgo/go/sync/map_test.go b/libgo/go/sync/map_test.go new file mode 100644 index 00000000000..b60a1c7bede --- /dev/null +++ b/libgo/go/sync/map_test.go @@ -0,0 +1,170 @@ +// Copyright 2016 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package sync_test + +import ( + "math/rand" + "reflect" + "runtime" + "sync" + "testing" + "testing/quick" +) + +type mapOp string + +const ( + opLoad = mapOp("Load") + opStore = mapOp("Store") + opLoadOrStore = mapOp("LoadOrStore") + opDelete = mapOp("Delete") +) + +var mapOps = [...]mapOp{opLoad, opStore, opLoadOrStore, opDelete} + +// mapCall is a quick.Generator for calls on mapInterface. +type mapCall struct { + op mapOp + k, v interface{} +} + +func (c mapCall) apply(m mapInterface) (interface{}, bool) { + switch c.op { + case opLoad: + return m.Load(c.k) + case opStore: + m.Store(c.k, c.v) + return nil, false + case opLoadOrStore: + return m.LoadOrStore(c.k, c.v) + case opDelete: + m.Delete(c.k) + return nil, false + default: + panic("invalid mapOp") + } +} + +type mapResult struct { + value interface{} + ok bool +} + +func randValue(r *rand.Rand) interface{} { + b := make([]byte, r.Intn(4)) + for i := range b { + b[i] = 'a' + byte(rand.Intn(26)) + } + return string(b) +} + +func (mapCall) Generate(r *rand.Rand, size int) reflect.Value { + c := mapCall{op: mapOps[rand.Intn(len(mapOps))], k: randValue(r)} + switch c.op { + case opStore, opLoadOrStore: + c.v = randValue(r) + } + return reflect.ValueOf(c) +} + +func applyCalls(m mapInterface, calls []mapCall) (results []mapResult, final map[interface{}]interface{}) { + for _, c := range calls { + v, ok := c.apply(m) + results = append(results, mapResult{v, ok}) + } + + final = make(map[interface{}]interface{}) + m.Range(func(k, v interface{}) bool { + final[k] = v + return true + }) + + return results, final +} + +func applyMap(calls []mapCall) ([]mapResult, map[interface{}]interface{}) { + return applyCalls(new(sync.Map), calls) +} + +func applyRWMutexMap(calls []mapCall) ([]mapResult, map[interface{}]interface{}) { + return applyCalls(new(RWMutexMap), calls) +} + +func applyDeepCopyMap(calls []mapCall) ([]mapResult, map[interface{}]interface{}) { + return applyCalls(new(DeepCopyMap), calls) +} + +func TestMapMatchesRWMutex(t *testing.T) { + if err := quick.CheckEqual(applyMap, applyRWMutexMap, nil); err != nil { + t.Error(err) + } +} + +func TestMapMatchesDeepCopy(t *testing.T) { + if err := quick.CheckEqual(applyMap, applyDeepCopyMap, nil); err != nil { + t.Error(err) + } +} + +func TestConcurrentRange(t *testing.T) { + const mapSize = 1 << 10 + + m := new(sync.Map) + for n := int64(1); n <= mapSize; n++ { + m.Store(n, int64(n)) + } + + done := make(chan struct{}) + var wg sync.WaitGroup + defer func() { + close(done) + wg.Wait() + }() + for g := int64(runtime.GOMAXPROCS(0)); g > 0; g-- { + r := rand.New(rand.NewSource(g)) + wg.Add(1) + go func(g int64) { + defer wg.Done() + for i := int64(0); ; i++ { + select { + case <-done: + return + default: + } + for n := int64(1); n < mapSize; n++ { + if r.Int63n(mapSize) == 0 { + m.Store(n, n*i*g) + } else { + m.Load(n) + } + } + } + }(g) + } + + iters := 1 << 10 + if testing.Short() { + iters = 16 + } + for n := iters; n > 0; n-- { + seen := make(map[int64]bool, mapSize) + + m.Range(func(ki, vi interface{}) bool { + k, v := ki.(int64), vi.(int64) + if v%k != 0 { + t.Fatalf("while Storing multiples of %v, Range saw value %v", k, v) + } + if seen[k] { + t.Fatalf("Range visited key %v twice", k) + } + seen[k] = true + return true + }) + + if len(seen) != mapSize { + t.Fatalf("Range visited %v elements of %v-element Map", len(seen), mapSize) + } + } +} diff --git a/libgo/go/sync/mutex.go b/libgo/go/sync/mutex.go index 8c9366f4fe1..1232c629b18 100644 --- a/libgo/go/sync/mutex.go +++ b/libgo/go/sync/mutex.go @@ -19,8 +19,7 @@ import ( func throw(string) // provided by runtime // A Mutex is a mutual exclusion lock. -// Mutexes can be created as part of other structures; -// the zero value for a Mutex is an unlocked mutex. +// The zero value for a Mutex is an unlocked mutex. // // A Mutex must not be copied after first use. type Mutex struct { @@ -37,7 +36,34 @@ type Locker interface { const ( mutexLocked = 1 << iota // mutex is locked mutexWoken + mutexStarving mutexWaiterShift = iota + + // Mutex fairness. + // + // Mutex can be in 2 modes of operations: normal and starvation. + // In normal mode waiters are queued in FIFO order, but a woken up waiter + // does not own the mutex and competes with new arriving goroutines over + // the ownership. New arriving goroutines have an advantage -- they are + // already running on CPU and there can be lots of them, so a woken up + // waiter has good chances of losing. In such case it is queued at front + // of the wait queue. If a waiter fails to acquire the mutex for more than 1ms, + // it switches mutex to the starvation mode. + // + // In starvation mode ownership of the mutex is directly handed off from + // the unlocking goroutine to the waiter at the front of the queue. + // New arriving goroutines don't try to acquire the mutex even if it appears + // to be unlocked, and don't try to spin. Instead they queue themselves at + // the tail of the wait queue. + // + // If a waiter receives ownership of the mutex and sees that either + // (1) it is the last waiter in the queue, or (2) it waited for less than 1 ms, + // it switches mutex back to normal operation mode. + // + // Normal mode has considerably better performance as a goroutine can acquire + // a mutex several times in a row even if there are blocked waiters. + // Starvation mode is important to prevent pathological cases of tail latency. + starvationThresholdNs = 1e6 ) // Lock locks m. @@ -52,41 +78,86 @@ func (m *Mutex) Lock() { return } + var waitStartTime int64 + starving := false awoke := false iter := 0 + old := m.state for { - old := m.state - new := old | mutexLocked - if old&mutexLocked != 0 { - if runtime_canSpin(iter) { - // Active spinning makes sense. - // Try to set mutexWoken flag to inform Unlock - // to not wake other blocked goroutines. - if !awoke && old&mutexWoken == 0 && old>>mutexWaiterShift != 0 && - atomic.CompareAndSwapInt32(&m.state, old, old|mutexWoken) { - awoke = true - } - runtime_doSpin() - iter++ - continue + // Don't spin in starvation mode, ownership is handed off to waiters + // so we won't be able to acquire the mutex anyway. + if old&(mutexLocked|mutexStarving) == mutexLocked && runtime_canSpin(iter) { + // Active spinning makes sense. + // Try to set mutexWoken flag to inform Unlock + // to not wake other blocked goroutines. + if !awoke && old&mutexWoken == 0 && old>>mutexWaiterShift != 0 && + atomic.CompareAndSwapInt32(&m.state, old, old|mutexWoken) { + awoke = true } - new = old + 1< starvationThresholdNs + old = m.state + if old&mutexStarving != 0 { + // If this goroutine was woken and mutex is in starvation mode, + // ownership was handed off to us but mutex is in somewhat + // inconsistent state: mutexLocked is not set and we are still + // accounted as waiter. Fix that. + if old&(mutexLocked|mutexWoken) != 0 || old>>mutexWaiterShift == 0 { + panic("sync: inconsistent mutex state") + } + delta := int32(mutexLocked - 1<>mutexWaiterShift == 1 { + // Exit starvation mode. + // Critical to do it here and consider wait time. + // Starvation mode is so inefficient, that two goroutines + // can go lock-step infinitely once they switch mutex + // to starvation mode. + delta -= mutexStarving + } + atomic.AddInt32(&m.state, delta) break } - runtime_SemacquireMutex(&m.sema) awoke = true iter = 0 + } else { + old = m.state } } @@ -110,22 +181,33 @@ func (m *Mutex) Unlock() { // Fast path: drop lock bit. new := atomic.AddInt32(&m.state, -mutexLocked) if (new+mutexLocked)&mutexLocked == 0 { - throw("sync: unlock of unlocked mutex") + panic("sync: unlock of unlocked mutex") } - - old := new - for { - // If there are no waiters or a goroutine has already - // been woken or grabbed the lock, no need to wake anyone. - if old>>mutexWaiterShift == 0 || old&(mutexLocked|mutexWoken) != 0 { - return - } - // Grab the right to wake someone. - new = (old - 1<>mutexWaiterShift == 0 || old&(mutexLocked|mutexWoken|mutexStarving) != 0 { + return + } + // Grab the right to wake someone. + new = (old - 1< 0 { + gids := make([]Gid_t, ngroups) for i, v := range cred.Groups { - groups[i] = Gid_t(v) + gids[i] = Gid_t(v) } - err2 := setgroups(ngroups, &groups[0]) + groups = &gids[0] + } + if !cred.NoSetGroups { + err2 := setgroups(ngroups, groups) if err2 == nil { err1 = 0 } else { err1 = err2.(Errno) } - } - if err1 != 0 { - goto childerror + if err1 != 0 { + goto childerror + } } err2 := Setgid(int(cred.Gid)) if err2 != nil { @@ -255,17 +255,3 @@ childerror: raw_exit(253) } } - -// Try to open a pipe with O_CLOEXEC set on both file descriptors. -func forkExecPipe(p []int) error { - err := Pipe(p) - if err != nil { - return err - } - _, err = fcntl(p[0], F_SETFD, FD_CLOEXEC) - if err != nil { - return err - } - _, err = fcntl(p[1], F_SETFD, FD_CLOEXEC) - return err -} diff --git a/libgo/go/syscall/exec_freebsd.go b/libgo/go/syscall/exec_freebsd.go new file mode 100644 index 00000000000..4ed32c0614f --- /dev/null +++ b/libgo/go/syscall/exec_freebsd.go @@ -0,0 +1,25 @@ +// Copyright 2017 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package syscall + +func forkExecPipe(p []int) error { + err := Pipe2(p, O_CLOEXEC) + if err == nil { + return nil + } + + // FreeBSD 9 fallback. + // TODO: remove this for Go 1.10 per Issue 19072 + err = Pipe(p) + if err != nil { + return err + } + _, err = fcntl(p[0], F_SETFD, FD_CLOEXEC) + if err != nil { + return err + } + _, err = fcntl(p[1], F_SETFD, FD_CLOEXEC) + return err +} diff --git a/libgo/go/syscall/exec_linux.go b/libgo/go/syscall/exec_linux.go index 8d6467a8720..6e2f83e9416 100644 --- a/libgo/go/syscall/exec_linux.go +++ b/libgo/go/syscall/exec_linux.go @@ -13,6 +13,12 @@ import ( //sysnb raw_prctl(option int, arg2 int, arg3 int, arg4 int, arg5 int) (ret int, err Errno) //prctl(option _C_int, arg2 _C_long, arg3 _C_long, arg4 _C_long, arg5 _C_long) _C_int +//sysnb rawUnshare(flags int) (err Errno) +//unshare(flags _C_int) _C_int + +//sysnb rawMount(source *byte, target *byte, fstype *byte, flags uintptr, data *byte) (err Errno) +//mount(source *byte, target *byte, fstype *byte, flags _C_long, data *byte) _C_int + // SysProcIDMap holds Container ID to Host ID mappings used for User Namespaces in Linux. // See user_namespaces(7). type SysProcIDMap struct { @@ -42,11 +48,18 @@ type SysProcAttr struct { // This parameter is no-op if GidMappings == nil. Otherwise for unprivileged // users this should be set to false for mappings work. GidMappingsEnableSetgroups bool + AmbientCaps []uintptr // Ambient capabilities (Linux only) } +var ( + none = [...]byte{'n', 'o', 'n', 'e', 0} + slash = [...]byte{'/', 0} +) + // Implemented in runtime package. func runtime_BeforeFork() func runtime_AfterFork() +func runtime_AfterForkInChild() // Implemented in clone_linux.c func rawClone(flags _C_ulong, child_stack *byte, ptid *Pid_t, ctid *Pid_t, regs unsafe.Pointer) _C_long @@ -62,16 +75,62 @@ func rawClone(flags _C_ulong, child_stack *byte, ptid *Pid_t, ctid *Pid_t, regs // functions that do not grow the stack. //go:norace func forkAndExecInChild(argv0 *byte, argv, envv []*byte, chroot, dir *byte, attr *ProcAttr, sys *SysProcAttr, pipe int) (pid int, err Errno) { + // Set up and fork. This returns immediately in the parent or + // if there's an error. + r1, err1, p, locked := forkAndExecInChild1(argv0, argv, envv, chroot, dir, attr, sys, pipe) + if locked { + runtime_AfterFork() + } + if err1 != 0 { + return 0, err1 + } + + // parent; return PID + pid = int(r1) + + if sys.UidMappings != nil || sys.GidMappings != nil { + Close(p[0]) + err := writeUidGidMappings(pid, sys) + var err2 Errno + if err != nil { + err2 = err.(Errno) + } + RawSyscall(SYS_WRITE, uintptr(p[1]), uintptr(unsafe.Pointer(&err2)), unsafe.Sizeof(err2)) + Close(p[1]) + } + + return pid, 0 +} + +// forkAndExecInChild1 implements the body of forkAndExecInChild up to +// the parent's post-fork path. This is a separate function so we can +// separate the child's and parent's stack frames if we're using +// vfork. +// +// This is go:noinline because the point is to keep the stack frames +// of this and forkAndExecInChild separate. +// +//go:noinline +//go:norace +func forkAndExecInChild1(argv0 *byte, argv, envv []*byte, chroot, dir *byte, attr *ProcAttr, sys *SysProcAttr, pipe int) (r1 uintptr, err1 Errno, p [2]int, locked bool) { + // Defined in linux/prctl.h starting with Linux 4.3. + const ( + PR_CAP_AMBIENT = 0x2f + PR_CAP_AMBIENT_RAISE = 0x2 + ) + + // vfork requires that the child not touch any of the parent's + // active stack frames. Hence, the child does all post-fork + // processing in this stack frame and never returns, while the + // parent returns immediately from this frame and does all + // post-fork processing in the outer frame. // Declare all variables at top in case any // declarations require heap allocation (e.g., err1). var ( - r1 uintptr - r2 _C_long - err1 Errno err2 Errno nextfd int i int - p [2]int + r2 int ) // Record parent PID so child can test if it has died. @@ -94,39 +153,42 @@ func forkAndExecInChild(argv0 *byte, argv, envv []*byte, chroot, dir *byte, attr // synchronizing writing of User ID/Group ID mappings. if sys.UidMappings != nil || sys.GidMappings != nil { if err := forkExecPipe(p[:]); err != nil { - return 0, err.(Errno) + err1 = err.(Errno) + return } } // About to call fork. // No more allocation or calls of non-assembly functions. runtime_BeforeFork() - r2 = rawClone(_C_ulong(uintptr(SIGCHLD)|sys.Cloneflags), nil, nil, nil, unsafe.Pointer(nil)) + locked = true + r2 = int(rawClone(_C_ulong(uintptr(SIGCHLD)|sys.Cloneflags), nil, nil, nil, unsafe.Pointer(nil))) if r2 < 0 { - runtime_AfterFork() - return 0, GetErrno() + err1 = GetErrno() } - if r2 != 0 { - // parent; return PID - runtime_AfterFork() - pid = int(r2) - - if sys.UidMappings != nil || sys.GidMappings != nil { - Close(p[0]) - err := writeUidGidMappings(pid, sys) - if err != nil { - err2 = err.(Errno) - } - RawSyscall(SYS_WRITE, uintptr(p[1]), uintptr(unsafe.Pointer(&err2)), unsafe.Sizeof(err2)) - Close(p[1]) - } - - return pid, 0 + // If we're in the parent, we must return immediately + // so we're not in the same stack frame as the child. + // This can at most use the return PC, which the child + // will not modify, and the results of + // rawVforkSyscall, which must have been written after + // the child was replaced. + r1 = uintptr(r2) + return } // Fork succeeded, now in child. + runtime_AfterForkInChild() + + // Enable the "keep capabilities" flag to set ambient capabilities later. + if len(sys.AmbientCaps) > 0 { + _, _, err1 = RawSyscall6(SYS_PRCTL, PR_SET_KEEPCAPS, 1, 0, 0, 0, 0) + if err1 != 0 { + goto childerror + } + } + // Wait for User ID/Group ID mappings to be written. if sys.UidMappings != nil || sys.GidMappings != nil { if _, _, err1 = RawSyscall(SYS_CLOSE, uintptr(p[1]), 0, 0); err1 != 0 { @@ -184,17 +246,30 @@ func forkAndExecInChild(argv0 *byte, argv, envv []*byte, chroot, dir *byte, attr } } - // Chroot - if chroot != nil { - err1 = raw_chroot(chroot) + // Unshare + if sys.Unshareflags != 0 { + err1 = rawUnshare(int(sys.Unshareflags)) if err1 != 0 { goto childerror } + // The unshare system call in Linux doesn't unshare mount points + // mounted with --shared. Systemd mounts / with --shared. For a + // long discussion of the pros and cons of this see debian bug 739593. + // The Go model of unsharing is more like Plan 9, where you ask + // to unshare and the namespaces are unconditionally unshared. + // To make this model work we must further mark / as MS_PRIVATE. + // This is what the standard unshare command does. + if sys.Unshareflags&CLONE_NEWNS == CLONE_NEWNS { + err1 = rawMount(&none[0], &slash[0], nil, MS_REC|MS_PRIVATE, nil) + if err1 != 0 { + goto childerror + } + } } - // Unshare - if sys.Unshareflags != 0 { - _, _, err1 = RawSyscall(SYS_UNSHARE, sys.Unshareflags, 0, 0) + // Chroot + if chroot != nil { + err1 = raw_chroot(chroot) if err1 != 0 { goto childerror } @@ -207,10 +282,7 @@ func forkAndExecInChild(argv0 *byte, argv, envv []*byte, chroot, dir *byte, attr if ngroups > 0 { groups = unsafe.Pointer(&cred.Groups[0]) } - // Don't call setgroups in case of user namespace, gid mappings - // and disabled setgroups, because otherwise unprivileged user namespace - // will fail with any non-empty SysProcAttr.Credential. - if !(sys.GidMappings != nil && !sys.GidMappingsEnableSetgroups && ngroups == 0) { + if !(sys.GidMappings != nil && !sys.GidMappingsEnableSetgroups && ngroups == 0) && !cred.NoSetGroups { err1 = raw_setgroups(ngroups, groups) if err1 != 0 { goto childerror @@ -226,6 +298,13 @@ func forkAndExecInChild(argv0 *byte, argv, envv []*byte, chroot, dir *byte, attr } } + for _, c := range sys.AmbientCaps { + _, _, err1 = RawSyscall6(SYS_PRCTL, PR_CAP_AMBIENT, uintptr(PR_CAP_AMBIENT_RAISE), c, 0, 0, 0) + if err1 != 0 { + goto childerror + } + } + // Chdir if dir != nil { err1 = raw_chdir(dir) @@ -321,7 +400,7 @@ func forkAndExecInChild(argv0 *byte, argv, envv []*byte, chroot, dir *byte, attr // Set the controlling TTY to Ctty if sys.Setctty { - _, err1 = raw_ioctl(sys.Ctty, TIOCSCTTY, 0) + _, err1 = raw_ioctl(sys.Ctty, TIOCSCTTY, sys.Ctty) if err1 != 0 { goto childerror } diff --git a/libgo/go/syscall/exec_linux_test.go b/libgo/go/syscall/exec_linux_test.go index 7a4b5717600..114deec5bbe 100644 --- a/libgo/go/syscall/exec_linux_test.go +++ b/libgo/go/syscall/exec_linux_test.go @@ -7,12 +7,20 @@ package syscall_test import ( + "flag" + "fmt" + "internal/testenv" + "io" "io/ioutil" "os" "os/exec" + "os/user" + "path/filepath" + "strconv" "strings" "syscall" "testing" + "unsafe" ) // Check if we are in a chroot by checking if the inode of / is @@ -49,6 +57,14 @@ func checkUserNS(t *testing.T) { t.Skip("kernel prohibits user namespace in unprivileged process") } } + // On Centos 7 make sure they set the kernel parameter user_namespace=1 + // See issue 16283 and 20796. + if _, err := os.Stat("/sys/module/user_namespace/parameters/enable"); err == nil { + buf, _ := ioutil.ReadFile("/sys/module/user_namespace/parameters/enabled") + if !strings.HasPrefix(string(buf), "Y") { + t.Skip("kernel doesn't support user namespaces") + } + } // When running under the Go continuous build, skip tests for // now when under Kubernetes. (where things are root but not quite) // Both of these are our own environment variables. @@ -174,6 +190,12 @@ func TestUnshare(t *testing.T) { } out, err := cmd.CombinedOutput() if err != nil { + if strings.Contains(err.Error(), "operation not permitted") { + // Issue 17206: despite all the checks above, + // this still reportedly fails for some users. + // (older kernels?). Just skip. + t.Skip("skipping due to permission error") + } t.Fatalf("Cmd failed with err %v, output: %s", err, out) } @@ -205,9 +227,10 @@ func TestGroupCleanup(t *testing.T) { t.Fatalf("Cmd failed with err %v, output: %s", err, out) } strOut := strings.TrimSpace(string(out)) - expected := "uid=0(root) gid=0(root) groups=0(root)" + expected := "uid=0(root) gid=0(root)" // Just check prefix because some distros reportedly output a // context parameter; see https://golang.org/issue/16224. + // Alpine does not output groups; see https://golang.org/issue/19938. if !strings.HasPrefix(strOut, expected) { t.Errorf("id command output: %q, expected prefix: %q", strOut, expected) } @@ -245,6 +268,7 @@ func TestGroupCleanupUserNamespace(t *testing.T) { "uid=0(root) gid=0(root) groups=0(root),65534(nobody)", "uid=0(root) gid=0(root) groups=0(root),65534(nogroup)", "uid=0(root) gid=0(root) groups=0(root),65534", + "uid=0(root) gid=0(root) groups=0(root),65534(nobody),65534(nobody),65534(nobody),65534(nobody),65534(nobody),65534(nobody),65534(nobody),65534(nobody),65534(nobody),65534(nobody)", // Alpine; see https://golang.org/issue/19938 } for _, e := range expected { if strOut == e { @@ -253,3 +277,282 @@ func TestGroupCleanupUserNamespace(t *testing.T) { } t.Errorf("id command output: %q, expected one of %q", strOut, expected) } + +// TestUnshareHelperProcess isn't a real test. It's used as a helper process +// for TestUnshareMountNameSpace. +func TestUnshareMountNameSpaceHelper(*testing.T) { + if os.Getenv("GO_WANT_HELPER_PROCESS") != "1" { + return + } + defer os.Exit(0) + if err := syscall.Mount("none", flag.Args()[0], "proc", 0, ""); err != nil { + fmt.Fprintf(os.Stderr, "unshare: mount %v failed: %v", os.Args, err) + os.Exit(2) + } +} + +// Test for Issue 38471: unshare fails because systemd has forced / to be shared +func TestUnshareMountNameSpace(t *testing.T) { + // Make sure we are running as root so we have permissions to use unshare + // and create a network namespace. + if os.Getuid() != 0 { + t.Skip("kernel prohibits unshare in unprivileged process, unless using user namespace") + } + + // When running under the Go continuous build, skip tests for + // now when under Kubernetes. (where things are root but not quite) + // Both of these are our own environment variables. + // See Issue 12815. + if os.Getenv("GO_BUILDER_NAME") != "" && os.Getenv("IN_KUBERNETES") == "1" { + t.Skip("skipping test on Kubernetes-based builders; see Issue 12815") + } + + d, err := ioutil.TempDir("", "unshare") + if err != nil { + t.Fatalf("tempdir: %v", err) + } + + cmd := exec.Command(os.Args[0], "-test.run=TestUnshareMountNameSpaceHelper", d) + cmd.Env = []string{"GO_WANT_HELPER_PROCESS=1"} + cmd.SysProcAttr = &syscall.SysProcAttr{Unshareflags: syscall.CLONE_NEWNS} + + o, err := cmd.CombinedOutput() + if err != nil { + if strings.Contains(err.Error(), ": permission denied") { + t.Skipf("Skipping test (golang.org/issue/19698); unshare failed due to permissions: %s, %v", o, err) + } + t.Fatalf("unshare failed: %s, %v", o, err) + } + + // How do we tell if the namespace was really unshared? It turns out + // to be simple: just try to remove the directory. If it's still mounted + // on the rm will fail with EBUSY. Then we have some cleanup to do: + // we must unmount it, then try to remove it again. + + if err := os.Remove(d); err != nil { + t.Errorf("rmdir failed on %v: %v", d, err) + if err := syscall.Unmount(d, syscall.MNT_FORCE); err != nil { + t.Errorf("Can't unmount %v: %v", d, err) + } + if err := os.Remove(d); err != nil { + t.Errorf("rmdir after unmount failed on %v: %v", d, err) + } + } +} + +// Test for Issue 20103: unshare fails when chroot is used +func TestUnshareMountNameSpaceChroot(t *testing.T) { + // Make sure we are running as root so we have permissions to use unshare + // and create a network namespace. + if os.Getuid() != 0 { + t.Skip("kernel prohibits unshare in unprivileged process, unless using user namespace") + } + + // When running under the Go continuous build, skip tests for + // now when under Kubernetes. (where things are root but not quite) + // Both of these are our own environment variables. + // See Issue 12815. + if os.Getenv("GO_BUILDER_NAME") != "" && os.Getenv("IN_KUBERNETES") == "1" { + t.Skip("skipping test on Kubernetes-based builders; see Issue 12815") + } + + d, err := ioutil.TempDir("", "unshare") + if err != nil { + t.Fatalf("tempdir: %v", err) + } + + // Since we are doing a chroot, we need the binary there, + // and it must be statically linked. + x := filepath.Join(d, "syscall.test") + cmd := exec.Command(testenv.GoToolPath(t), "test", "-c", "-o", x, "syscall") + cmd.Env = append(os.Environ(), "CGO_ENABLED=0") + if o, err := cmd.CombinedOutput(); err != nil { + t.Fatalf("Build of syscall in chroot failed, output %v, err %v", o, err) + } + + cmd = exec.Command("/syscall.test", "-test.run=TestUnshareMountNameSpaceHelper", "/") + cmd.Env = []string{"GO_WANT_HELPER_PROCESS=1"} + cmd.SysProcAttr = &syscall.SysProcAttr{Chroot: d, Unshareflags: syscall.CLONE_NEWNS} + + o, err := cmd.CombinedOutput() + if err != nil { + if strings.Contains(err.Error(), ": permission denied") { + t.Skipf("Skipping test (golang.org/issue/19698); unshare failed due to permissions: %s, %v", o, err) + } + t.Fatalf("unshare failed: %s, %v", o, err) + } + + // How do we tell if the namespace was really unshared? It turns out + // to be simple: just try to remove the executable. If it's still mounted + // on, the rm will fail. Then we have some cleanup to do: + // we must force unmount it, then try to remove it again. + + if err := os.Remove(x); err != nil { + t.Errorf("rm failed on %v: %v", x, err) + if err := syscall.Unmount(d, syscall.MNT_FORCE); err != nil { + t.Fatalf("Can't unmount %v: %v", d, err) + } + if err := os.Remove(x); err != nil { + t.Fatalf("rm failed on %v: %v", x, err) + } + } + + if err := os.Remove(d); err != nil { + t.Errorf("rmdir failed on %v: %v", d, err) + } +} + +type capHeader struct { + version uint32 + pid int +} + +type capData struct { + effective uint32 + permitted uint32 + inheritable uint32 +} + +const CAP_SYS_TIME = 25 + +type caps struct { + hdr capHeader + data [2]capData +} + +func getCaps() (caps, error) { + var c caps + + // Get capability version + if _, _, errno := syscall.Syscall(syscall.SYS_CAPGET, uintptr(unsafe.Pointer(&c.hdr)), uintptr(unsafe.Pointer(nil)), 0); errno != 0 { + return c, fmt.Errorf("SYS_CAPGET: %v", errno) + } + + // Get current capabilities + if _, _, errno := syscall.Syscall(syscall.SYS_CAPGET, uintptr(unsafe.Pointer(&c.hdr)), uintptr(unsafe.Pointer(&c.data[0])), 0); errno != 0 { + return c, fmt.Errorf("SYS_CAPGET: %v", errno) + } + + return c, nil +} + +func mustSupportAmbientCaps(t *testing.T) { + var uname syscall.Utsname + if err := syscall.Uname(&uname); err != nil { + t.Fatalf("Uname: %v", err) + } + var buf [65]byte + for i, b := range uname.Release { + buf[i] = byte(b) + } + ver := string(buf[:]) + if i := strings.Index(ver, "\x00"); i != -1 { + ver = ver[:i] + } + if strings.HasPrefix(ver, "2.") || + strings.HasPrefix(ver, "3.") || + strings.HasPrefix(ver, "4.1.") || + strings.HasPrefix(ver, "4.2.") { + t.Skipf("kernel version %q predates required 4.3; skipping test", ver) + } +} + +// TestAmbientCapsHelper isn't a real test. It's used as a helper process for +// TestAmbientCaps. +func TestAmbientCapsHelper(*testing.T) { + if os.Getenv("GO_WANT_HELPER_PROCESS") != "1" { + return + } + defer os.Exit(0) + + caps, err := getCaps() + if err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(2) + } + if caps.data[0].effective&(1< ctx.maxLen { @@ -398,6 +411,7 @@ func runBenchmarks(matchString func(pat, str string) (bool, error), benchmarks [ w: os.Stdout, chatty: *chatty, }, + importPath: importPath, benchFunc: func(b *B) { for _, Benchmark := range bs { b.Run(Benchmark.Name, Benchmark.F) @@ -462,7 +476,7 @@ func (ctx *benchContext) processBench(b *B) { // least once will not be measured itself and will be called once with N=1. // // Run may be called simultaneously from multiple goroutines, but all such -// calls must happen before the outer benchmark function for b returns. +// calls must return before the outer benchmark function for b returns. func (b *B) Run(name string, f func(b *B)) bool { // Since b has subbenchmarks, we will no longer run it as a benchmark itself. // Release the lock and acquire it on exit to ensure locks stay paired. @@ -470,9 +484,9 @@ func (b *B) Run(name string, f func(b *B)) bool { benchmarkLock.Unlock() defer benchmarkLock.Lock() - benchName, ok := b.name, true + benchName, ok, partial := b.name, true, false if b.context != nil { - benchName, ok = b.context.match.fullName(&b.common, name) + benchName, ok, partial = b.context.match.fullName(&b.common, name) } if !ok { return true @@ -486,9 +500,15 @@ func (b *B) Run(name string, f func(b *B)) bool { w: b.w, chatty: b.chatty, }, - benchFunc: f, - benchTime: b.benchTime, - context: b.context, + importPath: b.importPath, + benchFunc: f, + benchTime: b.benchTime, + context: b.context, + } + if partial { + // Partial name match, like -bench=X/Y matching BenchmarkX. + // Only process sub-benchmarks, if any. + atomic.StoreInt32(&sub.hasSub, 1) } if sub.run1() { sub.run() @@ -634,10 +654,10 @@ func Benchmark(f func(b *B)) BenchmarkResult { benchFunc: f, benchTime: *benchTime, } - if !b.run1() { - return BenchmarkResult{} + if b.run1() { + b.run() } - return b.run() + return b.result } type discard struct{} diff --git a/libgo/go/testing/helper_test.go b/libgo/go/testing/helper_test.go new file mode 100644 index 00000000000..f5cb27c317b --- /dev/null +++ b/libgo/go/testing/helper_test.go @@ -0,0 +1,70 @@ +// Copyright 2017 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package testing + +import ( + "bytes" + "regexp" + "strings" +) + +func TestTBHelper(t *T) { + var buf bytes.Buffer + ctx := newTestContext(1, newMatcher(regexp.MatchString, "", "")) + t1 := &T{ + common: common{ + signal: make(chan bool), + w: &buf, + }, + context: ctx, + } + t1.Run("Test", testHelper) + + want := `--- FAIL: Test (?s) +helperfuncs_test.go:12: 0 +helperfuncs_test.go:33: 1 +helperfuncs_test.go:21: 2 +helperfuncs_test.go:35: 3 +helperfuncs_test.go:42: 4 +helperfuncs_test.go:47: 5 +--- FAIL: Test/sub (?s) +helperfuncs_test.go:50: 6 +helperfuncs_test.go:21: 7 +helperfuncs_test.go:53: 8 +` + lines := strings.Split(buf.String(), "\n") + durationRE := regexp.MustCompile(`\(.*\)$`) + for i, line := range lines { + line = strings.TrimSpace(line) + line = durationRE.ReplaceAllString(line, "(?s)") + lines[i] = line + } + got := strings.Join(lines, "\n") + if got != want { + t.Errorf("got output:\n\n%s\nwant:\n\n%s", got, want) + } +} + +func TestTBHelperParallel(t *T) { + var buf bytes.Buffer + ctx := newTestContext(1, newMatcher(regexp.MatchString, "", "")) + t1 := &T{ + common: common{ + signal: make(chan bool), + w: &buf, + }, + context: ctx, + } + t1.Run("Test", parallelTestHelper) + + lines := strings.Split(strings.TrimSpace(buf.String()), "\n") + if len(lines) != 6 { + t.Fatalf("parallelTestHelper gave %d lines of output; want 6", len(lines)) + } + want := "helperfuncs_test.go:21: parallel" + if got := strings.TrimSpace(lines[1]); got != want { + t.Errorf("got output line %q; want %q", got, want) + } +} diff --git a/libgo/go/testing/helperfuncs_test.go b/libgo/go/testing/helperfuncs_test.go new file mode 100644 index 00000000000..7cb2e2cc56d --- /dev/null +++ b/libgo/go/testing/helperfuncs_test.go @@ -0,0 +1,67 @@ +// Copyright 2017 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package testing + +import "sync" + +// The line numbering of this file is important for TestTBHelper. + +func notHelper(t *T, msg string) { + t.Error(msg) +} + +func helper(t *T, msg string) { + t.Helper() + t.Error(msg) +} + +func notHelperCallingHelper(t *T, msg string) { + helper(t, msg) +} + +func helperCallingHelper(t *T, msg string) { + t.Helper() + helper(t, msg) +} + +func testHelper(t *T) { + // Check combinations of directly and indirectly + // calling helper functions. + notHelper(t, "0") + helper(t, "1") + notHelperCallingHelper(t, "2") + helperCallingHelper(t, "3") + + // Check a function literal closing over t that uses Helper. + fn := func(msg string) { + t.Helper() + t.Error(msg) + } + fn("4") + + // Check that calling Helper from inside this test entry function + // doesn't have an effect. + t.Helper() + t.Error("5") + + t.Run("sub", func(t *T) { + helper(t, "6") + notHelperCallingHelper(t, "7") + t.Helper() + t.Error("8") + }) +} + +func parallelTestHelper(t *T) { + var wg sync.WaitGroup + for i := 0; i < 5; i++ { + wg.Add(1) + go func() { + notHelperCallingHelper(t, "parallel") + wg.Done() + }() + } + wg.Wait() +} diff --git a/libgo/go/testing/internal/testdeps/deps.go b/libgo/go/testing/internal/testdeps/deps.go index b08300b5d60..042f69614ee 100644 --- a/libgo/go/testing/internal/testdeps/deps.go +++ b/libgo/go/testing/internal/testdeps/deps.go @@ -49,3 +49,10 @@ func (TestDeps) WriteHeapProfile(w io.Writer) error { func (TestDeps) WriteProfileTo(name string, w io.Writer, debug int) error { return pprof.Lookup(name).WriteTo(w, debug) } + +// ImportPath is the import path of the testing binary, set by the generated main function. +var ImportPath string + +func (TestDeps) ImportPath() string { + return ImportPath +} diff --git a/libgo/go/testing/match.go b/libgo/go/testing/match.go index 77510357602..89e30d01a75 100644 --- a/libgo/go/testing/match.go +++ b/libgo/go/testing/match.go @@ -47,7 +47,7 @@ func newMatcher(matchString func(pat, str string) (bool, error), patterns, name } } -func (m *matcher) fullName(c *common, subname string) (name string, ok bool) { +func (m *matcher) fullName(c *common, subname string) (name string, ok, partial bool) { name = subname m.mu.Lock() @@ -62,15 +62,16 @@ func (m *matcher) fullName(c *common, subname string) (name string, ok bool) { // We check the full array of paths each time to allow for the case that // a pattern contains a '/'. - for i, s := range strings.Split(name, "/") { + elem := strings.Split(name, "/") + for i, s := range elem { if i >= len(m.filter) { break } if ok, _ := m.matchFunc(m.filter[i], s); !ok { - return name, false + return name, false, false } } - return name, true + return name, true, len(elem) < len(m.filter) } func splitRegexp(s string) []string { diff --git a/libgo/go/testing/match_test.go b/libgo/go/testing/match_test.go index 8c1c5f4452c..8c09dc660fb 100644 --- a/libgo/go/testing/match_test.go +++ b/libgo/go/testing/match_test.go @@ -88,43 +88,44 @@ func TestMatcher(t *T) { pattern string parent, sub string ok bool + partial bool }{ // Behavior without subtests. - {"", "", "TestFoo", true}, - {"TestFoo", "", "TestFoo", true}, - {"TestFoo/", "", "TestFoo", true}, - {"TestFoo/bar/baz", "", "TestFoo", true}, - {"TestFoo", "", "TestBar", false}, - {"TestFoo/", "", "TestBar", false}, - {"TestFoo/bar/baz", "", "TestBar/bar/baz", false}, + {"", "", "TestFoo", true, false}, + {"TestFoo", "", "TestFoo", true, false}, + {"TestFoo/", "", "TestFoo", true, true}, + {"TestFoo/bar/baz", "", "TestFoo", true, true}, + {"TestFoo", "", "TestBar", false, false}, + {"TestFoo/", "", "TestBar", false, false}, + {"TestFoo/bar/baz", "", "TestBar/bar/baz", false, false}, // with subtests - {"", "TestFoo", "x", true}, - {"TestFoo", "TestFoo", "x", true}, - {"TestFoo/", "TestFoo", "x", true}, - {"TestFoo/bar/baz", "TestFoo", "bar", true}, + {"", "TestFoo", "x", true, false}, + {"TestFoo", "TestFoo", "x", true, false}, + {"TestFoo/", "TestFoo", "x", true, false}, + {"TestFoo/bar/baz", "TestFoo", "bar", true, true}, // Subtest with a '/' in its name still allows for copy and pasted names // to match. - {"TestFoo/bar/baz", "TestFoo", "bar/baz", true}, - {"TestFoo/bar/baz", "TestFoo/bar", "baz", true}, - {"TestFoo/bar/baz", "TestFoo", "x", false}, - {"TestFoo", "TestBar", "x", false}, - {"TestFoo/", "TestBar", "x", false}, - {"TestFoo/bar/baz", "TestBar", "x/bar/baz", false}, + {"TestFoo/bar/baz", "TestFoo", "bar/baz", true, false}, + {"TestFoo/bar/baz", "TestFoo/bar", "baz", true, false}, + {"TestFoo/bar/baz", "TestFoo", "x", false, false}, + {"TestFoo", "TestBar", "x", false, false}, + {"TestFoo/", "TestBar", "x", false, false}, + {"TestFoo/bar/baz", "TestBar", "x/bar/baz", false, false}, // subtests only - {"", "TestFoo", "x", true}, - {"/", "TestFoo", "x", true}, - {"./", "TestFoo", "x", true}, - {"./.", "TestFoo", "x", true}, - {"/bar/baz", "TestFoo", "bar", true}, - {"/bar/baz", "TestFoo", "bar/baz", true}, - {"//baz", "TestFoo", "bar/baz", true}, - {"//", "TestFoo", "bar/baz", true}, - {"/bar/baz", "TestFoo/bar", "baz", true}, - {"//foo", "TestFoo", "bar/baz", false}, - {"/bar/baz", "TestFoo", "x", false}, - {"/bar/baz", "TestBar", "x/bar/baz", false}, + {"", "TestFoo", "x", true, false}, + {"/", "TestFoo", "x", true, false}, + {"./", "TestFoo", "x", true, false}, + {"./.", "TestFoo", "x", true, false}, + {"/bar/baz", "TestFoo", "bar", true, true}, + {"/bar/baz", "TestFoo", "bar/baz", true, false}, + {"//baz", "TestFoo", "bar/baz", true, false}, + {"//", "TestFoo", "bar/baz", true, false}, + {"/bar/baz", "TestFoo/bar", "baz", true, false}, + {"//foo", "TestFoo", "bar/baz", false, false}, + {"/bar/baz", "TestFoo", "x", false, false}, + {"/bar/baz", "TestBar", "x/bar/baz", false, false}, } for _, tc := range testCases { @@ -134,9 +135,9 @@ func TestMatcher(t *T) { if tc.parent != "" { parent.level = 1 } - if n, ok := m.fullName(parent, tc.sub); ok != tc.ok { - t.Errorf("for pattern %q, fullName(parent=%q, sub=%q) = %q, ok %v; want ok %v", - tc.pattern, tc.parent, tc.sub, n, ok, tc.ok) + if n, ok, partial := m.fullName(parent, tc.sub); ok != tc.ok || partial != tc.partial { + t.Errorf("for pattern %q, fullName(parent=%q, sub=%q) = %q, ok %v partial %v; want ok %v partial %v", + tc.pattern, tc.parent, tc.sub, n, ok, partial, tc.ok, tc.partial) } } } @@ -178,7 +179,7 @@ func TestNaming(t *T) { } for i, tc := range testCases { - if got, _ := m.fullName(parent, tc.name); got != tc.want { + if got, _, _ := m.fullName(parent, tc.name); got != tc.want { t.Errorf("%d:%s: got %q; want %q", i, tc.name, got, tc.want) } } diff --git a/libgo/go/testing/quick/quick.go b/libgo/go/testing/quick/quick.go index 95860fda0fb..0457fc7571b 100644 --- a/libgo/go/testing/quick/quick.go +++ b/libgo/go/testing/quick/quick.go @@ -14,6 +14,7 @@ import ( "math/rand" "reflect" "strings" + "time" ) var defaultMaxCount *int = flag.Int("quickchecks", 100, "The default number of iterations for each check") @@ -43,8 +44,10 @@ func randFloat64(rand *rand.Rand) float64 { return f } -// randInt64 returns a random integer taking half the range of an int64. -func randInt64(rand *rand.Rand) int64 { return rand.Int63() - 1<<62 } +// randInt64 returns a random int64. +func randInt64(rand *rand.Rand) int64 { + return int64(rand.Uint64()) +} // complexSize is the maximum length of arbitrary values that contain other // values. @@ -172,19 +175,20 @@ func sizedValue(t reflect.Type, rand *rand.Rand, size int) (value reflect.Value, // A Config structure contains options for running a test. type Config struct { - // MaxCount sets the maximum number of iterations. If zero, - // MaxCountScale is used. + // MaxCount sets the maximum number of iterations. + // If zero, MaxCountScale is used. MaxCount int - // MaxCountScale is a non-negative scale factor applied to the default - // maximum. If zero, the default is unchanged. + // MaxCountScale is a non-negative scale factor applied to the + // default maximum. + // If zero, the default is unchanged. MaxCountScale float64 - // If non-nil, rand is a source of random numbers. Otherwise a default - // pseudo-random source will be used. + // Rand specifies a source of random numbers. + // If nil, a default pseudo-random source will be used. Rand *rand.Rand - // If non-nil, the Values function generates a slice of arbitrary - // reflect.Values that are congruent with the arguments to the function - // being tested. Otherwise, the top-level Value function is used - // to generate them. + // Values specifies a function to generate a slice of + // arbitrary reflect.Values that are congruent with the + // arguments to the function being tested. + // If nil, the top-level Value function is used to generate them. Values func([]reflect.Value, *rand.Rand) } @@ -193,7 +197,7 @@ var defaultConfig Config // getRand returns the *rand.Rand to use for a given Config. func (c *Config) getRand() *rand.Rand { if c.Rand == nil { - return rand.New(rand.NewSource(0)) + return rand.New(rand.NewSource(time.Now().UnixNano())) } return c.Rand } diff --git a/libgo/go/testing/quick/quick_test.go b/libgo/go/testing/quick/quick_test.go index fe443592f87..4246cd1d3ba 100644 --- a/libgo/go/testing/quick/quick_test.go +++ b/libgo/go/testing/quick/quick_test.go @@ -307,3 +307,21 @@ func TestNonZeroSliceAndMap(t *testing.T) { t.Fatal(err) } } + +func TestInt64(t *testing.T) { + var lo, hi int64 + f := func(x int64) bool { + if x < lo { + lo = x + } + if x > hi { + hi = x + } + return true + } + cfg := &Config{MaxCount: 100000} + Check(f, cfg) + if uint64(lo)>>62 == 0 || uint64(hi)>>62 == 0 { + t.Errorf("int64 returned range %#016x,%#016x; does not look like full range", lo, hi) + } +} diff --git a/libgo/go/testing/sub_test.go b/libgo/go/testing/sub_test.go index bb7b3e09255..acf5dea8785 100644 --- a/libgo/go/testing/sub_test.go +++ b/libgo/go/testing/sub_test.go @@ -8,11 +8,18 @@ import ( "bytes" "fmt" "regexp" + "runtime" "strings" + "sync" "sync/atomic" "time" ) +func init() { + // Make benchmark tests run 10* faster. + *benchTime = 100 * time.Millisecond +} + func TestTestContext(t *T) { const ( add1 = 0 @@ -455,8 +462,14 @@ func TestBRun(t *T) { _ = append([]byte(nil), buf[:]...) } } - b.Run("", func(b *B) { alloc(b) }) - b.Run("", func(b *B) { alloc(b) }) + b.Run("", func(b *B) { + alloc(b) + b.ReportAllocs() + }) + b.Run("", func(b *B) { + alloc(b) + b.ReportAllocs() + }) // runtime.MemStats sometimes reports more allocations than the // benchmark is responsible for. Luckily the point of this test is // to ensure that the results are not underreported, so we can @@ -517,6 +530,26 @@ func TestBenchmarkOutput(t *T) { Benchmark(func(b *B) {}) } +func TestBenchmarkStartsFrom1(t *T) { + var first = true + Benchmark(func(b *B) { + if first && b.N != 1 { + panic(fmt.Sprintf("Benchmark() first N=%v; want 1", b.N)) + } + first = false + }) +} + +func TestBenchmarkReadMemStatsBeforeFirstRun(t *T) { + var first = true + Benchmark(func(b *B) { + if first && (b.startAllocs == 0 || b.startBytes == 0) { + panic(fmt.Sprintf("ReadMemStats not called before first run")) + } + first = false + }) +} + func TestParallelSub(t *T) { c := make(chan int) block := make(chan int) @@ -532,3 +565,59 @@ func TestParallelSub(t *T) { <-c } } + +type funcWriter func([]byte) (int, error) + +func (fw funcWriter) Write(b []byte) (int, error) { return fw(b) } + +func TestRacyOutput(t *T) { + var runs int32 // The number of running Writes + var races int32 // Incremented for each race detected + raceDetector := func(b []byte) (int, error) { + // Check if some other goroutine is concurrently calling Write. + if atomic.LoadInt32(&runs) > 0 { + atomic.AddInt32(&races, 1) // Race detected! + } + atomic.AddInt32(&runs, 1) + defer atomic.AddInt32(&runs, -1) + runtime.Gosched() // Increase probability of a race + return len(b), nil + } + + var wg sync.WaitGroup + root := &T{ + common: common{w: funcWriter(raceDetector), chatty: true}, + context: newTestContext(1, newMatcher(regexp.MatchString, "", "")), + } + root.Run("", func(t *T) { + for i := 0; i < 100; i++ { + wg.Add(1) + go func(i int) { + defer wg.Done() + t.Run(fmt.Sprint(i), func(t *T) { + t.Logf("testing run %d", i) + }) + }(i) + } + }) + wg.Wait() + + if races > 0 { + t.Errorf("detected %d racy Writes", races) + } +} + +func TestBenchmark(t *T) { + res := Benchmark(func(b *B) { + for i := 0; i < 5; i++ { + b.Run("", func(b *B) { + for i := 0; i < b.N; i++ { + time.Sleep(time.Millisecond) + } + }) + } + }) + if res.NsPerOp() < 4000000 { + t.Errorf("want >5ms; got %v", time.Duration(res.NsPerOp())) + } +} diff --git a/libgo/go/testing/testing.go b/libgo/go/testing/testing.go index b002aa0bf10..a62974211fb 100644 --- a/libgo/go/testing/testing.go +++ b/libgo/go/testing/testing.go @@ -83,16 +83,30 @@ // ignores leading and trailing space.) These are examples of an example: // // func ExampleHello() { -// fmt.Println("hello") -// // Output: hello +// fmt.Println("hello") +// // Output: hello // } // // func ExampleSalutations() { -// fmt.Println("hello, and") -// fmt.Println("goodbye") -// // Output: -// // hello, and -// // goodbye +// fmt.Println("hello, and") +// fmt.Println("goodbye") +// // Output: +// // hello, and +// // goodbye +// } +// +// The comment prefix "Unordered output:" is like "Output:", but matches any +// line order: +// +// func ExamplePerm() { +// for _, value := range Perm(4) { +// fmt.Println(value) +// } +// // Unordered output: 4 +// // 2 +// // 1 +// // 3 +// // 0 // } // // Example functions without output comments are compiled but not executed. @@ -238,6 +252,7 @@ var ( chatty = flag.Bool("test.v", false, "verbose: print additional output") count = flag.Uint("test.count", 1, "run tests and benchmarks `n` times") coverProfile = flag.String("test.coverprofile", "", "write a coverage profile to `file`") + matchList = flag.String("test.list", "", "list tests, examples, and benchmarch maching `regexp` then exit") match = flag.String("test.run", "", "run only tests and examples matching `regexp`") memProfile = flag.String("test.memprofile", "", "write a memory profile to `file`") memProfileRate = flag.Int("test.memprofilerate", 0, "set memory profiling `rate` (see runtime.MemProfileRate)") @@ -247,7 +262,7 @@ var ( mutexProfile = flag.String("test.mutexprofile", "", "write a mutex contention profile to the named file after execution") mutexProfileFraction = flag.Int("test.mutexprofilefraction", 1, "if >= 0, calls runtime.SetMutexProfileFraction()") traceFile = flag.String("test.trace", "", "write an execution trace to `file`") - timeout = flag.Duration("test.timeout", 0, "fail test binary execution after duration `d` (0 means unlimited)") + timeout = flag.Duration("test.timeout", 0, "panic test binary after duration `d` (0 means unlimited)") cpuListStr = flag.String("test.cpu", "", "comma-separated `list` of cpu counts to run each test with") parallel = flag.Int("test.parallel", runtime.GOMAXPROCS(0), "run at most `n` tests in parallel") @@ -259,17 +274,20 @@ var ( // common holds the elements common between T and B and // captures common methods such as Errorf. type common struct { - mu sync.RWMutex // guards output, failed, and done. - output []byte // Output generated by test or benchmark. - w io.Writer // For flushToParent. - chatty bool // A copy of the chatty flag. - ran bool // Test or benchmark (or one of its subtests) was executed. - failed bool // Test or benchmark has failed. - skipped bool // Test of benchmark has been skipped. - finished bool // Test function has completed. - done bool // Test is finished and all subtests have completed. - hasSub int32 // written atomically - raceErrors int // number of races detected during test + mu sync.RWMutex // guards this group of fields + output []byte // Output generated by test or benchmark. + w io.Writer // For flushToParent. + ran bool // Test or benchmark (or one of its subtests) was executed. + failed bool // Test or benchmark has failed. + skipped bool // Test of benchmark has been skipped. + done bool // Test is finished and all subtests have completed. + helpers map[string]struct{} // functions to be skipped when writing file/line info + + chatty bool // A copy of the chatty flag. + finished bool // Test function has completed. + hasSub int32 // written atomically + raceErrors int // number of races detected during test + runner string // function name of tRunner running the test parent *common level int // Nesting depth of test or benchmark. @@ -298,10 +316,48 @@ func Verbose() bool { return *chatty } +// frameSkip searches, starting after skip frames, for the first caller frame +// in a function not marked as a helper and returns the frames to skip +// to reach that site. The search stops if it finds a tRunner function that +// was the entry point into the test. +// This function must be called with c.mu held. +func (c *common) frameSkip(skip int) int { + if c.helpers == nil { + return skip + } + var pc [50]uintptr + // Skip two extra frames to account for this function + // and runtime.Callers itself. + n := runtime.Callers(skip+2, pc[:]) + if n == 0 { + panic("testing: zero callers found") + } + frames := runtime.CallersFrames(pc[:n]) + var frame runtime.Frame + more := true + for i := 0; more; i++ { + frame, more = frames.Next() + if frame.Function == c.runner { + // We've gone up all the way to the tRunner calling + // the test function (so the user must have + // called tb.Helper from inside that test function). + // Only skip up to the test function itself. + return skip + i - 1 + } + if _, ok := c.helpers[frame.Function]; !ok { + // Found a frame that wasn't inside a helper function. + return skip + i + } + } + return skip +} + // decorate prefixes the string with the file and line of the call site // and inserts the final newline if needed and indentation tabs for formatting. -func decorate(s string) string { - _, file, line, ok := runtime.Caller(3) // decorate + log + public function. +// This function must be called with c.mu held. +func (c *common) decorate(s string) string { + skip := c.frameSkip(3) // decorate + log + public function. + _, file, line, ok := runtime.Caller(skip) if ok { // Truncate file name at last file name separator. if index := strings.LastIndex(file, "/"); index >= 0 { @@ -391,6 +447,7 @@ type TB interface { SkipNow() Skipf(format string, args ...interface{}) Skipped() bool + Helper() // A private method to prevent users implementing the // interface and so future additions to it will not @@ -450,8 +507,9 @@ func (c *common) Fail() { // Failed reports whether the function has failed. func (c *common) Failed() bool { c.mu.RLock() - defer c.mu.RUnlock() - return c.failed + failed := c.failed + c.mu.RUnlock() + return failed || c.raceErrors+race.Errors() > 0 } // FailNow marks the function as having failed and stops its execution. @@ -490,7 +548,7 @@ func (c *common) FailNow() { func (c *common) log(s string) { c.mu.Lock() defer c.mu.Unlock() - c.output = append(c.output, decorate(s)...) + c.output = append(c.output, c.decorate(s)...) } // Log formats its arguments using default formatting, analogous to Println, @@ -568,8 +626,38 @@ func (c *common) Skipped() bool { return c.skipped } +// Helper marks the calling function as a test helper function. +// When printing file and line information, that function will be skipped. +// Helper may be called simultaneously from multiple goroutines. +// Helper has no effect if it is called directly from a TestXxx/BenchmarkXxx +// function or a subtest/sub-benchmark function. +func (c *common) Helper() { + c.mu.Lock() + defer c.mu.Unlock() + if c.helpers == nil { + c.helpers = make(map[string]struct{}) + } + c.helpers[callerName(1)] = struct{}{} +} + +// callerName gives the function name (qualified with a package path) +// for the caller after skip frames (where 0 means the current function). +func callerName(skip int) string { + // Make room for the skip PC. + var pc [2]uintptr + n := runtime.Callers(skip+2, pc[:]) // skip + runtime.Callers + callerName + if n == 0 { + panic("testing: zero callers found") + } + frames := runtime.CallersFrames(pc[:n]) + frame, _ := frames.Next() + return frame.Function +} + // Parallel signals that this test is to be run in parallel with (and only with) -// other parallel tests. +// other parallel tests. When a test is run multiple times due to use of +// -test.count or -test.cpu, multiple instances of a single test never run in +// parallel with each other. func (t *T) Parallel() { if t.isParallel { panic("testing: t.Parallel called multiple times") @@ -600,13 +688,14 @@ type InternalTest struct { } func tRunner(t *T, fn func(t *T)) { + t.runner = callerName(0) + // When this goroutine is done, either because fn(t) // returned normally or because a test failure triggered // a call to runtime.Goexit, record the duration and send // a signal saying that the test is done. defer func() { - t.raceErrors += race.Errors() - if t.raceErrors > 0 { + if t.raceErrors+race.Errors() > 0 { t.Errorf("race detected during execution of test") } @@ -658,14 +747,15 @@ func tRunner(t *T, fn func(t *T)) { t.finished = true } -// Run runs f as a subtest of t called name. It reports whether f succeeded. -// Run will block until all its parallel subtests have completed. +// Run runs f as a subtest of t called name. It reports whether f succeeded. Run +// runs f in a separate goroutine and will block until all its parallel subtests +// have completed. // -// Run may be called simultaneously from multiple goroutines, but all such -// calls must happen before the outer test function for t returns. +// Run may be called simultaneously from multiple goroutines, but all such calls +// must return before the outer test function for t returns. func (t *T) Run(name string, f func(t *T)) bool { atomic.StoreInt32(&t.hasSub, 1) - testName, ok := t.context.match.fullName(&t.common, name) + testName, ok, _ := t.context.match.fullName(&t.common, name) if !ok { return true } @@ -687,7 +777,9 @@ func (t *T) Run(name string, f func(t *T)) bool { root := t.parent for ; root.parent != nil; root = root.parent { } + root.mu.Lock() fmt.Fprintf(root.w, "=== RUN %s\n", t.name) + root.mu.Unlock() } // Instead of reducing the running count of this test before calling the // tRunner and increasing it afterwards, we rely on tRunner keeping the @@ -764,6 +856,7 @@ func (f matchStringOnly) StartCPUProfile(w io.Writer) error { return e func (f matchStringOnly) StopCPUProfile() {} func (f matchStringOnly) WriteHeapProfile(w io.Writer) error { return errMain } func (f matchStringOnly) WriteProfileTo(string, io.Writer, int) error { return errMain } +func (f matchStringOnly) ImportPath() string { return "" } // Main is an internal function, part of the implementation of the "go test" command. // It was exported because it is cross-package and predates "internal" packages. @@ -793,6 +886,7 @@ type testDeps interface { StopCPUProfile() WriteHeapProfile(io.Writer) error WriteProfileTo(string, io.Writer, int) error + ImportPath() string } // MainStart is meant for use by tests generated by 'go test'. @@ -814,6 +908,11 @@ func (m *M) Run() int { flag.Parse() } + if len(*matchList) != 0 { + listTests(m.deps.MatchString, m.tests, m.benchmarks, m.examples) + return 0 + } + parseCpuList() m.before() @@ -825,7 +924,7 @@ func (m *M) Run() int { if !testRan && !exampleRan && *matchBenchmarks == "" { fmt.Fprintln(os.Stderr, "testing: warning: no tests to run") } - if !testOk || !exampleOk || !runBenchmarks(m.deps.MatchString, m.benchmarks) || race.Errors() > 0 { + if !testOk || !exampleOk || !runBenchmarks(m.deps.ImportPath(), m.deps.MatchString, m.benchmarks) || race.Errors() > 0 { fmt.Println("FAIL") m.after() return 1 @@ -853,6 +952,29 @@ func (t *T) report() { } } +func listTests(matchString func(pat, str string) (bool, error), tests []InternalTest, benchmarks []InternalBenchmark, examples []InternalExample) { + if _, err := matchString(*matchList, "non-empty"); err != nil { + fmt.Fprintf(os.Stderr, "testing: invalid regexp in -test.list (%q): %s\n", *matchList, err) + os.Exit(1) + } + + for _, test := range tests { + if ok, _ := matchString(*matchList, test.Name); ok { + fmt.Println(test.Name) + } + } + for _, bench := range benchmarks { + if ok, _ := matchString(*matchList, bench.Name); ok { + fmt.Println(bench.Name) + } + } + for _, example := range examples { + if ok, _ := matchString(*matchList, example.Name); ok { + fmt.Println(example.Name) + } + } +} + // An internal function but exported because it is cross-package; part of the implementation // of the "go test" command. func RunTests(matchString func(pat, str string) (bool, error), tests []InternalTest) (ok bool) { diff --git a/libgo/go/text/scanner/example_test.go b/libgo/go/text/scanner/example_test.go index f48c31daa0a..9e2d5b7c733 100644 --- a/libgo/go/text/scanner/example_test.go +++ b/libgo/go/text/scanner/example_test.go @@ -14,28 +14,25 @@ import ( func Example() { const src = ` - // This is scanned code. - if a > 10 { - someParsable = text - }` +// This is scanned code. +if a > 10 { + someParsable = text +}` var s scanner.Scanner - s.Filename = "example" s.Init(strings.NewReader(src)) - var tok rune - for tok != scanner.EOF { - tok = s.Scan() - fmt.Println("At position", s.Pos(), ":", s.TokenText()) + s.Filename = "example" + for tok := s.Scan(); tok != scanner.EOF; tok = s.Scan() { + fmt.Printf("%s: %s\n", s.Position, s.TokenText()) } // Output: - // At position example:3:4 : if - // At position example:3:6 : a - // At position example:3:8 : > - // At position example:3:11 : 10 - // At position example:3:13 : { - // At position example:4:15 : someParsable - // At position example:4:17 : = - // At position example:4:22 : text - // At position example:5:3 : } - // At position example:5:3 : + // example:3:1: if + // example:3:4: a + // example:3:6: > + // example:3:8: 10 + // example:3:11: { + // example:4:2: someParsable + // example:4:15: = + // example:4:17: text + // example:5:1: } } diff --git a/libgo/go/text/scanner/scanner.go b/libgo/go/text/scanner/scanner.go index e085f8a7d95..6fb0422fe5e 100644 --- a/libgo/go/text/scanner/scanner.go +++ b/libgo/go/text/scanner/scanner.go @@ -166,7 +166,8 @@ type Scanner struct { // The Filename field is always left untouched by the Scanner. // If an error is reported (via Error) and Position is invalid, // the scanner is not inside a token. Call Pos to obtain an error - // position in that case. + // position in that case, or to obtain the position immediately + // after the most recently scanned token. Position } @@ -637,6 +638,8 @@ redo: // Pos returns the position of the character immediately after // the character or token returned by the last call to Next or Scan. +// Use the Scanner's Position field for the start position of the most +// recently scanned token. func (s *Scanner) Pos() (pos Position) { pos.Filename = s.Filename pos.Offset = s.srcBufOffset + s.srcPos - s.lastCharLen diff --git a/libgo/go/text/template/doc.go b/libgo/go/text/template/doc.go index fe59e3f74e8..d174ebd9cfe 100644 --- a/libgo/go/text/template/doc.go +++ b/libgo/go/text/template/doc.go @@ -20,7 +20,8 @@ The input text for a template is UTF-8-encoded text in any format. "{{" and "}}"; all text outside actions is copied to the output unchanged. Except for raw strings, actions may not span newlines, although comments can. -Once parsed, a template may be executed safely in parallel. +Once parsed, a template may be executed safely in parallel, although if parallel +executions share a Writer the output may be interleaved. Here is a trivial example that prints "17 items are made of wool". @@ -80,14 +81,14 @@ data, defined in detail in the corresponding sections that follow. {{if pipeline}} T1 {{end}} If the value of the pipeline is empty, no output is generated; - otherwise, T1 is executed. The empty values are false, 0, any + otherwise, T1 is executed. The empty values are false, 0, any nil pointer or interface value, and any array, slice, map, or string of length zero. Dot is unaffected. {{if pipeline}} T1 {{else}} T0 {{end}} If the value of the pipeline is empty, T0 is executed; - otherwise, T1 is executed. Dot is unaffected. + otherwise, T1 is executed. Dot is unaffected. {{if pipeline}} T1 {{else if pipeline}} T0 {{end}} To simplify the appearance of if-else chains, the else action @@ -241,19 +242,19 @@ where $variable is the name of the variable. An action that declares a variable produces no output. If a "range" action initializes a variable, the variable is set to the -successive elements of the iteration. Also, a "range" may declare two +successive elements of the iteration. Also, a "range" may declare two variables, separated by a comma: range $index, $element := pipeline in which case $index and $element are set to the successive values of the -array/slice index or map key and element, respectively. Note that if there is +array/slice index or map key and element, respectively. Note that if there is only one variable, it is assigned the element; this is opposite to the convention in Go range clauses. A variable's scope extends to the "end" action of the control structure ("if", "with", or "range") in which it is declared, or to the end of the template if -there is no such control structure. A template invocation does not inherit +there is no such control structure. A template invocation does not inherit variables from the point of its invocation. When execution begins, $ is set to the data argument passed to Execute, that is, @@ -314,7 +315,8 @@ Predefined global functions are named as follows. or the returned error value is non-nil, execution stops. html Returns the escaped HTML equivalent of the textual - representation of its arguments. + representation of its arguments. This function is unavailable + in html/template, with a few exceptions. index Returns the result of indexing its first argument by the following arguments. Thus "index x 1 2 3" is, in Go syntax, @@ -340,6 +342,8 @@ Predefined global functions are named as follows. urlquery Returns the escaped value of the textual representation of its arguments in a form suitable for embedding in a URL query. + This function is unavailable in html/template, with a few + exceptions. The boolean functions take any zero value to be false and a non-zero value to be true. diff --git a/libgo/go/text/template/exec.go b/libgo/go/text/template/exec.go index 89d3e379b4b..29eb68fba75 100644 --- a/libgo/go/text/template/exec.go +++ b/libgo/go/text/template/exec.go @@ -155,7 +155,8 @@ func errRecover(errp *error) { // If an error occurs executing the template or writing its output, // execution stops, but partial results may already have been written to // the output writer. -// A template may be executed safely in parallel. +// A template may be executed safely in parallel, although if parallel +// executions share a Writer the output may be interleaved. func (t *Template) ExecuteTemplate(wr io.Writer, name string, data interface{}) error { var tmpl *Template if t.common != nil { @@ -172,7 +173,8 @@ func (t *Template) ExecuteTemplate(wr io.Writer, name string, data interface{}) // If an error occurs executing the template or writing its output, // execution stops, but partial results may already have been written to // the output writer. -// A template may be executed safely in parallel. +// A template may be executed safely in parallel, although if parallel +// executions share a Writer the output may be interleaved. // // If data is a reflect.Value, the template applies to the concrete // value that the reflect.Value holds, as in fmt.Print. @@ -553,7 +555,7 @@ func (s *state) evalField(dot reflect.Value, fieldName string, node parse.Node, // Unless it's an interface, need to get to a value of type *T to guarantee // we see all methods of T and *T. ptr := receiver - if ptr.Kind() != reflect.Interface && ptr.CanAddr() { + if ptr.Kind() != reflect.Interface && ptr.Kind() != reflect.Ptr && ptr.CanAddr() { ptr = ptr.Addr() } if method := ptr.MethodByName(fieldName); method.IsValid() { @@ -630,7 +632,7 @@ func (s *state) evalCall(dot, fun reflect.Value, node parse.Node, name string, a if numIn < numFixed { s.errorf("wrong number of args for %s: want at least %d got %d", name, typ.NumIn()-1, len(args)) } - } else if numIn < typ.NumIn()-1 || !typ.IsVariadic() && numIn != typ.NumIn() { + } else if numIn != typ.NumIn() { s.errorf("wrong number of args for %s: want %d got %d", name, typ.NumIn(), len(args)) } if !goodFunc(typ) { diff --git a/libgo/go/text/template/exec_test.go b/libgo/go/text/template/exec_test.go index 5892b27391b..9f7e637c190 100644 --- a/libgo/go/text/template/exec_test.go +++ b/libgo/go/text/template/exec_test.go @@ -147,6 +147,8 @@ var tVal = &T{ Tmpl: Must(New("x").Parse("test template")), // "x" is the value of .X } +var tSliceOfNil = []*T{nil} + // A non-empty interface. type I interface { Method0() string @@ -337,6 +339,7 @@ var execTests = []execTest{ "true", tVal, true}, {".NilOKFunc not nil", "{{call .NilOKFunc .PI}}", "false", tVal, true}, {".NilOKFunc nil", "{{call .NilOKFunc nil}}", "true", tVal, true}, + {"method on nil value from slice", "-{{range .}}{{.Method1 1234}}{{end}}-", "-1234-", tSliceOfNil, true}, // Function call builtin. {".BinaryFunc", "{{call .BinaryFunc `1` `2`}}", "[1=2]", tVal, true}, diff --git a/libgo/go/text/template/funcs.go b/libgo/go/text/template/funcs.go index 3047b272e57..91074310374 100644 --- a/libgo/go/text/template/funcs.go +++ b/libgo/go/text/template/funcs.go @@ -489,6 +489,7 @@ var ( htmlAmp = []byte("&") htmlLt = []byte("<") htmlGt = []byte(">") + htmlNull = []byte("\uFFFD") ) // HTMLEscape writes to w the escaped HTML equivalent of the plain text data b. @@ -497,6 +498,8 @@ func HTMLEscape(w io.Writer, b []byte) { for i, c := range b { var html []byte switch c { + case '\000': + html = htmlNull case '"': html = htmlQuot case '\'': @@ -520,7 +523,7 @@ func HTMLEscape(w io.Writer, b []byte) { // HTMLEscapeString returns the escaped HTML equivalent of the plain text data s. func HTMLEscapeString(s string) string { // Avoid allocation if we can. - if !strings.ContainsAny(s, `'"&<>`) { + if !strings.ContainsAny(s, "'\"&<>\000") { return s } var b bytes.Buffer diff --git a/libgo/go/text/template/parse/lex_test.go b/libgo/go/text/template/parse/lex_test.go index d655d788b3b..2c73bb623ae 100644 --- a/libgo/go/text/template/parse/lex_test.go +++ b/libgo/go/text/template/parse/lex_test.go @@ -498,7 +498,7 @@ func TestShutdown(t *testing.T) { // We need to duplicate template.Parse here to hold on to the lexer. const text = "erroneous{{define}}{{else}}1234" lexer := lex("foo", text, "{{", "}}") - _, err := New("root").parseLexer(lexer, text) + _, err := New("root").parseLexer(lexer) if err == nil { t.Fatalf("expected error") } @@ -511,7 +511,7 @@ func TestShutdown(t *testing.T) { // parseLexer is a local version of parse that lets us pass in the lexer instead of building it. // We expect an error, so the tree set and funcs list are explicitly nil. -func (t *Tree) parseLexer(lex *lexer, text string) (tree *Tree, err error) { +func (t *Tree) parseLexer(lex *lexer) (tree *Tree, err error) { defer t.recover(&err) t.ParseName = t.Name t.startParse(nil, lex, map[string]*Tree{}) diff --git a/libgo/go/text/template/parse/parse.go b/libgo/go/text/template/parse/parse.go index 6060c6d74b7..a91a544ce01 100644 --- a/libgo/go/text/template/parse/parse.go +++ b/libgo/go/text/template/parse/parse.go @@ -202,7 +202,6 @@ func (t *Tree) recover(errp *error) { } *errp = e.(error) } - return } // startParse initializes the parser, using the lexer. diff --git a/libgo/go/text/template/template.go b/libgo/go/text/template/template.go index 3b4f34b4db0..2246f676e65 100644 --- a/libgo/go/text/template/template.go +++ b/libgo/go/text/template/template.go @@ -159,6 +159,7 @@ func (t *Template) Delims(left, right string) *Template { } // Funcs adds the elements of the argument map to the template's function map. +// It must be called before the template is parsed. // It panics if a value in the map is not a function with appropriate return // type or if the name cannot be used syntactically as a function in a template. // It is legal to overwrite elements of the map. The return value is the template, diff --git a/libgo/go/time/example_test.go b/libgo/go/time/example_test.go index 7dc2bb5e7ee..aeb63caa552 100644 --- a/libgo/go/time/example_test.go +++ b/libgo/go/time/example_test.go @@ -256,6 +256,9 @@ func ExampleTime_Truncate() { for _, d := range trunc { fmt.Printf("t.Truncate(%5s) = %s\n", d, t.Truncate(d).Format("15:04:05.999999999")) } + // To round to the last midnight in the local timezone, create a new Date. + midnight := time.Date(t.Year(), t.Month(), t.Day(), 0, 0, 0, 0, time.Local) + _ = midnight // Output: // t.Truncate( 1ns) = 12:15:30.918273645 diff --git a/libgo/go/time/export_test.go b/libgo/go/time/export_test.go index 6cd535f6b19..4c08ab13afc 100644 --- a/libgo/go/time/export_test.go +++ b/libgo/go/time/export_test.go @@ -18,7 +18,20 @@ func ForceUSPacificForTesting() { localOnce.Do(initTestingZone) } +func ZoneinfoForTesting() *string { + return zoneinfo +} + +func ResetZoneinfoForTesting() { + zoneinfo = nil + zoneinfoOnce = sync.Once{} +} + var ( ForceZipFileForTesting = forceZipFileForTesting ParseTimeZone = parseTimeZone + SetMono = (*Time).setMono + GetMono = (*Time).mono + ErrLocation = errLocation + ReadFile = readFile ) diff --git a/libgo/go/time/format.go b/libgo/go/time/format.go index b903e1485c6..8c16e873f64 100644 --- a/libgo/go/time/format.go +++ b/libgo/go/time/format.go @@ -61,6 +61,8 @@ import "errors" // RFC822, RFC822Z, RFC1123, and RFC1123Z are useful for formatting; // when used with time.Parse they do not accept all the time formats // permitted by the RFCs. +// The RFC3339Nano format removes trailing zeros from the seconds field +// and thus may not sort correctly once formatted. const ( ANSIC = "Mon Jan _2 15:04:05 2006" UnixDate = "Mon Jan _2 15:04:05 MST 2006" @@ -424,8 +426,41 @@ func formatNano(b []byte, nanosec uint, n int, trim bool) []byte { // String returns the time formatted using the format string // "2006-01-02 15:04:05.999999999 -0700 MST" +// +// If the time has a monotonic clock reading, the returned string +// includes a final field "m=±", where value is the monotonic +// clock reading formatted as a decimal number of seconds. +// +// The returned string is meant for debugging; for a stable serialized +// representation, use t.MarshalText, t.MarshalBinary, or t.Format +// with an explicit format string. func (t Time) String() string { - return t.Format("2006-01-02 15:04:05.999999999 -0700 MST") + s := t.Format("2006-01-02 15:04:05.999999999 -0700 MST") + + // Format monotonic clock reading as m=±ddd.nnnnnnnnn. + if t.wall&hasMonotonic != 0 { + m2 := uint64(t.ext) + sign := byte('+') + if t.ext < 0 { + sign = '-' + m2 = -m2 + } + m1, m2 := m2/1e9, m2%1e9 + m0, m1 := m1/1e9, m1%1e9 + var buf []byte + buf = append(buf, " m="...) + buf = append(buf, sign) + wid := 0 + if m0 != 0 { + buf = appendInt(buf, int(m0), 0) + wid = 9 + } + buf = appendInt(buf, int(m1), wid) + buf = append(buf, '.') + buf = appendInt(buf, int(m2), 9) + s += string(buf) + } + return s } // Format returns a textual representation of the time value formatted @@ -725,11 +760,6 @@ func skip(value, prefix string) (string, error) { // location and zone in the returned time. Otherwise it records the time as // being in a fabricated location with time fixed at the given zone offset. // -// No checking is done that the day of the month is within the month's -// valid dates; any one- or two-digit value is accepted. For example -// February 31 and even February 99 are valid dates, specifying dates -// in March and May. This behavior is consistent with time.Date. -// // When parsing a time with a zone abbreviation like MST, if the zone abbreviation // has a defined offset in the current location, then that offset is used. // The zone abbreviation "UTC" is recognized as UTC regardless of location. @@ -1022,11 +1052,11 @@ func parse(layout, value string, defaultLocation, local *Location) (Time, error) if zoneOffset != -1 { t := Date(year, Month(month), day, hour, min, sec, nsec, UTC) - t.sec -= int64(zoneOffset) + t.addSec(-int64(zoneOffset)) // Look for local zone with the given offset. // If that zone was in effect at the given time, use it. - name, offset, _, _, _ := local.lookup(t.sec + internalToUnix) + name, offset, _, _, _ := local.lookup(t.unixSec()) if offset == zoneOffset && (zoneName == "" || name == zoneName) { t.setLoc(local) return t, nil @@ -1041,9 +1071,9 @@ func parse(layout, value string, defaultLocation, local *Location) (Time, error) t := Date(year, Month(month), day, hour, min, sec, nsec, UTC) // Look for local zone with the given offset. // If that zone was in effect at the given time, use it. - offset, _, ok := local.lookupName(zoneName, t.sec+internalToUnix) + offset, _, ok := local.lookupName(zoneName, t.unixSec()) if ok { - t.sec -= int64(offset) + t.addSec(-int64(offset)) t.setLoc(local) return t, nil } diff --git a/libgo/go/time/format_test.go b/libgo/go/time/format_test.go index 0e4a4174309..abaeb506096 100644 --- a/libgo/go/time/format_test.go +++ b/libgo/go/time/format_test.go @@ -380,8 +380,8 @@ func checkTime(time Time, test *ParseTest, t *testing.T) { func TestFormatAndParse(t *testing.T) { const fmt = "Mon MST " + RFC3339 // all fields f := func(sec int64) bool { - t1 := Unix(sec, 0) - if t1.Year() < 1000 || t1.Year() > 9999 { + t1 := Unix(sec/2, 0) + if t1.Year() < 1000 || t1.Year() > 9999 || t1.Unix() != sec { // not required to work return true } diff --git a/libgo/go/time/genzabbrs.go b/libgo/go/time/genzabbrs.go index 6281f73ce4a..824a67f15af 100644 --- a/libgo/go/time/genzabbrs.go +++ b/libgo/go/time/genzabbrs.go @@ -142,8 +142,8 @@ const prog = ` // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -// generated by genzabbrs.go from -// {{.URL}} +// Code generated by genzabbrs.go; DO NOT EDIT. +// Based on information from {{.URL}} package time diff --git a/libgo/go/time/mono_test.go b/libgo/go/time/mono_test.go new file mode 100644 index 00000000000..8778ab78a03 --- /dev/null +++ b/libgo/go/time/mono_test.go @@ -0,0 +1,261 @@ +// Copyright 2017 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package time_test + +import ( + "strings" + "testing" + . "time" +) + +func TestHasMonotonicClock(t *testing.T) { + yes := func(expr string, tt Time) { + if GetMono(&tt) == 0 { + t.Errorf("%s: missing monotonic clock reading", expr) + } + } + no := func(expr string, tt Time) { + if GetMono(&tt) != 0 { + t.Errorf("%s: unexpected monotonic clock reading", expr) + } + } + + yes("<-After(1)", <-After(1)) + ticker := NewTicker(1) + yes("<-Tick(1)", <-ticker.C) + ticker.Stop() + no("Date(2009, 11, 23, 0, 0, 0, 0, UTC)", Date(2009, 11, 23, 0, 0, 0, 0, UTC)) + tp, _ := Parse(UnixDate, "Sat Mar 7 11:06:39 PST 2015") + no(`Parse(UnixDate, "Sat Mar 7 11:06:39 PST 2015")`, tp) + no("Unix(1486057371, 0)", Unix(1486057371, 0)) + + yes("Now()", Now()) + + tu := Unix(1486057371, 0) + tm := tu + SetMono(&tm, 123456) + no("tu", tu) + yes("tm", tm) + + no("tu.Add(1)", tu.Add(1)) + no("tu.In(UTC)", tu.In(UTC)) + no("tu.AddDate(1, 1, 1)", tu.AddDate(1, 1, 1)) + no("tu.AddDate(0, 0, 0)", tu.AddDate(0, 0, 0)) + no("tu.Local()", tu.Local()) + no("tu.UTC()", tu.UTC()) + no("tu.Round(2)", tu.Round(2)) + no("tu.Truncate(2)", tu.Truncate(2)) + + yes("tm.Add(1)", tm.Add(1)) + no("tm.AddDate(1, 1, 1)", tm.AddDate(1, 1, 1)) + no("tm.AddDate(0, 0, 0)", tm.AddDate(0, 0, 0)) + no("tm.In(UTC)", tm.In(UTC)) + no("tm.Local()", tm.Local()) + no("tm.UTC()", tm.UTC()) + no("tm.Round(2)", tm.Round(2)) + no("tm.Truncate(2)", tm.Truncate(2)) +} + +func TestMonotonicAdd(t *testing.T) { + tm := Unix(1486057371, 123456) + SetMono(&tm, 123456789012345) + + t2 := tm.Add(1e8) + if t2.Nanosecond() != 100123456 { + t.Errorf("t2.Nanosecond() = %d, want 100123456", t2.Nanosecond()) + } + if GetMono(&t2) != 123456889012345 { + t.Errorf("t2.mono = %d, want 123456889012345", GetMono(&t2)) + } + + t3 := tm.Add(-9e18) // wall now out of range + if t3.Nanosecond() != 123456 { + t.Errorf("t3.Nanosecond() = %d, want 123456", t3.Nanosecond()) + } + if GetMono(&t3) != 0 { + t.Errorf("t3.mono = %d, want 0 (wall time out of range for monotonic reading)", GetMono(&t3)) + } + + t4 := tm.Add(+9e18) // wall now out of range + if t4.Nanosecond() != 123456 { + t.Errorf("t4.Nanosecond() = %d, want 123456", t4.Nanosecond()) + } + if GetMono(&t4) != 0 { + t.Errorf("t4.mono = %d, want 0 (wall time out of range for monotonic reading)", GetMono(&t4)) + } + + tn := Now() + tn1 := tn.Add(1 * Hour) + Sleep(100 * Millisecond) + d := Until(tn1) + if d < 59*Minute { + t.Errorf("Until(Now().Add(1*Hour)) = %v, wanted at least 59m", d) + } + now := Now() + if now.After(tn1) { + t.Errorf("Now().After(Now().Add(1*Hour)) = true, want false") + } + if !tn1.After(now) { + t.Errorf("Now().Add(1*Hour).After(now) = false, want true") + } + if tn1.Before(now) { + t.Errorf("Now().Add(1*Hour).Before(Now()) = true, want false") + } + if !now.Before(tn1) { + t.Errorf("Now().Before(Now().Add(1*Hour)) = false, want true") + } +} + +func TestMonotonicSub(t *testing.T) { + t1 := Unix(1483228799, 995e6) + SetMono(&t1, 123456789012345) + + t2 := Unix(1483228799, 5e6) + SetMono(&t2, 123456789012345+10e6) + + t3 := Unix(1483228799, 995e6) + SetMono(&t3, 123456789012345+1e9) + + t1w := t1.AddDate(0, 0, 0) + if GetMono(&t1w) != 0 { + t.Fatalf("AddDate didn't strip monotonic clock reading") + } + t2w := t2.AddDate(0, 0, 0) + if GetMono(&t2w) != 0 { + t.Fatalf("AddDate didn't strip monotonic clock reading") + } + t3w := t3.AddDate(0, 0, 0) + if GetMono(&t3w) != 0 { + t.Fatalf("AddDate didn't strip monotonic clock reading") + } + + sub := func(txs, tys string, tx, txw, ty, tyw Time, d, dw Duration) { + check := func(expr string, d, want Duration) { + if d != want { + t.Errorf("%s = %v, want %v", expr, d, want) + } + } + check(txs+".Sub("+tys+")", tx.Sub(ty), d) + check(txs+"w.Sub("+tys+")", txw.Sub(ty), dw) + check(txs+".Sub("+tys+"w)", tx.Sub(tyw), dw) + check(txs+"w.Sub("+tys+"w)", txw.Sub(tyw), dw) + } + sub("t1", "t1", t1, t1w, t1, t1w, 0, 0) + sub("t1", "t2", t1, t1w, t2, t2w, -10*Millisecond, 990*Millisecond) + sub("t1", "t3", t1, t1w, t3, t3w, -1000*Millisecond, 0) + + sub("t2", "t1", t2, t2w, t1, t1w, 10*Millisecond, -990*Millisecond) + sub("t2", "t2", t2, t2w, t2, t2w, 0, 0) + sub("t2", "t3", t2, t2w, t3, t3w, -990*Millisecond, -990*Millisecond) + + sub("t3", "t1", t3, t3w, t1, t1w, 1000*Millisecond, 0) + sub("t3", "t2", t3, t3w, t2, t2w, 990*Millisecond, 990*Millisecond) + sub("t3", "t3", t3, t3w, t3, t3w, 0, 0) + + cmp := func(txs, tys string, tx, txw, ty, tyw Time, c, cw int) { + check := func(expr string, b, want bool) { + if b != want { + t.Errorf("%s = %v, want %v", expr, b, want) + } + } + check(txs+".After("+tys+")", tx.After(ty), c > 0) + check(txs+"w.After("+tys+")", txw.After(ty), cw > 0) + check(txs+".After("+tys+"w)", tx.After(tyw), cw > 0) + check(txs+"w.After("+tys+"w)", txw.After(tyw), cw > 0) + + check(txs+".Before("+tys+")", tx.Before(ty), c < 0) + check(txs+"w.Before("+tys+")", txw.Before(ty), cw < 0) + check(txs+".Before("+tys+"w)", tx.Before(tyw), cw < 0) + check(txs+"w.Before("+tys+"w)", txw.Before(tyw), cw < 0) + + check(txs+".Equal("+tys+")", tx.Equal(ty), c == 0) + check(txs+"w.Equal("+tys+")", txw.Equal(ty), cw == 0) + check(txs+".Equal("+tys+"w)", tx.Equal(tyw), cw == 0) + check(txs+"w.Equal("+tys+"w)", txw.Equal(tyw), cw == 0) + } + + cmp("t1", "t1", t1, t1w, t1, t1w, 0, 0) + cmp("t1", "t2", t1, t1w, t2, t2w, -1, +1) + cmp("t1", "t3", t1, t1w, t3, t3w, -1, 0) + + cmp("t2", "t1", t2, t2w, t1, t1w, +1, -1) + cmp("t2", "t2", t2, t2w, t2, t2w, 0, 0) + cmp("t2", "t3", t2, t2w, t3, t3w, -1, -1) + + cmp("t3", "t1", t3, t3w, t1, t1w, +1, 0) + cmp("t3", "t2", t3, t3w, t2, t2w, +1, +1) + cmp("t3", "t3", t3, t3w, t3, t3w, 0, 0) +} + +func TestMonotonicOverflow(t *testing.T) { + t1 := Now().Add(-30 * Second) + d := Until(t1) + if d < -35*Second || -30*Second < d { + t.Errorf("Until(Now().Add(-30s)) = %v, want roughly -30s (-35s to -30s)", d) + } + + t1 = Now().Add(30 * Second) + d = Until(t1) + if d < 25*Second || 30*Second < d { + t.Errorf("Until(Now().Add(-30s)) = %v, want roughly 30s (25s to 30s)", d) + } + + t0 := Now() + t1 = t0.Add(Duration(1<<63 - 1)) + if GetMono(&t1) != 0 { + t.Errorf("Now().Add(maxDuration) has monotonic clock reading (%v => %v %d %d)", t0.String(), t1.String(), t0.Unix(), t1.Unix()) + } + t2 := t1.Add(-Duration(1<<63 - 1)) + d = Since(t2) + if d < -10*Second || 10*Second < d { + t.Errorf("Since(Now().Add(max).Add(-max)) = %v, want [-10s, 10s]", d) + } + + t0 = Now() + t1 = t0.Add(1 * Hour) + Sleep(100 * Millisecond) + t2 = Now().Add(-5 * Second) + if !t1.After(t2) { + t.Errorf("Now().Add(1*Hour).After(Now().Add(-5*Second)) = false, want true\nt1=%v\nt2=%v", t1, t2) + } + if t2.After(t1) { + t.Errorf("Now().Add(-5*Second).After(Now().Add(1*Hour)) = true, want false\nt1=%v\nt2=%v", t1, t2) + } + if t1.Before(t2) { + t.Errorf("Now().Add(1*Hour).Before(Now().Add(-5*Second)) = true, want false\nt1=%v\nt2=%v", t1, t2) + } + if !t2.Before(t1) { + t.Errorf("Now().Add(-5*Second).Before(Now().Add(1*Hour)) = false, want true\nt1=%v\nt2=%v", t1, t2) + } +} + +var monotonicStringTests = []struct { + mono int64 + want string +}{ + {0, "m=+0.000000000"}, + {123456789, "m=+0.123456789"}, + {-123456789, "m=-0.123456789"}, + {123456789000, "m=+123.456789000"}, + {-123456789000, "m=-123.456789000"}, + {9e18, "m=+9000000000.000000000"}, + {-9e18, "m=-9000000000.000000000"}, + {-1 << 63, "m=-9223372036.854775808"}, +} + +func TestMonotonicString(t *testing.T) { + t1 := Now() + t.Logf("Now() = %v", t1) + + for _, tt := range monotonicStringTests { + t1 := Now() + SetMono(&t1, tt.mono) + s := t1.String() + got := s[strings.LastIndex(s, " ")+1:] + if got != tt.want { + t.Errorf("with mono=%d: got %q; want %q", tt.mono, got, tt.want) + } + } +} diff --git a/libgo/go/time/sleep_test.go b/libgo/go/time/sleep_test.go index c286bd00273..546b28a28ac 100644 --- a/libgo/go/time/sleep_test.go +++ b/libgo/go/time/sleep_test.go @@ -227,7 +227,7 @@ func TestAfterQueuing(t *testing.T) { err := errors.New("!=nil") for i := 0; i < attempts && err != nil; i++ { delta := Duration(20+i*50) * Millisecond - if err = testAfterQueuing(t, delta); err != nil { + if err = testAfterQueuing(delta); err != nil { t.Logf("attempt %v failed: %v", i, err) } } @@ -248,7 +248,7 @@ func await(slot int, result chan<- afterResult, ac <-chan Time) { result <- afterResult{slot, <-ac} } -func testAfterQueuing(t *testing.T, delta Duration) error { +func testAfterQueuing(delta Duration) error { // make the result channel buffered because we don't want // to depend on channel queueing semantics that might // possibly change in the future. diff --git a/libgo/go/time/sys_plan9.go b/libgo/go/time/sys_plan9.go index 11365a791f7..9086a6e835f 100644 --- a/libgo/go/time/sys_plan9.go +++ b/libgo/go/time/sys_plan9.go @@ -19,6 +19,7 @@ func interrupt() { // readFile reads and returns the content of the named file. // It is a trivial implementation of ioutil.ReadFile, reimplemented // here to avoid depending on io/ioutil or os. +// It returns an error if name exceeds maxFileSize bytes. func readFile(name string) ([]byte, error) { f, err := syscall.Open(name, syscall.O_RDONLY) if err != nil { @@ -38,6 +39,9 @@ func readFile(name string) ([]byte, error) { if n == 0 || err != nil { break } + if len(ret) > maxFileSize { + return nil, fileSizeError(name) + } } return ret, err } diff --git a/libgo/go/time/sys_unix.go b/libgo/go/time/sys_unix.go index 4c68bbdc6d1..57152751800 100644 --- a/libgo/go/time/sys_unix.go +++ b/libgo/go/time/sys_unix.go @@ -19,6 +19,7 @@ func interrupt() { // readFile reads and returns the content of the named file. // It is a trivial implementation of ioutil.ReadFile, reimplemented // here to avoid depending on io/ioutil or os. +// It returns an error if name exceeds maxFileSize bytes. func readFile(name string) ([]byte, error) { f, err := syscall.Open(name, syscall.O_RDONLY, 0) if err != nil { @@ -38,6 +39,9 @@ func readFile(name string) ([]byte, error) { if n == 0 || err != nil { break } + if len(ret) > maxFileSize { + return nil, fileSizeError(name) + } } return ret, err } diff --git a/libgo/go/time/sys_windows.go b/libgo/go/time/sys_windows.go index a4a068f7849..9e381653937 100644 --- a/libgo/go/time/sys_windows.go +++ b/libgo/go/time/sys_windows.go @@ -16,6 +16,7 @@ func interrupt() { // readFile reads and returns the content of the named file. // It is a trivial implementation of ioutil.ReadFile, reimplemented // here to avoid depending on io/ioutil or os. +// It returns an error if name exceeds maxFileSize bytes. func readFile(name string) ([]byte, error) { f, err := syscall.Open(name, syscall.O_RDONLY, 0) if err != nil { @@ -35,6 +36,9 @@ func readFile(name string) ([]byte, error) { if n == 0 || err != nil { break } + if len(ret) > maxFileSize { + return nil, fileSizeError(name) + } } return ret, err } diff --git a/libgo/go/time/time.go b/libgo/go/time/time.go index 10b32461e1c..8a29eef2631 100644 --- a/libgo/go/time/time.go +++ b/libgo/go/time/time.go @@ -6,6 +6,69 @@ // // The calendrical calculations always assume a Gregorian calendar, with // no leap seconds. +// +// Monotonic Clocks +// +// Operating systems provide both a “wall clock,” which is subject to +// changes for clock synchronization, and a “monotonic clock,” which is +// not. The general rule is that the wall clock is for telling time and +// the monotonic clock is for measuring time. Rather than split the API, +// in this package the Time returned by time.Now contains both a wall +// clock reading and a monotonic clock reading; later time-telling +// operations use the wall clock reading, but later time-measuring +// operations, specifically comparisons and subtractions, use the +// monotonic clock reading. +// +// For example, this code always computes a positive elapsed time of +// approximately 20 milliseconds, even if the wall clock is changed during +// the operation being timed: +// +// start := time.Now() +// ... operation that takes 20 milliseconds ... +// t := time.Now() +// elapsed := t.Sub(start) +// +// Other idioms, such as time.Since(start), time.Until(deadline), and +// time.Now().Before(deadline), are similarly robust against wall clock +// resets. +// +// The rest of this section gives the precise details of how operations +// use monotonic clocks, but understanding those details is not required +// to use this package. +// +// The Time returned by time.Now contains a monotonic clock reading. +// If Time t has a monotonic clock reading, t.Add adds the same duration to +// both the wall clock and monotonic clock readings to compute the result. +// Because t.AddDate(y, m, d), t.Round(d), and t.Truncate(d) are wall time +// computations, they always strip any monotonic clock reading from their results. +// Because t.In, t.Local, and t.UTC are used for their effect on the interpretation +// of the wall time, they also strip any monotonic clock reading from their results. +// The canonical way to strip a monotonic clock reading is to use t = t.Round(0). +// +// If Times t and u both contain monotonic clock readings, the operations +// t.After(u), t.Before(u), t.Equal(u), and t.Sub(u) are carried out +// using the monotonic clock readings alone, ignoring the wall clock +// readings. If either t or u contains no monotonic clock reading, these +// operations fall back to using the wall clock readings. +// +// Because the monotonic clock reading has no meaning outside +// the current process, the serialized forms generated by t.GobEncode, +// t.MarshalBinary, t.MarshalJSON, and t.MarshalText omit the monotonic +// clock reading, and t.Format provides no format for it. Similarly, the +// constructors time.Date, time.Parse, time.ParseInLocation, and time.Unix, +// as well as the unmarshalers t.GobDecode, t.UnmarshalBinary. +// t.UnmarshalJSON, and t.UnmarshalText always create times with +// no monotonic clock reading. +// +// Note that the Go == operator compares not just the time instant but +// also the Location and the monotonic clock reading. See the +// documentation for the Time type for a discussion of equality +// testing for Time values. +// +// For debugging, the result of t.String does include the monotonic +// clock reading if present. If t != u because of different monotonic clock readings, +// that difference will be visible when printing t.String() and u.String(). +// package time import "errors" @@ -14,8 +77,11 @@ import "errors" // // Programs using times should typically store and pass them as values, // not pointers. That is, time variables and struct fields should be of -// type time.Time, not *time.Time. A Time value can be used by -// multiple goroutines simultaneously. +// type time.Time, not *time.Time. +// +// A Time value can be used by multiple goroutines simultaneously except +// that the methods GobDecode, UnmarshalBinary, UnmarshalJSON and +// UnmarshalText are not concurrency-safe. // // Time instants can be compared using the Before, After, and Equal methods. // The Sub method subtracts two instants, producing a Duration. @@ -33,19 +99,34 @@ import "errors" // computations described in earlier paragraphs. // // Note that the Go == operator compares not just the time instant but also the -// Location. Therefore, Time values should not be used as map or database keys -// without first guaranteeing that the identical Location has been set for all -// values, which can be achieved through use of the UTC or Local method. +// Location and the monotonic clock reading. Therefore, Time values should not +// be used as map or database keys without first guaranteeing that the +// identical Location has been set for all values, which can be achieved +// through use of the UTC or Local method, and that the monotonic clock reading +// has been stripped by setting t = t.Round(0). In general, prefer t.Equal(u) +// to t == u, since t.Equal uses the most accurate comparison available and +// correctly handles the case when only one of its arguments has a monotonic +// clock reading. +// +// In addition to the required “wall clock” reading, a Time may contain an optional +// reading of the current process's monotonic clock, to provide additional precision +// for comparison or subtraction. +// See the “Monotonic Clocks” section in the package documentation for details. // type Time struct { - // sec gives the number of seconds elapsed since - // January 1, year 1 00:00:00 UTC. - sec int64 - - // nsec specifies a non-negative nanosecond - // offset within the second named by Seconds. - // It must be in the range [0, 999999999]. - nsec int32 + // wall and ext encode the wall time seconds, wall time nanoseconds, + // and optional monotonic clock reading in nanoseconds. + // + // From high to low bit position, wall encodes a 1-bit flag (hasMonotonic), + // a 33-bit seconds field, and a 30-bit wall time nanoseconds field. + // The nanoseconds field is in the range [0, 999999999]. + // If the hasMonotonic bit is 0, then the 33-bit field must be zero + // and the full signed 64-bit wall seconds since Jan 1 year 1 is stored in ext. + // If the hasMonotonic bit is 1, then the 33-bit field holds a 33-bit + // unsigned wall seconds since Jan 1 year 1885, and ext holds a + // signed 64-bit monotonic clock reading, nanoseconds since process start. + wall uint64 + ext int64 // loc specifies the Location that should be used to // determine the minute, hour, month, day, and year @@ -55,29 +136,124 @@ type Time struct { loc *Location } +const ( + hasMonotonic = 1 << 63 + maxWall = wallToInternal + (1<<33 - 1) // year 2157 + minWall = wallToInternal // year 1885 + nsecMask = 1<<30 - 1 + nsecShift = 30 +) + +// These helpers for manipulating the wall and monotonic clock readings +// take pointer receivers, even when they don't modify the time, +// to make them cheaper to call. + +// nsec returns the time's nanoseconds. +func (t *Time) nsec() int32 { + return int32(t.wall & nsecMask) +} + +// sec returns the time's seconds since Jan 1 year 1. +func (t *Time) sec() int64 { + if t.wall&hasMonotonic != 0 { + return wallToInternal + int64(t.wall<<1>>(nsecShift+1)) + } + return int64(t.ext) +} + +// unixSec returns the time's seconds since Jan 1 1970 (Unix time). +func (t *Time) unixSec() int64 { return t.sec() + internalToUnix } + +// addSec adds d seconds to the time. +func (t *Time) addSec(d int64) { + if t.wall&hasMonotonic != 0 { + sec := int64(t.wall << 1 >> (nsecShift + 1)) + dsec := sec + d + if 0 <= dsec && dsec <= 1<<33-1 { + t.wall = t.wall&nsecMask | uint64(dsec)< u.sec || t.sec == u.sec && t.nsec > u.nsec + if t.wall&u.wall&hasMonotonic != 0 { + return t.ext > u.ext + } + ts := t.sec() + us := u.sec() + return ts > us || ts == us && t.nsec() > u.nsec() } // Before reports whether the time instant t is before u. func (t Time) Before(u Time) bool { - return t.sec < u.sec || t.sec == u.sec && t.nsec < u.nsec + if t.wall&u.wall&hasMonotonic != 0 { + return t.ext < u.ext + } + return t.sec() < u.sec() || t.sec() == u.sec() && t.nsec() < u.nsec() } // Equal reports whether t and u represent the same time instant. // Two times can be equal even if they are in different locations. // For example, 6:00 +0200 CEST and 4:00 UTC are Equal. -// Do not use == with Time values. +// See the documentation on the Time type for the pitfalls of using == with +// Time values; most code should use Equal instead. func (t Time) Equal(u Time) bool { - return t.sec == u.sec && t.nsec == u.nsec + if t.wall&u.wall&hasMonotonic != 0 { + return t.ext == u.ext + } + return t.sec() == u.sec() && t.nsec() == u.nsec() } // A Month specifies a month of the year (January = 1, ...). @@ -162,7 +338,7 @@ func (d Weekday) String() string { return days[d] } // The zero Time value does not force a specific epoch for the time // representation. For example, to use the Unix epoch internally, we // could define that to distinguish a zero value from Jan 1 1970, that -// time would be represented by sec=-1, nsec=1e9. However, it does +// time would be represented by sec=-1, nsec=1e9. However, it does // suggest a representation, namely using 1-1-1 00:00:00 UTC as the // epoch, and that's what we do. // @@ -194,7 +370,7 @@ func (d Weekday) String() string { return days[d] } // everywhere. // // The calendar runs on an exact 400 year cycle: a 400-year calendar -// printed for 1970-2469 will apply as well to 2370-2769. Even the days +// printed for 1970-2369 will apply as well to 2370-2769. Even the days // of the week match up. It simplifies the computations to choose the // cycle boundaries so that the exceptional years are always delayed as // long as possible. That means choosing a year equal to 1 mod 400, so @@ -208,7 +384,7 @@ func (d Weekday) String() string { return days[d] } // // These three considerations—choose an epoch as early as possible, that // uses a year equal to 1 mod 400, and that is no more than 2⁶³ seconds -// earlier than 1970—bring us to the year -292277022399. We refer to +// earlier than 1970—bring us to the year -292277022399. We refer to // this year as the absolute zero year, and to times measured as a uint64 // seconds since this year as absolute times. // @@ -219,9 +395,9 @@ func (d Weekday) String() string { return days[d] } // times. // // It is tempting to just use the year 1 as the absolute epoch, defining -// that the routines are only valid for years >= 1. However, the +// that the routines are only valid for years >= 1. However, the // routines would then be invalid when displaying the epoch in time zones -// west of UTC, since it is year 0. It doesn't seem tenable to say that +// west of UTC, since it is year 0. It doesn't seem tenable to say that // printing the zero time correctly isn't supported in half the time // zones. By comparison, it's reasonable to mishandle some times in // the year -292277022399. @@ -245,12 +421,15 @@ const ( unixToInternal int64 = (1969*365 + 1969/4 - 1969/100 + 1969/400) * secondsPerDay internalToUnix int64 = -unixToInternal + + wallToInternal int64 = (1884*365 + 1884/4 - 1884/100 + 1884/400) * secondsPerDay + internalToWall int64 = -wallToInternal ) // IsZero reports whether t represents the zero time instant, // January 1, year 1, 00:00:00 UTC. func (t Time) IsZero() bool { - return t.sec == 0 && t.nsec == 0 + return t.sec() == 0 && t.nsec() == 0 } // abs returns the time t as an absolute time, adjusted by the zone offset. @@ -261,7 +440,7 @@ func (t Time) abs() uint64 { if l == nil || l == &localLoc { l = l.get() } - sec := t.sec + internalToUnix + sec := t.unixSec() if l != &utcLoc { if l.cacheZone != nil && l.cacheStart <= sec && sec < l.cacheEnd { sec += int64(l.cacheZone.offset) @@ -281,7 +460,7 @@ func (t Time) locabs() (name string, offset int, abs uint64) { l = l.get() } // Avoid function call if we hit the local time cache. - sec := t.sec + internalToUnix + sec := t.unixSec() if l != &utcLoc { if l.cacheZone != nil && l.cacheStart <= sec && sec < l.cacheEnd { name = l.cacheZone.name @@ -425,7 +604,7 @@ func (t Time) Second() int { // Nanosecond returns the nanosecond offset within the second specified by t, // in the range [0, 999999999]. func (t Time) Nanosecond() int { - return int(t.nsec) + return int(t.nsec()) } // YearDay returns the day of the year specified by t, in the range [1,365] for non-leap years, @@ -543,8 +722,8 @@ func (d Duration) String() string { } // fmtFrac formats the fraction of v/10**prec (e.g., ".12345") into the -// tail of buf, omitting trailing zeros. it omits the decimal -// point too when the fraction is 0. It returns the index where the +// tail of buf, omitting trailing zeros. it omits the decimal +// point too when the fraction is 0. It returns the index where the // output bytes begin and the value v/10**prec. func fmtFrac(buf []byte, v uint64, prec int) (nw int, nv uint64) { // Omit trailing zeros up to and including decimal point. @@ -616,18 +795,73 @@ func (d Duration) Hours() float64 { return float64(hour) + float64(nsec)/(60*60*1e9) } +// Truncate returns the result of rounding d toward zero to a multiple of m. +// If m <= 0, Truncate returns d unchanged. +func (d Duration) Truncate(m Duration) Duration { + if m <= 0 { + return d + } + return d - d%m +} + +// lessThanHalf reports whether x+x < y but avoids overflow, +// assuming x and y are both positive (Duration is signed). +func lessThanHalf(x, y Duration) bool { + return uint64(x)+uint64(x) < uint64(y) +} + +// Round returns the result of rounding d to the nearest multiple of m. +// The rounding behavior for halfway values is to round away from zero. +// If the result exceeds the maximum (or minimum) +// value that can be stored in a Duration, +// Round returns the maximum (or minimum) duration. +// If m <= 0, Round returns d unchanged. +func (d Duration) Round(m Duration) Duration { + if m <= 0 { + return d + } + r := d % m + if d < 0 { + r = -r + if lessThanHalf(r, m) { + return d + r + } + if d1 := d - m + r; d1 < d { + return d1 + } + return minDuration // overflow + } + if lessThanHalf(r, m) { + return d - r + } + if d1 := d + m - r; d1 > d { + return d1 + } + return maxDuration // overflow +} + // Add returns the time t+d. func (t Time) Add(d Duration) Time { - t.sec += int64(d / 1e9) - nsec := t.nsec + int32(d%1e9) + dsec := int64(d / 1e9) + nsec := t.nsec() + int32(d%1e9) if nsec >= 1e9 { - t.sec++ + dsec++ nsec -= 1e9 } else if nsec < 0 { - t.sec-- + dsec-- nsec += 1e9 } - t.nsec = nsec + t.wall = t.wall&^nsecMask | uint64(nsec) // update nsec + t.addSec(dsec) + if t.wall&hasMonotonic != 0 { + te := t.ext + int64(d) + if d < 0 && te > int64(t.ext) || d > 0 && te < int64(t.ext) { + // Monotonic clock reading now out of range; degrade to wall-only. + t.stripMono() + } else { + t.ext = te + } + } return t } @@ -636,7 +870,19 @@ func (t Time) Add(d Duration) Time { // will be returned. // To compute t-d for a duration d, use t.Add(-d). func (t Time) Sub(u Time) Duration { - d := Duration(t.sec-u.sec)*Second + Duration(t.nsec-u.nsec) + if t.wall&u.wall&hasMonotonic != 0 { + te := int64(t.ext) + ue := int64(u.ext) + d := Duration(te - ue) + if d < 0 && te > ue { + return maxDuration // t - u is positive out of range + } + if d > 0 && te < ue { + return minDuration // t - u is negative out of range + } + return d + } + d := Duration(t.sec()-u.sec())*Second + Duration(t.nsec()-u.nsec()) // Check for overflow or underflow. switch { case u.Add(d).Equal(t): @@ -671,7 +917,7 @@ func Until(t Time) Duration { func (t Time) AddDate(years int, months int, days int) Time { year, month, day := t.Date() hour, min, sec := t.Clock() - return Date(year+years, month+Month(months), day+days, hour, min, sec, int(t.nsec), t.Location()) + return Date(year+years, month+Month(months), day+days, hour, min, sec, int(t.nsec()), t.Location()) } const ( @@ -718,7 +964,7 @@ func absDate(abs uint64, full bool) (year int, month Month, day int, yday int) { // Cut off years within a 4-year cycle. // The last year is a leap year, so on the last day of that year, - // day / 365 will be 4 instead of 3. Cut it back down to 3 + // day / 365 will be 4 instead of 3. Cut it back down to 3 // by subtracting n>>2. n = d / 365 n -= n >> 2 @@ -791,12 +1037,20 @@ func daysIn(m Month, year int) int { } // Provided by package runtime. -func now() (sec int64, nsec int32) +func now() (sec int64, nsec int32, mono int64) // Now returns the current local time. func Now() Time { - sec, nsec := now() - return Time{sec + unixToInternal, nsec, Local} + sec, nsec, mono := now() + sec += unixToInternal - minWall + if uint64(sec)>>33 != 0 { + return Time{uint64(nsec), sec + minWall, Local} + } + return Time{hasMonotonic | uint64(sec)<> 56), // bytes 1-8: seconds - byte(t.sec >> 48), - byte(t.sec >> 40), - byte(t.sec >> 32), - byte(t.sec >> 24), - byte(t.sec >> 16), - byte(t.sec >> 8), - byte(t.sec), - byte(t.nsec >> 24), // bytes 9-12: nanoseconds - byte(t.nsec >> 16), - byte(t.nsec >> 8), - byte(t.nsec), + byte(sec >> 56), // bytes 1-8: seconds + byte(sec >> 48), + byte(sec >> 40), + byte(sec >> 32), + byte(sec >> 24), + byte(sec >> 16), + byte(sec >> 8), + byte(sec), + byte(nsec >> 24), // bytes 9-12: nanoseconds + byte(nsec >> 16), + byte(nsec >> 8), + byte(nsec), byte(offsetMin >> 8), // bytes 13-14: zone offset in minutes byte(offsetMin), } @@ -910,18 +1166,22 @@ func (t *Time) UnmarshalBinary(data []byte) error { } buf = buf[1:] - t.sec = int64(buf[7]) | int64(buf[6])<<8 | int64(buf[5])<<16 | int64(buf[4])<<24 | + sec := int64(buf[7]) | int64(buf[6])<<8 | int64(buf[5])<<16 | int64(buf[4])<<24 | int64(buf[3])<<32 | int64(buf[2])<<40 | int64(buf[1])<<48 | int64(buf[0])<<56 buf = buf[8:] - t.nsec = int32(buf[3]) | int32(buf[2])<<8 | int32(buf[1])<<16 | int32(buf[0])<<24 + nsec := int32(buf[3]) | int32(buf[2])<<8 | int32(buf[1])<<16 | int32(buf[0])<<24 buf = buf[4:] offset := int(int16(buf[1])|int16(buf[0])<<8) * 60 + *t = Time{} + t.wall = uint64(nsec) + t.ext = sec + if offset == -1*60 { t.setLoc(&utcLoc) - } else if _, localoff, _, _, _ := Local.lookup(t.sec + internalToUnix); offset == localoff { + } else if _, localoff, _, _, _ := Local.lookup(t.unixSec()); offset == localoff { t.setLoc(Local) } else { t.setLoc(FixedZone("", offset)) @@ -1008,7 +1268,7 @@ func Unix(sec int64, nsec int64) Time { sec-- } } - return Time{sec + unixToInternal, int32(nsec), Local} + return unixTime(sec, int32(nsec)) } func isLeap(year int) bool { @@ -1117,7 +1377,7 @@ func Date(year int, month Month, day, hour, min, sec, nsec int, loc *Location) T unix -= int64(offset) } - t := Time{unix + unixToInternal, int32(nsec), nil} + t := unixTime(unix, int32(nsec)) t.setLoc(loc) return t } @@ -1130,6 +1390,7 @@ func Date(year int, month Month, day, hour, min, sec, nsec int, loc *Location) T // time. Thus, Truncate(Hour) may return a time with a non-zero // minute, depending on the time's Location. func (t Time) Truncate(d Duration) Time { + t.stripMono() if d <= 0 { return t } @@ -1146,11 +1407,12 @@ func (t Time) Truncate(d Duration) Time { // time. Thus, Round(Hour) may return a time with a non-zero // minute, depending on the time's Location. func (t Time) Round(d Duration) Time { + t.stripMono() if d <= 0 { return t } _, r := div(t, d) - if r+r < d { + if lessThanHalf(r, d) { return t.Add(-r) } return t.Add(d - r) @@ -1161,15 +1423,16 @@ func (t Time) Round(d Duration) Time { // but it's still here in case we change our minds. func div(t Time, d Duration) (qmod2 int, r Duration) { neg := false - nsec := t.nsec - if t.sec < 0 { + nsec := t.nsec() + sec := t.sec() + if sec < 0 { // Operate on absolute value. neg = true - t.sec = -t.sec + sec = -sec nsec = -nsec if nsec < 0 { nsec += 1e9 - t.sec-- // t.sec >= 1 before the -- so safe + sec-- // sec >= 1 before the -- so safe } } @@ -1182,8 +1445,8 @@ func div(t Time, d Duration) (qmod2 int, r Duration) { // Special case: d is a multiple of 1 second. case d%Second == 0: d1 := int64(d / Second) - qmod2 = int(t.sec/d1) & 1 - r = Duration(t.sec%d1)*Second + Duration(nsec) + qmod2 = int(sec/d1) & 1 + r = Duration(sec%d1)*Second + Duration(nsec) // General case. // This could be faster if more cleverness were applied, @@ -1191,7 +1454,7 @@ func div(t Time, d Duration) (qmod2 int, r Duration) { // No one will care about these cases. default: // Compute nanoseconds as 128-bit number. - sec := uint64(t.sec) + sec := uint64(sec) tmp := (sec >> 32) * 1e9 u1 := tmp >> 32 u0 := tmp << 32 diff --git a/libgo/go/time/time_test.go b/libgo/go/time/time_test.go index 2922560f097..dba8e0dadcd 100644 --- a/libgo/go/time/time_test.go +++ b/libgo/go/time/time_test.go @@ -11,7 +11,9 @@ import ( "fmt" "math/big" "math/rand" + "os" "runtime" + "strings" "testing" "testing/quick" . "time" @@ -231,6 +233,7 @@ var truncateRoundTests = []struct { {Date(-1, January, 1, 12, 15, 31, 5e8, UTC), 3}, {Date(2012, January, 1, 12, 15, 30, 5e8, UTC), Second}, {Date(2012, January, 1, 12, 15, 31, 5e8, UTC), Second}, + {Unix(-19012425939, 649146258), 7435029458905025217}, // 5.8*d rounds to 6*d, but .8*d+.8*d < 0 < d } func TestTruncateRound(t *testing.T) { @@ -1056,6 +1059,66 @@ func TestDurationHours(t *testing.T) { } } +var durationTruncateTests = []struct { + d Duration + m Duration + want Duration +}{ + {0, Second, 0}, + {Minute, -7 * Second, Minute}, + {Minute, 0, Minute}, + {Minute, 1, Minute}, + {Minute + 10*Second, 10 * Second, Minute + 10*Second}, + {2*Minute + 10*Second, Minute, 2 * Minute}, + {10*Minute + 10*Second, 3 * Minute, 9 * Minute}, + {Minute + 10*Second, Minute + 10*Second + 1, 0}, + {Minute + 10*Second, Hour, 0}, + {-Minute, Second, -Minute}, + {-10 * Minute, 3 * Minute, -9 * Minute}, + {-10 * Minute, Hour, 0}, +} + +func TestDurationTruncate(t *testing.T) { + for _, tt := range durationTruncateTests { + if got := tt.d.Truncate(tt.m); got != tt.want { + t.Errorf("Duration(%s).Truncate(%s) = %s; want: %s", tt.d, tt.m, got, tt.want) + } + } +} + +var durationRoundTests = []struct { + d Duration + m Duration + want Duration +}{ + {0, Second, 0}, + {Minute, -11 * Second, Minute}, + {Minute, 0, Minute}, + {Minute, 1, Minute}, + {2 * Minute, Minute, 2 * Minute}, + {2*Minute + 10*Second, Minute, 2 * Minute}, + {2*Minute + 30*Second, Minute, 3 * Minute}, + {2*Minute + 50*Second, Minute, 3 * Minute}, + {-Minute, 1, -Minute}, + {-2 * Minute, Minute, -2 * Minute}, + {-2*Minute - 10*Second, Minute, -2 * Minute}, + {-2*Minute - 30*Second, Minute, -3 * Minute}, + {-2*Minute - 50*Second, Minute, -3 * Minute}, + {8e18, 3e18, 9e18}, + {9e18, 5e18, 1<<63 - 1}, + {-8e18, 3e18, -9e18}, + {-9e18, 5e18, -1 << 63}, + {3<<61 - 1, 3 << 61, 3 << 61}, +} + +func TestDurationRound(t *testing.T) { + for _, tt := range durationRoundTests { + if got := tt.d.Round(tt.m); got != tt.want { + t.Errorf("Duration(%s).Round(%s) = %s; want: %s", tt.d, tt.m, got, tt.want) + } + } +} + var defaultLocTests = []struct { name string f func(t1, t2 Time) bool @@ -1254,3 +1317,14 @@ func TestZeroMonthString(t *testing.T) { t.Errorf("zero month = %q; want %q", got, want) } } + +func TestReadFileLimit(t *testing.T) { + const zero = "/dev/zero" + if _, err := os.Stat(zero); err != nil { + t.Skip("skipping test without a /dev/zero") + } + _, err := ReadFile(zero) + if err == nil || !strings.Contains(err.Error(), "is too large") { + t.Errorf("readFile(%q) error = %v; want error containing 'is too large'", zero, err) + } +} diff --git a/libgo/go/time/zoneinfo.go b/libgo/go/time/zoneinfo.go index fb0aa392404..f4d4df95d36 100644 --- a/libgo/go/time/zoneinfo.go +++ b/libgo/go/time/zoneinfo.go @@ -5,6 +5,7 @@ package time import ( + "errors" "sync" "syscall" ) @@ -81,7 +82,7 @@ func (l *Location) get() *Location { } // String returns a descriptive name for the time zone information, -// corresponding to the argument to LoadLocation. +// corresponding to the name argument to LoadLocation or FixedZone. func (l *Location) String() string { return l.get().name } @@ -256,7 +257,10 @@ func (l *Location) lookupName(name string, unix int64) (offset int, isDST bool, // NOTE(rsc): Eventually we will need to accept the POSIX TZ environment // syntax too, but I don't feel like implementing it today. -var zoneinfo, _ = syscall.Getenv("ZONEINFO") +var errLocation = errors.New("time: invalid location name") + +var zoneinfo *string +var zoneinfoOnce sync.Once // LoadLocation returns the Location with the given name. // @@ -279,11 +283,33 @@ func LoadLocation(name string) (*Location, error) { if name == "Local" { return Local, nil } - if zoneinfo != "" { - if z, err := loadZoneFile(zoneinfo, name); err == nil { + if containsDotDot(name) || name[0] == '/' || name[0] == '\\' { + // No valid IANA Time Zone name contains a single dot, + // much less dot dot. Likewise, none begin with a slash. + return nil, errLocation + } + zoneinfoOnce.Do(func() { + env, _ := syscall.Getenv("ZONEINFO") + zoneinfo = &env + }) + if zoneinfo != nil && *zoneinfo != "" { + if z, err := loadZoneFile(*zoneinfo, name); err == nil { z.name = name return z, nil } } return loadLocation(name) } + +// containsDotDot reports whether s contains "..". +func containsDotDot(s string) bool { + if len(s) < 2 { + return false + } + for i := 0; i < len(s)-1; i++ { + if s[i] == '.' && s[i+1] == '.' { + return true + } + } + return false +} diff --git a/libgo/go/time/zoneinfo_abbrs_windows.go b/libgo/go/time/zoneinfo_abbrs_windows.go index 9425db844ca..db0bbfd74f4 100644 --- a/libgo/go/time/zoneinfo_abbrs_windows.go +++ b/libgo/go/time/zoneinfo_abbrs_windows.go @@ -2,8 +2,8 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -// generated by genzabbrs.go from -// http://unicode.org/cldr/data/common/supplemental/windowsZones.xml +// Code generated by genzabbrs.go; DO NOT EDIT. +// Based on information from http://unicode.org/cldr/data/common/supplemental/windowsZones.xml package time @@ -22,124 +22,128 @@ var abbrs = map[string]abbr{ "Namibia Standard Time": {"WAT", "WAST"}, // Africa/Windhoek "Aleutian Standard Time": {"HST", "HDT"}, // America/Adak "Alaskan Standard Time": {"AKST", "AKDT"}, // America/Anchorage - "Tocantins Standard Time": {"BRT", "BRT"}, // America/Araguaina - "Paraguay Standard Time": {"PYT", "PYST"}, // America/Asuncion - "Bahia Standard Time": {"BRT", "BRT"}, // America/Bahia - "SA Pacific Standard Time": {"COT", "COT"}, // America/Bogota - "Argentina Standard Time": {"ART", "ART"}, // America/Buenos_Aires + "Tocantins Standard Time": {"-03", "-03"}, // America/Araguaina + "Paraguay Standard Time": {"-04", "-03"}, // America/Asuncion + "Bahia Standard Time": {"-03", "-03"}, // America/Bahia + "SA Pacific Standard Time": {"-05", "-05"}, // America/Bogota + "Argentina Standard Time": {"-03", "-03"}, // America/Buenos_Aires "Eastern Standard Time (Mexico)": {"EST", "EST"}, // America/Cancun - "Venezuela Standard Time": {"VET", "VET"}, // America/Caracas - "SA Eastern Standard Time": {"GFT", "GFT"}, // America/Cayenne + "Venezuela Standard Time": {"-04", "-04"}, // America/Caracas + "SA Eastern Standard Time": {"-03", "-03"}, // America/Cayenne "Central Standard Time": {"CST", "CDT"}, // America/Chicago "Mountain Standard Time (Mexico)": {"MST", "MDT"}, // America/Chihuahua - "Central Brazilian Standard Time": {"AMT", "AMST"}, // America/Cuiaba + "Central Brazilian Standard Time": {"-04", "-03"}, // America/Cuiaba "Mountain Standard Time": {"MST", "MDT"}, // America/Denver - "Greenland Standard Time": {"WGT", "WGST"}, // America/Godthab + "Greenland Standard Time": {"-03", "-02"}, // America/Godthab "Turks And Caicos Standard Time": {"AST", "AST"}, // America/Grand_Turk "Central America Standard Time": {"CST", "CST"}, // America/Guatemala "Atlantic Standard Time": {"AST", "ADT"}, // America/Halifax "Cuba Standard Time": {"CST", "CDT"}, // America/Havana "US Eastern Standard Time": {"EST", "EDT"}, // America/Indianapolis - "SA Western Standard Time": {"BOT", "BOT"}, // America/La_Paz + "SA Western Standard Time": {"-04", "-04"}, // America/La_Paz "Pacific Standard Time": {"PST", "PDT"}, // America/Los_Angeles "Central Standard Time (Mexico)": {"CST", "CDT"}, // America/Mexico_City - "Saint Pierre Standard Time": {"PMST", "PMDT"}, // America/Miquelon - "Montevideo Standard Time": {"UYT", "UYT"}, // America/Montevideo + "Saint Pierre Standard Time": {"-03", "-02"}, // America/Miquelon + "Montevideo Standard Time": {"-03", "-03"}, // America/Montevideo "Eastern Standard Time": {"EST", "EDT"}, // America/New_York "US Mountain Standard Time": {"MST", "MST"}, // America/Phoenix - "Haiti Standard Time": {"EST", "EST"}, // America/Port-au-Prince + "Haiti Standard Time": {"EST", "EDT"}, // America/Port-au-Prince + "Magallanes Standard Time": {"-03", "-03"}, // America/Punta_Arenas "Canada Central Standard Time": {"CST", "CST"}, // America/Regina - "Pacific SA Standard Time": {"CLT", "CLST"}, // America/Santiago - "E. South America Standard Time": {"BRT", "BRST"}, // America/Sao_Paulo + "Pacific SA Standard Time": {"-04", "-03"}, // America/Santiago + "E. South America Standard Time": {"-03", "-02"}, // America/Sao_Paulo "Newfoundland Standard Time": {"NST", "NDT"}, // America/St_Johns "Pacific Standard Time (Mexico)": {"PST", "PDT"}, // America/Tijuana "Central Asia Standard Time": {"+06", "+06"}, // Asia/Almaty "Jordan Standard Time": {"EET", "EEST"}, // Asia/Amman - "Arabic Standard Time": {"AST", "AST"}, // Asia/Baghdad - "Azerbaijan Standard Time": {"AZT", "AZT"}, // Asia/Baku - "SE Asia Standard Time": {"ICT", "ICT"}, // Asia/Bangkok - "Altai Standard Time": {"+06", "+07"}, // Asia/Barnaul + "Arabic Standard Time": {"+03", "+03"}, // Asia/Baghdad + "Azerbaijan Standard Time": {"+04", "+04"}, // Asia/Baku + "SE Asia Standard Time": {"+07", "+07"}, // Asia/Bangkok + "Altai Standard Time": {"+07", "+07"}, // Asia/Barnaul "Middle East Standard Time": {"EET", "EEST"}, // Asia/Beirut "India Standard Time": {"IST", "IST"}, // Asia/Calcutta - "Transbaikal Standard Time": {"IRKT", "YAKT"}, // Asia/Chita - "Sri Lanka Standard Time": {"IST", "IST"}, // Asia/Colombo + "Transbaikal Standard Time": {"+09", "+09"}, // Asia/Chita + "Sri Lanka Standard Time": {"+0530", "+0530"}, // Asia/Colombo "Syria Standard Time": {"EET", "EEST"}, // Asia/Damascus - "Bangladesh Standard Time": {"BDT", "BDT"}, // Asia/Dhaka - "Arabian Standard Time": {"GST", "GST"}, // Asia/Dubai + "Bangladesh Standard Time": {"+06", "+06"}, // Asia/Dhaka + "Arabian Standard Time": {"+04", "+04"}, // Asia/Dubai "West Bank Standard Time": {"EET", "EEST"}, // Asia/Hebron - "W. Mongolia Standard Time": {"HOVT", "HOVST"}, // Asia/Hovd - "North Asia East Standard Time": {"IRKT", "IRKT"}, // Asia/Irkutsk + "W. Mongolia Standard Time": {"+07", "+07"}, // Asia/Hovd + "North Asia East Standard Time": {"+08", "+08"}, // Asia/Irkutsk "Israel Standard Time": {"IST", "IDT"}, // Asia/Jerusalem - "Afghanistan Standard Time": {"AFT", "AFT"}, // Asia/Kabul - "Russia Time Zone 11": {"PETT", "PETT"}, // Asia/Kamchatka + "Afghanistan Standard Time": {"+0430", "+0430"}, // Asia/Kabul + "Russia Time Zone 11": {"+12", "+12"}, // Asia/Kamchatka "Pakistan Standard Time": {"PKT", "PKT"}, // Asia/Karachi - "Nepal Standard Time": {"NPT", "NPT"}, // Asia/Katmandu - "North Asia Standard Time": {"KRAT", "KRAT"}, // Asia/Krasnoyarsk - "Magadan Standard Time": {"MAGT", "MAGT"}, // Asia/Magadan - "N. Central Asia Standard Time": {"+06", "+07"}, // Asia/Novosibirsk + "Nepal Standard Time": {"+0545", "+0545"}, // Asia/Katmandu + "North Asia Standard Time": {"+07", "+07"}, // Asia/Krasnoyarsk + "Magadan Standard Time": {"+11", "+11"}, // Asia/Magadan + "N. Central Asia Standard Time": {"+07", "+07"}, // Asia/Novosibirsk + "Omsk Standard Time": {"+06", "+06"}, // Asia/Omsk "North Korea Standard Time": {"KST", "KST"}, // Asia/Pyongyang - "Myanmar Standard Time": {"MMT", "MMT"}, // Asia/Rangoon - "Arab Standard Time": {"AST", "AST"}, // Asia/Riyadh - "Sakhalin Standard Time": {"SAKT", "SAKT"}, // Asia/Sakhalin + "Myanmar Standard Time": {"+0630", "+0630"}, // Asia/Rangoon + "Arab Standard Time": {"+03", "+03"}, // Asia/Riyadh + "Sakhalin Standard Time": {"+11", "+11"}, // Asia/Sakhalin "Korea Standard Time": {"KST", "KST"}, // Asia/Seoul "China Standard Time": {"CST", "CST"}, // Asia/Shanghai - "Singapore Standard Time": {"SGT", "SGT"}, // Asia/Singapore - "Russia Time Zone 10": {"SRET", "SRET"}, // Asia/Srednekolymsk + "Singapore Standard Time": {"+08", "+08"}, // Asia/Singapore + "Russia Time Zone 10": {"+11", "+11"}, // Asia/Srednekolymsk "Taipei Standard Time": {"CST", "CST"}, // Asia/Taipei - "West Asia Standard Time": {"UZT", "UZT"}, // Asia/Tashkent - "Georgian Standard Time": {"GET", "GET"}, // Asia/Tbilisi - "Iran Standard Time": {"IRST", "IRDT"}, // Asia/Tehran + "West Asia Standard Time": {"+05", "+05"}, // Asia/Tashkent + "Georgian Standard Time": {"+04", "+04"}, // Asia/Tbilisi + "Iran Standard Time": {"+0330", "+0430"}, // Asia/Tehran "Tokyo Standard Time": {"JST", "JST"}, // Asia/Tokyo - "Tomsk Standard Time": {"+06", "+07"}, // Asia/Tomsk - "Ulaanbaatar Standard Time": {"ULAT", "ULAST"}, // Asia/Ulaanbaatar - "Vladivostok Standard Time": {"VLAT", "VLAT"}, // Asia/Vladivostok - "Yakutsk Standard Time": {"YAKT", "YAKT"}, // Asia/Yakutsk - "Ekaterinburg Standard Time": {"YEKT", "YEKT"}, // Asia/Yekaterinburg - "Caucasus Standard Time": {"AMT", "AMT"}, // Asia/Yerevan - "Azores Standard Time": {"AZOT", "AZOST"}, // Atlantic/Azores - "Cape Verde Standard Time": {"CVT", "CVT"}, // Atlantic/Cape_Verde + "Tomsk Standard Time": {"+07", "+07"}, // Asia/Tomsk + "Ulaanbaatar Standard Time": {"+08", "+08"}, // Asia/Ulaanbaatar + "Vladivostok Standard Time": {"+10", "+10"}, // Asia/Vladivostok + "Yakutsk Standard Time": {"+09", "+09"}, // Asia/Yakutsk + "Ekaterinburg Standard Time": {"+05", "+05"}, // Asia/Yekaterinburg + "Caucasus Standard Time": {"+04", "+04"}, // Asia/Yerevan + "Azores Standard Time": {"-01", "+00"}, // Atlantic/Azores + "Cape Verde Standard Time": {"-01", "-01"}, // Atlantic/Cape_Verde "Greenwich Standard Time": {"GMT", "GMT"}, // Atlantic/Reykjavik "Cen. Australia Standard Time": {"ACST", "ACDT"}, // Australia/Adelaide "E. Australia Standard Time": {"AEST", "AEST"}, // Australia/Brisbane "AUS Central Standard Time": {"ACST", "ACST"}, // Australia/Darwin - "Aus Central W. Standard Time": {"ACWST", "ACWST"}, // Australia/Eucla + "Aus Central W. Standard Time": {"+0845", "+0845"}, // Australia/Eucla "Tasmania Standard Time": {"AEST", "AEDT"}, // Australia/Hobart - "Lord Howe Standard Time": {"LHST", "LHDT"}, // Australia/Lord_Howe + "Lord Howe Standard Time": {"+1030", "+11"}, // Australia/Lord_Howe "W. Australia Standard Time": {"AWST", "AWST"}, // Australia/Perth "AUS Eastern Standard Time": {"AEST", "AEDT"}, // Australia/Sydney - "UTC": {"GMT", "GMT"}, // Etc/GMT - "UTC-11": {"GMT+11", "GMT+11"}, // Etc/GMT+11 - "Dateline Standard Time": {"GMT+12", "GMT+12"}, // Etc/GMT+12 - "UTC-02": {"GMT+2", "GMT+2"}, // Etc/GMT+2 - "UTC-08": {"GMT+8", "GMT+8"}, // Etc/GMT+8 - "UTC-09": {"GMT+9", "GMT+9"}, // Etc/GMT+9 - "UTC+12": {"GMT-12", "GMT-12"}, // Etc/GMT-12 - "Astrakhan Standard Time": {"+03", "+04"}, // Europe/Astrakhan - "W. Europe Standard Time": {"CET", "CEST"}, // Europe/Berlin - "GTB Standard Time": {"EET", "EEST"}, // Europe/Bucharest - "Central Europe Standard Time": {"CET", "CEST"}, // Europe/Budapest - "E. Europe Standard Time": {"EET", "EEST"}, // Europe/Chisinau - "Turkey Standard Time": {"EET", "EEST"}, // Europe/Istanbul - "Kaliningrad Standard Time": {"EET", "EET"}, // Europe/Kaliningrad - "FLE Standard Time": {"EET", "EEST"}, // Europe/Kiev - "GMT Standard Time": {"GMT", "BST"}, // Europe/London - "Belarus Standard Time": {"MSK", "MSK"}, // Europe/Minsk - "Russian Standard Time": {"MSK", "MSK"}, // Europe/Moscow - "Romance Standard Time": {"CET", "CEST"}, // Europe/Paris - "Russia Time Zone 3": {"SAMT", "SAMT"}, // Europe/Samara - "Central European Standard Time": {"CET", "CEST"}, // Europe/Warsaw - "Mauritius Standard Time": {"MUT", "MUT"}, // Indian/Mauritius - "Samoa Standard Time": {"WSST", "WSDT"}, // Pacific/Apia - "New Zealand Standard Time": {"NZST", "NZDT"}, // Pacific/Auckland - "Bougainville Standard Time": {"BST", "BST"}, // Pacific/Bougainville - "Chatham Islands Standard Time": {"CHAST", "CHADT"}, // Pacific/Chatham - "Easter Island Standard Time": {"EAST", "EASST"}, // Pacific/Easter - "Fiji Standard Time": {"FJT", "FJST"}, // Pacific/Fiji - "Central Pacific Standard Time": {"SBT", "SBT"}, // Pacific/Guadalcanal - "Hawaiian Standard Time": {"HST", "HST"}, // Pacific/Honolulu - "Line Islands Standard Time": {"LINT", "LINT"}, // Pacific/Kiritimati - "Marquesas Standard Time": {"MART", "MART"}, // Pacific/Marquesas - "Norfolk Standard Time": {"NFT", "NFT"}, // Pacific/Norfolk - "West Pacific Standard Time": {"PGT", "PGT"}, // Pacific/Port_Moresby - "Tonga Standard Time": {"TOT", "TOT"}, // Pacific/Tongatapu + "UTC": {"GMT", "GMT"}, // Etc/GMT + "UTC-11": {"-11", "-11"}, // Etc/GMT+11 + "Dateline Standard Time": {"-12", "-12"}, // Etc/GMT+12 + "UTC-02": {"-02", "-02"}, // Etc/GMT+2 + "UTC-08": {"-08", "-08"}, // Etc/GMT+8 + "UTC-09": {"-09", "-09"}, // Etc/GMT+9 + "UTC+12": {"+12", "+12"}, // Etc/GMT-12 + "UTC+13": {"+13", "+13"}, // Etc/GMT-13 + "Astrakhan Standard Time": {"+04", "+04"}, // Europe/Astrakhan + "W. Europe Standard Time": {"CET", "CEST"}, // Europe/Berlin + "GTB Standard Time": {"EET", "EEST"}, // Europe/Bucharest + "Central Europe Standard Time": {"CET", "CEST"}, // Europe/Budapest + "E. Europe Standard Time": {"EET", "EEST"}, // Europe/Chisinau + "Turkey Standard Time": {"+03", "+03"}, // Europe/Istanbul + "Kaliningrad Standard Time": {"EET", "EET"}, // Europe/Kaliningrad + "FLE Standard Time": {"EET", "EEST"}, // Europe/Kiev + "GMT Standard Time": {"GMT", "BST"}, // Europe/London + "Belarus Standard Time": {"+03", "+03"}, // Europe/Minsk + "Russian Standard Time": {"MSK", "MSK"}, // Europe/Moscow + "Romance Standard Time": {"CET", "CEST"}, // Europe/Paris + "Russia Time Zone 3": {"+04", "+04"}, // Europe/Samara + "Saratov Standard Time": {"+03", "+04"}, // Europe/Saratov + "Central European Standard Time": {"CET", "CEST"}, // Europe/Warsaw + "Mauritius Standard Time": {"+04", "+04"}, // Indian/Mauritius + "Samoa Standard Time": {"+13", "+14"}, // Pacific/Apia + "New Zealand Standard Time": {"NZST", "NZDT"}, // Pacific/Auckland + "Bougainville Standard Time": {"+11", "+11"}, // Pacific/Bougainville + "Chatham Islands Standard Time": {"+1245", "+1345"}, // Pacific/Chatham + "Easter Island Standard Time": {"-06", "-05"}, // Pacific/Easter + "Fiji Standard Time": {"+12", "+13"}, // Pacific/Fiji + "Central Pacific Standard Time": {"+11", "+11"}, // Pacific/Guadalcanal + "Hawaiian Standard Time": {"HST", "HST"}, // Pacific/Honolulu + "Line Islands Standard Time": {"+14", "+14"}, // Pacific/Kiritimati + "Marquesas Standard Time": {"-0930", "-0930"}, // Pacific/Marquesas + "Norfolk Standard Time": {"+11", "+11"}, // Pacific/Norfolk + "West Pacific Standard Time": {"+10", "+10"}, // Pacific/Port_Moresby + "Tonga Standard Time": {"+13", "+14"}, // Pacific/Tongatapu } diff --git a/libgo/go/time/zoneinfo_plan9.go b/libgo/go/time/zoneinfo_plan9.go index 0694f0a9904..26637a151f0 100644 --- a/libgo/go/time/zoneinfo_plan9.go +++ b/libgo/go/time/zoneinfo_plan9.go @@ -95,7 +95,7 @@ func loadZoneDataPlan9(s string) (l *Location, err error) { // Fill in the cache with information about right now, // since that will be the most common lookup. - sec, _ := now() + sec, _, _ := now() for i := range tx { if tx[i].when <= sec && (i+1 == len(tx) || sec < tx[i+1].when) { l.cacheStart = tx[i].when diff --git a/libgo/go/time/zoneinfo_read.go b/libgo/go/time/zoneinfo_read.go index 19cd40d8477..b0cd9da9230 100644 --- a/libgo/go/time/zoneinfo_read.go +++ b/libgo/go/time/zoneinfo_read.go @@ -11,6 +11,17 @@ package time import "errors" +// maxFileSize is the max permitted size of files read by readFile. +// As reference, the zoneinfo.zip distributed by Go is ~350 KB, +// so 10MB is overkill. +const maxFileSize = 10 << 20 + +type fileSizeError string + +func (f fileSizeError) Error() string { + return "time: file " + string(f) + " is too large" +} + // Copies of io.Seek* constants to avoid importing "io": const ( seekStart = 0 @@ -188,7 +199,7 @@ func loadZoneData(bytes []byte) (l *Location, err error) { // Fill in the cache with information about right now, // since that will be the most common lookup. - sec, _ := now() + sec, _, _ := now() for i := range tx { if tx[i].when <= sec && (i+1 == len(tx) || sec < tx[i+1].when) { l.cacheStart = tx[i].when diff --git a/libgo/go/time/zoneinfo_test.go b/libgo/go/time/zoneinfo_test.go index 5b6a4dc4e4e..8a4caa0c445 100644 --- a/libgo/go/time/zoneinfo_test.go +++ b/libgo/go/time/zoneinfo_test.go @@ -5,10 +5,56 @@ package time_test import ( + "fmt" + "os" "testing" "time" ) +func init() { + if time.ZoneinfoForTesting() != nil { + panic(fmt.Errorf("zoneinfo initialized before first LoadLocation")) + } +} + +func TestEnvVarUsage(t *testing.T) { + time.ResetZoneinfoForTesting() + + const testZoneinfo = "foo.zip" + const env = "ZONEINFO" + + defer os.Setenv(env, os.Getenv(env)) + os.Setenv(env, testZoneinfo) + + // Result isn't important, we're testing the side effect of this command + time.LoadLocation("Asia/Jerusalem") + defer time.ResetZoneinfoForTesting() + + if zoneinfo := time.ZoneinfoForTesting(); testZoneinfo != *zoneinfo { + t.Errorf("zoneinfo does not match env variable: got %q want %q", zoneinfo, testZoneinfo) + } +} + +func TestLoadLocationValidatesNames(t *testing.T) { + time.ResetZoneinfoForTesting() + const env = "ZONEINFO" + defer os.Setenv(env, os.Getenv(env)) + os.Setenv(env, "") + + bad := []string{ + "/usr/foo/Foo", + "\\UNC\foo", + "..", + "a..", + } + for _, v := range bad { + _, err := time.LoadLocation(v) + if err != time.ErrLocation { + t.Errorf("LoadLocation(%q) error = %v; want ErrLocation", v, err) + } + } +} + func TestVersion3(t *testing.T) { t.Skip("gccgo does not use the zip file") time.ForceZipFileForTesting(true) @@ -44,8 +90,8 @@ func TestFirstZone(t *testing.T) { { "Pacific/Fakaofo", 1325242799, - "Thu, 29 Dec 2011 23:59:59 -1100 (TKT)", - "Sat, 31 Dec 2011 00:00:00 +1300 (TKT)", + "Thu, 29 Dec 2011 23:59:59 -1100 (-11)", + "Sat, 31 Dec 2011 00:00:00 +1300 (+13)", }, } diff --git a/libgo/go/time/zoneinfo_windows.go b/libgo/go/time/zoneinfo_windows.go index a6e227b5b0d..c201f4b55ee 100644 --- a/libgo/go/time/zoneinfo_windows.go +++ b/libgo/go/time/zoneinfo_windows.go @@ -132,7 +132,7 @@ func pseudoUnix(year int, d *syscall.Systemtime) int64 { day -= 7 } } - return t.sec + int64(day-1)*secondsPerDay + internalToUnix + return t.sec() + int64(day-1)*secondsPerDay + internalToUnix } func initLocalFromTZI(i *syscall.Timezoneinformation) { diff --git a/libgo/go/unicode/letter.go b/libgo/go/unicode/letter.go index b43cc66e7d3..90b0b414da2 100644 --- a/libgo/go/unicode/letter.go +++ b/libgo/go/unicode/letter.go @@ -46,7 +46,7 @@ type Range32 struct { // CaseRange represents a range of Unicode code points for simple (one // code point to one code point) case conversion. -// The range runs from Lo to Hi inclusive, with a fixed stride of 1. Deltas +// The range runs from Lo to Hi inclusive, with a fixed stride of 1. Deltas // are the number to add to the code point to reach the code point for a // different case for that character. They may be negative. If zero, it // means the character is in the corresponding case. There is a special @@ -308,7 +308,7 @@ func (special SpecialCase) ToLower(r rune) rune { } // caseOrbit is defined in tables.go as []foldPair. Right now all the -// entries fit in uint16, so use uint16. If that changes, compilation +// entries fit in uint16, so use uint16. If that changes, compilation // will fail (the constants in the composite literal will not fit in uint16) // and the types here can change to uint32. type foldPair struct { diff --git a/libgo/go/unicode/tables.go b/libgo/go/unicode/tables.go index 15fecd954f1..90323367d55 100644 --- a/libgo/go/unicode/tables.go +++ b/libgo/go/unicode/tables.go @@ -2,9 +2,9 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -// Generated by running +// Code generated by maketables; DO NOT EDIT. +// To regenerate, run: // maketables --tables=all --data=http://www.unicode.org/Public/9.0.0/ucd/UnicodeData.txt --casefolding=http://www.unicode.org/Public/9.0.0/ucd/CaseFolding.txt -// DO NOT EDIT package unicode @@ -7311,34 +7311,12 @@ var caseOrbit = []foldPair{ // simple case folding to code points inside the category. // If there is no entry for a category name, there are no such points. var FoldCategory = map[string]*RangeTable{ - "Common": foldCommon, - "Greek": foldGreek, - "Inherited": foldInherited, - "L": foldL, - "Ll": foldLl, - "Lt": foldLt, - "Lu": foldLu, - "M": foldM, - "Mn": foldMn, -} - -var foldCommon = &RangeTable{ - R16: []Range16{ - {0x039c, 0x03bc, 32}, - }, -} - -var foldGreek = &RangeTable{ - R16: []Range16{ - {0x00b5, 0x0345, 656}, - }, -} - -var foldInherited = &RangeTable{ - R16: []Range16{ - {0x0399, 0x03b9, 32}, - {0x1fbe, 0x1fbe, 1}, - }, + "L": foldL, + "Ll": foldLl, + "Lt": foldLt, + "Lu": foldLu, + "M": foldM, + "Mn": foldMn, } var foldL = &RangeTable{ @@ -7609,7 +7587,30 @@ var foldMn = &RangeTable{ // code points outside the script that are equivalent under // simple case folding to code points inside the script. // If there is no entry for a script name, there are no such points. -var FoldScript = map[string]*RangeTable{} +var FoldScript = map[string]*RangeTable{ + "Common": foldCommon, + "Greek": foldGreek, + "Inherited": foldInherited, +} + +var foldCommon = &RangeTable{ + R16: []Range16{ + {0x039c, 0x03bc, 32}, + }, +} + +var foldGreek = &RangeTable{ + R16: []Range16{ + {0x00b5, 0x0345, 656}, + }, +} + +var foldInherited = &RangeTable{ + R16: []Range16{ + {0x0399, 0x03b9, 32}, + {0x1fbe, 0x1fbe, 1}, + }, +} // Range entries: 3576 16-bit, 1454 32-bit, 5030 total. // Range bytes: 21456 16-bit, 17448 32-bit, 38904 total. diff --git a/libgo/godeps.sh b/libgo/godeps.sh index 0da5d07bdf9..87cd09e3d9d 100644 --- a/libgo/godeps.sh +++ b/libgo/godeps.sh @@ -25,7 +25,8 @@ shift files=$* deps=`for f in $files; do cat $f; done | sed -n -e '/^import.*"/p; /^import[ ]*(/,/^)/p' | - grep '"' | + sed -e 's/^import //' | + grep '^[ ]*"' | grep -v '"unsafe"' | sed -e 's/^.*"\([^"]*\)".*$/\1/' -e 's/$/.gox/' | sort -u` diff --git a/libgo/match.sh b/libgo/match.sh index f87a51d8f1f..fac75eabec6 100755 --- a/libgo/match.sh +++ b/libgo/match.sh @@ -151,18 +151,18 @@ for f in $gofiles; do fi match=false ;; - $goos | $goarch | $cgotag | $cmdlinetag | "gccgo") + $goos | $goarch | $cgotag | $cmdlinetag | "gccgo" | go1.[0-9]) match=true ;; - "!"$goos | "!"$goarch | "!"$cgotag | "!"$cmdlinetag | "!gccgo") + "!"$goos | "!"$goarch | "!"$cgotag | "!"$cmdlinetag | "!gccgo" | "!"go1.[0-9]) ;; *,*) cmatch=true for ctag in `echo $tag | sed -e 's/,/ /g'`; do case $ctag in - $goos | $goarch | $cgotag | $cmdlinetag | "gccgo") + $goos | $goarch | $cgotag | $cmdlinetag | "gccgo" | go1.[0-9]) ;; - "!"$goos | "!"$goarch | "!"$cgotag | "!"$cmdlinetag | "!gccgo") + "!"$goos | "!"$goarch | "!"$cgotag | "!"$cmdlinetag | "!gccgo" | "!"go1.[0-9]) cmatch=false ;; "!"*) diff --git a/libgo/merge.sh b/libgo/merge.sh index bc24504e2a0..66686654476 100755 --- a/libgo/merge.sh +++ b/libgo/merge.sh @@ -128,7 +128,7 @@ echo ${rev} > VERSION (cd ${NEWDIR}/src && find . -name '*.go' -print) | while read f; do skip=false case "$f" in - ./cmd/cgo/* | ./cmd/go/* | ./cmd/gofmt/* | ./cmd/internal/browser/*) + ./cmd/cgo/* | ./cmd/go/* | ./cmd/gofmt/* | ./cmd/internal/browser/* | ./cmd/internal/objabi/*) ;; ./cmd/*) skip=true diff --git a/libgo/misc/cgo/errors/issue18452.go b/libgo/misc/cgo/errors/issue18452.go new file mode 100644 index 00000000000..36ef7f54e12 --- /dev/null +++ b/libgo/misc/cgo/errors/issue18452.go @@ -0,0 +1,18 @@ +// Copyright 2017 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Issue 18452: show pos info in undefined name errors + +package p + +import ( + "C" + "fmt" +) + +func a() { + fmt.Println("Hello, world!") + C.function_that_does_not_exist() // line 16 + C.pi // line 17 +} diff --git a/libgo/misc/cgo/errors/issue18889.go b/libgo/misc/cgo/errors/issue18889.go new file mode 100644 index 00000000000..bba6b8f9bb1 --- /dev/null +++ b/libgo/misc/cgo/errors/issue18889.go @@ -0,0 +1,7 @@ +package main + +import "C" + +func main() { + _ = C.malloc // ERROR HERE +} diff --git a/libgo/misc/cgo/errors/ptr.go b/libgo/misc/cgo/errors/ptr.go index 4dafbdf3c01..3e117666bff 100644 --- a/libgo/misc/cgo/errors/ptr.go +++ b/libgo/misc/cgo/errors/ptr.go @@ -343,6 +343,14 @@ var ptrTests = []ptrTest{ body: `var b C.char; p := &b; C.f((*C.u)(unsafe.Pointer(&p)))`, fail: false, }, + { + // Issue #21306. + name: "preempt-during-call", + c: `void f() {}`, + imports: []string{"runtime", "sync"}, + body: `var wg sync.WaitGroup; wg.Add(100); for i := 0; i < 100; i++ { go func(i int) { for j := 0; j < 100; j++ { C.f(); runtime.GOMAXPROCS(i) }; wg.Done() }(i) }; wg.Wait()`, + fail: false, + }, } func main() { diff --git a/libgo/misc/cgo/errors/test.bash b/libgo/misc/cgo/errors/test.bash index 05261e9d767..ed0b0946925 100644 --- a/libgo/misc/cgo/errors/test.bash +++ b/libgo/misc/cgo/errors/test.bash @@ -17,7 +17,7 @@ check() { expect() { file=$1 shift - if go build $file >errs 2>&1; then + if go build -gcflags=-C $file >errs 2>&1; then echo 1>&2 misc/cgo/errors/test.bash: BUG: expected cgo to fail on $file but it succeeded exit 1 fi @@ -47,6 +47,8 @@ expect issue13635.go C.uchar C.schar C.ushort C.uint C.ulong C.longlong C.ulongl check issue13830.go check issue16116.go check issue16591.go +check issue18889.go +expect issue18452.go issue18452.go:16 issue18452.go:17 if ! go build issue14669.go; then exit 1 diff --git a/libgo/misc/cgo/fortran/test.bash b/libgo/misc/cgo/fortran/test.bash index 3d1bc9de8e9..1e0d59ea1c6 100644 --- a/libgo/misc/cgo/fortran/test.bash +++ b/libgo/misc/cgo/fortran/test.bash @@ -12,7 +12,7 @@ FC=$1 goos=$(go env GOOS) libext="so" -if [ "$goos" == "darwin" ]; then +if [ "$goos" = "darwin" ]; then libext="dylib" fi diff --git a/libgo/misc/cgo/test/cgo_test.go b/libgo/misc/cgo/test/cgo_test.go index a6de999752b..f7cf6f613c4 100644 --- a/libgo/misc/cgo/test/cgo_test.go +++ b/libgo/misc/cgo/test/cgo_test.go @@ -76,5 +76,9 @@ func TestThreadLock(t *testing.T) { testThreadLockFunc(t) } func TestCheckConst(t *testing.T) { testCheckConst(t) } func Test17537(t *testing.T) { test17537(t) } func Test18126(t *testing.T) { test18126(t) } +func Test20369(t *testing.T) { test20369(t) } +func Test18720(t *testing.T) { test18720(t) } +func Test20266(t *testing.T) { test20266(t) } +func Test20129(t *testing.T) { test20129(t) } func BenchmarkCgoCall(b *testing.B) { benchCgoCall(b) } diff --git a/libgo/misc/cgo/test/issue18720.go b/libgo/misc/cgo/test/issue18720.go new file mode 100644 index 00000000000..a93304498e0 --- /dev/null +++ b/libgo/misc/cgo/test/issue18720.go @@ -0,0 +1,28 @@ +// Copyright 2017 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package cgotest + +/* +#define HELLO "hello" +#define WORLD "world" +#define HELLO_WORLD HELLO "\000" WORLD + +struct foo { char c; }; +#define SIZE_OF(x) sizeof(x) +#define SIZE_OF_FOO SIZE_OF(struct foo) +*/ +import "C" +import "testing" + +func test18720(t *testing.T) { + if C.HELLO_WORLD != "hello\000world" { + t.Fatalf(`expected "hello\000world", but got %q`, C.HELLO_WORLD) + } + + // Issue 20125. + if got, want := C.SIZE_OF_FOO, 1; got != want { + t.Errorf("C.SIZE_OF_FOO == %v, expected %v", got, want) + } +} diff --git a/libgo/misc/cgo/test/issue20129.go b/libgo/misc/cgo/test/issue20129.go new file mode 100644 index 00000000000..e69e0e16efa --- /dev/null +++ b/libgo/misc/cgo/test/issue20129.go @@ -0,0 +1,33 @@ +// Copyright 2017 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package cgotest + +/* +int issue20129 = 0; +typedef void issue20129Void; +issue20129Void issue20129Foo() { + issue20129 = 1; +} +typedef issue20129Void issue20129Void2; +issue20129Void2 issue20129Bar() { + issue20129 = 2; +} +*/ +import "C" +import "testing" + +func test20129(t *testing.T) { + if C.issue20129 != 0 { + t.Fatal("test is broken") + } + C.issue20129Foo() + if C.issue20129 != 1 { + t.Errorf("got %v but expected %v", C.issue20129, 1) + } + C.issue20129Bar() + if C.issue20129 != 2 { + t.Errorf("got %v but expected %v", C.issue20129, 2) + } +} diff --git a/libgo/misc/cgo/test/issue20266.go b/libgo/misc/cgo/test/issue20266.go new file mode 100644 index 00000000000..9f95086cc7b --- /dev/null +++ b/libgo/misc/cgo/test/issue20266.go @@ -0,0 +1,21 @@ +// Copyright 2017 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Issue 20266: use -I with a relative path. + +package cgotest + +/* +#cgo CFLAGS: -I issue20266 -Iissue20266 -Ddef20266 +#include "issue20266.h" +*/ +import "C" + +import "testing" + +func test20266(t *testing.T) { + if got, want := C.issue20266, 20266; got != want { + t.Errorf("got %d, want %d", got, want) + } +} diff --git a/libgo/misc/cgo/test/issue20266/issue20266.h b/libgo/misc/cgo/test/issue20266/issue20266.h new file mode 100644 index 00000000000..8d3258ec6b8 --- /dev/null +++ b/libgo/misc/cgo/test/issue20266/issue20266.h @@ -0,0 +1,9 @@ +// Copyright 2017 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +#define issue20266 20266 + +#ifndef def20266 +#error "expected def20266 to be defined" +#endif diff --git a/libgo/misc/cgo/test/issue20369.go b/libgo/misc/cgo/test/issue20369.go new file mode 100644 index 00000000000..37b4b78dfe1 --- /dev/null +++ b/libgo/misc/cgo/test/issue20369.go @@ -0,0 +1,20 @@ +// Copyright 2017 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package cgotest + +/* +#define UINT64_MAX 18446744073709551615ULL +*/ +import "C" +import ( + "math" + "testing" +) + +func test20369(t *testing.T) { + if C.UINT64_MAX != math.MaxUint64 { + t.Fatalf("got %v, want %v", uint64(C.UINT64_MAX), uint64(math.MaxUint64)) + } +} diff --git a/libgo/misc/cgo/test/issue6612.go b/libgo/misc/cgo/test/issue6612.go index c337f911d90..15a12fab38f 100644 --- a/libgo/misc/cgo/test/issue6612.go +++ b/libgo/misc/cgo/test/issue6612.go @@ -74,18 +74,15 @@ func testNaming(t *testing.T) { } } - // This would be nice, but it has never worked. - /* - if c := C.myfloat_def; c != 1.5 { - t.Errorf("C.myint_def = %v, want 1.5", c) - } - { - const c = C.myfloat_def - if c != 1.5 { + if c := C.myfloat_def; c != 1.5 { + t.Errorf("C.myint_def = %v, want 1.5", c) + } + { + const c = C.myfloat_def + if c != 1.5 { t.Errorf("C.myint as const = %v, want 1.5", c) - } } - */ + } if s := C.mystring_def; s != "hello" { t.Errorf("C.mystring_def = %q, want %q", s, "hello") diff --git a/libgo/misc/cgo/testcarchive/carchive_test.go b/libgo/misc/cgo/testcarchive/carchive_test.go index a2ad9c56418..4865b806a3e 100644 --- a/libgo/misc/cgo/testcarchive/carchive_test.go +++ b/libgo/misc/cgo/testcarchive/carchive_test.go @@ -120,8 +120,10 @@ func init() { func goEnv(key string) string { out, err := exec.Command("go", "env", key).Output() if err != nil { - fmt.Fprintf(os.Stderr, "go env %s failed:\n%s", key, err) - fmt.Fprintf(os.Stderr, "%s", err.(*exec.ExitError).Stderr) + fmt.Fprintf(os.Stderr, "go env %s failed:\n%s\n", key, err) + if ee, ok := err.(*exec.ExitError); ok { + fmt.Fprintf(os.Stderr, "%s", ee.Stderr) + } os.Exit(2) } return strings.TrimSpace(string(out)) @@ -238,15 +240,7 @@ func TestEarlySignalHandler(t *testing.T) { } func TestSignalForwarding(t *testing.T) { - switch GOOS { - case "darwin": - switch GOARCH { - case "arm", "arm64": - t.Skipf("skipping on %s/%s; see https://golang.org/issue/13701", GOOS, GOARCH) - } - case "windows": - t.Skip("skipping signal test on Windows") - } + checkSignalForwardingTest(t) defer func() { os.Remove("libgo2.a") @@ -274,32 +268,19 @@ func TestSignalForwarding(t *testing.T) { cmd = exec.Command(bin[0], append(bin[1:], "1")...) out, err := cmd.CombinedOutput() + t.Logf("%s", out) + expectSignal(t, err, syscall.SIGSEGV) - if err == nil { - t.Logf("%s", out) - t.Error("test program succeeded unexpectedly") - } else if ee, ok := err.(*exec.ExitError); !ok { - t.Logf("%s", out) - t.Errorf("error (%v) has type %T; expected exec.ExitError", err, err) - } else if ws, ok := ee.Sys().(syscall.WaitStatus); !ok { - t.Logf("%s", out) - t.Errorf("error.Sys (%v) has type %T; expected syscall.WaitStatus", ee.Sys(), ee.Sys()) - } else if !ws.Signaled() || ws.Signal() != syscall.SIGSEGV { - t.Logf("%s", out) - t.Errorf("got %v; expected SIGSEGV", ee) - } + // Test SIGPIPE forwarding + cmd = exec.Command(bin[0], append(bin[1:], "3")...) + + out, err = cmd.CombinedOutput() + t.Logf("%s", out) + expectSignal(t, err, syscall.SIGPIPE) } func TestSignalForwardingExternal(t *testing.T) { - switch GOOS { - case "darwin": - switch GOARCH { - case "arm", "arm64": - t.Skipf("skipping on %s/%s; see https://golang.org/issue/13701", GOOS, GOARCH) - } - case "windows": - t.Skip("skipping signal test on Windows") - } + checkSignalForwardingTest(t) defer func() { os.Remove("libgo2.a") @@ -370,14 +351,7 @@ func TestSignalForwardingExternal(t *testing.T) { continue } - if ee, ok := err.(*exec.ExitError); !ok { - t.Errorf("error (%v) has type %T; expected exec.ExitError", err, err) - } else if ws, ok := ee.Sys().(syscall.WaitStatus); !ok { - t.Errorf("error.Sys (%v) has type %T; expected syscall.WaitStatus", ee.Sys(), ee.Sys()) - } else if !ws.Signaled() || ws.Signal() != syscall.SIGSEGV { - t.Errorf("got %v; expected SIGSEGV", ee) - } else { - // We got the error we expected. + if expectSignal(t, err, syscall.SIGSEGV) { return } } @@ -385,6 +359,38 @@ func TestSignalForwardingExternal(t *testing.T) { t.Errorf("program succeeded unexpectedly %d times", tries) } +// checkSignalForwardingTest calls t.Skip if the SignalForwarding test +// doesn't work on this platform. +func checkSignalForwardingTest(t *testing.T) { + switch GOOS { + case "darwin": + switch GOARCH { + case "arm", "arm64": + t.Skipf("skipping on %s/%s; see https://golang.org/issue/13701", GOOS, GOARCH) + } + case "windows": + t.Skip("skipping signal test on Windows") + } +} + +// expectSignal checks that err, the exit status of a test program, +// shows a failure due to a specific signal. Returns whether we found +// the expected signal. +func expectSignal(t *testing.T, err error, sig syscall.Signal) bool { + if err == nil { + t.Error("test program succeeded unexpectedly") + } else if ee, ok := err.(*exec.ExitError); !ok { + t.Errorf("error (%v) has type %T; expected exec.ExitError", err, err) + } else if ws, ok := ee.Sys().(syscall.WaitStatus); !ok { + t.Errorf("error.Sys (%v) has type %T; expected syscall.WaitStatus", ee.Sys(), ee.Sys()) + } else if !ws.Signaled() || ws.Signal() != sig { + t.Errorf("got %v; expected signal %v", ee, sig) + } else { + return true + } + return false +} + func TestOsSignal(t *testing.T) { switch GOOS { case "windows": @@ -585,3 +591,85 @@ func hasDynTag(t *testing.T, f *elf.File, tag elf.DynTag) bool { } return false } + +func TestSIGPROF(t *testing.T) { + switch GOOS { + case "windows", "plan9": + t.Skipf("skipping SIGPROF test on %s", GOOS) + } + + t.Parallel() + + defer func() { + os.Remove("testp6" + exeSuffix) + os.Remove("libgo6.a") + os.Remove("libgo6.h") + }() + + cmd := exec.Command("go", "build", "-buildmode=c-archive", "-o", "libgo6.a", "libgo6") + cmd.Env = gopathEnv + if out, err := cmd.CombinedOutput(); err != nil { + t.Logf("%s", out) + t.Fatal(err) + } + + ccArgs := append(cc, "-o", "testp6"+exeSuffix, "main6.c", "libgo6.a") + if runtime.Compiler == "gccgo" { + ccArgs = append(ccArgs, "-lgo") + } + if out, err := exec.Command(ccArgs[0], ccArgs[1:]...).CombinedOutput(); err != nil { + t.Logf("%s", out) + t.Fatal(err) + } + + argv := cmdToRun("./testp6") + cmd = exec.Command(argv[0], argv[1:]...) + if out, err := cmd.CombinedOutput(); err != nil { + t.Logf("%s", out) + t.Fatal(err) + } +} + +// TestCompileWithoutShared tests that if we compile code without the +// -shared option, we can put it into an archive. When we use the go +// tool with -buildmode=c-archive, it passes -shared to the compiler, +// so we override that. The go tool doesn't work this way, but Bazel +// will likely do it in the future. And it ought to work. This test +// was added because at one time it did not work on PPC GNU/Linux. +func TestCompileWithoutShared(t *testing.T) { + // For simplicity, reuse the signal forwarding test. + checkSignalForwardingTest(t) + + defer func() { + os.Remove("libgo2.a") + os.Remove("libgo2.h") + }() + + cmd := exec.Command("go", "build", "-buildmode=c-archive", "-gcflags=-shared=false", "-o", "libgo2.a", "libgo2") + cmd.Env = gopathEnv + t.Log(cmd.Args) + out, err := cmd.CombinedOutput() + t.Logf("%s", out) + if err != nil { + t.Fatal(err) + } + + exe := "./testnoshared" + exeSuffix + ccArgs := append(cc, "-o", exe, "main5.c", "libgo2.a") + if runtime.Compiler == "gccgo" { + ccArgs = append(ccArgs, "-lgo") + } + t.Log(ccArgs) + out, err = exec.Command(ccArgs[0], ccArgs[1:]...).CombinedOutput() + t.Logf("%s", out) + if err != nil { + t.Fatal(err) + } + defer os.Remove(exe) + + binArgs := append(cmdToRun(exe), "3") + t.Log(binArgs) + out, err = exec.Command(binArgs[0], binArgs[1:]...).CombinedOutput() + t.Logf("%s", out) + expectSignal(t, err, syscall.SIGPIPE) +} diff --git a/libgo/misc/cgo/testcarchive/main2.c b/libgo/misc/cgo/testcarchive/main2.c index 774e014a162..769cd497e6c 100644 --- a/libgo/misc/cgo/testcarchive/main2.c +++ b/libgo/misc/cgo/testcarchive/main2.c @@ -17,6 +17,7 @@ #include #include #include +#include #include "libgo2.h" @@ -26,6 +27,7 @@ static void die(const char* msg) { } static volatile sig_atomic_t sigioSeen; +static volatile sig_atomic_t sigpipeSeen; // Use up some stack space. static void recur(int i, char *p) { @@ -37,6 +39,10 @@ static void recur(int i, char *p) { } } +static void pipeHandler(int signo, siginfo_t* info, void* ctxt) { + sigpipeSeen = 1; +} + // Signal handler that uses up more stack space than a goroutine will have. static void ioHandler(int signo, siginfo_t* info, void* ctxt) { char a[1024]; @@ -106,6 +112,10 @@ static void init() { die("sigaction"); } + sa.sa_sigaction = pipeHandler; + if (sigaction(SIGPIPE, &sa, NULL) < 0) { + die("sigaction"); + } } int main(int argc, char** argv) { @@ -167,7 +177,30 @@ int main(int argc, char** argv) { nanosleep(&ts, NULL); i++; if (i > 5000) { - fprintf(stderr, "looping too long waiting for signal\n"); + fprintf(stderr, "looping too long waiting for SIGIO\n"); + exit(EXIT_FAILURE); + } + } + + if (verbose) { + printf("provoking SIGPIPE\n"); + } + + GoRaiseSIGPIPE(); + + if (verbose) { + printf("waiting for sigpipeSeen\n"); + } + + // Wait until the signal has been delivered. + i = 0; + while (!sigpipeSeen) { + ts.tv_sec = 0; + ts.tv_nsec = 1000000; + nanosleep(&ts, NULL); + i++; + if (i > 5000) { + fprintf(stderr, "looping too long waiting for SIGPIPE\n"); exit(EXIT_FAILURE); } } diff --git a/libgo/misc/cgo/testcarchive/main3.c b/libgo/misc/cgo/testcarchive/main3.c index 0a6c0d3f74e..60a16cf5fc4 100644 --- a/libgo/misc/cgo/testcarchive/main3.c +++ b/libgo/misc/cgo/testcarchive/main3.c @@ -11,6 +11,7 @@ #include #include #include +#include #include "libgo3.h" @@ -25,6 +26,31 @@ static void ioHandler(int signo, siginfo_t* info, void* ctxt) { sigioSeen = 1; } +// Set up the SIGPIPE signal handler in a high priority constructor, so +// that it is installed before the Go code starts. + +static void pipeHandler(int signo, siginfo_t* info, void* ctxt) { + const char *s = "unexpected SIGPIPE\n"; + write(2, s, strlen(s)); + exit(EXIT_FAILURE); +} + +static void init(void) __attribute__ ((constructor (200))); + +static void init() { + struct sigaction sa; + + memset(&sa, 0, sizeof sa); + sa.sa_sigaction = pipeHandler; + if (sigemptyset(&sa.sa_mask) < 0) { + die("sigemptyset"); + } + sa.sa_flags = SA_SIGINFO; + if (sigaction(SIGPIPE, &sa, NULL) < 0) { + die("sigaction"); + } +} + int main(int argc, char** argv) { int verbose; struct sigaction sa; @@ -34,6 +60,14 @@ int main(int argc, char** argv) { verbose = argc > 2; setvbuf(stdout, NULL, _IONBF, 0); + if (verbose) { + printf("raising SIGPIPE\n"); + } + + // Test that the Go runtime handles SIGPIPE, even if we installed + // a non-default SIGPIPE handler before the runtime initializes. + ProvokeSIGPIPE(); + if (verbose) { printf("calling sigaction\n"); } diff --git a/libgo/misc/cgo/testcarchive/main5.c b/libgo/misc/cgo/testcarchive/main5.c index 9fadf0801e1..2437bf07c58 100644 --- a/libgo/misc/cgo/testcarchive/main5.c +++ b/libgo/misc/cgo/testcarchive/main5.c @@ -68,6 +68,24 @@ int main(int argc, char** argv) { break; } + case 3: { + if (verbose) { + printf("attempting SIGPIPE\n"); + } + + int fd[2]; + if (pipe(fd) != 0) { + printf("pipe(2) failed\n"); + return 0; + } + // Close the reading end. + close(fd[0]); + // Expect that write(2) fails (EPIPE) + if (write(fd[1], "some data", 9) != -1) { + printf("write(2) unexpectedly succeeded\n"); + return 0; + } + } default: printf("Unknown test: %d\n", test); return 0; diff --git a/libgo/misc/cgo/testcarchive/main6.c b/libgo/misc/cgo/testcarchive/main6.c new file mode 100644 index 00000000000..2745eb9dc5e --- /dev/null +++ b/libgo/misc/cgo/testcarchive/main6.c @@ -0,0 +1,34 @@ +// Copyright 2016 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Test that using the Go profiler in a C program does not crash. + +#include +#include + +#include "libgo6.h" + +int main(int argc, char **argv) { + struct timeval tvstart, tvnow; + int diff; + + gettimeofday(&tvstart, NULL); + + go_start_profile(); + + // Busy wait so we have something to profile. + // If we just sleep the profiling signal will never fire. + while (1) { + gettimeofday(&tvnow, NULL); + diff = (tvnow.tv_sec - tvstart.tv_sec) * 1000 * 1000 + (tvnow.tv_usec - tvstart.tv_usec); + + // Profile frequency is 100Hz so we should definitely + // get a signal in 50 milliseconds. + if (diff > 50 * 1000) + break; + } + + go_stop_profile(); + return 0; +} diff --git a/libgo/misc/cgo/testcarchive/src/libgo2/libgo2.go b/libgo/misc/cgo/testcarchive/src/libgo2/libgo2.go index fbed493b93f..19c8e1a6dcb 100644 --- a/libgo/misc/cgo/testcarchive/src/libgo2/libgo2.go +++ b/libgo/misc/cgo/testcarchive/src/libgo2/libgo2.go @@ -4,6 +4,30 @@ package main +/* +#include +#include +#include +#include + +// Raise SIGPIPE. +static void CRaiseSIGPIPE() { + int fds[2]; + + if (pipe(fds) == -1) { + perror("pipe"); + exit(EXIT_FAILURE); + } + // Close the reader end + close(fds[0]); + // Write to the writer end to provoke a SIGPIPE + if (write(fds[1], "some data", 9) != -1) { + fprintf(stderr, "write to a closed pipe succeeded\n"); + exit(EXIT_FAILURE); + } + close(fds[1]); +} +*/ import "C" import ( @@ -46,5 +70,11 @@ func TestSEGV() { func Noop() { } +// Raise SIGPIPE. +//export GoRaiseSIGPIPE +func GoRaiseSIGPIPE() { + C.CRaiseSIGPIPE() +} + func main() { } diff --git a/libgo/misc/cgo/testcarchive/src/libgo3/libgo3.go b/libgo/misc/cgo/testcarchive/src/libgo3/libgo3.go index 94e5d21c14a..e276a3c347a 100644 --- a/libgo/misc/cgo/testcarchive/src/libgo3/libgo3.go +++ b/libgo/misc/cgo/testcarchive/src/libgo3/libgo3.go @@ -40,5 +40,17 @@ func SawSIGIO() C.int { } } +// ProvokeSIGPIPE provokes a kernel-initiated SIGPIPE. +//export ProvokeSIGPIPE +func ProvokeSIGPIPE() { + r, w, err := os.Pipe() + if err != nil { + panic(err) + } + r.Close() + defer w.Close() + w.Write([]byte("some data")) +} + func main() { } diff --git a/libgo/misc/cgo/testcarchive/src/libgo6/sigprof.go b/libgo/misc/cgo/testcarchive/src/libgo6/sigprof.go new file mode 100644 index 00000000000..4cb05dc6178 --- /dev/null +++ b/libgo/misc/cgo/testcarchive/src/libgo6/sigprof.go @@ -0,0 +1,25 @@ +// Copyright 2016 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package main + +import ( + "io/ioutil" + "runtime/pprof" +) + +import "C" + +//export go_start_profile +func go_start_profile() { + pprof.StartCPUProfile(ioutil.Discard) +} + +//export go_stop_profile +func go_stop_profile() { + pprof.StopCPUProfile() +} + +func main() { +} diff --git a/libgo/misc/cgo/testcshared/main0.c b/libgo/misc/cgo/testcshared/main0.c index 1274b8950eb..39ef7e30513 100644 --- a/libgo/misc/cgo/testcshared/main0.c +++ b/libgo/misc/cgo/testcshared/main0.c @@ -12,6 +12,7 @@ // int8_t DidInitRun(); // int8_t DidMainRun(); // int32_t FromPkg(); +// uint32_t Divu(uint32_t, uint32_t); int main(void) { int8_t ran_init = DidInitRun(); if (!ran_init) { @@ -30,6 +31,11 @@ int main(void) { fprintf(stderr, "ERROR: FromPkg=%d, want %d\n", from_pkg, 1024); return 1; } + uint32_t divu = Divu(2264, 31); + if (divu != 73) { + fprintf(stderr, "ERROR: Divu(2264, 31)=%d, want %d\n", divu, 73); + return 1; + } // test.bash looks for "PASS" to ensure this program has reached the end. printf("PASS\n"); return 0; diff --git a/libgo/misc/cgo/testcshared/src/p/p.go b/libgo/misc/cgo/testcshared/src/p/p.go index 82b445c1210..0f02cf3ce6c 100644 --- a/libgo/misc/cgo/testcshared/src/p/p.go +++ b/libgo/misc/cgo/testcshared/src/p/p.go @@ -8,3 +8,6 @@ import "C" //export FromPkg func FromPkg() int32 { return 1024 } + +//export Divu +func Divu(a, b uint32) uint32 { return a / b } diff --git a/libgo/misc/cgo/testcshared/test.bash b/libgo/misc/cgo/testcshared/test.bash index 0315fb07f57..315a0d40367 100644 --- a/libgo/misc/cgo/testcshared/test.bash +++ b/libgo/misc/cgo/testcshared/test.bash @@ -27,7 +27,7 @@ fi # Directory where cgo headers and outputs will be installed. # The installation directory format varies depending on the platform. installdir=pkg/${goos}_${goarch}_testcshared_shared -if [ "${goos}" == "darwin" ]; then +if [ "${goos}" = "darwin" ]; then installdir=pkg/${goos}_${goarch}_testcshared fi @@ -40,13 +40,13 @@ function cleanup() { rm -f testp testp2 testp3 testp4 testp5 rm -rf pkg "${goroot}/${installdir}" - if [ "$goos" == "android" ]; then + if [ "$goos" = "android" ]; then adb shell rm -rf "$androidpath" fi } trap cleanup EXIT -if [ "$goos" == "android" ]; then +if [ "$goos" = "android" ]; then adb shell mkdir -p "$androidpath" fi @@ -69,7 +69,7 @@ function run() { function binpush() { bin=${1} - if [ "$goos" == "android" ]; then + if [ "$goos" = "android" ]; then adb push "$bin" "${androidpath}/${bin}" 2>/dev/null fi } @@ -79,7 +79,7 @@ rm -rf pkg suffix="-installsuffix testcshared" libext="so" -if [ "$goos" == "darwin" ]; then +if [ "$goos" = "darwin" ]; then libext="dylib" fi @@ -89,7 +89,7 @@ GOPATH=$(pwd) go install -buildmode=c-shared $suffix libgo GOPATH=$(pwd) go build -buildmode=c-shared $suffix -o libgo.$libext src/libgo/libgo.go binpush libgo.$libext -if [ "$goos" == "linux" ] || [ "$goos" == "android" ] ; then +if [ "$goos" = "linux" ] || [ "$goos" = "android" ] ; then if readelf -d libgo.$libext | grep TEXTREL >/dev/null; then echo "libgo.$libext has TEXTREL set" exit 1 @@ -97,8 +97,8 @@ if [ "$goos" == "linux" ] || [ "$goos" == "android" ] ; then fi GOGCCFLAGS=$(go env GOGCCFLAGS) -if [ "$goos" == "android" ]; then - GOGCCFLAGS="${GOGCCFLAGS} -pie" +if [ "$goos" = "android" ]; then + GOGCCFLAGS="${GOGCCFLAGS} -pie -fuse-ld=gold" fi status=0 @@ -127,7 +127,7 @@ fi GOPATH=$(pwd) go build -buildmode=c-shared $suffix -o libgo2.$libext libgo2 binpush libgo2.$libext linkflags="-Wl,--no-as-needed" -if [ "$goos" == "darwin" ]; then +if [ "$goos" = "darwin" ]; then linkflags="" fi $(go env CC) ${GOGCCFLAGS} -o testp2 main2.c $linkflags libgo2.$libext @@ -139,7 +139,7 @@ if [ "$output" != "PASS" ]; then fi # test3: tests main.main is exported on android. -if [ "$goos" == "android" ]; then +if [ "$goos" = "android" ]; then $(go env CC) ${GOGCCFLAGS} -o testp3 main3.c -ldl binpush testp3 output=$(run ./testp ./libgo.so) diff --git a/libgo/misc/cgo/testplugin/src/issue19534/main.go b/libgo/misc/cgo/testplugin/src/issue19534/main.go new file mode 100644 index 00000000000..de263b6f0f2 --- /dev/null +++ b/libgo/misc/cgo/testplugin/src/issue19534/main.go @@ -0,0 +1,23 @@ +// Copyright 2017 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package main + +import "plugin" + +func main() { + p, err := plugin.Open("plugin.so") + if err != nil { + panic(err) + } + + sym, err := p.Lookup("Foo") + if err != nil { + panic(err) + } + f := sym.(func() int) + if f() != 42 { + panic("expected f() == 42") + } +} diff --git a/libgo/misc/cgo/testplugin/src/issue19534/plugin.go b/libgo/misc/cgo/testplugin/src/issue19534/plugin.go new file mode 100644 index 00000000000..582d33305c9 --- /dev/null +++ b/libgo/misc/cgo/testplugin/src/issue19534/plugin.go @@ -0,0 +1,9 @@ +// Copyright 2017 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package main + +func Foo() int { + return 42 +} diff --git a/libgo/misc/cgo/testplugin/test.bash b/libgo/misc/cgo/testplugin/test.bash index ab7430acc37..69df5bd2bfa 100644 --- a/libgo/misc/cgo/testplugin/test.bash +++ b/libgo/misc/cgo/testplugin/test.bash @@ -16,7 +16,7 @@ goarch=$(go env GOARCH) function cleanup() { rm -f plugin*.so unnamed*.so iface*.so - rm -rf host pkg sub iface issue18676 + rm -rf host pkg sub iface issue18676 issue19534 } trap cleanup EXIT @@ -44,3 +44,9 @@ LD_LIBRARY_PATH=$(pwd) ./iface GOPATH=$(pwd) go build -buildmode=plugin -o plugin.so src/issue18676/plugin.go GOPATH=$(pwd) go build -o issue18676 src/issue18676/main.go timeout 10s ./issue18676 + +# Test for issue 19534 - that we can load a plugin built in a path with non-alpha +# characters +GOPATH=$(pwd) go build -buildmode=plugin -ldflags='-pluginpath=issue.19534' -o plugin.so src/issue19534/plugin.go +GOPATH=$(pwd) go build -o issue19534 src/issue19534/main.go +./issue19534 diff --git a/libgo/misc/cgo/testplugin/unnamed1.go b/libgo/misc/cgo/testplugin/unnamed1.go index 102edaf3e29..5c1df086d76 100644 --- a/libgo/misc/cgo/testplugin/unnamed1.go +++ b/libgo/misc/cgo/testplugin/unnamed1.go @@ -9,4 +9,15 @@ import "C" func FuncInt() int { return 1 } +// Add a recursive type to to check that type equality across plugins doesn't +// crash. See https://golang.org/issues/19258 +func FuncRecursive() X { return X{} } + +type Y struct { + X *X +} +type X struct { + Y Y +} + func main() {} diff --git a/libgo/misc/cgo/testplugin/unnamed2.go b/libgo/misc/cgo/testplugin/unnamed2.go index 55070d5e9f7..7ef66109c5c 100644 --- a/libgo/misc/cgo/testplugin/unnamed2.go +++ b/libgo/misc/cgo/testplugin/unnamed2.go @@ -9,4 +9,13 @@ import "C" func FuncInt() int { return 2 } +func FuncRecursive() X { return X{} } + +type Y struct { + X *X +} +type X struct { + Y Y +} + func main() {} diff --git a/libgo/misc/cgo/testsanitizers/test.bash b/libgo/misc/cgo/testsanitizers/test.bash index 4da85020d89..9f80af6c507 100644 --- a/libgo/misc/cgo/testsanitizers/test.bash +++ b/libgo/misc/cgo/testsanitizers/test.bash @@ -72,12 +72,12 @@ testmsanshared() { goos=$(go env GOOS) suffix="-installsuffix testsanitizers" libext="so" - if [ "$goos" == "darwin" ]; then + if [ "$goos" = "darwin" ]; then libext="dylib" fi go build -msan -buildmode=c-shared $suffix -o ${TMPDIR}/libmsanshared.$libext msan_shared.go - echo 'int main() { return 0; }' > ${TMPDIR}/testmsanshared.c + echo 'int main() { return 0; }' > ${TMPDIR}/testmsanshared.c $CC $(go env GOGCCFLAGS) -fsanitize=memory -o ${TMPDIR}/testmsanshared ${TMPDIR}/testmsanshared.c ${TMPDIR}/libmsanshared.$libext if ! LD_LIBRARY_PATH=. ${TMPDIR}/testmsanshared; then @@ -131,21 +131,43 @@ if test "$msan" = "yes"; then testmsanshared fi +testtsanshared() { + goos=$(go env GOOS) + suffix="-installsuffix tsan" + libext="so" + if [ "$goos" = "darwin" ]; then + libext="dylib" + fi + go build -buildmode=c-shared $suffix -o ${TMPDIR}/libtsanshared.$libext tsan_shared.go + + echo 'int main() { return 0; }' > ${TMPDIR}/testtsanshared.c + $CC $(go env GOGCCFLAGS) -fsanitize=thread -o ${TMPDIR}/testtsanshared ${TMPDIR}/testtsanshared.c ${TMPDIR}/libtsanshared.$libext + + if ! LD_LIBRARY_PATH=. ${TMPDIR}/testtsanshared; then + echo "FAIL: tsan_shared" + status=1 + fi + rm -f ${TMPDIR}/{testtsanshared,testtsanshared.c,libtsanshared.$libext} +} + if test "$tsan" = "yes"; then echo 'int main() { return 0; }' > ${TMPDIR}/testsanitizers$$.c ok=yes if ! $CC -fsanitize=thread ${TMPDIR}/testsanitizers$$.c -o ${TMPDIR}/testsanitizers$$ &> ${TMPDIR}/testsanitizers$$.err; then ok=no fi - if grep "unrecognized" ${TMPDIR}/testsanitizers$$.err >& /dev/null; then + if grep "unrecognized" ${TMPDIR}/testsanitizers$$.err >& /dev/null; then echo "skipping tsan tests: -fsanitize=thread not supported" tsan=no - elif test "$ok" != "yes"; then - cat ${TMPDIR}/testsanitizers$$.err - echo "skipping tsan tests: -fsanitizer=thread build failed" - tsan=no - fi - rm -f ${TMPDIR}/testsanitizers$$* + elif test "$ok" != "yes"; then + cat ${TMPDIR}/testsanitizers$$.err + echo "skipping tsan tests: -fsanitizer=thread build failed" + tsan=no + elif ! ${TMPDIR}/testsanitizers$$ 2>&1; then + echo "skipping tsan tests: running tsan program failed" + tsan=no + fi + rm -f ${TMPDIR}/testsanitizers$$* fi # Run a TSAN test. @@ -177,8 +199,10 @@ if test "$tsan" = "yes"; then # These tests are only reliable using clang or GCC version 7 or later. # Otherwise runtime/cgo/libcgo.h can't tell whether TSAN is in use. ok=false + clang=false if ${CC} --version | grep clang >/dev/null 2>&1; then ok=true + clang=true else ver=$($CC -dumpversion) major=$(echo $ver | sed -e 's/\([0-9]*\).*/\1/') @@ -190,14 +214,19 @@ if test "$tsan" = "yes"; then fi if test "$ok" = "true"; then - # This test requires rebuilding os/user with -fsanitize=thread. + # These tests require rebuilding os/user with -fsanitize=thread. testtsan tsan5.go "CGO_CFLAGS=-fsanitize=thread CGO_LDFLAGS=-fsanitize=thread" "-installsuffix=tsan" - - # This test requires rebuilding runtime/cgo with -fsanitize=thread. testtsan tsan6.go "CGO_CFLAGS=-fsanitize=thread CGO_LDFLAGS=-fsanitize=thread" "-installsuffix=tsan" - - # This test requires rebuilding runtime/cgo with -fsanitize=thread. testtsan tsan7.go "CGO_CFLAGS=-fsanitize=thread CGO_LDFLAGS=-fsanitize=thread" "-installsuffix=tsan" + + # The remaining tests reportedly hang when built with GCC; issue #21196. + if test "$clang" = "true"; then + testtsan tsan10.go "CGO_CFLAGS=-fsanitize=thread CGO_LDFLAGS=-fsanitize=thread" "-installsuffix=tsan" + testtsan tsan11.go "CGO_CFLAGS=-fsanitize=thread CGO_LDFLAGS=-fsanitize=thread" "-installsuffix=tsan" + testtsan tsan12.go "CGO_CFLAGS=-fsanitize=thread CGO_LDFLAGS=-fsanitize=thread" "-installsuffix=tsan" + fi + + testtsanshared fi fi diff --git a/libgo/misc/cgo/testsanitizers/tsan10.go b/libgo/misc/cgo/testsanitizers/tsan10.go new file mode 100644 index 00000000000..a40f2455537 --- /dev/null +++ b/libgo/misc/cgo/testsanitizers/tsan10.go @@ -0,0 +1,31 @@ +// Copyright 2017 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package main + +// This program hung when run under the C/C++ ThreadSanitizer. +// TSAN defers asynchronous signals until the signaled thread calls into libc. +// Since the Go runtime makes direct futex syscalls, Go runtime threads could +// run for an arbitrarily long time without triggering the libc interceptors. +// See https://golang.org/issue/18717. + +import ( + "os" + "os/signal" + "syscall" +) + +/* +#cgo CFLAGS: -g -fsanitize=thread +#cgo LDFLAGS: -g -fsanitize=thread +*/ +import "C" + +func main() { + c := make(chan os.Signal, 1) + signal.Notify(c, syscall.SIGUSR1) + defer signal.Stop(c) + syscall.Kill(syscall.Getpid(), syscall.SIGUSR1) + <-c +} diff --git a/libgo/misc/cgo/testsanitizers/tsan11.go b/libgo/misc/cgo/testsanitizers/tsan11.go new file mode 100644 index 00000000000..70ac9c8ae2c --- /dev/null +++ b/libgo/misc/cgo/testsanitizers/tsan11.go @@ -0,0 +1,55 @@ +// Copyright 2017 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package main + +// This program hung when run under the C/C++ ThreadSanitizer. TSAN defers +// asynchronous signals until the signaled thread calls into libc. The runtime's +// sysmon goroutine idles itself using direct usleep syscalls, so it could +// run for an arbitrarily long time without triggering the libc interceptors. +// See https://golang.org/issue/18717. + +import ( + "os" + "os/signal" + "syscall" +) + +/* +#cgo CFLAGS: -g -fsanitize=thread +#cgo LDFLAGS: -g -fsanitize=thread + +#include +#include +#include +#include + +static void raise_usr2(int signo) { + raise(SIGUSR2); +} + +static void register_handler(int signo) { + struct sigaction sa; + memset(&sa, 0, sizeof(sa)); + sigemptyset(&sa.sa_mask); + sa.sa_flags = SA_ONSTACK; + sa.sa_handler = raise_usr2; + + if (sigaction(SIGUSR1, &sa, NULL) != 0) { + perror("failed to register SIGUSR1 handler"); + exit(EXIT_FAILURE); + } +} +*/ +import "C" + +func main() { + ch := make(chan os.Signal) + signal.Notify(ch, syscall.SIGUSR2) + + C.register_handler(C.int(syscall.SIGUSR1)) + syscall.Kill(syscall.Getpid(), syscall.SIGUSR1) + + <-ch +} diff --git a/libgo/misc/cgo/testsanitizers/tsan12.go b/libgo/misc/cgo/testsanitizers/tsan12.go new file mode 100644 index 00000000000..3e767eee1f8 --- /dev/null +++ b/libgo/misc/cgo/testsanitizers/tsan12.go @@ -0,0 +1,35 @@ +// Copyright 2017 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package main + +// This program hung when run under the C/C++ ThreadSanitizer. TSAN installs a +// libc interceptor that writes signal handlers to a global variable within the +// TSAN runtime instead of making a sigaction system call. A bug in +// syscall.runtime_AfterForkInChild corrupted TSAN's signal forwarding table +// during calls to (*os/exec.Cmd).Run, causing the parent process to fail to +// invoke signal handlers. + +import ( + "fmt" + "os" + "os/exec" + "os/signal" + "syscall" +) + +import "C" + +func main() { + ch := make(chan os.Signal) + signal.Notify(ch, syscall.SIGUSR1) + + if err := exec.Command("true").Run(); err != nil { + fmt.Fprintf(os.Stderr, "Unexpected error from `true`: %v", err) + os.Exit(1) + } + + syscall.Kill(syscall.Getpid(), syscall.SIGUSR1) + <-ch +} diff --git a/libgo/misc/cgo/testsanitizers/tsan_shared.go b/libgo/misc/cgo/testsanitizers/tsan_shared.go new file mode 100644 index 00000000000..55ff67ecbaf --- /dev/null +++ b/libgo/misc/cgo/testsanitizers/tsan_shared.go @@ -0,0 +1,63 @@ +// Copyright 2017 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package main + +// This program failed with SIGSEGV when run under the C/C++ ThreadSanitizer. +// The Go runtime had re-registered the C handler with the wrong flags due to a +// typo, resulting in null pointers being passed for the info and context +// parameters to the handler. + +/* +#cgo CFLAGS: -fsanitize=thread +#cgo LDFLAGS: -fsanitize=thread + +#include +#include +#include +#include +#include + +void check_params(int signo, siginfo_t *info, void *context) { + ucontext_t* uc = (ucontext_t*)(context); + + if (info->si_signo != signo) { + fprintf(stderr, "info->si_signo does not match signo.\n"); + abort(); + } + + if (uc->uc_stack.ss_size == 0) { + fprintf(stderr, "uc_stack has size 0.\n"); + abort(); + } +} + + +// Set up the signal handler in a high priority constructor, so +// that it is installed before the Go code starts. + +static void register_handler(void) __attribute__ ((constructor (200))); + +static void register_handler() { + struct sigaction sa; + memset(&sa, 0, sizeof(sa)); + sigemptyset(&sa.sa_mask); + sa.sa_flags = SA_SIGINFO; + sa.sa_sigaction = check_params; + + if (sigaction(SIGUSR1, &sa, NULL) != 0) { + perror("failed to register SIGUSR1 handler"); + exit(EXIT_FAILURE); + } +} +*/ +import "C" + +import "syscall" + +func init() { + C.raise(C.int(syscall.SIGUSR1)) +} + +func main() {} diff --git a/libgo/misc/cgo/testshared/shared_test.go b/libgo/misc/cgo/testshared/shared_test.go index f0766e511ec..9e682a2fb59 100644 --- a/libgo/misc/cgo/testshared/shared_test.go +++ b/libgo/misc/cgo/testshared/shared_test.go @@ -10,7 +10,6 @@ import ( "debug/elf" "encoding/binary" "errors" - "flag" "fmt" "go/build" "io" @@ -166,7 +165,6 @@ func TestMain(m *testing.M) { // That won't work if GOBIN is set. os.Unsetenv("GOBIN") - flag.Parse() exitCode, err := testMain(m) if err != nil { log.Fatal(err) @@ -402,6 +400,12 @@ func TestTrivialExecutablePIE(t *testing.T) { AssertHasRPath(t, "./trivial.pie", gorootInstallDir) } +// Build a division test program and check it runs. +func TestDivisionExecutable(t *testing.T) { + goCmd(t, "install", "-linkshared", "division") + run(t, "division executable", "./bin/division") +} + // Build an executable that uses cgo linked against the shared runtime and check it // runs. func TestCgoExecutable(t *testing.T) { @@ -759,6 +763,13 @@ func appendFile(path, content string) { } } +func writeFile(path, content string) { + err := ioutil.WriteFile(path, []byte(content), 0644) + if err != nil { + log.Fatalf("ioutil.WriteFile failed: %v", err) + } +} + func TestABIChecking(t *testing.T) { goCmd(t, "install", "-buildmode=shared", "-linkshared", "depBase") goCmd(t, "install", "-linkshared", "exe") @@ -797,9 +808,10 @@ func TestABIChecking(t *testing.T) { run(t, "rebuilt exe", "./bin/exe") // If we make a change which does not break ABI (such as adding an unexported - // function) and rebuild libdepBase.so, exe still works. + // function) and rebuild libdepBase.so, exe still works, even if new function + // is in a file by itself. resetFileStamps() - appendFile("src/depBase/dep.go", "func noABIBreak() {}\n") + writeFile("src/depBase/dep2.go", "package depBase\nfunc noABIBreak() {}\n") goCmd(t, "install", "-buildmode=shared", "-linkshared", "depBase") run(t, "after non-ABI breaking change", "./bin/exe") } diff --git a/libgo/misc/cgo/testshared/src/division/division.go b/libgo/misc/cgo/testshared/src/division/division.go new file mode 100644 index 00000000000..bb5fc984602 --- /dev/null +++ b/libgo/misc/cgo/testshared/src/division/division.go @@ -0,0 +1,17 @@ +// Copyright 2017 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package main + +//go:noinline +func div(x, y uint32) uint32 { + return x / y +} + +func main() { + a := div(97, 11) + if a != 8 { + panic("FAIL") + } +} diff --git a/libgo/runtime/go-now.c b/libgo/runtime/go-now.c index d24e6ee76af..13e8f517722 100644 --- a/libgo/runtime/go-now.c +++ b/libgo/runtime/go-now.c @@ -8,13 +8,22 @@ #include "runtime.h" -// Return current time. This is the implementation of time.now(). +// Return current time. This is the implementation of time.walltime(). -struct time_now_ret +struct walltime_ret +{ + int64_t sec; + int32_t nsec; +}; + +struct walltime_ret now() __asm__ (GOSYM_PREFIX "runtime.walltime") + __attribute__ ((no_split_stack)); + +struct walltime_ret now() { struct timespec ts; - struct time_now_ret ret; + struct walltime_ret ret; clock_gettime (CLOCK_REALTIME, &ts); ret.sec = ts.tv_sec; diff --git a/libgo/runtime/proc.c b/libgo/runtime/proc.c index 3e3437e6a47..e591824b140 100644 --- a/libgo/runtime/proc.c +++ b/libgo/runtime/proc.c @@ -375,6 +375,8 @@ bool runtime_isarchive; extern void kickoff(void) __asm__(GOSYM_PREFIX "runtime.kickoff"); +extern void minit(void) + __asm__(GOSYM_PREFIX "runtime.minit"); extern void mstart1(void) __asm__(GOSYM_PREFIX "runtime.mstart1"); extern void stopm(void) @@ -476,6 +478,10 @@ runtime_mstart(void *arg) gp->entry = nil; gp->param = nil; + // We have to call minit before we call getcontext, + // because getcontext will copy the signal mask. + minit(); + initcontext(); // Record top of stack for use by mcall. diff --git a/libgo/runtime/runtime.h b/libgo/runtime/runtime.h index 3324038a57d..dd5a958888f 100644 --- a/libgo/runtime/runtime.h +++ b/libgo/runtime/runtime.h @@ -193,16 +193,6 @@ enum { }; void runtime_hashinit(void); -void runtime_traceback(int32) - __asm__ (GOSYM_PREFIX "runtime.traceback"); -void runtime_tracebackothers(G*) - __asm__ (GOSYM_PREFIX "runtime.tracebackothers"); -enum -{ - // The maximum number of frames we print for a traceback - TracebackMaxFrames = 100, -}; - /* * external data */ @@ -217,7 +207,6 @@ extern M* runtime_getallm(void) extern Sched* runtime_sched; extern uint32 runtime_panicking(void) __asm__ (GOSYM_PREFIX "runtime.getPanicking"); -extern struct debugVars runtime_debug; extern bool runtime_isstarted; extern bool runtime_isarchive; @@ -253,10 +242,6 @@ void runtime_schedinit(void) __asm__ (GOSYM_PREFIX "runtime.schedinit"); void runtime_initsig(bool) __asm__ (GOSYM_PREFIX "runtime.initsig"); -void runtime_goroutineheader(G*) - __asm__ (GOSYM_PREFIX "runtime.goroutineheader"); -void runtime_printtrace(Slice, G*) - __asm__ (GOSYM_PREFIX "runtime.printtrace"); #define runtime_open(p, f, m) open((p), (f), (m)) #define runtime_read(d, v, n) read((d), (v), (n)) #define runtime_write(d, v, n) write((d), (v), (n)) @@ -327,8 +312,6 @@ G* __go_go(void (*pfn)(void*), void*); int32 runtime_callers(int32, Location*, int32, bool keep_callers); int64 runtime_nanotime(void) // monotonic time __asm__(GOSYM_PREFIX "runtime.nanotime"); -int64 runtime_unixnanotime(void) // real time, can skip - __asm__ (GOSYM_PREFIX "runtime.unixnanotime"); void runtime_dopanic(int32) __attribute__ ((noreturn)); void runtime_startpanic(void) __asm__ (GOSYM_PREFIX "runtime.startpanic"); @@ -343,22 +326,11 @@ void runtime_blockevent(int64, int32); extern int64 runtime_blockprofilerate; G* runtime_netpoll(bool) __asm__ (GOSYM_PREFIX "runtime.netpoll"); -void runtime_crash(void) - __asm__ (GOSYM_PREFIX "runtime.crash"); void runtime_parsedebugvars(void) __asm__(GOSYM_PREFIX "runtime.parsedebugvars"); void _rt0_go(void); G* runtime_timejump(void); -void runtime_callStopTheWorldWithSema(void) - __asm__(GOSYM_PREFIX "runtime.callStopTheWorldWithSema"); -void runtime_callStartTheWorldWithSema(void) - __asm__(GOSYM_PREFIX "runtime.callStartTheWorldWithSema"); -void runtime_acquireWorldsema(void) - __asm__(GOSYM_PREFIX "runtime.acquireWorldsema"); -void runtime_releaseWorldsema(void) - __asm__(GOSYM_PREFIX "runtime.releaseWorldsema"); - /* * mutual exclusion locks. in the uncontended case, * as fast as spin locks (just a few user-level instructions), @@ -403,17 +375,6 @@ bool runtime_notetsleep(Note*, int64) // false - timeout bool runtime_notetsleepg(Note*, int64) // false - timeout __asm__ (GOSYM_PREFIX "runtime.notetsleepg"); -/* - * Lock-free stack. - * Initialize uint64 head to 0, compare with 0 to test for emptiness. - * The stack does not keep pointers to nodes, - * so they can be garbage collected if there are no other pointers to nodes. - */ -void runtime_lfstackpush(uint64 *head, LFNode *node) - __asm__ (GOSYM_PREFIX "runtime.lfstackpush"); -void* runtime_lfstackpop(uint64 *head) - __asm__ (GOSYM_PREFIX "runtime.lfstackpop"); - /* * low level C-called */ @@ -454,9 +415,6 @@ void runtime_procyield(uint32) void runtime_osyield(void) __asm__(GOSYM_PREFIX "runtime.osyield"); -void runtime_printcreatedby(G*) - __asm__(GOSYM_PREFIX "runtime.printcreatedby"); - uintptr runtime_memlimit(void); #define ISNAN(f) __builtin_isnan(f) @@ -491,15 +449,6 @@ void runtime_badsignal(int); Defer* runtime_newdefer(void); void runtime_freedefer(Defer*); -struct time_now_ret -{ - int64_t sec; - int32_t nsec; -}; - -struct time_now_ret now() __asm__ (GOSYM_PREFIX "time.now") - __attribute__ ((no_split_stack)); - extern void _cgo_wait_runtime_init_done (void); extern void _cgo_notify_runtime_init_done (void) __asm__ (GOSYM_PREFIX "runtime._cgo_notify_runtime_init_done"); diff --git a/libgo/runtime/runtime_c.c b/libgo/runtime/runtime_c.c index 6da35210440..9a6672d602e 100644 --- a/libgo/runtime/runtime_c.c +++ b/libgo/runtime/runtime_c.c @@ -10,6 +10,10 @@ #include #endif +#ifdef __linux__ +#include +#endif + #include "config.h" #include "runtime.h" @@ -81,13 +85,6 @@ runtime_signalstack(byte *p, uintptr n) *(int *)0xf1 = 0xf1; } -struct debugVars runtime_debug; - -void -runtime_setdebug(struct debugVars* d) { - runtime_debug = *d; -} - int32 go_open(char *, int32, int32) __asm__ (GOSYM_PREFIX "runtime.open"); @@ -184,3 +181,18 @@ publicationBarrier() { __atomic_thread_fence(__ATOMIC_RELEASE); } + +#ifdef __linux__ + +/* Currently sbrk0 is only called on GNU/Linux. */ + +uintptr sbrk0(void) + __asm__ (GOSYM_PREFIX "runtime.sbrk0"); + +uintptr +sbrk0() +{ + return syscall(SYS_brk, (uintptr)(0)); +} + +#endif /* __linux__ */ diff --git a/libgo/testsuite/gotest b/libgo/testsuite/gotest index ad3a485d14f..83f78d4d9b9 100755 --- a/libgo/testsuite/gotest +++ b/libgo/testsuite/gotest @@ -348,18 +348,18 @@ x) fi match=false ;; - $goos | $goarch | cgo) + $goos | $goarch | cgo | go1.[0-9]) match=true ;; - "!"$goos | "!"$goarch | "!cgo") + "!"$goos | "!"$goarch | "!cgo" | "!"go1.[0-9]) ;; *,*) cmatch=true for ctag in `echo $tag | sed -e 's/,/ /g'`; do case $ctag in - $goos | $goarch | cgo) + $goos | $goarch | cgo | go1.[0-9]) ;; - "!"$goos | "!"$goarch | "!cgo") + "!"$goos | "!"$goarch | "!cgo" | "!"go1.[0-9]) cmatch=false ;; "!"*)