From 2b6d99468d4d988fd5f5ea3e9be41a3dc95a1291 Mon Sep 17 00:00:00 2001 From: Ian Lance Taylor Date: Fri, 10 Jul 2020 10:51:40 -0700 Subject: [PATCH] libgo: update to Go 1.14.4 release Reviewed-on: https://go-review.googlesource.com/c/gofrontend/+/241999 --- gcc/go/gofrontend/MERGE | 2 +- libgo/MERGE | 2 +- libgo/VERSION | 2 +- libgo/go/cmd/cgo/gcc.go | 12 ++- libgo/go/encoding/json/decode.go | 5 ++ libgo/go/encoding/json/decode_test.go | 33 ++++++- libgo/go/encoding/json/encode.go | 11 +-- libgo/go/encoding/json/encode_test.go | 87 ++++++++++++------- libgo/go/encoding/json/stream_test.go | 8 +- libgo/go/go/doc/example.go | 6 +- libgo/go/go/doc/example_test.go | 76 ++++++++++++---- libgo/go/go/parser/interface.go | 8 +- libgo/go/math/big/nat.go | 15 +++- libgo/go/math/big/nat_test.go | 18 ++++ libgo/go/os/os_test.go | 35 ++++++++ libgo/go/runtime/crash_test.go | 14 ++- libgo/go/runtime/mgcscavenge.go | 22 +++++ libgo/go/runtime/mpagecache.go | 13 ++- libgo/go/runtime/mpagecache_test.go | 33 ++++++- libgo/go/runtime/proc.go | 6 ++ libgo/go/runtime/proc_test.go | 24 +++++ .../runtime/testdata/testprog/lockosthread.go | 49 +++++++++++ libgo/misc/cgo/test/testx.go | 10 +++ 23 files changed, 409 insertions(+), 82 deletions(-) diff --git a/gcc/go/gofrontend/MERGE b/gcc/go/gofrontend/MERGE index ecef60400cc..64b13f410e0 100644 --- a/gcc/go/gofrontend/MERGE +++ b/gcc/go/gofrontend/MERGE @@ -1,4 +1,4 @@ -30674246ef60ab74566a21f362a7de7a09b99955 +2ad0970e9da95024110cd3244e9e21313af70a5f The first line of this file holds the git revision number of the last merge done from the gofrontend repository. diff --git a/libgo/MERGE b/libgo/MERGE index 8cae45f6b8e..07547d064a1 100644 --- a/libgo/MERGE +++ b/libgo/MERGE @@ -1,4 +1,4 @@ -96745b980cfde139e8611772e2bc0c59a8e6cdf7 +83b181c68bf332ac7948f145f33d128377a09c42 The first line of this file holds the git revision number of the last merge done from the master library sources. diff --git a/libgo/VERSION b/libgo/VERSION index 864916ef499..d8281a2e996 100644 --- a/libgo/VERSION +++ b/libgo/VERSION @@ -1 +1 @@ -go1.14.2 +go1.14.4 diff --git a/libgo/go/cmd/cgo/gcc.go b/libgo/go/cmd/cgo/gcc.go index 310316bdda4..e38972913be 100644 --- a/libgo/go/cmd/cgo/gcc.go +++ b/libgo/go/cmd/cgo/gcc.go @@ -2082,6 +2082,10 @@ var goIdent = make(map[string]*ast.Ident) // that may contain a pointer. This is used for cgo pointer checking. var unionWithPointer = make(map[ast.Expr]bool) +// anonymousStructTag provides a consistent tag for an anonymous struct. +// The same dwarf.StructType pointer will always get the same tag. +var anonymousStructTag = make(map[*dwarf.StructType]string) + func (c *typeConv) Init(ptrSize, intSize int64) { c.ptrSize = ptrSize c.intSize = intSize @@ -2430,8 +2434,12 @@ func (c *typeConv) loadType(dtype dwarf.Type, pos token.Pos, parent string) *Typ break } if tag == "" { - tag = "__" + strconv.Itoa(tagGen) - tagGen++ + tag = anonymousStructTag[dt] + if tag == "" { + tag = "__" + strconv.Itoa(tagGen) + tagGen++ + anonymousStructTag[dt] = tag + } } else if t.C.Empty() { t.C.Set(dt.Kind + " " + tag) } diff --git a/libgo/go/encoding/json/decode.go b/libgo/go/encoding/json/decode.go index b43484692e1..b60e2bb0b2c 100644 --- a/libgo/go/encoding/json/decode.go +++ b/libgo/go/encoding/json/decode.go @@ -1217,6 +1217,11 @@ func (d *decodeState) unquoteBytes(s []byte) (t []byte, ok bool) { if r == -1 { return s, true } + // Only perform up to one safe unquote for each re-scanned string + // literal. In some edge cases, the decoder unquotes a literal a second + // time, even after another literal has been re-scanned. Thus, only the + // first unquote can safely use safeUnquote. + d.safeUnquote = 0 b := make([]byte, len(s)+2*utf8.UTFMax) w := copy(b, s[0:r]) diff --git a/libgo/go/encoding/json/decode_test.go b/libgo/go/encoding/json/decode_test.go index 498bd97b46e..a49181e9823 100644 --- a/libgo/go/encoding/json/decode_test.go +++ b/libgo/go/encoding/json/decode_test.go @@ -2419,7 +2419,7 @@ func (m *textUnmarshalerString) UnmarshalText(text []byte) error { return nil } -// Test unmarshal to a map, with map key is a user defined type. +// Test unmarshal to a map, where the map key is a user defined type. // See golang.org/issues/34437. func TestUnmarshalMapWithTextUnmarshalerStringKey(t *testing.T) { var p map[textUnmarshalerString]string @@ -2428,6 +2428,35 @@ func TestUnmarshalMapWithTextUnmarshalerStringKey(t *testing.T) { } if _, ok := p["foo"]; !ok { - t.Errorf(`Key "foo" is not existed in map: %v`, p) + t.Errorf(`Key "foo" does not exist in map: %v`, p) + } +} + +func TestUnmarshalRescanLiteralMangledUnquote(t *testing.T) { + // See golang.org/issues/38105. + var p map[textUnmarshalerString]string + if err := Unmarshal([]byte(`{"开源":"12345开源"}`), &p); err != nil { + t.Fatalf("Unmarshal unexpected error: %v", err) + } + if _, ok := p["开源"]; !ok { + t.Errorf(`Key "开源" does not exist in map: %v`, p) + } + + // See golang.org/issues/38126. + type T struct { + F1 string `json:"F1,string"` + } + t1 := T{"aaa\tbbb"} + + b, err := Marshal(t1) + if err != nil { + t.Fatalf("Marshal unexpected error: %v", err) + } + var t2 T + if err := Unmarshal(b, &t2); err != nil { + t.Fatalf("Unmarshal unexpected error: %v", err) + } + if t1 != t2 { + t.Errorf("Marshal and Unmarshal roundtrip mismatch: want %q got %q", t1, t2) } } diff --git a/libgo/go/encoding/json/encode.go b/libgo/go/encoding/json/encode.go index 39cdaebde7b..b351cf3f449 100644 --- a/libgo/go/encoding/json/encode.go +++ b/libgo/go/encoding/json/encode.go @@ -635,11 +635,12 @@ func stringEncoder(e *encodeState, v reflect.Value, opts encOpts) { return } if opts.quoted { - b := make([]byte, 0, v.Len()+2) - b = append(b, '"') - b = append(b, []byte(v.String())...) - b = append(b, '"') - e.stringBytes(b, opts.escapeHTML) + e2 := newEncodeState() + // Since we encode the string twice, we only need to escape HTML + // the first time. + e2.string(v.String(), opts.escapeHTML) + e.stringBytes(e2.Bytes(), false) + encodeStatePool.Put(e2) } else { e.string(v.String(), opts.escapeHTML) } diff --git a/libgo/go/encoding/json/encode_test.go b/libgo/go/encoding/json/encode_test.go index 5110c7de9b1..7290eca06f0 100644 --- a/libgo/go/encoding/json/encode_test.go +++ b/libgo/go/encoding/json/encode_test.go @@ -79,37 +79,66 @@ type StringTag struct { NumberStr Number `json:",string"` } -var stringTagExpected = `{ - "BoolStr": "true", - "IntStr": "42", - "UintptrStr": "44", - "StrStr": "\"xzbit\"", - "NumberStr": "46" -}` - -func TestStringTag(t *testing.T) { - var s StringTag - s.BoolStr = true - s.IntStr = 42 - s.UintptrStr = 44 - s.StrStr = "xzbit" - s.NumberStr = "46" - got, err := MarshalIndent(&s, "", " ") - if err != nil { - t.Fatal(err) - } - if got := string(got); got != stringTagExpected { - t.Fatalf(" got: %s\nwant: %s\n", got, stringTagExpected) +func TestRoundtripStringTag(t *testing.T) { + tests := []struct { + name string + in StringTag + want string // empty to just test that we roundtrip + }{ + { + name: "AllTypes", + in: StringTag{ + BoolStr: true, + IntStr: 42, + UintptrStr: 44, + StrStr: "xzbit", + NumberStr: "46", + }, + want: `{ + "BoolStr": "true", + "IntStr": "42", + "UintptrStr": "44", + "StrStr": "\"xzbit\"", + "NumberStr": "46" + }`, + }, + { + // See golang.org/issues/38173. + name: "StringDoubleEscapes", + in: StringTag{ + StrStr: "\b\f\n\r\t\"\\", + NumberStr: "0", // just to satisfy the roundtrip + }, + want: `{ + "BoolStr": "false", + "IntStr": "0", + "UintptrStr": "0", + "StrStr": "\"\\u0008\\u000c\\n\\r\\t\\\"\\\\\"", + "NumberStr": "0" + }`, + }, } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + // Indent with a tab prefix to make the multi-line string + // literals in the table nicer to read. + got, err := MarshalIndent(&test.in, "\t\t\t", "\t") + if err != nil { + t.Fatal(err) + } + if got := string(got); got != test.want { + t.Fatalf(" got: %s\nwant: %s\n", got, test.want) + } - // Verify that it round-trips. - var s2 StringTag - err = NewDecoder(bytes.NewReader(got)).Decode(&s2) - if err != nil { - t.Fatalf("Decode: %v", err) - } - if !reflect.DeepEqual(s, s2) { - t.Fatalf("decode didn't match.\nsource: %#v\nEncoded as:\n%s\ndecode: %#v", s, string(got), s2) + // Verify that it round-trips. + var s2 StringTag + if err := Unmarshal(got, &s2); err != nil { + t.Fatalf("Decode: %v", err) + } + if !reflect.DeepEqual(test.in, s2) { + t.Fatalf("decode didn't match.\nsource: %#v\nEncoded as:\n%s\ndecode: %#v", test.in, string(got), s2) + } + }) } } diff --git a/libgo/go/encoding/json/stream_test.go b/libgo/go/encoding/json/stream_test.go index ebb4f231d15..c9e5334337d 100644 --- a/libgo/go/encoding/json/stream_test.go +++ b/libgo/go/encoding/json/stream_test.go @@ -144,14 +144,15 @@ func TestEncoderSetEscapeHTML(t *testing.T) { }, { "stringOption", stringOption, - `{"bar":"\"\u003chtml\u003efoobar\u003c/html\u003e\""}`, + `{"bar":"\"\\u003chtml\\u003efoobar\\u003c/html\\u003e\""}`, `{"bar":"\"foobar\""}`, }, } { var buf bytes.Buffer enc := NewEncoder(&buf) if err := enc.Encode(tt.v); err != nil { - t.Fatalf("Encode(%s): %s", tt.name, err) + t.Errorf("Encode(%s): %s", tt.name, err) + continue } if got := strings.TrimSpace(buf.String()); got != tt.wantEscape { t.Errorf("Encode(%s) = %#q, want %#q", tt.name, got, tt.wantEscape) @@ -159,7 +160,8 @@ func TestEncoderSetEscapeHTML(t *testing.T) { buf.Reset() enc.SetEscapeHTML(false) if err := enc.Encode(tt.v); err != nil { - t.Fatalf("SetEscapeHTML(false) Encode(%s): %s", tt.name, err) + t.Errorf("SetEscapeHTML(false) Encode(%s): %s", tt.name, err) + continue } if got := strings.TrimSpace(buf.String()); got != tt.want { t.Errorf("SetEscapeHTML(false) Encode(%s) = %#q, want %#q", diff --git a/libgo/go/go/doc/example.go b/libgo/go/go/doc/example.go index a010d3a85ab..ebf81189b5b 100644 --- a/libgo/go/go/doc/example.go +++ b/libgo/go/go/doc/example.go @@ -62,9 +62,6 @@ func Examples(testFiles ...*ast.File) []*Example { if !ok || f.Recv != nil { continue } - if params := f.Type.Params; len(params.List) != 0 { - continue // function has params; not a valid example - } numDecl++ name := f.Name.Name if isTest(name, "Test") || isTest(name, "Benchmark") { @@ -74,6 +71,9 @@ func Examples(testFiles ...*ast.File) []*Example { if !isTest(name, "Example") { continue } + if params := f.Type.Params; len(params.List) != 0 { + continue // function has params; not a valid example + } if f.Body == nil { // ast.File.Body nil dereference (see issue 28044) continue } diff --git a/libgo/go/go/doc/example_test.go b/libgo/go/go/doc/example_test.go index cd2f469c2fb..32db3cd7da7 100644 --- a/libgo/go/go/doc/example_test.go +++ b/libgo/go/go/doc/example_test.go @@ -331,25 +331,65 @@ func main() { } ` +const exampleWholeFileFunction = `package foo_test + +func Foo(x int) { +} + +func Example() { + fmt.Println("Hello, world!") + // Output: Hello, world! +} +` + +const exampleWholeFileFunctionOutput = `package main + +func Foo(x int) { +} + +func main() { + fmt.Println("Hello, world!") +} +` + +var exampleWholeFileTestCases = []struct { + Title, Source, Play, Output string +}{ + { + "Methods", + exampleWholeFile, + exampleWholeFileOutput, + "Hello, world!\n", + }, + { + "Function", + exampleWholeFileFunction, + exampleWholeFileFunctionOutput, + "Hello, world!\n", + }, +} + func TestExamplesWholeFile(t *testing.T) { - fset := token.NewFileSet() - file, err := parser.ParseFile(fset, "test.go", strings.NewReader(exampleWholeFile), parser.ParseComments) - if err != nil { - t.Fatal(err) - } - es := doc.Examples(file) - if len(es) != 1 { - t.Fatalf("wrong number of examples; got %d want 1", len(es)) - } - e := es[0] - if e.Name != "" { - t.Errorf("got Name == %q, want %q", e.Name, "") - } - if g, w := formatFile(t, fset, e.Play), exampleWholeFileOutput; g != w { - t.Errorf("got Play == %q, want %q", g, w) - } - if g, w := e.Output, "Hello, world!\n"; g != w { - t.Errorf("got Output == %q, want %q", g, w) + for _, c := range exampleWholeFileTestCases { + fset := token.NewFileSet() + file, err := parser.ParseFile(fset, "test.go", strings.NewReader(c.Source), parser.ParseComments) + if err != nil { + t.Fatal(err) + } + es := doc.Examples(file) + if len(es) != 1 { + t.Fatalf("%s: wrong number of examples; got %d want 1", c.Title, len(es)) + } + e := es[0] + if e.Name != "" { + t.Errorf("%s: got Name == %q, want %q", c.Title, e.Name, "") + } + if g, w := formatFile(t, fset, e.Play), c.Play; g != w { + t.Errorf("%s: got Play == %q, want %q", c.Title, g, w) + } + if g, w := e.Output, c.Output; g != w { + t.Errorf("%s: got Output == %q, want %q", c.Title, g, w) + } } } diff --git a/libgo/go/go/parser/interface.go b/libgo/go/go/parser/interface.go index 500c98d4966..54f9d7b80ac 100644 --- a/libgo/go/go/parser/interface.go +++ b/libgo/go/go/parser/interface.go @@ -133,13 +133,7 @@ func ParseFile(fset *token.FileSet, filename string, src interface{}, mode Mode) // first error encountered are returned. // func ParseDir(fset *token.FileSet, path string, filter func(os.FileInfo) bool, mode Mode) (pkgs map[string]*ast.Package, first error) { - fd, err := os.Open(path) - if err != nil { - return nil, err - } - defer fd.Close() - - list, err := fd.Readdir(-1) + list, err := ioutil.ReadDir(path) if err != nil { return nil, err } diff --git a/libgo/go/math/big/nat.go b/libgo/go/math/big/nat.go index 1b771ca7c63..c31ec5156b8 100644 --- a/libgo/go/math/big/nat.go +++ b/libgo/go/math/big/nat.go @@ -740,7 +740,8 @@ func (z nat) divLarge(u, uIn, vIn nat) (q, r nat) { // The remainder overwrites input u. // // Precondition: -// - len(q) >= len(u)-len(v) +// - q is large enough to hold the quotient u / v +// which has a maximum length of len(u)-len(v)+1. func (q nat) divBasic(u, v nat) { n := len(v) m := len(u) - n @@ -779,6 +780,8 @@ func (q nat) divBasic(u, v nat) { } // D4. + // Compute the remainder u - (q̂*v) << (_W*j). + // The subtraction may overflow if q̂ estimate was off by one. qhatv[n] = mulAddVWW(qhatv[0:n], v, qhat, 0) qhl := len(qhatv) if j+qhl > len(u) && qhatv[n] == 0 { @@ -787,7 +790,11 @@ func (q nat) divBasic(u, v nat) { c := subVV(u[j:j+qhl], u[j:], qhatv) if c != 0 { c := addVV(u[j:j+n], u[j:], v) - u[j+n] += c + // If n == qhl, the carry from subVV and the carry from addVV + // cancel out and don't affect u[j+n]. + if n < qhl { + u[j+n] += c + } qhat-- } @@ -827,6 +834,10 @@ func (z nat) divRecursive(u, v nat) { putNat(tmp) } +// divRecursiveStep computes the division of u by v. +// - z must be large enough to hold the quotient +// - the quotient will overwrite z +// - the remainder will overwrite u func (z nat) divRecursiveStep(u, v nat, depth int, tmp *nat, temps []*nat) { u = u.norm() v = v.norm() diff --git a/libgo/go/math/big/nat_test.go b/libgo/go/math/big/nat_test.go index 32f29e38769..89e913fc16e 100644 --- a/libgo/go/math/big/nat_test.go +++ b/libgo/go/math/big/nat_test.go @@ -786,3 +786,21 @@ func TestNatDiv(t *testing.T) { } } } + +// TestIssue37499 triggers the edge case of divBasic where +// the inaccurate estimate of the first word's quotient +// happens at the very beginning of the loop. +func TestIssue37499(t *testing.T) { + // Choose u and v such that v is slightly larger than u >> N. + // This tricks divBasic into choosing 1 as the first word + // of the quotient. This works in both 32-bit and 64-bit settings. + u := natFromString("0x2b6c385a05be027f5c22005b63c42a1165b79ff510e1706b39f8489c1d28e57bb5ba4ef9fd9387a3e344402c0a453381") + v := natFromString("0x2b6c385a05be027f5c22005b63c42a1165b79ff510e1706c") + + q := nat(nil).make(8) + q.divBasic(u, v) + q = q.norm() + if s := string(q.utoa(16)); s != "fffffffffffffffffffffffffffffffffffffffffffffffb" { + t.Fatalf("incorrect quotient: %s", s) + } +} diff --git a/libgo/go/os/os_test.go b/libgo/go/os/os_test.go index a19b46d2964..8ec6de7c078 100644 --- a/libgo/go/os/os_test.go +++ b/libgo/go/os/os_test.go @@ -2450,3 +2450,38 @@ func TestDirSeek(t *testing.T) { } } } + +// Test that opening a file does not change its permissions. Issue 38225. +func TestOpenFileKeepsPermissions(t *testing.T) { + t.Parallel() + dir, err := ioutil.TempDir("", "TestOpenFileKeepsPermissions") + if err != nil { + t.Fatal(err) + } + defer RemoveAll(dir) + name := filepath.Join(dir, "x") + f, err := Create(name) + if err != nil { + t.Fatal(err) + } + if err := f.Close(); err != nil { + t.Error(err) + } + f, err = OpenFile(name, O_WRONLY|O_CREATE|O_TRUNC, 0) + if err != nil { + t.Fatal(err) + } + if fi, err := f.Stat(); err != nil { + t.Error(err) + } else if fi.Mode()&0222 == 0 { + t.Errorf("f.Stat.Mode after OpenFile is %v, should be writable", fi.Mode()) + } + if err := f.Close(); err != nil { + t.Error(err) + } + if fi, err := Stat(name); err != nil { + t.Error(err) + } else if fi.Mode()&0222 == 0 { + t.Errorf("Stat after OpenFile is %v, should be writable", fi.Mode()) + } +} diff --git a/libgo/go/runtime/crash_test.go b/libgo/go/runtime/crash_test.go index 6268f2ed726..aa97cf7d568 100644 --- a/libgo/go/runtime/crash_test.go +++ b/libgo/go/runtime/crash_test.go @@ -55,6 +55,16 @@ func runTestProg(t *testing.T, binary, name string, env ...string) string { t.Fatal(err) } + return runBuiltTestProg(t, exe, name, env...) +} + +func runBuiltTestProg(t *testing.T, exe, name string, env ...string) string { + if *flagQuick { + t.Skip("-quick") + } + + testenv.MustHaveGoBuild(t) + cmd := testenv.CleanCmdEnv(exec.Command(exe, name)) cmd.Env = append(cmd.Env, env...) if testing.Short() { @@ -64,7 +74,7 @@ func runTestProg(t *testing.T, binary, name string, env ...string) string { cmd.Stdout = &b cmd.Stderr = &b if err := cmd.Start(); err != nil { - t.Fatalf("starting %s %s: %v", binary, name, err) + t.Fatalf("starting %s %s: %v", exe, name, err) } // If the process doesn't complete within 1 minute, @@ -92,7 +102,7 @@ func runTestProg(t *testing.T, binary, name string, env ...string) string { }() if err := cmd.Wait(); err != nil { - t.Logf("%s %s exit status: %v", binary, name, err) + t.Logf("%s %s exit status: %v", exe, name, err) } close(done) diff --git a/libgo/go/runtime/mgcscavenge.go b/libgo/go/runtime/mgcscavenge.go index 3b60b3d51fd..d4b527c0713 100644 --- a/libgo/go/runtime/mgcscavenge.go +++ b/libgo/go/runtime/mgcscavenge.go @@ -288,6 +288,28 @@ func bgscavenge(c chan int) { continue } + if released < physPageSize { + // If this happens, it means that we may have attempted to release part + // of a physical page, but the likely effect of that is that it released + // the whole physical page, some of which may have still been in-use. + // This could lead to memory corruption. Throw. + throw("released less than one physical page of memory") + } + + // On some platforms we may see crit as zero if the time it takes to scavenge + // memory is less than the minimum granularity of its clock (e.g. Windows). + // In this case, just assume scavenging takes 10 µs per regular physical page + // (determined empirically), and conservatively ignore the impact of huge pages + // on timing. + // + // We shouldn't ever see a crit value less than zero unless there's a bug of + // some kind, either on our side or in the platform we're running on, but be + // defensive in that case as well. + const approxCritNSPerPhysicalPage = 10e3 + if crit <= 0 { + crit = approxCritNSPerPhysicalPage * float64(released/physPageSize) + } + // Multiply the critical time by 1 + the ratio of the costs of using // scavenged memory vs. scavenging memory. This forces us to pay down // the cost of reusing this memory eagerly by sleeping for a longer period diff --git a/libgo/go/runtime/mpagecache.go b/libgo/go/runtime/mpagecache.go index 9fc338bd8e4..a0749618405 100644 --- a/libgo/go/runtime/mpagecache.go +++ b/libgo/go/runtime/mpagecache.go @@ -148,9 +148,14 @@ func (s *pageAlloc) allocToCache() pageCache { // Update as an allocation, but note that it's not contiguous. s.update(c.base, pageCachePages, false, true) - // We're always searching for the first free page, and we always know the - // up to pageCache size bits will be allocated, so we can always move the - // searchAddr past the cache. - s.searchAddr = c.base + pageSize*pageCachePages + // Set the search address to the last page represented by the cache. + // Since all of the pages in this block are going to the cache, and we + // searched for the first free page, we can confidently start at the + // next page. + // + // However, s.searchAddr is not allowed to point into unmapped heap memory + // unless it is maxSearchAddr, so make it the last page as opposed to + // the page after. + s.searchAddr = c.base + pageSize*(pageCachePages-1) return c } diff --git a/libgo/go/runtime/mpagecache_test.go b/libgo/go/runtime/mpagecache_test.go index b8cc0bd965f..2ed0c0aa6a0 100644 --- a/libgo/go/runtime/mpagecache_test.go +++ b/libgo/go/runtime/mpagecache_test.go @@ -260,12 +260,13 @@ func TestPageAllocAllocToCache(t *testing.T) { if GOOS == "openbsd" && testing.Short() { t.Skip("skipping because virtual memory is limited; see #36210") } - tests := map[string]struct { + type test struct { before map[ChunkIdx][]BitRange scav map[ChunkIdx][]BitRange hits []PageCache // expected base addresses and patterns after map[ChunkIdx][]BitRange - }{ + } + tests := map[string]test{ "AllFree": { before: map[ChunkIdx][]BitRange{ BaseChunkIdx: {}, @@ -349,6 +350,34 @@ func TestPageAllocAllocToCache(t *testing.T) { }, }, } + if PageAlloc64Bit != 0 { + const chunkIdxBigJump = 0x100000 // chunk index offset which translates to O(TiB) + + // This test is similar to the one with the same name for + // pageAlloc.alloc and serves the same purpose. + // See mpagealloc_test.go for details. + sumsPerPhysPage := ChunkIdx(PhysPageSize / PallocSumBytes) + baseChunkIdx := BaseChunkIdx &^ (sumsPerPhysPage - 1) + tests["DiscontiguousMappedSumBoundary"] = test{ + before: map[ChunkIdx][]BitRange{ + baseChunkIdx + sumsPerPhysPage - 1: {{0, PallocChunkPages - 1}}, + baseChunkIdx + chunkIdxBigJump: {{1, PallocChunkPages - 1}}, + }, + scav: map[ChunkIdx][]BitRange{ + baseChunkIdx + sumsPerPhysPage - 1: {}, + baseChunkIdx + chunkIdxBigJump: {}, + }, + hits: []PageCache{ + NewPageCache(PageBase(baseChunkIdx+sumsPerPhysPage-1, PallocChunkPages-64), 1<<63, 0), + NewPageCache(PageBase(baseChunkIdx+chunkIdxBigJump, 0), 1, 0), + NewPageCache(0, 0, 0), + }, + after: map[ChunkIdx][]BitRange{ + baseChunkIdx + sumsPerPhysPage - 1: {{0, PallocChunkPages}}, + baseChunkIdx + chunkIdxBigJump: {{0, PallocChunkPages}}, + }, + } + } for name, v := range tests { v := v t.Run(name, func(t *testing.T) { diff --git a/libgo/go/runtime/proc.go b/libgo/go/runtime/proc.go index f75cacf3f43..e0981377512 100644 --- a/libgo/go/runtime/proc.go +++ b/libgo/go/runtime/proc.go @@ -1704,10 +1704,16 @@ func startTemplateThread() { if GOARCH == "wasm" { // no threads on wasm yet return } + + // Disable preemption to guarantee that the template thread will be + // created before a park once haveTemplateThread is set. + mp := acquirem() if !atomic.Cas(&newmHandoff.haveTemplateThread, 0, 1) { + releasem(mp) return } newm(templateThread, nil) + releasem(mp) } // templateThread is a thread in a known-good state that exists solely diff --git a/libgo/go/runtime/proc_test.go b/libgo/go/runtime/proc_test.go index a693937aa1e..5f96d648d98 100644 --- a/libgo/go/runtime/proc_test.go +++ b/libgo/go/runtime/proc_test.go @@ -6,6 +6,7 @@ package runtime_test import ( "fmt" + "internal/testenv" "math" "net" "runtime" @@ -930,6 +931,29 @@ func TestLockOSThreadAvoidsStatePropagation(t *testing.T) { } } +func TestLockOSThreadTemplateThreadRace(t *testing.T) { + testenv.MustHaveGoRun(t) + + exe, err := buildTestProg(t, "testprog") + if err != nil { + t.Fatal(err) + } + + iterations := 100 + if testing.Short() { + // Reduce run time to ~100ms, with much lower probability of + // catching issues. + iterations = 5 + } + for i := 0; i < iterations; i++ { + want := "OK\n" + output := runBuiltTestProg(t, exe, "LockOSThreadTemplateThreadRace") + if output != want { + t.Fatalf("run %d: want %q, got %q", i, want, output) + } + } +} + // fakeSyscall emulates a system call. //go:nosplit func fakeSyscall(duration time.Duration) { diff --git a/libgo/go/runtime/testdata/testprog/lockosthread.go b/libgo/go/runtime/testdata/testprog/lockosthread.go index fd3123e6474..098cc4dd722 100644 --- a/libgo/go/runtime/testdata/testprog/lockosthread.go +++ b/libgo/go/runtime/testdata/testprog/lockosthread.go @@ -7,6 +7,7 @@ package main import ( "os" "runtime" + "sync" "time" ) @@ -30,6 +31,7 @@ func init() { runtime.LockOSThread() }) register("LockOSThreadAvoidsStatePropagation", LockOSThreadAvoidsStatePropagation) + register("LockOSThreadTemplateThreadRace", LockOSThreadTemplateThreadRace) } func LockOSThreadMain() { @@ -195,3 +197,50 @@ func LockOSThreadAvoidsStatePropagation() { runtime.UnlockOSThread() println("OK") } + +func LockOSThreadTemplateThreadRace() { + // This test attempts to reproduce the race described in + // golang.org/issue/38931. To do so, we must have a stop-the-world + // (achieved via ReadMemStats) racing with two LockOSThread calls. + // + // While this test attempts to line up the timing, it is only expected + // to fail (and thus hang) around 2% of the time if the race is + // present. + + // Ensure enough Ps to actually run everything in parallel. Though on + // <4 core machines, we are still at the whim of the kernel scheduler. + runtime.GOMAXPROCS(4) + + go func() { + // Stop the world; race with LockOSThread below. + var m runtime.MemStats + for { + runtime.ReadMemStats(&m) + } + }() + + // Try to synchronize both LockOSThreads. + start := time.Now().Add(10*time.Millisecond) + + var wg sync.WaitGroup + wg.Add(2) + + for i := 0; i < 2; i++ { + go func() { + for time.Now().Before(start) { + } + + // Add work to the local runq to trigger early startm + // in handoffp. + go func(){}() + + runtime.LockOSThread() + runtime.Gosched() // add a preemption point. + wg.Done() + }() + } + + wg.Wait() + // If both LockOSThreads completed then we did not hit the race. + println("OK") +} diff --git a/libgo/misc/cgo/test/testx.go b/libgo/misc/cgo/test/testx.go index 42979b5f4d5..eb9d7fa47d3 100644 --- a/libgo/misc/cgo/test/testx.go +++ b/libgo/misc/cgo/test/testx.go @@ -124,6 +124,11 @@ typedef struct { } Issue31891B; void callIssue31891(void); + +typedef struct { + int i; +} Issue38408, *PIssue38408; + */ import "C" @@ -552,3 +557,8 @@ func useIssue31891B(c *C.Issue31891B) {} func test31891(t *testing.T) { C.callIssue31891() } + +// issue 38408 +// A typedef pointer can be used as the element type. +// No runtime test; just make sure it compiles. +var _ C.PIssue38408 = &C.Issue38408{i: 1} -- 2.30.2