cmd/link: process data symbols with slices

First (and largest single) step to switching cmd/link from linked
lists of symbols to slices.

Sort sections independently and concurrently.
This reduces jujud link times on linux/amd64 by ~4%.

Updates #15374

Change-Id: I452bc8f33081039468636502fe3c1cc8d6ed9efa
Reviewed-on: https://go-review.googlesource.com/22205
Reviewed-by: Michael Hudson-Doyle <michael.hudson@canonical.com>

src/cmd/link/internal/ld/data.go

@@ -32,7 +32,6 @@
 package ld
 
 import (
-	"bytes"
 	"cmd/internal/gcprog"
 	"cmd/internal/obj"
 	"cmd/internal/sys"
@@ -42,6 +41,7 @@ import (
 	"sort"
 	"strconv"
 	"strings"
+	"sync"
 )
 
 func Symgrow(ctxt *Link, s *LSym, siz int64) {
@@ -651,8 +651,8 @@ func reloc() {
 	for s := Ctxt.Textp; s != nil; s = s.Next {
 		relocsym(s)
 	}
-	for s := datap; s != nil; s = s.Next {
+	for _, sym := range datap {
-		relocsym(s)
+		relocsym(sym)
 	}
 	for s := dwarfp; s != nil; s = s.Next {
 		relocsym(s)
@@ -713,7 +713,7 @@ func dynrelocsym(s *LSym) {
 	}
 }
 
-func dynreloc() {
+func dynreloc(data *[obj.SXREF][]*LSym) {
 	// -d suppresses dynamic loader format, so we may as well not
 	// compute these sections or mark their symbols as reachable.
 	if Debug['d'] != 0 && HEADTYPE != obj.Hwindows {
@@ -727,8 +727,10 @@ func dynreloc() {
 	for s := Ctxt.Textp; s != nil; s = s.Next {
 		dynrelocsym(s)
 	}
-	for s := datap; s != nil; s = s.Next {
+	for _, syms := range data {
-		dynrelocsym(s)
+		for _, sym := range syms {
+			dynrelocsym(sym)
+		}
 	}
 	if Iself {
 		elfdynhash()
@@ -849,27 +851,77 @@ func Codeblk(addr int64, size int64) {
 	Bso.Flush()
 }
 
+// blkSlice is a variant of blk that processes slices.
+// After text symbols are converted from a linked list to a slice,
+// delete blk and give this function its name.
+func blkSlice(syms []*LSym, addr, size int64) {
+	for i, s := range syms {
+		if s.Type&obj.SSUB == 0 && s.Value >= addr {
+			syms = syms[i:]
+			break
+		}
+	}
+
+	eaddr := addr + size
+	for _, s := range syms {
+		if s.Type&obj.SSUB != 0 {
+			continue
+		}
+		if s.Value >= eaddr {
+			break
+		}
+		Ctxt.Cursym = s
+		if s.Value < addr {
+			Diag("phase error: addr=%#x but sym=%#x type=%d", addr, s.Value, s.Type)
+			errorexit()
+		}
+		if addr < s.Value {
+			strnput("", int(s.Value-addr))
+			addr = s.Value
+		}
+		Cwrite(s.P)
+		addr += int64(len(s.P))
+		if addr < s.Value+s.Size {
+			strnput("", int(s.Value+s.Size-addr))
+			addr = s.Value + s.Size
+		}
+		if addr != s.Value+s.Size {
+			Diag("phase error: addr=%#x value+size=%#x", addr, s.Value+s.Size)
+			errorexit()
+		}
+		if s.Value+s.Size >= eaddr {
+			break
+		}
+	}
+
+	if addr < eaddr {
+		strnput("", int(eaddr-addr))
+	}
+	Cflush()
+}
+
 func Datblk(addr int64, size int64) {
 	if Debug['a'] != 0 {
 		fmt.Fprintf(Bso, "datblk [%#x,%#x) at offset %#x\n", addr, addr+size, Cpos())
 	}
 
-	blk(datap, addr, size)
+	blkSlice(datap, addr, size)
 
 	/* again for printing */
 	if Debug['a'] == 0 {
 		return
 	}
 
-	var sym *LSym
+	syms := datap
-	for sym = datap; sym != nil; sym = sym.Next {
+	for i, sym := range syms {
 		if sym.Value >= addr {
+			syms = syms[i:]
 			break
 		}
 	}
 
 	eaddr := addr + size
-	for ; sym != nil; sym = sym.Next {
+	for _, sym := range syms {
 		if sym.Value >= eaddr {
 			break
 		}
@@ -1080,18 +1132,15 @@ func aligndatsize(datsize int64, s *LSym) int64 {
 }
 
 // maxalign returns the maximum required alignment for
-// the list of symbols s; the list stops when s->type exceeds type.
+// the slice of symbols syms
-func maxalign(s *LSym, type_ int) int32 {
+func maxalign(syms []*LSym) int32 {
-	var align int32
+	var max int32
-
+	for _, sym := range syms {
-	max := int32(0)
+		align := symalign(sym)
-	for ; s != nil && int(s.Type) <= type_; s = s.Next {
-		align = symalign(s)
 		if max < align {
 			max = align
 		}
 	}
-
 	return max
 }
 
@@ -1156,41 +1205,24 @@ func (p *GCProg) AddSym(s *LSym) {
 	p.w.Append(prog[4:], nptr)
 }
 
+// dataSortKey is used to sort a slice of data symbol *LSym pointers.
+// The sort keys are kept inline to improve cache behaviour while sorting.
 type dataSortKey struct {
-	// keep sort keys inline to improve cache behaviour while sorting
+	size int64
-	Type int16
+	name string
-	Size int64
+	lsym *LSym
-	Name string
-
-	Lsym *LSym
 }
 
-type dataSlice []dataSortKey
+type bySizeAndName []dataSortKey
 
-func (d dataSlice) Len() int      { return len(d) }
+func (d bySizeAndName) Len() int      { return len(d) }
-func (d dataSlice) Swap(i, j int) { d[i], d[j] = d[j], d[i] }
+func (d bySizeAndName) Swap(i, j int) { d[i], d[j] = d[j], d[i] }
-func (d dataSlice) Less(i, j int) bool {
+func (d bySizeAndName) Less(i, j int) bool {
-	s1, s2 := &d[i], &d[j]
+	s1, s2 := d[i], d[j]
-	if s1.Type != s2.Type {
+	if s1.size != s2.size {
-		return s1.Type < s2.Type
+		return s1.size < s2.size
 	}
-
+	return s1.name < s2.name
-	// For ppc64, we want to interleave the .got and .toc sections
-	// from input files. Both are type SELFGOT, so in that case
-	// fall through to the name comparison (conveniently, .got
-	// sorts before .toc).
-	if s1.Type != obj.SELFGOT && s1.Size != s2.Size {
-		return s1.Size < s2.Size
-	}
-
-	// Sort typelinks by the string field.
-	if strings.HasPrefix(s1.Name, "go.typelink.") && strings.HasPrefix(s2.Name, "go.typelink.") {
-		s1n := decodetype_string(s1.Lsym.R[0].Sym)
-		s2n := decodetype_string(s2.Lsym.R[0].Sym)
-		return bytes.Compare(s1n, s2n) < 0
-	}
-
-	return s1.Name < s2.Name
 }
 
 func growdatsize(datsizep *int64, s *LSym) {
@@ -1207,38 +1239,17 @@ func growdatsize(datsizep *int64, s *LSym) {
 	*datsizep = datsize + s.Size
 }
 
-func list2Slice(head *LSym) dataSlice {
+func list2slice(s *LSym) []*LSym {
-	n := 0
+	var syms []*LSym
-	for s := datap; s != nil; s = s.Next {
+	for ; s != nil; s = s.Next {
-		n++
+		syms = append(syms, s)
 	}
-	slice := make(dataSlice, n)
+	return syms
-	i := 0
-	for s := datap; s != nil; s = s.Next {
-		k := &slice[i]
-		k.Type = s.Type
-		k.Size = s.Size
-		k.Name = s.Name
-		k.Lsym = s
-
-		i++
-	}
-	return slice
 }
 
-func slice2List(d dataSlice) *LSym {
+// datap is a collection of reachable data symbols in address order.
-	for i := 0; i < len(d)-1; i++ {
+// Generated by dodata.
-		d[i].Lsym.Next = d[i+1].Lsym
+var datap []*LSym
-	}
-	d[len(d)-1].Lsym.Next = nil
-	return d[0].Lsym
-}
-
-func dataSort(head *LSym) *LSym {
-	d := list2Slice(head)
-	sort.Sort(d)
-	return slice2List(d)
-}
 
 func dodata() {
 	if Debug['v'] != 0 {
@@ -1246,135 +1257,103 @@ func dodata() {
 	}
 	Bso.Flush()
 
-	var last *LSym
+	// Collect data symbols by type into data.
-	datap = nil
+	var data [obj.SXREF][]*LSym
-
 	for _, s := range Ctxt.Allsym {
 		if !s.Attr.Reachable() || s.Attr.Special() {
 			continue
 		}
-		if obj.STEXT < s.Type && s.Type < obj.SXREF {
+		if s.Type <= obj.STEXT || s.Type >= obj.SXREF {
-			if s.Attr.OnList() {
+			continue
-				log.Fatalf("symbol %s listed multiple times", s.Name)
-			}
-			s.Attr |= AttrOnList
-			if last == nil {
-				datap = s
-			} else {
-				last.Next = s
-			}
-			s.Next = nil
-			last = s
 		}
+		data[s.Type] = append(data[s.Type], s)
 	}
 
-	for s := datap; s != nil; s = s.Next {
+	// Now that we have the data symbols, but before we start
-		if int64(len(s.P)) > s.Size {
+	// to assign addresses, record all the necessary
-			Diag("%s: initialize bounds (%d < %d)", s.Name, s.Size, len(s.P))
+	// dynamic relocations. These will grow the relocation
-		}
+	// symbol, which is itself data.
-	}
+	//
-
+	// On darwin, we need the symbol table numbers for dynreloc.
-	/*
-	 * now that we have the datap list, but before we start
-	 * to assign addresses, record all the necessary
-	 * dynamic relocations.  these will grow the relocation
-	 * symbol, which is itself data.
-	 *
-	 * on darwin, we need the symbol table numbers for dynreloc.
-	 */
 	if HEADTYPE == obj.Hdarwin {
 		machosymorder()
 	}
-	dynreloc()
+	dynreloc(&data)
-
-	/* some symbols may no longer belong in datap (Mach-O) */
-	var l **LSym
-	var s *LSym
-	for l = &datap; ; {
-		s = *l
-		if s == nil {
-			break
-		}
-
-		if s.Type <= obj.STEXT || obj.SXREF <= s.Type {
-			*l = s.Next
-		} else {
-			l = &s.Next
-		}
-	}
-
-	*l = nil
 
 	if UseRelro() {
 		// "read only" data with relocations needs to go in its own section
 		// when building a shared library. We do this by boosting objects of
 		// type SXXX with relocations to type SXXXRELRO.
-		for s := datap; s != nil; s = s.Next {
+		for symnro := int16(obj.STYPE); symnro < obj.STYPERELRO; symnro++ {
-			if (s.Type >= obj.STYPE && s.Type <= obj.SFUNCTAB && len(s.R) > 0) || s.Type == obj.STYPE || s.Type == obj.SGOSTRINGHDR {
+			symnrelro := symnro + obj.STYPERELRO - obj.STYPE
-				s.Type += (obj.STYPERELRO - obj.STYPE)
+
+			ro := []*LSym{}
+			relro := data[symnrelro]
+
+			for _, s := range data[symnro] {
+				isRelro := len(s.R) > 0
+				switch s.Type {
+				case obj.STYPE, obj.SGOSTRINGHDR, obj.STYPERELRO, obj.SGOSTRINGHDRRELRO:
+					// Symbols are not sorted yet, so it is possible
+					// that an Outer symbol has been changed to a
+					// relro Type before it reaches here.
+					isRelro = true
+				}
+				if isRelro {
+					s.Type = symnrelro
 					if s.Outer != nil {
 						s.Outer.Type = s.Type
 					}
+					relro = append(relro, s)
+				} else {
+					ro = append(ro, s)
 				}
 			}
+
 			// Check that we haven't made two symbols with the same .Outer into
 			// different types (because references two symbols with non-nil Outer
 			// become references to the outer symbol + offset it's vital that the
 			// symbol and the outer end up in the same section).
-		for s := datap; s != nil; s = s.Next {
+			for _, s := range relro {
 				if s.Outer != nil && s.Outer.Type != s.Type {
 					Diag("inconsistent types for %s and its Outer %s (%d != %d)",
 						s.Name, s.Outer.Name, s.Type, s.Outer.Type)
 				}
 			}
 
-	}
+			data[symnro] = ro
-
+			data[symnrelro] = relro
-	datap = dataSort(datap)
-
-	if Iself {
-		// Make .rela and .rela.plt contiguous, the ELF ABI requires this
-		// and Solaris actually cares.
-		var relplt *LSym
-		for l = &datap; *l != nil; l = &(*l).Next {
-			if (*l).Name == ".rel.plt" || (*l).Name == ".rela.plt" {
-				relplt = (*l)
-				*l = (*l).Next
-				break
-			}
-		}
-		if relplt != nil {
-			for s = datap; s != nil; s = s.Next {
-				if s.Name == ".rel" || s.Name == ".rela" {
-					relplt.Next = s.Next
-					s.Next = relplt
-				}
-			}
 		}
 	}
 
-	/*
+	// Sort symbols.
-	 * allocate sections.  list is sorted by type,
+	var wg sync.WaitGroup
-	 * so we can just walk it for each piece we want to emit.
+	for symn := range data {
-	 * segdata is processed before segtext, because we need
+		symn := symn
-	 * to see all symbols in the .data and .bss sections in order
+		wg.Add(1)
-	 * to generate garbage collection information.
+		go func() {
-	 */
+			data[symn] = dodataSect(symn, data[symn])
-
+			wg.Done()
-	/* begin segdata */
+		}()
-
-	/* skip symbols belonging to segtext */
-	s = datap
-
-	for ; s != nil && s.Type < obj.SELFSECT; s = s.Next {
 	}
+	wg.Wait()
 
-	/* writable ELF sections */
+	// Allocate sections.
+	// Data is processed before segtext, because we need
+	// to see all symbols in the .data and .bss sections in order
+	// to generate garbage collection information.
 	datsize := int64(0)
 
-	var sect *Section
+	// Writable sections.
-	for ; s != nil && s.Type < obj.SELFGOT; s = s.Next {
+	writableSects := []int{
-		sect = addsection(&Segdata, s.Name, 06)
+		obj.SELFSECT,
+		obj.SMACHO,
+		obj.SMACHOGOT,
+		obj.SWINDOWS,
+	}
+	for _, symn := range writableSects {
+		for _, s := range data[symn] {
+			sect := addsection(&Segdata, s.Name, 06)
 			sect.Align = symalign(s)
 			datsize = Rnd(datsize, int64(sect.Align))
 			sect.Vaddr = uint64(datsize)
@@ -1384,15 +1363,16 @@ func dodata() {
 			growdatsize(&datsize, s)
 			sect.Length = uint64(datsize) - sect.Vaddr
 		}
+	}
 
-	/* .got (and .toc on ppc64) */
+	// .got (and .toc on ppc64)
-	if s.Type == obj.SELFGOT {
+	if len(data[obj.SELFGOT]) > 0 {
 		sect := addsection(&Segdata, ".got", 06)
-		sect.Align = maxalign(s, obj.SELFGOT)
+		sect.Align = maxalign(data[obj.SELFGOT])
 		datsize = Rnd(datsize, int64(sect.Align))
 		sect.Vaddr = uint64(datsize)
 		var toc *LSym
-		for ; s != nil && s.Type == obj.SELFGOT; s = s.Next {
+		for _, s := range data[obj.SELFGOT] {
 			datsize = aligndatsize(datsize, s)
 			s.Sect = sect
 			s.Type = obj.SDATA
@@ -1400,7 +1380,6 @@ func dodata() {
 
 			// Resolve .TOC. symbol for this object file (ppc64)
 			toc = Linkrlookup(Ctxt, ".TOC.", int(s.Version))
-
 			if toc != nil {
 				toc.Sect = sect
 				toc.Outer = s
@@ -1412,26 +1391,24 @@ func dodata() {
 
 			growdatsize(&datsize, s)
 		}
-
 		sect.Length = uint64(datsize) - sect.Vaddr
 	}
 
 	/* pointer-free data */
-	sect = addsection(&Segdata, ".noptrdata", 06)
+	sect := addsection(&Segdata, ".noptrdata", 06)
 
-	sect.Align = maxalign(s, obj.SINITARR-1)
+	sect.Align = maxalign(data[obj.SNOPTRDATA])
 	datsize = Rnd(datsize, int64(sect.Align))
 	sect.Vaddr = uint64(datsize)
 	Linklookup(Ctxt, "runtime.noptrdata", 0).Sect = sect
 	Linklookup(Ctxt, "runtime.enoptrdata", 0).Sect = sect
-	for ; s != nil && s.Type < obj.SINITARR; s = s.Next {
+	for _, s := range data[obj.SNOPTRDATA] {
 		datsize = aligndatsize(datsize, s)
 		s.Sect = sect
 		s.Type = obj.SDATA
 		s.Value = int64(uint64(datsize) - sect.Vaddr)
 		growdatsize(&datsize, s)
 	}
-
 	sect.Length = uint64(datsize) - sect.Vaddr
 
 	hasinitarr := Linkshared
@@ -1441,37 +1418,30 @@ func dodata() {
 	case BuildmodeCArchive, BuildmodeCShared, BuildmodeShared:
 		hasinitarr = true
 	}
-
 	if hasinitarr {
 		sect := addsection(&Segdata, ".init_array", 06)
-		sect.Align = maxalign(s, obj.SINITARR)
+		sect.Align = maxalign(data[obj.SINITARR])
 		datsize = Rnd(datsize, int64(sect.Align))
 		sect.Vaddr = uint64(datsize)
-		for ; s != nil && s.Type == obj.SINITARR; s = s.Next {
+		for _, s := range data[obj.SINITARR] {
 			datsize = aligndatsize(datsize, s)
 			s.Sect = sect
 			s.Value = int64(uint64(datsize) - sect.Vaddr)
 			growdatsize(&datsize, s)
 		}
-
 		sect.Length = uint64(datsize) - sect.Vaddr
 	}
 
 	/* data */
 	sect = addsection(&Segdata, ".data", 06)
-	sect.Align = maxalign(s, obj.SBSS-1)
+	sect.Align = maxalign(data[obj.SDATA])
 	datsize = Rnd(datsize, int64(sect.Align))
 	sect.Vaddr = uint64(datsize)
 	Linklookup(Ctxt, "runtime.data", 0).Sect = sect
 	Linklookup(Ctxt, "runtime.edata", 0).Sect = sect
 	var gc GCProg
 	gc.Init("runtime.gcdata")
-	for ; s != nil && s.Type < obj.SBSS; s = s.Next {
+	for _, s := range data[obj.SDATA] {
-		if s.Type == obj.SINITARR {
-			Ctxt.Cursym = s
-			Diag("unexpected symbol type %d", s.Type)
-		}
-
 		s.Sect = sect
 		s.Type = obj.SDATA
 		datsize = aligndatsize(datsize, s)
@@ -1484,14 +1454,14 @@ func dodata() {
 
 	/* bss */
 	sect = addsection(&Segdata, ".bss", 06)
-	sect.Align = maxalign(s, obj.SNOPTRBSS-1)
+	sect.Align = maxalign(data[obj.SBSS])
 	datsize = Rnd(datsize, int64(sect.Align))
 	sect.Vaddr = uint64(datsize)
 	Linklookup(Ctxt, "runtime.bss", 0).Sect = sect
 	Linklookup(Ctxt, "runtime.ebss", 0).Sect = sect
 	gc = GCProg{}
 	gc.Init("runtime.gcbss")
-	for ; s != nil && s.Type < obj.SNOPTRBSS; s = s.Next {
+	for _, s := range data[obj.SBSS] {
 		s.Sect = sect
 		datsize = aligndatsize(datsize, s)
 		s.Value = int64(uint64(datsize) - sect.Vaddr)
@@ -1504,12 +1474,12 @@ func dodata() {
 	/* pointer-free bss */
 	sect = addsection(&Segdata, ".noptrbss", 06)
 
-	sect.Align = maxalign(s, obj.SNOPTRBSS)
+	sect.Align = maxalign(data[obj.SNOPTRBSS])
 	datsize = Rnd(datsize, int64(sect.Align))
 	sect.Vaddr = uint64(datsize)
 	Linklookup(Ctxt, "runtime.noptrbss", 0).Sect = sect
 	Linklookup(Ctxt, "runtime.enoptrbss", 0).Sect = sect
-	for ; s != nil && s.Type == obj.SNOPTRBSS; s = s.Next {
+	for _, s := range data[obj.SNOPTRBSS] {
 		datsize = aligndatsize(datsize, s)
 		s.Sect = sect
 		s.Value = int64(uint64(datsize) - sect.Vaddr)
@@ -1519,22 +1489,21 @@ func dodata() {
 	sect.Length = uint64(datsize) - sect.Vaddr
 	Linklookup(Ctxt, "runtime.end", 0).Sect = sect
 
-	// 6g uses 4-byte relocation offsets, so the entire segment must fit in 32 bits.
+	// The compiler uses 4-byte relocation offsets, so the entire segment must fit in 32 bits.
 	if datsize != int64(uint32(datsize)) {
 		Diag("data or bss segment too large")
 	}
 
-	if s != nil && s.Type == obj.STLSBSS {
+	if len(data[obj.STLSBSS]) > 0 {
+		var sect *Section
 		if Iself && (Linkmode == LinkExternal || Debug['d'] == 0) && HEADTYPE != obj.Hopenbsd {
 			sect = addsection(&Segdata, ".tbss", 06)
 			sect.Align = int32(SysArch.PtrSize)
 			sect.Vaddr = 0
-		} else {
-			sect = nil
 		}
 		datsize = 0
 
-		for ; s != nil && s.Type == obj.STLSBSS; s = s.Next {
+		for _, s := range data[obj.STLSBSS] {
 			datsize = aligndatsize(datsize, s)
 			s.Sect = sect
 			s.Value = datsize
@@ -1546,11 +1515,6 @@ func dodata() {
 		}
 	}
 
-	if s != nil {
-		Ctxt.Cursym = nil
-		Diag("unexpected symbol type %d for %s", s.Type, s.Name)
-	}
-
 	/*
 	 * We finished data, begin read-only data.
 	 * Not all systems support a separate read-only non-executable data section.
@@ -1568,13 +1532,14 @@ func dodata() {
 		segro = &Segtext
 	}
 
-	s = datap
-
 	datsize = 0
 
 	/* read-only executable ELF, Mach-O sections */
-	for ; s != nil && s.Type < obj.STYPE; s = s.Next {
+	if len(data[obj.STEXT]) != 0 {
-		sect = addsection(&Segtext, s.Name, 04)
+		Diag("dodata found an STEXT symbol: %s", data[obj.STEXT][0].Name)
+	}
+	for _, s := range data[obj.SELFRXSECT] {
+		sect := addsection(&Segtext, s.Name, 04)
 		sect.Align = symalign(s)
 		datsize = Rnd(datsize, int64(sect.Align))
 		sect.Vaddr = uint64(datsize)
@@ -1588,8 +1553,6 @@ func dodata() {
 	/* read-only data */
 	sect = addsection(segro, ".rodata", 04)
 
-	sect.Align = maxalign(s, obj.STYPERELRO-1)
-	datsize = Rnd(datsize, int64(sect.Align))
 	sect.Vaddr = 0
 	Linklookup(Ctxt, "runtime.rodata", 0).Sect = sect
 	Linklookup(Ctxt, "runtime.erodata", 0).Sect = sect
@@ -1597,14 +1560,32 @@ func dodata() {
 		Linklookup(Ctxt, "runtime.types", 0).Sect = sect
 		Linklookup(Ctxt, "runtime.etypes", 0).Sect = sect
 	}
-	for ; s != nil && s.Type < obj.STYPERELRO; s = s.Next {
+	roSects := []int{
+		obj.STYPE,
+		obj.SSTRING,
+		obj.SGOSTRING,
+		obj.SGOSTRINGHDR,
+		obj.SGOFUNC,
+		obj.SGCBITS,
+		obj.SRODATA,
+		obj.SFUNCTAB,
+	}
+	for _, symn := range roSects {
+		align := maxalign(data[symn])
+		if sect.Align < align {
+			sect.Align = align
+		}
+	}
+	datsize = Rnd(datsize, int64(sect.Align))
+	for _, symn := range roSects {
+		for _, s := range data[symn] {
 			datsize = aligndatsize(datsize, s)
 			s.Sect = sect
 			s.Type = obj.SRODATA
 			s.Value = int64(uint64(datsize) - sect.Vaddr)
 			growdatsize(&datsize, s)
 		}
-
+	}
 	sect.Length = uint64(datsize) - sect.Vaddr
 
 	// There is some data that are conceptually read-only but are written to by
@@ -1626,12 +1607,28 @@ func dodata() {
 		/* data only written by relocations */
 		sect = addsection(segro, ".data.rel.ro", 06)
 
-		sect.Align = maxalign(s, obj.STYPELINK-1)
-		datsize = Rnd(datsize, int64(sect.Align))
 		sect.Vaddr = 0
 		Linklookup(Ctxt, "runtime.types", 0).Sect = sect
 		Linklookup(Ctxt, "runtime.etypes", 0).Sect = sect
-		for ; s != nil && s.Type < obj.STYPELINK; s = s.Next {
+		relroSects := []int{
+			obj.STYPERELRO,
+			obj.SSTRINGRELRO,
+			obj.SGOSTRINGRELRO,
+			obj.SGOSTRINGHDRRELRO,
+			obj.SGOFUNCRELRO,
+			obj.SGCBITSRELRO,
+			obj.SRODATARELRO,
+			obj.SFUNCTABRELRO,
+		}
+		for _, symn := range relroSects {
+			align := maxalign(data[symn])
+			if sect.Align < align {
+				sect.Align = align
+			}
+		}
+		datsize = Rnd(datsize, int64(sect.Align))
+		for _, symn := range relroSects {
+			for _, s := range data[symn] {
 				datsize = aligndatsize(datsize, s)
 				if s.Outer != nil && s.Outer.Sect != nil && s.Outer.Sect != sect {
 					Diag("s.Outer (%s) in different section from s (%s)", s.Outer.Name, s.Name)
@@ -1641,6 +1638,7 @@ func dodata() {
 				s.Value = int64(uint64(datsize) - sect.Vaddr)
 				growdatsize(&datsize, s)
 			}
+		}
 
 		sect.Length = uint64(datsize) - sect.Vaddr
 
@@ -1648,78 +1646,85 @@ func dodata() {
 
 	/* typelink */
 	sect = addsection(segro, relro_prefix+".typelink", relro_perms)
-
+	sect.Align = maxalign(data[obj.STYPELINK])
-	sect.Align = maxalign(s, obj.STYPELINK)
 	datsize = Rnd(datsize, int64(sect.Align))
 	sect.Vaddr = uint64(datsize)
 	Linklookup(Ctxt, "runtime.typelink", 0).Sect = sect
 	Linklookup(Ctxt, "runtime.etypelink", 0).Sect = sect
-	for ; s != nil && s.Type == obj.STYPELINK; s = s.Next {
+	for _, s := range data[obj.STYPELINK] {
 		datsize = aligndatsize(datsize, s)
 		s.Sect = sect
 		s.Type = obj.SRODATA
 		s.Value = int64(uint64(datsize) - sect.Vaddr)
 		growdatsize(&datsize, s)
 	}
-
 	sect.Length = uint64(datsize) - sect.Vaddr
 
 	/* itablink */
 	sect = addsection(segro, relro_prefix+".itablink", relro_perms)
 
-	sect.Align = maxalign(s, obj.SITABLINK)
+	sect.Align = maxalign(data[obj.SITABLINK])
 	datsize = Rnd(datsize, int64(sect.Align))
 	sect.Vaddr = uint64(datsize)
 	Linklookup(Ctxt, "runtime.itablink", 0).Sect = sect
 	Linklookup(Ctxt, "runtime.eitablink", 0).Sect = sect
-	for ; s != nil && s.Type == obj.SITABLINK; s = s.Next {
+	for _, s := range data[obj.SITABLINK] {
 		datsize = aligndatsize(datsize, s)
 		s.Sect = sect
 		s.Type = obj.SRODATA
 		s.Value = int64(uint64(datsize) - sect.Vaddr)
 		growdatsize(&datsize, s)
 	}
-
 	sect.Length = uint64(datsize) - sect.Vaddr
 
 	/* gosymtab */
 	sect = addsection(segro, relro_prefix+".gosymtab", relro_perms)
 
-	sect.Align = maxalign(s, obj.SPCLNTAB-1)
+	sect.Align = maxalign(data[obj.SSYMTAB])
 	datsize = Rnd(datsize, int64(sect.Align))
 	sect.Vaddr = uint64(datsize)
 	Linklookup(Ctxt, "runtime.symtab", 0).Sect = sect
 	Linklookup(Ctxt, "runtime.esymtab", 0).Sect = sect
-	for ; s != nil && s.Type < obj.SPCLNTAB; s = s.Next {
+	for _, s := range data[obj.SSYMTAB] {
 		datsize = aligndatsize(datsize, s)
 		s.Sect = sect
 		s.Type = obj.SRODATA
 		s.Value = int64(uint64(datsize) - sect.Vaddr)
 		growdatsize(&datsize, s)
 	}
-
 	sect.Length = uint64(datsize) - sect.Vaddr
 
 	/* gopclntab */
 	sect = addsection(segro, relro_prefix+".gopclntab", relro_perms)
 
-	sect.Align = maxalign(s, obj.SELFROSECT-1)
+	sect.Align = maxalign(data[obj.SPCLNTAB])
 	datsize = Rnd(datsize, int64(sect.Align))
 	sect.Vaddr = uint64(datsize)
 	Linklookup(Ctxt, "runtime.pclntab", 0).Sect = sect
 	Linklookup(Ctxt, "runtime.epclntab", 0).Sect = sect
-	for ; s != nil && s.Type < obj.SELFROSECT; s = s.Next {
+	for _, s := range data[obj.SPCLNTAB] {
 		datsize = aligndatsize(datsize, s)
 		s.Sect = sect
 		s.Type = obj.SRODATA
 		s.Value = int64(uint64(datsize) - sect.Vaddr)
 		growdatsize(&datsize, s)
 	}
-
 	sect.Length = uint64(datsize) - sect.Vaddr
 
 	/* read-only ELF, Mach-O sections */
-	for ; s != nil && s.Type < obj.SELFSECT; s = s.Next {
+	for _, s := range data[obj.SELFROSECT] {
+		sect = addsection(segro, s.Name, 04)
+		sect.Align = symalign(s)
+		datsize = Rnd(datsize, int64(sect.Align))
+		sect.Vaddr = uint64(datsize)
+		s.Sect = sect
+		s.Type = obj.SRODATA
+		s.Value = int64(uint64(datsize) - sect.Vaddr)
+		growdatsize(&datsize, s)
+		sect.Length = uint64(datsize) - sect.Vaddr
+	}
+
+	for _, s := range data[obj.SMACHOPLT] {
 		sect = addsection(segro, s.Name, 04)
 		sect.Align = symalign(s)
 		datsize = Rnd(datsize, int64(sect.Align))
@@ -1736,8 +1741,13 @@ func dodata() {
 		Diag("read-only data segment too large")
 	}
 
+	for symn := obj.SELFRXSECT; symn < obj.SXREF; symn++ {
+		datap = append(datap, data[symn]...)
+	}
+
 	dwarfgeneratedebugsyms()
 
+	var s *LSym
 	for s = dwarfp; s != nil && s.Type == obj.SDWARFSECT; s = s.Next {
 		sect = addsection(&Segdwarf, s.Name, 04)
 		sect.Align = 1
@@ -1791,6 +1801,84 @@ func dodata() {
 	}
 }
 
+func dodataSect(symn int, syms []*LSym) []*LSym {
+	if HEADTYPE == obj.Hdarwin {
+		// Some symbols may no longer belong in syms
+		// due to movement in machosymorder.
+		newSyms := make([]*LSym, 0, len(syms))
+		for _, s := range syms {
+			if int(s.Type) == symn {
+				newSyms = append(newSyms, s)
+			}
+		}
+		syms = newSyms
+	}
+
+	symsSort := make([]dataSortKey, len(syms))
+	for i, s := range syms {
+		if s.Attr.OnList() {
+			log.Fatalf("symbol %s listed multiple times", s.Name)
+		}
+		s.Attr |= AttrOnList
+		if int64(len(s.P)) > s.Size {
+			Diag("%s: initialize bounds (%d < %d)", s.Name, s.Size, len(s.P))
+		}
+
+		symsSort[i] = dataSortKey{
+			size: s.Size,
+			name: s.Name,
+			lsym: s,
+		}
+
+		switch s.Type {
+		case obj.SELFGOT:
+			// For ppc64, we want to interleave the .got and .toc sections
+			// from input files. Both are type SELFGOT, so in that case
+			// we skip size comparison and fall through to the name
+			// comparison (conveniently, .got sorts before .toc).
+			symsSort[i].size = 0
+		case obj.STYPELINK:
+			// Sort typelinks by the rtype.string field so the reflect
+			// package can binary search type links.
+			symsSort[i].name = string(decodetype_string(s.R[0].Sym))
+		}
+	}
+
+	sort.Sort(bySizeAndName(symsSort))
+
+	for i, symSort := range symsSort {
+		syms[i] = symSort.lsym
+	}
+
+	if Iself && symn == obj.SELFROSECT {
+		// Make .rela and .rela.plt contiguous, the ELF ABI requires this
+		// and Solaris actually cares.
+		reli, plti := -1, -1
+		for i, s := range syms {
+			switch s.Name {
+			case ".rel.plt", ".rela.plt":
+				plti = i
+			case ".rel", ".rela":
+				reli = i
+			}
+		}
+		if reli >= 0 && plti >= 0 && plti != reli+1 {
+			newSyms := make([]*LSym, 0, len(syms))
+			plt := syms[plti]
+			newSyms = append(newSyms, syms[:reli+1]...)
+			newSyms = append(newSyms, plt)
+			newSyms = append(newSyms, syms[reli+1:plti]...)
+			newSyms = append(newSyms, syms[plti+1:]...)
+			if len(newSyms) != len(syms) {
+				Diag("plt move failed: len %d/%d", len(newSyms), len(syms))
+			}
+			syms = newSyms
+		}
+	}
+
+	return syms
+}
+
 // Add buildid to beginning of text segment, on non-ELF systems.
 // Non-ELF binary formats are not always flexible enough to
 // give us a place to put the Go build ID. On those systems, we put it
@@ -1816,8 +1904,6 @@ func textbuildid() {
 
 // assign addresses to text
 func textaddress() {
-	var sub *LSym
-
 	addsection(&Segtext, ".text", 05)
 
 	// Assign PCs in text segment.
@@ -1844,7 +1930,7 @@ func textaddress() {
 			va = uint64(Rnd(int64(va), int64(Funcalign)))
 		}
 		sym.Value = 0
-		for sub = sym; sub != nil; sub = sub.Sub {
+		for sub := sym; sub != nil; sub = sub.Sub {
 			sub.Value += int64(va)
 		}
 		if sym.Size == 0 && sym.Sub != nil {
@@ -1982,22 +2068,22 @@ func address() {
 	symtab := itablink.Next
 	pclntab := symtab.Next
 
-	var sub *LSym
+	for _, s := range datap {
-	for sym := datap; sym != nil; sym = sym.Next {
+		Ctxt.Cursym = s
-		Ctxt.Cursym = sym
+		if s.Sect != nil {
-		if sym.Sect != nil {
+			s.Value += int64(s.Sect.Vaddr)
-			sym.Value += int64(sym.Sect.Vaddr)
 		}
-		for sub = sym.Sub; sub != nil; sub = sub.Sub {
+		for sub := s.Sub; sub != nil; sub = sub.Sub {
-			sub.Value += sym.Value
+			sub.Value += s.Value
 		}
 	}
+
 	for sym := dwarfp; sym != nil; sym = sym.Next {
 		Ctxt.Cursym = sym
 		if sym.Sect != nil {
 			sym.Value += int64(sym.Sect.Vaddr)
 		}
-		for sub = sym.Sub; sub != nil; sub = sub.Sub {
+		for sub := sym.Sub; sub != nil; sub = sub.Sub {
 			sub.Value += sym.Value
 		}
 	}

src/cmd/link/internal/ld/elf.go

@@ -1670,7 +1670,7 @@ func elfshreloc(sect *Section) *ElfShdr {
 	return sh
 }
 
-func elfrelocsect(sect *Section, first *LSym) {
+func elfrelocsect(sect *Section, syms []*LSym) {
 	// If main section is SHT_NOBITS, nothing to relocate.
 	// Also nothing to relocate in .shstrtab.
 	if sect.Vaddr >= sect.Seg.Vaddr+sect.Seg.Filelen {
@@ -1681,18 +1681,18 @@ func elfrelocsect(sect *Section, first *LSym) {
 	}
 
 	sect.Reloff = uint64(Cpos())
-	var sym *LSym
+	for i, s := range syms {
-	for sym = first; sym != nil; sym = sym.Next {
+		if !s.Attr.Reachable() {
-		if !sym.Attr.Reachable() {
 			continue
 		}
-		if uint64(sym.Value) >= sect.Vaddr {
+		if uint64(s.Value) >= sect.Vaddr {
+			syms = syms[i:]
 			break
 		}
 	}
 
 	eaddr := int32(sect.Vaddr + sect.Length)
-	for ; sym != nil; sym = sym.Next {
+	for _, sym := range syms {
 		if !sym.Attr.Reachable() {
 			continue
 		}
@@ -1710,7 +1710,6 @@ func elfrelocsect(sect *Section, first *LSym) {
 				Diag("missing xsym in relocation")
 				continue
 			}
-
 			if r.Xsym.ElfsymForReloc() == 0 {
 				Diag("reloc %d to non-elf symbol %s (outer=%s) %d", r.Type, r.Sym.Name, r.Xsym.Name, r.Sym.Type)
 			}
@@ -1728,7 +1727,7 @@ func Elfemitreloc() {
 		Cput(0)
 	}
 
-	elfrelocsect(Segtext.Sect, Ctxt.Textp)
+	elfrelocsect(Segtext.Sect, list2slice(Ctxt.Textp))
 	for sect := Segtext.Sect.Next; sect != nil; sect = sect.Next {
 		elfrelocsect(sect, datap)
 	}
@@ -1739,7 +1738,7 @@ func Elfemitreloc() {
 		elfrelocsect(sect, datap)
 	}
 	for sect := Segdwarf.Sect; sect != nil; sect = sect.Next {
-		elfrelocsect(sect, dwarfp)
+		elfrelocsect(sect, list2slice(dwarfp))
 	}
 }
 

src/cmd/link/internal/ld/lib.go

@@ -130,7 +130,6 @@ func (r *Rpath) String() string {
 
 var (
 	Thearch Arch
-	datap   *LSym
 	Debug   [128]int
 	Lcsize  int32
 	rpath   Rpath
@@ -2109,7 +2108,7 @@ func undef() {
 	for s := Ctxt.Textp; s != nil; s = s.Next {
 		undefsym(s)
 	}
-	for s := datap; s != nil; s = s.Next {
+	for _, s := range datap {
 		undefsym(s)
 	}
 	if nerrors > 0 {

src/cmd/link/internal/ld/macho.go

@@ -806,25 +806,25 @@ func Domacholink() int64 {
 	return Rnd(int64(size), int64(INITRND))
 }
 
-func machorelocsect(sect *Section, first *LSym) {
+func machorelocsect(sect *Section, syms []*LSym) {
 	// If main section has no bits, nothing to relocate.
 	if sect.Vaddr >= sect.Seg.Vaddr+sect.Seg.Filelen {
 		return
 	}
 
 	sect.Reloff = uint64(Cpos())
-	var sym *LSym
+	for i, s := range syms {
-	for sym = first; sym != nil; sym = sym.Next {
+		if !s.Attr.Reachable() {
-		if !sym.Attr.Reachable() {
 			continue
 		}
-		if uint64(sym.Value) >= sect.Vaddr {
+		if uint64(s.Value) >= sect.Vaddr {
+			syms = syms[i:]
 			break
 		}
 	}
 
 	eaddr := int32(sect.Vaddr + sect.Length)
-	for ; sym != nil; sym = sym.Next {
+	for _, sym := range syms {
 		if !sym.Attr.Reachable() {
 			continue
 		}
@@ -852,7 +852,7 @@ func Machoemitreloc() {
 		Cput(0)
 	}
 
-	machorelocsect(Segtext.Sect, Ctxt.Textp)
+	machorelocsect(Segtext.Sect, list2slice(Ctxt.Textp))
 	for sect := Segtext.Sect.Next; sect != nil; sect = sect.Next {
 		machorelocsect(sect, datap)
 	}
@@ -860,6 +860,6 @@ func Machoemitreloc() {
 		machorelocsect(sect, datap)
 	}
 	for sect := Segdwarf.Sect; sect != nil; sect = sect.Next {
-		machorelocsect(sect, dwarfp)
+		machorelocsect(sect, list2slice(dwarfp))
 	}
 }

src/cmd/link/internal/ld/pe.go

@@ -764,7 +764,7 @@ func addexports() {
 
 // perelocsect relocates symbols from first in section sect, and returns
 // the total number of relocations emitted.
-func perelocsect(sect *Section, first *LSym) int {
+func perelocsect(sect *Section, syms []*LSym) int {
 	// If main section has no bits, nothing to relocate.
 	if sect.Vaddr >= sect.Seg.Vaddr+sect.Seg.Filelen {
 		return 0
@@ -773,18 +773,18 @@ func perelocsect(sect *Section, first *LSym) int {
 	relocs := 0
 
 	sect.Reloff = uint64(Cpos())
-	var sym *LSym
+	for i, s := range syms {
-	for sym = first; sym != nil; sym = sym.Next {
+		if !s.Attr.Reachable() {
-		if !sym.Attr.Reachable() {
 			continue
 		}
-		if uint64(sym.Value) >= sect.Vaddr {
+		if uint64(s.Value) >= sect.Vaddr {
+			syms = syms[i:]
 			break
 		}
 	}
 
 	eaddr := int32(sect.Vaddr + sect.Length)
-	for ; sym != nil; sym = sym.Next {
+	for _, sym := range syms {
 		if !sym.Attr.Reachable() {
 			continue
 		}
@@ -831,7 +831,7 @@ func peemitreloc(text, data, ctors *IMAGE_SECTION_HEADER) {
 	Lputl(0)
 	Wputl(0)
 
-	n := perelocsect(Segtext.Sect, Ctxt.Textp) + 1
+	n := perelocsect(Segtext.Sect, list2slice(Ctxt.Textp)) + 1
 	for sect := Segtext.Sect.Next; sect != nil; sect = sect.Next {
 		n += perelocsect(sect, datap)
 	}