codekingpro/portable-devtools

1// run2 3// Copyright 2009 The Go Authors. All rights reserved.4// Use of this source code is governed by a BSD-style5// license that can be found in the LICENSE file.6 7// Test heap sampling logic.8 9package main10 11import (12	"fmt"13	"math"14	"runtime"15)16 17var a16 *[16]byte18var a512 *[512]byte19var a256 *[256]byte20var a1k *[1024]byte21var a16k *[16 * 1024]byte22var a17k *[17 * 1024]byte23var a18k *[18 * 1024]byte24 25// This test checks that heap sampling produces reasonable results.26// Note that heap sampling uses randomization, so the results vary for27// run to run. To avoid flakes, this test performs multiple28// experiments and only complains if all of them consistently fail.29func main() {30	// Sample at 16K instead of default 512K to exercise sampling more heavily.31	runtime.MemProfileRate = 16 * 102432 33	if err := testInterleavedAllocations(); err != nil {34		panic(err.Error())35	}36	if err := testSmallAllocations(); err != nil {37		panic(err.Error())38	}39}40 41// Repeatedly exercise a set of allocations and check that the heap42// profile collected by the runtime unsamples to a reasonable43// value. Because sampling is based on randomization, there can be44// significant variability on the unsampled data. To account for that,45// the testcase allows for a 10% margin of error, but only fails if it46// consistently fails across three experiments, avoiding flakes.47func testInterleavedAllocations() error {48	const iters = 5000049	// Sizes of the allocations performed by each experiment.50	frames := []string{"main.allocInterleaved1", "main.allocInterleaved2", "main.allocInterleaved3"}51	leafFrame := "main.allocInterleaved"52 53	// Pass if at least one of three experiments has no errors. Use a separate54	// function for each experiment to identify each experiment in the profile.55	allocInterleaved1(iters)56	if checkAllocations(getMemProfileRecords(), leafFrame, frames[0:1], iters, allocInterleavedSizes) == nil {57		// Passed on first try, report no error.58		return nil59	}60	allocInterleaved2(iters)61	if checkAllocations(getMemProfileRecords(), leafFrame, frames[0:2], iters, allocInterleavedSizes) == nil {62		// Passed on second try, report no error.63		return nil64	}65	allocInterleaved3(iters)66	// If it fails a third time, we may be onto something.67	return checkAllocations(getMemProfileRecords(), leafFrame, frames[0:3], iters, allocInterleavedSizes)68}69 70var allocInterleavedSizes = []int64{17 * 1024, 1024, 18 * 1024, 512, 16 * 1024, 256}71 72// allocInterleaved stress-tests the heap sampling logic by interleaving large and small allocations.73func allocInterleaved(n int) {74	for i := 0; i < n; i++ {75		// Test verification depends on these lines being contiguous.76		a17k = new([17 * 1024]byte)77		a1k = new([1024]byte)78		a18k = new([18 * 1024]byte)79		a512 = new([512]byte)80		a16k = new([16 * 1024]byte)81		a256 = new([256]byte)82		// Test verification depends on these lines being contiguous.83 84		// Slow down the allocation rate to avoid #52433.85		runtime.Gosched()86	}87}88 89func allocInterleaved1(n int) {90	allocInterleaved(n)91}92 93func allocInterleaved2(n int) {94	allocInterleaved(n)95}96 97func allocInterleaved3(n int) {98	allocInterleaved(n)99}100 101// Repeatedly exercise a set of allocations and check that the heap102// profile collected by the runtime unsamples to a reasonable103// value. Because sampling is based on randomization, there can be104// significant variability on the unsampled data. To account for that,105// the testcase allows for a 10% margin of error, but only fails if it106// consistently fails across three experiments, avoiding flakes.107func testSmallAllocations() error {108	const iters = 50000109	// Sizes of the allocations performed by each experiment.110	sizes := []int64{1024, 512, 256}111	frames := []string{"main.allocSmall1", "main.allocSmall2", "main.allocSmall3"}112	leafFrame := "main.allocSmall"113 114	// Pass if at least one of three experiments has no errors. Use a separate115	// function for each experiment to identify each experiment in the profile.116	allocSmall1(iters)117	if checkAllocations(getMemProfileRecords(), leafFrame, frames[0:1], iters, sizes) == nil {118		// Passed on first try, report no error.119		return nil120	}121	allocSmall2(iters)122	if checkAllocations(getMemProfileRecords(), leafFrame, frames[0:2], iters, sizes) == nil {123		// Passed on second try, report no error.124		return nil125	}126	allocSmall3(iters)127	// If it fails a third time, we may be onto something.128	return checkAllocations(getMemProfileRecords(), leafFrame, frames[0:3], iters, sizes)129}130 131// allocSmall performs only small allocations for sanity testing.132func allocSmall(n int) {133	for i := 0; i < n; i++ {134		// Test verification depends on these lines being contiguous.135		a1k = new([1024]byte)136		a512 = new([512]byte)137		a256 = new([256]byte)138 139		// Slow down the allocation rate to avoid #52433.140		runtime.Gosched()141	}142}143 144// Three separate instances of testing to avoid flakes. Will report an error145// only if they all consistently report failures.146func allocSmall1(n int) {147	allocSmall(n)148}149 150func allocSmall2(n int) {151	allocSmall(n)152}153 154func allocSmall3(n int) {155	allocSmall(n)156}157 158// checkAllocations validates that the profile records collected for159// the named function are consistent with count contiguous allocations160// of the specified sizes.161// Check multiple functions and only report consistent failures across162// multiple tests.163// Look only at samples that include the named frames, and group the164// allocations by their line number. All these allocations are done from165// the same leaf function, so their line numbers are the same.166func checkAllocations(records []runtime.MemProfileRecord, leafFrame string, frames []string, count int64, size []int64) error {167	objectsPerLine := map[int][]int64{}168	bytesPerLine := map[int][]int64{}169	totalCount := []int64{}170	// Compute the line number of the first allocation. All the171	// allocations are from the same leaf, so pick the first one.172	var firstLine int173	for ln := range allocObjects(records, leafFrame, frames[0]) {174		if firstLine == 0 || firstLine > ln {175			firstLine = ln176		}177	}178	for _, frame := range frames {179		var objectCount int64180		a := allocObjects(records, leafFrame, frame)181		for s := range size {182			// Allocations of size size[s] should be on line firstLine + s.183			ln := firstLine + s184			objectsPerLine[ln] = append(objectsPerLine[ln], a[ln].objects)185			bytesPerLine[ln] = append(bytesPerLine[ln], a[ln].bytes)186			objectCount += a[ln].objects187		}188		totalCount = append(totalCount, objectCount)189	}190	for i, w := range size {191		ln := firstLine + i192		if err := checkValue(frames[0], ln, "objects", count, objectsPerLine[ln]); err != nil {193			return err194		}195		if err := checkValue(frames[0], ln, "bytes", count*w, bytesPerLine[ln]); err != nil {196			return err197		}198	}199	return checkValue(frames[0], 0, "total", count*int64(len(size)), totalCount)200}201 202// checkValue checks an unsampled value against its expected value.203// Given that this is a sampled value, it will be unexact and will change204// from run to run. Only report it as a failure if all the values land205// consistently far from the expected value.206func checkValue(fname string, ln int, testName string, want int64, got []int64) error {207	if got == nil {208		return fmt.Errorf("Unexpected empty result")209	}210	min, max := got[0], got[0]211	for _, g := range got[1:] {212		if g < min {213			min = g214		}215		if g > max {216			max = g217		}218	}219	margin := want / 10 // 10% margin.220	if min > want+margin || max < want-margin {221		return fmt.Errorf("%s:%d want %s in [%d: %d], got %v", fname, ln, testName, want-margin, want+margin, got)222	}223	return nil224}225 226func getMemProfileRecords() []runtime.MemProfileRecord {227	// Force the runtime to update the object and byte counts.228	// This can take up to two GC cycles to get a complete229	// snapshot of the current point in time.230	runtime.GC()231	runtime.GC()232 233	// Find out how many records there are (MemProfile(nil, true)),234	// allocate that many records, and get the data.235	// There's a race—more records might be added between236	// the two calls—so allocate a few extra records for safety237	// and also try again if we're very unlucky.238	// The loop should only execute one iteration in the common case.239	var p []runtime.MemProfileRecord240	n, ok := runtime.MemProfile(nil, true)241	for {242		// Allocate room for a slightly bigger profile,243		// in case a few more entries have been added244		// since the call to MemProfile.245		p = make([]runtime.MemProfileRecord, n+50)246		n, ok = runtime.MemProfile(p, true)247		if ok {248			p = p[0:n]249			break250		}251		// Profile grew; try again.252	}253	return p254}255 256type allocStat struct {257	bytes, objects int64258}259 260// allocObjects examines the profile records for samples including the261// named function and returns the allocation stats aggregated by262// source line number of the allocation (at the leaf frame).263func allocObjects(records []runtime.MemProfileRecord, leafFrame, function string) map[int]allocStat {264	a := make(map[int]allocStat)265	for _, r := range records {266		var pcs []uintptr267		for _, s := range r.Stack0 {268			if s == 0 {269				break270			}271			pcs = append(pcs, s)272		}273		frames := runtime.CallersFrames(pcs)274		line := 0275		for {276			frame, more := frames.Next()277			name := frame.Function278			if name == leafFrame && line == 0 {279				line = frame.Line280			}281			if name == function {282				allocStat := a[line]283				allocStat.bytes += r.AllocBytes284				allocStat.objects += r.AllocObjects285				a[line] = allocStat286			}287			if !more {288				break289			}290		}291	}292	for line, stats := range a {293		objects, bytes := scaleHeapSample(stats.objects, stats.bytes, int64(runtime.MemProfileRate))294		a[line] = allocStat{bytes, objects}295	}296	return a297}298 299// scaleHeapSample unsamples heap allocations.300// Taken from src/cmd/pprof/internal/profile/legacy_profile.go301func scaleHeapSample(count, size, rate int64) (int64, int64) {302	if count == 0 || size == 0 {303		return 0, 0304	}305 306	if rate <= 1 {307		// if rate==1 all samples were collected so no adjustment is needed.308		// if rate<1 treat as unknown and skip scaling.309		return count, size310	}311 312	avgSize := float64(size) / float64(count)313	scale := 1 / (1 - math.Exp(-avgSize/float64(rate)))314 315	return int64(float64(count) * scale), int64(float64(size) * scale)316}317