Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
48 changes: 37 additions & 11 deletions errors.go
Original file line number Diff line number Diff line change
Expand Up @@ -64,6 +64,8 @@ var retryableCodes = []string{
ErrRateLimited,
}

const stackChainSeparator = "---"

// Error is terror's error. It implements Go's error interface.
type Error struct {
Code string `json:"code"`
Expand Down Expand Up @@ -176,16 +178,13 @@ func StackStringWithMaxSize(p *Error, sizeLimit int) string {
var causalDepth int
outer:
for terr != nil {
if buffer.Len() != 0 && len(terr.StackFrames) > 0 {
fmt.Fprintf(&buffer, "\n---")
s := terr.StackFrames
if buffer.Len() != 0 && len(s) > 0 {
fmt.Fprintf(&buffer, "\n%s", stackChainSeparator)
}
for _, frame := range terr.StackFrames {
// 10 seems like a reasonable estimate of how large the rest of the line would be.
estimatedLineLen := len(frame.Filename) + len(frame.Method) + 16
if estimatedLineLen+buffer.Len() > sizeLimit {
break outer
}
fmt.Fprintf(&buffer, "\n %s:%d in %s", frame.Filename, frame.Line, frame.Method)

if s.WriteWithMaxSize(&buffer, sizeLimit) {
break
}

if tcause, ok := terr.cause.(*Error); ok && causalDepth < maxCausalDepth {
Expand Down Expand Up @@ -343,6 +342,24 @@ func (p *Error) PrefixMatches(prefixParts ...string) bool {
return strings.HasPrefix(p.Code, prefix)
}

func (p *Error) hasCommonStackAncestry(currentRoot stack.Stack) bool {
// find the nearest causative error with a stacktrace
terr := p
for {
if len(terr.StackFrames) > 0 {
break
}
if next, ok := terr.cause.(*Error); ok {
terr = next
} else {
return false
}
}

// does our current stacktrace have a common ancestry with the terror being augmented?
return currentRoot.HasCommonAncestry(terr.StackFrames)
}

// Matches returns true if the error is a terror error and the string returned from error.Error() contains the given
// param string. This means you can match the error on different levels e.g. dotted codes `bad_request` or
// `bad_request.missing_param` or even on the more descriptive message
Expand Down Expand Up @@ -394,13 +411,22 @@ func Augment(err error, context string, params map[string]string) error {
switch err := err.(type) {
case *Error:
withMergedParams := addParams(err, params)
// The underlying terror will already have a stack, so we don't take a new trace here.
// If we know that the current stack has a common root with the already captured
// stack, then we know that the existing terror already captures our context.
// However, if it doesn't, we need to decorate the terror with the stacktrace for
// the current context.
var stackFrames stack.Stack
currentStack := stack.BuildStack(2)
if !err.hasCommonStackAncestry(currentStack) {
stackFrames = currentStack
}

return &Error{
Code: err.Code,
Message: context,
MessageChain: append([]string{err.Message}, err.MessageChain...),
Params: withMergedParams.Params,
StackFrames: stack.Stack{},
StackFrames: stackFrames,
IsRetryable: err.IsRetryable,
IsUnexpected: err.IsUnexpected,
MarshalCount: err.MarshalCount,
Expand Down
64 changes: 52 additions & 12 deletions errors_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,9 @@ package terrors
import (
"errors"
"fmt"
"github.com/stretchr/testify/require"
"os"
"strings"
"testing"

"github.com/stretchr/testify/assert"
Expand Down Expand Up @@ -48,9 +50,9 @@ func TestErrorConstructors(t *testing.T) {
},
{
Unauthorized, "service.foo", "test params", map[string]string{
"some key": "some value",
"another key": "another value",
}, ErrUnauthorized,
"some key": "some value",
"another key": "another value",
}, ErrUnauthorized,
},
{
PreconditionFailed, "service.foo", "precondition_failed.service.foo", nil, ErrPreconditionFailed,
Expand Down Expand Up @@ -272,14 +274,36 @@ func TestAugmentTerror(t *testing.T) {
"new": "meta",
})
terr := newErr.(*Error)

t.Logf("Stack: %s", terr.VerboseString())
assert.Equal(t, "not_found.foo", terr.Code)
assert.Equal(t, "added context", terr.Message)
assert.Empty(t, terr.StackFrames)

assert.Equal(t, "not_found.foo: added context: failed to find foo", terr.Error())
assert.Equal(t, base, terr.cause)
}

func TestAugmentTerrorDoesNotRepeatStackTrace(t *testing.T) {
newErr := failingChildFunction()

newErr = Augment(newErr, "added context", map[string]string{
"new": "meta",
})
terr := newErr.(*Error)

t.Logf("%s", terr.VerboseString())

segments := strings.Split(terr.StackString(), stackChainSeparator)

assert.Len(t, segments, 1)
}

func failingChildFunction() error {
return NotFound("foo", "failed to find foo", map[string]string{
"base": "meta",
})
}

func TestAugmentTerrorWithWrap(t *testing.T) {
base := NotFound("foo", "failed to find foo", map[string]string{"base": "meta"})
augmentedErr := Augment(base, "added context", map[string]string{"new": "meta"})
Expand Down Expand Up @@ -604,6 +628,30 @@ func TestStackStringChasesCausalChain(t *testing.T) {
assert.Contains(t, ss, "failyFunction")
}

func TestStackStringSegmentsCausalChain(t *testing.T) {
a := failyFunction().(*Error)
a.StackFrames = stack.Stack{
{Filename: "foo.go", Line: 42, Method: "Foo"},
}
err := Augment(a, "something may be up", nil)
terr := err.(*Error)
terr.StackFrames = stack.Stack{
{Filename: "bar.go", Line: 43, Method: "Bar"},
}

ss := terr.StackString()
t.Log(ss)

segments := strings.Split(ss, stackChainSeparator)
require.Len(t, segments, 2)

// The outermost stackframes should be at the start
assert.Contains(t, segments[0], "bar.go")

// Then the inner stackframes
assert.Contains(t, segments[1], "foo.go")
}

func TestCircularErrorProducesFiniteOutputWithStackFrames(t *testing.T) {
orig := failyFunction()
err := Augment(orig, "something may be up", nil)
Expand All @@ -616,11 +664,3 @@ func TestCircularErrorProducesFiniteOutputWithStackFrames(t *testing.T) {
assert.Less(t, len(ss), 32000)
assert.GreaterOrEqual(t, len(ss), 32000-1000)
}
func TestCircularErrorProducesFiniteOutputWithoutStackFrames(t *testing.T) {
err := Augment(failyFunction(), "something may be up", nil)
terr := err.(*Error)
terr.cause = terr
ss := terr.StackString()
// There's no actual stack in the causal cycle, so we don't render anything here.
assert.Empty(t, ss)
}
92 changes: 90 additions & 2 deletions stack/stack.go
Original file line number Diff line number Diff line change
Expand Up @@ -73,13 +73,101 @@ func (s Stack) Fingerprint() string {
return fmt.Sprintf("%x", hash.Sum32())
}

func (s Stack) HasCommonAncestry(otherRoot Stack) bool {
startIdx := len(otherRoot) - len(s)
if startIdx < 0 {
return false
}

other := otherRoot[startIdx:]

if len(other) == 0 {
return true
}

// Special case the frame where we call terrors.Augment, because for cases like the following:
//
// err := something();
// if err != nil {
// return terrors.Augment(err, "context", nil()
// }
//
// Just comparing the program counter isn't enough, because while yes, the calls
// to something() and terrors.Augment() are at different points, we still want to
// consider them as the "same" stack frame. So we fall back to comparing the file
// and method names too.
Comment on lines +95 to +98

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think I might be being dense.

In that example, the PC for the two traces will be different, no? One will be pointing at the call to something, and the other will be the call to terrors.Augment.

Will we not always return false on the first frame? 🤔

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I'd hope that my tests demonstrate otherwise :) You're right that the PC check would always fail on the first frame, which is why we fall back to comparing the file and method names down on line 110. Does that help at all?

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

🤦‍♀️

thisFrame.PC != otherFrame.PC && thisFrame.Filename != otherFrame.Filename && thisFrame.Method != otherFrame.Method

I was being dense - this returns false if all of them don't match, not if any of them don't match. (Yes, the tests were a clue I was missing something. 😆)

But I'm still not sure I understand the intent behind this check. If the filename matches, but the method doesn't, we consider it a match. Do we not want something like:

// If PC differs _and_ we're not in the same method & file, we're not in the same function.

if thisFrame.PC != otherFrame.PC &&
    (thisFrame.Filename != otherFrame.Filename || thisFrame.Method != otherFrame.Method) {
    return false
}

if !topFrameEqualByFunctionName(other[0], s[0]) {
return false
}

thisRemaining := s[1:]
otherRemaining := other[1:]

for i, thisFrame := range thisRemaining {
if !equalByPC(otherRemaining[i], thisFrame) {
return false
}
}
return true
}

func topFrameEqualByFunctionName(otherFrame *Frame, thisFrame *Frame) bool {
// We also assume that a program counter of zero means it is remote, and thus
// never equal to a local frame, since a) we don't currently transfer that value
// over the wire (see the protobuf representations for details), and b) there are
// no instructions mapped at address zero.
if otherFrame.PC == 0 {
return false
}

if thisFrame.PC == otherFrame.PC {
// The other properties are all derived from the program counter, so we know that
// if the program counters are the same we can skip the remaining checks.
return true
} else if thisFrame.Filename == otherFrame.Filename && thisFrame.Method == otherFrame.Method {
return true
}
return false
}

func equalByPC(otherFrame *Frame, thisFrame *Frame) bool {
// A frame from a remote source; can't match a locally generated stack
if otherFrame.PC == 0 {
return false
}

// if the program counter values are the same, then that's fine, and all we need to care about for frames above the caller of terrors.Augment
return thisFrame.PC == otherFrame.PC
}

func (s Stack) String() string {
var buf strings.Builder
s.WriteWithMaxSize(&buf, 32000)
return buf.String()
}

// WriteWithMaxSize writes the stack to the provided buffer, not going above
func (s Stack) WriteWithMaxSize(buffer *strings.Builder, sizeLimit int) bool {
for _, frame := range s {
// 10 seems like a reasonable estimate of how large the rest of the line would be.
estimatedLineLen := len(frame.Filename) + len(frame.Method) + 16
if estimatedLineLen+buffer.Len() > sizeLimit {
return true
}
fmt.Fprintf(buffer, "\n %s:%d in %s", frame.Filename, frame.Line, frame.Method)
}

return false
}

// Remove un-needed information from the source file path. This makes them
// shorter in Rollbar UI as well as making them the same, regardless of the
// machine the code was compiled on.
//
// Examples:
// /usr/local/go/src/pkg/runtime/proc.c -> pkg/runtime/proc.c
// /home/foo/go/src/github.com/rollbar/rollbar.go -> github.com/rollbar/rollbar.go
//
// /usr/local/go/src/pkg/runtime/proc.c -> pkg/runtime/proc.c
// /home/foo/go/src/github.com/rollbar/rollbar.go -> github.com/rollbar/rollbar.go
func shortenFilePath(s string) string {
idx := strings.Index(s, "/src/pkg/")
if idx != -1 {
Expand Down
Loading
Loading