diff --git a/language/lexer/lexer.go b/language/lexer/lexer.go index 1988c5fdc..e20106f68 100644 --- a/language/lexer/lexer.go +++ b/language/lexer/lexer.go @@ -625,7 +625,7 @@ func positionAfterWhitespace(body []byte, startPosition int) (position int, rune // SourceCharacter but not LineTerminator (code > 0x001F || code == 0x0009) && code != 0x000A && code != 0x000D { position += n - runePosition++ + runePosition += n continue } else { break diff --git a/language/lexer/lexer_test.go b/language/lexer/lexer_test.go index c476d8fa4..001a44fa7 100644 --- a/language/lexer/lexer_test.go +++ b/language/lexer/lexer_test.go @@ -153,6 +153,18 @@ func TestLexer_SkipsWhiteSpace(t *testing.T) { Value: "", }, }, + { + Body: ` + #comment фы世界 + foo#comment +`, + Expected: Token{ + Kind: NAME, + Start: 30, + End: 33, + Value: "foo", + }, + }, } for _, test := range tests { token, err := Lex(&source.Source{Body: []byte(test.Body)})(0) diff --git a/language/parser/parser_test.go b/language/parser/parser_test.go index 8f0e0715d..6a4f7cd0a 100644 --- a/language/parser/parser_test.go +++ b/language/parser/parser_test.go @@ -346,6 +346,84 @@ func TestParsesMultiByteCharacters_UnicodeText(t *testing.T) { } } +func TestMultiByteCharactersAmongstOtherTokens(t *testing.T) { + doc := ` + { + # This comment has a фы世界 multi-byte character. + field(arg: "Has a фы世界 multi-byte character.") + } + ` + astDoc := parse(t, doc) + + expectedASTDoc := ast.NewDocument(&ast.Document{ + Loc: ast.NewLocation(&ast.Location{ + Start: 67, + End: 121, + }), + Definitions: []ast.Node{ + ast.NewOperationDefinition(&ast.OperationDefinition{ + Loc: ast.NewLocation(&ast.Location{ + Start: 67, + End: 119, + }), + Operation: "query", + SelectionSet: ast.NewSelectionSet(&ast.SelectionSet{ + Loc: ast.NewLocation(&ast.Location{ + Start: 67, + End: 119, + }), + Selections: []ast.Selection{ + ast.NewField(&ast.Field{ + Loc: ast.NewLocation(&ast.Location{ + Start: 67, + End: 117, + }), + Name: ast.NewName(&ast.Name{ + Loc: ast.NewLocation(&ast.Location{ + Start: 69, + End: 74, + }), + Value: "field", + }), + Arguments: []*ast.Argument{ + ast.NewArgument(&ast.Argument{ + Loc: ast.NewLocation(&ast.Location{ + Start: 75, + End: 116, + }), + Name: ast.NewName(&ast.Name{ + + Loc: ast.NewLocation(&ast.Location{ + Start: 75, + End: 78, + }), + Value: "arg", + }), + Value: ast.NewStringValue(&ast.StringValue{ + + Loc: ast.NewLocation(&ast.Location{ + Start: 80, + End: 116, + }), + Value: "Has a фы世界 multi-byte character.", + }), + }), + }, + }), + }, + }), + }), + }, + }) + + astDocQuery := printer.Print(astDoc) + expectedASTDocQuery := printer.Print(expectedASTDoc) + + if !reflect.DeepEqual(astDocQuery, expectedASTDocQuery) { + t.Fatalf("unexpected document, expected: %v, got: %v", expectedASTDocQuery, astDocQuery) + } +} + func TestParsesKitchenSink(t *testing.T) { b, err := ioutil.ReadFile("../../kitchen-sink.graphql") if err != nil {