Skip to content

Commit c30ebee

Browse files
nycdotnetnycdotnet
committed
Implemented an asynchronous tokenizer that handles whitespace and newlines only
1 parent 6f02c19 commit c30ebee

13 files changed

+407
-18
lines changed

‎Spike/App.config

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
<?xml version="1.0" encoding="utf-8" ?>
2+
<configuration>
3+
<startup>
4+
<supportedRuntime version="v4.0" sku=".NETFramework,Version=v4.5" />
5+
</startup>
6+
</configuration>

‎Spike/Program.cs

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,40 @@
1+
using System;
2+
using System.Collections.Generic;
3+
using System.Linq;
4+
using System.Text;
5+
using System.Threading.Tasks;
6+
using TSqlFlex.SqlParser;
7+
8+
namespace Spike
9+
{
10+
class Program
11+
{
12+
static void Main(string[] args)
13+
{
14+
DoStuff();
15+
}
16+
17+
static async void DoStuff()
18+
{
19+
var actualTask = SqlTokenizer.TokenizeAsync(" \n ");
20+
var expected = new List<SqlToken>();
21+
expected.Add(new SqlToken(SqlToken.TokenTypes.Whitespace, 1, 1));
22+
expected[0].Text = " ";
23+
expected.Add(new SqlToken(SqlToken.TokenTypes.Newline, 3, 1));
24+
expected[1].Text = "\n";
25+
expected.Add(new SqlToken(SqlToken.TokenTypes.Whitespace, 1, 2));
26+
expected[2].Text = " ";
27+
var actual = await actualTask;
28+
29+
//Assert.AreEqual(expected, actual);
30+
Console.Write(actual[0].StartCharacterIndex);
31+
//Assert.AreEqual(expected, actual);
32+
/*Assert.AreEqual(expected[0].TokenType, actual[0].TokenType);
33+
Assert.AreEqual(expected[0].LineNumber, actual[0].LineNumber);
34+
Assert.AreEqual(expected[0].StartCharacterIndex, actual[0].StartCharacterIndex);
35+
Assert.AreEqual(expected[0].Length, actual[0].Length);
36+
Assert.AreEqual(expected[0].Text, actual[0].Text);
37+
* */
38+
}
39+
}
40+
}

‎Spike/Properties/AssemblyInfo.cs

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,36 @@
1+
using System.Reflection;
2+
using System.Runtime.CompilerServices;
3+
using System.Runtime.InteropServices;
4+
5+
// General Information about an assembly is controlled through the following
6+
// set of attributes. Change these attribute values to modify the information
7+
// associated with an assembly.
8+
[assembly: AssemblyTitle("Spike")]
9+
[assembly: AssemblyDescription("")]
10+
[assembly: AssemblyConfiguration("")]
11+
[assembly: AssemblyCompany("")]
12+
[assembly: AssemblyProduct("Spike")]
13+
[assembly: AssemblyCopyright("Copyright © 2014")]
14+
[assembly: AssemblyTrademark("")]
15+
[assembly: AssemblyCulture("")]
16+
17+
// Setting ComVisible to false makes the types in this assembly not visible
18+
// to COM components. If you need to access a type in this assembly from
19+
// COM, set the ComVisible attribute to true on that type.
20+
[assembly: ComVisible(false)]
21+
22+
// The following GUID is for the ID of the typelib if this project is exposed to COM
23+
[assembly: Guid("138b89a4-48ee-47b1-bae1-104081f1deb7")]
24+
25+
// Version information for an assembly consists of the following four values:
26+
//
27+
// Major Version
28+
// Minor Version
29+
// Build Number
30+
// Revision
31+
//
32+
// You can specify all the values or you can default the Build and Revision Numbers
33+
// by using the '*' as shown below:
34+
// [assembly: AssemblyVersion("1.0.*")]
35+
[assembly: AssemblyVersion("1.0.0.0")]
36+
[assembly: AssemblyFileVersion("1.0.0.0")]

‎Spike/Spike.csproj

Lines changed: 68 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,68 @@
1+
<?xml version="1.0" encoding="utf-8"?>
2+
<Project ToolsVersion="12.0" DefaultTargets="Build" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
3+
<Import Project="$(MSBuildExtensionsPath)\$(MSBuildToolsVersion)\Microsoft.Common.props" Condition="Exists('$(MSBuildExtensionsPath)\$(MSBuildToolsVersion)\Microsoft.Common.props')" />
4+
<PropertyGroup>
5+
<Configuration Condition=" '$(Configuration)' == '' ">Debug</Configuration>
6+
<Platform Condition=" '$(Platform)' == '' ">AnyCPU</Platform>
7+
<ProjectGuid>{2DE449CB-81B9-4735-B058-8E4828E26CE6}</ProjectGuid>
8+
<OutputType>Exe</OutputType>
9+
<AppDesignerFolder>Properties</AppDesignerFolder>
10+
<RootNamespace>Spike</RootNamespace>
11+
<AssemblyName>Spike</AssemblyName>
12+
<TargetFrameworkVersion>v4.5</TargetFrameworkVersion>
13+
<FileAlignment>512</FileAlignment>
14+
</PropertyGroup>
15+
<PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Debug|AnyCPU' ">
16+
<PlatformTarget>AnyCPU</PlatformTarget>
17+
<DebugSymbols>true</DebugSymbols>
18+
<DebugType>full</DebugType>
19+
<Optimize>false</Optimize>
20+
<OutputPath>bin\Debug\</OutputPath>
21+
<DefineConstants>DEBUG;TRACE</DefineConstants>
22+
<ErrorReport>prompt</ErrorReport>
23+
<WarningLevel>4</WarningLevel>
24+
</PropertyGroup>
25+
<PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Release|AnyCPU' ">
26+
<PlatformTarget>AnyCPU</PlatformTarget>
27+
<DebugType>pdbonly</DebugType>
28+
<Optimize>true</Optimize>
29+
<OutputPath>bin\Release\</OutputPath>
30+
<DefineConstants>TRACE</DefineConstants>
31+
<ErrorReport>prompt</ErrorReport>
32+
<WarningLevel>4</WarningLevel>
33+
</PropertyGroup>
34+
<ItemGroup>
35+
<Reference Include="System" />
36+
<Reference Include="System.Core" />
37+
<Reference Include="System.Xml.Linq" />
38+
<Reference Include="System.Data.DataSetExtensions" />
39+
<Reference Include="Microsoft.CSharp" />
40+
<Reference Include="System.Data" />
41+
<Reference Include="System.Xml" />
42+
</ItemGroup>
43+
<ItemGroup>
44+
<Compile Include="Program.cs" />
45+
<Compile Include="Properties\AssemblyInfo.cs" />
46+
</ItemGroup>
47+
<ItemGroup>
48+
<None Include="App.config" />
49+
</ItemGroup>
50+
<ItemGroup>
51+
<ProjectReference Include="..\TSqlFlex.SqlParser.Tests\TSqlFlex.SqlParser.Tests.csproj">
52+
<Project>{c95c7cc7-59bb-4a4e-adae-9272bb145ec9}</Project>
53+
<Name>TSqlFlex.SqlParser.Tests</Name>
54+
</ProjectReference>
55+
<ProjectReference Include="..\TSqlFlex.SqlParser\TSqlFlex.SqlParser.csproj">
56+
<Project>{7633e837-631c-49bd-9264-c9b46e1d02b8}</Project>
57+
<Name>TSqlFlex.SqlParser</Name>
58+
</ProjectReference>
59+
</ItemGroup>
60+
<Import Project="$(MSBuildToolsPath)\Microsoft.CSharp.targets" />
61+
<!-- To modify your build process, add your task inside one of the targets below and uncomment it.
62+
Other similar extension points exist, see Microsoft.Common.targets.
63+
<Target Name="BeforeBuild">
64+
</Target>
65+
<Target Name="AfterBuild">
66+
</Target>
67+
-->
68+
</Project>

‎TSqlFlex.SqlParser.Tests/ParserTests.cs

Lines changed: 1 addition & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,6 @@
11
using System;
22
using System.Collections.Generic;
33
using System.Linq;
4-
using System.IO;
5-
using System.Reflection;
64
using System.Text;
75
using System.Threading.Tasks;
86
using NUnit.Framework;
@@ -24,26 +22,12 @@ public void CreateParser_DoesNotCrash()
2422
[Test()]
2523
public void SqlBatch_ReturnsSameBatchTextAsPassed()
2624
{
27-
string s = Load_SimpleSelect();
25+
string s = StaticFiles.SimpleSelect();
2826
SqlBatch b = new SqlBatch(s);
2927

3028
Assert.AreEqual(s, b.BatchText());
3129
Assert.AreEqual(21, b.BatchText().Length, "expected this specific length because that's what is in the file.");
3230
}
3331

34-
35-
36-
private string Load_SimpleSelect()
37-
{
38-
string resourceName = "TSqlFlex.SqlParser.Tests.SqlScripts.SimpleSelect.sql";
39-
string result;
40-
Assembly assembly = Assembly.GetExecutingAssembly();
41-
using (Stream stream = assembly.GetManifestResourceStream(resourceName))
42-
using (StreamReader reader = new StreamReader(stream))
43-
{
44-
result = reader.ReadToEnd();
45-
}
46-
return result;
47-
}
4832
}
4933
}

‎TSqlFlex.SqlParser.Tests/SqlScripts/CommentsStringsAndWhitespace.sql

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
--This is a one-line comment preceeded by whitespace.
2+
3+
/* this is a one-line block comment. */
4+
/* this is a multi-line block comment
5+
6+
ending on the third line.*/
7+
8+
-- 'This is a string inside a comment'
9+
10+
/* 'This is a string inside a block comment.' */
11+
12+
PRINT 'This is not -- a line comment.'
13+
PRINT 'THIS /* is not a multi line
14+
15+
comment */'

‎TSqlFlex.SqlParser.Tests/StaticFiles.cs

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,35 @@
1+
using System;
2+
using System.Collections.Generic;
3+
using System.IO;
4+
using System.Reflection;
5+
using System.Linq;
6+
using System.Text;
7+
using System.Threading.Tasks;
8+
9+
namespace TSqlFlex.SqlParser.Tests
10+
{
11+
public class StaticFiles
12+
{
13+
static public string SimpleSelect()
14+
{
15+
return GetResourceByName("TSqlFlex.SqlParser.Tests.SqlScripts.SimpleSelect.sql");
16+
}
17+
18+
static public string ComentsStringsAndWhitespace()
19+
{
20+
return GetResourceByName("TSqlFlex.SqlParser.Tests.SqlScripts.CommentsStringsAndWhitespace.sql");
21+
}
22+
23+
static private string GetResourceByName(string resourceName)
24+
{
25+
string result;
26+
Assembly assembly = Assembly.GetExecutingAssembly();
27+
using (Stream stream = assembly.GetManifestResourceStream(resourceName))
28+
using (StreamReader reader = new StreamReader(stream))
29+
{
30+
result = reader.ReadToEnd();
31+
}
32+
return result;
33+
}
34+
}
35+
}

‎TSqlFlex.SqlParser.Tests/TSqlFlex.SqlParser.Tests.csproj

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,8 @@
3535
</Reference>
3636
<Reference Include="System" />
3737
<Reference Include="System.Core" />
38+
<Reference Include="System.Web" />
39+
<Reference Include="System.Web.Extensions" />
3840
<Reference Include="System.Xml.Linq" />
3941
<Reference Include="System.Data.DataSetExtensions" />
4042
<Reference Include="Microsoft.CSharp" />
@@ -44,6 +46,8 @@
4446
<ItemGroup>
4547
<Compile Include="ParserTests.cs" />
4648
<Compile Include="Properties\AssemblyInfo.cs" />
49+
<Compile Include="StaticFiles.cs" />
50+
<Compile Include="TokenizerTests.cs" />
4751
</ItemGroup>
4852
<ItemGroup>
4953
<None Include="packages.config" />
@@ -57,6 +61,9 @@
5761
<ItemGroup>
5862
<EmbeddedResource Include="SqlScripts\SimpleSelect.sql" />
5963
</ItemGroup>
64+
<ItemGroup>
65+
<EmbeddedResource Include="SqlScripts\CommentsStringsAndWhitespace.sql" />
66+
</ItemGroup>
6067
<Import Project="$(MSBuildToolsPath)\Microsoft.CSharp.targets" />
6168
<!-- To modify your build process, add your task inside one of the targets below and uncomment it.
6269
Other similar extension points exist, see Microsoft.Common.targets.

‎TSqlFlex.SqlParser.Tests/TokenizerTests.cs

Lines changed: 62 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,62 @@
1+
using System;
2+
using System.Collections.Generic;
3+
using System.Linq;
4+
using System.Text;
5+
using System.Threading.Tasks;
6+
using System.Web;
7+
using NUnit.Framework;
8+
using TSqlFlex.SqlParser;
9+
10+
11+
namespace TSqlFlex.SqlParser.Tests
12+
{
13+
[TestFixture()]
14+
class TokenizerTests
15+
{
16+
[Test()]
17+
public async void WhenPassedEmptyString_ReturnsEmptyArray()
18+
{
19+
var actualTask = SqlTokenizer.TokenizeAsync("");
20+
var expected = new List<SqlToken>();
21+
var actual = await actualTask;
22+
Assert.AreEqual(expected, actual);
23+
}
24+
25+
[Test()]
26+
public async void WhenPassedSingleSpace_ReturnsArrayWithOneWhitespaceToken()
27+
{
28+
var actualTask = SqlTokenizer.TokenizeAsync(" ");
29+
var expected = new List<SqlToken>();
30+
expected.Add (new SqlToken(SqlToken.TokenTypes.Whitespace, 1, 1));
31+
expected[0].Text = " ";
32+
var actual = await actualTask;
33+
AssertArePropertiesEqual(expected, actual);
34+
}
35+
36+
[Test()]
37+
public async void WhenPassedWhitespaceSeparatedByNewline_ReturnsWhitespaceSeparatedByNewline()
38+
{
39+
var actualTask = SqlTokenizer.TokenizeAsync(" \n ");
40+
var expected = new List<SqlToken>();
41+
expected.Add(new SqlToken(SqlToken.TokenTypes.Whitespace, 1, 1));
42+
expected[0].Text = " ";
43+
expected.Add(new SqlToken(SqlToken.TokenTypes.Newline, 3, 1));
44+
expected[1].Text = "\r\n";
45+
expected.Add(new SqlToken(SqlToken.TokenTypes.Whitespace, 1, 2));
46+
expected[2].Text = " ";
47+
var actual = await actualTask;
48+
49+
AssertArePropertiesEqual(expected, actual);
50+
}
51+
52+
53+
//Thanks: http://stackoverflow.com/questions/318210/compare-equality-between-two-objects-in-nunit/
54+
public static void AssertArePropertiesEqual(object expected, object actual)
55+
{
56+
var serializer = new System.Web.Script.Serialization.JavaScriptSerializer();
57+
var expectedJson = serializer.Serialize(expected);
58+
var actualJson = serializer.Serialize(actual);
59+
Assert.AreEqual(expectedJson, actualJson, "expected: " + expectedJson + "\r\n" + "actual: " + actualJson);
60+
}
61+
}
62+
}

‎TSqlFlex.SqlParser.sln

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,12 +1,14 @@
11

22
Microsoft Visual Studio Solution File, Format Version 12.00
33
# Visual Studio 2013
4-
VisualStudioVersion = 12.0.30501.0
4+
VisualStudioVersion = 12.0.30626.0
55
MinimumVisualStudioVersion = 10.0.40219.1
66
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "TSqlFlex.SqlParser", "TSqlFlex.SqlParser\TSqlFlex.SqlParser.csproj", "{7633E837-631C-49BD-9264-C9B46E1D02B8}"
77
EndProject
88
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "TSqlFlex.SqlParser.Tests", "TSqlFlex.SqlParser.Tests\TSqlFlex.SqlParser.Tests.csproj", "{C95C7CC7-59BB-4A4E-ADAE-9272BB145EC9}"
99
EndProject
10+
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Spike", "Spike\Spike.csproj", "{2DE449CB-81B9-4735-B058-8E4828E26CE6}"
11+
EndProject
1012
Global
1113
GlobalSection(SolutionConfigurationPlatforms) = preSolution
1214
Debug|Any CPU = Debug|Any CPU
@@ -21,6 +23,10 @@ Global
2123
{C95C7CC7-59BB-4A4E-ADAE-9272BB145EC9}.Debug|Any CPU.Build.0 = Debug|Any CPU
2224
{C95C7CC7-59BB-4A4E-ADAE-9272BB145EC9}.Release|Any CPU.ActiveCfg = Release|Any CPU
2325
{C95C7CC7-59BB-4A4E-ADAE-9272BB145EC9}.Release|Any CPU.Build.0 = Release|Any CPU
26+
{2DE449CB-81B9-4735-B058-8E4828E26CE6}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
27+
{2DE449CB-81B9-4735-B058-8E4828E26CE6}.Debug|Any CPU.Build.0 = Debug|Any CPU
28+
{2DE449CB-81B9-4735-B058-8E4828E26CE6}.Release|Any CPU.ActiveCfg = Release|Any CPU
29+
{2DE449CB-81B9-4735-B058-8E4828E26CE6}.Release|Any CPU.Build.0 = Release|Any CPU
2430
EndGlobalSection
2531
GlobalSection(SolutionProperties) = preSolution
2632
HideSolutionNode = FALSE

0 commit comments

Comments
 (0)