Skip to content

instarepo automatic PR #20

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 1 commit into
base: master
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 6 additions & 3 deletions Factory.pas
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,8 @@ implementation

uses
PascalParser, CppParser, EditorConfigParser,
FormatterBase, RTFFormatter, HTMLFormatter, Recognizers, TokenTypes, Parsers, ParseResult, Tokenizers;
FormatterBase, RTFFormatter, HTMLFormatter, Recognizers, TokenTypes,
Parsers, ParseResult, Tokenizers;

function CreateFormatter(FormatterType: TFormatterType;
OutputStream: TStream): TFormatterBase;
Expand All @@ -46,7 +47,8 @@ function CreateRecognizers(DocumentType: TDocumentType): TTokenRecognizers;
dtCpp: Result := CppParser.CreateRecognizers;
dtPascal: Result := PascalParser.CreateRecognizers;
dtEditorConfig: Result := EditorConfigParser.CreateRecognizers;
else raise Exception.Create('Not implemented!')
else
raise Exception.Create('Not implemented!')
end;
end;

Expand All @@ -56,7 +58,8 @@ function CreateParser(DocumentType: TDocumentType): TParser<TFmt>;
dtCpp: Result := CppParser.CreateParser;
dtPascal: Result := PascalParser.CreateParser;
dtEditorConfig: Result := EditorConfigParser.CreateParser;
else raise Exception.Create('Not implemented!')
else
raise Exception.Create('Not implemented!')
end;
end;

Expand Down
6 changes: 4 additions & 2 deletions HTMLFormatter.pas
Original file line number Diff line number Diff line change
Expand Up @@ -14,12 +14,14 @@ THTMLFormatter = class(TFormatterBase)
public
procedure WriteFooter; override;
procedure WriteHeader; override;
procedure WriteToken(const Token: String; const TokenType: THigherTokenType); override;
procedure WriteToken(const Token: String; const TokenType: THigherTokenType);
override;
end;

implementation

procedure THTMLFormatter.WriteToken(const Token: String; const TokenType: THigherTokenType);
procedure THTMLFormatter.WriteToken(const Token: String;
const TokenType: THigherTokenType);
var
escapedToken, FormatToken: String;
begin
Expand Down
6 changes: 4 additions & 2 deletions RTFFormatter.pas
Original file line number Diff line number Diff line change
Expand Up @@ -14,12 +14,14 @@ TRTFFormatter = class(TFormatterBase)
public
procedure WriteFooter; override;
procedure WriteHeader; override;
procedure WriteToken(const Token: String; const TokenType: THigherTokenType); override;
procedure WriteToken(const Token: String; const TokenType: THigherTokenType);
override;
end;

implementation

procedure TRTFFormatter.WriteToken(const Token: String; const TokenType: THigherTokenType);
procedure TRTFFormatter.WriteToken(const Token: String;
const TokenType: THigherTokenType);
var
escapedToken, FormatToken: String;
begin
Expand Down
2 changes: 1 addition & 1 deletion TokenTypes.pas
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@ interface
htString,
htSymbol,
htUnknown
);
);

TFmt = record
Text: String;
Expand Down
4 changes: 2 additions & 2 deletions binaryparsers.pas
Original file line number Diff line number Diff line change
Expand Up @@ -82,10 +82,10 @@ function TAbstractAndParser<L, R, T>.Parse(Source: TUndoTokenizer): TParseResult
begin
FLeft.Undo(Source, Left.Data);
Result := FailedParseResult<T>();
end
end;
end
else
Result := FailedParseResult<T>()
Result := FailedParseResult<T>();
end;

(* And *)
Expand Down
3 changes: 2 additions & 1 deletion codefmtparsers.pas
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,8 @@ function IdentifierRecognizer: TTokenRecognizer;

(* TListToFmtMapper *)

constructor TListToFmtMapper.Create(Parser: TParser<TTokenLinkedList>; Kind: THigherTokenType);
constructor TListToFmtMapper.Create(Parser: TParser<TTokenLinkedList>;
Kind: THigherTokenType);
begin
inherited Create(Parser);
FKind := Kind;
Expand Down
29 changes: 18 additions & 11 deletions cppparser.pas
Original file line number Diff line number Diff line change
Expand Up @@ -32,8 +32,10 @@ implementation
'unsigned', 'virtual', 'void', 'volatile', 'while');

type
TTokenType = (ttEol, ttWhiteSpace, ttDigits, ttPound, ttDoubleQuote, ttDoubleSlash, ttKeyword, ttIdentifier, ttUnknown);
TTokenType = (ttEol, ttWhiteSpace, ttDigits, ttPound, ttDoubleQuote,
ttDoubleSlash, ttKeyword, ttIdentifier, ttUnknown);
TTokenTypeSet = set of TTokenType;

const
AllTokenTypes: TTokenTypeSet = [ttEol..ttUnknown];

Expand All @@ -49,7 +51,8 @@ function CreateRecognizer(TokenType: TTokenType): TTokenRecognizer;
ttKeyword: Result := TKeywordRecognizer.Create(CppKeyWords, csSensitive);
ttIdentifier: Result := IdentifierRecognizer;
ttUnknown: Result := TAnyRecognizer.Create;
else raise Exception.Create('Unknown token type')
else
raise Exception.Create('Unknown token type')
end;
end;

Expand Down Expand Up @@ -88,15 +91,19 @@ function FilterTokens(TokenTypes: TTokenTypeSet): TParser<TToken>;

(* Simple Parser maps tokens almost as-is from one enum to another *)

function SimpleParser(TokenType: TTokenType; HigherTokenType: THigherTokenType): TParser<TFmt>; overload;
function SimpleParser(TokenType: TTokenType;
HigherTokenType: THigherTokenType): TParser<TFmt>; overload;
begin
Result := TListToFmtMapper.Create(MapTokenToList(FilterToken(TokenType)), HigherTokenType);
Result := TListToFmtMapper.Create(MapTokenToList(FilterToken(TokenType)),
HigherTokenType);
end;

function SimpleParser: TParser<TFmt>; overload;
var
SourceTokens: array of TTokenType = [ttEol, ttWhiteSpace, ttDigits, ttKeyword, ttIdentifier, ttUnknown];
DestTokens: array of THigherTokenType = [htCRLF, htSpace, htNumber, htKeyword, htIdentifier, htUnknown];
SourceTokens: array of TTokenType = [ttEol, ttWhiteSpace, ttDigits,
ttKeyword, ttIdentifier, ttUnknown];
DestTokens: array of THigherTokenType = [htCRLF, htSpace, htNumber,
htKeyword, htIdentifier, htUnknown];
i: Integer;
begin
Result := nil;
Expand All @@ -122,17 +129,17 @@ function SlashComments: TParser<TTokenLinkedList>;

function StringParser: TParser<TTokenLinkedList>;
begin
Result := Seq(
Seq(FilterToken(ttDoubleQuote), ManyTokens(FilterTokens(AllTokenTypes - [ttEol, ttDoubleQuote]))),
FilterToken(ttDoubleQuote)
);
Result := Seq(Seq(FilterToken(ttDoubleQuote),
ManyTokens(FilterTokens(AllTokenTypes - [ttEol, ttDoubleQuote]))),
FilterToken(ttDoubleQuote));
end;

// Pre-processor directive

function PreProcessorDirective: TParser<TTokenLinkedList>;
begin
Result := Seq(FilterToken(ttPound), ManyTokens(FilterTokens([ttIdentifier, ttKeyword])));
Result := Seq(FilterToken(ttPound),
ManyTokens(FilterTokens([ttIdentifier, ttKeyword])));
end;

function CreateParser: TParser<TFmt>;
Expand Down
38 changes: 19 additions & 19 deletions editorconfigparser.pas
Original file line number Diff line number Diff line change
Expand Up @@ -17,8 +17,10 @@ implementation
(* Recognizers *)

type
TTokenType = (ttEol, ttWhiteSpace, ttDigits, ttPound, ttLeftBracket, ttRightBracket, ttIdentifier, ttUnknown);
TTokenType = (ttEol, ttWhiteSpace, ttDigits, ttPound, ttLeftBracket,
ttRightBracket, ttIdentifier, ttUnknown);
TTokenTypeSet = set of TTokenType;

const
AllTokenTypes: TTokenTypeSet = [ttEol..ttUnknown];

Expand All @@ -33,7 +35,8 @@ function CreateRecognizer(TokenType: TTokenType): TTokenRecognizer;
ttRightBracket: Result := TSingleCharRecognizer.Create(']');
ttIdentifier: Result := IdentifierRecognizer;
ttUnknown: Result := TAnyRecognizer.Create;
else raise Exception.Create('Unknown token type')
else
raise Exception.Create('Unknown token type')
end;
end;

Expand Down Expand Up @@ -72,15 +75,19 @@ function FilterTokens(TokenTypes: TTokenTypeSet): TParser<TToken>;

(* Simple Parser maps tokens almost as-is from one enum to another *)

function SimpleParser(TokenType: TTokenType; HigherTokenType: THigherTokenType): TParser<TFmt>; overload;
function SimpleParser(TokenType: TTokenType;
HigherTokenType: THigherTokenType): TParser<TFmt>; overload;
begin
Result := TListToFmtMapper.Create(MapTokenToList(FilterToken(TokenType)), HigherTokenType);
Result := TListToFmtMapper.Create(MapTokenToList(FilterToken(TokenType)),
HigherTokenType);
end;

function SimpleParser: TParser<TFmt>; overload;
var
SourceTokens: array of TTokenType = [ttEol, ttWhiteSpace, ttDigits, ttIdentifier, ttUnknown];
DestTokens: array of THigherTokenType = [htCRLF, htSpace, htNumber, htIdentifier, htUnknown];
SourceTokens: array of TTokenType = [ttEol, ttWhiteSpace, ttDigits,
ttIdentifier, ttUnknown];
DestTokens: array of THigherTokenType = [htCRLF, htSpace, htNumber,
htIdentifier, htUnknown];
i: Integer;
begin
Result := nil;
Expand All @@ -93,15 +100,10 @@ function SimpleParser: TParser<TFmt>; overload;
function SectionParser: TParser<TFmt>;
begin
Result := TListToFmtMapper.Create(
Seq(
Seq(
FilterToken(ttLeftBracket),
ManyTokens(FilterTokens([ttWhiteSpace, ttDigits, ttIdentifier, ttUnknown]))
),
FilterToken(ttRightBracket)
),
htDirective
);
Seq(Seq(FilterToken(ttLeftBracket),
ManyTokens(FilterTokens([ttWhiteSpace, ttDigits, ttIdentifier,
ttUnknown]))),
FilterToken(ttRightBracket)), htDirective);
end;

function NoEol: TParser<TTokenLinkedList>;
Expand All @@ -111,10 +113,8 @@ function NoEol: TParser<TTokenLinkedList>;

function CommentParser: TParser<TFmt>;
begin
Result := TListToFmtMapper.Create(
Seq(FilterToken(ttPound), NoEol),
htComment
);
Result := TListToFmtMapper.Create(Seq(FilterToken(ttPound), NoEol),
htComment);
end;

function CreateParser: TParser<TFmt>;
Expand Down
8 changes: 4 additions & 4 deletions parsers.pas
Original file line number Diff line number Diff line change
Expand Up @@ -73,7 +73,7 @@ function OrElse<T>(Left: TParser<T>; Right: TParser<T>): TParser<T>;
else
Result := Left
else
Result := Right
Result := Right;
end;

(* Filter *)
Expand Down Expand Up @@ -102,10 +102,10 @@ function TFilterParser<T>.Parse(Source: TUndoTokenizer): TParseResult<T>;
begin
Undo(Source, Next.Data);
Result := FailedParseResult<T>();
end
end;
end
else
Result := Next
Result := Next;
end;

procedure TFilterParser<T>.Undo(Source: TUndoTokenizer; Data: T);
Expand Down Expand Up @@ -161,7 +161,7 @@ function TMapParser<T, U>.Parse(Source: TUndoTokenizer): TParseResult<U>;
if Next.Success then
Result := SuccessParseResult<U>(Map(Next.Data))
else
Result := FailedParseResult<U>()
Result := FailedParseResult<U>();
end;

end.
66 changes: 23 additions & 43 deletions pascalparser.pas
Original file line number Diff line number Diff line change
Expand Up @@ -51,8 +51,9 @@ implementation
ttKeyword,
ttIdentifier,
ttUnknown
);
);
TTokenTypeSet = set of TTokenType;

const
AllTokenTypes: TTokenTypeSet = [ttEol..ttUnknown];

Expand Down Expand Up @@ -84,12 +85,14 @@ function CreateRecognizer(TokenType: TTokenType): TTokenRecognizer;
ttAnsiCommentEnd: Result := TStringRecognizer.Create('*)');
ttBraceOpen: Result := TSingleCharRecognizer.Create('{');
ttBraceClose: Result := TSingleCharRecognizer.Create('}');
ttHexNumber: Result := TLeadingPredicateRecognizer.Create(IsDollarSign, IsHexDigit, 1);
ttHexNumber: Result := TLeadingPredicateRecognizer.Create(IsDollarSign,
IsHexDigit, 1);
ttChar: Result := TLeadingPredicateRecognizer.Create(IsPound, IsDigit);
ttKeyword: Result := TKeywordRecognizer.Create(PasKeyWords, csInsensitive);
ttIdentifier: Result := IdentifierRecognizer;
ttUnknown: Result := TAnyRecognizer.Create;
else raise Exception.Create('Unknown token type')
else
raise Exception.Create('Unknown token type')
end;
end;

Expand Down Expand Up @@ -128,33 +131,19 @@ function FilterTokens(TokenTypes: TTokenTypeSet): TParser<TToken>;

(* Simple Parser maps tokens almost as-is from one enum to another *)

function SimpleParser(TokenType: TTokenType; HigherTokenType: THigherTokenType): TParser<TFmt>; overload;
function SimpleParser(TokenType: TTokenType;
HigherTokenType: THigherTokenType): TParser<TFmt>; overload;
begin
Result := TListToFmtMapper.Create(MapTokenToList(FilterToken(TokenType)), HigherTokenType);
Result := TListToFmtMapper.Create(MapTokenToList(FilterToken(TokenType)),
HigherTokenType);
end;

function SimpleParser: TParser<TFmt>; overload;
var
SourceTokens: array of TTokenType = [
ttEol,
ttWhiteSpace,
ttDigits,
ttHexNumber,
ttChar,
ttKeyword,
ttIdentifier,
ttUnknown
];
DestTokens: array of THigherTokenType = [
htCRLF,
htSpace,
htNumber,
htNumber,
htString,
htKeyword,
htIdentifier,
htUnknown
];
SourceTokens: array of TTokenType = [ttEol, ttWhiteSpace,
ttDigits, ttHexNumber, ttChar, ttKeyword, ttIdentifier, ttUnknown];
DestTokens: array of THigherTokenType = [htCRLF, htSpace,
htNumber, htNumber, htString, htKeyword, htIdentifier, htUnknown];
i: Integer;
begin
Result := nil;
Expand All @@ -180,36 +169,27 @@ function SlashComments: TParser<TTokenLinkedList>;

function StringParser: TParser<TTokenLinkedList>;
begin
Result := Seq(
Seq(FilterToken(ttSingleQuote), ManyTokens(FilterTokens(AllTokenTypes - [ttEol, ttSingleQuote]))),
FilterToken(ttSingleQuote)
);
Result := Seq(Seq(FilterToken(ttSingleQuote),
ManyTokens(FilterTokens(AllTokenTypes - [ttEol, ttSingleQuote]))),
FilterToken(ttSingleQuote));
end;

// Ansi Comments

function AnsiCommentsParser: TParser<TTokenLinkedList>;
begin
Result := Seq(
Seq(
FilterToken(ttAnsiCommentBegin),
ManyTokens(FilterTokens(AllTokenTypes - [ttAnsiCommentEnd]))
),
FilterToken(ttAnsiCommentEnd)
);
Result := Seq(Seq(FilterToken(ttAnsiCommentBegin),
ManyTokens(FilterTokens(AllTokenTypes - [ttAnsiCommentEnd]))),
FilterToken(ttAnsiCommentEnd));
end;

// Borland Comments

function BorlandCommentsParser: TParser<TTokenLinkedList>;
begin
Result := Seq(
Seq(
FilterToken(ttBraceOpen),
ManyTokens(FilterTokens(AllTokenTypes - [ttBraceClose]))
),
FilterToken(ttBraceClose)
);
Result := Seq(Seq(FilterToken(ttBraceOpen),
ManyTokens(FilterTokens(AllTokenTypes - [ttBraceClose]))),
FilterToken(ttBraceClose));
end;

function CreateParser: TParser<TFmt>;
Expand Down
Loading