46 lines
1.4 KiB
C#
46 lines
1.4 KiB
C#
using System.Collections;
|
||
using System.Collections.Generic;
|
||
using System.Text.RegularExpressions;
|
||
|
||
namespace Rokojori
|
||
{
|
||
public class WordLexer:Lexer
|
||
{
|
||
public static readonly LexerMatcher LineBreak = new LexerMatcher( "LineBreak", @"\r\n|\n|\r" );
|
||
public static readonly LexerMatcher Space = new LexerMatcher( "Space", @"\s" );
|
||
public static readonly LexerMatcher Number =new LexerMatcher( "Number", @"[+-]?\d{1,3}(?:[.,]\d{3})*(?:[.,]\d+)?" );
|
||
public static readonly LexerMatcher Word = new LexerMatcher( "Word", @"\p{L}+(?:[-’']\p{L}+)*" );
|
||
public static readonly LexerMatcher Currency = new LexerMatcher( "Currency", @"[\p{Sc}]\s?\d+(?:[.,]\d+)?" );
|
||
public static readonly LexerMatcher Punctuation = new LexerMatcher( "Punctuation", @"[\p{P}]" );
|
||
public static readonly LexerMatcher Other = new LexerMatcher( "Other", @"." );
|
||
|
||
public static List<LexerEvent> Lex( string source )
|
||
{
|
||
var lexer = new WordLexer();
|
||
var events = lexer.LexToList( source );
|
||
|
||
if ( lexer.hasError )
|
||
{
|
||
return null;
|
||
}
|
||
|
||
events.ForEach( ev => { ev.GrabMatch( source ); } );
|
||
return events;
|
||
}
|
||
|
||
public WordLexer()
|
||
{
|
||
|
||
AddAllMatchers(
|
||
LineBreak,
|
||
Space,
|
||
Number,
|
||
Word,
|
||
Currency,
|
||
Punctuation,
|
||
Other
|
||
);
|
||
|
||
}
|
||
}
|
||
} |