46 lines
1.4 KiB
C#
46 lines
1.4 KiB
C#
|
using System.Collections;
|
|||
|
|
using System.Collections.Generic;
|
|||
|
|
using System.Text.RegularExpressions;
|
|||
|
|
|
|||
|
|
namespace Rokojori
|
|||
|
|
{
|
|||
|
|
public class WordLexer:Lexer
|
|||
|
|
{
|
|||
|
|
public static readonly LexerMatcher LineBreak = new LexerMatcher( "LineBreak", @"\r\n|\n|\r" );
|
|||
|
|
public static readonly LexerMatcher Space = new LexerMatcher( "Space", @"\s" );
|
|||
|
|
public static readonly LexerMatcher Number =new LexerMatcher( "Number", @"[+-]?\d{1,3}(?:[.,]\d{3})*(?:[.,]\d+)?" );
|
|||
|
|
public static readonly LexerMatcher Word = new LexerMatcher( "Word", @"\p{L}+(?:[-’']\p{L}+)*" );
|
|||
|
|
public static readonly LexerMatcher Currency = new LexerMatcher( "Currency", @"[\p{Sc}]\s?\d+(?:[.,]\d+)?" );
|
|||
|
|
public static readonly LexerMatcher Punctuation = new LexerMatcher( "Punctuation", @"[\p{P}]" );
|
|||
|
|
public static readonly LexerMatcher Other = new LexerMatcher( "Other", @"." );
|
|||
|
|
|
|||
|
|
public static List<LexerEvent> Lex( string source )
|
|||
|
|
{
|
|||
|
|
var lexer = new WordLexer();
|
|||
|
|
var events = lexer.LexToList( source );
|
|||
|
|
|
|||
|
|
if ( lexer.hasError )
|
|||
|
|
{
|
|||
|
|
return null;
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
events.ForEach( ev => { ev.GrabMatch( source ); } );
|
|||
|
|
return events;
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
public WordLexer()
|
|||
|
|
{
|
|||
|
|
|
|||
|
|
AddAllMatchers(
|
|||
|
|
LineBreak,
|
|||
|
|
Space,
|
|||
|
|
Number,
|
|||
|
|
Word,
|
|||
|
|
Currency,
|
|||
|
|
Punctuation,
|
|||
|
|
Other
|
|||
|
|
);
|
|||
|
|
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
}
|