using System.Collections; using System.Collections.Generic; using System.Text.RegularExpressions; using System.Text; using System.Globalization; using Godot; namespace Rokojori; public class Token:ASTNode { // public class PredicateData // { // public string type; // public string match; // public bool Matches( ASTNode node ) // { // return node.IsToken( type, match ); // } // public System.Predicate predicate // { // get // { // System.Predicate pred = ( tk ) => tk.Is( type, match ); // return pred; // } // } // public static PredicateData Create( string type, string match = null ) // { // var pd = new PredicateData(); // pd.type = type; // pd.match = match; // return pd; // } // public static TokenPredicateData Lexed( Lexer lexer, string value ) // { // var lexedSequence = CreateLexedSequenceData( lexer, value ); // return lexedSequence[ 0 ]; // } // } public LexerEvent lexerEvent; public string match => lexerEvent.match; public string type => lexerEvent.type; public override string ToString() { var matchInfo = match.Replace( "\n", "\\n" ).Replace( "\r", "\\r" ); return $"Token{{ '{matchInfo}' ({type}) }}"; } public bool Is( string type, string match = null ) { return lexerEvent.Is( type, match ); } public bool Is( LexerMatcher matcher, string match = null ) { return lexerEvent.Is( matcher.type, match ); } ASTFileRoot _root; public string lineInfo { get { if ( _root == null ) { _root = ASTWalker.instance.GetInParents( this, n => n is ASTFileRoot ) as ASTFileRoot; } var textLine = _root.GetTextLinesMapper().GetLine( lexerEvent.offset ); return "[ " + textLine.textEditorLineIndex + " ] | " + textLine.GetContent( _root.GetSource() ); } } public static Token Create( LexerEvent le, ASTNode parent = null ) { var t = new Token(); t.lexerEvent = le; t.parent = parent; return t; } public System.Predicate ToPredicate( Lexer lexer = null ) { var isVariableMatch = lexer == null ? Lexer.IsVariableMatchType( type ) : lexer.HasVariableMatch( type ); var match = isVariableMatch ? null : this.match; System.Predicate predicate = ( tk ) => tk.Is( type, match ); return predicate; } public TokenPredicateData ToPredicateData( Lexer lexer = null ) { var isVariableMatch = lexer == null ? Lexer.IsVariableMatchType( type ) : lexer.HasVariableMatch( type ); var match = isVariableMatch ? null : this.match; return TokenPredicateData.Create( type, match ); } public static System.Predicate CreateRawPredicate( LexerMatcher matcher, string match = null ) { System.Predicate predicate = ( tk ) => tk.Is( matcher, match ); return predicate; } public static List> CreateLexedSequence( Lexer lexer, string sequence ) { lexer.Clear(); var lexedSequence = lexer.LexToList( sequence ); if ( lexer.hasError ) { throw new System.Exception( "Lexer has error at char " + lexedSequence.Last().offset + " >> '" + sequence + "'" ); } lexedSequence.Pop(); lexer.GrabMatches( lexedSequence, sequence ); var filteredEvents = lexedSequence.Filter( le => ! le.IsAnyOf( LexerMatcherLibrary.Ignore ) ); // RJLog.Log( "Creating predicates:", "'" + sequence + "'", ">", filteredEvents.Count, ">>", filteredEvents.Map( e => e.type + " : " + e.match ).Join( ", " ) ); var tokenSequence = filteredEvents.Map( le => Create( le ).ToPredicate( lexer ) ); return tokenSequence; } public static List CreateLexedSequenceData( Lexer lexer, string sequence ) { lexer.Clear(); var lexedSequence = lexer.LexToList( sequence ); if ( lexer.hasError ) { throw new System.Exception( "Lexer has error at char " + lexedSequence.Last().offset + " >> '" + sequence + "'" ); } lexedSequence.Pop(); lexer.GrabMatches( lexedSequence, sequence ); var filteredEvents = lexedSequence.Filter( le => ! le.IsAnyOf( LexerMatcherLibrary.Ignore ) ); // RJLog.Log( "Creating predicates:", "'" + sequence + "'", ">", filteredEvents.Count, ">>", filteredEvents.Map( e => e.type + " : " + e.match ).Join( ", " ) ); var tokenSequence = filteredEvents.Map( le => Create( le ).ToPredicateData( lexer ) ); return tokenSequence; } }