Interpreter/Lexer.cpp at master · Teagueporter/Interpreter · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
#include "Lexer.h"
#include "Token.h"
#include "CommaAutomaton.h"
#include "ColonAutomaton.h"
#include "ColonDashAutomaton.h"
#include "QueriesAutomaton.h"
#include "Q_markAutomaton.h"
#include "L_ParenAutomaton.h"
#include "R_Paren.h"
#include "AddAutomaton.h"
#include "MultiplyAutomaton.h"
#include "SchemesAutomaton.h"
#include "PeriodAutomaton.h"
#include "FactsAutomaton.h"
#include "RulesAutomaton.h"
#include "StringAutomaton.h"
#include "IdAutomaton.h"
#include "SCommentAutomaton.h"
#include "MCommentAutomaton.h"
#include "UndefinedBlockComment.h"
#include <cctype>
#include <sstream>

using namespace std;

Lexer::Lexer() {
    CreateAutomata();
}

Lexer::~Lexer() {
    // TODO: need to clean up the memory in `automata` and `tokens`

}

void Lexer::CreateAutomata() {
   automata.push_back(new ColonAutomaton());
   automata.push_back(new ColonDashAutomaton());
   automata.push_back(new CommaAutomaton());
   automata.push_back(new PeriodAutomaton());
   automata.push_back(new Q_markAutomaton());
   automata.push_back(new L_ParenAutomaton());
   automata.push_back(new R_ParenAutomaton());
   automata.push_back(new MultiplyAutomaton());
   automata.push_back(new AddAutomaton());
   automata.push_back(new RulesAutomaton());
   automata.push_back(new SchemesAutomaton());
   automata.push_back(new FactsAutomaton());
   automata.push_back(new QueriesAutomaton());
   automata.push_back(new IdAutomaton());
   automata.push_back(new StringAutomaton());
   automata.push_back(new UndefinedBlockAutomaton());
   automata.push_back(new SCommentAutomaton());
   automata.push_back(new MCommentAutomaton());

}

void Lexer::Run(std::string& input) {

    lineNumber = 1;
    int maxRead;
    int inputRead = 0;

    while(input.size() > 0){

        maxRead = 0;

        Automaton* maxAutomaton;
        maxAutomaton = automata.front();

        while(isspace(input[0])){
            if(input[0] == '\n') {
                lineNumber++;
            }
            input.erase(0,1);
        }
        for(size_t i = 0; i < automata.size(); ++i) {

            inputRead = automata.at(i)->Start(input);
            //inputRead += automata.at(i)->NewLinesRead();
            if(inputRead > maxRead) {
                maxRead = inputRead;
                maxAutomaton = automata.at(i);
            }
        }

        if (maxRead > 0){
            //determine if it is a comment and don't add it
            if(maxAutomaton->CreateToken(input.substr(0, maxRead), lineNumber)->getTokenType() != TokenType::COMMENT){
                tokens.push_back(maxAutomaton->CreateToken(input.substr(0, maxRead), lineNumber));
                lineNumber += maxAutomaton->NewLinesRead();
            }
        }

        else {
            if (input.size() > 0){
                maxRead = 1;
                tokens.push_back(new Token(TokenType::UNDEFINED, input.substr(0, maxRead), lineNumber));
            }
        }

        input.erase(0, maxRead);

    }

    //TODO: add eof token to the token list
    tokens.push_back(new Token(TokenType::_EOF, "", lineNumber));

}
string Lexer::toStringLexar(){
    stringstream oss;
    for(size_t i = 0; i < tokens.size(); ++i){
        oss << tokens.at(i)->toString() << endl;
    }
    oss << "Total Tokens = " << tokens.size();
    return oss.str();
}