Browse Source

Util: Implement GenericLexer in Json::Lexer

master
Riyyi 2 years ago
parent
commit
e3041bfdc7
  1. 49
      src/util/json/lexer.cpp
  2. 13
      src/util/json/lexer.h

49
src/util/json/lexer.cpp

@ -13,7 +13,8 @@
namespace Json { namespace Json {
Lexer::Lexer(Job* job) Lexer::Lexer(Job* job)
: m_job(job) : GenericLexer(job->input())
, m_job(job)
, m_tokens(job->tokens()) , m_tokens(job->tokens())
{ {
} }
@ -26,7 +27,7 @@ Lexer::~Lexer()
void Lexer::analyze() void Lexer::analyze()
{ {
while (m_index < m_job->input().length()) { while (m_index < m_input.length()) {
switch (peek()) { switch (peek()) {
case '{': case '{':
m_tokens->push_back({ Token::Type::BraceOpen, m_line, m_column, "{" }); m_tokens->push_back({ Token::Type::BraceOpen, m_line, m_column, "{" });
@ -100,7 +101,7 @@ void Lexer::analyze()
case '\t': case '\t':
break; break;
case '\r': case '\r':
if (peekNext() == '\n') { // CRLF \r\n if (peek(1) == '\n') { // CRLF \r\n
break; break;
} }
m_column = -1; m_column = -1;
@ -119,41 +120,13 @@ void Lexer::analyze()
break; break;
} }
increment(); ignore();
m_column++;
} }
} }
// ----------------------------------------- // -----------------------------------------
char Lexer::peek()
{
return m_job->input()[m_index];
}
char Lexer::peekNext()
{
return m_job->input()[m_index + 1];
}
void Lexer::increment()
{
m_index++;
m_column++;
}
void Lexer::decrement()
{
m_index--;
m_column--;
}
char Lexer::consume()
{
char character = peek();
increment();
return character;
}
bool Lexer::getString() bool Lexer::getString()
{ {
size_t column = m_column; size_t column = m_column;
@ -166,7 +139,7 @@ bool Lexer::getString()
if (!escape && character == '\\') { if (!escape && character == '\\') {
symbol += '\\'; symbol += '\\';
increment(); ignore();
escape = true; escape = true;
continue; continue;
} }
@ -180,7 +153,7 @@ bool Lexer::getString()
} }
symbol += character; symbol += character;
increment(); ignore();
if (escape) { if (escape) {
escape = false; escape = false;
@ -221,13 +194,13 @@ bool Lexer::getNumberOrLiteral(Token::Type type)
break; break;
} }
increment(); ignore();
} }
m_tokens->push_back({ type, m_line, column, m_tokens->push_back({ type, m_line, column,
m_job->input().substr(index, m_index - index) }); std::string(m_input.substr(index, m_index - index)) });
decrement(); retreat();
return true; return true;
} }

13
src/util/json/lexer.h

@ -11,10 +11,11 @@
// https://www.rfc-editor.org/rfc/pdfrfc/rfc8259.txt.pdf // https://www.rfc-editor.org/rfc/pdfrfc/rfc8259.txt.pdf
#include <cstddef> // size_t #include <cstddef> // size_t
#include <memory> // shared_ptr
#include <string> #include <string>
#include <vector> #include <vector>
#include "util/genericlexer.h"
namespace Json { namespace Json {
class Job; class Job;
@ -40,7 +41,7 @@ struct Token {
}; };
// Lexical analyzer // Lexical analyzer
class Lexer { class Lexer final : public Util::GenericLexer {
public: public:
Lexer(Job* job); Lexer(Job* job);
virtual ~Lexer(); virtual ~Lexer();
@ -48,13 +49,6 @@ public:
void analyze(); void analyze();
private: private:
char peek();
char peekNext();
void increment();
void decrement();
char consume();
bool getString(); bool getString();
bool getNumberOrLiteral(Token::Type type); bool getNumberOrLiteral(Token::Type type);
bool getNumber(); bool getNumber();
@ -62,7 +56,6 @@ private:
Job* m_job { nullptr }; Job* m_job { nullptr };
size_t m_index { 0 };
size_t m_column { 0 }; size_t m_column { 0 };
size_t m_line { 0 }; size_t m_line { 0 };

Loading…
Cancel
Save