diff --git a/src/yycc/carton/lexer61.cpp b/src/yycc/carton/lexer61.cpp index dff829a..db19253 100644 --- a/src/yycc/carton/lexer61.cpp +++ b/src/yycc/carton/lexer61.cpp @@ -2,11 +2,12 @@ namespace yycc::carton::lexer61 { - Lexer61::Lexer61() : m_ArgsCollection(), m_CurrentArg(), m_CurrentChar(u8'\0'), m_State(LexerState::Space), m_PrevState(LexerState::Space) {} + Lexer61::Lexer61() : + m_ArgsCollection(), m_CurrentArg(), m_CurrentChar(u8'\0'), m_State(LexerState::Space), m_PrevState(LexerState::Space) {} Lexer61::~Lexer61() {} - LexerResult> Lexer61::lex(const std::u8string_view &cmd) { + LexerResult> Lexer61::lex(const std::u8string_view &cmd) { // Clear variables when we start a new lex. this->reset(); @@ -67,20 +68,6 @@ namespace yycc::carton::lexer61 { } } - LexerResult> Lexer61::owend_lex(const std::u8string_view &cmd) { - auto rv = this->lex(cmd); - if (rv.has_value()) { - auto source = std::move(rv.value()); - std::vector elems; - for (const auto &strl_view : source) { - elems.emplace_back(std::u8string(strl_view)); - } - return elems; - } else { - std::unexpected(rv.error()); - } - } - void Lexer61::reset() { // Because these value may be moved, so we need assign them with new value, // rather clear them. diff --git a/src/yycc/carton/lexer61.hpp b/src/yycc/carton/lexer61.hpp index f64f43e..139a8df 100644 --- a/src/yycc/carton/lexer61.hpp +++ b/src/yycc/carton/lexer61.hpp @@ -24,8 +24,7 @@ namespace yycc::carton::lexer61 { YYCC_DEFAULT_COPY_MOVE(Lexer61) public: - LexerResult> lex(const std::u8string_view& cmd); - LexerResult> owend_lex(const std::u8string_view& cmd); + LexerResult> lex(const std::u8string_view& cmd); private: void reset(); @@ -36,11 +35,11 @@ namespace yycc::carton::lexer61 { void proc_escape(); void proc_normal(); - std::vector m_ArgsCollection; ///< Internal result holder. - std::u8string m_CurrentArg; ///< Holding current building commandline argument. - char8_t m_CurrentChar; ///< Holding current char analysing. - LexerState m_State; ///< Recording current state. - LexerState m_PrevState; ///< Recording previous state. + std::vector m_ArgsCollection; ///< Internal result holder. + std::u8string m_CurrentArg; ///< Holding current building commandline argument. + char8_t m_CurrentChar; ///< Holding current char analysing. + LexerState m_State; ///< Recording current state. + LexerState m_PrevState; ///< Recording previous state. }; } // namespace yycc::carton::lexer61 diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 3c72060..31e788a 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -42,6 +42,7 @@ PRIVATE yycc/carton/tabulate.cpp yycc/carton/clap.cpp yycc/carton/binstore.cpp + yycc/carton/lexer61.cpp yycc/carton/fft.cpp ) target_sources(YYCCTest diff --git a/test/yycc/carton/lexer61.cpp b/test/yycc/carton/lexer61.cpp new file mode 100644 index 0000000..1396ae5 --- /dev/null +++ b/test/yycc/carton/lexer61.cpp @@ -0,0 +1,166 @@ +#include +#include +#include + +#define LEXER61 ::yycc::carton::lexer61 + +namespace yycctest::carton::lexer61 { + + TEST(CartonLexer61, EmptyCommand) { + LEXER61::Lexer61 lexer; + auto result = lexer.lex(u8""); + ASSERT_TRUE(result.has_value()); + + auto value = std::move(result.value()); + EXPECT_TRUE(value.empty()); + } + + TEST(CartonLexer61, SingleArgument) { + LEXER61::Lexer61 lexer; + auto result = lexer.lex(u8"program"); + ASSERT_TRUE(result.has_value()); + + auto value = std::move(result.value()); + ASSERT_EQ(value.size(), 1); + EXPECT_EQ(value[0], u8"program"); + } + + TEST(CartonLexer61, MultipleArguments) { + LEXER61::Lexer61 lexer; + auto result = lexer.lex(u8"program arg1 arg2 arg3"); + ASSERT_TRUE(result.has_value()); + + auto value = std::move(result.value()); + ASSERT_EQ(value.size(), 4); + EXPECT_EQ(value[0], u8"program"); + EXPECT_EQ(value[1], u8"arg1"); + EXPECT_EQ(value[2], u8"arg2"); + EXPECT_EQ(value[3], u8"arg3"); + } + + TEST(CartonLexer61, LeadingSpaces) { + LEXER61::Lexer61 lexer; + auto result = lexer.lex(u8" program arg"); + ASSERT_TRUE(result.has_value()); + + auto value = std::move(result.value()); + ASSERT_EQ(value.size(), 2); + EXPECT_EQ(value[0], u8"program"); + EXPECT_EQ(value[1], u8"arg"); + } + + TEST(CartonLexer61, TrailingSpaces) { + LEXER61::Lexer61 lexer; + auto result = lexer.lex(u8"program arg "); + ASSERT_TRUE(result.has_value()); + + auto value = std::move(result.value()); + ASSERT_EQ(value.size(), 2); + EXPECT_EQ(value[0], u8"program"); + EXPECT_EQ(value[1], u8"arg"); + } + + TEST(CartonLexer61, EmbeddedSpaces) { + LEXER61::Lexer61 lexer; + auto result = lexer.lex(u8"program arg1 arg2"); + ASSERT_TRUE(result.has_value()); + + auto value = std::move(result.value()); + ASSERT_EQ(value.size(), 3); + EXPECT_EQ(value[0], u8"program"); + EXPECT_EQ(value[1], u8"arg1"); + EXPECT_EQ(value[2], u8"arg2"); + } + + TEST(CartonLexer61, SingleQuotedArgs) { + LEXER61::Lexer61 lexer; + auto result = lexer.lex(u8"program 'arg with spaces' arg2"); + ASSERT_TRUE(result.has_value()); + + auto value = std::move(result.value()); + ASSERT_EQ(value.size(), 3); + EXPECT_EQ(value[0], u8"program"); + EXPECT_EQ(value[1], u8"arg with spaces"); + EXPECT_EQ(value[2], u8"arg2"); + } + + TEST(CartonLexer61, DoubleQuotedArgs) { + LEXER61::Lexer61 lexer; + auto result = lexer.lex(u8R"(program "arg with spaces" arg2)"); + ASSERT_TRUE(result.has_value()); + + auto value = std::move(result.value()); + ASSERT_EQ(value.size(), 3); + EXPECT_EQ(value[0], u8"program"); + EXPECT_EQ(value[1], u8"arg with spaces"); + EXPECT_EQ(value[2], u8"arg2"); + } + + TEST(CartonLexer61, MixedQuotes) { + LEXER61::Lexer61 lexer; + auto result = lexer.lex(u8R"(program "double quoted 'single inside'" 'single quoted "double inside"')"); + ASSERT_TRUE(result.has_value()); + + auto value = std::move(result.value()); + ASSERT_EQ(value.size(), 3); + EXPECT_EQ(value[0], u8"program"); + EXPECT_EQ(value[1], u8"double quoted 'single inside'"); + EXPECT_EQ(value[2], u8"single quoted \"double inside\""); + } + + TEST(CartonLexer61, EscapedCharacters) { + LEXER61::Lexer61 lexer; + auto result = lexer.lex(u8R"(program escaped\ space "quoted with \" quote" 'single with \' quote')"); + ASSERT_TRUE(result.has_value()); + + auto value = std::move(result.value()); + ASSERT_EQ(value.size(), 4); + EXPECT_EQ(value[0], u8"program"); + EXPECT_EQ(value[1], u8"escaped space"); + EXPECT_EQ(value[2], u8"quoted with \" quote"); + EXPECT_EQ(value[3], u8"single with \' quote"); + } + + TEST(CartonLexer61, ChineseCharacters) { + LEXER61::Lexer61 lexer; + auto result = lexer.lex(u8"程序 中文 参数"); + ASSERT_TRUE(result.has_value()); + + auto value = std::move(result.value()); + ASSERT_EQ(value.size(), 3); + EXPECT_EQ(value[0], u8"程序"); + EXPECT_EQ(value[1], u8"中文"); + EXPECT_EQ(value[2], u8"参数"); + } + + TEST(CartonLexer61, ChineseWithQuotes) { + LEXER61::Lexer61 lexer; + auto result = lexer.lex(u8R"(程序 "中文 参数" '另一个"引号"参数')"); + ASSERT_TRUE(result.has_value()); + + auto value = std::move(result.value()); + ASSERT_EQ(value.size(), 3); + EXPECT_EQ(value[0], u8"程序"); + EXPECT_EQ(value[1], u8"中文 参数"); + EXPECT_EQ(value[2], u8"另一个\"引号\"参数"); + } + + TEST(CartonLexer61, UnclosedQuoteError) { + LEXER61::Lexer61 lexer; + auto result = lexer.lex(u8R"(program "unclosed quote)"); + ASSERT_FALSE(result.has_value()); + } + + TEST(CartonLexer61, EmptyArgumentInQuotes) { + LEXER61::Lexer61 lexer; + auto result = lexer.lex(u8R"(program "" '')"); + ASSERT_TRUE(result.has_value()); + + auto value = std::move(result.value()); + ASSERT_EQ(value.size(), 3); + EXPECT_EQ(value[0], u8"program"); + EXPECT_EQ(value[1], u8""); + EXPECT_EQ(value[2], u8""); + } + +} // namespace yycctest::carton::lexer61 \ No newline at end of file