From e8abb063ad0e8bbf8f150d52c8cb28bb699e46e4 Mon Sep 17 00:00:00 2001 From: Jacob Lifshay Date: Thu, 1 Jun 2017 23:24:42 -0700 Subject: [PATCH] json parser compiles --- .gitignore | 3 +- build/.cproject | 60 +- src/CMakeLists.txt | 3 +- src/generate_spirv_parser/CMakeLists.txt | 24 + .../generate_spirv_parser.cpp | 69 ++ src/json/CMakeLists.txt | 3 +- src/json/json.cpp | 95 ++- src/json/json.h | 149 +++- src/json/parser.cpp | 702 ++++++++++++++++++ src/json/parser.h | 243 ++++++ src/util/variant.cpp | 28 + src/util/variant.h | 31 +- 12 files changed, 1324 insertions(+), 86 deletions(-) create mode 100644 src/generate_spirv_parser/CMakeLists.txt create mode 100644 src/generate_spirv_parser/generate_spirv_parser.cpp create mode 100644 src/json/parser.cpp create mode 100644 src/json/parser.h diff --git a/.gitignore b/.gitignore index d56ce4a..9130b65 100644 --- a/.gitignore +++ b/.gitignore @@ -5,4 +5,5 @@ cmake_install.cmake CMakeCache.txt Makefile /build/.settings/ -/test-files/test.spv \ No newline at end of file +/test-files/test.spv +/build/compile_commands.json diff --git a/build/.cproject b/build/.cproject index a89bc8a..ae99ecc 100644 --- a/build/.cproject +++ b/build/.cproject @@ -5,54 +5,47 @@ - - + - + + - - - - - - - + + + + + + - - + + - - - - - + + + + + + + + + - - - - - - @@ -63,4 +56,17 @@ + + + + + + + + + + + + + diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 28989b5..a4c3944 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -22,4 +22,5 @@ cmake_minimum_required(VERSION 3.1 FATAL_ERROR) add_subdirectory(spirv) add_subdirectory(demo) add_subdirectory(json) -add_subdirectory(util) \ No newline at end of file +add_subdirectory(util) +add_subdirectory(generate_spirv_parser) \ No newline at end of file diff --git a/src/generate_spirv_parser/CMakeLists.txt b/src/generate_spirv_parser/CMakeLists.txt new file mode 100644 index 0000000..e2954a0 --- /dev/null +++ b/src/generate_spirv_parser/CMakeLists.txt @@ -0,0 +1,24 @@ +# Copyright 2017 Jacob Lifshay +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# +cmake_minimum_required(VERSION 3.1 FATAL_ERROR) +set(sources generate_spirv_parser.cpp) +add_executable(generate_spirv_parser ${sources}) +target_link_libraries(generate_spirv_parser util json) diff --git a/src/generate_spirv_parser/generate_spirv_parser.cpp b/src/generate_spirv_parser/generate_spirv_parser.cpp new file mode 100644 index 0000000..df3b453 --- /dev/null +++ b/src/generate_spirv_parser/generate_spirv_parser.cpp @@ -0,0 +1,69 @@ +/* + * Copyright 2017 Jacob Lifshay + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + */ +#include +#include "../json/json.h" +#include "../json/parser.h" + +namespace vulkan_cpu +{ +namespace generate_spirv_parser +{ +namespace ast = json::ast; + +int generate_spirv_parser_main(int argc, char **argv) +{ + std::string file_name; + if(argc >= 2) + file_name = argv[1]; + if(argc != 2 || (file_name.size() > 1 && file_name[0] == '-')) + { + std::cerr << "usage: " << argv[0] << " " << std::endl; + return 1; + } + try + { + const auto source = file_name == "-" ? json::source::load_stdin() : + json::source::load_file(std::move(file_name)); + auto value = json::parse(&source); + ast::write(std::cout, value); + std::cout << std::endl; + } + catch(json::parse_error &e) + { + std::cerr << "error: " << e.what() << std::endl; + return 1; + } + catch(std::exception &e) + { + std::cerr << "error: " << e.what() << std::endl; + return 1; + } + return 0; +} +} +} + +int main(int argc, char **argv) +{ + return vulkan_cpu::generate_spirv_parser::generate_spirv_parser_main(argc, argv); +} diff --git a/src/json/CMakeLists.txt b/src/json/CMakeLists.txt index 9666a76..4fd34f8 100644 --- a/src/json/CMakeLists.txt +++ b/src/json/CMakeLists.txt @@ -19,6 +19,7 @@ # SOFTWARE. # cmake_minimum_required(VERSION 3.1 FATAL_ERROR) -set(sources json.cpp) +set(sources json.cpp + parser.cpp) add_library(json STATIC ${sources}) target_link_libraries(json util) \ No newline at end of file diff --git a/src/json/json.cpp b/src/json/json.cpp index 8e2a938..cb213aa 100644 --- a/src/json/json.cpp +++ b/src/json/json.cpp @@ -337,24 +337,23 @@ void write_number(Write_Char write_char, } } -std::string number_value::to_string(std::string buffer_in, unsigned base) const +std::string number_value::append_double_to_string(double value, std::string buffer, unsigned base) { - buffer_in.clear(); - std::string retval = std::move(buffer_in); write_number( [&](char ch) { - retval += ch; + buffer += ch; }, value, base); - return retval; + return buffer; } -std::size_t number_value::to_string(char *output_buffer, - std::size_t output_buffer_size, - bool require_null_terminator, - unsigned base) const noexcept +std::size_t number_value::double_to_buffer(double value, + char *output_buffer, + std::size_t output_buffer_size, + bool require_null_terminator, + unsigned base) noexcept { if(output_buffer_size == 0) return 0; @@ -375,6 +374,84 @@ std::size_t number_value::to_string(char *output_buffer, return used_buffer_size; // report used buffer excluding the null terminator } +std::string number_value::append_unsigned_integer_to_string(std::uint64_t value, + std::string buffer, + unsigned base) +{ + write_unsigned_integer( + [&](char ch) + { + buffer += ch; + }, + value, + base); + return buffer; +} + +std::size_t number_value::unsigned_integer_to_buffer(std::uint64_t value, + char *output_buffer, + std::size_t output_buffer_size, + bool require_null_terminator, + unsigned base) noexcept +{ + if(output_buffer_size == 0) + return 0; + std::size_t used_buffer_size = 0; + std::size_t output_buffer_size_without_terminator = output_buffer_size; + if(require_null_terminator) + output_buffer_size_without_terminator--; + write_unsigned_integer( + [&](char ch) + { + if(used_buffer_size < output_buffer_size_without_terminator) + output_buffer[used_buffer_size++] = ch; + }, + value, + base); + if(used_buffer_size < output_buffer_size) + output_buffer[used_buffer_size] = '\0'; // add the null terminator if there is space + return used_buffer_size; // report used buffer excluding the null terminator +} + +std::string number_value::append_signed_integer_to_string(std::int64_t value, + std::string buffer, + unsigned base) +{ + write_unsigned_integer( + [&](char ch) + { + buffer += ch; + }, + value, + base); + return buffer; +} + +std::size_t number_value::signed_integer_to_buffer(std::int64_t value, + char *output_buffer, + std::size_t output_buffer_size, + bool require_null_terminator, + unsigned base) noexcept +{ + if(output_buffer_size == 0) + return 0; + std::size_t used_buffer_size = 0; + std::size_t output_buffer_size_without_terminator = output_buffer_size; + if(require_null_terminator) + output_buffer_size_without_terminator--; + write_unsigned_integer( + [&](char ch) + { + if(used_buffer_size < output_buffer_size_without_terminator) + output_buffer[used_buffer_size++] = ch; + }, + value, + base); + if(used_buffer_size < output_buffer_size) + output_buffer[used_buffer_size] = '\0'; // add the null terminator if there is space + return used_buffer_size; // report used buffer excluding the null terminator +} + void number_value::write(std::ostream &os, unsigned base) const { write_number( diff --git a/src/json/json.h b/src/json/json.h index 08b2e9c..d54fa90 100644 --- a/src/json/json.h +++ b/src/json/json.h @@ -31,6 +31,7 @@ #include #include #include +#include #include "../util/variant.h" namespace vulkan_cpu @@ -39,16 +40,12 @@ namespace json { namespace ast { -struct composite_value -{ - composite_value() = default; - virtual ~composite_value() = default; - virtual void write(std::ostream &os) const = 0; - virtual std::unique_ptr duplicate() const = 0; -}; - struct null_value final { + constexpr null_value() noexcept = default; + constexpr null_value(std::nullptr_t) noexcept + { + } void write(std::ostream &os) const; null_value duplicate() const noexcept { @@ -67,7 +64,8 @@ struct null_value final struct boolean_value final { bool value; - constexpr boolean_value(bool value) noexcept : value(value) + template ::value>::type> + constexpr boolean_value(T value) noexcept : value(value) { } void write(std::ostream &os) const; @@ -88,10 +86,11 @@ struct boolean_value final struct string_value final { std::string value; - string_value(std::string value) noexcept : value(std::move(value)) - { - } - string_value(const char *value) : value(std::move(value)) + template < + typename T, + typename = typename std::enable_if::value + && std::is_convertible::value>::type> + string_value(T value) noexcept : value(std::move(value)) { } static void write(std::ostream &os, const std::string &value); @@ -118,7 +117,10 @@ struct number_value final double value; static_assert(std::numeric_limits::is_iec559 && std::numeric_limits::radix == 2, "double is not a ieee754 float64"); - number_value(double value) noexcept : value(value) + template ::value + && !std::is_same::value>::type> + number_value(T value) noexcept : value(value) { } explicit operator std::string() const @@ -128,11 +130,67 @@ struct number_value final static constexpr unsigned max_base = 36; static constexpr unsigned min_base = 2; static constexpr unsigned default_base = 10; // the json spec only supports base 10 - std::string to_string(std::string buffer_in = {}, unsigned base = default_base) const; - std::size_t to_string(char *output_buffer, + std::string append_to_string(std::string buffer, unsigned base = default_base) const + { + return append_double_to_string(value, std::move(buffer), base); + } + std::string to_string(std::string buffer = {}, unsigned base = default_base) const + { + return double_to_string(value, std::move(buffer), base); + } + std::size_t to_buffer(char *output_buffer, std::size_t output_buffer_size, bool require_null_terminator = true, - unsigned base = default_base) const noexcept; + unsigned base = default_base) const noexcept + { + return double_to_buffer( + value, output_buffer, output_buffer_size, require_null_terminator, base); + } + static std::string append_unsigned_integer_to_string(std::uint64_t value, + std::string buffer, + unsigned base = default_base); + static std::string unsigned_integer_to_string(std::uint64_t value, + std::string buffer = {}, + unsigned base = default_base) + { + buffer.clear(); + return append_unsigned_integer_to_string(value, std::move(buffer), base); + } + static std::size_t unsigned_integer_to_buffer(std::uint64_t value, + char *output_buffer, + std::size_t output_buffer_size, + bool require_null_terminator = true, + unsigned base = default_base) noexcept; + static std::string append_signed_integer_to_string(std::int64_t value, + std::string buffer, + unsigned base = default_base); + static std::string signed_integer_to_string(std::int64_t value, + std::string buffer = {}, + unsigned base = default_base) + { + buffer.clear(); + return append_signed_integer_to_string(value, std::move(buffer), base); + } + static std::size_t signed_integer_to_buffer(std::int64_t value, + char *output_buffer, + std::size_t output_buffer_size, + bool require_null_terminator = true, + unsigned base = default_base) noexcept; + static std::string append_double_to_string(double value, + std::string buffer, + unsigned base = default_base); + static std::string double_to_string(double value, + std::string buffer = {}, + unsigned base = default_base) + { + buffer.clear(); + return append_double_to_string(value, std::move(buffer), base); + } + static std::size_t double_to_buffer(double value, + char *output_buffer, + std::size_t output_buffer_size, + bool require_null_terminator = true, + unsigned base = default_base) noexcept; void write(std::ostream &os, unsigned base = default_base) const; number_value duplicate() const noexcept { @@ -148,9 +206,44 @@ struct number_value final } }; +struct composite_value; + +class composite_value_pointer +{ +private: + std::shared_ptr value; + +public: + constexpr composite_value_pointer() noexcept = default; + template ::value>::type> + composite_value_pointer(std::shared_ptr value) noexcept : value(std::move(value)) + { + } + composite_value *operator->() const noexcept + { + return value.operator->(); + } + composite_value &operator*() const noexcept + { + return *value; + } +}; + typedef util:: - variant> - value; + variant value; + +struct composite_value +{ + composite_value() = default; + virtual ~composite_value() = default; + virtual void write(std::ostream &os) const = 0; + virtual composite_value_pointer duplicate() const = 0; + operator value() const + { + return duplicate(); + } +}; inline value duplicate(const value &v) { @@ -182,14 +275,14 @@ struct object final : public composite_value { } virtual void write(std::ostream &os) const override; - virtual std::unique_ptr duplicate() const override + virtual composite_value_pointer duplicate() const override { std::unordered_map new_values; for(auto &entry : values) { - new_values.emplace(std::get<0>(entry), ast::duplicate(std::get<1>(entry))); + new_values.emplace(std::get<0>(entry), ast::duplicate(std::get<1>(entry))); } - return std::unique_ptr(new object(std::move(new_values))); + return std::make_shared(std::move(new_values)); } }; @@ -203,13 +296,13 @@ struct array final : public composite_value { } virtual void write(std::ostream &os) const override; - virtual std::unique_ptr duplicate() const override + virtual composite_value_pointer duplicate() const override { - std::vector new_values; - new_values.reserve(values.size()); - for(auto &value : values) - new_values.emplace_back(ast::duplicate(value)); - return std::unique_ptr(new array(std::move(new_values))); + std::vector new_values; + new_values.reserve(values.size()); + for(auto &value : values) + new_values.emplace_back(ast::duplicate(value)); + return std::make_shared(std::move(new_values)); } }; } diff --git a/src/json/parser.cpp b/src/json/parser.cpp new file mode 100644 index 0000000..56a25de --- /dev/null +++ b/src/json/parser.cpp @@ -0,0 +1,702 @@ +/* +* Copyright 2017 Jacob Lifshay +* +* Permission is hereby granted, free of charge, to any person obtaining a copy +* of this software and associated documentation files (the "Software"), to deal +* in the Software without restriction, including without limitation the rights +* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +* copies of the Software, and to permit persons to whom the Software is +* furnished to do so, subject to the following conditions: +* +* The above copyright notice and this permission notice shall be included in all +* copies or substantial portions of the Software. +* +* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +* SOFTWARE. +* +*/ +#include "parser.h" +#include +#include +#include +#include +#include +#include "../util/soft_float.h" + +namespace vulkan_cpu +{ +namespace json +{ +namespace +{ +constexpr bool is_new_line(char ch) noexcept +{ + return ch == '\r' || ch == '\n'; +} + +constexpr bool is_new_line_pair(char ch1, char ch2) noexcept +{ + return ch1 == '\r' && ch2 == '\n'; +} + +template +void find_line_start_indexes_helper(Add_Index &&add_index, + const char *contents, + std::size_t contents_size) +{ + for(std::size_t i = 0; i < contents_size; i++) + { + char ch = contents[i]; + if(i + 1 < contents_size) + { + char ch2 = contents[i + 1]; + if(is_new_line_pair(ch, ch2)) + { + add_index(i + 2); + i++; + continue; + } + } + if(is_new_line(ch)) + add_index(i + 1); + } +} +} + +std::vector source::find_line_start_indexes(const char *contents, + std::size_t contents_size) +{ + std::size_t retval_size = 0; + find_line_start_indexes_helper( + [&](std::size_t) + { + retval_size++; + }, + contents, + contents_size); + std::vector retval; + retval.reserve(retval_size); + find_line_start_indexes_helper( + [&](std::size_t index) + { + retval.push_back(index); + }, + contents, + contents_size); + return retval; +} + +source source::load_file(std::string file_name) +{ + // TODO: add code to use mmap + std::ifstream is; + is.exceptions(std::ios::badbit | std::ios::failbit); + is.open(file_name); + std::vector buffer; + while(is.peek() != std::char_traits::eof()) + { + if(buffer.size() == buffer.capacity()) + buffer.reserve(buffer.size() * 2); + buffer.push_back(is.get()); + } + is.close(); + buffer.shrink_to_fit(); + std::size_t contents_size = buffer.size(); + auto buffer_ptr = std::make_shared>(std::move(buffer)); + std::shared_ptr contents(buffer_ptr, buffer_ptr->data()); + return source(std::move(file_name), std::move(contents), contents_size); +} + +source source::load_stdin() +{ + auto &is = std::cin; + is.clear(); + auto previous_exceptions = is.exceptions(); + std::vector buffer; + try + { + is.exceptions(std::ios::badbit | std::ios::failbit); + while(is.peek() != std::char_traits::eof()) + { + if(buffer.size() == buffer.capacity()) + buffer.reserve(buffer.size() * 2); + buffer.push_back(is.get()); + } + } + catch(...) + { + is.clear(); + is.exceptions(previous_exceptions); + } + is.clear(); + is.exceptions(previous_exceptions); + buffer.shrink_to_fit(); + std::size_t contents_size = buffer.size(); + auto buffer_ptr = std::make_shared>(std::move(buffer)); + std::shared_ptr contents(buffer_ptr, buffer_ptr->data()); + return source("stdin", std::move(contents), contents_size); +} + +std::ostream &operator<<(std::ostream &os, const source::line_and_column &v) +{ + os << v.to_string(); + return os; +} + +source::line_and_index source::get_line_and_start_index(std::size_t char_index) const noexcept +{ + std::size_t line = + 1 + line_start_indexes.size() + + (line_start_indexes.rbegin() - std::lower_bound(line_start_indexes.rbegin(), + line_start_indexes.rend(), + char_index, + std::greater())); + return line_and_index(line, line <= 1 ? 0 : line_start_indexes[line - 2]); +} + +namespace +{ +constexpr std::size_t get_column_after_tab(std::size_t column, std::size_t tab_size) noexcept +{ + return tab_size == 0 || column == 0 ? column + 1 : + column + (tab_size - (column - 1) % tab_size); +} +} + +source::line_and_column source::get_line_and_column(std::size_t char_index, + std::size_t tab_size) const noexcept +{ + auto line_and_start_index = get_line_and_start_index(char_index); + std::size_t column = 1; + for(std::size_t i = line_and_start_index.index; i < char_index; i++) + { + int ch = contents.get()[i]; + if(ch == '\t') + column = get_column_after_tab(column, tab_size); + else + column++; + } + return line_and_column(line_and_start_index.line, column); +} + +std::ostream &operator<<(std::ostream &os, const location &v) +{ + os << v.to_string(); + return os; +} + +namespace +{ +enum class token_type +{ + eof, + l_bracket, + r_bracket, + l_brace, + r_brace, + colon, + comma, + true_literal, + false_literal, + null_literal, + string, + number, +}; + +class tokenizer final +{ +private: + std::size_t input_char_index; + static constexpr int eof = std::char_traits::eof(); + +public: + const source *const source; + const parse_options options; + location token_location; + ast::value token_value; + token_type token_type; + +private: + int peekc() const noexcept + { + if(input_char_index >= source->contents_size) + return eof; + return static_cast(source->contents.get()[input_char_index]); + } + int getc() noexcept + { + int retval = peekc(); + input_char_index++; + return retval; + } + static constexpr bool is_digit(int ch, unsigned base = 10) noexcept + { + return get_digit_value(ch) >= 0; + } + static constexpr int get_digit_value(int ch, unsigned base = 10) noexcept + { + unsigned retval{}; + if(ch >= '0' && ch <= '9') + retval = ch - '0'; + else if(ch >= 'a' && ch <= 'z') + retval = ch - 'a' + 0xA; + else if(ch >= 'A' && ch <= 'Z') + retval = ch - 'A' + 0xA; + else + return -1; + if(retval >= base) + return -1; + return retval; + } + static constexpr bool is_letter(int ch) noexcept + { + return (ch >= 'a' && ch <= 'z') || (ch >= 'A' && ch <= 'Z'); + } + static constexpr bool is_control_character(int ch) noexcept + { + return ch >= 0 && ch < 0x20U; + } + static constexpr bool is_whitespace(int ch) noexcept + { + return ch == '\t' || ch == '\n' || ch == '\r' || ch == ' '; + } + static bool match_buffer_with_string(const char *buffer, + std::size_t buffer_size, + const char *string) noexcept + { + for(; buffer_size != 0 && *string; string++, buffer++, buffer_size--) + if(*string != *buffer) + return false; + if(*string || buffer_size != 0) + return false; + return true; + } + std::uint32_t parse_4_hex_digits() + { + std::uint32_t retval = 0; + for(std::size_t i = 0; i < 4; i++) + { + int digit_char = peekc(); + int digit_value = get_digit_value(digit_char, 0x10); + if(digit_value < 0) + throw parse_error(location(source, input_char_index), "missing hex digit"); + getc(); + retval <<= 4; + retval |= digit_value; + } + return retval; + } + static std::string append_utf8(std::string buffer, std::uint32_t ch) + { + if(ch < 0x80U) + { + buffer += static_cast(ch); + } + else if(ch < 0x800U) + { + buffer += static_cast((ch >> 6) | 0xC0U); + buffer += static_cast((ch & 0x3FU) | 0x80U); + } + else if(ch < 0x10000UL) + { + buffer += static_cast((ch >> 12) | 0xE0U); + buffer += static_cast(((ch >> 6) & 0x3FU) | 0x80U); + buffer += static_cast((ch & 0x3FU) | 0x80U); + } + else + { + buffer += static_cast(((ch >> 18) & 0x7U) | 0xF0U); + buffer += static_cast(((ch >> 12) & 0x3FU) | 0xE0U); + buffer += static_cast(((ch >> 6) & 0x3FU) | 0x80U); + buffer += static_cast((ch & 0x3FU) | 0x80U); + } + return buffer; + } + +public: + tokenizer(const json::source *source, parse_options options) + : input_char_index(0), source(source), options(options), token_location(), token_value() + { + next(); + } + ast::value get() + { + auto retval = token_value; + next(); + return retval; + } + void next() + { + while(is_whitespace(peekc())) + getc(); + token_location = location(source, input_char_index); + token_value = nullptr; + bool got_minus = false, got_plus = false; + if(peekc() == '-') + { + getc(); + got_minus = true; + } + else if(options.allow_explicit_plus_sign_in_mantissa && peekc() == '+') + { + getc(); + got_plus = true; + } + if(is_letter(peekc())) + { + const char *name = source->contents.get() + input_char_index; + std::size_t name_size = 0; + while(is_letter(peekc()) || is_digit(peekc())) + { + getc(); + name_size++; + } + if(!got_minus && !got_plus) + { + if(match_buffer_with_string(name, name_size, "null")) + { + token_value = nullptr; + token_type = json::token_type::null_literal; + return; + } + if(match_buffer_with_string(name, name_size, "false")) + { + token_value = false; + token_type = json::token_type::false_literal; + return; + } + if(match_buffer_with_string(name, name_size, "true")) + { + token_value = true; + token_type = json::token_type::true_literal; + return; + } + } + if(options.allow_infinity_and_nan) + { + if(match_buffer_with_string(name, name_size, "NaN") + || match_buffer_with_string(name, name_size, "nan") + || match_buffer_with_string(name, name_size, "NAN")) + { + token_value = std::numeric_limits::quiet_NaN(); + token_type = json::token_type::number; + return; + } + if(match_buffer_with_string(name, name_size, "Infinity") + || match_buffer_with_string(name, name_size, "INFINITY") + || match_buffer_with_string(name, name_size, "infinity") + || match_buffer_with_string(name, name_size, "inf") + || match_buffer_with_string(name, name_size, "INF")) + { + token_value = got_minus ? -std::numeric_limits::infinity() : + std::numeric_limits::infinity(); + token_type = json::token_type::number; + return; + } + } + throw parse_error(token_location, + (got_minus || got_plus ? "invalid number: " : "invalid identifier: ") + + std::string(name, name_size)); + } + if(got_minus || got_plus || is_digit(peekc()) + || (options.allow_number_to_start_with_dot && peekc() == '.')) + { + auto mantissa = util::soft_float::ExtendedFloat::Zero(); + bool got_any_digit = false; + if(is_digit(peekc())) + { + if(peekc() == '0') + { + getc(); + got_any_digit = true; + if(is_digit(peekc())) + throw parse_error(location(source, input_char_index), + "extra leading zero not allowed in numbers"); + } + else + { + while(is_digit(peekc())) + { + std::int64_t digit = get_digit_value(getc()); + got_any_digit = true; + mantissa *= util::soft_float::ExtendedFloat(static_cast(10)); + mantissa += util::soft_float::ExtendedFloat(digit); + } + } + } + std::int64_t exponent_offset = 0; + if(peekc() == '.') + { + getc(); + while(is_digit(peekc())) + { + std::int64_t digit = get_digit_value(getc()); + got_any_digit = true; + mantissa *= util::soft_float::ExtendedFloat(static_cast(10)); + exponent_offset--; + mantissa += util::soft_float::ExtendedFloat(digit); + } + } + if(!got_any_digit) + throw parse_error(location(source, input_char_index), "missing digit"); + std::int64_t exponent = 0; + if(peekc() == 'e' || peekc() == 'E') + { + getc(); + bool exponent_is_negative = false; + if(peekc() == '-') + { + exponent_is_negative = true; + getc(); + } + else if(peekc() == '+') + { + getc(); + } + if(!is_digit(peekc())) + throw parse_error(location(source, input_char_index), "missing digit"); + while(is_digit(peekc())) + { + exponent *= 10; + exponent += get_digit_value(getc()); + } + if(exponent_is_negative) + exponent = -exponent; + } + exponent += exponent_offset; + auto value = + mantissa + * pow(util::soft_float::ExtendedFloat(static_cast(10)), exponent); + token_type = json::token_type::number; + token_value = static_cast(value); + return; + } + if(peekc() == '\"' || (options.allow_single_quote_strings && peekc() == '\'')) + { + int quote = getc(); + std::string value; + while(true) + { + if(peekc() == eof || is_control_character(peekc())) + throw parse_error(token_location, "string missing closing quote"); + if(peekc() == quote) + { + getc(); + break; + } + if(peekc() == '\\') + { + auto escape_location = location(source, input_char_index); + getc(); + switch(peekc()) + { + case '\"': + case '\\': + case '/': + value += getc(); + break; + case 'b': + value += '\b'; + getc(); + break; + case 'f': + value += '\f'; + getc(); + break; + case 'n': + value += '\n'; + getc(); + break; + case 'r': + value += '\r'; + getc(); + break; + case 't': + value += '\t'; + getc(); + break; + case 'u': + { + getc(); + std::uint32_t ch = parse_4_hex_digits(); + if(ch >= 0xD800U && ch < 0xDC00U && peekc() == '\\') + { + escape_location = location(source, input_char_index); + getc(); + if(peekc() == 'u') + { + getc(); + std::uint32_t ch2 = parse_4_hex_digits(); + if(ch2 >= 0xDC00U && ch2 < 0xE000U) + { + // got surrogate pair + ch = ((ch & 0x3FFU) >> 10) + (ch2 & 0x3FFU) + 0x10000UL; + } + else + { + input_char_index = escape_location.char_index; + } + } + else + { + input_char_index = escape_location.char_index; + } + } + value = append_utf8(std::move(value), ch); + break; + } + default: + if(options.allow_single_quote_strings && peekc() == '\'') + { + value += getc(); + break; + } + throw parse_error(escape_location, "invalid escape sequence"); + } + } + else + { + value += getc(); + } + } + token_type = json::token_type::string; + token_value = std::move(value); + return; + } + switch(peekc()) + { + case eof: + token_type = json::token_type::eof; + token_value = nullptr; + return; + case '[': + getc(); + token_type = json::token_type::l_bracket; + token_value = nullptr; + return; + case ']': + getc(); + token_type = json::token_type::r_bracket; + token_value = nullptr; + return; + case '{': + getc(); + token_type = json::token_type::l_brace; + token_value = nullptr; + return; + case '}': + getc(); + token_type = json::token_type::r_brace; + token_value = nullptr; + return; + case ':': + getc(); + token_type = json::token_type::colon; + token_value = nullptr; + return; + case ',': + getc(); + token_type = json::token_type::comma; + token_value = nullptr; + return; + } + throw parse_error(token_location, "invalid character"); + } +}; + +ast::value parse_value(tokenizer &tokenizer) +{ + switch(tokenizer.token_type) + { + case token_type::eof: + throw parse_error(tokenizer.token_location, "missing value"); + case token_type::number: + case token_type::string: + case token_type::true_literal: + case token_type::false_literal: + case token_type::null_literal: + return tokenizer.get(); + case token_type::l_bracket: + { + std::vector values; + tokenizer.next(); + if(tokenizer.token_type == token_type::r_bracket) + { + tokenizer.next(); + } + else + { + while(true) + { + values.push_back(parse_value(tokenizer)); + if(tokenizer.token_type == token_type::comma) + { + tokenizer.next(); + continue; + } + if(tokenizer.token_type == token_type::r_bracket) + { + tokenizer.next(); + break; + } + throw parse_error(tokenizer.token_location, "missing , or ]"); + } + } + return ast::array(std::move(values)); + } + case token_type::l_brace: + { + std::unordered_map values; + tokenizer.next(); + if(tokenizer.token_type == token_type::r_brace) + { + tokenizer.next(); + } + else + { + while(true) + { + if(tokenizer.token_type != token_type::string) + throw parse_error(tokenizer.token_location, "missing string"); + auto string_value = std::move(util::get(tokenizer.get()).value); + if(tokenizer.token_type != token_type::colon) + throw parse_error(tokenizer.token_location, "missing ':'"); + tokenizer.next(); + values.emplace(std::move(string_value), parse_value(tokenizer)); + if(tokenizer.token_type == token_type::comma) + { + tokenizer.next(); + continue; + } + if(tokenizer.token_type == token_type::r_brace) + { + tokenizer.next(); + break; + } + throw parse_error(tokenizer.token_location, "missing ',' or '}'"); + } + } + return ast::object(std::move(values)); + } + default: + break; + } + throw parse_error(tokenizer.token_location, "token not allowed here"); +} +} + +ast::value parse(const source *source, parse_options options) +{ + tokenizer tokenizer(source, options); + auto retval = parse_value(tokenizer); + if(tokenizer.token_type != token_type::eof) + throw parse_error(tokenizer.token_location, "unexpected token"); + return retval; +} +} +} diff --git a/src/json/parser.h b/src/json/parser.h new file mode 100644 index 0000000..747a404 --- /dev/null +++ b/src/json/parser.h @@ -0,0 +1,243 @@ +/* + * Copyright 2017 Jacob Lifshay + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + */ + +#ifndef JSON_PARSER_H_ +#define JSON_PARSER_H_ + +#include +#include +#include +#include +#include +#include "json.h" +#include "../util/optional.h" + +namespace vulkan_cpu +{ +namespace json +{ +struct source +{ + std::string file_name; + std::shared_ptr contents; // use a shared_ptr so you can use mmap-ed memory + std::size_t contents_size; + /** doesn't have first line to save memory */ + std::vector line_start_indexes; + static std::vector find_line_start_indexes(const char *contents, + std::size_t contents_size); + source(source &&) = default; + source(const source &) = delete; + source &operator=(source &&) = default; + source &operator=(const source &) = delete; + source() : file_name(), contents(), contents_size(0), line_start_indexes() + { + } + explicit source(std::string file_name) noexcept : file_name(std::move(file_name)), + contents(), + contents_size(0), + line_start_indexes() + { + } + source(std::string file_name, + std::shared_ptr contents, + std::size_t contents_size) noexcept + : file_name(std::move(file_name)), + contents(std::move(contents)), + contents_size(contents_size), + line_start_indexes(find_line_start_indexes(this->contents.get(), contents_size)) + { + } + source(std::string file_name, std::string contents_in) + : file_name(file_name), + contents(), + contents_size(contents_in.size()), + line_start_indexes(find_line_start_indexes(contents_in.data(), contents_size)) + { + auto str = std::make_shared(std::move(contents_in)); + contents = std::shared_ptr(str, str->data()); + } + source(std::string file_name, std::vector contents_in) + : file_name(file_name), + contents(), + contents_size(contents_in.size()), + line_start_indexes(find_line_start_indexes(contents_in.data(), contents_size)) + { + auto str = std::make_shared>(std::move(contents_in)); + contents = std::shared_ptr(str, str->data()); + } + source(std::string file_name, std::vector contents_in) + : file_name(file_name), + contents(), + contents_size(contents_in.size()), + line_start_indexes(find_line_start_indexes( + reinterpret_cast(contents_in.data()), contents_size)) + { + auto str = std::make_shared>(std::move(contents_in)); + contents = std::shared_ptr(str, reinterpret_cast(str->data())); + } + explicit operator bool() const noexcept + { + return contents != nullptr; + } + static source load_file(std::string file_name); + static source load_stdin(); + struct line_and_index + { + std::size_t line; + std::size_t index; + constexpr line_and_index() noexcept : line(), index() + { + } + constexpr line_and_index(std::size_t line, std::size_t index) noexcept : line(line), + index(index) + { + } + }; + struct line_and_column + { + std::size_t line; + std::size_t column; + constexpr line_and_column() noexcept : line(), column() + { + } + constexpr line_and_column(std::size_t line, std::size_t column) noexcept : line(line), + column(column) + { + } + std::string append_to_string(std::string buffer) const + { + buffer = ast::number_value::append_unsigned_integer_to_string(line, std::move(buffer)); + buffer += ':'; + buffer = + ast::number_value::append_unsigned_integer_to_string(column, std::move(buffer)); + return buffer; + } + std::string to_string(std::string buffer = {}) const + { + buffer.clear(); + return append_to_string(std::move(buffer)); + } + friend std::ostream &operator<<(std::ostream &os, const line_and_column &v); + }; + static constexpr std::size_t default_tab_size = 8; + line_and_index get_line_and_start_index(std::size_t char_index) const noexcept; + line_and_column get_line_and_column(std::size_t char_index, + std::size_t tab_size = default_tab_size) const noexcept; +}; + +struct location +{ + const source *source; + std::size_t char_index; + constexpr location() noexcept : source(nullptr), char_index(0) + { + } + constexpr location(const json::source *source, std::size_t char_index) noexcept + : source(source), + char_index(char_index) + { + } + json::source::line_and_index get_line_and_start_index() const noexcept + { + if(!source) + return {}; + return source->get_line_and_start_index(char_index); + } + json::source::line_and_column get_line_and_column( + std::size_t tab_size = json::source::default_tab_size) const noexcept + { + if(!source) + return {}; + return source->get_line_and_column(char_index, tab_size); + } + std::string to_string(std::string buffer = {}, + std::size_t tab_size = json::source::default_tab_size) const + { + buffer.clear(); + return append_to_string(std::move(buffer)); + } + std::string append_to_string(std::string buffer, + std::size_t tab_size = json::source::default_tab_size) const + { + if(!source || source->file_name.empty()) + buffer += ""; + else + buffer += source->file_name; + buffer += ':'; + buffer = get_line_and_column(tab_size).append_to_string(std::move(buffer)); + return buffer; + } + friend std::ostream &operator<<(std::ostream &os, const location &v); +}; + +class parse_error : public std::runtime_error +{ +public: + location location; + parse_error(json::location location, const std::string &message) + : runtime_error(location.to_string() + ": " + message) + { + } + parse_error(json::location location, const char *message) + : runtime_error(location.to_string() + ": " + message) + { + } +}; + +struct parse_options +{ + bool allow_infinity_and_nan; + bool allow_explicit_plus_sign_in_mantissa; + bool allow_single_quote_strings; + bool allow_number_to_start_with_dot; + constexpr parse_options() noexcept : allow_infinity_and_nan(false), + allow_explicit_plus_sign_in_mantissa(false), + allow_single_quote_strings(false), + allow_number_to_start_with_dot(false) + { + } + constexpr parse_options(bool allow_infinity_and_nan, + bool allow_explicit_plus_sign_in_mantissa, + bool allow_single_quote_strings, + bool allow_number_to_start_with_dot) noexcept + : allow_infinity_and_nan(allow_infinity_and_nan), + allow_explicit_plus_sign_in_mantissa(allow_explicit_plus_sign_in_mantissa), + allow_single_quote_strings(allow_single_quote_strings), + allow_number_to_start_with_dot(allow_number_to_start_with_dot) + { + } + static constexpr parse_options default_options() noexcept + { + return parse_options(); + } + static constexpr parse_options relaxed_options() noexcept + { + return parse_options(true, true, true, true); + } +}; + +ast::value parse(const source *source, parse_options options = parse_options::default_options()); +} +} + +#endif /* JSON_PARSER_H_ */ \ No newline at end of file diff --git a/src/util/variant.cpp b/src/util/variant.cpp index 5854282..eb5d0c7 100644 --- a/src/util/variant.cpp +++ b/src/util/variant.cpp @@ -21,3 +21,31 @@ * */ #include "variant.h" + +#if 0 +#include +#include +#include +namespace vulkan_cpu +{ +namespace util +{ +namespace +{ +void test() +{ + variant v = "abc"; +} + +struct Test +{ + Test() + { + test(); + std::exit(0); + } +} tester; +} +} +} +#endif diff --git a/src/util/variant.h b/src/util/variant.h index 118629c..3e49e29 100644 --- a/src/util/variant.h +++ b/src/util/variant.h @@ -670,21 +670,6 @@ VULKAN_CPU_UTIL_VARIANT_DISPATCH(, &&) VULKAN_CPU_UTIL_VARIANT_DISPATCH(const, &&) #undef VULKAN_CPU_UTIL_VARIANT_DISPATCH -template ::fn(std::declval()))::type, - std::size_t Index = variant_values::template index_from_type(), - typename = typename std::enable_if<(Index < sizeof...(Types))>::type> -constexpr std::size_t variant_conversion_deduce_index() noexcept -{ - return Index; -} - -template -using variant_conversion_deduce_type = - variant_alternative_t(), variant>; - template struct variant_index_type { @@ -948,9 +933,13 @@ public: } template < typename T, - std::size_t Index = detail::variant_conversion_deduce_index(), + typename Deduced_Type = typename decltype( + detail::variant_hypothetical_overload_set::fn(std::declval()))::type, + std::size_t Index = + detail::variant_values::template index_from_type(), typename = typename std:: - enable_if::type, variant>::value + enable_if<(Index < sizeof...(Types)) + && !std::is_same::type, variant>::value && !detail::variant_is_in_place_index::type>::value && !detail::variant_is_in_place_type::type>::value && std::is_constructible>, @@ -1033,9 +1022,13 @@ public: } template < typename T, - std::size_t Index = detail::variant_conversion_deduce_index(), + typename Deduced_Type = typename decltype( + detail::variant_hypothetical_overload_set::fn(std::declval()))::type, + std::size_t Index = + detail::variant_values::template index_from_type(), typename = typename std:: - enable_if::type, variant>::value + enable_if<(Index < sizeof...(Types)) + && !std::is_same::type, variant>::value && !detail::variant_is_in_place_index::type>::value && !detail::variant_is_in_place_type::type>::value && std::is_constructible>, -- 2.30.2