From 7960b6bfaa99391c4980574016a150653611b99a Mon Sep 17 00:00:00 2001 From: Jacob Lifshay Date: Thu, 29 Jun 2017 00:29:08 -0700 Subject: [PATCH] add code to parse extinst json files; still need to add to generate.cpp --- src/generate_spirv_parser/ast.cpp | 60 +++++- src/generate_spirv_parser/ast.h | 63 ++++++- .../generate_spirv_parser.cpp | 19 +- src/generate_spirv_parser/parser.cpp | 176 ++++++++++++++++-- src/generate_spirv_parser/parser.h | 5 +- src/spirv/CMakeLists.txt | 11 +- 6 files changed, 292 insertions(+), 42 deletions(-) diff --git a/src/generate_spirv_parser/ast.cpp b/src/generate_spirv_parser/ast.cpp index 12dc6f1..5619228 100644 --- a/src/generate_spirv_parser/ast.cpp +++ b/src/generate_spirv_parser/ast.cpp @@ -28,6 +28,11 @@ namespace generate_spirv_parser { namespace ast { +constexpr const char *Extension_instruction_set::json_file_name_prefix; +constexpr const char *Extension_instruction_set::json_file_name_suffix; + +constexpr const char *Top_level::core_grammar_json_file_name; + namespace { std::string to_hex_string(std::uint32_t v, std::size_t min_digit_count) @@ -201,18 +206,53 @@ json::ast::Value Operand_kinds::to_json() const return json::ast::Value(make_empty_location(), std::move(retval)); } -json::ast::Value Top_level::to_json() const +util::optional Extension_instruction_set::get_import_name_from_instruction_set_name( + util::string_view instruction_set_name) { - json::ast::Object retval; - retval.values["copyright"] = copyright.to_json(); - retval.values["magic_number"] = + if(instruction_set_name == "glsl.std.450") + return "GLSL.std.450"; + if(instruction_set_name == "opencl.std.100") + return "OpenCL.std"; + return {}; +} + +Json_file Extension_instruction_set::to_json() const +{ + json::ast::Object grammar; + grammar.values["copyright"] = copyright.to_json(); + grammar.values["version"] = json::ast::Value(make_empty_location(), version); + grammar.values["revision"] = json::ast::Value(make_empty_location(), revision); + grammar.values["instructions"] = instructions.to_json(); + auto file_name = json_file_name_prefix + instruction_set_name + json_file_name_suffix; + for(char &ch : file_name) + { + if(ch >= 'A' && ch <= 'Z') + ch = ch - 'A' + 'a'; // to lower + } + return Json_file(std::move(file_name), + json::ast::Value(make_empty_location(), std::move(grammar)), + import_name); +} + +std::vector Top_level::to_json() const +{ + json::ast::Object core_grammar; + core_grammar.values["copyright"] = copyright.to_json(); + core_grammar.values["magic_number"] = json::ast::Value(make_empty_location(), to_hex_string(magic_number, 8)); - retval.values["major_version"] = json::ast::Value(make_empty_location(), major_version); - retval.values["minor_version"] = json::ast::Value(make_empty_location(), minor_version); - retval.values["revision"] = json::ast::Value(make_empty_location(), revision); - retval.values["instructions"] = instructions.to_json(); - retval.values["operand_kinds"] = operand_kinds.to_json(); - return json::ast::Value(make_empty_location(), std::move(retval)); + core_grammar.values["major_version"] = json::ast::Value(make_empty_location(), major_version); + core_grammar.values["minor_version"] = json::ast::Value(make_empty_location(), minor_version); + core_grammar.values["revision"] = json::ast::Value(make_empty_location(), revision); + core_grammar.values["instructions"] = instructions.to_json(); + core_grammar.values["operand_kinds"] = operand_kinds.to_json(); + std::vector retval; + retval.reserve(extension_instruction_sets.size() + 1); + retval.push_back(Json_file("spirv.core.grammar.json", + json::ast::Value(make_empty_location(), std::move(core_grammar)), + util::nullopt)); + for(auto &extension_instruction_set : extension_instruction_sets) + retval.push_back(extension_instruction_set.to_json()); + return retval; } } } diff --git a/src/generate_spirv_parser/ast.h b/src/generate_spirv_parser/ast.h index e664051..fe4ece8 100644 --- a/src/generate_spirv_parser/ast.h +++ b/src/generate_spirv_parser/ast.h @@ -475,8 +475,60 @@ struct Operand_kinds } }; +struct Json_file +{ + std::string file_name; + json::ast::Value json; + util::optional extension_instruction_set_import_name; + Json_file(std::string file_name, + json::ast::Value json, + util::optional extension_instruction_set_import_name) + : file_name(std::move(file_name)), + json(std::move(json)), + extension_instruction_set_import_name(std::move(extension_instruction_set_import_name)) + { + } +}; + +struct Extension_instruction_set +{ + std::string instruction_set_name; + std::string import_name; + Copyright copyright; + std::size_t version; + std::size_t revision; + Instructions instructions; + Extension_instruction_set(std::string instruction_set_name, + std::string import_name, + Copyright copyright, + std::size_t version, + std::size_t revision, + Instructions instructions) + : instruction_set_name(std::move(instruction_set_name)), + import_name(std::move(import_name)), + copyright(std::move(copyright)), + version(version), + revision(revision), + instructions(std::move(instructions)) + { + } + static constexpr const char *json_file_name_prefix = "extinst."; + static constexpr const char *json_file_name_suffix = ".grammar.json"; + static util::optional get_import_name_from_instruction_set_name( + util::string_view instruction_set_name); + Json_file to_json() const; + template + void visit(Fn fn) const + { + fn(*this); + copyright.visit(fn); + instructions.visit(fn); + } +}; + struct Top_level { + static constexpr const char *core_grammar_json_file_name = "spirv.core.grammar.json"; Copyright copyright; std::uint32_t magic_number; std::size_t major_version; @@ -484,23 +536,26 @@ struct Top_level std::size_t revision; Instructions instructions; Operand_kinds operand_kinds; + std::vector extension_instruction_sets; Top_level(Copyright copyright, std::uint32_t magic_number, std::size_t major_version, std::size_t minor_version, std::size_t revision, Instructions instructions, - Operand_kinds operand_kinds) + Operand_kinds operand_kinds, + std::vector extension_instruction_sets) : copyright(std::move(copyright)), magic_number(magic_number), major_version(major_version), minor_version(minor_version), revision(revision), instructions(std::move(instructions)), - operand_kinds(std::move(operand_kinds)) + operand_kinds(std::move(operand_kinds)), + extension_instruction_sets(std::move(extension_instruction_sets)) { } - json::ast::Value to_json() const; + std::vector to_json() const; template void visit(Fn fn) const { @@ -508,6 +563,8 @@ struct Top_level copyright.visit(fn); instructions.visit(fn); operand_kinds.visit(fn); + for(auto &extension_instruction_set : extension_instruction_sets) + extension_instruction_set.visit(fn); } }; } diff --git a/src/generate_spirv_parser/generate_spirv_parser.cpp b/src/generate_spirv_parser/generate_spirv_parser.cpp index c03e25d..e065e8f 100644 --- a/src/generate_spirv_parser/generate_spirv_parser.cpp +++ b/src/generate_spirv_parser/generate_spirv_parser.cpp @@ -34,26 +34,25 @@ namespace generate_spirv_parser { int generate_spirv_parser_main(int argc, char **argv) { - std::string file_name; + std::string input_directory; std::string output_directory; if(argc >= 2) - file_name = argv[1]; + input_directory = argv[1]; if(argc >= 3) output_directory = argv[2]; - if(argc != 3 || (file_name.size() > 1 && file_name[0] == '-') - || (output_directory.size() > 1 && output_directory[0] == '-')) + if(argc != 3 || input_directory.empty() || input_directory[0] == '-' || output_directory.empty() + || output_directory[0] == '-') { - std::cerr << "usage: " << argv[0] << " " << std::endl; + std::cerr << "usage: " << argv[0] << " " << std::endl; return 1; } try { - auto source = file_name == "-" ? json::Source::load_stdin() : - json::Source::load_file(std::move(file_name)); + std::shared_ptr> required_files; // outside of try so try { - auto json_in = json::parse(&source); - auto ast = parser::parse(json_in.duplicate()); + required_files = parser::read_required_files(std::move(input_directory)); + auto ast = parser::parse(std::move(*required_files)); for(auto *patch : Ast_patches::get_patches()) patch->run(ast, &std::cout); for(auto &generator : generate::Generators::make_all_generators()) @@ -73,7 +72,7 @@ int generate_spirv_parser_main(int argc, char **argv) return 1; } } - catch(std::exception &e) + catch(std::runtime_error &e) { std::cerr << "error: " << e.what() << std::endl; return 1; diff --git a/src/generate_spirv_parser/parser.cpp b/src/generate_spirv_parser/parser.cpp index f7f7c76..f48eb1e 100644 --- a/src/generate_spirv_parser/parser.cpp +++ b/src/generate_spirv_parser/parser.cpp @@ -22,10 +22,12 @@ */ #include "parser.h" #include "util/optional.h" +#include "util/string_view.h" #include #include #include #include +#include namespace vulkan_cpu { @@ -248,10 +250,10 @@ std::string parse_identifier_string(json::ast::Value value, for(char ch : string_value.value) if(!is_identifier_continue(ch)) throw Parse_error( - value.location, - parent_path_builder->path(), - std::string(name) - + ": invalid identifier in string: character is not a letter, digit, or underline"); + value.location, + parent_path_builder->path(), + std::string(name) + + ": invalid identifier in string: character is not a letter, digit, or underline"); return std::move(string_value.value); } @@ -735,14 +737,157 @@ ast::Instructions parse_instructions(json::ast::Value value, } return ast::Instructions(std::move(instructions)); } -} -ast::Top_level parse(json::ast::Value &&top_level_value) -{ +ast::Extension_instruction_set parse_extension_instruction_set(json::ast::Value top_level_value, + std::string file_name, + std::string import_name) +{ + util::string_view file_name_prefix = ast::Extension_instruction_set::json_file_name_prefix; + util::string_view file_name_suffix = ast::Extension_instruction_set::json_file_name_suffix; + if(file_name.size() <= file_name_prefix.size() + file_name_suffix.size() + || util::string_view(file_name).compare(0, file_name_prefix.size(), file_name_prefix) != 0 + || util::string_view(file_name).compare( + file_name.size() - file_name_suffix.size(), file_name_suffix.size(), file_name_suffix) + != 0) + throw Parse_error(top_level_value.location, {}, "file name is unrecognizable"); + auto instruction_set_name = std::move(file_name); + instruction_set_name.erase(instruction_set_name.size() - file_name_suffix.size(), file_name_suffix.size()); + instruction_set_name.erase(0, file_name_prefix.size()); if(top_level_value.get_value_kind() != json::ast::Value_kind::object) throw Parse_error(top_level_value.location, {}, "top level value is not an object"); auto &top_level_object = top_level_value.get_object(); util::optional copyright; + util::optional version; + util::optional revision; + util::optional instructions; + for(auto &entry : top_level_object.values) + { + const auto &key = std::get<0>(entry); + auto &entry_value = std::get<1>(entry); + Path_builder path_builder(&key, nullptr); + if(key == "copyright") + { + copyright = parse_copyright(std::move(entry_value), &path_builder); + } + else if(key == "version") + { + version = parse_integer(entry_value, &path_builder, "version"); + } + else if(key == "revision") + { + revision = parse_integer(entry_value, &path_builder, "revision"); + } + else if(key == "instructions") + { + instructions = parse_instructions(std::move(entry_value), &path_builder); + } + else + { + throw Parse_error(entry_value.location, path_builder.path(), "unknown key"); + } + } + return ast::Extension_instruction_set( + std::move(instruction_set_name), + std::move(import_name), + get_value_or_throw_parse_error( + std::move(copyright), top_level_value.location, nullptr, "missing copyright"), + get_value_or_throw_parse_error( + version, top_level_value.location, nullptr, "missing version"), + get_value_or_throw_parse_error( + revision, top_level_value.location, nullptr, "missing revision"), + get_value_or_throw_parse_error( + std::move(instructions), top_level_value.location, nullptr, "missing instructions")); +} +} + +std::shared_ptr> read_required_files( + const util::filesystem::path &dir_path) +{ + struct Result_holder + { + std::vector retval; + std::list sources; + }; + auto result_holder = std::make_shared(); + auto retval = + std::shared_ptr>(result_holder, &result_holder->retval); + auto &sources = result_holder->sources; + retval->push_back(ast::Json_file( + ast::Top_level::core_grammar_json_file_name, json::ast::Value({}, nullptr), {})); + util::string_view extension_grammar_prefix = + ast::Extension_instruction_set::json_file_name_prefix; + util::string_view extension_grammar_suffix = + ast::Extension_instruction_set::json_file_name_suffix; + for(auto &entry : util::filesystem::directory_iterator(dir_path)) + { + auto filename = entry.path().filename().string(); + if(filename == ast::Top_level::core_grammar_json_file_name) + { + // already added; just check file type + } + else if(filename.size() > extension_grammar_prefix.size() + extension_grammar_suffix.size() + && util::string_view(filename) + .compare(0, extension_grammar_prefix.size(), extension_grammar_prefix) + == 0 + && util::string_view(filename) + .compare(filename.size() - extension_grammar_suffix.size(), + extension_grammar_suffix.size(), + extension_grammar_suffix) + == 0) + { + util::string_view instruction_set_name = filename; + instruction_set_name.remove_prefix(extension_grammar_prefix.size()); + instruction_set_name.remove_suffix(extension_grammar_suffix.size()); + auto import_name = + ast::Extension_instruction_set::get_import_name_from_instruction_set_name( + instruction_set_name); + if(!import_name) + { + std::cerr << "Warning: unknown extended instruction set grammar file -- ignored: " + << entry.path() << std::endl; + continue; + } + retval->push_back(ast::Json_file( + std::move(filename), json::ast::Value({}, nullptr), std::move(*import_name))); + } + else + continue; + if(!entry.is_regular_file()) + throw Parse_error({}, {}, "file is not a regular file: " + entry.path().string()); + } + for(auto &file : *retval) + { + sources.push_back(json::Source::load_file(dir_path / file.file_name)); + auto &source = sources.back(); + file.json = json::parse(&source); + } + return retval; +} + +ast::Top_level parse(std::vector &&json_files) +{ + util::optional top_level_value; + std::vector extension_instruction_sets; + if(!json_files.empty()) + extension_instruction_sets.reserve(json_files.size() - 1); + for(auto &file : json_files) + { + if(file.extension_instruction_set_import_name) + extension_instruction_sets.push_back(parse_extension_instruction_set( + std::move(file.json), + std::move(file.file_name), + std::move(*file.extension_instruction_set_import_name))); + else if(top_level_value) + throw Parse_error(top_level_value->location, {}, "multiple core grammar files"); + else + top_level_value = std::move(file.json); + } + if(!top_level_value) + throw Parse_error(top_level_value->location, {}, "no core grammar file"); + if(top_level_value->get_value_kind() != json::ast::Value_kind::object) + throw Parse_error(top_level_value->location, {}, "top level value is not an object"); + auto &top_level_object = top_level_value->get_object(); + util::optional copyright; util::optional magic_number; util::optional major_version; util::optional minor_version; @@ -790,19 +935,20 @@ ast::Top_level parse(json::ast::Value &&top_level_value) } return ast::Top_level( get_value_or_throw_parse_error( - std::move(copyright), top_level_value.location, nullptr, "missing copyright"), + std::move(copyright), top_level_value->location, nullptr, "missing copyright"), get_value_or_throw_parse_error( - magic_number, top_level_value.location, nullptr, "missing magic_number"), + magic_number, top_level_value->location, nullptr, "missing magic_number"), get_value_or_throw_parse_error( - major_version, top_level_value.location, nullptr, "missing major_version"), + major_version, top_level_value->location, nullptr, "missing major_version"), get_value_or_throw_parse_error( - minor_version, top_level_value.location, nullptr, "missing minor_version"), + minor_version, top_level_value->location, nullptr, "missing minor_version"), get_value_or_throw_parse_error( - revision, top_level_value.location, nullptr, "missing revision"), + revision, top_level_value->location, nullptr, "missing revision"), get_value_or_throw_parse_error( - std::move(instructions), top_level_value.location, nullptr, "missing instructions"), + std::move(instructions), top_level_value->location, nullptr, "missing instructions"), get_value_or_throw_parse_error( - std::move(operand_kinds), top_level_value.location, nullptr, "missing operand_kinds")); + std::move(operand_kinds), top_level_value->location, nullptr, "missing operand_kinds"), + std::move(extension_instruction_sets)); } #if 0 @@ -815,7 +961,7 @@ void test_fn() std::size_t path_index = 0; Path_builder path_builder(&path_index, nullptr); std::cout << parse_hex_integer_string( - json::ast::Value({}, "0x1234"), &path_builder, "test", 1, 8) + json::ast::Value({}, "0x1234"), &path_builder, "test", 1, 8) << std::endl; } catch(std::exception &e) diff --git a/src/generate_spirv_parser/parser.h b/src/generate_spirv_parser/parser.h index acf2457..dd2a21e 100644 --- a/src/generate_spirv_parser/parser.h +++ b/src/generate_spirv_parser/parser.h @@ -32,6 +32,7 @@ #include "util/variant.h" #include "json/json.h" #include "json/parser.h" +#include "util/filesystem.h" namespace vulkan_cpu { @@ -110,7 +111,9 @@ public: } }; -ast::Top_level parse(json::ast::Value &&top_level_value); +std::shared_ptr> read_required_files(const util::filesystem::path &dir_path); + +ast::Top_level parse(std::vector &&json_files); } } } diff --git a/src/spirv/CMakeLists.txt b/src/spirv/CMakeLists.txt index f85af00..e1fefdf 100644 --- a/src/spirv/CMakeLists.txt +++ b/src/spirv/CMakeLists.txt @@ -26,14 +26,19 @@ set(spirv_parser_generated_include_dir ${CMAKE_CURRENT_BINARY_DIR}/generated) set(spirv_parser_generated_dir ${spirv_parser_generated_include_dir}/spirv) set(spirv_parser_sources ${spirv_parser_generated_dir}/parser.cpp ${spirv_parser_generated_dir}/spirv.cpp) set(spirv_parser_headers ${spirv_parser_generated_dir}/parser.h ${spirv_parser_generated_dir}/spirv.h) -set(spirv_core_grammar_json ${CMAKE_CURRENT_SOURCE_DIR}/../khronos-spirv/spirv.core.grammar.json) +set(spirv_grammar_json_dir ${CMAKE_CURRENT_SOURCE_DIR}/../khronos-spirv) +set(spirv_core_grammar_json ${spirv_grammar_json_dir}/spirv.core.grammar.json) +file(GLOB spirv_extension_instruction_set_json_files ${spirv_grammar_json_dir}/extinst.*.grammar.json) message(STATUS "spirv_parser_generated_dir: " ${spirv_parser_generated_dir}) +foreach(i ${spirv_extension_instruction_set_json_files}) + message(STATUS "spirv_extension_instruction_set_json_files: " ${i}) +endforeach(i) add_custom_command(OUTPUT ${spirv_parser_sources} ${spirv_parser_headers} COMMAND ${CMAKE_COMMAND} -E make_directory ${spirv_parser_generated_dir} - COMMAND ${CMAKE_COMMAND} -E chdir ${spirv_parser_generated_include_dir} $ ${spirv_core_grammar_json} spirv + COMMAND ${CMAKE_COMMAND} -E chdir ${spirv_parser_generated_include_dir} $ ${spirv_grammar_json_dir} spirv MAIN_DEPENDENCY ${spirv_core_grammar_json} - DEPENDS $ + DEPENDS $ ${spirv_extension_instruction_set_json_files} VERBATIM COMMENT "Generating SPIR-V Parser") set(sources ${sources} ${spirv_parser_sources}) -- 2.30.2