From 458c88d2996e1524f4539a40a3f7761fde522bdb Mon Sep 17 00:00:00 2001 From: Jacob Lifshay Date: Sun, 21 Oct 2018 21:42:26 -0700 Subject: [PATCH] working on parser generator --- Cargo.toml | 2 + spirv-parser-generator/Cargo.toml | 15 ++ spirv-parser-generator/src/ast.rs | 320 +++++++++++++++++++++++++++++ spirv-parser-generator/src/lib.rs | 192 +++++++++++++++++ spirv-parser-generator/src/util.rs | 170 +++++++++++++++ spirv-parser/Cargo.toml | 12 ++ spirv-parser/src/lib.rs | 2 + 7 files changed, 713 insertions(+) create mode 100644 spirv-parser-generator/Cargo.toml create mode 100644 spirv-parser-generator/src/ast.rs create mode 100644 spirv-parser-generator/src/lib.rs create mode 100644 spirv-parser-generator/src/util.rs create mode 100644 spirv-parser/Cargo.toml create mode 100644 spirv-parser/src/lib.rs diff --git a/Cargo.toml b/Cargo.toml index be9c384..0b50934 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -4,6 +4,8 @@ members = [ "shader-compiler-backend", "shader-compiler-backend-llvm-7", + "spirv-parser", + "spirv-parser-generator", "vulkan-driver", ] diff --git a/spirv-parser-generator/Cargo.toml b/spirv-parser-generator/Cargo.toml new file mode 100644 index 0000000..84e83cb --- /dev/null +++ b/spirv-parser-generator/Cargo.toml @@ -0,0 +1,15 @@ +# SPDX-License-Identifier: LGPL-2.1-or-later +# Copyright 2018 Jacob Lifshay +[package] +name = "spirv-parser-generator" +version = "0.1.0" +authors = ["Jacob Lifshay "] +license = "LGPL-2.1-or-later" + +[lib] +crate-type = ["rlib"] + +[dependencies] +serde_json = "1.0" +serde = "1.0" +serde_derive = "1.0" diff --git a/spirv-parser-generator/src/ast.rs b/spirv-parser-generator/src/ast.rs new file mode 100644 index 0000000..54696ba --- /dev/null +++ b/spirv-parser-generator/src/ast.rs @@ -0,0 +1,320 @@ +// SPDX-License-Identifier: LGPL-2.1-or-later +// Copyright 2018 Jacob Lifshay + +use serde::de::{self, Deserialize, Deserializer}; +use std::fmt; +use util::NameFormat::*; +use util::WordIterator; + +#[derive(Copy, Clone)] +pub enum QuotedInteger { + U16Hex(u16), + U32Hex(u32), +} + +impl fmt::Display for QuotedInteger { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + match *self { + QuotedInteger::U16Hex(v) => write!(f, "{:#06X}", v), + QuotedInteger::U32Hex(v) => write!(f, "{:#010X}", v), + } + } +} + +impl fmt::Debug for QuotedInteger { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + struct DisplayQuotedInteger(self::QuotedInteger); + impl fmt::Debug for DisplayQuotedInteger { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + fmt::Display::fmt(&self.0, f) + } + } + #[derive(Debug)] + struct QuotedInteger(DisplayQuotedInteger); + QuotedInteger(DisplayQuotedInteger(*self)).fmt(f) + } +} + +impl<'de> Deserialize<'de> for QuotedInteger { + fn deserialize>(deserializer: D) -> Result { + let s = String::deserialize(deserializer)?; + let prefix = "0x"; + if !s.starts_with(prefix) { + return Err(de::Error::custom(format!( + "invalid quoted integer -- must start with {:?}", + prefix + ))); + } + let digits = s.split_at(prefix.len()).1; + let radix = 0x10; + if digits.find(|c: char| !c.is_digit(radix)).is_some() { + return Err(de::Error::custom( + "invalid quoted integer -- not a hexadecimal digit", + )); + } + let retval = match digits.len() { + 4 => QuotedInteger::U16Hex(u16::from_str_radix(digits, radix).unwrap()), + 8 => QuotedInteger::U32Hex(u32::from_str_radix(digits, radix).unwrap()), + _ => { + return Err(de::Error::custom( + "invalid quoted integer -- wrong number of hex digits", + )); + } + }; + Ok(retval) + } +} + +#[derive(Copy, Clone, Eq, PartialEq, Hash, Debug)] +pub enum SPIRVVersion { + Any, + None, + AtLeast { major: u32, minor: u32 }, +} + +impl Default for SPIRVVersion { + fn default() -> Self { + SPIRVVersion::Any + } +} + +impl<'de> Deserialize<'de> for SPIRVVersion { + fn deserialize>(deserializer: D) -> Result { + let s = String::deserialize(deserializer)?; + if s == "None" { + return Ok(SPIRVVersion::None); + } + let dot_pos = s + .find('.') + .ok_or_else(|| de::Error::custom("invalid SPIR-V version -- no decimal place"))?; + let (major_digits, minor_digits) = s.split_at(dot_pos); + let minor_digits = minor_digits.split_at(1).1; + let parse_digits = |digits: &str| -> Result { + if digits == "" { + return Err(de::Error::custom( + "invalid SPIR-V version -- expected a decimal digit", + )); + } + if digits.find(|c: char| !c.is_ascii_digit()).is_some() { + return Err(de::Error::custom( + "invalid SPIR-V version -- expected a decimal digit", + )); + } + if digits.len() > 5 { + return Err(de::Error::custom( + "invalid SPIR-V version -- too many digits", + )); + } + Ok(digits.parse().unwrap()) + }; + let major = parse_digits(major_digits)?; + let minor = parse_digits(minor_digits)?; + Ok(SPIRVVersion::AtLeast { major, minor }) + } +} + +#[derive(Copy, Clone, Eq, PartialEq, Hash, Debug, Deserialize)] +pub enum Quantifier { + #[serde(rename = "?")] + Optional, + #[serde(rename = "*")] + Variadic, +} + +#[derive(Deserialize, Debug)] +#[serde(deny_unknown_fields)] +pub struct InstructionOperand { + kind: String, + name: Option, + quantifier: Option, +} + +impl InstructionOperand { + pub fn guess_name(&mut self) -> Result<(), ::Error> { + if self.name.is_none() { + self.name = Some( + SnakeCase + .name_from_words(WordIterator::new(&self.kind)) + .ok_or(::Error::DeducingNameForInstructionOperandFailed)?, + ); + } + Ok(()) + } +} + +#[derive(Deserialize, Debug)] +#[serde(deny_unknown_fields)] +pub struct Instruction { + opname: String, + opcode: u16, + #[serde(default)] + operands: Vec, + #[serde(default)] + capabilities: Vec, + #[serde(default)] + extensions: Vec, + #[serde(default)] + version: SPIRVVersion, +} + +impl Instruction { + pub fn guess_names(&mut self) -> Result<(), ::Error> { + for operand in self.operands.iter_mut() { + operand.guess_name()?; + } + Ok(()) + } +} + +#[derive(Deserialize, Debug)] +#[serde(deny_unknown_fields)] +pub struct ExtensionInstruction { + opname: String, + opcode: u16, + #[serde(default)] + operands: Vec, + #[serde(default)] + capabilities: Vec, +} + +impl ExtensionInstruction { + pub fn guess_names(&mut self) -> Result<(), ::Error> { + for operand in self.operands.iter_mut() { + operand.guess_name()?; + } + Ok(()) + } +} + +#[derive(Deserialize, Debug)] +#[serde(deny_unknown_fields)] +pub struct EnumerantParameter { + kind: String, + name: Option, +} + +impl EnumerantParameter { + pub fn guess_name(&mut self) -> Result<(), ::Error> { + if self.name.is_none() { + self.name = Some( + SnakeCase + .name_from_words(WordIterator::new(&self.kind)) + .ok_or(::Error::DeducingNameForEnumerantParameterFailed)?, + ); + } + Ok(()) + } +} + +#[derive(Deserialize, Debug)] +#[serde(deny_unknown_fields)] +pub struct Enumerant { + enumerant: String, + value: Value, + #[serde(default)] + capabilities: Vec, + #[serde(default)] + parameters: Vec, + #[serde(default)] + extensions: Vec, + #[serde(default)] + version: SPIRVVersion, +} + +impl Enumerant { + pub fn guess_names(&mut self) -> Result<(), ::Error> { + for parameter in self.parameters.iter_mut() { + parameter.guess_name()?; + } + Ok(()) + } +} + +#[derive(Deserialize, Debug)] +#[serde(deny_unknown_fields)] +#[serde(tag = "category")] +pub enum OperandKind { + BitEnum { + kind: String, + enumerants: Vec>, + }, + ValueEnum { + kind: String, + enumerants: Vec>, + }, + Id { + kind: String, + doc: Option, + }, + Literal { + kind: String, + doc: Option, + }, + Composite { + kind: String, + bases: Vec, + }, +} + +impl OperandKind { + pub fn guess_names(&mut self) -> Result<(), ::Error> { + match self { + OperandKind::BitEnum { enumerants, .. } => { + for enumerant in enumerants.iter_mut() { + enumerant.guess_names()?; + } + } + OperandKind::ValueEnum { enumerants, .. } => { + for enumerant in enumerants.iter_mut() { + enumerant.guess_names()?; + } + } + OperandKind::Id { .. } + | OperandKind::Literal { .. } + | OperandKind::Composite { .. } => {} + } + Ok(()) + } +} + +#[derive(Deserialize, Debug)] +#[serde(deny_unknown_fields)] +pub struct CoreGrammar { + copyright: Vec, + magic_number: QuotedInteger, + major_version: u16, + minor_version: u16, + revision: u32, + instructions: Vec, + operand_kinds: Vec, +} + +impl CoreGrammar { + pub fn guess_names(&mut self) -> Result<(), ::Error> { + for instruction in self.instructions.iter_mut() { + instruction.guess_names()?; + } + for operand_kind in self.operand_kinds.iter_mut() { + operand_kind.guess_names()?; + } + Ok(()) + } +} + +#[derive(Deserialize, Debug)] +#[serde(deny_unknown_fields)] +pub struct ExtensionInstructionSet { + copyright: Vec, + version: u32, + revision: u32, + instructions: Vec, +} + +impl ExtensionInstructionSet { + pub fn guess_names(&mut self) -> Result<(), ::Error> { + for instruction in self.instructions.iter_mut() { + instruction.guess_names()?; + } + Ok(()) + } +} diff --git a/spirv-parser-generator/src/lib.rs b/spirv-parser-generator/src/lib.rs new file mode 100644 index 0000000..d931a6a --- /dev/null +++ b/spirv-parser-generator/src/lib.rs @@ -0,0 +1,192 @@ +// SPDX-License-Identifier: LGPL-2.1-or-later +// Copyright 2018 Jacob Lifshay + +#[macro_use] +extern crate serde_derive; +extern crate serde; +extern crate serde_json; + +use std::collections::HashMap; +use std::error; +use std::fmt; +use std::fs::File; +use std::io; +use std::path::Path; +use std::path::PathBuf; + +mod ast; +mod util; + +pub const SPIRV_CORE_GRAMMAR_JSON_FILE_NAME: &str = "spirv.core.grammar.json"; + +#[derive(Copy, Clone, Eq, PartialEq, Hash, Debug)] +pub enum ExtensionInstructionSet { + GLSLStd450, + OpenCLStd, +} + +impl ExtensionInstructionSet { + pub fn get_grammar_json_file_name(self) -> &'static str { + match self { + ExtensionInstructionSet::GLSLStd450 => "extinst.glsl.std.450.grammar.json", + ExtensionInstructionSet::OpenCLStd => "extinst.opencl.std.100.grammar.json", + } + } +} + +#[derive(Debug)] +pub enum Error { + IOError(io::Error), + JSONError(serde_json::Error), + DeducingNameForInstructionOperandFailed, + DeducingNameForEnumerantParameterFailed, +} + +impl From for Error { + fn from(v: io::Error) -> Error { + Error::IOError(v) + } +} + +impl From for Error { + fn from(v: serde_json::Error) -> Error { + if let serde_json::error::Category::Io = v.classify() { + Error::IOError(v.into()) + } else { + Error::JSONError(v) + } + } +} + +impl fmt::Display for Error { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + match self { + Error::IOError(v) => fmt::Display::fmt(v, f), + Error::JSONError(v) => fmt::Display::fmt(v, f), + Error::DeducingNameForInstructionOperandFailed => { + write!(f, "deducing name for InstructionOperand failed") + } + Error::DeducingNameForEnumerantParameterFailed => { + write!(f, "deducing name for EnumerantParameter failed") + } + } + } +} + +impl error::Error for Error {} + +impl From for io::Error { + fn from(error: Error) -> Self { + match error { + Error::IOError(v) => v, + Error::JSONError(v) => v.into(), + error @ Error::DeducingNameForInstructionOperandFailed + | error @ Error::DeducingNameForEnumerantParameterFailed => { + io::Error::new(io::ErrorKind::Other, format!("{}", error)) + } + } + } +} + +pub struct Output {} + +pub struct Input { + spirv_core_grammar_json_path: PathBuf, + extension_instruction_sets: HashMap, +} + +impl Input { + pub fn new>(spirv_core_grammar_json_path: T) -> Input { + Input { + spirv_core_grammar_json_path: spirv_core_grammar_json_path.as_ref().into(), + extension_instruction_sets: HashMap::new(), + } + } + pub fn add_extension_instruction_set>( + mut self, + extension_instruction_set: ExtensionInstructionSet, + path: T, + ) -> Self { + assert!( + self.extension_instruction_sets + .insert(extension_instruction_set, path.as_ref().into()) + .is_none(), + "duplicate extension instruction set: {:?}", + extension_instruction_set + ); + self + } + pub fn generate(self) -> Result { + let Input { + spirv_core_grammar_json_path, + extension_instruction_sets, + } = self; + let mut core_grammar: ast::CoreGrammar = + serde_json::from_reader(File::open(spirv_core_grammar_json_path)?)?; + core_grammar.guess_names()?; + let mut parsed_extension_instruction_sets: HashMap< + ExtensionInstructionSet, + ast::ExtensionInstructionSet, + > = Default::default(); + for (extension_instruction_set, path) in extension_instruction_sets { + let mut parsed_extension_instruction_set: ast::ExtensionInstructionSet = + serde_json::from_reader(File::open(path)?)?; + parsed_extension_instruction_set.guess_names()?; + assert!( + parsed_extension_instruction_sets + .insert(extension_instruction_set, parsed_extension_instruction_set) + .is_none() + ); + } + unimplemented!() + } +} + +#[cfg(test)] +mod tests { + use super::*; + fn get_spirv_grammar_path>(name: T) -> PathBuf { + Path::new(env!("CARGO_MANIFEST_DIR")) + .join("../external/SPIRV-Headers/include/spirv/unified1") + .join(name) + } + + fn create_input(extension_instruction_sets: &[ExtensionInstructionSet]) -> Input { + let mut retval = Input::new(get_spirv_grammar_path("spirv.core.grammar.json")); + for &extension_instruction_set in extension_instruction_sets { + retval = retval.add_extension_instruction_set( + extension_instruction_set, + get_spirv_grammar_path(extension_instruction_set.get_grammar_json_file_name()), + ); + } + retval + } + + #[test] + fn parse_core_grammar() -> Result<(), Error> { + create_input(&[]).generate()?; + Ok(()) + } + + #[test] + fn parse_core_grammar_with_opencl() -> Result<(), Error> { + create_input(&[ExtensionInstructionSet::OpenCLStd]).generate()?; + Ok(()) + } + + #[test] + fn parse_core_grammar_with_opencl_and_glsl() -> Result<(), Error> { + create_input(&[ + ExtensionInstructionSet::OpenCLStd, + ExtensionInstructionSet::GLSLStd450, + ]) + .generate()?; + Ok(()) + } + + #[test] + fn parse_core_grammar_with_glsl() -> Result<(), Error> { + create_input(&[ExtensionInstructionSet::GLSLStd450]).generate()?; + Ok(()) + } +} diff --git a/spirv-parser-generator/src/util.rs b/spirv-parser-generator/src/util.rs new file mode 100644 index 0000000..e82265d --- /dev/null +++ b/spirv-parser-generator/src/util.rs @@ -0,0 +1,170 @@ +// SPDX-License-Identifier: LGPL-2.1-or-later +// Copyright 2018 Jacob Lifshay + +use std::borrow::Borrow; +use std::iter::Iterator; + +#[derive(Copy, Clone, Eq, PartialEq, Hash, Debug)] +enum CharClass { + Uppercase, + OtherIdentifier, + Number, + WordSeparator, +} + +impl From for CharClass { + fn from(v: char) -> CharClass { + match v { + 'A'...'Z' => CharClass::Uppercase, + 'a'...'z' => CharClass::OtherIdentifier, + '0'...'9' => CharClass::Number, + _ => CharClass::WordSeparator, + } + } +} + +pub struct WordIterator<'a> { + word: Option<&'a str>, + words: &'a str, +} + +impl<'a> WordIterator<'a> { + pub fn new(words: &'a str) -> Self { + WordIterator { word: None, words } + } +} + +impl<'a> Iterator for WordIterator<'a> { + type Item = &'a str; + fn next(&mut self) -> Option<&'a str> { + let mut word_start = None; + let mut last_char_class = CharClass::WordSeparator; + for (i, ch) in self.words.char_indices() { + let current_char_class = CharClass::from(ch); + if word_start.is_some() { + match current_char_class { + CharClass::WordSeparator => { + self.word = Some(&self.words[word_start.unwrap()..i]); + self.words = &self.words[i..]; + return self.word; + } + CharClass::Uppercase => { + if last_char_class != CharClass::Uppercase + && last_char_class != CharClass::Number + { + self.word = Some(&self.words[word_start.unwrap()..i]); + self.words = &self.words[i..]; + return self.word; + } + if self.words[i..].chars().nth(1).map(CharClass::from) + == Some(CharClass::OtherIdentifier) + { + self.word = Some(&self.words[word_start.unwrap()..i]); + self.words = &self.words[i..]; + return self.word; + } + } + _ => {} + } + } else if current_char_class != CharClass::WordSeparator { + word_start = Some(i); + } + last_char_class = current_char_class; + } + if let Some(word_start) = word_start { + self.word = Some(&self.words[word_start..]); + } else { + self.word = None; + } + self.words = ""; + self.word + } +} + +pub const RUST_RESERVED_WORDS: &[&str] = &[ + "_", "Self", "abstract", "alignof", "as", "become", "box", "break", "const", "continue", + "crate", "do", "else", "enum", "extern", "false", "final", "fn", "for", "if", "impl", "in", + "let", "loop", "macro", "match", "mod", "move", "mut", "offsetof", "override", "priv", "proc", + "pub", "pure", "ref", "return", "self", "sizeof", "static", "struct", "super", "trait", "true", + "type", "typeof", "unsafe", "unsized", "use", "virtual", "where", "while", "yield", +]; + +#[derive(Copy, Clone, Eq, PartialEq, Hash, Debug)] +pub enum CharacterCase { + Upper, + Lower, +} + +impl CharacterCase { + pub fn convert_ascii_case>(self, string: T) -> String { + let mut retval = string.into(); + match self { + CharacterCase::Upper => retval.make_ascii_uppercase(), + CharacterCase::Lower => retval.make_ascii_lowercase(), + } + retval + } + pub fn convert_initial_ascii_case>(self, string: T) -> String { + let mut retval = string.into(); + if let Some(first) = retval.get_mut(0..1) { + match self { + CharacterCase::Upper => first.make_ascii_uppercase(), + CharacterCase::Lower => first.make_ascii_lowercase(), + } + } + retval + } +} + +#[derive(Copy, Clone, Eq, PartialEq, Hash, Debug)] +pub enum NameFormat { + SnakeCase, + ScreamingSnakeCase, + CamelCase, +} + +impl NameFormat { + pub fn word_separator(self) -> &'static str { + match self { + NameFormat::SnakeCase | NameFormat::ScreamingSnakeCase => "_", + NameFormat::CamelCase => "", + } + } + pub fn word_initial_char_case(self) -> CharacterCase { + match self { + NameFormat::CamelCase | NameFormat::ScreamingSnakeCase => CharacterCase::Upper, + NameFormat::SnakeCase => CharacterCase::Lower, + } + } + pub fn word_char_case(self) -> CharacterCase { + match self { + NameFormat::ScreamingSnakeCase => CharacterCase::Upper, + NameFormat::CamelCase | NameFormat::SnakeCase => CharacterCase::Lower, + } + } + pub fn name_from_words, I: Iterator>( + self, + words: I, + ) -> Option { + let mut retval: Option = None; + for word in words { + let word = word.borrow(); + let word = self.word_char_case().convert_ascii_case(word); + let word = self + .word_initial_char_case() + .convert_initial_ascii_case(word); + retval = Some(if let Some(mut s) = retval { + s + self.word_separator() + &word + } else { + word + }); + } + let retval = retval?; + for &reserved_word in RUST_RESERVED_WORDS { + if retval == reserved_word { + return Some(retval + "_"); + } + } + Some(retval) + } +} diff --git a/spirv-parser/Cargo.toml b/spirv-parser/Cargo.toml new file mode 100644 index 0000000..cd65d77 --- /dev/null +++ b/spirv-parser/Cargo.toml @@ -0,0 +1,12 @@ +# SPDX-License-Identifier: LGPL-2.1-or-later +# Copyright 2018 Jacob Lifshay +[package] +name = "spirv-parser" +version = "0.1.0" +authors = ["Jacob Lifshay "] +license = "LGPL-2.1-or-later" + +[lib] +crate-type = ["rlib"] + +[dependencies] diff --git a/spirv-parser/src/lib.rs b/spirv-parser/src/lib.rs new file mode 100644 index 0000000..84e9d79 --- /dev/null +++ b/spirv-parser/src/lib.rs @@ -0,0 +1,2 @@ +// SPDX-License-Identifier: LGPL-2.1-or-later +// Copyright 2018 Jacob Lifshay -- 2.30.2