2 * yosys -- Yosys Open SYnthesis Suite
4 * Copyright (C) 2019 whitequark <whitequark@whitequark.org>
6 * Permission to use, copy, modify, and/or distribute this software for any
7 * purpose with or without fee is hereby granted, provided that the above
8 * copyright notice and this permission notice appear in all copies.
10 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
11 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
12 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
13 * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
14 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
15 * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
16 * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
20 // The reason the -path mode of connect_rpc uses byte-oriented and not message-oriented sockets, even though
21 // it is a message-oriented interface, is that the system can place various limits on the message size, which
22 // are not always transparent or easy to change. Given that generated HDL code get be extremely large, it is
23 // unwise to rely on those limits being large enough, and using byte-oriented sockets is guaranteed to work.
29 #include <sys/socket.h>
31 extern char **environ
;
34 #include "libs/json11/json11.hpp"
35 #include "libs/sha1/sha1.h"
36 #include "kernel/yosys.h"
41 static std::wstring
str2wstr(const std::string
&in
) {
42 if(in
== "") return L
"";
44 out
.resize(MultiByteToWideChar(/*CodePage=*/CP_UTF8
, /*dwFlags=*/0, /*lpMultiByteStr=*/&in
[0], /*cbMultiByte=*/(int)in
.length(), /*lpWideCharStr=*/NULL
, /*cchWideChar=*/0));
45 int written
= MultiByteToWideChar(/*CodePage=*/CP_UTF8
, /*dwFlags=*/0, /*lpMultiByteStr=*/&in
[0], /*cbMultiByte=*/(int)in
.length(), /*lpWideCharStr=*/&out
[0], /*cchWideChar=*/(int)out
.length());
46 log_assert(written
== (int)out
.length());
50 static std::string
wstr2str(const std::wstring
&in
) {
51 if(in
== L
"") return "";
53 out
.resize(WideCharToMultiByte(/*CodePage=*/CP_UTF8
, /*dwFlags=*/0, /*lpWideCharStr=*/&in
[0], /*cchWideChar=*/(int)in
.length(), /*lpMultiByteStr=*/NULL
, /*cbMultiByte=*/0, /*lpDefaultChar=*/NULL
, /*lpUsedDefaultChar=*/NULL
));
54 int written
= WideCharToMultiByte(/*CodePage=*/CP_UTF8
, /*dwFlags=*/0, /*lpWideCharStr=*/&in
[0], /*cchWideChar=*/(int)in
.length(), /*lpMultiByteStr=*/&out
[0], /*cbMultiByte=*/(int)out
.length(), /*lpDefaultChar=*/NULL
, /*lpUsedDefaultChar=*/NULL
);
55 log_assert(written
== (int)out
.length());
59 static std::string
get_last_error_str() {
60 DWORD last_error
= GetLastError();
62 DWORD size_w
= FormatMessageW(/*dwFlags=*/FORMAT_MESSAGE_FROM_SYSTEM
|FORMAT_MESSAGE_ALLOCATE_BUFFER
|FORMAT_MESSAGE_IGNORE_INSERTS
, /*lpSource=*/NULL
, /*dwMessageId=*/last_error
, /*dwLanguageId=*/0, /*lpBuffer=*/(LPWSTR
)&out_w
, /*nSize=*/0, /*Arguments=*/NULL
);
64 return std::to_string(last_error
);
65 std::string out
= wstr2str(std::wstring(out_w
, size_w
));
76 RpcServer(const std::string
&name
) : name(name
) { }
77 virtual ~RpcServer() { }
79 virtual void write(const std::string
&data
) = 0;
80 virtual std::string
read() = 0;
82 Json
call(const Json
&json_request
) {
84 json_request
.dump(request
);
86 log_debug("RPC frontend request: %s", request
.c_str());
89 std::string response
= read();
90 log_debug("RPC frontend response: %s", response
.c_str());
92 Json json_response
= Json::parse(response
, error
);
93 if (json_response
.is_null())
94 log_cmd_error("parsing JSON failed: %s\n", error
.c_str());
95 if (json_response
["error"].is_string())
96 log_cmd_error("RPC frontend returned an error: %s\n", json_response
["error"].string_value().c_str());
100 std::vector
<std::string
> get_module_names() {
101 Json response
= call(Json::object
{
102 { "method", "modules" },
104 bool is_valid
= true;
105 std::vector
<std::string
> modules
;
106 if (response
["modules"].is_array()) {
107 for (auto &json_module
: response
["modules"].array_items()) {
108 if (json_module
.is_string())
109 modules
.push_back(json_module
.string_value());
110 else is_valid
= false;
112 } else is_valid
= false;
114 log_cmd_error("RPC frontend returned malformed response: %s\n", response
.dump().c_str());
118 std::pair
<std::string
, std::string
> derive_module(const std::string
&module
, const dict
<RTLIL::IdString
, RTLIL::Const
> ¶meters
) {
119 Json::object json_parameters
;
120 for (auto ¶m
: parameters
) {
121 std::string type
, value
;
122 if (param
.second
.flags
& RTLIL::CONST_FLAG_REAL
) {
124 value
= param
.second
.decode_string();
125 } else if (param
.second
.flags
& RTLIL::CONST_FLAG_STRING
) {
127 value
= param
.second
.decode_string();
128 } else if ((param
.second
.flags
& ~RTLIL::CONST_FLAG_SIGNED
) == RTLIL::CONST_FLAG_NONE
) {
129 type
= (param
.second
.flags
& RTLIL::CONST_FLAG_SIGNED
) ? "signed" : "unsigned";
130 value
= param
.second
.as_string();
132 log_cmd_error("Unserializable constant flags 0x%x\n", param
.second
.flags
);
133 json_parameters
[param
.first
.str()] = Json::object
{
138 Json response
= call(Json::object
{
139 { "method", "derive" },
140 { "module", module
},
141 { "parameters", json_parameters
},
143 bool is_valid
= true;
144 std::string frontend
, source
;
145 if (response
["frontend"].is_string())
146 frontend
= response
["frontend"].string_value();
147 else is_valid
= false;
148 if (response
["source"].is_string())
149 source
= response
["source"].string_value();
150 else is_valid
= false;
152 log_cmd_error("RPC frontend returned malformed response: %s\n", response
.dump().c_str());
153 return std::make_pair(frontend
, source
);
157 struct RpcModule
: RTLIL::Module
{
158 std::shared_ptr
<RpcServer
> server
;
160 RTLIL::IdString
derive(RTLIL::Design
*design
, const dict
<RTLIL::IdString
, RTLIL::Const
> ¶meters
, bool /*mayfail*/) YS_OVERRIDE
{
161 std::string stripped_name
= name
.str();
162 if (stripped_name
.compare(0, 9, "$abstract") == 0)
163 stripped_name
= stripped_name
.substr(9);
164 log_assert(stripped_name
[0] == '\\');
166 log_header(design
, "Executing RPC frontend `%s' for module `%s'.\n", server
->name
.c_str(), stripped_name
.c_str());
168 std::string parameter_info
;
169 for (auto ¶m
: parameters
) {
170 log("Parameter %s = %s\n", param
.first
.c_str(), log_signal(RTLIL::SigSpec(param
.second
)));
171 parameter_info
+= stringf("%s=%s", param
.first
.c_str(), log_signal(RTLIL::SigSpec(param
.second
)));
174 std::string derived_name
;
175 if (parameters
.empty())
176 derived_name
= stripped_name
;
177 else if (parameter_info
.size() > 60)
178 derived_name
= "$paramod$" + sha1(parameter_info
) + stripped_name
;
180 derived_name
= "$paramod" + stripped_name
+ parameter_info
;
182 if (design
->has(derived_name
)) {
183 log("Found cached RTLIL representation for module `%s'.\n", derived_name
.c_str());
185 std::string command
, input
;
186 std::tie(command
, input
) = server
->derive_module(stripped_name
.substr(1), parameters
);
188 std::istringstream
input_stream(input
);
189 RTLIL::Design
*derived_design
= new RTLIL::Design
;
190 Frontend::frontend_call(derived_design
, &input_stream
, "<rpc>" + derived_name
.substr(8), command
);
191 derived_design
->check();
193 dict
<std::string
, std::string
> name_mangling
;
194 bool found_derived_top
= false;
195 for (auto module
: derived_design
->modules()) {
196 std::string original_name
= module
->name
.str();
197 if (original_name
== stripped_name
) {
198 found_derived_top
= true;
199 name_mangling
[original_name
] = derived_name
;
201 name_mangling
[original_name
] = derived_name
+ module
->name
.str();
204 if (!found_derived_top
)
205 log_cmd_error("RPC frontend did not return requested module `%s`!\n", stripped_name
.c_str());
207 for (auto module
: derived_design
->modules())
208 for (auto cell
: module
->cells())
209 if (name_mangling
.count(cell
->type
.str()))
210 cell
->type
= name_mangling
[cell
->type
.str()];
212 for (auto module
: derived_design
->modules_
) {
213 std::string mangled_name
= name_mangling
[module
.first
.str()];
215 log("Importing `%s' as `%s'.\n", log_id(module
.first
), log_id(mangled_name
));
217 module
.second
->name
= mangled_name
;
218 module
.second
->design
= design
;
219 module
.second
->attributes
.erase(ID::top
);
220 if (!module
.second
->has_attribute(ID::hdlname
))
221 module
.second
->set_string_attribute(ID::hdlname
, module
.first
.str());
222 design
->modules_
[mangled_name
] = module
.second
;
223 derived_design
->modules_
.erase(module
.first
);
226 delete derived_design
;
232 RTLIL::Module
*clone() const YS_OVERRIDE
{
233 RpcModule
*new_mod
= new RpcModule
;
234 new_mod
->server
= server
;
242 #if defined(_MSC_VER)
244 typedef SSIZE_T ssize_t
;
247 struct HandleRpcServer
: RpcServer
{
250 HandleRpcServer(const std::string
&name
, HANDLE hsend
, HANDLE hrecv
)
251 : RpcServer(name
), hsend(hsend
), hrecv(hrecv
) { }
253 void write(const std::string
&data
) YS_OVERRIDE
{
254 log_assert(data
.length() >= 1 && data
.find('\n') == data
.length() - 1);
258 if (!WriteFile(hsend
, &data
[offset
], data
.length() - offset
, &data_written
, /*lpOverlapped=*/NULL
))
259 log_cmd_error("WriteFile failed: %s\n", get_last_error_str().c_str());
260 offset
+= data_written
;
261 } while(offset
< (ssize_t
)data
.length());
264 std::string
read() YS_OVERRIDE
{
267 while (data
.length() == 0 || data
[data
.length() - 1] != '\n') {
268 data
.resize(data
.length() + 1024);
270 if (!ReadFile(hrecv
, &data
[offset
], data
.length() - offset
, &data_read
, /*lpOverlapped=*/NULL
))
271 log_cmd_error("ReadFile failed: %s\n", get_last_error_str().c_str());
274 size_t term_pos
= data
.find('\n', offset
);
275 if (term_pos
!= data
.length() - 1 && term_pos
!= std::string::npos
)
276 log_cmd_error("read failed: more than one response\n");
290 struct FdRpcServer
: RpcServer
{
294 FdRpcServer(const std::string
&name
, int fdsend
, int fdrecv
, pid_t pid
= -1)
295 : RpcServer(name
), fdsend(fdsend
), fdrecv(fdrecv
), pid(pid
) { }
298 if (pid
== -1) return;
299 // If we're communicating with a process, check that it's still running, or we may get killed with SIGPIPE.
300 pid_t wait_result
= ::waitpid(pid
, NULL
, WNOHANG
);
301 if (wait_result
== -1)
302 log_cmd_error("waitpid failed: %s\n", strerror(errno
));
303 if (wait_result
== pid
)
304 log_cmd_error("RPC frontend terminated unexpectedly\n");
307 void write(const std::string
&data
) YS_OVERRIDE
{
308 log_assert(data
.length() >= 1 && data
.find('\n') == data
.length() - 1);
312 ssize_t result
= ::write(fdsend
, &data
[offset
], data
.length() - offset
);
314 log_cmd_error("write failed: %s\n", strerror(errno
));
316 } while(offset
< (ssize_t
)data
.length());
319 std::string
read() YS_OVERRIDE
{
322 while (data
.length() == 0 || data
[data
.length() - 1] != '\n') {
323 data
.resize(data
.length() + 1024);
325 ssize_t result
= ::read(fdrecv
, &data
[offset
], data
.length() - offset
);
327 log_cmd_error("read failed: %s\n", strerror(errno
));
330 size_t term_pos
= data
.find('\n', offset
);
331 if (term_pos
!= data
.length() - 1 && term_pos
!= std::string::npos
)
332 log_cmd_error("read failed: more than one response\n");
339 if (fdrecv
!= fdsend
)
346 // RpcFrontend does not inherit from Frontend since it does not read files.
347 struct RpcFrontend
: public Pass
{
348 RpcFrontend() : Pass("connect_rpc", "connect to RPC frontend") { }
349 void help() YS_OVERRIDE
351 // |---v---|---v---|---v---|---v---|---v---|---v---|---v---|---v---|---v---|---v---|
353 log(" connect_rpc -exec <command> [args...]\n");
354 log(" connect_rpc -path <path>\n");
356 log("Load modules using an out-of-process frontend.\n");
358 log(" -exec <command> [args...]\n");
359 log(" run <command> with arguments [args...]. send requests on stdin, read\n");
360 log(" responses from stdout.\n");
362 log(" -path <path>\n");
363 log(" connect to Unix domain socket at <path>. (Unix)\n");
364 log(" connect to bidirectional byte-type named pipe at <path>. (Windows)\n");
366 log("A simple JSON-based, newline-delimited protocol is used for communicating with\n");
367 log("the frontend. Yosys requests data from the frontend by sending exactly 1 line\n");
368 log("of JSON. Frontend responds with data or error message by replying with exactly\n");
369 log("1 line of JSON as well.\n");
371 log(" -> {\"method\": \"modules\"}\n");
372 log(" <- {\"modules\": [\"<module-name>\", ...]}\n");
373 log(" <- {\"error\": \"<error-message>\"}\n");
374 log(" request for the list of modules that can be derived by this frontend.\n");
375 log(" the 'hierarchy' command will call back into this frontend if a cell\n");
376 log(" with type <module-name> is instantiated in the design.\n");
378 log(" -> {\"method\": \"derive\", \"module\": \"<module-name\">, \"parameters\": {\n");
379 log(" \"<param-name>\": {\"type\": \"[unsigned|signed|string|real]\",\n");
380 log(" \"value\": \"<param-value>\"}, ...}}\n");
381 log(" <- {\"frontend\": \"[ilang|verilog|...]\",\"source\": \"<source>\"}}\n");
382 log(" <- {\"error\": \"<error-message>\"}\n");
383 log(" request for the module <module-name> to be derived for a specific set of\n");
384 log(" parameters. <param-name> starts with \\ for named parameters, and with $\n");
385 log(" for unnamed parameters, which are numbered starting at 1.<param-value>\n");
386 log(" for integer parameters is always specified as a binary string of unlimited\n");
387 log(" precision. the <source> returned by the frontend is hygienically parsed\n");
388 log(" by a built-in Yosys <frontend>, allowing the RPC frontend to return any\n");
389 log(" convenient representation of the module. the derived module is cached,\n");
390 log(" so the response should be the same whenever the same set of parameters\n");
391 log(" is provided.\n");
393 void execute(std::vector
<std::string
> args
, RTLIL::Design
*design
) YS_OVERRIDE
395 log_header(design
, "Connecting to RPC frontend.\n");
397 std::vector
<std::string
> command
;
400 for (argidx
= 1; argidx
< args
.size(); argidx
++) {
401 std::string arg
= args
[argidx
];
402 if (arg
== "-exec" && argidx
+1 < args
.size()) {
403 command
.insert(command
.begin(), args
.begin() + argidx
+ 1, args
.end());
406 if (arg
== "-path" && argidx
+1 < args
.size()) {
407 path
= args
[argidx
+1];
412 extra_args(args
, argidx
, design
);
414 if ((!command
.empty()) + (!path
.empty()) != 1)
415 log_cmd_error("Exactly one of -exec, -unix must be specified.\n");
417 std::shared_ptr
<RpcServer
> server
;
418 if (!command
.empty()) {
419 std::string command_line
;
421 for (auto &arg
: command
) {
422 if (!first
) command_line
+= ' ';
428 std::wstring command_w
= str2wstr(command
[0]);
429 std::wstring command_path_w
;
430 std::wstring command_line_w
= str2wstr(command_line
);
431 DWORD command_path_len_w
;
432 SECURITY_ATTRIBUTES pipe_attr
= {};
433 HANDLE send_r
= NULL
, send_w
= NULL
, recv_r
= NULL
, recv_w
= NULL
;
434 STARTUPINFOW startup_info
= {};
435 PROCESS_INFORMATION proc_info
= {};
437 command_path_len_w
= SearchPathW(/*lpPath=*/NULL
, /*lpFileName=*/command_w
.c_str(), /*lpExtension=*/L
".exe", /*nBufferLength=*/0, /*lpBuffer=*/NULL
, /*lpFilePart=*/NULL
);
438 if (command_path_len_w
== 0) {
439 log_error("SearchPathW failed: %s\n", get_last_error_str().c_str());
442 command_path_w
.resize(command_path_len_w
- 1);
443 command_path_len_w
= SearchPathW(/*lpPath=*/NULL
, /*lpFileName=*/command_w
.c_str(), /*lpExtension=*/L
".exe", /*nBufferLength=*/command_path_len_w
, /*lpBuffer=*/&command_path_w
[0], /*lpFilePart=*/NULL
);
444 log_assert(command_path_len_w
== command_path_w
.length());
446 pipe_attr
.nLength
= sizeof(pipe_attr
);
447 pipe_attr
.bInheritHandle
= TRUE
;
448 pipe_attr
.lpSecurityDescriptor
= NULL
;
449 if (!CreatePipe(&send_r
, &send_w
, &pipe_attr
, /*nSize=*/0)) {
450 log_error("CreatePipe failed: %s\n", get_last_error_str().c_str());
453 if (!SetHandleInformation(send_w
, HANDLE_FLAG_INHERIT
, 0)) {
454 log_error("SetHandleInformation failed: %s\n", get_last_error_str().c_str());
457 if (!CreatePipe(&recv_r
, &recv_w
, &pipe_attr
, /*nSize=*/0)) {
458 log_error("CreatePipe failed: %s\n", get_last_error_str().c_str());
461 if (!SetHandleInformation(recv_r
, HANDLE_FLAG_INHERIT
, 0)) {
462 log_error("SetHandleInformation failed: %s\n", get_last_error_str().c_str());
466 startup_info
.cb
= sizeof(startup_info
);
467 startup_info
.hStdInput
= send_r
;
468 startup_info
.hStdOutput
= recv_w
;
469 startup_info
.hStdError
= GetStdHandle(STD_ERROR_HANDLE
);
470 startup_info
.dwFlags
|= STARTF_USESTDHANDLES
;
471 if (!CreateProcessW(/*lpApplicationName=*/command_path_w
.c_str(), /*lpCommandLine=*/&command_line_w
[0], /*lpProcessAttributes=*/NULL
, /*lpThreadAttributes=*/NULL
, /*bInheritHandles=*/TRUE
, /*dwCreationFlags=*/0, /*lpEnvironment=*/NULL
, /*lpCurrentDirectory=*/NULL
, &startup_info
, &proc_info
)) {
472 log_error("CreateProcessW failed: %s\n", get_last_error_str().c_str());
475 CloseHandle(proc_info
.hProcess
);
476 CloseHandle(proc_info
.hThread
);
478 server
= std::make_shared
<HandleRpcServer
>(path
, send_w
, recv_r
);
483 if (send_r
!= NULL
) CloseHandle(send_r
);
484 if (send_w
!= NULL
) CloseHandle(send_w
);
485 if (recv_r
!= NULL
) CloseHandle(recv_r
);
486 if (recv_w
!= NULL
) CloseHandle(recv_w
);
488 std::vector
<char *> argv
;
489 int send
[2] = {-1,-1}, recv
[2] = {-1,-1};
490 posix_spawn_file_actions_t file_actions
, *file_actions_p
= NULL
;
493 for (auto &arg
: command
)
494 argv
.push_back(&arg
[0]);
495 argv
.push_back(nullptr);
497 if (pipe(send
) != 0) {
498 log_error("pipe failed: %s\n", strerror(errno
));
501 if (pipe(recv
) != 0) {
502 log_error("pipe failed: %s\n", strerror(errno
));
506 if (posix_spawn_file_actions_init(&file_actions
) != 0) {
507 log_error("posix_spawn_file_actions_init failed: %s\n", strerror(errno
));
510 file_actions_p
= &file_actions
;
511 if (posix_spawn_file_actions_adddup2(file_actions_p
, send
[0], STDIN_FILENO
) != 0) {
512 log_error("posix_spawn_file_actions_adddup2 failed: %s\n", strerror(errno
));
515 if (posix_spawn_file_actions_addclose(file_actions_p
, send
[1]) != 0) {
516 log_error("posix_spawn_file_actions_addclose failed: %s\n", strerror(errno
));
519 if (posix_spawn_file_actions_adddup2(file_actions_p
, recv
[1], STDOUT_FILENO
) != 0) {
520 log_error("posix_spawn_file_actions_adddup2 failed: %s\n", strerror(errno
));
523 if (posix_spawn_file_actions_addclose(file_actions_p
, recv
[0]) != 0) {
524 log_error("posix_spawn_file_actions_addclose failed: %s\n", strerror(errno
));
528 if (posix_spawnp(&pid
, argv
[0], file_actions_p
, /*attrp=*/NULL
, argv
.data(), environ
) != 0) {
529 log_error("posix_spawnp failed: %s\n", strerror(errno
));
533 server
= std::make_shared
<FdRpcServer
>(command_line
, send
[1], recv
[0], pid
);
538 if (send
[0] != -1) close(send
[0]);
539 if (send
[1] != -1) close(send
[1]);
540 if (recv
[0] != -1) close(recv
[0]);
541 if (recv
[1] != -1) close(recv
[1]);
542 if (file_actions_p
!= NULL
)
543 posix_spawn_file_actions_destroy(file_actions_p
);
545 } else if (!path
.empty()) {
547 std::wstring path_w
= str2wstr(path
);
550 h
= CreateFileW(path_w
.c_str(), GENERIC_READ
|GENERIC_WRITE
, /*dwShareMode=*/0, /*lpSecurityAttributes=*/NULL
, /*dwCreationDisposition=*/OPEN_EXISTING
, /*dwFlagsAndAttributes=*/0, /*hTemplateFile=*/NULL
);
551 if (h
== INVALID_HANDLE_VALUE
) {
552 log_error("CreateFileW failed: %s\n", get_last_error_str().c_str());
556 server
= std::make_shared
<HandleRpcServer
>(path
, h
, h
);
561 struct sockaddr_un addr
;
562 addr
.sun_family
= AF_UNIX
;
563 strncpy(addr
.sun_path
, path
.c_str(), sizeof(addr
.sun_path
) - 1);
565 int fd
= socket(AF_UNIX
, SOCK_STREAM
, 0);
567 log_error("socket failed: %s\n", strerror(errno
));
571 if (connect(fd
, (struct sockaddr
*)&addr
, sizeof(addr
)) != 0) {
572 log_error("connect failed: %s\n", strerror(errno
));
576 server
= std::make_shared
<FdRpcServer
>(path
, fd
, fd
);
580 if (fd
!= -1) close(fd
);
585 log_cmd_error("Failed to connect to RPC frontend.\n");
587 for (auto &module_name
: server
->get_module_names()) {
588 log("Linking module `%s'.\n", module_name
.c_str());
589 RpcModule
*module
= new RpcModule
;
590 module
->name
= "$abstract\\" + module_name
;
591 module
->server
= server
;