Merge pull request #951 from YosysHQ/clifford/logdebug
[yosys.git] / passes / techmap / techmap.cc
index 29ce96766e7b00224592cbeada94901013b26fa7..ab0bd3b54f034361ae8cf68083e1fda8bf3ad317 100644 (file)
@@ -2,11 +2,11 @@
  *  yosys -- Yosys Open SYnthesis Suite
  *
  *  Copyright (C) 2012  Clifford Wolf <clifford@clifford.at>
- *  
+ *
  *  Permission to use, copy, modify, and/or distribute this software for any
  *  purpose with or without fee is hereby granted, provided that the above
  *  copyright notice and this permission notice appear in all copies.
- *  
+ *
  *  THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
  *  WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
  *  MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
@@ -18,7 +18,7 @@
  */
 
 #include "kernel/yosys.h"
-#include "kernel/toposort.h"
+#include "kernel/utils.h"
 #include "kernel/sigtools.h"
 #include "libs/sha1/sha1.h"
 
 #include <stdio.h>
 #include <string.h>
 
+#include "simplemap.h"
 #include "passes/techmap/techmap.inc"
 
-// see simplemap.cc
-extern void simplemap_get_mappers(std::map<RTLIL::IdString, void(*)(RTLIL::Module*, RTLIL::Cell*)> &mappers);
+YOSYS_NAMESPACE_BEGIN
+
+// see maccmap.cc
+extern void maccmap(RTLIL::Module *module, RTLIL::Cell *cell, bool unmap = false);
+
+YOSYS_NAMESPACE_END
 
-static void apply_prefix(std::string prefix, std::string &id)
+USING_YOSYS_NAMESPACE
+PRIVATE_NAMESPACE_BEGIN
+
+void apply_prefix(std::string prefix, std::string &id)
 {
        if (id[0] == '\\')
                id = prefix + "." + id.substr(1);
@@ -39,9 +47,9 @@ static void apply_prefix(std::string prefix, std::string &id)
                id = "$techmap" + prefix + "." + id;
 }
 
-static void apply_prefix(std::string prefix, RTLIL::SigSpec &sig, RTLIL::Module *module)
+void apply_prefix(std::string prefix, RTLIL::SigSpec &sig, RTLIL::Module *module)
 {
-       std::vector<RTLIL::SigChunk> chunks = sig;
+       vector<SigChunk> chunks = sig;
        for (auto &chunk : chunks)
                if (chunk.wire != NULL) {
                        std::string wire_name = chunk.wire->name.str();
@@ -58,6 +66,13 @@ struct TechmapWorker
        std::map<std::pair<RTLIL::IdString, std::map<RTLIL::IdString, RTLIL::Const>>, RTLIL::Module*> techmap_cache;
        std::map<RTLIL::Module*, bool> techmap_do_cache;
        std::set<RTLIL::Module*, RTLIL::IdString::compare_ptr_by_name<RTLIL::Module>> module_queue;
+       dict<Module*, SigMap> sigmaps;
+
+       pool<IdString> flatten_do_list;
+       pool<IdString> flatten_done_list;
+       pool<Cell*> flatten_keep_list;
+
+       pool<string> log_msg_cache;
 
        struct TechmapWireData {
                RTLIL::Wire *wire;
@@ -70,6 +85,8 @@ struct TechmapWorker
        bool assert_mode;
        bool flatten_mode;
        bool recursive_mode;
+       bool autoproc_mode;
+       bool ignore_wb;
 
        TechmapWorker()
        {
@@ -77,6 +94,8 @@ struct TechmapWorker
                assert_mode = false;
                flatten_mode = false;
                recursive_mode = false;
+               autoproc_mode = false;
+               ignore_wb = false;
        }
 
        std::string constmap_tpl_name(SigMap &sigmap, RTLIL::Module *tpl, RTLIL::Cell *cell, bool verbose)
@@ -85,7 +104,7 @@ struct TechmapWorker
                std::map<RTLIL::SigBit, std::pair<RTLIL::IdString, int>> connbits_map;
 
                for (auto conn : cell->connections())
-                       for (int i = 0; i < SIZE(conn.second); i++) {
+                       for (int i = 0; i < GetSize(conn.second); i++) {
                                RTLIL::SigBit bit = sigmap(conn.second[i]);
                                if (bit.wire == nullptr) {
                                        if (verbose)
@@ -97,8 +116,10 @@ struct TechmapWorker
                                                                connbits_map.at(bit).second, log_id(connbits_map.at(bit).first));
                                        constmap_info += stringf("|%s %d %s %d", log_id(conn.first), i,
                                                        log_id(connbits_map.at(bit).first), connbits_map.at(bit).second);
-                               } else
-                                       connbits_map[bit] = std::pair<RTLIL::IdString, int>(conn.first, i);stringf("%s %d", log_id(conn.first), i, bit.data);
+                               } else {
+                                       connbits_map[bit] = std::pair<RTLIL::IdString, int>(conn.first, i);
+                                       constmap_info += stringf("|%s %d", log_id(conn.first), i);
+                               }
                        }
 
                return stringf("$paramod$constmap:%s%s", sha1(constmap_info).c_str(), tpl->name.c_str());
@@ -117,7 +138,7 @@ struct TechmapWorker
                                continue;
 
                        const char *q = strrchr(p+1, '.');
-                       p = q ? q : p+1;
+                       p = q ? q+1 : p+1;
 
                        if (!strncmp(p, "_TECHMAP_", 9)) {
                                TechmapWireData record;
@@ -141,24 +162,47 @@ struct TechmapWorker
 
        void techmap_module_worker(RTLIL::Design *design, RTLIL::Module *module, RTLIL::Cell *cell, RTLIL::Module *tpl)
        {
-               if (tpl->memories.size() != 0)
-                       log_error("Technology map yielded memories -> this is not supported.\n");
-
                if (tpl->processes.size() != 0) {
-                       log("Technology map yielded processes:\n");
+                       log("Technology map yielded processes:");
                        for (auto &it : tpl->processes)
-                               log("  %s",RTLIL::id2cstr(it.first));
-                       log_error("Technology map yielded processes -> this is not supported.\n");
+                               log(" %s",RTLIL::id2cstr(it.first));
+                       log("\n");
+                       if (autoproc_mode) {
+                               Pass::call_on_module(tpl->design, tpl, "proc");
+                               log_assert(GetSize(tpl->processes) == 0);
+                       } else
+                               log_error("Technology map yielded processes -> this is not supported (use -autoproc to run 'proc' automatically).\n");
                }
 
                std::string orig_cell_name;
-               if (!flatten_mode)
+               pool<string> extra_src_attrs = cell->get_strpool_attribute("\\src");
+
+               if (!flatten_mode) {
                        for (auto &it : tpl->cells_)
                                if (it.first == "\\_TECHMAP_REPLACE_") {
                                        orig_cell_name = cell->name.str();
                                        module->rename(cell, stringf("$techmap%d", autoidx++) + cell->name.str());
                                        break;
                                }
+               }
+
+               dict<IdString, IdString> memory_renames;
+
+               for (auto &it : tpl->memories) {
+                       std::string m_name = it.first.str();
+                       apply_prefix(cell->name.str(), m_name);
+                       RTLIL::Memory *m = new RTLIL::Memory;
+                       m->name = m_name;
+                       m->width = it.second->width;
+                       m->start_offset = it.second->start_offset;
+                       m->size = it.second->size;
+                       m->attributes = it.second->attributes;
+                       if (m->attributes.count("\\src"))
+                               m->add_strpool_attribute("\\src", extra_src_attrs);
+                       module->memories[m->name] = m;
+                       memory_renames[it.first] = m->name;
+                       design->select(module, m);
+               }
 
                std::map<RTLIL::IdString, RTLIL::IdString> positional_ports;
 
@@ -173,12 +217,28 @@ struct TechmapWorker
                        w->port_id = 0;
                        if (it.second->get_bool_attribute("\\_techmap_special_"))
                                w->attributes.clear();
+                       if (w->attributes.count("\\src"))
+                               w->add_strpool_attribute("\\src", extra_src_attrs);
                        design->select(module, w);
                }
 
+               SigMap tpl_sigmap(tpl);
+               pool<SigBit> tpl_written_bits;
+
+               for (auto &it1 : tpl->cells_)
+               for (auto &it2 : it1.second->connections_)
+                       if (it1.second->output(it2.first))
+                               for (auto bit : tpl_sigmap(it2.second))
+                                       tpl_written_bits.insert(bit);
+               for (auto &it1 : tpl->connections_)
+                       for (auto bit : tpl_sigmap(it1.first))
+                               tpl_written_bits.insert(bit);
+
                SigMap port_signal_map;
+               SigSig port_signal_assign;
 
-               for (auto &it : cell->connections()) {
+               for (auto &it : cell->connections())
+               {
                        RTLIL::IdString portname = it.first;
                        if (positional_ports.count(portname) > 0)
                                portname = positional_ports.at(portname);
@@ -187,41 +247,93 @@ struct TechmapWorker
                                        log_error("Can't map port `%s' of cell `%s' to template `%s'!\n", portname.c_str(), cell->name.c_str(), tpl->name.c_str());
                                continue;
                        }
+
+                       if (GetSize(it.second) == 0)
+                               continue;
+
                        RTLIL::Wire *w = tpl->wires_.at(portname);
-                       RTLIL::SigSig c;
-                       if (w->port_output) {
+                       RTLIL::SigSig c, extra_connect;
+
+                       if (w->port_output && !w->port_input) {
                                c.first = it.second;
                                c.second = RTLIL::SigSpec(w);
                                apply_prefix(cell->name.str(), c.second, module);
-                       } else {
+                               extra_connect.first = c.second;
+                               extra_connect.second = c.first;
+                       } else if (!w->port_output && w->port_input) {
                                c.first = RTLIL::SigSpec(w);
                                c.second = it.second;
                                apply_prefix(cell->name.str(), c.first, module);
+                               extra_connect.first = c.first;
+                               extra_connect.second = c.second;
+                       } else {
+                               SigSpec sig_tpl = w, sig_tpl_pf = w, sig_mod = it.second;
+                               apply_prefix(cell->name.str(), sig_tpl_pf, module);
+                               for (int i = 0; i < GetSize(sig_tpl) && i < GetSize(sig_mod); i++) {
+                                       if (tpl_written_bits.count(tpl_sigmap(sig_tpl[i]))) {
+                                               c.first.append(sig_mod[i]);
+                                               c.second.append(sig_tpl_pf[i]);
+                                       } else {
+                                               c.first.append(sig_tpl_pf[i]);
+                                               c.second.append(sig_mod[i]);
+                                       }
+                               }
+                               extra_connect.first = sig_tpl_pf;
+                               extra_connect.second = sig_mod;
                        }
+
                        if (c.second.size() > c.first.size())
                                c.second.remove(c.first.size(), c.second.size() - c.first.size());
+
                        if (c.second.size() < c.first.size())
                                c.second.append(RTLIL::SigSpec(RTLIL::State::S0, c.first.size() - c.second.size()));
+
                        log_assert(c.first.size() == c.second.size());
-                       if (flatten_mode) {
+
+                       if (flatten_mode)
+                       {
                                // more conservative approach:
                                // connect internal and external wires
+
+                               if (sigmaps.count(module) == 0)
+                                       sigmaps[module].set(module);
+
+                               if (sigmaps.at(module)(c.first).has_const())
+                                       log_error("Mismatch in directionality for cell port %s.%s.%s: %s <= %s\n",
+                                               log_id(module), log_id(cell), log_id(it.first), log_signal(c.first), log_signal(c.second));
+
                                module->connect(c);
-                       } else {
+                       }
+                       else
+                       {
                                // approach that yields nicer outputs:
                                // replace internal wires that are connected to external wires
-                               if (w->port_output)
+
+                               if (w->port_output && !w->port_input) {
                                        port_signal_map.add(c.second, c.first);
-                               else
+                               } else
+                               if (!w->port_output && w->port_input) {
                                        port_signal_map.add(c.first, c.second);
+                               } else {
+                                       module->connect(c);
+                                       extra_connect = SigSig();
+                               }
+
+                               for (auto &attr : w->attributes) {
+                                       if (attr.first == "\\src")
+                                               continue;
+                                       module->connect(extra_connect);
+                                       break;
+                               }
                        }
                }
 
                for (auto &it : tpl->cells_)
                {
                        std::string c_name = it.second->name.str();
+                       bool techmap_replace_cell = (!flatten_mode) && (c_name == "\\_TECHMAP_REPLACE_");
 
-                       if (!flatten_mode && c_name == "\\_TECHMAP_REPLACE_")
+                       if (techmap_replace_cell)
                                c_name = orig_cell_name;
                        else
                                apply_prefix(cell->name.str(), c_name);
@@ -236,6 +348,26 @@ struct TechmapWorker
                                apply_prefix(cell->name.str(), it2.second, module);
                                port_signal_map.apply(it2.second);
                        }
+
+                       if (c->type == "$memrd" || c->type == "$memwr" || c->type == "$meminit") {
+                               IdString memid = c->getParam("\\MEMID").decode_string();
+                               log_assert(memory_renames.count(memid) != 0);
+                               c->setParam("\\MEMID", Const(memory_renames[memid].str()));
+                       }
+
+                       if (c->type == "$mem") {
+                               string memid = c->getParam("\\MEMID").decode_string();
+                               apply_prefix(cell->name.str(), memid);
+                               c->setParam("\\MEMID", Const(memid));
+                       }
+
+                       if (c->attributes.count("\\src"))
+                               c->add_strpool_attribute("\\src", extra_src_attrs);
+
+                       if (techmap_replace_cell)
+                               for (auto attr : cell->attributes)
+                                       if (!c->attributes.count(attr.first))
+                                               c->attributes[attr.first] = attr.second;
                }
 
                for (auto &it : tpl->connections()) {
@@ -251,19 +383,20 @@ struct TechmapWorker
        }
 
        bool techmap_module(RTLIL::Design *design, RTLIL::Module *module, RTLIL::Design *map, std::set<RTLIL::Cell*> &handled_cells,
-                       const std::map<RTLIL::IdString, std::set<RTLIL::IdString>> &celltypeMap, bool in_recursion)
+                       const std::map<RTLIL::IdString, std::set<RTLIL::IdString, RTLIL::sort_by_id_str>> &celltypeMap, bool in_recursion)
        {
                std::string mapmsg_prefix = in_recursion ? "Recursively mapping" : "Mapping";
 
-               if (!design->selected(module))
+               if (!design->selected(module) || module->get_blackbox_attribute(ignore_wb))
                        return false;
 
                bool log_continue = false;
                bool did_something = false;
+               LogMakeDebugHdl mkdebug;
 
                SigMap sigmap(module);
 
-               TopoSort<RTLIL::Cell*> cells;
+               TopoSort<RTLIL::Cell*, RTLIL::IdString::compare_ptr_by_name<RTLIL::Cell>> cells;
                std::map<RTLIL::Cell*, std::set<RTLIL::SigBit>> cell_to_inbit;
                std::map<RTLIL::SigBit, std::set<RTLIL::Cell*>> outbit_to_cell;
 
@@ -282,12 +415,28 @@ struct TechmapWorker
                                continue;
                        }
 
+                       if (flatten_mode) {
+                               bool keepit = cell->get_bool_attribute("\\keep_hierarchy");
+                               for (auto &tpl_name : celltypeMap.at(cell_type))
+                                       if (map->modules_[tpl_name]->get_bool_attribute("\\keep_hierarchy"))
+                                               keepit = true;
+                               if (keepit) {
+                                       if (!flatten_keep_list[cell]) {
+                                               log("Keeping %s.%s (found keep_hierarchy property).\n", log_id(module), log_id(cell));
+                                               flatten_keep_list.insert(cell);
+                                       }
+                                       if (!flatten_done_list[cell->type])
+                                               flatten_do_list.insert(cell->type);
+                                       continue;
+                               }
+                       }
+
                        for (auto &conn : cell->connections())
                        {
                                RTLIL::SigSpec sig = sigmap(conn.second);
                                sig.remove_const();
 
-                               if (SIZE(sig) == 0)
+                               if (GetSize(sig) == 0)
                                        continue;
 
                                for (auto &tpl_name : celltypeMap.at(cell_type)) {
@@ -318,6 +467,7 @@ struct TechmapWorker
                        bool mapped_cell = false;
 
                        std::string cell_type = cell->type.str();
+
                        if (in_recursion && cell_type.substr(0, 2) == "\\$")
                                cell_type = cell_type.substr(1);
 
@@ -325,34 +475,51 @@ struct TechmapWorker
                        {
                                RTLIL::IdString derived_name = tpl_name;
                                RTLIL::Module *tpl = map->modules_[tpl_name];
-                               std::map<RTLIL::IdString, RTLIL::Const> parameters = cell->parameters;
+                               std::map<RTLIL::IdString, RTLIL::Const> parameters(cell->parameters.begin(), cell->parameters.end());
 
-                               if (tpl->get_bool_attribute("\\blackbox"))
+                               if (tpl->get_blackbox_attribute(ignore_wb))
                                        continue;
 
                                if (!flatten_mode)
                                {
+                                       std::string extmapper_name;
+
                                        if (tpl->get_bool_attribute("\\techmap_simplemap"))
+                                               extmapper_name = "simplemap";
+
+                                       if (tpl->get_bool_attribute("\\techmap_maccmap"))
+                                               extmapper_name = "maccmap";
+
+                                       if (tpl->attributes.count("\\techmap_wrap"))
+                                               extmapper_name = "wrap";
+
+                                       if (!extmapper_name.empty())
                                        {
                                                cell->type = cell_type;
 
-                                               if (extern_mode && !in_recursion)
+                                               if ((extern_mode && !in_recursion) || extmapper_name == "wrap")
                                                {
-                                                       std::string m_name = stringf("$extern:simplemap:%s", log_id(cell->type));
+                                                       std::string m_name = stringf("$extern:%s:%s", extmapper_name.c_str(), log_id(cell->type));
 
                                                        for (auto &c : cell->parameters)
                                                                m_name += stringf(":%s=%s", log_id(c.first), log_signal(c.second));
 
-                                                       RTLIL::Module *simplemap_module = design->module(m_name);
+                                                       if (extmapper_name == "wrap")
+                                                               m_name += ":" + sha1(tpl->attributes.at("\\techmap_wrap").decode_string());
 
-                                                       if (simplemap_module == nullptr)
+                                                       RTLIL::Design *extmapper_design = extern_mode && !in_recursion ? design : tpl->design;
+                                                       RTLIL::Module *extmapper_module = extmapper_design->module(m_name);
+
+                                                       if (extmapper_module == nullptr)
                                                        {
-                                                               simplemap_module = design->addModule(m_name);
-                                                               RTLIL::Cell *simplemap_cell = simplemap_module->addCell(cell->type, cell);
+                                                               extmapper_module = extmapper_design->addModule(m_name);
+                                                               RTLIL::Cell *extmapper_cell = extmapper_module->addCell(cell->type, cell);
+
+                                                               extmapper_cell->set_src_attribute(cell->get_src_attribute());
 
                                                                int port_counter = 1;
-                                                               for (auto &c : simplemap_cell->connections_) {
-                                                                       RTLIL::Wire *w = simplemap_module->addWire(c.first, SIZE(c.second));
+                                                               for (auto &c : extmapper_cell->connections_) {
+                                                                       RTLIL::Wire *w = extmapper_module->addWire(c.first, GetSize(c.second));
                                                                        if (w->name == "\\Y" || w->name == "\\Q")
                                                                                w->port_output = true;
                                                                        else
@@ -361,25 +528,70 @@ struct TechmapWorker
                                                                        c.second = w;
                                                                }
 
-                                                               simplemap_module->check();
+                                                               extmapper_module->fixup_ports();
+                                                               extmapper_module->check();
+
+                                                               if (extmapper_name == "simplemap") {
+                                                                       log("Creating %s with simplemap.\n", log_id(extmapper_module));
+                                                                       if (simplemap_mappers.count(extmapper_cell->type) == 0)
+                                                                               log_error("No simplemap mapper for cell type %s found!\n", log_id(extmapper_cell->type));
+                                                                       simplemap_mappers.at(extmapper_cell->type)(extmapper_module, extmapper_cell);
+                                                                       extmapper_module->remove(extmapper_cell);
+                                                               }
+
+                                                               if (extmapper_name == "maccmap") {
+                                                                       log("Creating %s with maccmap.\n", log_id(extmapper_module));
+                                                                       if (extmapper_cell->type != "$macc")
+                                                                               log_error("The maccmap mapper can only map $macc (not %s) cells!\n", log_id(extmapper_cell->type));
+                                                                       maccmap(extmapper_module, extmapper_cell);
+                                                                       extmapper_module->remove(extmapper_cell);
+                                                               }
 
-                                                               log("Creating %s with simplemap.\n", log_id(simplemap_module));
-                                                               if (simplemap_mappers.count(simplemap_cell->type) == 0)
-                                                                       log_error("No simplemap mapper for cell type %s found!\n", RTLIL::id2cstr(simplemap_cell->type));
-                                                               simplemap_mappers.at(simplemap_cell->type)(simplemap_module, simplemap_cell);
-                                                               simplemap_module->remove(simplemap_cell);
+                                                               if (extmapper_name == "wrap") {
+                                                                       std::string cmd_string = tpl->attributes.at("\\techmap_wrap").decode_string();
+                                                                       log("Running \"%s\" on wrapper %s.\n", cmd_string.c_str(), log_id(extmapper_module));
+                                                                       mkdebug.on();
+                                                                       Pass::call_on_module(extmapper_design, extmapper_module, cmd_string);
+                                                                       log_continue = true;
+                                                               }
                                                        }
 
-                                                       log("%s %s.%s (%s) to %s.\n", mapmsg_prefix.c_str(), log_id(module), log_id(cell), log_id(cell->type), log_id(simplemap_module));
-                                                       cell->type = simplemap_module->name;
+                                                       cell->type = extmapper_module->name;
                                                        cell->parameters.clear();
+
+                                                       if (!extern_mode || in_recursion) {
+                                                               tpl = extmapper_module;
+                                                               goto use_wrapper_tpl;
+                                                       }
+
+                                                       auto msg = stringf("Using extmapper %s for cells of type %s.", log_id(extmapper_module), log_id(cell->type));
+                                                       if (!log_msg_cache.count(msg)) {
+                                                               log_msg_cache.insert(msg);
+                                                               log("%s\n", msg.c_str());
+                                                       }
+                                                       log_debug("%s %s.%s (%s) to %s.\n", mapmsg_prefix.c_str(), log_id(module), log_id(cell), log_id(cell->type), log_id(extmapper_module));
                                                }
                                                else
                                                {
-                                                       log("%s %s.%s (%s) with simplemap.\n", mapmsg_prefix.c_str(), log_id(module), log_id(cell), log_id(cell->type));
-                                                       if (simplemap_mappers.count(cell->type) == 0)
-                                                               log_error("No simplemap mapper for cell type %s found!\n", RTLIL::id2cstr(cell->type));
-                                                       simplemap_mappers.at(cell->type)(module, cell);
+                                                       auto msg = stringf("Using extmapper %s for cells of type %s.", extmapper_name.c_str(), log_id(cell->type));
+                                                       if (!log_msg_cache.count(msg)) {
+                                                               log_msg_cache.insert(msg);
+                                                               log("%s\n", msg.c_str());
+                                                       }
+                                                       log_debug("%s %s.%s (%s) with %s.\n", mapmsg_prefix.c_str(), log_id(module), log_id(cell), log_id(cell->type), extmapper_name.c_str());
+
+                                                       if (extmapper_name == "simplemap") {
+                                                               if (simplemap_mappers.count(cell->type) == 0)
+                                                                       log_error("No simplemap mapper for cell type %s found!\n", RTLIL::id2cstr(cell->type));
+                                                               simplemap_mappers.at(cell->type)(module, cell);
+                                                       }
+
+                                                       if (extmapper_name == "maccmap") {
+                                                               if (cell->type != "$macc")
+                                                                       log_error("The maccmap mapper can only map $macc (not %s) cells!\n", log_id(cell->type));
+                                                               maccmap(module, cell);
+                                                       }
+
                                                        module->remove(cell);
                                                        cell = NULL;
                                                }
@@ -455,16 +667,22 @@ struct TechmapWorker
                                                }
                                }
 
-                               std::pair<RTLIL::IdString, std::map<RTLIL::IdString, RTLIL::Const>> key(tpl_name, parameters);
-                               if (techmap_cache.count(key) > 0) {
-                                       tpl = techmap_cache[key];
+                               if (0) {
+                       use_wrapper_tpl:;
+                                       // do not register techmap_wrap modules with techmap_cache
                                } else {
-                                       if (cell->parameters.size() != 0) {
-                                               derived_name = tpl->derive(map, parameters);
-                                               tpl = map->module(derived_name);
-                                               log_continue = true;
+                                       std::pair<RTLIL::IdString, std::map<RTLIL::IdString, RTLIL::Const>> key(tpl_name, parameters);
+                                       if (techmap_cache.count(key) > 0) {
+                                               tpl = techmap_cache[key];
+                                       } else {
+                                               if (parameters.size() != 0) {
+                                                       mkdebug.on();
+                                                       derived_name = tpl->derive(map, dict<RTLIL::IdString, RTLIL::Const>(parameters.begin(), parameters.end()));
+                                                       tpl = map->module(derived_name);
+                                                       log_continue = true;
+                                               }
+                                               techmap_cache[key] = tpl;
                                        }
-                                       techmap_cache[key] = tpl;
                                }
 
                                if (flatten_mode) {
@@ -560,7 +778,7 @@ struct TechmapWorker
                                                                }
 
                                                                for (auto conn : cell->connections())
-                                                                       for (int i = 0; i < SIZE(conn.second); i++)
+                                                                       for (int i = 0; i < GetSize(conn.second); i++)
                                                                        {
                                                                                RTLIL::SigBit bit = sigmap(conn.second[i]);
                                                                                RTLIL::SigBit tplbit(tpl->wire(conn.first), i);
@@ -626,8 +844,9 @@ struct TechmapWorker
 
                                        if (recursive_mode) {
                                                if (log_continue) {
-                                                       log_header("Continuing TECHMAP pass.\n");
+                                                       log_header(design, "Continuing TECHMAP pass.\n");
                                                        log_continue = false;
+                                                       mkdebug.off();
                                                }
                                                while (techmap_module(map, tpl, map, handled_cells, celltypeMap, true)) { }
                                        }
@@ -637,8 +856,9 @@ struct TechmapWorker
                                        continue;
 
                                if (log_continue) {
-                                       log_header("Continuing TECHMAP pass.\n");
+                                       log_header(design, "Continuing TECHMAP pass.\n");
                                        log_continue = false;
+                                       mkdebug.off();
                                }
 
                                if (extern_mode && !in_recursion)
@@ -658,13 +878,18 @@ struct TechmapWorker
                                                module_queue.insert(m);
                                        }
 
-                                       log("%s %s.%s to imported %s.\n", mapmsg_prefix.c_str(), log_id(module), log_id(cell), log_id(m_name));
+                                       log_debug("%s %s.%s to imported %s.\n", mapmsg_prefix.c_str(), log_id(module), log_id(cell), log_id(m_name));
                                        cell->type = m_name;
                                        cell->parameters.clear();
                                }
                                else
                                {
-                                       log("%s %s.%s using %s.\n", mapmsg_prefix.c_str(), log_id(module), log_id(cell), log_id(tpl));
+                                       auto msg = stringf("Using template %s for cells of type %s.", log_id(tpl), log_id(cell->type));
+                                       if (!log_msg_cache.count(msg)) {
+                                               log_msg_cache.insert(msg);
+                                               log("%s\n", msg.c_str());
+                                       }
+                                       log_debug("%s %s.%s (%s) using %s.\n", mapmsg_prefix.c_str(), log_id(module), log_id(cell), log_id(cell->type), log_id(tpl));
                                        techmap_module_worker(design, module, cell, tpl);
                                        cell = NULL;
                                }
@@ -680,8 +905,9 @@ struct TechmapWorker
                }
 
                if (log_continue) {
-                       log_header("Continuing TECHMAP pass.\n");
+                       log_header(design, "Continuing TECHMAP pass.\n");
                        log_continue = false;
+                       mkdebug.off();
                }
 
                return did_something;
@@ -690,14 +916,14 @@ struct TechmapWorker
 
 struct TechmapPass : public Pass {
        TechmapPass() : Pass("techmap", "generic technology mapper") { }
-       virtual void help()
+       void help() YS_OVERRIDE
        {
                //   |---v---|---v---|---v---|---v---|---v---|---v---|---v---|---v---|---v---|---v---|
                log("\n");
                log("    techmap [-map filename] [selection]\n");
                log("\n");
                log("This pass implements a very simple technology mapper that replaces cells in\n");
-               log("the design with implementations given in form of a verilog or ilang source\n");
+               log("the design with implementations given in form of a Verilog or ilang source\n");
                log("file.\n");
                log("\n");
                log("    -map filename\n");
@@ -706,10 +932,8 @@ struct TechmapPass : public Pass {
                log("        transforms the internal RTL cells to the internal gate\n");
                log("        library.\n");
                log("\n");
-               log("    -share_map filename\n");
-               log("        like -map, but look for the file in the share directory (where the\n");
-               log("        yosys data files are). this is mainly used internally when techmap\n");
-               log("        is called from other commands.\n");
+               log("    -map %%<design-name>\n");
+               log("        like -map above, but with an in-memory design instead of a file.\n");
                log("\n");
                log("    -extern\n");
                log("        load the cell implementations as separate modules into the design\n");
@@ -720,18 +944,24 @@ struct TechmapPass : public Pass {
                log("\n");
                log("    -recursive\n");
                log("        instead of the iterative breadth-first algorithm use a recursive\n");
-               log("        depth-first algorithm. both methods should yield equivialent results,\n");
+               log("        depth-first algorithm. both methods should yield equivalent results,\n");
                log("        but may differ in performance.\n");
                log("\n");
+               log("    -autoproc\n");
+               log("        Automatically call \"proc\" on implementations that contain processes.\n");
+               log("\n");
+               log("    -wb\n");
+               log("        Ignore the 'whitebox' attribute on cell implementations.\n");
+               log("\n");
                log("    -assert\n");
                log("        this option will cause techmap to exit with an error if it can't map\n");
                log("        a selected cell. only cell types that end on an underscore are accepted\n");
                log("        as final cell types by this mode.\n");
                log("\n");
                log("    -D <define>, -I <incdir>\n");
-               log("        this options are passed as-is to the verilog frontend for loading the\n");
-               log("        map file. Note that the verilog frontend is also called with the\n");
-               log("        '-ignore_redef' option set.\n");
+               log("        this options are passed as-is to the Verilog frontend for loading the\n");
+               log("        map file. Note that the Verilog frontend is also called with the\n");
+               log("        '-nooverwrite' option set.\n");
                log("\n");
                log("When a module in the map file has the 'techmap_celltype' attribute set, it will\n");
                log("match cells with a type that match the text value of this attribute. Otherwise\n");
@@ -740,6 +970,13 @@ struct TechmapPass : public Pass {
                log("When a module in the map file has the 'techmap_simplemap' attribute set, techmap\n");
                log("will use 'simplemap' (see 'help simplemap') to map cells matching the module.\n");
                log("\n");
+               log("When a module in the map file has the 'techmap_maccmap' attribute set, techmap\n");
+               log("will use 'maccmap' (see 'help maccmap') to map cells matching the module.\n");
+               log("\n");
+               log("When a module in the map file has the 'techmap_wrap' attribute set, techmap\n");
+               log("will create a wrapper for the cell and then run the command string that the\n");
+               log("attribute is set to on the wrapper module.\n");
+               log("\n");
                log("All wires in the modules from the map file matching the pattern _TECHMAP_*\n");
                log("or *._TECHMAP_* are special wires that are used to pass instructions from\n");
                log("the mapping module to the techmap command. At the moment the following special\n");
@@ -769,7 +1006,7 @@ struct TechmapPass : public Pass {
                log("        of constant inputs and shorted inputs at this point and import the\n");
                log("        constant and connected bits into the map module. All further commands\n");
                log("        are executed in this copy. This is a very convenient way of creating\n");
-               log("        optimizied specializations of techmap modules without using the special\n");
+               log("        optimized specializations of techmap modules without using the special\n");
                log("        parameters described below.\n");
                log("\n");
                log("        A _TECHMAP_DO_* command may start with the special token 'RECURSION; '.\n");
@@ -805,37 +1042,32 @@ struct TechmapPass : public Pass {
                log("constant value.\n");
                log("\n");
                log("A cell with the name _TECHMAP_REPLACE_ in the map file will inherit the name\n");
-               log("of the cell that is beeing replaced.\n");
+               log("and attributes of the cell that is being replaced.\n");
                log("\n");
                log("See 'help extract' for a pass that does the opposite thing.\n");
                log("\n");
                log("See 'help flatten' for a pass that does flatten the design (which is\n");
-               log("esentially techmap but using the design itself as map library).\n");
+               log("essentially techmap but using the design itself as map library).\n");
                log("\n");
        }
-       virtual void execute(std::vector<std::string> args, RTLIL::Design *design)
+       void execute(std::vector<std::string> args, RTLIL::Design *design) YS_OVERRIDE
        {
-               log_header("Executing TECHMAP pass (map to technology primitives).\n");
+               log_header(design, "Executing TECHMAP pass (map to technology primitives).\n");
                log_push();
 
                TechmapWorker worker;
                simplemap_get_mappers(worker.simplemap_mappers);
 
                std::vector<std::string> map_files;
-               std::string verilog_frontend = "verilog -ignore_redef";
+               std::string verilog_frontend = "verilog -nooverwrite -noblackbox";
                int max_iter = -1;
 
                size_t argidx;
-               std::string proc_share_path = proc_share_dirname();
                for (argidx = 1; argidx < args.size(); argidx++) {
                        if (args[argidx] == "-map" && argidx+1 < args.size()) {
                                map_files.push_back(args[++argidx]);
                                continue;
                        }
-                       if (args[argidx] == "-share_map" && argidx+1 < args.size()) {
-                               map_files.push_back(proc_share_path + args[++argidx]);
-                               continue;
-                       }
                        if (args[argidx] == "-max_iter" && argidx+1 < args.size()) {
                                max_iter = atoi(args[++argidx].c_str());
                                continue;
@@ -860,16 +1092,23 @@ struct TechmapPass : public Pass {
                                worker.recursive_mode = true;
                                continue;
                        }
+                       if (args[argidx] == "-autoproc") {
+                               worker.autoproc_mode = true;
+                               continue;
+                       }
+                       if (args[argidx] == "-wb") {
+                               worker.ignore_wb = true;
+                               continue;
+                       }
                        break;
                }
                extra_args(args, argidx, design);
 
                RTLIL::Design *map = new RTLIL::Design;
                if (map_files.empty()) {
-                       FILE *f = fmemopen(stdcells_code, strlen(stdcells_code), "rt");
-                       Frontend::frontend_call(map, f, "<techmap.v>", verilog_frontend);
-                       fclose(f);
-               } else
+                       std::istringstream f(stdcells_code);
+                       Frontend::frontend_call(map, &f, "<techmap.v>", verilog_frontend);
+               } else {
                        for (auto &fn : map_files)
                                if (fn.substr(0, 1) == "%") {
                                        if (!saved_designs.count(fn.substr(1))) {
@@ -880,30 +1119,31 @@ struct TechmapPass : public Pass {
                                                if (!map->has(mod->name))
                                                        map->add(mod->clone());
                                } else {
-                                       FILE *f = fopen(fn.c_str(), "rt");
-                                       if (f == NULL)
+                                       std::ifstream f;
+                                       rewrite_filename(fn);
+                                       f.open(fn.c_str());
+                                       yosys_input_files.insert(fn);
+                                       if (f.fail())
                                                log_cmd_error("Can't open map file `%s'\n", fn.c_str());
-                                       Frontend::frontend_call(map, f, fn, (fn.size() > 3 && fn.substr(fn.size()-3) == ".il") ? "ilang" : verilog_frontend);
-                                       fclose(f);
+                                       Frontend::frontend_call(map, &f, fn, (fn.size() > 3 && fn.substr(fn.size()-3) == ".il") ? "ilang" : verilog_frontend);
                                }
-
-               std::map<RTLIL::IdString, RTLIL::Module*> modules_new;
-               for (auto &it : map->modules_) {
-                       if (it.first.substr(0, 2) == "\\$")
-                               it.second->name = it.first.substr(1);
-                       modules_new[it.second->name] = it.second;
                }
-               map->modules_.swap(modules_new);
 
-               std::map<RTLIL::IdString, std::set<RTLIL::IdString>> celltypeMap;
+               log_header(design, "Continuing TECHMAP pass.\n");
+
+               std::map<RTLIL::IdString, std::set<RTLIL::IdString, RTLIL::sort_by_id_str>> celltypeMap;
                for (auto &it : map->modules_) {
                        if (it.second->attributes.count("\\techmap_celltype") && !it.second->attributes.at("\\techmap_celltype").bits.empty()) {
                                char *p = strdup(it.second->attributes.at("\\techmap_celltype").decode_string().c_str());
                                for (char *q = strtok(p, " \t\r\n"); q; q = strtok(NULL, " \t\r\n"))
                                        celltypeMap[RTLIL::escape_id(q)].insert(it.first);
                                free(p);
-                       } else
-                               celltypeMap[it.first].insert(it.first);
+                       } else {
+                               string module_name = it.first.str();
+                               if (module_name.substr(0, 2) == "\\$")
+                                       module_name = module_name.substr(1);
+                               celltypeMap[module_name].insert(it.first);
+                       }
                }
 
                for (auto module : design->modules())
@@ -933,33 +1173,48 @@ struct TechmapPass : public Pass {
                log_pop();
        }
 } TechmapPass;
+
 struct FlattenPass : public Pass {
        FlattenPass() : Pass("flatten", "flatten design") { }
-       virtual void help()
+       void help() YS_OVERRIDE
        {
                //   |---v---|---v---|---v---|---v---|---v---|---v---|---v---|---v---|---v---|---v---|
                log("\n");
-               log("    flatten [selection]\n");
+               log("    flatten [options] [selection]\n");
                log("\n");
                log("This pass flattens the design by replacing cells by their implementation. This\n");
-               log("pass is very simmilar to the 'techmap' pass. The only difference is that this\n");
+               log("pass is very similar to the 'techmap' pass. The only difference is that this\n");
                log("pass is using the current design as mapping library.\n");
                log("\n");
+               log("Cells and/or modules with the 'keep_hierarchy' attribute set will not be\n");
+               log("flattened by this command.\n");
+               log("\n");
+               log("    -wb\n");
+               log("        Ignore the 'whitebox' attribute on cell implementations.\n");
+               log("\n");
        }
-       virtual void execute(std::vector<std::string> args, RTLIL::Design *design)
+       void execute(std::vector<std::string> args, RTLIL::Design *design) YS_OVERRIDE
        {
-               log_header("Executing FLATTEN pass (flatten design).\n");
+               log_header(design, "Executing FLATTEN pass (flatten design).\n");
                log_push();
 
-               extra_args(args, 1, design);
-
                TechmapWorker worker;
                worker.flatten_mode = true;
 
-               std::map<RTLIL::IdString, std::set<RTLIL::IdString>> celltypeMap;
-               for (auto &it : design->modules_)
-                       celltypeMap[it.first].insert(it.first);
+               size_t argidx;
+               for (argidx = 1; argidx < args.size(); argidx++) {
+                       if (args[argidx] == "-wb") {
+                               worker.ignore_wb = true;
+                               continue;
+                       }
+                       break;
+               }
+               extra_args(args, argidx, design);
+
+
+               std::map<RTLIL::IdString, std::set<RTLIL::IdString, RTLIL::sort_by_id_str>> celltypeMap;
+               for (auto module : design->modules())
+                       celltypeMap[module->name].insert(module->name);
 
                RTLIL::Module *top_mod = NULL;
                if (design->full_selection())
@@ -967,26 +1222,42 @@ struct FlattenPass : public Pass {
                                if (mod->get_bool_attribute("\\top"))
                                        top_mod = mod;
 
-               bool did_something = true;
                std::set<RTLIL::Cell*> handled_cells;
-               while (did_something) {
-                       did_something = false;
-                       if (top_mod != NULL) {
-                               if (worker.techmap_module(design, top_mod, design, handled_cells, celltypeMap, false))
-                                       did_something = true;
-                       } else {
-                               for (auto mod : design->modules())
-                                       if (worker.techmap_module(design, mod, design, handled_cells, celltypeMap, false))
-                                               did_something = true;
+               if (top_mod != NULL) {
+                       worker.flatten_do_list.insert(top_mod->name);
+                       while (!worker.flatten_do_list.empty()) {
+                               auto mod = design->module(*worker.flatten_do_list.begin());
+                               while (worker.techmap_module(design, mod, design, handled_cells, celltypeMap, false)) { }
+                               worker.flatten_done_list.insert(mod->name);
+                               worker.flatten_do_list.erase(mod->name);
+                       }
+               } else {
+                       for (auto mod : vector<Module*>(design->modules())) {
+                               while (worker.techmap_module(design, mod, design, handled_cells, celltypeMap, false)) { }
                        }
                }
 
+               log_suppressed();
                log("No more expansions possible.\n");
 
-               if (top_mod != NULL) {
-                       std::map<RTLIL::IdString, RTLIL::Module*> new_modules;
-                       for (auto mod : design->modules())
-                               if (mod == top_mod || mod->get_bool_attribute("\\blackbox")) {
+               if (top_mod != NULL)
+               {
+                       pool<RTLIL::IdString> used_modules, new_used_modules;
+                       new_used_modules.insert(top_mod->name);
+                       while (!new_used_modules.empty()) {
+                               pool<RTLIL::IdString> queue;
+                               queue.swap(new_used_modules);
+                               for (auto modname : queue)
+                                       used_modules.insert(modname);
+                               for (auto modname : queue)
+                                       for (auto cell : design->module(modname)->cells())
+                                               if (design->module(cell->type) && !used_modules[cell->type])
+                                                       new_used_modules.insert(cell->type);
+                       }
+
+                       dict<RTLIL::IdString, RTLIL::Module*> new_modules;
+                       for (auto mod : vector<Module*>(design->modules()))
+                               if (used_modules[mod->name] || mod->get_blackbox_attribute(worker.ignore_wb)) {
                                        new_modules[mod->name] = mod;
                                } else {
                                        log("Deleting now unused module %s.\n", log_id(mod));
@@ -999,3 +1270,4 @@ struct FlattenPass : public Pass {
        }
 } FlattenPass;
 
+PRIVATE_NAMESPACE_END