kernel/mem.cc

   1 /*
   2  *  yosys -- Yosys Open SYnthesis Suite
   3  *
   4  *  Copyright (C) 2020  Marcelina Kościelnicka <mwk@0x04.net>
   5  *
   6  *  Permission to use, copy, modify, and/or distribute this software for any
   7  *  purpose with or without fee is hereby granted, provided that the above
   8  *  copyright notice and this permission notice appear in all copies.
   9  *
  10  *  THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
  11  *  WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
  12  *  MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
  13  *  ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
  14  *  WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
  15  *  ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
  16  *  OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
  17  *
  18  */
  19
  20 #include "kernel/mem.h"
  21 #include "kernel/ff.h"
  22
  23 USING_YOSYS_NAMESPACE
  24
  25 void Mem::remove() {
  26         if (cell) {
  27                 module->remove(cell);
  28                 cell = nullptr;
  29         }
  30         if (mem) {
  31                 module->memories.erase(mem->name);
  32                 delete mem;
  33                 mem = nullptr;
  34         }
  35         for (auto &port : rd_ports) {
  36                 if (port.cell) {
  37                         module->remove(port.cell);
  38                         port.cell = nullptr;
  39                 }
  40         }
  41         for (auto &port : wr_ports) {
  42                 if (port.cell) {
  43                         module->remove(port.cell);
  44                         port.cell = nullptr;
  45                 }
  46         }
  47         for (auto &init : inits) {
  48                 if (init.cell) {
  49                         module->remove(init.cell);
  50                         init.cell = nullptr;
  51                 }
  52         }
  53 }
  54
  55 void Mem::emit() {
  56         check();
  57         std::vector<int> rd_left;
  58         for (int i = 0; i < GetSize(rd_ports); i++) {
  59                 auto &port = rd_ports[i];
  60                 if (port.removed) {
  61                         if (port.cell) {
  62                                 module->remove(port.cell);
  63                         }
  64                 } else {
  65                         rd_left.push_back(i);
  66                 }
  67         }
  68         std::vector<int> wr_left;
  69         for (int i = 0; i < GetSize(wr_ports); i++) {
  70                 auto &port = wr_ports[i];
  71                 if (port.removed) {
  72                         if (port.cell) {
  73                                 module->remove(port.cell);
  74                         }
  75                 } else {
  76                         wr_left.push_back(i);
  77                 }
  78         }
  79         for (int i = 0; i < GetSize(rd_left); i++)
  80                 if (i != rd_left[i])
  81                         std::swap(rd_ports[i], rd_ports[rd_left[i]]);
  82         rd_ports.resize(GetSize(rd_left));
  83         for (int i = 0; i < GetSize(wr_left); i++)
  84                 if (i != wr_left[i])
  85                         std::swap(wr_ports[i], wr_ports[wr_left[i]]);
  86         wr_ports.resize(GetSize(wr_left));
  87
  88         // for future: handle transparency mask here
  89
  90         for (auto &port : wr_ports) {
  91                 for (int i = 0; i < GetSize(wr_left); i++)
  92                         port.priority_mask[i] = port.priority_mask[wr_left[i]];
  93                 port.priority_mask.resize(GetSize(wr_left));
  94         }
  95
  96         if (packed) {
  97                 if (mem) {
  98                         module->memories.erase(mem->name);
  99                         delete mem;
 100                         mem = nullptr;
 101                 }
 102                 if (!cell) {
 103                         if (memid.empty())
 104                                 memid = NEW_ID;
 105                         cell = module->addCell(memid, ID($mem));
 106                 }
 107                 cell->attributes = attributes;
 108                 cell->parameters[ID::MEMID] = Const(memid.str());
 109                 cell->parameters[ID::WIDTH] = Const(width);
 110                 cell->parameters[ID::OFFSET] = Const(start_offset);
 111                 cell->parameters[ID::SIZE] = Const(size);
 112                 Const rd_wide_continuation, rd_clk_enable, rd_clk_polarity, rd_transparent;
 113                 Const wr_wide_continuation, wr_clk_enable, wr_clk_polarity;
 114                 SigSpec rd_clk, rd_en, rd_addr, rd_data;
 115                 SigSpec wr_clk, wr_en, wr_addr, wr_data;
 116                 int abits = 0;
 117                 for (auto &port : rd_ports)
 118                         abits = std::max(abits, GetSize(port.addr));
 119                 for (auto &port : wr_ports)
 120                         abits = std::max(abits, GetSize(port.addr));
 121                 cell->parameters[ID::ABITS] = Const(abits);
 122                 for (auto &port : rd_ports) {
 123                         // TODO: remove
 124                         log_assert(port.arst == State::S0);
 125                         log_assert(port.srst == State::S0);
 126                         log_assert(port.init_value == Const(State::Sx, width << port.wide_log2));
 127                         if (port.cell) {
 128                                 module->remove(port.cell);
 129                                 port.cell = nullptr;
 130                         }
 131                         for (int sub = 0; sub < (1 << port.wide_log2); sub++)
 132                         {
 133                                 rd_wide_continuation.bits.push_back(State(sub != 0));
 134                                 rd_clk_enable.bits.push_back(State(port.clk_enable));
 135                                 rd_clk_polarity.bits.push_back(State(port.clk_polarity));
 136                                 rd_transparent.bits.push_back(State(port.transparent));
 137                                 rd_clk.append(port.clk);
 138                                 rd_en.append(port.en);
 139                                 SigSpec addr = port.sub_addr(sub);
 140                                 addr.extend_u0(abits, false);
 141                                 rd_addr.append(addr);
 142                                 log_assert(GetSize(addr) == abits);
 143                         }
 144                         rd_data.append(port.data);
 145                 }
 146                 if (rd_ports.empty()) {
 147                         rd_wide_continuation = State::S0;
 148                         rd_clk_enable = State::S0;
 149                         rd_clk_polarity = State::S0;
 150                         rd_transparent = State::S0;
 151                 }
 152                 cell->parameters[ID::RD_PORTS] = Const(GetSize(rd_clk));
 153                 cell->parameters[ID::RD_CLK_ENABLE] = rd_clk_enable;
 154                 cell->parameters[ID::RD_CLK_POLARITY] = rd_clk_polarity;
 155                 cell->parameters[ID::RD_TRANSPARENT] = rd_transparent;
 156                 cell->setPort(ID::RD_CLK, rd_clk);
 157                 cell->setPort(ID::RD_EN, rd_en);
 158                 cell->setPort(ID::RD_ADDR, rd_addr);
 159                 cell->setPort(ID::RD_DATA, rd_data);
 160                 for (auto &port : wr_ports) {
 161                         if (port.cell) {
 162                                 module->remove(port.cell);
 163                                 port.cell = nullptr;
 164                         }
 165                         for (int sub = 0; sub < (1 << port.wide_log2); sub++)
 166                         {
 167                                 wr_wide_continuation.bits.push_back(State(sub != 0));
 168                                 wr_clk_enable.bits.push_back(State(port.clk_enable));
 169                                 wr_clk_polarity.bits.push_back(State(port.clk_polarity));
 170                                 wr_clk.append(port.clk);
 171                                 SigSpec addr = port.sub_addr(sub);
 172                                 addr.extend_u0(abits, false);
 173                                 wr_addr.append(addr);
 174                                 log_assert(GetSize(addr) == abits);
 175                         }
 176                         wr_en.append(port.en);
 177                         wr_data.append(port.data);
 178                 }
 179                 if (wr_ports.empty()) {
 180                         wr_wide_continuation = State::S0;
 181                         wr_clk_enable = State::S0;
 182                         wr_clk_polarity = State::S0;
 183                 }
 184                 cell->parameters[ID::WR_PORTS] = Const(GetSize(wr_clk));
 185                 cell->parameters[ID::WR_CLK_ENABLE] = wr_clk_enable;
 186                 cell->parameters[ID::WR_CLK_POLARITY] = wr_clk_polarity;
 187                 cell->setPort(ID::WR_CLK, wr_clk);
 188                 cell->setPort(ID::WR_EN, wr_en);
 189                 cell->setPort(ID::WR_ADDR, wr_addr);
 190                 cell->setPort(ID::WR_DATA, wr_data);
 191                 for (auto &init : inits) {
 192                         if (init.cell) {
 193                                 module->remove(init.cell);
 194                                 init.cell = nullptr;
 195                         }
 196                 }
 197                 cell->parameters[ID::INIT] = get_init_data();
 198         } else {
 199                 if (cell) {
 200                         module->remove(cell);
 201                         cell = nullptr;
 202                 }
 203                 if (!mem) {
 204                         if (memid.empty())
 205                                 memid = NEW_ID;
 206                         mem = new RTLIL::Memory;
 207                         mem->name = memid;
 208                         module->memories[memid] = mem;
 209                 }
 210                 mem->width = width;
 211                 mem->start_offset = start_offset;
 212                 mem->size = size;
 213                 for (auto &port : rd_ports) {
 214                         // TODO: remove
 215                         log_assert(port.arst == State::S0);
 216                         log_assert(port.srst == State::S0);
 217                         log_assert(port.init_value == Const(State::Sx, width << port.wide_log2));
 218                         if (!port.cell)
 219                                 port.cell = module->addCell(NEW_ID, ID($memrd));
 220                         port.cell->parameters[ID::MEMID] = memid.str();
 221                         port.cell->parameters[ID::ABITS] = GetSize(port.addr);
 222                         port.cell->parameters[ID::WIDTH] = width << port.wide_log2;
 223                         port.cell->parameters[ID::CLK_ENABLE] = port.clk_enable;
 224                         port.cell->parameters[ID::CLK_POLARITY] = port.clk_polarity;
 225                         port.cell->parameters[ID::TRANSPARENT] = port.transparent;
 226                         port.cell->setPort(ID::CLK, port.clk);
 227                         port.cell->setPort(ID::EN, port.en);
 228                         port.cell->setPort(ID::ADDR, port.addr);
 229                         port.cell->setPort(ID::DATA, port.data);
 230                 }
 231                 int idx = 0;
 232                 for (auto &port : wr_ports) {
 233                         if (!port.cell)
 234                                 port.cell = module->addCell(NEW_ID, ID($memwr));
 235                         port.cell->parameters[ID::MEMID] = memid.str();
 236                         port.cell->parameters[ID::ABITS] = GetSize(port.addr);
 237                         port.cell->parameters[ID::WIDTH] = width << port.wide_log2;
 238                         port.cell->parameters[ID::CLK_ENABLE] = port.clk_enable;
 239                         port.cell->parameters[ID::CLK_POLARITY] = port.clk_polarity;
 240                         port.cell->parameters[ID::PRIORITY] = idx++;
 241                         port.cell->setPort(ID::CLK, port.clk);
 242                         port.cell->setPort(ID::EN, port.en);
 243                         port.cell->setPort(ID::ADDR, port.addr);
 244                         port.cell->setPort(ID::DATA, port.data);
 245                 }
 246                 idx = 0;
 247                 for (auto &init : inits) {
 248                         if (!init.cell)
 249                                 init.cell = module->addCell(NEW_ID, ID($meminit));
 250                         init.cell->parameters[ID::MEMID] = memid.str();
 251                         init.cell->parameters[ID::ABITS] = GetSize(init.addr);
 252                         init.cell->parameters[ID::WIDTH] = width;
 253                         init.cell->parameters[ID::WORDS] = GetSize(init.data) / width;
 254                         init.cell->parameters[ID::PRIORITY] = idx++;
 255                         init.cell->setPort(ID::ADDR, init.addr);
 256                         init.cell->setPort(ID::DATA, init.data);
 257                 }
 258         }
 259 }
 260
 261 void Mem::clear_inits() {
 262         for (auto &init : inits)
 263                 if (init.cell)
 264                         module->remove(init.cell);
 265         inits.clear();
 266 }
 267
 268 Const Mem::get_init_data() const {
 269         Const init_data(State::Sx, width * size);
 270         for (auto &init : inits) {
 271                 int offset = (init.addr.as_int() - start_offset) * width;
 272                 for (int i = 0; i < GetSize(init.data); i++)
 273                         if (0 <= i+offset && i+offset < GetSize(init_data))
 274                                 init_data.bits[i+offset] = init.data.bits[i];
 275         }
 276         return init_data;
 277 }
 278
 279 void Mem::check() {
 280         int max_wide_log2 = 0;
 281         for (auto &port : rd_ports) {
 282                 if (port.removed)
 283                         continue;
 284                 log_assert(GetSize(port.clk) == 1);
 285                 log_assert(GetSize(port.en) == 1);
 286                 log_assert(GetSize(port.arst) == 1);
 287                 log_assert(GetSize(port.srst) == 1);
 288                 log_assert(GetSize(port.data) == (width << port.wide_log2));
 289                 log_assert(GetSize(port.init_value) == (width << port.wide_log2));
 290                 log_assert(GetSize(port.arst_value) == (width << port.wide_log2));
 291                 log_assert(GetSize(port.srst_value) == (width << port.wide_log2));
 292                 if (!port.clk_enable) {
 293                         log_assert(!port.transparent);
 294                         log_assert(port.arst == State::S0);
 295                         log_assert(port.srst == State::S0);
 296                 }
 297                 for (int j = 0; j < port.wide_log2; j++) {
 298                         log_assert(port.addr[j] == State::S0);
 299                 }
 300                 max_wide_log2 = std::max(max_wide_log2, port.wide_log2);
 301         }
 302         for (int i = 0; i < GetSize(wr_ports); i++) {
 303                 auto &port = wr_ports[i];
 304                 if (port.removed)
 305                         continue;
 306                 log_assert(GetSize(port.clk) == 1);
 307                 log_assert(GetSize(port.en) == (width << port.wide_log2));
 308                 log_assert(GetSize(port.data) == (width << port.wide_log2));
 309                 for (int j = 0; j < port.wide_log2; j++) {
 310                         log_assert(port.addr[j] == State::S0);
 311                 }
 312                 max_wide_log2 = std::max(max_wide_log2, port.wide_log2);
 313                 log_assert(GetSize(port.priority_mask) == GetSize(wr_ports));
 314                 for (int j = 0; j < GetSize(wr_ports); j++) {
 315                         auto &wport = wr_ports[j];
 316                         if (port.priority_mask[j] && !wport.removed) {
 317                                 log_assert(j < i);
 318                                 log_assert(port.clk_enable == wport.clk_enable);
 319                                 if (port.clk_enable) {
 320                                         log_assert(port.clk == wport.clk);
 321                                         log_assert(port.clk_polarity == wport.clk_polarity);
 322                                 }
 323                         }
 324                 }
 325         }
 326         int mask = (1 << max_wide_log2) - 1;
 327         log_assert(!(start_offset & mask));
 328         log_assert(!(size & mask));
 329 }
 330
 331 namespace {
 332
 333         struct MemIndex {
 334                 dict<IdString, pool<Cell *>> rd_ports;
 335                 dict<IdString, pool<Cell *>> wr_ports;
 336                 dict<IdString, pool<Cell *>> inits;
 337                 MemIndex (Module *module) {
 338                         for (auto cell: module->cells()) {
 339                                 if (cell->type == ID($memwr))
 340                                         wr_ports[cell->parameters.at(ID::MEMID).decode_string()].insert(cell);
 341                                 else if (cell->type == ID($memrd))
 342                                         rd_ports[cell->parameters.at(ID::MEMID).decode_string()].insert(cell);
 343                                 else if (cell->type == ID($meminit))
 344                                         inits[cell->parameters.at(ID::MEMID).decode_string()].insert(cell);
 345                         }
 346                 }
 347         };
 348
 349         Mem mem_from_memory(Module *module, RTLIL::Memory *mem, const MemIndex &index) {
 350                 Mem res(module, mem->name, mem->width, mem->start_offset, mem->size);
 351                 res.packed = false;
 352                 res.mem = mem;
 353                 res.attributes = mem->attributes;
 354                 if (index.rd_ports.count(mem->name)) {
 355                         for (auto cell : index.rd_ports.at(mem->name)) {
 356                                 MemRd mrd;
 357                                 mrd.cell = cell;
 358                                 mrd.attributes = cell->attributes;
 359                                 mrd.clk_enable = cell->parameters.at(ID::CLK_ENABLE).as_bool();
 360                                 mrd.clk_polarity = cell->parameters.at(ID::CLK_POLARITY).as_bool();
 361                                 mrd.transparent = cell->parameters.at(ID::TRANSPARENT).as_bool();
 362                                 mrd.clk = cell->getPort(ID::CLK);
 363                                 mrd.en = cell->getPort(ID::EN);
 364                                 mrd.addr = cell->getPort(ID::ADDR);
 365                                 mrd.data = cell->getPort(ID::DATA);
 366                                 mrd.wide_log2 = ceil_log2(GetSize(mrd.data) / mem->width);
 367                                 mrd.ce_over_srst = false;
 368                                 mrd.arst_value = Const(State::Sx, mem->width << mrd.wide_log2);
 369                                 mrd.srst_value = Const(State::Sx, mem->width << mrd.wide_log2);
 370                                 mrd.init_value = Const(State::Sx, mem->width << mrd.wide_log2);
 371                                 mrd.srst = State::S0;
 372                                 mrd.arst = State::S0;
 373                                 res.rd_ports.push_back(mrd);
 374                         }
 375                 }
 376                 if (index.wr_ports.count(mem->name)) {
 377                         std::vector<std::pair<int, MemWr>> ports;
 378                         for (auto cell : index.wr_ports.at(mem->name)) {
 379                                 MemWr mwr;
 380                                 mwr.cell = cell;
 381                                 mwr.attributes = cell->attributes;
 382                                 mwr.clk_enable = cell->parameters.at(ID::CLK_ENABLE).as_bool();
 383                                 mwr.clk_polarity = cell->parameters.at(ID::CLK_POLARITY).as_bool();
 384                                 mwr.clk = cell->getPort(ID::CLK);
 385                                 mwr.en = cell->getPort(ID::EN);
 386                                 mwr.addr = cell->getPort(ID::ADDR);
 387                                 mwr.data = cell->getPort(ID::DATA);
 388                                 mwr.wide_log2 = ceil_log2(GetSize(mwr.data) / mem->width);
 389                                 ports.push_back(std::make_pair(cell->parameters.at(ID::PRIORITY).as_int(), mwr));
 390                         }
 391                         std::sort(ports.begin(), ports.end(), [](const std::pair<int, MemWr> &a, const std::pair<int, MemWr> &b) { return a.first < b.first; });
 392                         for (auto &it : ports)
 393                                 res.wr_ports.push_back(it.second);
 394                 }
 395                 if (index.inits.count(mem->name)) {
 396                         std::vector<std::pair<int, MemInit>> inits;
 397                         for (auto cell : index.inits.at(mem->name)) {
 398                                 MemInit init;
 399                                 init.cell = cell;
 400                                 init.attributes = cell->attributes;
 401                                 auto addr = cell->getPort(ID::ADDR);
 402                                 auto data = cell->getPort(ID::DATA);
 403                                 if (!addr.is_fully_const())
 404                                         log_error("Non-constant address %s in memory initialization %s.\n", log_signal(addr), log_id(cell));
 405                                 if (!data.is_fully_const())
 406                                         log_error("Non-constant data %s in memory initialization %s.\n", log_signal(data), log_id(cell));
 407                                 init.addr = addr.as_const();
 408                                 init.data = data.as_const();
 409                                 inits.push_back(std::make_pair(cell->parameters.at(ID::PRIORITY).as_int(), init));
 410                         }
 411                         std::sort(inits.begin(), inits.end(), [](const std::pair<int, MemInit> &a, const std::pair<int, MemInit> &b) { return a.first < b.first; });
 412                         for (auto &it : inits)
 413                                 res.inits.push_back(it.second);
 414                 }
 415                 for (int i = 0; i < GetSize(res.wr_ports); i++) {
 416                         auto &port = res.wr_ports[i];
 417                         port.priority_mask.resize(GetSize(res.wr_ports));
 418                         for (int j = 0; j < i; j++) {
 419                                 auto &oport = res.wr_ports[j];
 420                                 if (port.clk_enable != oport.clk_enable)
 421                                         continue;
 422                                 if (port.clk_enable && port.clk != oport.clk)
 423                                         continue;
 424                                 if (port.clk_enable && port.clk_polarity != oport.clk_polarity)
 425                                         continue;
 426                                 port.priority_mask[j] = true;
 427                         }
 428                 }
 429                 res.check();
 430                 return res;
 431         }
 432
 433         Mem mem_from_cell(Cell *cell) {
 434                 Mem res(cell->module, cell->parameters.at(ID::MEMID).decode_string(),
 435                         cell->parameters.at(ID::WIDTH).as_int(),
 436                         cell->parameters.at(ID::OFFSET).as_int(),
 437                         cell->parameters.at(ID::SIZE).as_int()
 438                 );
 439                 int abits = cell->parameters.at(ID::ABITS).as_int();
 440                 res.packed = true;
 441                 res.cell = cell;
 442                 res.attributes = cell->attributes;
 443                 Const &init = cell->parameters.at(ID::INIT);
 444                 if (!init.is_fully_undef()) {
 445                         int pos = 0;
 446                         while (pos < res.size) {
 447                                 Const word = init.extract(pos * res.width, res.width, State::Sx);
 448                                 if (word.is_fully_undef()) {
 449                                         pos++;
 450                                 } else {
 451                                         int epos;
 452                                         for (epos = pos; epos < res.size; epos++) {
 453                                                 Const eword = init.extract(epos * res.width, res.width, State::Sx);
 454                                                 if (eword.is_fully_undef())
 455                                                         break;
 456                                         }
 457                                         MemInit minit;
 458                                         minit.addr = res.start_offset + pos;
 459                                         minit.data = init.extract(pos * res.width, (epos - pos) * res.width, State::Sx);
 460                                         res.inits.push_back(minit);
 461                                         pos = epos;
 462                                 }
 463                         }
 464                 }
 465                 for (int i = 0; i < cell->parameters.at(ID::RD_PORTS).as_int(); i++) {
 466                         MemRd mrd;
 467                         mrd.wide_log2 = 0;
 468                         mrd.clk_enable = cell->parameters.at(ID::RD_CLK_ENABLE).extract(i, 1).as_bool();
 469                         mrd.clk_polarity = cell->parameters.at(ID::RD_CLK_POLARITY).extract(i, 1).as_bool();
 470                         mrd.transparent = cell->parameters.at(ID::RD_TRANSPARENT).extract(i, 1).as_bool();
 471                         mrd.clk = cell->getPort(ID::RD_CLK).extract(i, 1);
 472                         mrd.en = cell->getPort(ID::RD_EN).extract(i, 1);
 473                         mrd.addr = cell->getPort(ID::RD_ADDR).extract(i * abits, abits);
 474                         mrd.data = cell->getPort(ID::RD_DATA).extract(i * res.width, res.width);
 475                         mrd.ce_over_srst = false;
 476                         mrd.arst_value = Const(State::Sx, res.width << mrd.wide_log2);
 477                         mrd.srst_value = Const(State::Sx, res.width << mrd.wide_log2);
 478                         mrd.init_value = Const(State::Sx, res.width << mrd.wide_log2);
 479                         mrd.srst = State::S0;
 480                         mrd.arst = State::S0;
 481                         res.rd_ports.push_back(mrd);
 482                 }
 483                 for (int i = 0; i < cell->parameters.at(ID::WR_PORTS).as_int(); i++) {
 484                         MemWr mwr;
 485                         mwr.wide_log2 = 0;
 486                         mwr.clk_enable = cell->parameters.at(ID::WR_CLK_ENABLE).extract(i, 1).as_bool();
 487                         mwr.clk_polarity = cell->parameters.at(ID::WR_CLK_POLARITY).extract(i, 1).as_bool();
 488                         mwr.clk = cell->getPort(ID::WR_CLK).extract(i, 1);
 489                         mwr.en = cell->getPort(ID::WR_EN).extract(i * res.width, res.width);
 490                         mwr.addr = cell->getPort(ID::WR_ADDR).extract(i * abits, abits);
 491                         mwr.data = cell->getPort(ID::WR_DATA).extract(i * res.width, res.width);
 492                         res.wr_ports.push_back(mwr);
 493                 }
 494                 for (int i = 0; i < GetSize(res.wr_ports); i++) {
 495                         auto &port = res.wr_ports[i];
 496                         port.priority_mask.resize(GetSize(res.wr_ports));
 497                         for (int j = 0; j < i; j++) {
 498                                 auto &oport = res.wr_ports[j];
 499                                 if (port.clk_enable != oport.clk_enable)
 500                                         continue;
 501                                 if (port.clk_enable && port.clk != oport.clk)
 502                                         continue;
 503                                 if (port.clk_enable && port.clk_polarity != oport.clk_polarity)
 504                                         continue;
 505                                 port.priority_mask[j] = true;
 506                         }
 507                 }
 508                 res.check();
 509                 return res;
 510         }
 511
 512 }
 513
 514 std::vector<Mem> Mem::get_all_memories(Module *module) {
 515         std::vector<Mem> res;
 516         MemIndex index(module);
 517         for (auto it: module->memories) {
 518                 res.push_back(mem_from_memory(module, it.second, index));
 519         }
 520         for (auto cell: module->cells()) {
 521                 if (cell->type == ID($mem))
 522                         res.push_back(mem_from_cell(cell));
 523         }
 524         return res;
 525 }
 526
 527 std::vector<Mem> Mem::get_selected_memories(Module *module) {
 528         std::vector<Mem> res;
 529         MemIndex index(module);
 530         for (auto it: module->memories) {
 531                 if (module->design->selected(module, it.second))
 532                         res.push_back(mem_from_memory(module, it.second, index));
 533         }
 534         for (auto cell: module->selected_cells()) {
 535                 if (cell->type == ID($mem))
 536                         res.push_back(mem_from_cell(cell));
 537         }
 538         return res;
 539 }
 540
 541 Cell *Mem::extract_rdff(int idx, FfInitVals *initvals) {
 542         MemRd &port = rd_ports[idx];
 543
 544         if (!port.clk_enable)
 545                 return nullptr;
 546
 547         Cell *c;
 548
 549         // There are two ways to handle rdff extraction when transparency is involved:
 550         //
 551         // - if all of the following conditions are true, put the FF on address input:
 552         //
 553         //   - the port has no clock enable, no reset, and no initial value
 554         //   - the port is transparent wrt all write ports (implying they also share
 555         //     the clock domain)
 556         //
 557         // - otherwise, put the FF on the data output, and make bypass paths for
 558         //   all write ports wrt which this port is transparent
 559         bool trans_use_addr = port.transparent;
 560
 561         // If there are no write ports at all, we could possibly use either way; do data
 562         // FF in this case.
 563         if (GetSize(wr_ports) == 0)
 564                 trans_use_addr = false;
 565
 566         if (port.en != State::S1 || port.srst != State::S0 || port.arst != State::S0 || !port.init_value.is_fully_undef())
 567                 trans_use_addr = false;
 568
 569         if (trans_use_addr)
 570         {
 571                 // Do not put a register in front of constant address bits — this is both
 572                 // unnecessary and will break wide ports.
 573                 int width = 0;
 574                 for (int i = 0; i < GetSize(port.addr); i++)
 575                         if (port.addr[i].wire)
 576                                 width++;
 577
 578                 if (width)
 579                 {
 580                         SigSpec sig_q = module->addWire(stringf("$%s$rdreg[%d]$q", memid.c_str(), idx), width);
 581                         SigSpec sig_d;
 582
 583                         int pos = 0;
 584                         for (int i = 0; i < GetSize(port.addr); i++)
 585                                 if (port.addr[i].wire) {
 586                                         sig_d.append(port.addr[i]);
 587                                         port.addr[i] = sig_q[pos++];
 588                                 }
 589
 590                         c = module->addDff(stringf("$%s$rdreg[%d]", memid.c_str(), idx), port.clk, sig_d, sig_q, port.clk_polarity);
 591                 } else {
 592                         c = nullptr;
 593                 }
 594         }
 595         else
 596         {
 597                 log_assert(port.arst == State::S0 || port.srst == State::S0);
 598
 599                 SigSpec async_d = module->addWire(stringf("$%s$rdreg[%d]$d", memid.c_str(), idx), GetSize(port.data));
 600                 SigSpec sig_d = async_d;
 601
 602                 for (int i = 0; i < GetSize(wr_ports); i++) {
 603                         auto &wport = wr_ports[i];
 604                         if (port.transparent) {
 605                                 log_assert(wport.clk_enable);
 606                                 log_assert(wport.clk == port.clk);
 607                                 log_assert(wport.clk_enable == port.clk_enable);
 608                                 int min_wide_log2 = std::min(port.wide_log2, wport.wide_log2);
 609                                 int max_wide_log2 = std::max(port.wide_log2, wport.wide_log2);
 610                                 bool wide_write = wport.wide_log2 > port.wide_log2;
 611                                 for (int sub = 0; sub < (1 << max_wide_log2); sub += (1 << min_wide_log2)) {
 612                                         SigSpec raddr = port.addr;
 613                                         SigSpec waddr = wport.addr;
 614                                         if (wide_write)
 615                                                 waddr = wport.sub_addr(sub);
 616                                         else
 617                                                 raddr = port.sub_addr(sub);
 618                                         SigSpec addr_eq;
 619                                         if (raddr != waddr)
 620                                                 addr_eq = module->Eq(stringf("$%s$rdtransen[%d][%d][%d]$d", memid.c_str(), idx, i, sub), raddr, waddr);
 621                                         int pos = 0;
 622                                         int ewidth = width << min_wide_log2;
 623                                         int wsub = wide_write ? sub : 0;
 624                                         int rsub = wide_write ? 0 : sub;
 625                                         while (pos < ewidth) {
 626                                                 int epos = pos;
 627                                                 while (epos < ewidth && wport.en[epos + wsub * width] == wport.en[pos + wsub * width])
 628                                                         epos++;
 629                                                 SigSpec cur = sig_d.extract(pos + rsub * width, epos-pos);
 630                                                 SigSpec other = wport.data.extract(pos + wsub * width, epos-pos);
 631                                                 SigSpec cond;
 632                                                 if (raddr != waddr)
 633                                                         cond = module->And(stringf("$%s$rdtransgate[%d][%d][%d][%d]$d", memid.c_str(), idx, i, sub, pos), wport.en[pos + wsub * width], addr_eq);
 634                                                 else
 635                                                         cond = wport.en[pos + wsub * width];
 636                                                 SigSpec merged = module->Mux(stringf("$%s$rdtransmux[%d][%d][%d][%d]$d", memid.c_str(), idx, i, sub, pos), cur, other, cond);
 637                                                 sig_d.replace(pos + rsub * width, merged);
 638                                                 pos = epos;
 639                                         }
 640                                 }
 641                         }
 642                 }
 643
 644                 IdString name = stringf("$%s$rdreg[%d]", memid.c_str(), idx);
 645                 FfData ff(initvals);
 646                 ff.width = GetSize(port.data);
 647                 ff.has_clk = true;
 648                 ff.sig_clk = port.clk;
 649                 ff.pol_clk = port.clk_polarity;
 650                 if (port.en != State::S1) {
 651                         ff.has_en = true;
 652                         ff.pol_en = true;
 653                         ff.sig_en = port.en;
 654                 }
 655                 if (port.arst != State::S0) {
 656                         ff.has_arst = true;
 657                         ff.pol_arst = true;
 658                         ff.sig_arst = port.arst;
 659                         ff.val_arst = port.arst_value;
 660                 }
 661                 if (port.srst != State::S0) {
 662                         ff.has_srst = true;
 663                         ff.pol_srst = true;
 664                         ff.sig_srst = port.srst;
 665                         ff.val_srst = port.srst_value;
 666                         ff.ce_over_srst = ff.has_en && port.ce_over_srst;
 667                 }
 668                 ff.sig_d = sig_d;
 669                 ff.sig_q = port.data;
 670                 ff.val_init = port.init_value;
 671                 port.data = async_d;
 672                 c = ff.emit(module, name);
 673         }
 674
 675         log("Extracted %s FF from read port %d of %s.%s: %s\n", trans_use_addr ? "addr" : "data",
 676                         idx, log_id(module), log_id(memid), log_id(c));
 677
 678         port.en = State::S1;
 679         port.clk = State::S0;
 680         port.arst = State::S0;
 681         port.srst = State::S0;
 682         port.clk_enable = false;
 683         port.clk_polarity = true;
 684         port.transparent = false;
 685         port.ce_over_srst = false;
 686         port.arst_value = Const(State::Sx, GetSize(port.data));
 687         port.srst_value = Const(State::Sx, GetSize(port.data));
 688         port.init_value = Const(State::Sx, GetSize(port.data));
 689
 690         return c;
 691 }
 692
 693 void Mem::narrow() {
 694         std::vector<MemRd> new_rd_ports;
 695         std::vector<MemWr> new_wr_ports;
 696         std::vector<std::pair<int, int>> new_rd_map;
 697         std::vector<std::pair<int, int>> new_wr_map;
 698         for (int i = 0; i < GetSize(rd_ports); i++) {
 699                 auto &port = rd_ports[i];
 700                 for (int sub = 0; sub < (1 << port.wide_log2); sub++) {
 701                         new_rd_map.push_back(std::make_pair(i, sub));
 702                 }
 703         }
 704         for (int i = 0; i < GetSize(wr_ports); i++) {
 705                 auto &port = wr_ports[i];
 706                 for (int sub = 0; sub < (1 << port.wide_log2); sub++) {
 707                         new_wr_map.push_back(std::make_pair(i, sub));
 708                 }
 709         }
 710         for (auto &it : new_rd_map) {
 711                 MemRd &orig = rd_ports[it.first];
 712                 MemRd port = orig;
 713                 if (it.second != 0)
 714                         port.cell = nullptr;
 715                 if (port.wide_log2) {
 716                         port.data = port.data.extract(it.second * width, width);
 717                         port.init_value = port.init_value.extract(it.second * width, width);
 718                         port.arst_value = port.arst_value.extract(it.second * width, width);
 719                         port.srst_value = port.srst_value.extract(it.second * width, width);
 720                         port.addr = port.sub_addr(it.second);
 721                         port.wide_log2 = 0;
 722                 }
 723                 new_rd_ports.push_back(port);
 724         }
 725         for (auto &it : new_wr_map) {
 726                 MemWr &orig = wr_ports[it.first];
 727                 MemWr port = orig;
 728                 if (it.second != 0)
 729                         port.cell = nullptr;
 730                 if (port.wide_log2) {
 731                         port.data = port.data.extract(it.second * width, width);
 732                         port.en = port.en.extract(it.second * width, width);
 733                         port.addr = port.sub_addr(it.second);
 734                         port.wide_log2 = 0;
 735                 }
 736                 port.priority_mask.clear();
 737                 for (auto &it2 : new_wr_map)
 738                         port.priority_mask.push_back(orig.priority_mask[it2.first]);
 739                 new_wr_ports.push_back(port);
 740         }
 741         std::swap(rd_ports, new_rd_ports);
 742         std::swap(wr_ports, new_wr_ports);
 743 }
 744
 745 void Mem::emulate_priority(int idx1, int idx2)
 746 {
 747         auto &port1 = wr_ports[idx1];
 748         auto &port2 = wr_ports[idx2];
 749         if (!port2.priority_mask[idx1])
 750                 return;
 751         int min_wide_log2 = std::min(port1.wide_log2, port2.wide_log2);
 752         int max_wide_log2 = std::max(port1.wide_log2, port2.wide_log2);
 753         bool wide1 = port1.wide_log2 > port2.wide_log2;
 754         for (int sub = 0; sub < (1 << max_wide_log2); sub += (1 << min_wide_log2)) {
 755                 SigSpec addr1 = port1.addr;
 756                 SigSpec addr2 = port2.addr;
 757                 if (wide1)
 758                         addr1 = port1.sub_addr(sub);
 759                 else
 760                         addr2 = port2.sub_addr(sub);
 761                 SigSpec addr_eq = module->Eq(NEW_ID, addr1, addr2);
 762                 int ewidth = width << min_wide_log2;
 763                 int sub1 = wide1 ? sub : 0;
 764                 int sub2 = wide1 ? 0 : sub;
 765                 dict<std::pair<SigBit, SigBit>, SigBit> cache;
 766                 for (int pos = 0; pos < ewidth; pos++) {
 767                         SigBit &en1 = port1.en[pos + sub1 * width];
 768                         SigBit &en2 = port2.en[pos + sub2 * width];
 769                         std::pair<SigBit, SigBit> key(en1, en2);
 770                         if (cache.count(key)) {
 771                                 en1 = cache[key];
 772                         } else {
 773                                 SigBit active2 = module->And(NEW_ID, addr_eq, en2);
 774                                 SigBit nactive2 = module->Not(NEW_ID, active2);
 775                                 en1 = cache[key] = module->And(NEW_ID, en1, nactive2);
 776                         }
 777                 }
 778         }
 779         port2.priority_mask[idx1] = false;
 780 }
 781
 782 void Mem::prepare_wr_merge(int idx1, int idx2) {
 783         log_assert(idx1 < idx2);
 784         auto &port1 = wr_ports[idx1];
 785         auto &port2 = wr_ports[idx2];
 786         // If port 2 has priority over a port before port 1, make port 1 have priority too.
 787         for (int i = 0; i < idx1; i++)
 788                 if (port2.priority_mask[i])
 789                         port1.priority_mask[i] = true;
 790         // If port 2 has priority over a port after port 1, emulate it.
 791         for (int i = idx1 + 1; i < idx2; i++)
 792                 if (port2.priority_mask[i])
 793                         emulate_priority(i, idx2);
 794         // If some port had priority over port 2, make it have priority over the merged port too.
 795         for (int i = idx2 + 1; i < GetSize(wr_ports); i++) {
 796                 auto &oport = wr_ports[i];
 797                 if (oport.priority_mask[idx2])
 798                         oport.priority_mask[idx1] = true;
 799         }
 800 }