kernel/mem.cc

   1 /*
   2  *  yosys -- Yosys Open SYnthesis Suite
   3  *
   4  *  Copyright (C) 2020  Marcelina Kościelnicka <mwk@0x04.net>
   5  *
   6  *  Permission to use, copy, modify, and/or distribute this software for any
   7  *  purpose with or without fee is hereby granted, provided that the above
   8  *  copyright notice and this permission notice appear in all copies.
   9  *
  10  *  THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
  11  *  WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
  12  *  MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
  13  *  ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
  14  *  WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
  15  *  ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
  16  *  OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
  17  *
  18  */
  19
  20 #include "kernel/mem.h"
  21 #include "kernel/ff.h"
  22
  23 USING_YOSYS_NAMESPACE
  24
  25 void Mem::remove() {
  26         if (cell) {
  27                 module->remove(cell);
  28                 cell = nullptr;
  29         }
  30         if (mem) {
  31                 module->memories.erase(mem->name);
  32                 delete mem;
  33                 mem = nullptr;
  34         }
  35         for (auto &port : rd_ports) {
  36                 if (port.cell) {
  37                         module->remove(port.cell);
  38                         port.cell = nullptr;
  39                 }
  40         }
  41         for (auto &port : wr_ports) {
  42                 if (port.cell) {
  43                         module->remove(port.cell);
  44                         port.cell = nullptr;
  45                 }
  46         }
  47         for (auto &init : inits) {
  48                 if (init.cell) {
  49                         module->remove(init.cell);
  50                         init.cell = nullptr;
  51                 }
  52         }
  53 }
  54
  55 void Mem::emit() {
  56         check();
  57         std::vector<int> rd_left;
  58         for (int i = 0; i < GetSize(rd_ports); i++) {
  59                 auto &port = rd_ports[i];
  60                 if (port.removed) {
  61                         if (port.cell) {
  62                                 module->remove(port.cell);
  63                         }
  64                 } else {
  65                         rd_left.push_back(i);
  66                 }
  67         }
  68         std::vector<int> wr_left;
  69         for (int i = 0; i < GetSize(wr_ports); i++) {
  70                 auto &port = wr_ports[i];
  71                 if (port.removed) {
  72                         if (port.cell) {
  73                                 module->remove(port.cell);
  74                         }
  75                 } else {
  76                         wr_left.push_back(i);
  77                 }
  78         }
  79         for (int i = 0; i < GetSize(rd_left); i++)
  80                 if (i != rd_left[i])
  81                         std::swap(rd_ports[i], rd_ports[rd_left[i]]);
  82         rd_ports.resize(GetSize(rd_left));
  83         for (int i = 0; i < GetSize(wr_left); i++)
  84                 if (i != wr_left[i])
  85                         std::swap(wr_ports[i], wr_ports[wr_left[i]]);
  86         wr_ports.resize(GetSize(wr_left));
  87
  88         // for future: handle transparency mask here
  89
  90         for (auto &port : wr_ports) {
  91                 for (int i = 0; i < GetSize(wr_left); i++)
  92                         port.priority_mask[i] = port.priority_mask[wr_left[i]];
  93                 port.priority_mask.resize(GetSize(wr_left));
  94         }
  95
  96         if (packed) {
  97                 if (mem) {
  98                         module->memories.erase(mem->name);
  99                         delete mem;
 100                         mem = nullptr;
 101                 }
 102                 if (!cell) {
 103                         if (memid.empty())
 104                                 memid = NEW_ID;
 105                         cell = module->addCell(memid, ID($mem));
 106                 }
 107                 cell->attributes = attributes;
 108                 cell->parameters[ID::MEMID] = Const(memid.str());
 109                 cell->parameters[ID::WIDTH] = Const(width);
 110                 cell->parameters[ID::OFFSET] = Const(start_offset);
 111                 cell->parameters[ID::SIZE] = Const(size);
 112                 Const rd_wide_continuation, rd_clk_enable, rd_clk_polarity, rd_transparent;
 113                 Const wr_wide_continuation, wr_clk_enable, wr_clk_polarity;
 114                 SigSpec rd_clk, rd_en, rd_addr, rd_data;
 115                 SigSpec wr_clk, wr_en, wr_addr, wr_data;
 116                 int abits = 0;
 117                 for (auto &port : rd_ports)
 118                         abits = std::max(abits, GetSize(port.addr));
 119                 for (auto &port : wr_ports)
 120                         abits = std::max(abits, GetSize(port.addr));
 121                 cell->parameters[ID::ABITS] = Const(abits);
 122                 for (auto &port : rd_ports) {
 123                         // TODO: remove
 124                         log_assert(port.arst == State::S0);
 125                         log_assert(port.srst == State::S0);
 126                         log_assert(port.init_value == Const(State::Sx, width << port.wide_log2));
 127                         if (port.cell) {
 128                                 module->remove(port.cell);
 129                                 port.cell = nullptr;
 130                         }
 131                         for (int sub = 0; sub < (1 << port.wide_log2); sub++)
 132                         {
 133                                 rd_wide_continuation.bits.push_back(State(sub != 0));
 134                                 rd_clk_enable.bits.push_back(State(port.clk_enable));
 135                                 rd_clk_polarity.bits.push_back(State(port.clk_polarity));
 136                                 rd_transparent.bits.push_back(State(port.transparent));
 137                                 rd_clk.append(port.clk);
 138                                 rd_en.append(port.en);
 139                                 SigSpec addr = port.addr;
 140                                 addr.extend_u0(abits, false);
 141                                 for (int i = 0; i < port.wide_log2; i++)
 142                                         addr[i] = State(sub >> i & 1);
 143                                 rd_addr.append(addr);
 144                                 log_assert(GetSize(addr) == abits);
 145                         }
 146                         rd_data.append(port.data);
 147                 }
 148                 if (rd_ports.empty()) {
 149                         rd_wide_continuation = State::S0;
 150                         rd_clk_enable = State::S0;
 151                         rd_clk_polarity = State::S0;
 152                         rd_transparent = State::S0;
 153                 }
 154                 cell->parameters[ID::RD_PORTS] = Const(GetSize(rd_clk));
 155                 cell->parameters[ID::RD_CLK_ENABLE] = rd_clk_enable;
 156                 cell->parameters[ID::RD_CLK_POLARITY] = rd_clk_polarity;
 157                 cell->parameters[ID::RD_TRANSPARENT] = rd_transparent;
 158                 cell->setPort(ID::RD_CLK, rd_clk);
 159                 cell->setPort(ID::RD_EN, rd_en);
 160                 cell->setPort(ID::RD_ADDR, rd_addr);
 161                 cell->setPort(ID::RD_DATA, rd_data);
 162                 for (auto &port : wr_ports) {
 163                         if (port.cell) {
 164                                 module->remove(port.cell);
 165                                 port.cell = nullptr;
 166                         }
 167                         for (int sub = 0; sub < (1 << port.wide_log2); sub++)
 168                         {
 169                                 wr_wide_continuation.bits.push_back(State(sub != 0));
 170                                 wr_clk_enable.bits.push_back(State(port.clk_enable));
 171                                 wr_clk_polarity.bits.push_back(State(port.clk_polarity));
 172                                 wr_clk.append(port.clk);
 173                                 SigSpec addr = port.addr;
 174                                 addr.extend_u0(abits, false);
 175                                 for (int i = 0; i < port.wide_log2; i++)
 176                                         addr[i] = State(sub >> i & 1);
 177                                 wr_addr.append(addr);
 178                                 log_assert(GetSize(addr) == abits);
 179                         }
 180                         wr_en.append(port.en);
 181                         wr_data.append(port.data);
 182                 }
 183                 if (wr_ports.empty()) {
 184                         wr_wide_continuation = State::S0;
 185                         wr_clk_enable = State::S0;
 186                         wr_clk_polarity = State::S0;
 187                 }
 188                 cell->parameters[ID::WR_PORTS] = Const(GetSize(wr_clk));
 189                 cell->parameters[ID::WR_CLK_ENABLE] = wr_clk_enable;
 190                 cell->parameters[ID::WR_CLK_POLARITY] = wr_clk_polarity;
 191                 cell->setPort(ID::WR_CLK, wr_clk);
 192                 cell->setPort(ID::WR_EN, wr_en);
 193                 cell->setPort(ID::WR_ADDR, wr_addr);
 194                 cell->setPort(ID::WR_DATA, wr_data);
 195                 for (auto &init : inits) {
 196                         if (init.cell) {
 197                                 module->remove(init.cell);
 198                                 init.cell = nullptr;
 199                         }
 200                 }
 201                 cell->parameters[ID::INIT] = get_init_data();
 202         } else {
 203                 if (cell) {
 204                         module->remove(cell);
 205                         cell = nullptr;
 206                 }
 207                 if (!mem) {
 208                         if (memid.empty())
 209                                 memid = NEW_ID;
 210                         mem = new RTLIL::Memory;
 211                         mem->name = memid;
 212                         module->memories[memid] = mem;
 213                 }
 214                 mem->width = width;
 215                 mem->start_offset = start_offset;
 216                 mem->size = size;
 217                 for (auto &port : rd_ports) {
 218                         // TODO: remove
 219                         log_assert(port.arst == State::S0);
 220                         log_assert(port.srst == State::S0);
 221                         log_assert(port.init_value == Const(State::Sx, width << port.wide_log2));
 222                         if (!port.cell)
 223                                 port.cell = module->addCell(NEW_ID, ID($memrd));
 224                         port.cell->parameters[ID::MEMID] = memid.str();
 225                         port.cell->parameters[ID::ABITS] = GetSize(port.addr);
 226                         port.cell->parameters[ID::WIDTH] = width << port.wide_log2;
 227                         port.cell->parameters[ID::CLK_ENABLE] = port.clk_enable;
 228                         port.cell->parameters[ID::CLK_POLARITY] = port.clk_polarity;
 229                         port.cell->parameters[ID::TRANSPARENT] = port.transparent;
 230                         port.cell->setPort(ID::CLK, port.clk);
 231                         port.cell->setPort(ID::EN, port.en);
 232                         port.cell->setPort(ID::ADDR, port.addr);
 233                         port.cell->setPort(ID::DATA, port.data);
 234                 }
 235                 int idx = 0;
 236                 for (auto &port : wr_ports) {
 237                         if (!port.cell)
 238                                 port.cell = module->addCell(NEW_ID, ID($memwr));
 239                         port.cell->parameters[ID::MEMID] = memid.str();
 240                         port.cell->parameters[ID::ABITS] = GetSize(port.addr);
 241                         port.cell->parameters[ID::WIDTH] = width << port.wide_log2;
 242                         port.cell->parameters[ID::CLK_ENABLE] = port.clk_enable;
 243                         port.cell->parameters[ID::CLK_POLARITY] = port.clk_polarity;
 244                         port.cell->parameters[ID::PRIORITY] = idx++;
 245                         port.cell->setPort(ID::CLK, port.clk);
 246                         port.cell->setPort(ID::EN, port.en);
 247                         port.cell->setPort(ID::ADDR, port.addr);
 248                         port.cell->setPort(ID::DATA, port.data);
 249                 }
 250                 idx = 0;
 251                 for (auto &init : inits) {
 252                         if (!init.cell)
 253                                 init.cell = module->addCell(NEW_ID, ID($meminit));
 254                         init.cell->parameters[ID::MEMID] = memid.str();
 255                         init.cell->parameters[ID::ABITS] = GetSize(init.addr);
 256                         init.cell->parameters[ID::WIDTH] = width;
 257                         init.cell->parameters[ID::WORDS] = GetSize(init.data) / width;
 258                         init.cell->parameters[ID::PRIORITY] = idx++;
 259                         init.cell->setPort(ID::ADDR, init.addr);
 260                         init.cell->setPort(ID::DATA, init.data);
 261                 }
 262         }
 263 }
 264
 265 void Mem::clear_inits() {
 266         for (auto &init : inits)
 267                 if (init.cell)
 268                         module->remove(init.cell);
 269         inits.clear();
 270 }
 271
 272 Const Mem::get_init_data() const {
 273         Const init_data(State::Sx, width * size);
 274         for (auto &init : inits) {
 275                 int offset = (init.addr.as_int() - start_offset) * width;
 276                 for (int i = 0; i < GetSize(init.data); i++)
 277                         if (0 <= i+offset && i+offset < GetSize(init_data))
 278                                 init_data.bits[i+offset] = init.data.bits[i];
 279         }
 280         return init_data;
 281 }
 282
 283 void Mem::check() {
 284         int max_wide_log2 = 0;
 285         for (auto &port : rd_ports) {
 286                 if (port.removed)
 287                         continue;
 288                 log_assert(GetSize(port.clk) == 1);
 289                 log_assert(GetSize(port.en) == 1);
 290                 log_assert(GetSize(port.arst) == 1);
 291                 log_assert(GetSize(port.srst) == 1);
 292                 log_assert(GetSize(port.data) == (width << port.wide_log2));
 293                 log_assert(GetSize(port.init_value) == (width << port.wide_log2));
 294                 log_assert(GetSize(port.arst_value) == (width << port.wide_log2));
 295                 log_assert(GetSize(port.srst_value) == (width << port.wide_log2));
 296                 if (!port.clk_enable) {
 297                         log_assert(!port.transparent);
 298                         log_assert(port.arst == State::S0);
 299                         log_assert(port.srst == State::S0);
 300                 }
 301                 for (int j = 0; j < port.wide_log2; j++) {
 302                         log_assert(port.addr[j] == State::S0);
 303                 }
 304                 max_wide_log2 = std::max(max_wide_log2, port.wide_log2);
 305         }
 306         for (int i = 0; i < GetSize(wr_ports); i++) {
 307                 auto &port = wr_ports[i];
 308                 if (port.removed)
 309                         continue;
 310                 log_assert(GetSize(port.clk) == 1);
 311                 log_assert(GetSize(port.en) == (width << port.wide_log2));
 312                 log_assert(GetSize(port.data) == (width << port.wide_log2));
 313                 for (int j = 0; j < port.wide_log2; j++) {
 314                         log_assert(port.addr[j] == State::S0);
 315                 }
 316                 max_wide_log2 = std::max(max_wide_log2, port.wide_log2);
 317                 log_assert(GetSize(port.priority_mask) == GetSize(wr_ports));
 318                 for (int j = 0; j < GetSize(wr_ports); j++) {
 319                         auto &wport = wr_ports[j];
 320                         if (port.priority_mask[j] && !wport.removed) {
 321                                 log_assert(j < i);
 322                                 log_assert(port.clk_enable == wport.clk_enable);
 323                                 if (port.clk_enable) {
 324                                         log_assert(port.clk == wport.clk);
 325                                         log_assert(port.clk_polarity == wport.clk_polarity);
 326                                 }
 327                         }
 328                 }
 329         }
 330         int mask = (1 << max_wide_log2) - 1;
 331         log_assert(!(start_offset & mask));
 332         log_assert(!(size & mask));
 333 }
 334
 335 namespace {
 336
 337         struct MemIndex {
 338                 dict<IdString, pool<Cell *>> rd_ports;
 339                 dict<IdString, pool<Cell *>> wr_ports;
 340                 dict<IdString, pool<Cell *>> inits;
 341                 MemIndex (Module *module) {
 342                         for (auto cell: module->cells()) {
 343                                 if (cell->type == ID($memwr))
 344                                         wr_ports[cell->parameters.at(ID::MEMID).decode_string()].insert(cell);
 345                                 else if (cell->type == ID($memrd))
 346                                         rd_ports[cell->parameters.at(ID::MEMID).decode_string()].insert(cell);
 347                                 else if (cell->type == ID($meminit))
 348                                         inits[cell->parameters.at(ID::MEMID).decode_string()].insert(cell);
 349                         }
 350                 }
 351         };
 352
 353         Mem mem_from_memory(Module *module, RTLIL::Memory *mem, const MemIndex &index) {
 354                 Mem res(module, mem->name, mem->width, mem->start_offset, mem->size);
 355                 res.packed = false;
 356                 res.mem = mem;
 357                 res.attributes = mem->attributes;
 358                 if (index.rd_ports.count(mem->name)) {
 359                         for (auto cell : index.rd_ports.at(mem->name)) {
 360                                 MemRd mrd;
 361                                 mrd.cell = cell;
 362                                 mrd.attributes = cell->attributes;
 363                                 mrd.clk_enable = cell->parameters.at(ID::CLK_ENABLE).as_bool();
 364                                 mrd.clk_polarity = cell->parameters.at(ID::CLK_POLARITY).as_bool();
 365                                 mrd.transparent = cell->parameters.at(ID::TRANSPARENT).as_bool();
 366                                 mrd.clk = cell->getPort(ID::CLK);
 367                                 mrd.en = cell->getPort(ID::EN);
 368                                 mrd.addr = cell->getPort(ID::ADDR);
 369                                 mrd.data = cell->getPort(ID::DATA);
 370                                 mrd.wide_log2 = ceil_log2(GetSize(mrd.data) / mem->width);
 371                                 mrd.ce_over_srst = false;
 372                                 mrd.arst_value = Const(State::Sx, mem->width << mrd.wide_log2);
 373                                 mrd.srst_value = Const(State::Sx, mem->width << mrd.wide_log2);
 374                                 mrd.init_value = Const(State::Sx, mem->width << mrd.wide_log2);
 375                                 mrd.srst = State::S0;
 376                                 mrd.arst = State::S0;
 377                                 res.rd_ports.push_back(mrd);
 378                         }
 379                 }
 380                 if (index.wr_ports.count(mem->name)) {
 381                         std::vector<std::pair<int, MemWr>> ports;
 382                         for (auto cell : index.wr_ports.at(mem->name)) {
 383                                 MemWr mwr;
 384                                 mwr.cell = cell;
 385                                 mwr.attributes = cell->attributes;
 386                                 mwr.clk_enable = cell->parameters.at(ID::CLK_ENABLE).as_bool();
 387                                 mwr.clk_polarity = cell->parameters.at(ID::CLK_POLARITY).as_bool();
 388                                 mwr.clk = cell->getPort(ID::CLK);
 389                                 mwr.en = cell->getPort(ID::EN);
 390                                 mwr.addr = cell->getPort(ID::ADDR);
 391                                 mwr.data = cell->getPort(ID::DATA);
 392                                 mwr.wide_log2 = ceil_log2(GetSize(mwr.data) / mem->width);
 393                                 ports.push_back(std::make_pair(cell->parameters.at(ID::PRIORITY).as_int(), mwr));
 394                         }
 395                         std::sort(ports.begin(), ports.end(), [](const std::pair<int, MemWr> &a, const std::pair<int, MemWr> &b) { return a.first < b.first; });
 396                         for (auto &it : ports)
 397                                 res.wr_ports.push_back(it.second);
 398                 }
 399                 if (index.inits.count(mem->name)) {
 400                         std::vector<std::pair<int, MemInit>> inits;
 401                         for (auto cell : index.inits.at(mem->name)) {
 402                                 MemInit init;
 403                                 init.cell = cell;
 404                                 init.attributes = cell->attributes;
 405                                 auto addr = cell->getPort(ID::ADDR);
 406                                 auto data = cell->getPort(ID::DATA);
 407                                 if (!addr.is_fully_const())
 408                                         log_error("Non-constant address %s in memory initialization %s.\n", log_signal(addr), log_id(cell));
 409                                 if (!data.is_fully_const())
 410                                         log_error("Non-constant data %s in memory initialization %s.\n", log_signal(data), log_id(cell));
 411                                 init.addr = addr.as_const();
 412                                 init.data = data.as_const();
 413                                 inits.push_back(std::make_pair(cell->parameters.at(ID::PRIORITY).as_int(), init));
 414                         }
 415                         std::sort(inits.begin(), inits.end(), [](const std::pair<int, MemInit> &a, const std::pair<int, MemInit> &b) { return a.first < b.first; });
 416                         for (auto &it : inits)
 417                                 res.inits.push_back(it.second);
 418                 }
 419                 for (int i = 0; i < GetSize(res.wr_ports); i++) {
 420                         auto &port = res.wr_ports[i];
 421                         port.priority_mask.resize(GetSize(res.wr_ports));
 422                         for (int j = 0; j < i; j++) {
 423                                 auto &oport = res.wr_ports[j];
 424                                 if (port.clk_enable != oport.clk_enable)
 425                                         continue;
 426                                 if (port.clk_enable && port.clk != oport.clk)
 427                                         continue;
 428                                 if (port.clk_enable && port.clk_polarity != oport.clk_polarity)
 429                                         continue;
 430                                 port.priority_mask[j] = true;
 431                         }
 432                 }
 433                 res.check();
 434                 return res;
 435         }
 436
 437         Mem mem_from_cell(Cell *cell) {
 438                 Mem res(cell->module, cell->parameters.at(ID::MEMID).decode_string(),
 439                         cell->parameters.at(ID::WIDTH).as_int(),
 440                         cell->parameters.at(ID::OFFSET).as_int(),
 441                         cell->parameters.at(ID::SIZE).as_int()
 442                 );
 443                 int abits = cell->parameters.at(ID::ABITS).as_int();
 444                 res.packed = true;
 445                 res.cell = cell;
 446                 res.attributes = cell->attributes;
 447                 Const &init = cell->parameters.at(ID::INIT);
 448                 if (!init.is_fully_undef()) {
 449                         int pos = 0;
 450                         while (pos < res.size) {
 451                                 Const word = init.extract(pos * res.width, res.width, State::Sx);
 452                                 if (word.is_fully_undef()) {
 453                                         pos++;
 454                                 } else {
 455                                         int epos;
 456                                         for (epos = pos; epos < res.size; epos++) {
 457                                                 Const eword = init.extract(epos * res.width, res.width, State::Sx);
 458                                                 if (eword.is_fully_undef())
 459                                                         break;
 460                                         }
 461                                         MemInit minit;
 462                                         minit.addr = res.start_offset + pos;
 463                                         minit.data = init.extract(pos * res.width, (epos - pos) * res.width, State::Sx);
 464                                         res.inits.push_back(minit);
 465                                         pos = epos;
 466                                 }
 467                         }
 468                 }
 469                 for (int i = 0; i < cell->parameters.at(ID::RD_PORTS).as_int(); i++) {
 470                         MemRd mrd;
 471                         mrd.wide_log2 = 0;
 472                         mrd.clk_enable = cell->parameters.at(ID::RD_CLK_ENABLE).extract(i, 1).as_bool();
 473                         mrd.clk_polarity = cell->parameters.at(ID::RD_CLK_POLARITY).extract(i, 1).as_bool();
 474                         mrd.transparent = cell->parameters.at(ID::RD_TRANSPARENT).extract(i, 1).as_bool();
 475                         mrd.clk = cell->getPort(ID::RD_CLK).extract(i, 1);
 476                         mrd.en = cell->getPort(ID::RD_EN).extract(i, 1);
 477                         mrd.addr = cell->getPort(ID::RD_ADDR).extract(i * abits, abits);
 478                         mrd.data = cell->getPort(ID::RD_DATA).extract(i * res.width, res.width);
 479                         mrd.ce_over_srst = false;
 480                         mrd.arst_value = Const(State::Sx, res.width << mrd.wide_log2);
 481                         mrd.srst_value = Const(State::Sx, res.width << mrd.wide_log2);
 482                         mrd.init_value = Const(State::Sx, res.width << mrd.wide_log2);
 483                         mrd.srst = State::S0;
 484                         mrd.arst = State::S0;
 485                         res.rd_ports.push_back(mrd);
 486                 }
 487                 for (int i = 0; i < cell->parameters.at(ID::WR_PORTS).as_int(); i++) {
 488                         MemWr mwr;
 489                         mwr.wide_log2 = 0;
 490                         mwr.clk_enable = cell->parameters.at(ID::WR_CLK_ENABLE).extract(i, 1).as_bool();
 491                         mwr.clk_polarity = cell->parameters.at(ID::WR_CLK_POLARITY).extract(i, 1).as_bool();
 492                         mwr.clk = cell->getPort(ID::WR_CLK).extract(i, 1);
 493                         mwr.en = cell->getPort(ID::WR_EN).extract(i * res.width, res.width);
 494                         mwr.addr = cell->getPort(ID::WR_ADDR).extract(i * abits, abits);
 495                         mwr.data = cell->getPort(ID::WR_DATA).extract(i * res.width, res.width);
 496                         res.wr_ports.push_back(mwr);
 497                 }
 498                 for (int i = 0; i < GetSize(res.wr_ports); i++) {
 499                         auto &port = res.wr_ports[i];
 500                         port.priority_mask.resize(GetSize(res.wr_ports));
 501                         for (int j = 0; j < i; j++) {
 502                                 auto &oport = res.wr_ports[j];
 503                                 if (port.clk_enable != oport.clk_enable)
 504                                         continue;
 505                                 if (port.clk_enable && port.clk != oport.clk)
 506                                         continue;
 507                                 if (port.clk_enable && port.clk_polarity != oport.clk_polarity)
 508                                         continue;
 509                                 port.priority_mask[j] = true;
 510                         }
 511                 }
 512                 res.check();
 513                 return res;
 514         }
 515
 516 }
 517
 518 std::vector<Mem> Mem::get_all_memories(Module *module) {
 519         std::vector<Mem> res;
 520         MemIndex index(module);
 521         for (auto it: module->memories) {
 522                 res.push_back(mem_from_memory(module, it.second, index));
 523         }
 524         for (auto cell: module->cells()) {
 525                 if (cell->type == ID($mem))
 526                         res.push_back(mem_from_cell(cell));
 527         }
 528         return res;
 529 }
 530
 531 std::vector<Mem> Mem::get_selected_memories(Module *module) {
 532         std::vector<Mem> res;
 533         MemIndex index(module);
 534         for (auto it: module->memories) {
 535                 if (module->design->selected(module, it.second))
 536                         res.push_back(mem_from_memory(module, it.second, index));
 537         }
 538         for (auto cell: module->selected_cells()) {
 539                 if (cell->type == ID($mem))
 540                         res.push_back(mem_from_cell(cell));
 541         }
 542         return res;
 543 }
 544
 545 Cell *Mem::extract_rdff(int idx, FfInitVals *initvals) {
 546         MemRd &port = rd_ports[idx];
 547
 548         if (!port.clk_enable)
 549                 return nullptr;
 550
 551         Cell *c;
 552
 553         // There are two ways to handle rdff extraction when transparency is involved:
 554         //
 555         // - if all of the following conditions are true, put the FF on address input:
 556         //
 557         //   - the port has no clock enable, no reset, and no initial value
 558         //   - the port is transparent wrt all write ports (implying they also share
 559         //     the clock domain)
 560         //
 561         // - otherwise, put the FF on the data output, and make bypass paths for
 562         //   all write ports wrt which this port is transparent
 563         bool trans_use_addr = port.transparent;
 564
 565         // If there are no write ports at all, we could possibly use either way; do data
 566         // FF in this case.
 567         if (GetSize(wr_ports) == 0)
 568                 trans_use_addr = false;
 569
 570         if (port.en != State::S1 || port.srst != State::S0 || port.arst != State::S0 || !port.init_value.is_fully_undef())
 571                 trans_use_addr = false;
 572
 573         if (trans_use_addr)
 574         {
 575                 // Do not put a register in front of constant address bits — this is both
 576                 // unnecessary and will break wide ports.
 577                 int width = 0;
 578                 for (int i = 0; i < GetSize(port.addr); i++)
 579                         if (port.addr[i].wire)
 580                                 width++;
 581
 582                 if (width)
 583                 {
 584                         SigSpec sig_q = module->addWire(stringf("$%s$rdreg[%d]$q", memid.c_str(), idx), width);
 585                         SigSpec sig_d;
 586
 587                         int pos = 0;
 588                         for (int i = 0; i < GetSize(port.addr); i++)
 589                                 if (port.addr[i].wire) {
 590                                         sig_d.append(port.addr[i]);
 591                                         port.addr[i] = sig_q[pos++];
 592                                 }
 593
 594                         c = module->addDff(stringf("$%s$rdreg[%d]", memid.c_str(), idx), port.clk, sig_d, sig_q, port.clk_polarity);
 595                 } else {
 596                         c = nullptr;
 597                 }
 598         }
 599         else
 600         {
 601                 log_assert(port.arst == State::S0 || port.srst == State::S0);
 602
 603                 SigSpec async_d = module->addWire(stringf("$%s$rdreg[%d]$d", memid.c_str(), idx), GetSize(port.data));
 604                 SigSpec sig_d = async_d;
 605
 606                 for (int i = 0; i < GetSize(wr_ports); i++) {
 607                         auto &wport = wr_ports[i];
 608                         if (port.transparent) {
 609                                 log_assert(wport.clk_enable);
 610                                 log_assert(wport.clk == port.clk);
 611                                 log_assert(wport.clk_enable == port.clk_enable);
 612                                 int min_wide_log2 = std::min(port.wide_log2, wport.wide_log2);
 613                                 int max_wide_log2 = std::max(port.wide_log2, wport.wide_log2);
 614                                 bool wide_write = wport.wide_log2 > port.wide_log2;
 615                                 for (int sub = 0; sub < (1 << max_wide_log2); sub += (1 << min_wide_log2)) {
 616                                         SigSpec raddr = port.addr;
 617                                         SigSpec waddr = wport.addr;
 618                                         for (int j = min_wide_log2; j < max_wide_log2; j++)
 619                                                 if (wide_write)
 620                                                         waddr[j] = State(sub >> j & 1);
 621                                                 else
 622                                                         raddr[j] = State(sub >> j & 1);
 623                                         SigSpec addr_eq;
 624                                         if (raddr != waddr)
 625                                                 addr_eq = module->Eq(stringf("$%s$rdtransen[%d][%d][%d]$d", memid.c_str(), idx, i, sub), raddr, waddr);
 626                                         int pos = 0;
 627                                         int ewidth = width << min_wide_log2;
 628                                         int wsub = wide_write ? sub : 0;
 629                                         int rsub = wide_write ? 0 : sub;
 630                                         while (pos < ewidth) {
 631                                                 int epos = pos;
 632                                                 while (epos < ewidth && wport.en[epos + wsub * width] == wport.en[pos + wsub * width])
 633                                                         epos++;
 634                                                 SigSpec cur = sig_d.extract(pos + rsub * width, epos-pos);
 635                                                 SigSpec other = wport.data.extract(pos + wsub * width, epos-pos);
 636                                                 SigSpec cond;
 637                                                 if (raddr != waddr)
 638                                                         cond = module->And(stringf("$%s$rdtransgate[%d][%d][%d][%d]$d", memid.c_str(), idx, i, sub, pos), wport.en[pos + wsub * width], addr_eq);
 639                                                 else
 640                                                         cond = wport.en[pos + wsub * width];
 641                                                 SigSpec merged = module->Mux(stringf("$%s$rdtransmux[%d][%d][%d][%d]$d", memid.c_str(), idx, i, sub, pos), cur, other, cond);
 642                                                 sig_d.replace(pos + rsub * width, merged);
 643                                                 pos = epos;
 644                                         }
 645                                 }
 646                         }
 647                 }
 648
 649                 IdString name = stringf("$%s$rdreg[%d]", memid.c_str(), idx);
 650                 FfData ff(initvals);
 651                 ff.width = GetSize(port.data);
 652                 ff.has_clk = true;
 653                 ff.sig_clk = port.clk;
 654                 ff.pol_clk = port.clk_polarity;
 655                 if (port.en != State::S1) {
 656                         ff.has_en = true;
 657                         ff.pol_en = true;
 658                         ff.sig_en = port.en;
 659                 }
 660                 if (port.arst != State::S0) {
 661                         ff.has_arst = true;
 662                         ff.pol_arst = true;
 663                         ff.sig_arst = port.arst;
 664                         ff.val_arst = port.arst_value;
 665                 }
 666                 if (port.srst != State::S0) {
 667                         ff.has_srst = true;
 668                         ff.pol_srst = true;
 669                         ff.sig_srst = port.srst;
 670                         ff.val_srst = port.srst_value;
 671                         ff.ce_over_srst = ff.has_en && port.ce_over_srst;
 672                 }
 673                 ff.sig_d = sig_d;
 674                 ff.sig_q = port.data;
 675                 ff.val_init = port.init_value;
 676                 port.data = async_d;
 677                 c = ff.emit(module, name);
 678         }
 679
 680         log("Extracted %s FF from read port %d of %s.%s: %s\n", trans_use_addr ? "addr" : "data",
 681                         idx, log_id(module), log_id(memid), log_id(c));
 682
 683         port.en = State::S1;
 684         port.clk = State::S0;
 685         port.arst = State::S0;
 686         port.srst = State::S0;
 687         port.clk_enable = false;
 688         port.clk_polarity = true;
 689         port.transparent = false;
 690         port.ce_over_srst = false;
 691         port.arst_value = Const(State::Sx, GetSize(port.data));
 692         port.srst_value = Const(State::Sx, GetSize(port.data));
 693         port.init_value = Const(State::Sx, GetSize(port.data));
 694
 695         return c;
 696 }
 697
 698 void Mem::narrow() {
 699         std::vector<MemRd> new_rd_ports;
 700         std::vector<MemWr> new_wr_ports;
 701         std::vector<std::pair<int, int>> new_rd_map;
 702         std::vector<std::pair<int, int>> new_wr_map;
 703         for (int i = 0; i < GetSize(rd_ports); i++) {
 704                 auto &port = rd_ports[i];
 705                 for (int sub = 0; sub < (1 << port.wide_log2); sub++) {
 706                         new_rd_map.push_back(std::make_pair(i, sub));
 707                 }
 708         }
 709         for (int i = 0; i < GetSize(wr_ports); i++) {
 710                 auto &port = wr_ports[i];
 711                 for (int sub = 0; sub < (1 << port.wide_log2); sub++) {
 712                         new_wr_map.push_back(std::make_pair(i, sub));
 713                 }
 714         }
 715         for (auto &it : new_rd_map) {
 716                 MemRd &orig = rd_ports[it.first];
 717                 MemRd port = orig;
 718                 if (it.second != 0)
 719                         port.cell = nullptr;
 720                 if (port.wide_log2) {
 721                         port.data = port.data.extract(it.second * width, width);
 722                         port.init_value = port.init_value.extract(it.second * width, width);
 723                         port.arst_value = port.arst_value.extract(it.second * width, width);
 724                         port.srst_value = port.srst_value.extract(it.second * width, width);
 725                         for (int i = 0; i < port.wide_log2; i++)
 726                                 port.addr[i] = State(it.second >> i & 1);
 727                         port.wide_log2 = 0;
 728                 }
 729                 new_rd_ports.push_back(port);
 730         }
 731         for (auto &it : new_wr_map) {
 732                 MemWr &orig = wr_ports[it.first];
 733                 MemWr port = orig;
 734                 if (it.second != 0)
 735                         port.cell = nullptr;
 736                 if (port.wide_log2) {
 737                         port.data = port.data.extract(it.second * width, width);
 738                         port.en = port.en.extract(it.second * width, width);
 739                         for (int i = 0; i < port.wide_log2; i++)
 740                                 port.addr[i] = State(it.second >> i & 1);
 741                         port.wide_log2 = 0;
 742                 }
 743                 port.priority_mask.clear();
 744                 for (auto &it2 : new_wr_map)
 745                         port.priority_mask.push_back(orig.priority_mask[it2.first]);
 746                 new_wr_ports.push_back(port);
 747         }
 748         std::swap(rd_ports, new_rd_ports);
 749         std::swap(wr_ports, new_wr_ports);
 750 }
 751
 752 void Mem::emulate_priority(int idx1, int idx2)
 753 {
 754         auto &port1 = wr_ports[idx1];
 755         auto &port2 = wr_ports[idx2];
 756         if (!port2.priority_mask[idx1])
 757                 return;
 758         int min_wide_log2 = std::min(port1.wide_log2, port2.wide_log2);
 759         int max_wide_log2 = std::max(port1.wide_log2, port2.wide_log2);
 760         bool wide1 = port1.wide_log2 > port2.wide_log2;
 761         for (int sub = 0; sub < (1 << max_wide_log2); sub += (1 << min_wide_log2)) {
 762                 SigSpec addr1 = port1.addr;
 763                 SigSpec addr2 = port2.addr;
 764                 for (int j = min_wide_log2; j < max_wide_log2; j++)
 765                         if (wide1)
 766                                 addr1[j] = State(sub >> j & 1);
 767                         else
 768                                 addr2[j] = State(sub >> j & 1);
 769                 SigSpec addr_eq = module->Eq(NEW_ID, addr1, addr2);
 770                 int ewidth = width << min_wide_log2;
 771                 int sub1 = wide1 ? sub : 0;
 772                 int sub2 = wide1 ? 0 : sub;
 773                 dict<std::pair<SigBit, SigBit>, SigBit> cache;
 774                 for (int pos = 0; pos < ewidth; pos++) {
 775                         SigBit &en1 = port1.en[pos + sub1 * width];
 776                         SigBit &en2 = port2.en[pos + sub2 * width];
 777                         std::pair<SigBit, SigBit> key(en1, en2);
 778                         if (cache.count(key)) {
 779                                 en1 = cache[key];
 780                         } else {
 781                                 SigBit active2 = module->And(NEW_ID, addr_eq, en2);
 782                                 SigBit nactive2 = module->Not(NEW_ID, active2);
 783                                 en1 = cache[key] = module->And(NEW_ID, en1, nactive2);
 784                         }
 785                 }
 786         }
 787         port2.priority_mask[idx1] = false;
 788 }
 789
 790 void Mem::prepare_wr_merge(int idx1, int idx2) {
 791         log_assert(idx1 < idx2);
 792         auto &port1 = wr_ports[idx1];
 793         auto &port2 = wr_ports[idx2];
 794         // If port 2 has priority over a port before port 1, make port 1 have priority too.
 795         for (int i = 0; i < idx1; i++)
 796                 if (port2.priority_mask[i])
 797                         port1.priority_mask[i] = true;
 798         // If port 2 has priority over a port after port 1, emulate it.
 799         for (int i = idx1 + 1; i < idx2; i++)
 800                 if (port2.priority_mask[i])
 801                         emulate_priority(i, idx2);
 802         // If some port had priority over port 2, make it have priority over the merged port too.
 803         for (int i = idx2 + 1; i < GetSize(wr_ports); i++) {
 804                 auto &oport = wr_ports[i];
 805                 if (oport.priority_mask[idx2])
 806                         oport.priority_mask[idx1] = true;
 807         }
 808 }