util/multi/tcp_server.hh

   1 /*
   2  * Copyright (c) 2015 ARM Limited
   3  * All rights reserved
   4  *
   5  * The license below extends only to copyright in the software and shall
   6  * not be construed as granting a license to any other intellectual
   7  * property including but not limited to intellectual property relating
   8  * to a hardware implementation of the functionality of the software
   9  * licensed hereunder.  You may use the software subject to the license
  10  * terms below provided that you ensure that this notice is replicated
  11  * unmodified and in its entirety in all distributions of the software,
  12  * modified or unmodified, in source code or in binary form.
  13  *
  14  * Copyright (c) 2008 The Regents of The University of Michigan
  15  * All rights reserved.
  16  *
  17  * Redistribution and use in source and binary forms, with or without
  18  * modification, are permitted provided that the following conditions are
  19  * met: redistributions of source code must retain the above copyright
  20  * notice, this list of conditions and the following disclaimer;
  21  * redistributions in binary form must reproduce the above copyright
  22  * notice, this list of conditions and the following disclaimer in the
  23  * documentation and/or other materials provided with the distribution;
  24  * neither the name of the copyright holders nor the names of its
  25  * contributors may be used to endorse or promote products derived from
  26  * this software without specific prior written permission.
  27  *
  28  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  29  * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  30  * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  31  * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  32  * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  33  * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  34  * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  35  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  36  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  37  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  38  * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  39  *
  40  * Authors: Gabor Dozsa
  41  */
  42
  43 /* @file
  44  *  Message server using TCP stream sockets for parallel gem5 runs.
  45  *
  46  * For a high level description about multi gem5 see comments in
  47  * header files src/dev/multi_iface.hh and src/dev/tcp_iface.hh.
  48  *
  49  * This file implements the central message server process for multi gem5.
  50  * The server is responsible the following tasks.
  51  * 1. Establishing a TCP socket connection for each gem5 process (clients).
  52  *
  53  * 2. Process data messages coming in from clients. The server checks
  54  * the MAC addresses in the header message and transfers the message
  55  * to the target(s) client(s).
  56  *
  57  * 3. Processing synchronisation related control messages. Synchronisation
  58  * is performed as follows. The server waits for a 'barrier enter' message
  59  * from all the clients. When the last such control message arrives, the
  60  * server sends out a 'barrier leave' control message to all the clients.
  61  *
  62  * 4. Triggers complete termination in case a client exits. A client may
  63  * exit either by calling 'm5 exit' pseudo instruction or due to a fatal
  64  * error. In either case, we assume that the entire multi simulation needs to
  65  * terminate. The server triggers full termination by tearing down the
  66  * open TCP sockets.
  67  *
  68  * The TCPServer class is instantiated as a singleton object.
  69  *
  70  * The server can be built independently from the rest of gem5 (and it is
  71  * architecture agnostic). See the Makefile in the same directory.
  72  *
  73  */
  74
  75 #include <poll.h>
  76
  77 #include <map>
  78 #include <vector>
  79
  80 #include "dev/etherpkt.hh"
  81 #include "dev/multi_packet.hh"
  82
  83 /**
  84  * The maximum length of an Ethernet packet (allowing Jumbo frames).
  85  */
  86 #define MAX_ETH_PACKET_LENGTH 9014
  87
  88 class TCPServer
  89 {
  90   public:
  91     typedef MultiHeaderPkt::AddressType AddressType;
  92     typedef MultiHeaderPkt::Header Header;
  93     typedef MultiHeaderPkt::MsgType MsgType;
  94
  95   private:
  96
  97     enum
  98     class SyncState { periodic, ckpt, asyncCkpt, atomic, idle };
  99     /**
 100      * The Channel class encapsulates all the information about a client
 101      * and its current status.
 102      */
 103     class Channel
 104     {
 105       private:
 106         /**
 107          * The MAC address of the client.
 108          */
 109         AddressType address;
 110
 111         /**
 112          * Update the client MAC address. It is called every time a new data
 113          * packet is to come in.
 114          */
 115         void updateAddress(const AddressType &new_addr);
 116         /**
 117          * Process an incoming command message.
 118          */
 119         void processCmd(MultiHeaderPkt::MsgType cmd, Tick send_tick);
 120
 121
 122       public:
 123         /**
 124          * TCP stream socket.
 125          */
 126         int fd;
 127         /**
 128          * Is client connected?
 129          */
 130         bool isAlive;
 131         /**
 132          * Current state of the channel wrt. multi synchronisation.
 133          */
 134         SyncState state;
 135         /**
 136          * Multi rank of the client
 137          */
 138         unsigned rank;
 139
 140       public:
 141         Channel();
 142         ~Channel () {}
 143
 144
 145         /**
 146          * Receive and process the next incoming header packet.
 147          */
 148         void headerPktIn();
 149         /**
 150          * Send raw data to the connected client.
 151          *
 152          * @param data The data to send.
 153          * @param size Size of the data (in bytes).
 154          */
 155         void sendRaw(const void *data, unsigned size) const;
 156         /**
 157          * Receive raw data from the connected client.
 158          *
 159          * @param buf The buffer to store the incoming data into.
 160          * @param size Size of data to receive (in bytes).
 161          * @return In case of success, it returns size. Zero is returned
 162          * if the socket is already closed by the client.
 163          */
 164         unsigned recvRaw(void *buf, unsigned size) const;
 165     };
 166
 167     /**
 168      * The array of socket descriptors needed by the poll() system call.
 169      */
 170     std::vector<struct pollfd> clientsPollFd;
 171     /**
 172      * Array holding all clients info.
 173      */
 174     std::vector<Channel> clientsChannel;
 175
 176
 177     /**
 178      * We use a map to select the target client based on the destination
 179      * MAC address.
 180      */
 181     struct AddressCompare
 182     {
 183         bool operator()(const AddressType *a1, const AddressType *a2)
 184         {
 185             return MultiHeaderPkt::isAddressLess(*a1, *a2);
 186         }
 187     };
 188     std::map<const AddressType *, Channel *, AddressCompare> addressMap;
 189
 190     /**
 191      * As we dealt with only one message at a time, we can allocate and re-use
 192      * a single packet buffer (to hold any incoming data packet).
 193      */
 194     uint8_t packetBuffer[MAX_ETH_PACKET_LENGTH];
 195     /**
 196      * Send tick of the current periodic sync. It is used for sanity check.
 197      */
 198     Tick _periodicSyncTick;
 199     /**
 200      * The singleton server object.
 201      */
 202     static TCPServer *instance;
 203
 204     /**
 205      * Set up the socket connections to all the clients.
 206      *
 207      * @param listen_port The port we are listening on for new client
 208      * connection requests.
 209      * @param nclients The number of clients to connect to.
 210      * @param timeout Timeout in sec to complete the setup phase
 211      * (i.e. all gem5 establish socket connections)
 212      */
 213     void construct(unsigned listen_port, unsigned nclients, int timeout);
 214     /**
 215      * Transfer the header and the follow up data packet to the target(s)
 216      * clients.
 217      *
 218      * @param hdr The header message structure.
 219      * @param ch The source channel for the message.
 220      */
 221     void xferData(const Header &hdr, const Channel &ch);
 222     /**
 223      * Check if the current round of a synchronisation is completed and notify
 224      * the clients if it is so.
 225      *
 226      * @param st The state all channels should have if sync is complete.
 227      * @param ack The type of ack message to send out if the sync is compete.
 228      */
 229     void syncTryComplete(SyncState st, MultiHeaderPkt::MsgType ack);
 230     /**
 231      * Broadcast a request for checkpoint sync.
 232      *
 233      * @param ch The source channel of the checkpoint sync request.
 234      */
 235     void ckptPropagate(Channel &ch);
 236     /**
 237      * Setter for current periodic send tick.
 238      */
 239     void periodicSyncTick(Tick t) { _periodicSyncTick = t; }
 240     /**
 241      * Getter for current periodic send tick.
 242      */
 243     Tick periodicSyncTick() { return _periodicSyncTick; }
 244
 245   public:
 246
 247     TCPServer(unsigned clients_num, unsigned listen_port, int timeout_in_sec);
 248     ~TCPServer();
 249
 250     /**
 251      * The main server loop that waits for and processes incoming messages.
 252      */
 253     void run();
 254 };