Clean up some mysql stuff to make things work again and

hopefully improve performance a tad.

base/stats/mysql.cc:
    - it's not called sample anymore, it's called tick
    - don't bother to cleanup deleted runs.  Doing this for each run
    is not necessary, it can be done all at once
    - don't query for a bin id every time, just do it once.
    - use locking in a few places to prevent two processes from
    stepping on eachother.
    - don't duplicate subdata ids.  use -1,y and x,-1
base/stats/mysql.hh:
    It's not called sample anymore, it's called tick

--HG--
extra : convert_revision : 95de8498b627c9175da28a66604ec7c719f7804c

diff --git a/base/stats/mysql.cc b/base/stats/mysql.cc
index 4c2a601271cd06cdccd7b83c19f30fa3ae53ead8..a749b573a62ec0e6d04374205e74a18578a2143b 100644 (file)
--- a/base/stats/mysql.cc
+++ b/base/stats/mysql.cc
@@ -67,9 +67,16 @@ MySqlRun::connect(const string &host, const string &user, const string &passwd,
     if (mysql.error)
         panic("could not connect to database server\n%s\n", mysql.error);
 
+    mysql.query("LOCK TABLES runs WRITE");
+    if (mysql.error)
+        panic("could not lock tables\n%s\n", mysql.error);
+
     remove(name);
-    cleanup();
+//    cleanup();
     setup(name, sample, user, project);
+    mysql.query("UNLOCK TABLES");
+    if (mysql.error)
+        panic("could not unlock tables\n%s\n", mysql.error);
 }
 
 void
@@ -100,6 +107,8 @@ MySqlRun::remove(const string &name)
     stringstream sql;
     ccprintf(sql, "DELETE FROM runs WHERE rn_name=\"%s\"", name);
     mysql.query(sql);
+    if (mysql.error)
+        panic("could not delete run\n%s\n", mysql.error);
 }
 
 void
@@ -195,13 +204,13 @@ SetupStat::setup()
     mysql.query(select);
     MySQL::Result result = mysql.store_result();
     if (!result)
-        panic("could not get a run\n%s\n", mysql.error);
+        panic("could not find stat\n%s\n", mysql.error);
 
 
     assert(result.num_fields() == 16);
     MySQL::Row row = result.fetch_row();
     if (!row)
-        panic("could not get a run\n%s\n", mysql.error);
+        panic("could not get stat row\n%s\n", mysql.error);
 
     bool tb;
     int8_t ti8;
@@ -274,33 +283,52 @@ SetupStat::setup()
 unsigned
 SetupBin(const string &bin)
 {
-    MySQL::Connection &mysql = MySqlDB.conn();
-    assert(mysql.connected());
+    static map<string, int> binmap;
 
     using namespace MySQL;
+    map<string,int>::const_iterator i = binmap.find(bin);
+    if (i != binmap.end())
+        return (*i).second;
+
+    Connection &mysql = MySqlDB.conn();
+    assert(mysql.connected());
+
+    mysql.query("LOCK TABLES bins WRITE");
+    if (mysql.error)
+        panic("could not lock bin table\n%s\n", mysql.error);
+
+    uint16_t bin_id;
+
     stringstream select;
+    stringstream insert;
     ccprintf(select, "SELECT bn_id FROM bins WHERE bn_name=\"%s\"", bin);
 
     mysql.query(select);
     MySQL::Result result = mysql.store_result();
     if (result) {
         assert(result.num_fields() == 1);
-        Row row = result.fetch_row();
+        MySQL::Row row = result.fetch_row();
         if (row) {
-            uint16_t bin_id;
             to_number(row[0], bin_id);
-            return bin_id;
+            goto exit;
         }
     }
 
-    stringstream insert;
     ccprintf(insert, "INSERT INTO bins(bn_name) values(\"%s\")", bin);
 
     mysql.query(insert);
     if (mysql.error)
-        panic("could not get a run\n%s\n", mysql.error);
+        panic("could not get a bin\n%s\n", mysql.error);
+
+    bin_id = mysql.insert_id();
+    binmap.insert(make_pair(bin, bin_id));
+
+  exit:
+    mysql.query("UNLOCK TABLES");
+    if (mysql.error)
+        panic("could not unlock tables\n%s\n", mysql.error);
 
-    return mysql.insert_id();
+    return bin_id;
 }
 
 InsertData::InsertData()
@@ -322,13 +350,15 @@ InsertData::flush()
         MySQL::Connection &mysql = MySqlDB.conn();
         assert(mysql.connected());
         mysql.query(query);
+        if (mysql.error)
+            panic("could not insert data\n%s\n", mysql.error);
     }
 
     query[0] = '\0';
     size = 0;
     first = true;
     strcpy(query, "INSERT INTO "
-           "data(dt_stat,dt_x,dt_y,dt_run,dt_sample,dt_bin,dt_data) "
+           "data(dt_stat,dt_x,dt_y,dt_run,dt_tick,dt_bin,dt_data) "
            "values");
     size = strlen(query);
 }
@@ -347,7 +377,7 @@ InsertData::insert()
     first = false;
 
     size += sprintf(query + size, "(%u,%d,%d,%u,%llu,%u,\"%f\")",
-                    stat, x, y, MySqlDB.run(), (unsigned long long)sample,
+                    stat, x, y, MySqlDB.run(), (unsigned long long)tick,
                     bin, data);
 }
 
@@ -374,6 +404,8 @@ InsertSubData::setup()
              stat, x, y, name, descr);
 
     mysql.query(insert);
+//    if (mysql.error)
+//     panic("could not insert subdata\n%s\n", mysql.error);
 }
 
 void
@@ -387,13 +419,17 @@ InsertFormula(uint16_t stat, const string &formula)
              stat, formula);
 
     mysql.query(insert_formula);
+//    if (mysql.error)
+//     panic("could not insert formula\n%s\n", mysql.error);
 
     stringstream insert_ref;
     ccprintf(insert_ref,
              "INSERT INTO formula_ref(fr_stat,fr_run) values(%d, %d)",
              stat, MySqlDB.run());
 
-        mysql.query(insert_ref);
+    mysql.query(insert_ref);
+//    if (mysql.error)
+//     panic("could not insert formula reference\n%s\n", mysql.error);
 }
 
 void
@@ -405,6 +441,8 @@ UpdatePrereq(uint16_t stat, uint16_t prereq)
     ccprintf(update, "UPDATE stats SET st_prereq=%d WHERE st_id=%d",
              prereq, stat);
     mysql.query(update);
+    if (mysql.error)
+        panic("could not update prereq\n%s\n", mysql.error);
 }
 
 void
@@ -414,6 +452,17 @@ MySql::configure()
      * set up all stats!
      */
     using namespace Database;
+
+    MySQL::Connection &mysql = MySqlDB.conn();
+    mysql.query("LOCK TABLES "
+                "stats WRITE, "
+                "bins WRITE, "
+                "subdata WRITE, "
+                "formulas WRITE, "
+                "formula_ref WRITE");
+    if (mysql.error)
+        panic("could not lock tables\n%s\n", mysql.error);
+
     stat_list_t::const_iterator i, end = stats().end();
     for (i = stats().begin(); i != end; ++i)
         (*i)->visit(*this);
@@ -429,6 +478,10 @@ MySql::configure()
         }
     }
 
+    mysql.query("UNLOCK TABLES");
+    if (mysql.error)
+        panic("could not unlock tables\n%s\n", mysql.error);
+
     configured = true;
 }
 
@@ -531,7 +584,7 @@ MySql::configure(const Vector2dData &data)
     if (!data.subnames.empty()) {
         InsertSubData subdata;
         subdata.stat = statid;
-        subdata.y = 0;
+        subdata.y = -1;
         for (int i = 0; i < data.subnames.size(); ++i) {
             subdata.x = i;
             subdata.name = data.subnames[i];
@@ -544,7 +597,7 @@ MySql::configure(const Vector2dData &data)
     if (!data.y_subnames.empty()) {
         InsertSubData subdata;
         subdata.stat = statid;
-        subdata.x = 0;
+        subdata.x = -1;
         subdata.descr = "";
         for (int i = 0; i < data.y_subnames.size(); ++i) {
             subdata.y = i;
@@ -562,6 +615,7 @@ MySql::configure(const FormulaData &data)
 {
     configure(data, "FORMULA");
     insert(data.id, stat.setup());
+    InsertFormula(find(data.id), data.str());
 }
 
 void
@@ -591,7 +645,7 @@ MySql::output()
         configure();
 
     // store sample #
-    newdata.sample = curTick;
+    newdata.tick = curTick;
 
     if (bins().empty()) {
         output(string(""));
@@ -722,7 +776,6 @@ MySql::output(const Vector2dData &data)
 void
 MySql::output(const FormulaData &data)
 {
-    InsertFormula(find(data.id), data.str());
 }
 
 /*

diff --git a/base/stats/mysql.hh b/base/stats/mysql.hh
index 5780009d7afe97ba7d56c23b5795116f23e6a461..fcbe8c5e0f6ae94bbe02a00da6a0b747cf335aab 100644 (file)
--- a/base/stats/mysql.hh
+++ b/base/stats/mysql.hh
@@ -76,7 +76,7 @@ class InsertData
     MySqlRun *run;
 
   public:
-    uint64_t sample;
+    uint64_t tick;
     double data;
     uint16_t stat;
     uint16_t bin;