stats: unify the two stats distribution type better
[gem5.git] / src / base / stats / mysql.cc
1 /*
2 * Copyright (c) 2004-2005 The Regents of The University of Michigan
3 * All rights reserved.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions are
7 * met: redistributions of source code must retain the above copyright
8 * notice, this list of conditions and the following disclaimer;
9 * redistributions in binary form must reproduce the above copyright
10 * notice, this list of conditions and the following disclaimer in the
11 * documentation and/or other materials provided with the distribution;
12 * neither the name of the copyright holders nor the names of its
13 * contributors may be used to endorse or promote products derived from
14 * this software without specific prior written permission.
15 *
16 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
17 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
18 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
19 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
20 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
21 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
22 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
23 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
24 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
25 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
26 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
27 *
28 * Authors: Nathan Binkert
29 */
30
31 #include <cassert>
32 #include <cstdio>
33 #include <map>
34 #include <sstream>
35 #include <string>
36 #include <vector>
37
38 #include "base/misc.hh"
39 #include "base/mysql.hh"
40 #include "base/statistics.hh"
41 #include "base/stats/info.hh"
42 #include "base/stats/mysql.hh"
43 #include "base/stats/mysql_run.hh"
44 #include "base/stats/types.hh"
45 #include "base/str.hh"
46 #include "base/types.hh"
47 #include "base/userinfo.hh"
48
49 using namespace std;
50
51 namespace Stats {
52
53 void
54 MySqlRun::connect(const string &host, const string &user, const string &passwd,
55 const string &db, const string &name, const string &sample,
56 const string &project)
57 {
58 if (connected())
59 panic("can only get one database connection at this time!");
60
61 mysql.connect(host, user, passwd, db);
62 if (mysql.error)
63 panic("could not connect to database server\n%s\n", mysql.error);
64
65 if (mysql.autocommit(false))
66 panic("could not set autocommit\n%s\n", mysql.error);
67
68 remove(name);
69 //cleanup();
70 setup(name, sample, user, project);
71 }
72
73 void
74 MySqlRun::setup(const string &name, const string &sample, const string &user,
75 const string &project)
76 {
77 assert(mysql.connected());
78
79 stringstream insert;
80 ccprintf(insert,
81 "INSERT INTO "
82 "runs(rn_name,rn_sample,rn_user,rn_project,rn_date,rn_expire)"
83 "values(\"%s\", \"%s\", \"%s\", \"%s\", NOW(),"
84 "DATE_ADD(CURDATE(), INTERVAL 31 DAY))",
85 name, sample, user, project);
86
87 mysql.query(insert);
88 if (mysql.error)
89 panic("could not get a run\n%s\n", mysql.error);
90
91 run_id = mysql.insert_id();
92 if (mysql.commit())
93 panic("could not commit transaction\n%s\n", mysql.error);
94 }
95
96 void
97 MySqlRun::remove(const string &name)
98 {
99 assert(mysql.connected());
100 stringstream sql;
101 ccprintf(sql, "DELETE FROM runs WHERE rn_name=\"%s\"", name);
102 mysql.query(sql);
103 if (mysql.error)
104 panic("could not delete run\n%s\n", mysql.error);
105 if (mysql.commit())
106 panic("could not commit transaction\n%s\n", mysql.error);
107 }
108
109 void
110 MySqlRun::cleanup()
111 {
112 assert(mysql.connected());
113
114 mysql.query("DELETE data "
115 "FROM data "
116 "LEFT JOIN runs ON dt_run=rn_id "
117 "WHERE rn_id IS NULL");
118
119 if (mysql.commit())
120 panic("could not commit transaction\n%s\n", mysql.error);
121
122 mysql.query("DELETE formula_ref "
123 "FROM formula_ref "
124 "LEFT JOIN runs ON fr_run=rn_id "
125 "WHERE rn_id IS NULL");
126
127 if (mysql.commit())
128 panic("could not commit transaction\n%s\n", mysql.error);
129
130 mysql.query("DELETE formulas "
131 "FROM formulas "
132 "LEFT JOIN formula_ref ON fm_stat=fr_stat "
133 "WHERE fr_stat IS NULL");
134
135 if (mysql.commit())
136 panic("could not commit transaction\n%s\n", mysql.error);
137
138 mysql.query("DELETE stats "
139 "FROM stats "
140 "LEFT JOIN data ON st_id=dt_stat "
141 "WHERE dt_stat IS NULL");
142
143 if (mysql.commit())
144 panic("could not commit transaction\n%s\n", mysql.error);
145
146 mysql.query("DELETE subdata "
147 "FROM subdata "
148 "LEFT JOIN data ON sd_stat=dt_stat "
149 "WHERE dt_stat IS NULL");
150
151 if (mysql.commit())
152 panic("could not commit transaction\n%s\n", mysql.error);
153 }
154
155 void
156 SetupStat::init()
157 {
158 name = "";
159 descr = "";
160 type = "";
161 print = false;
162 prereq = 0;
163 prec = -1;
164 nozero = false;
165 nonan = false;
166 total = false;
167 pdf = false;
168 cdf = false;
169 min = 0;
170 max = 0;
171 bktsize = 0;
172 size = 0;
173 }
174
175 unsigned
176 SetupStat::setup(MySqlRun *run)
177 {
178 MySQL::Connection &mysql = run->conn();
179
180 stringstream insert;
181 ccprintf(insert,
182 "INSERT INTO "
183 "stats(st_name, st_descr, st_type, st_print, st_prereq, "
184 "st_prec, st_nozero, st_nonan, st_total, st_pdf, st_cdf, "
185 "st_min, st_max, st_bktsize, st_size)"
186 "values(\"%s\",\"%s\",\"%s\","
187 " %d,%d,%d,%d,%d,%d,%d,%d,%d,%d,%d,%d)",
188 name, descr, type, print, prereq, (int)prec, nozero, nonan,
189 total, pdf, cdf,
190 min, max, bktsize, size);
191
192 mysql.query(insert);
193 if (!mysql.error) {
194 int id = mysql.insert_id();
195 if (mysql.commit())
196 panic("could not commit transaction\n%s\n", mysql.error);
197 return id;
198 }
199
200 stringstream select;
201 ccprintf(select, "SELECT * FROM stats WHERE st_name=\"%s\"", name);
202
203 mysql.query(select);
204 MySQL::Result result = mysql.store_result();
205 if (!result)
206 panic("could not find stat\n%s\n", mysql.error);
207
208 assert(result.num_fields() == 16);
209 MySQL::Row row = result.fetch_row();
210 if (!row)
211 panic("could not get stat row\n%s\n", mysql.error);
212
213 bool tb;
214 int8_t ti8;
215 uint16_t tu16;
216 int64_t ti64;
217 uint64_t tu64;
218
219 if (name != (char *)row[1])
220 panic("failed stat check on %s:name. %s != %s\n",
221 name, name, row[1]);
222
223 if (descr != (char *)row[2])
224 panic("failed stat check on %s:descr. %s != %s\n",
225 name, descr, row[2]);
226
227 if (type != (char *)row[3])
228 panic("failed stat check on %s:type. %s != %s\n",
229 name, type, row[3]);
230
231 if (!to_number(row[4], tb) || print != tb)
232 panic("failed stat check on %s:print. %d != %d\n",
233 name, print, tb);
234
235 if (!to_number(row[6], ti8) || prec != ti8)
236 panic("failed stat check on %s:prec. %d != %d\n",
237 name, prec, ti8);
238
239 if (!to_number(row[7], tb) || nozero != tb)
240 panic("failed stat check on %s:nozero. %d != %d\n",
241 name, nozero, tb);
242
243 if (!to_number(row[8], tb) || nonan != tb)
244 panic("failed stat check on %s:nonan. %d != %d\n",
245 name, nonan, tb);
246
247 if (!to_number(row[9], tb) || total != tb)
248 panic("failed stat check on %s:total. %d != %d\n",
249 name, total, tb);
250
251 if (!to_number(row[10], tb) || pdf != tb)
252 panic("failed stat check on %s:pdf. %d != %d\n",
253 name, pdf, tb);
254
255 if (!to_number(row[11], tb) || cdf != tb)
256 panic("failed stat check on %s:cdf. %d != %d\n",
257 name, cdf, tb);
258
259 if (!to_number(row[12], ti64) || min != ti64)
260 panic("failed stat check on %s:min. %d != %d\n",
261 name, min, ti64);
262
263 if (!to_number(row[13], ti64) || max != ti64)
264 panic("failed stat check on %s:max. %d != %d\n",
265 name, max, ti64);
266
267 if (!to_number(row[14], tu64) || bktsize != tu64)
268 panic("failed stat check on %s:bktsize. %d != %d\n",
269 name, bktsize, tu64);
270
271 if (!to_number(row[15], tu16) || size != tu16)
272 panic("failed stat check on %s:size. %d != %d\n",
273 name, size, tu16);
274
275 to_number(row[5], prereq);
276 uint16_t statid;
277 to_number(row[0], statid);
278 return statid;
279 }
280
281 InsertData::InsertData(MySqlRun *_run)
282 : run(_run)
283 {
284 query = new char[maxsize + 1];
285 size = 0;
286 flush();
287 }
288
289 InsertData::~InsertData()
290 {
291 delete [] query;
292 }
293
294 void
295 InsertData::flush()
296 {
297 if (size) {
298 MySQL::Connection &mysql = run->conn();
299 assert(mysql.connected());
300 mysql.query(query);
301 if (mysql.error)
302 panic("could not insert data\n%s\n", mysql.error);
303 if (mysql.commit())
304 panic("could not commit transaction\n%s\n", mysql.error);
305 }
306
307 query[0] = '\0';
308 size = 0;
309 first = true;
310 strcpy(query, "INSERT INTO "
311 "data(dt_stat,dt_x,dt_y,dt_run,dt_tick,dt_data) "
312 "values");
313 size = strlen(query);
314 }
315
316 void
317 InsertData::insert()
318 {
319 if (size + 1024 > maxsize)
320 flush();
321
322 if (!first) {
323 query[size++] = ',';
324 query[size] = '\0';
325 }
326
327 first = false;
328
329 size += sprintf(query + size, "(%u,%d,%d,%u,%llu,\"%f\")",
330 stat, x, y, run->run(), (unsigned long long)tick,
331 data);
332 }
333
334 struct InsertSubData
335 {
336 uint16_t stat;
337 int16_t x;
338 int16_t y;
339 string name;
340 string descr;
341
342 void setup(MySqlRun *run);
343 };
344
345 void
346 InsertSubData::setup(MySqlRun *run)
347 {
348 MySQL::Connection &mysql = run->conn();
349 assert(mysql.connected());
350 stringstream insert;
351 ccprintf(insert,
352 "INSERT INTO subdata(sd_stat,sd_x,sd_y,sd_name,sd_descr) "
353 "values(%d,%d,%d,\"%s\",\"%s\")",
354 stat, x, y, name, descr);
355
356 mysql.query(insert);
357 // if (mysql.error)
358 // panic("could not insert subdata\n%s\n", mysql.error);
359
360 if (mysql.commit())
361 panic("could not commit transaction\n%s\n", mysql.error);
362 }
363
364 MySql::MySql()
365 : run(new MySqlRun), newdata(run)
366 {}
367
368 MySql::~MySql()
369 {
370 delete run;
371 }
372
373 void
374 MySql::connect(const string &host, const string &user, const string &passwd,
375 const string &db, const string &name, const string &sample,
376 const string &project)
377 {
378 run->connect(host, user, passwd, db, name, sample, project);
379 }
380
381 bool
382 MySql::connected() const
383 {
384 return run->connected();
385 }
386
387 void
388 MySql::configure()
389 {
390 /*
391 * set up all stats!
392 */
393 MySQL::Connection &mysql = run->conn();
394
395 list<Info *>::const_iterator i, end = statsList().end();
396 for (i = statsList().begin(); i != end; ++i) {
397 (*i)->visit(*this);
398 }
399
400 for (i = statsList().begin(); i != end; ++i) {
401 Info *info = *i;
402 if (info->prereq) {
403 // update the prerequisite
404 uint16_t stat_id = find(info->id);
405 uint16_t prereq_id = find(info->prereq->id);
406 assert(stat_id && prereq_id);
407
408 stringstream update;
409 ccprintf(update, "UPDATE stats SET st_prereq=%d WHERE st_id=%d",
410 prereq_id, stat_id);
411 mysql.query(update);
412 if (mysql.error)
413 panic("could not update prereq\n%s\n", mysql.error);
414
415 if (mysql.commit())
416 panic("could not commit transaction\n%s\n", mysql.error);
417 }
418 }
419
420 if (mysql.commit())
421 panic("could not commit transaction\n%s\n", mysql.error);
422
423 configured = true;
424 }
425
426 bool
427 MySql::configure(const Info &info, string type)
428 {
429 stat.init();
430 stat.name = info.name;
431 stat.descr = info.desc;
432 stat.type = type;
433 stat.print = info.flags & display;
434 stat.prec = info.precision;
435 stat.nozero = info.flags & nozero;
436 stat.nonan = info.flags & nonan;
437 stat.total = info.flags & total;
438 stat.pdf = info.flags & pdf;
439 stat.cdf = info.flags & cdf;
440
441 return stat.print;
442 }
443
444 void
445 MySql::configure(const ScalarInfo &info)
446 {
447 if (!configure(info, "SCALAR"))
448 return;
449
450 insert(info.id, stat.setup(run));
451 }
452
453 void
454 MySql::configure(const VectorInfo &info)
455 {
456 if (!configure(info, "VECTOR"))
457 return;
458
459 uint16_t statid = stat.setup(run);
460
461 if (!info.subnames.empty()) {
462 InsertSubData subdata;
463 subdata.stat = statid;
464 subdata.y = 0;
465 for (off_type i = 0; i < info.subnames.size(); ++i) {
466 subdata.x = i;
467 subdata.name = info.subnames[i];
468 subdata.descr = info.subdescs.empty() ? "" : info.subdescs[i];
469
470 if (!subdata.name.empty() || !subdata.descr.empty())
471 subdata.setup(run);
472 }
473 }
474
475 insert(info.id, statid);
476 }
477
478 void
479 MySql::configure(const DistInfo &info)
480 {
481 if (!configure(info, "DIST"))
482 return;
483
484 const DistStor::Params *params =
485 dynamic_cast<const DistStor::Params *>(info.storageParams);
486 if (params) {
487 assert(params->type == Dist);
488 stat.size = params->buckets;
489 stat.min = params->min;
490 stat.max = params->max;
491 stat.bktsize = params->bucket_size;
492 }
493 insert(info.id, stat.setup(run));
494 }
495
496 void
497 MySql::configure(const VectorDistInfo &info)
498 {
499 if (!configure(info, "VECTORDIST"))
500 return;
501
502 const DistStor::Params *params =
503 dynamic_cast<const DistStor::Params *>(info.storageParams);
504 if (params) {
505 assert(params->type == Dist);
506 stat.size = params->buckets;
507 stat.min = params->min;
508 stat.max = params->max;
509 stat.bktsize = params->bucket_size;
510 }
511
512 uint16_t statid = stat.setup(run);
513
514 if (!info.subnames.empty()) {
515 InsertSubData subdata;
516 subdata.stat = statid;
517 subdata.y = 0;
518 for (off_type i = 0; i < info.subnames.size(); ++i) {
519 subdata.x = i;
520 subdata.name = info.subnames[i];
521 subdata.descr = info.subdescs.empty() ? "" : info.subdescs[i];
522 if (!subdata.name.empty() || !subdata.descr.empty())
523 subdata.setup(run);
524 }
525 }
526
527 insert(info.id, statid);
528 }
529
530 void
531 MySql::configure(const Vector2dInfo &info)
532 {
533 if (!configure(info, "VECTOR2D"))
534 return;
535
536 uint16_t statid = stat.setup(run);
537
538 if (!info.subnames.empty()) {
539 InsertSubData subdata;
540 subdata.stat = statid;
541 subdata.y = -1;
542 for (off_type i = 0; i < info.subnames.size(); ++i) {
543 subdata.x = i;
544 subdata.name = info.subnames[i];
545 subdata.descr = info.subdescs.empty() ? "" : info.subdescs[i];
546 if (!subdata.name.empty() || !subdata.descr.empty())
547 subdata.setup(run);
548 }
549 }
550
551 if (!info.y_subnames.empty()) {
552 InsertSubData subdata;
553 subdata.stat = statid;
554 subdata.x = -1;
555 subdata.descr = "";
556 for (off_type i = 0; i < info.y_subnames.size(); ++i) {
557 subdata.y = i;
558 subdata.name = info.y_subnames[i];
559 if (!subdata.name.empty())
560 subdata.setup(run);
561 }
562 }
563
564 insert(info.id, statid);
565 }
566
567 void
568 MySql::configure(const FormulaInfo &info)
569 {
570 MySQL::Connection &mysql = run->conn();
571 assert(mysql.connected());
572
573 configure(info, "FORMULA");
574 insert(info.id, stat.setup(run));
575
576 uint16_t stat = find(info.id);
577 string formula = info.str();
578
579 stringstream insert_formula;
580 ccprintf(insert_formula,
581 "INSERT INTO formulas(fm_stat,fm_formula) values(%d, \"%s\")",
582 stat, formula);
583
584 mysql.query(insert_formula);
585 // if (mysql.error)
586 // panic("could not insert formula\n%s\n", mysql.error);
587
588 stringstream insert_ref;
589 ccprintf(insert_ref,
590 "INSERT INTO formula_ref(fr_stat,fr_run) values(%d, %d)",
591 stat, run->run());
592
593 mysql.query(insert_ref);
594 // if (mysql.error)
595 // panic("could not insert formula reference\n%s\n", mysql.error);
596
597 if (mysql.commit())
598 panic("could not commit transaction\n%s\n", mysql.error);
599 }
600
601 bool
602 MySql::valid() const
603 {
604 return run->connected();
605 }
606
607 void
608 MySql::output()
609 {
610 assert(valid());
611
612 if (!configured)
613 configure();
614
615 // store sample #
616 newdata.tick = curTick;
617
618 MySQL::Connection &mysql = run->conn();
619
620 list<Info *>::const_iterator i, end = statsList().end();
621 for (i = statsList().begin(); i != end; ++i) {
622 Info *stat = *i;
623 stat->visit(*this);
624 if (mysql.commit())
625 panic("could not commit transaction\n%s\n", mysql.error);
626 }
627
628 newdata.flush();
629 }
630
631 void
632 MySql::output(const ScalarInfo &info)
633 {
634 if (!(info.flags & display))
635 return;
636
637 newdata.stat = find(info.id);
638 newdata.x = 0;
639 newdata.y = 0;
640 newdata.data = info.value();
641
642 newdata.insert();
643 }
644
645 void
646 MySql::output(const VectorInfo &info)
647 {
648 if (!(info.flags & display))
649 return;
650
651 newdata.stat = find(info.id);
652 newdata.y = 0;
653
654 const VCounter &cvec = info.value();
655 size_type size = info.size();
656 for (off_type x = 0; x < size; x++) {
657 newdata.x = x;
658 newdata.data = cvec[x];
659 newdata.insert();
660 }
661 }
662
663 void
664 MySql::output(const DistData &data, const DistParams *params)
665 {
666 const int db_sum = -1;
667 const int db_squares = -2;
668 const int db_samples = -3;
669 const int db_min_val = -4;
670 const int db_max_val = -5;
671 const int db_underflow = -6;
672 const int db_overflow = -7;
673
674 newdata.x = db_sum;
675 newdata.data = data.sum;
676 newdata.insert();
677
678 newdata.x = db_squares;
679 newdata.data = data.squares;
680 newdata.insert();
681
682 newdata.x = db_samples;
683 newdata.data = data.samples;
684 newdata.insert();
685
686 if (data.samples && params->type == Dist) {
687 newdata.x = db_min_val;
688 newdata.data = data.min_val;
689 newdata.insert();
690
691 newdata.x = db_max_val;
692 newdata.data = data.max_val;
693 newdata.insert();
694
695 newdata.x = db_underflow;
696 newdata.data = data.underflow;
697 newdata.insert();
698
699 newdata.x = db_overflow;
700 newdata.data = data.overflow;
701 newdata.insert();
702
703 size_type size = data.cvec.size();
704 for (off_type x = 0; x < size; x++) {
705 newdata.x = x;
706 newdata.data = data.cvec[x];
707 newdata.insert();
708 }
709 }
710 }
711
712 void
713 MySql::output(const DistInfo &info)
714 {
715 if (!(info.flags & display))
716 return;
717
718 newdata.stat = find(info.id);
719 newdata.y = 0;
720 output(info.data, safe_cast<const DistParams *>(info.storageParams));
721 }
722
723 void
724 MySql::output(const VectorDistInfo &info)
725 {
726 if (!(info.flags & display))
727 return;
728
729 newdata.stat = find(info.id);
730
731 size_type size = info.data.size();
732 for (off_type y = 0; y < size; ++y) {
733 newdata.y = y;
734 output(info.data[y],
735 safe_cast<const DistParams *>(info.storageParams));
736 }
737 }
738
739 void
740 MySql::output(const Vector2dInfo &info)
741 {
742 if (!(info.flags & display))
743 return;
744
745 newdata.stat = find(info.id);
746
747 off_type index = 0;
748 for (off_type x = 0; x < info.x; x++) {
749 newdata.x = x;
750 for (off_type y = 0; y < info.y; y++) {
751 newdata.y = y;
752 newdata.data = info.cvec[index++];
753 newdata.insert();
754 }
755 }
756 }
757
758 void
759 MySql::output(const FormulaInfo &info)
760 {
761 }
762
763 /*
764 * Implement the visitor
765 */
766 void
767 MySql::visit(const ScalarInfo &info)
768 {
769 if (!configured)
770 configure(info);
771 else
772 output(info);
773 }
774
775 void
776 MySql::visit(const VectorInfo &info)
777 {
778 if (!configured)
779 configure(info);
780 else
781 output(info);
782 }
783
784 void
785 MySql::visit(const DistInfo &info)
786 {
787 return;
788 if (!configured)
789 configure(info);
790 else
791 output(info);
792 }
793
794 void
795 MySql::visit(const VectorDistInfo &info)
796 {
797 return;
798 if (!configured)
799 configure(info);
800 else
801 output(info);
802 }
803
804 void
805 MySql::visit(const Vector2dInfo &info)
806 {
807 return;
808 if (!configured)
809 configure(info);
810 else
811 output(info);
812 }
813
814 void
815 MySql::visit(const FormulaInfo &info)
816 {
817 if (!configured)
818 configure(info);
819 else
820 output(info);
821 }
822
823 bool
824 initMySQL(string host, string user, string password, string database,
825 string project, string name, string sample)
826 {
827 extern list<Output *> OutputList;
828 static MySql mysql;
829
830 if (mysql.connected())
831 return false;
832
833 mysql.connect(host, user, password, database, name, sample, project);
834 OutputList.push_back(&mysql);
835
836 return true;
837 }
838
839 /* end namespace Stats */ }