Fix matmul performance on gcc 4.9

It's just loop interchange in the register blocking loop.

diff --git a/benchmarks/mm/gen.scala b/benchmarks/mm/gen.scala
index 2d3dc34a2978a5b6583e7a86f95b5f1c060afe34..38d9e6badb9ab478234febc6796dd004d49feb9c 100644 (file)
--- a/benchmarks/mm/gen.scala
+++ b/benchmarks/mm/gen.scala
@@ -39,7 +39,7 @@ object MMGen {
         s += init("t*", r("a", i), "&"+ar("a", "lda*"+i))
       for (k <- 0 until p)
         s += init("t*", r("b", k), "&"+ar("b", "ldb*"+k))
-      for (i <- 0 until m; j <- 0 until n; k <- 0 until p)
+      for (k <- 0 until p; i <- 0 until m; j <- 0 until n)
         s += assign(r("c", i, j), "fma(" + ar(r("a", i), k) + ", " + ar(r("b", k), j) + ", " + r("c", i, j) + ")")
     }
 

diff --git a/benchmarks/mm/rb.h b/benchmarks/mm/rb.h
index fb08386104d8b9205f98c7ab7f4a8461f328e094..593b54f134f2b5818c74711708c37da93e56a092 100644 (file)
--- a/benchmarks/mm/rb.h
+++ b/benchmarks/mm/rb.h
@@ -41,124 +41,124 @@ static inline void kloop(size_t p, t* a0, size_t lda, t* b0, size_t ldb, t* c, s
     t* b_4 = &b[ldb*4];
     t* b_5 = &b[ldb*5];
     c_0_0 = fma(a_0[0], b_0[0], c_0_0);
-    c_0_0 = fma(a_0[1], b_1[0], c_0_0);
-    c_0_0 = fma(a_0[2], b_2[0], c_0_0);
-    c_0_0 = fma(a_0[3], b_3[0], c_0_0);
-    c_0_0 = fma(a_0[4], b_4[0], c_0_0);
-    c_0_0 = fma(a_0[5], b_5[0], c_0_0);
     c_0_1 = fma(a_0[0], b_0[1], c_0_1);
-    c_0_1 = fma(a_0[1], b_1[1], c_0_1);
-    c_0_1 = fma(a_0[2], b_2[1], c_0_1);
-    c_0_1 = fma(a_0[3], b_3[1], c_0_1);
-    c_0_1 = fma(a_0[4], b_4[1], c_0_1);
-    c_0_1 = fma(a_0[5], b_5[1], c_0_1);
     c_0_2 = fma(a_0[0], b_0[2], c_0_2);
-    c_0_2 = fma(a_0[1], b_1[2], c_0_2);
-    c_0_2 = fma(a_0[2], b_2[2], c_0_2);
-    c_0_2 = fma(a_0[3], b_3[2], c_0_2);
-    c_0_2 = fma(a_0[4], b_4[2], c_0_2);
-    c_0_2 = fma(a_0[5], b_5[2], c_0_2);
     c_0_3 = fma(a_0[0], b_0[3], c_0_3);
-    c_0_3 = fma(a_0[1], b_1[3], c_0_3);
-    c_0_3 = fma(a_0[2], b_2[3], c_0_3);
-    c_0_3 = fma(a_0[3], b_3[3], c_0_3);
-    c_0_3 = fma(a_0[4], b_4[3], c_0_3);
-    c_0_3 = fma(a_0[5], b_5[3], c_0_3);
     c_0_4 = fma(a_0[0], b_0[4], c_0_4);
-    c_0_4 = fma(a_0[1], b_1[4], c_0_4);
-    c_0_4 = fma(a_0[2], b_2[4], c_0_4);
-    c_0_4 = fma(a_0[3], b_3[4], c_0_4);
-    c_0_4 = fma(a_0[4], b_4[4], c_0_4);
-    c_0_4 = fma(a_0[5], b_5[4], c_0_4);
     c_1_0 = fma(a_1[0], b_0[0], c_1_0);
-    c_1_0 = fma(a_1[1], b_1[0], c_1_0);
-    c_1_0 = fma(a_1[2], b_2[0], c_1_0);
-    c_1_0 = fma(a_1[3], b_3[0], c_1_0);
-    c_1_0 = fma(a_1[4], b_4[0], c_1_0);
-    c_1_0 = fma(a_1[5], b_5[0], c_1_0);
     c_1_1 = fma(a_1[0], b_0[1], c_1_1);
-    c_1_1 = fma(a_1[1], b_1[1], c_1_1);
-    c_1_1 = fma(a_1[2], b_2[1], c_1_1);
-    c_1_1 = fma(a_1[3], b_3[1], c_1_1);
-    c_1_1 = fma(a_1[4], b_4[1], c_1_1);
-    c_1_1 = fma(a_1[5], b_5[1], c_1_1);
     c_1_2 = fma(a_1[0], b_0[2], c_1_2);
-    c_1_2 = fma(a_1[1], b_1[2], c_1_2);
-    c_1_2 = fma(a_1[2], b_2[2], c_1_2);
-    c_1_2 = fma(a_1[3], b_3[2], c_1_2);
-    c_1_2 = fma(a_1[4], b_4[2], c_1_2);
-    c_1_2 = fma(a_1[5], b_5[2], c_1_2);
     c_1_3 = fma(a_1[0], b_0[3], c_1_3);
-    c_1_3 = fma(a_1[1], b_1[3], c_1_3);
-    c_1_3 = fma(a_1[2], b_2[3], c_1_3);
-    c_1_3 = fma(a_1[3], b_3[3], c_1_3);
-    c_1_3 = fma(a_1[4], b_4[3], c_1_3);
-    c_1_3 = fma(a_1[5], b_5[3], c_1_3);
     c_1_4 = fma(a_1[0], b_0[4], c_1_4);
-    c_1_4 = fma(a_1[1], b_1[4], c_1_4);
-    c_1_4 = fma(a_1[2], b_2[4], c_1_4);
-    c_1_4 = fma(a_1[3], b_3[4], c_1_4);
-    c_1_4 = fma(a_1[4], b_4[4], c_1_4);
-    c_1_4 = fma(a_1[5], b_5[4], c_1_4);
     c_2_0 = fma(a_2[0], b_0[0], c_2_0);
-    c_2_0 = fma(a_2[1], b_1[0], c_2_0);
-    c_2_0 = fma(a_2[2], b_2[0], c_2_0);
-    c_2_0 = fma(a_2[3], b_3[0], c_2_0);
-    c_2_0 = fma(a_2[4], b_4[0], c_2_0);
-    c_2_0 = fma(a_2[5], b_5[0], c_2_0);
     c_2_1 = fma(a_2[0], b_0[1], c_2_1);
-    c_2_1 = fma(a_2[1], b_1[1], c_2_1);
-    c_2_1 = fma(a_2[2], b_2[1], c_2_1);
-    c_2_1 = fma(a_2[3], b_3[1], c_2_1);
-    c_2_1 = fma(a_2[4], b_4[1], c_2_1);
-    c_2_1 = fma(a_2[5], b_5[1], c_2_1);
     c_2_2 = fma(a_2[0], b_0[2], c_2_2);
-    c_2_2 = fma(a_2[1], b_1[2], c_2_2);
-    c_2_2 = fma(a_2[2], b_2[2], c_2_2);
-    c_2_2 = fma(a_2[3], b_3[2], c_2_2);
-    c_2_2 = fma(a_2[4], b_4[2], c_2_2);
-    c_2_2 = fma(a_2[5], b_5[2], c_2_2);
     c_2_3 = fma(a_2[0], b_0[3], c_2_3);
-    c_2_3 = fma(a_2[1], b_1[3], c_2_3);
-    c_2_3 = fma(a_2[2], b_2[3], c_2_3);
-    c_2_3 = fma(a_2[3], b_3[3], c_2_3);
-    c_2_3 = fma(a_2[4], b_4[3], c_2_3);
-    c_2_3 = fma(a_2[5], b_5[3], c_2_3);
     c_2_4 = fma(a_2[0], b_0[4], c_2_4);
-    c_2_4 = fma(a_2[1], b_1[4], c_2_4);
-    c_2_4 = fma(a_2[2], b_2[4], c_2_4);
-    c_2_4 = fma(a_2[3], b_3[4], c_2_4);
-    c_2_4 = fma(a_2[4], b_4[4], c_2_4);
-    c_2_4 = fma(a_2[5], b_5[4], c_2_4);
     c_3_0 = fma(a_3[0], b_0[0], c_3_0);
-    c_3_0 = fma(a_3[1], b_1[0], c_3_0);
-    c_3_0 = fma(a_3[2], b_2[0], c_3_0);
-    c_3_0 = fma(a_3[3], b_3[0], c_3_0);
-    c_3_0 = fma(a_3[4], b_4[0], c_3_0);
-    c_3_0 = fma(a_3[5], b_5[0], c_3_0);
     c_3_1 = fma(a_3[0], b_0[1], c_3_1);
-    c_3_1 = fma(a_3[1], b_1[1], c_3_1);
-    c_3_1 = fma(a_3[2], b_2[1], c_3_1);
-    c_3_1 = fma(a_3[3], b_3[1], c_3_1);
-    c_3_1 = fma(a_3[4], b_4[1], c_3_1);
-    c_3_1 = fma(a_3[5], b_5[1], c_3_1);
     c_3_2 = fma(a_3[0], b_0[2], c_3_2);
-    c_3_2 = fma(a_3[1], b_1[2], c_3_2);
-    c_3_2 = fma(a_3[2], b_2[2], c_3_2);
-    c_3_2 = fma(a_3[3], b_3[2], c_3_2);
-    c_3_2 = fma(a_3[4], b_4[2], c_3_2);
-    c_3_2 = fma(a_3[5], b_5[2], c_3_2);
     c_3_3 = fma(a_3[0], b_0[3], c_3_3);
-    c_3_3 = fma(a_3[1], b_1[3], c_3_3);
-    c_3_3 = fma(a_3[2], b_2[3], c_3_3);
-    c_3_3 = fma(a_3[3], b_3[3], c_3_3);
-    c_3_3 = fma(a_3[4], b_4[3], c_3_3);
-    c_3_3 = fma(a_3[5], b_5[3], c_3_3);
     c_3_4 = fma(a_3[0], b_0[4], c_3_4);
+    c_0_0 = fma(a_0[1], b_1[0], c_0_0);
+    c_0_1 = fma(a_0[1], b_1[1], c_0_1);
+    c_0_2 = fma(a_0[1], b_1[2], c_0_2);
+    c_0_3 = fma(a_0[1], b_1[3], c_0_3);
+    c_0_4 = fma(a_0[1], b_1[4], c_0_4);
+    c_1_0 = fma(a_1[1], b_1[0], c_1_0);
+    c_1_1 = fma(a_1[1], b_1[1], c_1_1);
+    c_1_2 = fma(a_1[1], b_1[2], c_1_2);
+    c_1_3 = fma(a_1[1], b_1[3], c_1_3);
+    c_1_4 = fma(a_1[1], b_1[4], c_1_4);
+    c_2_0 = fma(a_2[1], b_1[0], c_2_0);
+    c_2_1 = fma(a_2[1], b_1[1], c_2_1);
+    c_2_2 = fma(a_2[1], b_1[2], c_2_2);
+    c_2_3 = fma(a_2[1], b_1[3], c_2_3);
+    c_2_4 = fma(a_2[1], b_1[4], c_2_4);
+    c_3_0 = fma(a_3[1], b_1[0], c_3_0);
+    c_3_1 = fma(a_3[1], b_1[1], c_3_1);
+    c_3_2 = fma(a_3[1], b_1[2], c_3_2);
+    c_3_3 = fma(a_3[1], b_1[3], c_3_3);
     c_3_4 = fma(a_3[1], b_1[4], c_3_4);
+    c_0_0 = fma(a_0[2], b_2[0], c_0_0);
+    c_0_1 = fma(a_0[2], b_2[1], c_0_1);
+    c_0_2 = fma(a_0[2], b_2[2], c_0_2);
+    c_0_3 = fma(a_0[2], b_2[3], c_0_3);
+    c_0_4 = fma(a_0[2], b_2[4], c_0_4);
+    c_1_0 = fma(a_1[2], b_2[0], c_1_0);
+    c_1_1 = fma(a_1[2], b_2[1], c_1_1);
+    c_1_2 = fma(a_1[2], b_2[2], c_1_2);
+    c_1_3 = fma(a_1[2], b_2[3], c_1_3);
+    c_1_4 = fma(a_1[2], b_2[4], c_1_4);
+    c_2_0 = fma(a_2[2], b_2[0], c_2_0);
+    c_2_1 = fma(a_2[2], b_2[1], c_2_1);
+    c_2_2 = fma(a_2[2], b_2[2], c_2_2);
+    c_2_3 = fma(a_2[2], b_2[3], c_2_3);
+    c_2_4 = fma(a_2[2], b_2[4], c_2_4);
+    c_3_0 = fma(a_3[2], b_2[0], c_3_0);
+    c_3_1 = fma(a_3[2], b_2[1], c_3_1);
+    c_3_2 = fma(a_3[2], b_2[2], c_3_2);
+    c_3_3 = fma(a_3[2], b_2[3], c_3_3);
     c_3_4 = fma(a_3[2], b_2[4], c_3_4);
+    c_0_0 = fma(a_0[3], b_3[0], c_0_0);
+    c_0_1 = fma(a_0[3], b_3[1], c_0_1);
+    c_0_2 = fma(a_0[3], b_3[2], c_0_2);
+    c_0_3 = fma(a_0[3], b_3[3], c_0_3);
+    c_0_4 = fma(a_0[3], b_3[4], c_0_4);
+    c_1_0 = fma(a_1[3], b_3[0], c_1_0);
+    c_1_1 = fma(a_1[3], b_3[1], c_1_1);
+    c_1_2 = fma(a_1[3], b_3[2], c_1_2);
+    c_1_3 = fma(a_1[3], b_3[3], c_1_3);
+    c_1_4 = fma(a_1[3], b_3[4], c_1_4);
+    c_2_0 = fma(a_2[3], b_3[0], c_2_0);
+    c_2_1 = fma(a_2[3], b_3[1], c_2_1);
+    c_2_2 = fma(a_2[3], b_3[2], c_2_2);
+    c_2_3 = fma(a_2[3], b_3[3], c_2_3);
+    c_2_4 = fma(a_2[3], b_3[4], c_2_4);
+    c_3_0 = fma(a_3[3], b_3[0], c_3_0);
+    c_3_1 = fma(a_3[3], b_3[1], c_3_1);
+    c_3_2 = fma(a_3[3], b_3[2], c_3_2);
+    c_3_3 = fma(a_3[3], b_3[3], c_3_3);
     c_3_4 = fma(a_3[3], b_3[4], c_3_4);
+    c_0_0 = fma(a_0[4], b_4[0], c_0_0);
+    c_0_1 = fma(a_0[4], b_4[1], c_0_1);
+    c_0_2 = fma(a_0[4], b_4[2], c_0_2);
+    c_0_3 = fma(a_0[4], b_4[3], c_0_3);
+    c_0_4 = fma(a_0[4], b_4[4], c_0_4);
+    c_1_0 = fma(a_1[4], b_4[0], c_1_0);
+    c_1_1 = fma(a_1[4], b_4[1], c_1_1);
+    c_1_2 = fma(a_1[4], b_4[2], c_1_2);
+    c_1_3 = fma(a_1[4], b_4[3], c_1_3);
+    c_1_4 = fma(a_1[4], b_4[4], c_1_4);
+    c_2_0 = fma(a_2[4], b_4[0], c_2_0);
+    c_2_1 = fma(a_2[4], b_4[1], c_2_1);
+    c_2_2 = fma(a_2[4], b_4[2], c_2_2);
+    c_2_3 = fma(a_2[4], b_4[3], c_2_3);
+    c_2_4 = fma(a_2[4], b_4[4], c_2_4);
+    c_3_0 = fma(a_3[4], b_4[0], c_3_0);
+    c_3_1 = fma(a_3[4], b_4[1], c_3_1);
+    c_3_2 = fma(a_3[4], b_4[2], c_3_2);
+    c_3_3 = fma(a_3[4], b_4[3], c_3_3);
     c_3_4 = fma(a_3[4], b_4[4], c_3_4);
+    c_0_0 = fma(a_0[5], b_5[0], c_0_0);
+    c_0_1 = fma(a_0[5], b_5[1], c_0_1);
+    c_0_2 = fma(a_0[5], b_5[2], c_0_2);
+    c_0_3 = fma(a_0[5], b_5[3], c_0_3);
+    c_0_4 = fma(a_0[5], b_5[4], c_0_4);
+    c_1_0 = fma(a_1[5], b_5[0], c_1_0);
+    c_1_1 = fma(a_1[5], b_5[1], c_1_1);
+    c_1_2 = fma(a_1[5], b_5[2], c_1_2);
+    c_1_3 = fma(a_1[5], b_5[3], c_1_3);
+    c_1_4 = fma(a_1[5], b_5[4], c_1_4);
+    c_2_0 = fma(a_2[5], b_5[0], c_2_0);
+    c_2_1 = fma(a_2[5], b_5[1], c_2_1);
+    c_2_2 = fma(a_2[5], b_5[2], c_2_2);
+    c_2_3 = fma(a_2[5], b_5[3], c_2_3);
+    c_2_4 = fma(a_2[5], b_5[4], c_2_4);
+    c_3_0 = fma(a_3[5], b_5[0], c_3_0);
+    c_3_1 = fma(a_3[5], b_5[1], c_3_1);
+    c_3_2 = fma(a_3[5], b_5[2], c_3_2);
+    c_3_3 = fma(a_3[5], b_5[3], c_3_3);
     c_3_4 = fma(a_3[5], b_5[4], c_3_4);
   }
   for (t *a = a0 + p/RBK*RBK, *b = b0 + p/RBK*RBK*ldb; a < a0 + p; a++, b += ldb)